Consume output form the subprocesses

one of the times I was building the srf files the illumina2srf programs
stopped while the files were still incomplete, but with no CPU time.

It appears that the output from the child processes reached a point
where linux decided to block the process. This patch reads the output
from illumina2srf and if you run it with debug mode on it'll log it
otherwise it just ignores it.

diff --git a/htsworkflow/util/queuecommands.py b/htsworkflow/util/queuecommands.py
index 649fad66422160b1acad89ac1b3712158fc2c8ce..2227516aecda32491eeffe9c6548224d91eb56c4 100644 (file)
--- a/htsworkflow/util/queuecommands.py
+++ b/htsworkflow/util/queuecommands.py
@@ -3,6 +3,7 @@ Run up to N simultanous jobs from provided of commands
 """
 
 import logging
+import os
 from subprocess import PIPE
 import subprocess
 import select
@@ -74,8 +75,8 @@ class QueueCommands(object):
 
             # wait for something to finish
             # wl= write list, xl=exception list (not used so get bad names)
-            read_list, wl, xl = select.select(fds, [], fds)
-        
+            read_list, wl, xl = select.select(fds, [], fds, 1 )
+
             # for everything that might have finished...
             for pending_fd in read_list:
                 pending = self.running[pending_fd]
@@ -84,4 +85,10 @@ class QueueCommands(object):
                     queue_log.info("Process %d finished [%d]",
                                    pending.pid, pending.returncode)
                     del self.running[pending_fd]
+                else:
+                    # It's still running, but there's some output
+                    buffer = pending_fd.readline()
+                    buffer = buffer.strip()
+                    msg = "%d:(%d) %s" %(pending.pid, len(buffer), buffer)
+                    logging.debug(msg)
             time.sleep(1)