2 Run up to N simultanous jobs from provided of commands
6 from subprocess import PIPE
12 class QueueCommands(object):
14 Queue up N commands from cmd_list, launching more jobs as the first
18 def __init__(self, cmd_list, N=0, cwd=None):
20 cmd_list is a list of elements suitable for subprocess
21 N is the number of simultanious processes to run.
24 WARNING: this will not work on windows
25 (It depends on being able to pass local file descriptors to the
26 select call with isn't supported by the Win32 API)
28 self.to_run = cmd_list[:]
33 def under_process_limit(self):
35 are we still under the total number of allowable jobs?
40 if len(self.running) < self.N:
47 Launch jobs until we have the maximum allowable running
48 (or have run out of jobs)
50 queue_log = logging.getLogger('queue')
51 queue_log.info('using %s as cwd' % (self.cwd,))
53 while (len(self.to_run) > 0) and self.under_process_limit():
54 queue_log.info('%d left to run', len(self.to_run))
55 cmd = self.to_run.pop(0)
56 p = subprocess.Popen(cmd, stdout=PIPE, cwd=self.cwd, shell=True)
57 self.running[p.stdout] = p
58 queue_log.info("Created process %d from %s" % (p.pid, str(cmd)))
62 run up to N jobs until we run out of jobs
64 queue_log = logging.getLogger('queue')
66 # to_run slowly gets consumed by start_jobs
67 while len(self.to_run) > 0 or len(self.running) > 0:
68 # fill any empty spots in our job queue
71 # build a list of file descriptors
73 fds = [ x.stdout for x in self.running.values()]
75 # wait for something to finish
76 # wl= write list, xl=exception list (not used so get bad names)
77 read_list, wl, xl = select.select(fds, [], fds)
79 # for everything that might have finished...
80 for pending_fd in read_list:
81 pending = self.running[pending_fd]
82 # if it really did finish, remove it from running jobs
83 if pending.poll() is not None:
84 queue_log.info("Process %d finished [%d]",
85 pending.pid, pending.returncode)
86 del self.running[pending_fd]