2 Run up to N simultanous jobs from provided of commands
6 from subprocess import PIPE
11 class QueueCommands(object):
13 Queue up N commands from cmd_list, launching more jobs as the first
17 def __init__(self, cmd_list, N=0, cwd=None):
19 cmd_list is a list of elements suitable for subprocess
20 N is the number of simultanious processes to run.
23 WARNING: this will not work on windows
24 (It depends on being able to pass local file descriptors to the
25 select call with isn't supported by the Win32 API)
27 self.to_run = cmd_list[:]
32 def under_process_limit(self):
34 are we still under the total number of allowable jobs?
39 if len(self.running) < self.N:
46 Launch jobs until we have the maximum allowable running
47 (or have run out of jobs)
49 queue_log = logging.getLogger('queue')
50 queue_log.info('using %s as cwd' % (self.cwd,))
52 while (len(self.to_run) > 0) and self.under_process_limit():
53 queue_log.info('%d left to run', len(self.to_run))
54 cmd = self.to_run.pop(0)
55 p = subprocess.Popen(cmd, stdout=PIPE, cwd=self.cwd, shell=True)
56 self.running[p.stdout] = p
57 queue_log.info("Created process %d from %s" % (p.pid, str(cmd)))
61 run up to N jobs until we run out of jobs
63 queue_log = logging.getLogger('queue')
65 # to_run slowly gets consumed by start_jobs
66 while len(self.to_run) > 0 or len(self.running) > 0:
67 # fill any empty spots in our job queue
70 # build a list of file descriptors
72 fds = [ x.stdout for x in self.running.values()]
74 # wait for something to finish
75 # wl= write list, xl=exception list (not used so get bad names)
76 read_list, wl, xl = select.select(fds, [], fds)
78 # for everything that might have finished...
79 for pending_fd in read_list:
80 pending = self.running[pending_fd]
81 # if it really did finish, remove it from running jobs
82 if pending.poll() is not None:
83 queue_log.info("Process %d finished [%d]",
84 pending.pid, pending.returncode)
85 del self.running[pending_fd]