From 06c4f66bf7ec31c5351832ee102d5e4c9051260d Mon Sep 17 00:00:00 2001 From: Diane Trout Date: Mon, 11 Aug 2008 23:22:15 +0000 Subject: [PATCH] A bit of refactoring toward making the run progress report code work by walking the directory instead of just watching via pyinotify. mostly this was move where the report formatting code was stored to someplace a little more shared, and by moving the thread that watches the directory tree. --- gaworkflow/automation/runner.py | 20 +------ gaworkflow/pipeline/monitors.py | 64 -------------------- gaworkflow/pipeline/run_status.py | 99 +++++++++++++++++++++++++++++-- scripts/configure_pipeline | 2 +- 4 files changed, 97 insertions(+), 88 deletions(-) delete mode 100644 gaworkflow/pipeline/monitors.py diff --git a/gaworkflow/automation/runner.py b/gaworkflow/automation/runner.py index 84e9af2..a5f6843 100644 --- a/gaworkflow/automation/runner.py +++ b/gaworkflow/automation/runner.py @@ -10,7 +10,6 @@ import threading from benderjab import rpc from gaworkflow.pipeline.configure_run import * -from gaworkflow.pipeline.monitors import _percentCompleted #s_fc = re.compile('FC[0-9]+') s_fc = re.compile('_[0-9a-zA-Z]*$') @@ -104,24 +103,7 @@ class Runner(rpc.XmlRpcBot): return "No status information for %s yet." \ " Probably still in configure step. Try again later." % (fc_num) - fc,ft = status.statusFirecrest() - bc,bt = status.statusBustard() - gc,gt = status.statusGerald() - - tc,tt = status.statusTotal() - - fp = _percentCompleted(fc, ft) - bp = _percentCompleted(bc, bt) - gp = _percentCompleted(gc, gt) - tp = _percentCompleted(tc, tt) - - output = [] - - output.append(u'Firecrest: %s%% (%s/%s)' % (fp, fc, ft)) - output.append(u' Bustard: %s%% (%s/%s)' % (bp, bc, bt)) - output.append(u' Gerald: %s%% (%s/%s)' % (gp, gc, gt)) - output.append(u'-----------------------') - output.append(u' Total: %s%% (%s/%s)' % (tp, tc, tt)) + output = status.statusReport() return '\n'.join(output) diff --git a/gaworkflow/pipeline/monitors.py b/gaworkflow/pipeline/monitors.py deleted file mode 100644 index 78380fd..0000000 --- a/gaworkflow/pipeline/monitors.py +++ /dev/null @@ -1,64 +0,0 @@ -import time -import threading - -###################### -# Utility functions -def _percentCompleted(completed, total): - """ - Returns precent completed as float - """ - return (completed / float(total)) * 100 - - -################################################## -# Functions to be called by Thread(target=) -def _cmdLineStatusMonitorFunc(conf_info): - """ - Given a ConfigInfo object, provides status to stdout. - - You should probably use startCmdLineStatusMonitor() - instead of ths function. - - Use with: - t = threading.Thread(target=_cmdLineStatusMonitorFunc, - args=[conf_info]) - t.setDaemon(True) - t.start() - """ - SLEEP_AMOUNT = 30 - - while 1: - if conf_info.status is None: - print "No status object yet." - time.sleep(SLEEP_AMOUNT) - continue - - fc, ft = conf_info.status.statusFirecrest() - bc, bt = conf_info.status.statusBustard() - gc, gt = conf_info.status.statusGerald() - tc, tt = conf_info.status.statusTotal() - - fp = _percentCompleted(fc, ft) - bp = _percentCompleted(bc, bt) - gp = _percentCompleted(gc, gt) - tp = _percentCompleted(tc, tt) - - print 'Firecrest: %s%% (%s/%s)' % (fp, fc, ft) - print ' Bustard: %s%% (%s/%s)' % (bp, bc, bt) - print ' Gerald: %s%% (%s/%s)' % (gp, gc, gt) - print '-----------------------' - print ' Total: %s%% (%s/%s)' % (tp, tc, tt) - print '' - - time.sleep(SLEEP_AMOUNT) - - -############################################# -# Start monitor thread convenience functions -def startCmdLineStatusMonitor(conf_info): - """ - Starts a command line status monitor given a conf_info object. - """ - t = threading.Thread(target=_cmdLineStatusMonitorFunc, args=[conf_info]) - t.setDaemon(True) - t.start() diff --git a/gaworkflow/pipeline/run_status.py b/gaworkflow/pipeline/run_status.py index 8e19c01..39dc54c 100644 --- a/gaworkflow/pipeline/run_status.py +++ b/gaworkflow/pipeline/run_status.py @@ -1,6 +1,9 @@ import glob import re import os +import sys +import time +import threading s_comment = re.compile('^#') s_general_read_len = re.compile('^READ_LENGTH ') @@ -117,7 +120,6 @@ def _p2f(pattern, lane, tile=None, cycle=None): return pattern % (lane) - class GARunStatus(object): def __init__(self, conf_filepath): @@ -316,9 +318,6 @@ class GARunStatus(object): #gerald['Tile.htm.tmp'] = False gerald['finished.txt'] = False - - - def statusFirecrest(self): """ returns (, ) @@ -366,6 +365,34 @@ class GARunStatus(object): return (fc+bc+gc, ft+bt+gt) + def statusReport(self): + """ + Generate the basic percent complete report + """ + def _percentCompleted(completed, total): + """ + Returns precent completed as float + """ + return (completed / float(total)) * 100 + + fc, ft = self.statusFirecrest() + bc, bt = self.statusBustard() + gc, gt = self.statusGerald() + tc, tt = self.statusTotal() + + fp = _percentCompleted(fc, ft) + bp = _percentCompleted(bc, bt) + gp = _percentCompleted(gc, gt) + tp = _percentCompleted(tc, tt) + + report = ['Firecrest: %s%% (%s/%s)' % (fp, fc, ft), + ' Bustard: %s%% (%s/%s)' % (bp, bc, bt), + ' Gerald: %s%% (%s/%s)' % (gp, gc, gt), + '-----------------------', + ' Total: %s%% (%s/%s)' % (tp, tc, tt), + ] + return report + def updateFirecrest(self, filename): """ Marks firecrest filename as being completed. @@ -385,3 +412,67 @@ class GARunStatus(object): Marks gerald filename as being completed. """ self.status['gerald'][filename] = True + + + +################################################## +# Functions to be called by Thread(target=) +def _cmdLineStatusMonitorFunc(conf_info): + """ + Given a ConfigInfo object, provides status to stdout. + + You should probably use startCmdLineStatusMonitor() + instead of ths function. + + Use with: + t = threading.Thread(target=_cmdLineStatusMonitorFunc, + args=[conf_info]) + t.setDaemon(True) + t.start() + """ + SLEEP_AMOUNT = 30 + + while 1: + if conf_info.status is None: + print "No status object yet." + time.sleep(SLEEP_AMOUNT) + continue + + report = conf_info.status.statusReport() + print os.linesep.join(report) + print + + time.sleep(SLEEP_AMOUNT) + + +############################################# +# Start monitor thread convenience functions +def startCmdLineStatusMonitor(conf_info): + """ + Starts a command line status monitor given a conf_info object. + """ + t = threading.Thread(target=_cmdLineStatusMonitorFunc, args=[conf_info]) + t.setDaemon(True) + t.start() + +from optparse import OptionParser +def make_parser(): + usage = "%prog: config file" + + parser = OptionParser() + return parser + +def main(cmdline=None): + parser = make_parser() + opt, args = parser.parse_args(cmdline) + + if len(args) != 1: + parser.error("need name of configuration file") + + status = GARunStatus(args[0]) + print os.linesep.join(status.statusReport()) + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) + diff --git a/scripts/configure_pipeline b/scripts/configure_pipeline index 6083d51..6c94b91 100644 --- a/scripts/configure_pipeline +++ b/scripts/configure_pipeline @@ -4,7 +4,7 @@ import sys import re from gaworkflow.pipeline.configure_run import * from gaworkflow.pipeline import retrieve_config as _rc -from gaworkflow.pipeline.monitors import startCmdLineStatusMonitor +from gaworkflow.pipeline.run_status import startCmdLineStatusMonitor logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s', -- 2.30.2