import stat
import subprocess
import sys
+import tarfile
import time
try:
VERSION_RE = "([0-9\.]+)"
USER_RE = "([a-zA-Z0-9]+)"
LANES_PER_FLOWCELL = 8
+LANE_LIST = range(1, LANES_PER_FLOWCELL+1)
from htsworkflow.util.alphanum import alphanum
from htsworkflow.util.ethelp import indent, flatten
+from htsworkflow.util.queuecommands import QueueCommands
+
+from htsworkflow.pipelines import srf
class PipelineRun(object):
"""
tree = ElementTree.parse(filename).getroot()
self.set_elements(tree)
+def load_pipeline_run_xml(pathname):
+ """
+ Load and instantiate a Pipeline run from a run xml file
+
+ :Parameters:
+ - `pathname` : location of an run xml file
+
+ :Returns: initialized PipelineRun object
+ """
+ tree = ElementTree.parse(pathname).getroot()
+ run = PipelineRun(xml=tree)
+ return run
+
def get_runs(runfolder):
"""
Search through a run folder for all the various sub component runs
def scan_post_image_analysis(runs, runfolder, image_analysis, pathname):
logging.info("Looking for bustard directories in %s" % (pathname,))
- bustard_glob = os.path.join(pathname, "Bustard*")
- for bustard_pathname in glob(bustard_glob):
+ bustard_dirs = glob(os.path.join(pathname, "Bustard*"))
+ # RTA BaseCalls looks enough like Bustard.
+ bustard_dirs.extend(glob(os.path.join(pathname, "BaseCalls")))
+ for bustard_pathname in bustard_dirs:
logging.info("Found bustard directory %s" % (bustard_pathname,))
b = bustard.bustard(bustard_pathname)
gerald_glob = os.path.join(bustard_pathname, 'GERALD*')
p.gerald = g
runs.append(p)
except IOError, e:
- print "Ignoring", str(e)
+ logging.error("Ignoring " + str(e))
datadir = os.path.join(runfolder, 'Data')
for firecrest_pathname in glob(os.path.join(datadir,"*Firecrest*")):
logging.info('Found firecrest in ' + datadir)
image_analysis = firecrest.firecrest(firecrest_pathname)
- scan_post_image_analysis(runs, runfolder, image_analysis, firecrest_pathname)
+ if image_analysis is None:
+ logging.warn(
+ "%s is an empty or invalid firecrest directory" % (firecrest_pathname,)
+ )
+ else:
+ scan_post_image_analysis(
+ runs, runfolder, image_analysis, firecrest_pathname
+ )
# scan for IPAR directories
- for ipar_pathname in glob(os.path.join(datadir,"IPAR_*")):
+ ipar_dirs = glob(os.path.join(datadir, "IPAR_*"))
+ # The Intensities directory from the RTA software looks a lot like IPAR
+ ipar_dirs.extend(glob(os.path.join(datadir, 'Intensities')))
+ for ipar_pathname in ipar_dirs:
logging.info('Found ipar directories in ' + datadir)
image_analysis = ipar.ipar(ipar_pathname)
- scan_post_image_analysis(runs, runfolder, image_analysis, ipar_pathname)
+ if image_analysis is None:
+ logging.warn(
+ "%s is an empty or invalid IPAR directory" %(ipar_pathname,)
+ )
+ else:
+ scan_post_image_analysis(
+ runs, runfolder, image_analysis, ipar_pathname
+ )
return runs
+def get_specific_run(gerald_dir):
+ """
+ Given a gerald directory, construct a PipelineRun out of its parents
+
+ Basically this allows specifying a particular run instead of the previous
+ get_runs which scans a runfolder for various combinations of
+ firecrest/ipar/bustard/gerald runs.
+ """
+ from htsworkflow.pipelines import firecrest
+ from htsworkflow.pipelines import ipar
+ from htsworkflow.pipelines import bustard
+ from htsworkflow.pipelines import gerald
+
+ gerald_dir = os.path.expanduser(gerald_dir)
+ bustard_dir = os.path.abspath(os.path.join(gerald_dir, '..'))
+ image_dir = os.path.abspath(os.path.join(gerald_dir, '..', '..'))
+
+ runfolder_dir = os.path.abspath(os.path.join(image_dir, '..','..'))
+
+ logging.info('--- use-run detected options ---')
+ logging.info('runfolder: %s' % (runfolder_dir,))
+ logging.info('image_dir: %s' % (image_dir,))
+ logging.info('bustard_dir: %s' % (bustard_dir,))
+ logging.info('gerald_dir: %s' % (gerald_dir,))
+
+ # find our processed image dir
+ image_run = None
+ # split into parent, and leaf directory
+ # leaf directory should be an IPAR or firecrest directory
+ data_dir, short_image_dir = os.path.split(image_dir)
+ logging.info('data_dir: %s' % (data_dir,))
+ logging.info('short_iamge_dir: %s' %(short_image_dir,))
+
+ # guess which type of image processing directory we have by looking
+ # in the leaf directory name
+ if re.search('Firecrest', short_image_dir, re.IGNORECASE) is not None:
+ image_run = firecrest.firecrest(image_dir)
+ elif re.search('IPAR', short_image_dir, re.IGNORECASE) is not None:
+ image_run = ipar.ipar(image_dir)
+ elif re.search('Intensities', short_image_dir, re.IGNORECASE) is not None:
+ image_run = ipar.ipar(image_dir)
+
+ # if we din't find a run, report the error and return
+ if image_run is None:
+ msg = '%s does not contain an image processing step' % (image_dir,)
+ logging.error(msg)
+ return None
+
+ # find our base calling
+ base_calling_run = bustard.bustard(bustard_dir)
+ if base_calling_run is None:
+ logging.error('%s does not contain a bustard run' % (bustard_dir,))
+ return None
+
+ # find alignments
+ gerald_run = gerald.gerald(gerald_dir)
+ if gerald_run is None:
+ logging.error('%s does not contain a gerald run' % (gerald_dir,))
+ return None
+
+ p = PipelineRun(runfolder_dir)
+ p.image_analysis = image_run
+ p.bustard = base_calling_run
+ p.gerald = gerald_run
+
+ logging.info('Constructed PipelineRun from %s' % (gerald_dir,))
+ return p
def extract_run_parameters(runs):
"""
for run in runs:
run.save()
-def summarize_mapped_reads(mapped_reads):
+def summarize_mapped_reads(genome_map, mapped_reads):
"""
Summarize per chromosome reads into a genome count
But handle spike-in/contamination symlinks seperately.
genome = 'unknown'
for k, v in mapped_reads.items():
path, k = os.path.split(k)
- if len(path) > 0:
+ if len(path) > 0 and not genome_map.has_key(path):
genome = path
genome_reads += v
else:
eland_result = gerald.eland_results.results[end][lane_id]
report.append("Sample name %s" % (eland_result.sample_name))
report.append("Lane id %s end %s" % (eland_result.lane_id, end))
- cluster = summary_results[end][eland_result.lane_id].cluster
- report.append("Clusters %d +/- %d" % (cluster[0], cluster[1]))
+ if end < len(summary_results) and summary_results[end].has_key(eland_result.lane_id):
+ cluster = summary_results[end][eland_result.lane_id].cluster
+ report.append("Clusters %d +/- %d" % (cluster[0], cluster[1]))
report.append("Total Reads: %d" % (eland_result.reads))
- mc = eland_result._match_codes
- nm = mc['NM']
- nm_percent = float(nm)/eland_result.reads * 100
- qc = mc['QC']
- qc_percent = float(qc)/eland_result.reads * 100
-
- report.append("No Match: %d (%2.2g %%)" % (nm, nm_percent))
- report.append("QC Failed: %d (%2.2g %%)" % (qc, qc_percent))
- report.append('Unique (0,1,2 mismatches) %d %d %d' % \
- (mc['U0'], mc['U1'], mc['U2']))
- report.append('Repeat (0,1,2 mismatches) %d %d %d' % \
- (mc['R0'], mc['R1'], mc['R2']))
- report.append("Mapped Reads")
- mapped_reads = summarize_mapped_reads(eland_result.mapped_reads)
- for name, counts in mapped_reads.items():
- report.append(" %s: %d" % (name, counts))
+
+ if hasattr(eland_result, 'match_codes'):
+ mc = eland_result.match_codes
+ nm = mc['NM']
+ nm_percent = float(nm)/eland_result.reads * 100
+ qc = mc['QC']
+ qc_percent = float(qc)/eland_result.reads * 100
+
+ report.append("No Match: %d (%2.2g %%)" % (nm, nm_percent))
+ report.append("QC Failed: %d (%2.2g %%)" % (qc, qc_percent))
+ report.append('Unique (0,1,2 mismatches) %d %d %d' % \
+ (mc['U0'], mc['U1'], mc['U2']))
+ report.append('Repeat (0,1,2 mismatches) %d %d %d' % \
+ (mc['R0'], mc['R1'], mc['R2']))
+
+ if hasattr(eland_result, 'genome_map'):
+ report.append("Mapped Reads")
+ mapped_reads = summarize_mapped_reads(eland_result.genome_map, eland_result.mapped_reads)
+ for name, counts in mapped_reads.items():
+ report.append(" %s: %d" % (name, counts))
+
report.append('')
return report
else:
return False
-def extract_results(runs, output_base_dir=None):
+def save_flowcell_reports(data_dir, cycle_dir):
+ """
+ Save the flowcell quality reports
+ """
+ data_dir = os.path.abspath(data_dir)
+ status_file = os.path.join(data_dir, 'Status.xml')
+ reports_dir = os.path.join(data_dir, 'reports')
+ reports_dest = os.path.join(cycle_dir, 'flowcell-reports.tar.bz2')
+ if os.path.exists(reports_dir):
+ cmd_list = [ 'tar', 'cjvf', reports_dest, 'reports/' ]
+ if os.path.exists(status_file):
+ cmd_list.extend(['Status.xml', 'Status.xsl'])
+ logging.info("Saving reports from " + reports_dir)
+ cwd = os.getcwd()
+ os.chdir(data_dir)
+ q = QueueCommands([" ".join(cmd_list)])
+ q.run()
+ os.chdir(cwd)
+
+
+def save_summary_file(gerald_object, cycle_dir):
+ # Copy Summary.htm
+ summary_path = os.path.join(gerald_object.pathname, 'Summary.htm')
+ if os.path.exists(summary_path):
+ logging.info('Copying %s to %s' % (summary_path, cycle_dir))
+ shutil.copy(summary_path, cycle_dir)
+ else:
+ logging.info('Summary file %s was not found' % (summary_path,))
+
+def save_ivc_plot(bustard_object, cycle_dir):
+ """
+ Save the IVC page and its supporting images
+ """
+ plot_html = os.path.join(bustard_object.pathname, 'IVC.htm')
+ plot_image_path = os.path.join(bustard_object.pathname, 'Plots')
+ plot_images = os.path.join(plot_image_path, 's_?_[a-z]*.png')
+
+ plot_target_path = os.path.join(cycle_dir, 'Plots')
+
+ if os.path.exists(plot_html):
+ logging.debug("Saving %s" % ( plot_html, ))
+ logging.debug("Saving %s" % ( plot_images, ))
+ shutil.copy(plot_html, cycle_dir)
+ if not os.path.exists(plot_target_path):
+ os.mkdir(plot_target_path)
+ for plot_file in glob(plot_images):
+ shutil.copy(plot_file, plot_target_path)
+ else:
+ logging.warning('Missing IVC.html file, not archiving')
+
+
+def compress_score_files(bustard_object, cycle_dir):
+ """
+ Compress score files into our result directory
+ """
+ # check for g.pathname/Temp a new feature of 1.1rc1
+ scores_path = bustard_object.pathname
+ scores_path_temp = os.path.join(scores_path, 'Temp')
+ if os.path.isdir(scores_path_temp):
+ scores_path = scores_path_temp
+
+ # hopefully we have a directory that contains s_*_score files
+ score_files = []
+ for f in os.listdir(scores_path):
+ if re.match('.*_score.txt', f):
+ score_files.append(f)
+
+ tar_cmd = ['tar', 'c'] + score_files
+ bzip_cmd = [ 'bzip2', '-9', '-c' ]
+ tar_dest_name =os.path.join(cycle_dir, 'scores.tar.bz2')
+ tar_dest = open(tar_dest_name, 'w')
+ logging.info("Compressing score files from %s" % (scores_path,))
+ logging.info("Running tar: " + " ".join(tar_cmd[:10]))
+ logging.info("Running bzip2: " + " ".join(bzip_cmd))
+ logging.info("Writing to %s" %(tar_dest_name,))
+
+ env = {'BZIP': '-9'}
+ tar = subprocess.Popen(tar_cmd, stdout=subprocess.PIPE, shell=False, env=env,
+ cwd=scores_path)
+ bzip = subprocess.Popen(bzip_cmd, stdin=tar.stdout, stdout=tar_dest)
+ tar.wait()
+
+
+def compress_eland_results(gerald_object, cycle_dir, num_jobs=1):
+ """
+ Compress eland result files into the archive directory
+ """
+ # copy & bzip eland files
+ bz_commands = []
+
+ for lanes_dictionary in gerald_object.eland_results.results:
+ for eland_lane in lanes_dictionary.values():
+ source_name = eland_lane.pathname
+ path, name = os.path.split(eland_lane.pathname)
+ dest_name = os.path.join(cycle_dir, name)
+ logging.info("Saving eland file %s to %s" % \
+ (source_name, dest_name))
+
+ if is_compressed(name):
+ logging.info('Already compressed, Saving to %s' % (dest_name, ))
+ shutil.copy(source_name, dest_name)
+ else:
+ # not compressed
+ dest_name += '.bz2'
+ args = ['bzip2', '-9', '-c', source_name, '>', dest_name ]
+ bz_commands.append(" ".join(args))
+ #logging.info('Running: %s' % ( " ".join(args) ))
+ #bzip_dest = open(dest_name, 'w')
+ #bzip = subprocess.Popen(args, stdout=bzip_dest)
+ #logging.info('Saving to %s' % (dest_name, ))
+ #bzip.wait()
+
+ if len(bz_commands) > 0:
+ q = QueueCommands(bz_commands, num_jobs)
+ q.run()
+
+
+def extract_results(runs, output_base_dir=None, site="individual", num_jobs=1):
+ """
+ Iterate over runfolders in runs extracting the most useful information.
+ * run parameters (in run-*.xml)
+ * eland_result files
+ * score files
+ * Summary.htm
+ * srf files (raw sequence & qualities)
+ """
if output_base_dir is None:
output_base_dir = os.getcwd()
else:
os.mkdir(cycle_dir)
+ # save run file
+ r.save(cycle_dir)
+
+ # save illumina flowcell status report
+ save_flowcell_reports( os.path.join(r.image_analysis.pathname, '..'), cycle_dir )
+
+ # save stuff from bustard
+ # grab IVC plot
+ save_ivc_plot(r.bustard, cycle_dir)
+
+ # build base call saving commands
+ if site is not None:
+ lanes = range(1,9)
+ run_name = srf.pathname_to_run_name(r.pathname)
+ srf_cmds = srf.make_qseq_commands(run_name, r.bustard.pathname, lanes, site, cycle_dir)
+ srf.run_commands(r.bustard.pathname, srf_cmds, num_jobs)
+
+ # save stuff from GERALD
# copy stuff out of the main run
g = r.gerald
- # save run file
- r.save(cycle_dir)
+ # save summary file
+ save_summary_file(g, cycle_dir)
+
+ # compress eland result files
+ compress_eland_results(g, cycle_dir, num_jobs)
+
+ # md5 all the compressed files once we're done
+ md5_commands = srf.make_md5_commands(cycle_dir)
+ srf.run_commands(cycle_dir, md5_commands, num_jobs)
+
+def rm_list(files, dry_run=True):
+ for f in files:
+ if os.path.exists(f):
+ logging.info('deleting %s' % (f,))
+ if not dry_run:
+ if os.path.isdir(f):
+ shutil.rmtree(f)
+ else:
+ os.unlink(f)
+ else:
+ logging.warn("%s doesn't exist."% (f,))
- # Copy Summary.htm
- summary_path = os.path.join(r.gerald.pathname, 'Summary.htm')
- if os.path.exists(summary_path):
- logging.info('Copying %s to %s' % (summary_path, cycle_dir))
- shutil.copy(summary_path, cycle_dir)
- else:
- logging.info('Summary file %s was not found' % (summary_path,))
-
- # tar score files
- score_files = []
-
- # check for g.pathname/Temp a new feature of 1.1rc1
- scores_path = g.pathname
- scores_path_temp = os.path.join(scores_path, 'Temp')
- if os.path.isdir(scores_path_temp):
- scores_path = scores_path_temp
-
- # hopefully we have a directory that contains s_*_score files
- for f in os.listdir(scores_path):
- if re.match('.*_score.txt', f):
- score_files.append(f)
-
- tar_cmd = ['/bin/tar', 'c'] + score_files
- bzip_cmd = [ 'bzip2', '-9', '-c' ]
- tar_dest_name =os.path.join(cycle_dir, 'scores.tar.bz2')
- tar_dest = open(tar_dest_name, 'w')
- logging.info("Compressing score files from %s" % (scores_path,))
- logging.info("Running tar: " + " ".join(tar_cmd[:10]))
- logging.info("Running bzip2: " + " ".join(bzip_cmd))
- logging.info("Writing to %s" %(tar_dest_name))
-
- tar = subprocess.Popen(tar_cmd, stdout=subprocess.PIPE, shell=False,
- cwd=scores_path)
- bzip = subprocess.Popen(bzip_cmd, stdin=tar.stdout, stdout=tar_dest)
- tar.wait()
-
- # copy & bzip eland files
- for lanes_dictionary in g.eland_results.results:
- for eland_lane in lanes_dictionary.values():
- source_name = eland_lane.pathname
- path, name = os.path.split(eland_lane.pathname)
- dest_name = os.path.join(cycle_dir, name)
- logging.info("Saving eland file %s to %s" % \
- (source_name, dest_name))
-
- if is_compressed(name):
- logging.info('Already compressed, Saving to %s' % (dest_name, ))
- shutil.copy(source_name, dest_name)
- else:
- # not compressed
- dest_name += '.bz2'
- args = ['bzip2', '-9', '-c', source_name]
- logging.info('Running: %s' % ( " ".join(args) ))
- bzip_dest = open(dest_name, 'w')
- bzip = subprocess.Popen(args, stdout=bzip_dest)
- logging.info('Saving to %s' % (dest_name, ))
- bzip.wait()
-
-def clean_runs(runs):
+def clean_runs(runs, dry_run=True):
"""
Clean up run folders to optimize for compression.
"""
- # TODO: implement this.
- # rm RunLog*.xml
- # rm pipeline_*.txt
- # rm gclog.txt
- # rm NetCopy.log
- # rm nfn.log
- # rm Images/L*
- # cd Data/C1-*_Firecrest*
- # make clean_intermediate
-
- pass
+ if dry_run:
+ logging.info('In dry-run mode')
+
+ for run in runs:
+ logging.info('Cleaninging %s' % (run.pathname,))
+ # rm RunLog*.xml
+ runlogs = glob(os.path.join(run.pathname, 'RunLog*xml'))
+ rm_list(runlogs, dry_run)
+ # rm pipeline_*.txt
+ pipeline_logs = glob(os.path.join(run.pathname, 'pipeline*.txt'))
+ rm_list(pipeline_logs, dry_run)
+ # rm gclog.txt?
+ # rm NetCopy.log? Isn't this robocopy?
+ logs = glob(os.path.join(run.pathname, '*.log'))
+ rm_list(logs, dry_run)
+ # rm nfn.log?
+ # Calibration
+ calibration_dir = glob(os.path.join(run.pathname, 'Calibration_*'))
+ rm_list(calibration_dir, dry_run)
+ # rm Images/L*
+ logging.info("Cleaning images")
+ image_dirs = glob(os.path.join(run.pathname, 'Images', 'L*'))
+ rm_list(image_dirs, dry_run)
+ # cd Data/C1-*_Firecrest*
+ logging.info("Cleaning intermediate files")
+ # make clean_intermediate
+ if os.path.exists(os.path.join(run.image_analysis.pathname, 'Makefile')):
+ clean_process = subprocess.Popen(['make', 'clean_intermediate'],
+ cwd=run.image_analysis.pathname,)
+ clean_process.wait()
+
+
+
projects / htsworkflow.git / blobdiff