htsworkflow/pipelines/test/test_runfolder_ipar100.py

   1 #!/usr/bin/env python
   2
   3 from datetime import datetime, date
   4 import os
   5 import tempfile
   6 import shutil
   7 import unittest
   8
   9 from htsworkflow.pipelines import ipar
  10 from htsworkflow.pipelines import bustard
  11 from htsworkflow.pipelines import gerald
  12 from htsworkflow.pipelines import runfolder
  13 from htsworkflow.pipelines.runfolder import ElementTree
  14
  15 from htsworkflow.pipelines.test.simulate_runfolder import *
  16
  17
  18 def make_runfolder(obj=None):
  19     """
  20     Make a fake runfolder, attach all the directories to obj if defined
  21     """
  22     # make a fake runfolder directory
  23     temp_dir = tempfile.mkdtemp(prefix='tmp_runfolder_')
  24
  25     runfolder_dir = os.path.join(temp_dir,
  26                                  '080102_HWI-EAS229_0010_207BTAAXX')
  27     os.mkdir(runfolder_dir)
  28
  29     data_dir = os.path.join(runfolder_dir, 'Data')
  30     os.mkdir(data_dir)
  31
  32     ipar_dir = make_ipar_dir(data_dir)
  33
  34     matrix_dir = os.path.join(ipar_dir, 'Matrix')
  35     os.mkdir(matrix_dir)
  36     make_matrix(matrix_dir)
  37
  38     bustard_dir = os.path.join(ipar_dir,
  39                                'Bustard1.8.28_12-04-2008_diane')
  40     os.mkdir(bustard_dir)
  41     make_phasing_params(bustard_dir)
  42
  43     gerald_dir = os.path.join(bustard_dir,
  44                               'GERALD_12-04-2008_diane')
  45     os.mkdir(gerald_dir)
  46     make_gerald_config(gerald_dir)
  47     make_summary100_htm(gerald_dir)
  48     make_eland_results(gerald_dir)
  49
  50     if obj is not None:
  51         obj.temp_dir = temp_dir
  52         obj.runfolder_dir = runfolder_dir
  53         obj.data_dir = data_dir
  54         obj.image_analysis_dir = ipar_dir
  55         obj.matrix_dir = matrix_dir
  56         obj.bustard_dir = bustard_dir
  57         obj.gerald_dir = gerald_dir
  58
  59
  60 class RunfolderTests(unittest.TestCase):
  61     """
  62     Test components of the runfolder processing code
  63     which includes firecrest, bustard, and gerald
  64     """
  65     def setUp(self):
  66         # attaches all the directories to the object passed in
  67         make_runfolder(self)
  68
  69     def tearDown(self):
  70         shutil.rmtree(self.temp_dir)
  71
  72     def test_ipar(self):
  73         """
  74         Construct a firecrest object
  75         """
  76         i = ipar.ipar(self.image_analysis_dir)
  77         self.failUnlessEqual(i.version, '2.01.192.0')
  78         self.failUnlessEqual(i.start, 1)
  79         self.failUnlessEqual(i.stop, 37)
  80
  81         xml = i.get_elements()
  82         # just make sure that element tree can serialize the tree
  83         xml_str = ElementTree.tostring(xml)
  84
  85         i2 = ipar.IPAR(xml=xml)
  86         self.failUnlessEqual(i.version, i2.version)
  87         self.failUnlessEqual(i.start,   i2.start)
  88         self.failUnlessEqual(i.stop,    i2.stop)
  89         self.failUnlessEqual(i.date,    i2.date)
  90         self.failUnlessEqual(i.file_list(), i2.file_list())
  91
  92     def test_bustard(self):
  93         """
  94         construct a bustard object
  95         """
  96         b = bustard.bustard(self.bustard_dir)
  97         self.failUnlessEqual(b.version, '1.8.28')
  98         self.failUnlessEqual(b.date,    date(2008,4,12))
  99         self.failUnlessEqual(b.user,    'diane')
 100         self.failUnlessEqual(len(b.phasing), 8)
 101         self.failUnlessAlmostEqual(b.phasing[8].phasing, 0.0099)
 102
 103         xml = b.get_elements()
 104         b2 = bustard.Bustard(xml=xml)
 105         self.failUnlessEqual(b.version, b2.version)
 106         self.failUnlessEqual(b.date,    b2.date )
 107         self.failUnlessEqual(b.user,    b2.user)
 108         self.failUnlessEqual(len(b.phasing), len(b2.phasing))
 109         for key in b.phasing.keys():
 110             self.failUnlessEqual(b.phasing[key].lane,
 111                                  b2.phasing[key].lane)
 112             self.failUnlessEqual(b.phasing[key].phasing,
 113                                  b2.phasing[key].phasing)
 114             self.failUnlessEqual(b.phasing[key].prephasing,
 115                                  b2.phasing[key].prephasing)
 116
 117     def test_gerald(self):
 118         # need to update gerald and make tests for it
 119         g = gerald.gerald(self.gerald_dir)
 120
 121         self.failUnlessEqual(g.version,
 122             '@(#) Id: GERALD.pl,v 1.68.2.2 2007/06/13 11:08:49 km Exp')
 123         self.failUnlessEqual(g.date, datetime(2008,4,19,19,8,30))
 124         self.failUnlessEqual(len(g.lanes), len(g.lanes.keys()))
 125         self.failUnlessEqual(len(g.lanes), len(g.lanes.items()))
 126
 127
 128         # list of genomes, matches what was defined up in
 129         # make_gerald_config.
 130         # the first None is to offset the genomes list to be 1..9
 131         # instead of pythons default 0..8
 132         genomes = [None, '/g/dm3', '/g/equcab1', '/g/equcab1', '/g/canfam2',
 133                          '/g/hg18', '/g/hg18', '/g/hg18', '/g/hg18', ]
 134
 135         # test lane specific parameters from gerald config file
 136         for i in range(1,9):
 137             cur_lane = g.lanes[str(i)]
 138             self.failUnlessEqual(cur_lane.analysis, 'eland')
 139             self.failUnlessEqual(cur_lane.eland_genome, genomes[i])
 140             self.failUnlessEqual(cur_lane.read_length, '32')
 141             self.failUnlessEqual(cur_lane.use_bases, 'Y'*32)
 142
 143         # test data extracted from summary file
 144         clusters = [None,
 145                     (96483, 9074), (133738, 7938),
 146                     (152142, 10002), (15784, 2162),
 147                     (119735, 8465), (152177, 8146),
 148                     (84649, 7325), (54622, 4812),]
 149
 150         for i in range(1,9):
 151             summary_lane = g.summary[str(i)]
 152             self.failUnlessEqual(summary_lane.cluster, clusters[i])
 153             self.failUnlessEqual(summary_lane.lane, str(i))
 154
 155         xml = g.get_elements()
 156         # just make sure that element tree can serialize the tree
 157         xml_str = ElementTree.tostring(xml)
 158         g2 = gerald.Gerald(xml=xml)
 159
 160         # do it all again after extracting from the xml file
 161         self.failUnlessEqual(g.version, g2.version)
 162         self.failUnlessEqual(g.date, g2.date)
 163         self.failUnlessEqual(len(g.lanes.keys()), len(g2.lanes.keys()))
 164         self.failUnlessEqual(len(g.lanes.items()), len(g2.lanes.items()))
 165
 166         # test lane specific parameters from gerald config file
 167         for i in range(1,9):
 168             g_lane = g.lanes[str(i)]
 169             g2_lane = g2.lanes[str(i)]
 170             self.failUnlessEqual(g_lane.analysis, g2_lane.analysis)
 171             self.failUnlessEqual(g_lane.eland_genome, g2_lane.eland_genome)
 172             self.failUnlessEqual(g_lane.read_length, g2_lane.read_length)
 173             self.failUnlessEqual(g_lane.use_bases, g2_lane.use_bases)
 174
 175         # test (some) summary elements
 176         for i in range(1,9):
 177             g_summary = g.summary[str(i)]
 178             g2_summary = g2.summary[str(i)]
 179             self.failUnlessEqual(g_summary.cluster, g2_summary.cluster)
 180             self.failUnlessEqual(g_summary.lane, g2_summary.lane)
 181
 182             g_eland = g.eland_results
 183             g2_eland = g2.eland_results
 184             for lane in g_eland.keys():
 185                 self.failUnlessEqual(g_eland[lane].reads,
 186                                      g2_eland[lane].reads)
 187                 self.failUnlessEqual(len(g_eland[lane].mapped_reads),
 188                                      len(g2_eland[lane].mapped_reads))
 189                 for k in g_eland[lane].mapped_reads.keys():
 190                     self.failUnlessEqual(g_eland[lane].mapped_reads[k],
 191                                          g2_eland[lane].mapped_reads[k])
 192
 193                 self.failUnlessEqual(len(g_eland[lane].match_codes),
 194                                      len(g2_eland[lane].match_codes))
 195                 for k in g_eland[lane].match_codes.keys():
 196                     self.failUnlessEqual(g_eland[lane].match_codes[k],
 197                                          g2_eland[lane].match_codes[k])
 198
 199
 200     def test_eland(self):
 201         dm3_map = { 'chrUextra.fa' : 'dm3/chrUextra.fa',
 202                     'chr2L.fa': 'dm3/chr2L.fa',
 203                     'Lambda.fa': 'Lambda.fa'}
 204         genome_maps = { '1':dm3_map, '2':dm3_map, '3':dm3_map, '4':dm3_map,
 205                         '5':dm3_map, '6':dm3_map, '7':dm3_map, '8':dm3_map }
 206         eland = gerald.eland(self.gerald_dir, genome_maps=genome_maps)
 207
 208         for i in range(1,9):
 209             lane = eland[str(i)]
 210             self.failUnlessEqual(lane.reads, 4)
 211             self.failUnlessEqual(lane.sample_name, "s")
 212             self.failUnlessEqual(lane.lane_id, unicode(i))
 213             self.failUnlessEqual(len(lane.mapped_reads), 3)
 214             self.failUnlessEqual(lane.mapped_reads['Lambda.fa'], 1)
 215             self.failUnlessEqual(lane.mapped_reads['dm3/chr2L.fa'], 1)
 216             self.failUnlessEqual(lane.match_codes['U1'], 2)
 217             self.failUnlessEqual(lane.match_codes['NM'], 1)
 218
 219         xml = eland.get_elements()
 220         # just make sure that element tree can serialize the tree
 221         xml_str = ElementTree.tostring(xml)
 222         e2 = gerald.ELAND(xml=xml)
 223
 224         for i in range(1,9):
 225             l1 = eland[str(i)]
 226             l2 = e2[str(i)]
 227             self.failUnlessEqual(l1.reads, l2.reads)
 228             self.failUnlessEqual(l1.sample_name, l2.sample_name)
 229             self.failUnlessEqual(l1.lane_id, l2.lane_id)
 230             self.failUnlessEqual(len(l1.mapped_reads), len(l2.mapped_reads))
 231             self.failUnlessEqual(len(l1.mapped_reads), 3)
 232             for k in l1.mapped_reads.keys():
 233                 self.failUnlessEqual(l1.mapped_reads[k],
 234                                      l2.mapped_reads[k])
 235
 236             self.failUnlessEqual(len(l1.match_codes), 9)
 237             self.failUnlessEqual(len(l1.match_codes), len(l2.match_codes))
 238             for k in l1.match_codes.keys():
 239                 self.failUnlessEqual(l1.match_codes[k],
 240                                      l2.match_codes[k])
 241
 242     def test_runfolder(self):
 243         runs = runfolder.get_runs(self.runfolder_dir)
 244
 245         # do we get the flowcell id from the filename?
 246         self.failUnlessEqual(len(runs), 1)
 247         self.failUnlessEqual(runs[0].name, 'run_207BTAAXX_2008-10-30.xml')
 248
 249         # do we get the flowcell id from the FlowcellId.xml file
 250         make_flowcell_id(self.runfolder_dir, '207BTAAXY')
 251         runs = runfolder.get_runs(self.runfolder_dir)
 252         self.failUnlessEqual(len(runs), 1)
 253         self.failUnlessEqual(runs[0].name, 'run_207BTAAXY_2008-10-30.xml')
 254
 255         r1 = runs[0]
 256         xml = r1.get_elements()
 257         xml_str = ElementTree.tostring(xml)
 258
 259         r2 = runfolder.PipelineRun(xml=xml)
 260         self.failUnlessEqual(r1.name, r2.name)
 261         self.failIfEqual(r2.image_analysis, None)
 262         self.failIfEqual(r2.bustard, None)
 263         self.failIfEqual(r2.gerald, None)
 264
 265
 266 def suite():
 267     return unittest.makeSuite(RunfolderTests,'test')
 268
 269 if __name__ == "__main__":
 270     unittest.main(defaultTest="suite")
 271