htsworkflow/pipelines/test/test_runfolder_pair.py

   1 #!/usr/bin/env python
   2
   3 from datetime import datetime, date
   4 import os
   5 import tempfile
   6 import shutil
   7 import unittest
   8
   9 from htsworkflow.pipelines import firecrest
  10 from htsworkflow.pipelines import bustard
  11 from htsworkflow.pipelines import gerald
  12 from htsworkflow.pipelines import runfolder
  13 from htsworkflow.pipelines.runfolder import ElementTree
  14
  15 from htsworkflow.pipelines.test.simulate_runfolder import *
  16
  17
  18 def make_runfolder(obj=None):
  19     """
  20     Make a fake runfolder, attach all the directories to obj if defined
  21     """
  22     # make a fake runfolder directory
  23     temp_dir = tempfile.mkdtemp(prefix='tmp_runfolder_')
  24
  25     runfolder_dir = os.path.join(temp_dir,
  26                                  '080102_HWI-EAS229_0010_207BTAAXX')
  27     os.mkdir(runfolder_dir)
  28
  29     data_dir = os.path.join(runfolder_dir, 'Data')
  30     os.mkdir(data_dir)
  31
  32     ipar_dir = make_firecrest_dir(data_dir, "1.9.6", 1, 152)
  33
  34     matrix_dir = os.path.join(ipar_dir, 'Matrix')
  35     os.mkdir(matrix_dir)
  36     matrix_name = os.path.join(matrix_dir, 's_matrix.txt')
  37     make_matrix(matrix_name)
  38
  39     bustard_dir = os.path.join(ipar_dir,
  40                                'Bustard1.8.28_12-04-2008_diane')
  41     os.mkdir(bustard_dir)
  42     make_phasing_params(bustard_dir)
  43
  44     gerald_dir = os.path.join(bustard_dir,
  45                               'GERALD_12-04-2008_diane')
  46     os.mkdir(gerald_dir)
  47     make_gerald_config_100(gerald_dir)
  48     make_summary_paired_htm(gerald_dir)
  49     make_eland_multi(gerald_dir, paired=True)
  50
  51     if obj is not None:
  52         obj.temp_dir = temp_dir
  53         obj.runfolder_dir = runfolder_dir
  54         obj.data_dir = data_dir
  55         obj.image_analysis_dir = ipar_dir
  56         obj.matrix_dir = matrix_dir
  57         obj.bustard_dir = bustard_dir
  58         obj.gerald_dir = gerald_dir
  59
  60
  61 class RunfolderTests(unittest.TestCase):
  62     """
  63     Test components of the runfolder processing code
  64     which includes firecrest, bustard, and gerald
  65     """
  66     def setUp(self):
  67         # attaches all the directories to the object passed in
  68         make_runfolder(self)
  69
  70     def tearDown(self):
  71         shutil.rmtree(self.temp_dir)
  72
  73     def test_firecrest(self):
  74         """
  75         Construct a firecrest object
  76         """
  77         f = firecrest.firecrest(self.image_analysis_dir)
  78         self.failUnlessEqual(f.software, 'Firecrest')
  79         self.failUnlessEqual(f.version, '1.9.6')
  80         self.failUnlessEqual(f.start, 1)
  81         self.failUnlessEqual(f.stop, 152)
  82         self.failUnlessEqual(f.user, 'diane')
  83         # As of 2008-12-8, the date was being set in
  84         # simulate_runfolder.make_firecrest_dir
  85         self.failUnlessEqual(f.date, date(2008,4,12))
  86
  87         xml = f.get_elements()
  88         # just make sure that element tree can serialize the tree
  89         xml_str = ElementTree.tostring(xml)
  90
  91         f2 = firecrest.Firecrest(xml=xml)
  92         self.failUnlessEqual(f.software, f2.software)
  93         self.failUnlessEqual(f.version, f2.version)
  94         self.failUnlessEqual(f.start,   f2.start)
  95         self.failUnlessEqual(f.stop,    f2.stop)
  96         self.failUnlessEqual(f.user,    f2.user)
  97
  98     def test_bustard(self):
  99         """
 100         construct a bustard object
 101         """
 102         b = bustard.bustard(self.bustard_dir)
 103         self.failUnlessEqual(b.software, 'Bustard')
 104         self.failUnlessEqual(b.version, '1.8.28')
 105         self.failUnlessEqual(b.date,    date(2008,4,12))
 106         self.failUnlessEqual(b.user,    'diane')
 107         self.failUnlessEqual(len(b.phasing), 8)
 108         self.failUnlessAlmostEqual(b.phasing[8].phasing, 0.0099)
 109
 110         xml = b.get_elements()
 111         b2 = bustard.Bustard(xml=xml)
 112         self.failUnlessEqual(b.software, b2.software)
 113         self.failUnlessEqual(b.version, b2.version)
 114         self.failUnlessEqual(b.date,    b2.date )
 115         self.failUnlessEqual(b.user,    b2.user)
 116         self.failUnlessEqual(len(b.phasing), len(b2.phasing))
 117         for key in b.phasing.keys():
 118             self.failUnlessEqual(b.phasing[key].lane,
 119                                  b2.phasing[key].lane)
 120             self.failUnlessEqual(b.phasing[key].phasing,
 121                                  b2.phasing[key].phasing)
 122             self.failUnlessEqual(b.phasing[key].prephasing,
 123                                  b2.phasing[key].prephasing)
 124
 125     def test_gerald(self):
 126         # need to update gerald and make tests for it
 127         g = gerald.gerald(self.gerald_dir)
 128
 129         self.failUnlessEqual(g.software, 'GERALD')
 130         self.failUnlessEqual(g.version, '1.171')
 131         self.failUnlessEqual(g.date, datetime(2009,2,22,21,15,59))
 132         self.failUnlessEqual(len(g.lanes), len(g.lanes.keys()))
 133         self.failUnlessEqual(len(g.lanes), len(g.lanes.items()))
 134
 135
 136         # list of genomes, matches what was defined up in
 137         # make_gerald_config.
 138         # the first None is to offset the genomes list to be 1..9
 139         # instead of pythons default 0..8
 140         genomes = [None,
 141                    '/g/mm9',
 142                    '/g/mm9',
 143                    '/g/elegans190',
 144                    '/g/arabidopsis01222004',
 145                    '/g/mm9',
 146                    '/g/mm9',
 147                    '/g/mm9',
 148                    '/g/mm9', ]
 149
 150         # test lane specific parameters from gerald config file
 151         for i in range(1,9):
 152             cur_lane = g.lanes[i]
 153             self.failUnlessEqual(cur_lane.analysis, 'eland_extended')
 154             self.failUnlessEqual(cur_lane.eland_genome, genomes[i])
 155             self.failUnlessEqual(cur_lane.read_length, '37')
 156             self.failUnlessEqual(cur_lane.use_bases, 'Y'*37)
 157
 158         # I want to be able to use a simple iterator
 159         for l in g.lanes.values():
 160           self.failUnlessEqual(l.analysis, 'eland_extended')
 161           self.failUnlessEqual(l.read_length, '37')
 162           self.failUnlessEqual(l.use_bases, 'Y'*37)
 163
 164         # test data extracted from summary file
 165         clusters = [[None,
 166                     (103646, 4515), (106678, 4652),
 167                     (84583, 5963), (68813, 4782),
 168                     (104854, 4664), (43555, 1632),
 169                     (54265, 1588), (64363, 2697),],
 170                     [None,
 171                     (103647, 4516), (106679, 4653),
 172                     (84584, 5964), (68814, 4783),
 173                     (104855, 4665), (43556, 1633),
 174                     (54266, 1589), (64364, 2698),],]
 175
 176         for end in [0,1]:
 177             for lane in range(1,9):
 178                 summary_lane = g.summary[end][lane]
 179                 self.failUnlessEqual(summary_lane.cluster, clusters[end][lane])
 180                 self.failUnlessEqual(summary_lane.lane, lane)
 181
 182         xml = g.get_elements()
 183         # just make sure that element tree can serialize the tree
 184         xml_str = ElementTree.tostring(xml)
 185         g2 = gerald.Gerald(xml=xml)
 186
 187         # do it all again after extracting from the xml file
 188         self.failUnlessEqual(g.software, g2.software)
 189         self.failUnlessEqual(g.version, g2.version)
 190         self.failUnlessEqual(g.date, g2.date)
 191         self.failUnlessEqual(len(g.lanes.keys()), len(g2.lanes.keys()))
 192         self.failUnlessEqual(len(g.lanes.items()), len(g2.lanes.items()))
 193
 194         # test lane specific parameters from gerald config file
 195         for i in range(1,9):
 196             g_lane = g.lanes[i]
 197             g2_lane = g2.lanes[i]
 198             self.failUnlessEqual(g_lane.analysis, g2_lane.analysis)
 199             self.failUnlessEqual(g_lane.eland_genome, g2_lane.eland_genome)
 200             self.failUnlessEqual(g_lane.read_length, g2_lane.read_length)
 201             self.failUnlessEqual(g_lane.use_bases, g2_lane.use_bases)
 202
 203         # test (some) summary elements
 204         for end in [0,1]:
 205             for i in range(1,9):
 206                 g_summary = g.summary[end][i]
 207                 g2_summary = g2.summary[end][i]
 208                 self.failUnlessEqual(g_summary.cluster, g2_summary.cluster)
 209                 self.failUnlessEqual(g_summary.lane, g2_summary.lane)
 210
 211                 g_eland = g.eland_results
 212                 g2_eland = g2.eland_results
 213                 for lane in g_eland.results[end].keys():
 214                     g_results = g_eland.results[end][lane]
 215                     g2_results = g_eland.results[end][lane]
 216                     self.failUnlessEqual(g_results.reads,
 217                                          g2_results.reads)
 218                     self.failUnlessEqual(len(g_results.mapped_reads),
 219                                          len(g2_results.mapped_reads))
 220                     for k in g_results.mapped_reads.keys():
 221                         self.failUnlessEqual(g_results.mapped_reads[k],
 222                                              g2_results.mapped_reads[k])
 223
 224                     self.failUnlessEqual(len(g_results.match_codes),
 225                                          len(g2_results.match_codes))
 226                     for k in g_results.match_codes.keys():
 227                         self.failUnlessEqual(g_results.match_codes[k],
 228                                              g2_results.match_codes[k])
 229
 230
 231     def test_eland(self):
 232         hg_map = {'Lambda.fa': 'Lambda.fa'}
 233         for i in range(1,22):
 234           short_name = 'chr%d.fa' % (i,)
 235           long_name = 'hg18/chr%d.fa' % (i,)
 236           hg_map[short_name] = long_name
 237
 238         genome_maps = { 1:hg_map, 2:hg_map, 3:hg_map, 4:hg_map,
 239                         5:hg_map, 6:hg_map, 7:hg_map, 8:hg_map }
 240         eland = gerald.eland(self.gerald_dir, genome_maps=genome_maps)
 241
 242         # check first end
 243         for i in range(1,9):
 244             lane = eland.results[0][i]
 245             self.failUnlessEqual(lane.reads, 6)
 246             self.failUnlessEqual(lane.sample_name, "s")
 247             self.failUnlessEqual(lane.lane_id, i)
 248             self.failUnlessEqual(len(lane.mapped_reads), 17)
 249             self.failUnlessEqual(lane.mapped_reads['hg18/chr5.fa'], 4)
 250             self.failUnlessEqual(lane.match_codes['U0'], 3)
 251             self.failUnlessEqual(lane.match_codes['R0'], 2)
 252             self.failUnlessEqual(lane.match_codes['U1'], 1)
 253             self.failUnlessEqual(lane.match_codes['R1'], 9)
 254             self.failUnlessEqual(lane.match_codes['U2'], 0)
 255             self.failUnlessEqual(lane.match_codes['R2'], 12)
 256             self.failUnlessEqual(lane.match_codes['NM'], 1)
 257             self.failUnlessEqual(lane.match_codes['QC'], 0)
 258
 259         # check second end
 260         for i in range(1,9):
 261             lane = eland.results[1][i]
 262             self.failUnlessEqual(lane.reads, 7)
 263             self.failUnlessEqual(lane.sample_name, "s")
 264             self.failUnlessEqual(lane.lane_id, i)
 265             self.failUnlessEqual(len(lane.mapped_reads), 17)
 266             self.failUnlessEqual(lane.mapped_reads['hg18/chr5.fa'], 4)
 267             self.failUnlessEqual(lane.match_codes['U0'], 3)
 268             self.failUnlessEqual(lane.match_codes['R0'], 2)
 269             self.failUnlessEqual(lane.match_codes['U1'], 1)
 270             self.failUnlessEqual(lane.match_codes['R1'], 9)
 271             self.failUnlessEqual(lane.match_codes['U2'], 0)
 272             self.failUnlessEqual(lane.match_codes['R2'], 12)
 273             self.failUnlessEqual(lane.match_codes['NM'], 1)
 274             self.failUnlessEqual(lane.match_codes['QC'], 1)
 275
 276         xml = eland.get_elements()
 277         # just make sure that element tree can serialize the tree
 278         xml_str = ElementTree.tostring(xml)
 279         e2 = gerald.ELAND(xml=xml)
 280
 281         for end in [0, 1]:
 282             for i in range(1,9):
 283                 l1 = eland.results[end][i]
 284                 l2 = e2.results[end][i]
 285                 self.failUnlessEqual(l1.reads, l2.reads)
 286                 self.failUnlessEqual(l1.sample_name, l2.sample_name)
 287                 self.failUnlessEqual(l1.lane_id, l2.lane_id)
 288                 self.failUnlessEqual(len(l1.mapped_reads), len(l2.mapped_reads))
 289                 self.failUnlessEqual(len(l1.mapped_reads), 17)
 290                 for k in l1.mapped_reads.keys():
 291                     self.failUnlessEqual(l1.mapped_reads[k],
 292                                          l2.mapped_reads[k])
 293
 294                 self.failUnlessEqual(len(l1.match_codes), 9)
 295                 self.failUnlessEqual(len(l1.match_codes), len(l2.match_codes))
 296                 for k in l1.match_codes.keys():
 297                     self.failUnlessEqual(l1.match_codes[k],
 298                                          l2.match_codes[k])
 299
 300     def test_runfolder(self):
 301         runs = runfolder.get_runs(self.runfolder_dir)
 302
 303         # do we get the flowcell id from the filename?
 304         self.failUnlessEqual(len(runs), 1)
 305         # firecrest's date depends on filename not the create time.
 306         name = 'run_207BTAAXX_2009-02-22.xml'
 307         self.failUnlessEqual(runs[0].name, name)
 308
 309         # do we get the flowcell id from the FlowcellId.xml file
 310         make_flowcell_id(self.runfolder_dir, '207BTAAXY')
 311         runs = runfolder.get_runs(self.runfolder_dir)
 312         self.failUnlessEqual(len(runs), 1)
 313         name = 'run_207BTAAXY_2009-02-22.xml'
 314         self.failUnlessEqual(runs[0].name, name)
 315
 316         r1 = runs[0]
 317         xml = r1.get_elements()
 318         xml_str = ElementTree.tostring(xml)
 319
 320         r2 = runfolder.PipelineRun(xml=xml)
 321         self.failUnlessEqual(r1.name, r2.name)
 322         self.failIfEqual(r2.image_analysis, None)
 323         self.failIfEqual(r2.bustard, None)
 324         self.failIfEqual(r2.gerald, None)
 325
 326
 327 def suite():
 328     return unittest.makeSuite(RunfolderTests,'test')
 329
 330 if __name__ == "__main__":
 331     unittest.main(defaultTest="suite")
 332