htsworkflow/pipelines/test/test_runfolder_ipar130.py

   1 #!/usr/bin/env python
   2
   3 from datetime import datetime, date
   4 import os
   5 import tempfile
   6 import shutil
   7 import unittest
   8
   9 from htsworkflow.pipelines import ipar
  10 from htsworkflow.pipelines import bustard
  11 from htsworkflow.pipelines import gerald
  12 from htsworkflow.pipelines import runfolder
  13 from htsworkflow.pipelines.runfolder import ElementTree
  14
  15 from htsworkflow.pipelines.test.simulate_runfolder import *
  16
  17
  18 def make_runfolder(obj=None):
  19     """
  20     Make a fake runfolder, attach all the directories to obj if defined
  21     """
  22     # make a fake runfolder directory
  23     temp_dir = tempfile.mkdtemp(prefix='tmp_runfolder_')
  24
  25     runfolder_dir = os.path.join(temp_dir,
  26                                  '090313_HWI-EAS229_0101_3021JAAXX')
  27     os.mkdir(runfolder_dir)
  28
  29     data_dir = os.path.join(runfolder_dir, 'Data')
  30     os.mkdir(data_dir)
  31
  32     ipar_dir = make_ipar_dir(data_dir, '1.30')
  33
  34     bustard_dir = os.path.join(ipar_dir,
  35                                'Bustard1.3.2_15-03-2008_diane')
  36     os.mkdir(bustard_dir)
  37     make_phasing_params(bustard_dir)
  38     make_bustard_config132(bustard_dir)
  39
  40     gerald_dir = os.path.join(bustard_dir,
  41                               'GERALD_15-03-2008_diane')
  42     os.mkdir(gerald_dir)
  43     make_gerald_config_100(gerald_dir)
  44     make_summary_ipar130_htm(gerald_dir)
  45     make_eland_multi(gerald_dir)
  46
  47     if obj is not None:
  48         obj.temp_dir = temp_dir
  49         obj.runfolder_dir = runfolder_dir
  50         obj.data_dir = data_dir
  51         obj.image_analysis_dir = ipar_dir
  52         obj.bustard_dir = bustard_dir
  53         obj.gerald_dir = gerald_dir
  54
  55
  56 class RunfolderTests(unittest.TestCase):
  57     """
  58     Test components of the runfolder processing code
  59     which includes firecrest, bustard, and gerald
  60     """
  61     def setUp(self):
  62         # attaches all the directories to the object passed in
  63         make_runfolder(self)
  64
  65     def tearDown(self):
  66         shutil.rmtree(self.temp_dir)
  67
  68     def test_ipar(self):
  69         """
  70         Construct a firecrest object
  71         """
  72         i = ipar.ipar(self.image_analysis_dir)
  73         self.failUnlessEqual(i.version, '2.01.192.0')
  74         self.failUnlessEqual(i.start, 1)
  75         self.failUnlessEqual(i.stop, 37)
  76
  77         xml = i.get_elements()
  78         # just make sure that element tree can serialize the tree
  79         xml_str = ElementTree.tostring(xml)
  80
  81         i2 = ipar.IPAR(xml=xml)
  82         self.failUnlessEqual(i.version, i2.version)
  83         self.failUnlessEqual(i.start,   i2.start)
  84         self.failUnlessEqual(i.stop,    i2.stop)
  85         self.failUnlessEqual(i.date,    i2.date)
  86         self.failUnlessEqual(i.file_list(), i2.file_list())
  87
  88     def test_bustard(self):
  89         """
  90         construct a bustard object
  91         """
  92         def check_crosstalk(crosstalk):
  93             self.failUnlessAlmostEqual(crosstalk.base['A'][0],  1.27)
  94             self.failUnlessAlmostEqual(crosstalk.base['A'][1],  0.20999999999999)
  95             self.failUnlessAlmostEqual(crosstalk.base['A'][2], -0.02)
  96             self.failUnlessAlmostEqual(crosstalk.base['A'][3], -0.03)
  97
  98             self.failUnlessAlmostEqual(crosstalk.base['C'][0],  0.57)
  99             self.failUnlessAlmostEqual(crosstalk.base['C'][1],  0.58)
 100             self.failUnlessAlmostEqual(crosstalk.base['C'][2], -0.01)
 101             self.failUnlessAlmostEqual(crosstalk.base['C'][3], -0.01)
 102
 103             self.failUnlessAlmostEqual(crosstalk.base['T'][0], -0.02)
 104             self.failUnlessAlmostEqual(crosstalk.base['T'][1], -0.02)
 105             self.failUnlessAlmostEqual(crosstalk.base['T'][2],  0.80)
 106             self.failUnlessAlmostEqual(crosstalk.base['T'][3],  1.07)
 107
 108             self.failUnlessAlmostEqual(crosstalk.base['G'][0], -0.03)
 109             self.failUnlessAlmostEqual(crosstalk.base['G'][1], -0.04)
 110             self.failUnlessAlmostEqual(crosstalk.base['G'][2],  1.51)
 111             self.failUnlessAlmostEqual(crosstalk.base['G'][3], -0.02)
 112
 113         b = bustard.bustard(self.bustard_dir)
 114         self.failUnlessEqual(b.version, '1.3.2')
 115         self.failUnlessEqual(b.date,    date(2008,3,15))
 116         self.failUnlessEqual(b.user,    'diane')
 117         self.failUnlessEqual(len(b.phasing), 8)
 118         self.failUnlessAlmostEqual(b.phasing[8].phasing, 0.0099)
 119         self.failUnlessEqual(b.crosstalk.base.keys(), ['A','C','T','G'])
 120         check_crosstalk(b.crosstalk)
 121
 122         xml = b.get_elements()
 123         print ElementTree.dump(xml)
 124         b2 = bustard.Bustard(xml=xml)
 125         self.failUnlessEqual(b.version, b2.version)
 126         self.failUnlessEqual(b.date,    b2.date )
 127         self.failUnlessEqual(b.user,    b2.user)
 128         self.failUnlessEqual(len(b.phasing), len(b2.phasing))
 129         for key in b.phasing.keys():
 130             self.failUnlessEqual(b.phasing[key].lane,
 131                                  b2.phasing[key].lane)
 132             self.failUnlessEqual(b.phasing[key].phasing,
 133                                  b2.phasing[key].phasing)
 134             self.failUnlessEqual(b.phasing[key].prephasing,
 135                                  b2.phasing[key].prephasing)
 136         check_crosstalk(b2.crosstalk)
 137
 138     def test_gerald(self):
 139         # need to update gerald and make tests for it
 140         g = gerald.gerald(self.gerald_dir)
 141
 142         self.failUnlessEqual(g.version,
 143             '@(#) Id: GERALD.pl,v 1.171 2008/05/19 17:36:14 mzerara Exp')
 144         self.failUnlessEqual(g.date, datetime(2009,2,22,21,15,59))
 145         self.failUnlessEqual(len(g.lanes), len(g.lanes.keys()))
 146         self.failUnlessEqual(len(g.lanes), len(g.lanes.items()))
 147
 148
 149         # list of genomes, matches what was defined up in
 150         # make_gerald_config.
 151         # the first None is to offset the genomes list to be 1..9
 152         # instead of pythons default 0..8
 153         genomes = [None,
 154                    '/g/mm9',
 155                    '/g/mm9',
 156                    '/g/elegans190',
 157                    '/g/arabidopsis01222004',
 158                    '/g/mm9',
 159                    '/g/mm9',
 160                    '/g/mm9',
 161                    '/g/mm9', ]
 162
 163         # test lane specific parameters from gerald config file
 164         for i in range(1,9):
 165             cur_lane = g.lanes[i]
 166             self.failUnlessEqual(cur_lane.analysis, 'eland_extended')
 167             self.failUnlessEqual(cur_lane.eland_genome, genomes[i])
 168             self.failUnlessEqual(cur_lane.read_length, '37')
 169             self.failUnlessEqual(cur_lane.use_bases, 'Y'*37)
 170
 171         # I want to be able to use a simple iterator
 172         for l in g.lanes.values():
 173           self.failUnlessEqual(l.analysis, 'eland_extended')
 174           self.failUnlessEqual(l.read_length, '37')
 175           self.failUnlessEqual(l.use_bases, 'Y'*37)
 176
 177         # test data extracted from summary file
 178         clusters = [None,
 179                     (126910, 4300), (165739, 6792),
 180                     (196565, 8216), (153897, 8501),
 181                     (135536, 3908), (154083, 9315),
 182                     (159991, 9292), (198479, 17671),]
 183
 184         self.failUnlessEqual(len(g.summary), 1)
 185         for i in range(1,9):
 186             summary_lane = g.summary[0][i]
 187             self.failUnlessEqual(summary_lane.cluster, clusters[i])
 188             self.failUnlessEqual(summary_lane.lane, i)
 189
 190         xml = g.get_elements()
 191         # just make sure that element tree can serialize the tree
 192         xml_str = ElementTree.tostring(xml)
 193         g2 = gerald.Gerald(xml=xml)
 194
 195         # do it all again after extracting from the xml file
 196         self.failUnlessEqual(g.version, g2.version)
 197         self.failUnlessEqual(g.date, g2.date)
 198         self.failUnlessEqual(len(g.lanes.keys()), len(g2.lanes.keys()))
 199         self.failUnlessEqual(len(g.lanes.items()), len(g2.lanes.items()))
 200
 201         # test lane specific parameters from gerald config file
 202         for i in range(1,9):
 203             g_lane = g.lanes[i]
 204             g2_lane = g2.lanes[i]
 205             self.failUnlessEqual(g_lane.analysis, g2_lane.analysis)
 206             self.failUnlessEqual(g_lane.eland_genome, g2_lane.eland_genome)
 207             self.failUnlessEqual(g_lane.read_length, g2_lane.read_length)
 208             self.failUnlessEqual(g_lane.use_bases, g2_lane.use_bases)
 209
 210         # test (some) summary elements
 211         self.failUnlessEqual(len(g.summary), 1)
 212         for i in range(1,9):
 213             g_summary = g.summary[0][i]
 214             g2_summary = g2.summary[0][i]
 215             self.failUnlessEqual(g_summary.cluster, g2_summary.cluster)
 216             self.failUnlessEqual(g_summary.lane, g2_summary.lane)
 217
 218             g_eland = g.eland_results
 219             g2_eland = g2.eland_results
 220             for lane in g_eland.results[0].keys():
 221                 g_results = g_eland.results[0][lane]
 222                 g2_results = g2_eland.results[0][lane]
 223                 self.failUnlessEqual(g_results.reads,
 224                                      g2_results.reads)
 225                 self.failUnlessEqual(len(g_results.mapped_reads),
 226                                      len(g2_results.mapped_reads))
 227                 for k in g_results.mapped_reads.keys():
 228                     self.failUnlessEqual(g_results.mapped_reads[k],
 229                                          g2_results.mapped_reads[k])
 230
 231                 self.failUnlessEqual(len(g_results.match_codes),
 232                                      len(g2_results.match_codes))
 233                 for k in g_results.match_codes.keys():
 234                     self.failUnlessEqual(g_results.match_codes[k],
 235                                          g2_results.match_codes[k])
 236
 237
 238     def test_eland(self):
 239         hg_map = {'Lambda.fa': 'Lambda.fa'}
 240         for i in range(1,22):
 241           short_name = 'chr%d.fa' % (i,)
 242           long_name = 'hg18/chr%d.fa' % (i,)
 243           hg_map[short_name] = long_name
 244
 245         genome_maps = { 1:hg_map, 2:hg_map, 3:hg_map, 4:hg_map,
 246                         5:hg_map, 6:hg_map, 7:hg_map, 8:hg_map }
 247         eland = gerald.eland(self.gerald_dir, genome_maps=genome_maps)
 248
 249         for i in range(1,9):
 250             lane = eland.results[0][i]
 251             self.failUnlessEqual(lane.reads, 6)
 252             self.failUnlessEqual(lane.sample_name, "s")
 253             self.failUnlessEqual(lane.lane_id, i)
 254             self.failUnlessEqual(len(lane.mapped_reads), 17)
 255             self.failUnlessEqual(lane.mapped_reads['hg18/chr5.fa'], 4)
 256             self.failUnlessEqual(lane.match_codes['U0'], 3)
 257             self.failUnlessEqual(lane.match_codes['R0'], 2)
 258             self.failUnlessEqual(lane.match_codes['U1'], 1)
 259             self.failUnlessEqual(lane.match_codes['R1'], 9)
 260             self.failUnlessEqual(lane.match_codes['U2'], 0)
 261             self.failUnlessEqual(lane.match_codes['R2'], 12)
 262             self.failUnlessEqual(lane.match_codes['NM'], 1)
 263             self.failUnlessEqual(lane.match_codes['QC'], 0)
 264
 265         xml = eland.get_elements()
 266         # just make sure that element tree can serialize the tree
 267         xml_str = ElementTree.tostring(xml)
 268         e2 = gerald.ELAND(xml=xml)
 269
 270         for i in range(1,9):
 271             l1 = eland.results[0][i]
 272             l2 = e2.results[0][i]
 273             self.failUnlessEqual(l1.reads, l2.reads)
 274             self.failUnlessEqual(l1.sample_name, l2.sample_name)
 275             self.failUnlessEqual(l1.lane_id, l2.lane_id)
 276             self.failUnlessEqual(len(l1.mapped_reads), len(l2.mapped_reads))
 277             self.failUnlessEqual(len(l1.mapped_reads), 17)
 278             for k in l1.mapped_reads.keys():
 279                 self.failUnlessEqual(l1.mapped_reads[k],
 280                                      l2.mapped_reads[k])
 281
 282             self.failUnlessEqual(len(l1.match_codes), 9)
 283             self.failUnlessEqual(len(l1.match_codes), len(l2.match_codes))
 284             for k in l1.match_codes.keys():
 285                 self.failUnlessEqual(l1.match_codes[k],
 286                                      l2.match_codes[k])
 287
 288     def test_runfolder(self):
 289         runs = runfolder.get_runs(self.runfolder_dir)
 290
 291         # do we get the flowcell id from the filename?
 292         self.failUnlessEqual(len(runs), 1)
 293         name = 'run_3021JAAXX_%s.xml' % ( date.today().strftime('%Y-%m-%d'),)
 294         self.failUnlessEqual(runs[0].name, name)
 295
 296         # do we get the flowcell id from the FlowcellId.xml file
 297         make_flowcell_id(self.runfolder_dir, '207BTAAXY')
 298         runs = runfolder.get_runs(self.runfolder_dir)
 299         self.failUnlessEqual(len(runs), 1)
 300         name = 'run_207BTAAXY_%s.xml' % ( date.today().strftime('%Y-%m-%d'),)
 301         self.failUnlessEqual(runs[0].name, name)
 302
 303         r1 = runs[0]
 304         xml = r1.get_elements()
 305         xml_str = ElementTree.tostring(xml)
 306
 307         r2 = runfolder.PipelineRun(xml=xml)
 308         self.failUnlessEqual(r1.name, r2.name)
 309         self.failIfEqual(r2.image_analysis, None)
 310         self.failIfEqual(r2.bustard, None)
 311         self.failIfEqual(r2.gerald, None)
 312
 313
 314 def suite():
 315     return unittest.makeSuite(RunfolderTests,'test')
 316
 317 if __name__ == "__main__":
 318     unittest.main(defaultTest="suite")
 319