htsworkflow/pipelines/test/simulate_runfolder.py

   1 """
   2 Create simulated solexa/illumina runfolders for testing
   3 """
   4 import gzip
   5 import os
   6 import shutil
   7
   8 TEST_CODE_DIR = os.path.split(__file__)[0]
   9 TESTDATA_DIR = os.path.join(TEST_CODE_DIR, 'testdata')
  10 LANE_LIST = range(1,9)
  11 TILE_LIST = range(1,101)
  12 HISEQ_TILE_LIST = [1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108,
  13                    1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208,
  14                    2101, 2102, 2103, 2104, 2105, 2106, 2107, 2108,
  15                    2201, 2202, 2203, 2204, 2205, 2206, 2207, 2208,]
  16
  17 def make_firecrest_dir(data_dir, version="1.9.2", start=1, stop=37):
  18     firecrest_dir = os.path.join(data_dir,
  19                                  'C%d-%d_Firecrest%s_12-04-2008_diane' % (start, stop, version)
  20                                  )
  21     os.mkdir(firecrest_dir)
  22     return firecrest_dir
  23
  24 def make_ipar_dir(data_dir, version='1.01'):
  25     """
  26     Construct an artificial ipar parameter file and directory
  27     """
  28     ipar1_01_file = os.path.join(TESTDATA_DIR, 'IPAR1.01.params')
  29     shutil.copy(ipar1_01_file, os.path.join(data_dir, '.params'))
  30
  31     ipar_dir = os.path.join(data_dir, 'IPAR_%s' % (version,))
  32     if not os.path.exists(ipar_dir):
  33       os.mkdir(ipar_dir)
  34     return ipar_dir
  35
  36 def make_flowcell_id(runfolder_dir, flowcell_id=None):
  37     if flowcell_id is None:
  38         flowcell_id = '207BTAAXY'
  39
  40     config = """<?xml version="1.0"?>
  41 <FlowcellId>
  42   <Text>%s</Text>
  43 </FlowcellId>""" % (flowcell_id,)
  44     config_dir = os.path.join(runfolder_dir, 'Config')
  45
  46     if not os.path.exists(config_dir):
  47         os.mkdir(config_dir)
  48     pathname = os.path.join(config_dir, 'FlowcellId.xml')
  49     f = open(pathname,'w')
  50     f.write(config)
  51     f.close()
  52
  53 def make_runinfo(runfolder_dir, flowcell_id):
  54     """Simulate a RunInfo.xml file created by >= RTA 1.9
  55     """
  56     xml = '''<?xml version="1.0"?>
  57 <RunInfo xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" Version="2">
  58   <Run Id="{runfolder}" Number="101">
  59     <Flowcell>{flowcell}</Flowcell>
  60     <Instrument>SN787</Instrument>
  61     <Date>110815</Date>
  62     <Reads>
  63       <Read Number="1" NumCycles="50" IsIndexedRead="N" />
  64       <Read Number="2" NumCycles="7" IsIndexedRead="Y" />
  65     </Reads>
  66     <FlowcellLayout LaneCount="8" SurfaceCount="2" SwathCount="3" TileCount="8" />
  67     <AlignToPhiX />
  68   </Run>
  69 </RunInfo>
  70 '''
  71     path, runfolder = os.path.split(runfolder_dir)
  72     runinfo = os.path.join(runfolder_dir, 'RunInfo.xml')
  73     stream = open(runinfo, 'w')
  74     stream.write(xml.format(runfolder=runfolder, flowcell=flowcell_id))
  75     stream.close()
  76     return runinfo
  77
  78 def make_bustard_config132(image_dir):
  79     source = os.path.join(TESTDATA_DIR, 'bustard-config132.xml')
  80     destination = os.path.join(image_dir, 'config.xml')
  81     shutil.copy(source, destination)
  82
  83 def make_aligned_config_1_12(aligned_dir):
  84     """This is rouglhly equivalent to the old gerald file"""
  85     source = os.path.join(TESTDATA_DIR, '1_12', 'aligned_config_1_12.xml')
  86     destination = os.path.join(aligned_dir, 'config.xml')
  87     shutil.copy(source, destination)
  88
  89 def make_unaligned_config_1_12(unaligned_dir):
  90     demultiplex_pairs = [ # (src,
  91       # dest),
  92         (os.path.join(TESTDATA_DIR, '1_12', 'demultiplex_1.12.4.2.xml'),
  93          os.path.join(unaligned_dir, 'DemultiplexConfig.xml')),
  94         (os.path.join(TESTDATA_DIR, '1_12',
  95                       'demultiplexed_bustard_1.12.4.2.xml'),
  96          os.path.join(unaligned_dir, 'DemultiplexedBustardConfig.xml')),
  97         (os.path.join(TESTDATA_DIR, '1_12',
  98                       'demultiplexed_summary_1.12.4.2.xml'),
  99          os.path.join(unaligned_dir, 'DemultiplexedBustardSummary.xml')),
 100     ]
 101     for src, dest in demultiplex_pairs:
 102         shutil.copy(src, dest)
 103
 104 def make_rta_intensities_1460(data_dir, version='1.4.6.0'):
 105     """
 106     Construct an artificial RTA Intensities parameter file and directory
 107     """
 108     intensities_dir = os.path.join(data_dir, 'Intensities')
 109     if not os.path.exists(intensities_dir):
 110       os.mkdir(intensities_dir)
 111
 112     param_file = os.path.join(TESTDATA_DIR, 'rta_intensities_config.xml')
 113     shutil.copy(param_file, os.path.join(intensities_dir, 'config.xml'))
 114
 115     return intensities_dir
 116
 117 def make_rta_basecalls_1460(intensities_dir):
 118     """
 119     Construct an artificial RTA Intensities parameter file and directory
 120     """
 121     basecalls_dir = os.path.join(intensities_dir, 'BaseCalls')
 122     if not os.path.exists(basecalls_dir):
 123       os.mkdir(basecalls_dir)
 124
 125     param_file = os.path.join(TESTDATA_DIR, 'rta_basecalls_config.xml')
 126     shutil.copy(param_file, os.path.join(basecalls_dir, 'config.xml'))
 127
 128     return basecalls_dir
 129
 130 def make_rta_intensities_1870(data_dir, version='1.8.70.0'):
 131     """
 132     Construct an artificial RTA Intensities parameter file and directory
 133     """
 134     intensities_dir = os.path.join(data_dir, 'Intensities')
 135     if not os.path.exists(intensities_dir):
 136       os.mkdir(intensities_dir)
 137
 138     param_file = os.path.join(TESTDATA_DIR, 'rta_intensities_config_1870.xml')
 139     shutil.copy(param_file, os.path.join(intensities_dir, 'config.xml'))
 140
 141     return intensities_dir
 142
 143 def make_rta_intensities_1_10(data_dir, version='1.10.36.0'):
 144     """
 145     Construct an artificial RTA Intensities parameter file and directory
 146     """
 147     intensities_dir = os.path.join(data_dir, 'Intensities')
 148     if not os.path.exists(intensities_dir):
 149       os.mkdir(intensities_dir)
 150
 151     param_file = os.path.join(TESTDATA_DIR, 'rta_intensities_config_1.10.xml')
 152     shutil.copy(param_file, os.path.join(intensities_dir, 'config.xml'))
 153
 154     return intensities_dir
 155
 156 def make_rta_intensities_1_12(data_dir, version='1.12.4.2'):
 157     """
 158     Construct an artificial RTA Intensities parameter file and directory
 159     """
 160     intensities_dir = os.path.join(data_dir, 'Intensities')
 161     if not os.path.exists(intensities_dir):
 162       os.mkdir(intensities_dir)
 163
 164     param_file = os.path.join(TESTDATA_DIR, '1_12',
 165                               'rta_intensities_config_1.12.4.2.xml')
 166     shutil.copy(param_file, os.path.join(intensities_dir, 'RTAConfig.xml'))
 167
 168     return intensities_dir
 169
 170 def make_rta_basecalls_1870(intensities_dir):
 171     """
 172     Construct an artificial RTA Intensities parameter file and directory
 173     """
 174     basecalls_dir = os.path.join(intensities_dir, 'BaseCalls')
 175     if not os.path.exists(basecalls_dir):
 176       os.mkdir(basecalls_dir)
 177
 178     param_file = os.path.join(TESTDATA_DIR, 'rta_basecalls_config_1870.xml')
 179     shutil.copy(param_file, os.path.join(basecalls_dir, 'config.xml'))
 180
 181     return basecalls_dir
 182
 183 def make_rta_basecalls_1_10(intensities_dir):
 184     """
 185     Construct an artificial RTA Intensities parameter file and directory
 186     """
 187     basecalls_dir = os.path.join(intensities_dir, 'BaseCalls')
 188     if not os.path.exists(basecalls_dir):
 189         os.mkdir(basecalls_dir)
 190
 191     make_qseqs(basecalls_dir, basecall_info=ABXX_BASE_CALL_INFO)
 192     param_file = os.path.join(TESTDATA_DIR, 'rta_basecalls_config_1.10.xml')
 193     shutil.copy(param_file, os.path.join(basecalls_dir, 'config.xml'))
 194
 195     return basecalls_dir
 196
 197 def make_rta_basecalls_1_12(intensities_dir):
 198     """
 199     Construct an artificial RTA Intensities parameter file and directory
 200     """
 201     basecalls_dir = os.path.join(intensities_dir, 'BaseCalls')
 202     if not os.path.exists(basecalls_dir):
 203         os.mkdir(basecalls_dir)
 204
 205     make_qseqs(basecalls_dir, basecall_info=ABXX_BASE_CALL_INFO)
 206     param_file = os.path.join(TESTDATA_DIR, '1_12',
 207                               'rta_basecalls_config_1.12.4.2.xml')
 208     shutil.copy(param_file, os.path.join(basecalls_dir, 'config.xml'))
 209
 210     return basecalls_dir
 211
 212
 213 def make_qseqs(bustard_dir, basecall_info=None):
 214     """
 215     Fill gerald directory with qseq files
 216     """
 217     if basecall_info is None:
 218         qseq_file = '42BRJAAXX_8_1_0039_qseq.txt'
 219         tile_list = TILE_LIST
 220         summary_file = '42BRJAAXX_BustardSummary.xml'
 221     else:
 222         qseq_file = basecall_info.qseq_file
 223         tile_list = basecall_info.tile_list
 224         summary_file = basecall_info.basecall_summary
 225
 226     # 42BRJ 8 1 0039 happened to be a better than usual tile, in that there
 227     # was actually sequence at the start
 228     source = os.path.join(TESTDATA_DIR, qseq_file)
 229     destdir = bustard_dir
 230     if not os.path.isdir(destdir):
 231         os.mkdir(destdir)
 232
 233     for lane in LANE_LIST:
 234         for tile in tile_list:
 235             destination = os.path.join(bustard_dir, 's_%d_1_%04d_qseq.txt' % (lane, tile))
 236             shutil.copy(source, destination)
 237
 238     make_matrix_dir(bustard_dir)
 239     make_phasing_dir(bustard_dir)
 240
 241     summary_source = os.path.join(TESTDATA_DIR, summary_file)
 242     summary_dest = os.path.join(bustard_dir, 'BustardSummary.xml')
 243     shutil.copy(summary_source, summary_dest)
 244
 245     return destdir
 246
 247 def make_scores(gerald_dir, in_temp=True):
 248     """
 249     Fill gerald directory with score temp files
 250     will create the directory if it doesn't exist.
 251     """
 252     source = os.path.join(TESTDATA_DIR, 's_1_0001_score.txt')
 253     destdir = gerald_dir
 254     if in_temp:
 255         destdir = os.path.join(destdir, 'Temp')
 256     if not os.path.isdir(destdir):
 257         os.mkdir(destdir)
 258
 259     for lane in LANE_LIST:
 260         for tile in TILE_LIST:
 261             destination = os.path.join(destdir, 's_%d_%04d_score.txt' % (lane, tile))
 262             shutil.copy(source, destination)
 263
 264     return destdir
 265
 266 def make_matrix_dir(bustard_dir):
 267     """
 268     Create several matrix files in <bustard_dir>/Matrix/
 269
 270     from pipeline 1.4
 271     """
 272     destdir = os.path.join(bustard_dir, 'Matrix')
 273     if not os.path.isdir(destdir):
 274         os.mkdir(destdir)
 275
 276     source = os.path.join(TESTDATA_DIR, '42BRJAAXX_8_02_matrix.txt')
 277     for lane in LANE_LIST:
 278         destination = os.path.join(destdir, 's_%d_02_matrix.txt' % ( lane, ))
 279         shutil.copy(source, destination)
 280
 281 def make_matrix(matrix_filename):
 282     contents = """# Auto-generated frequency response matrix
 283 > A
 284 > C
 285 > G
 286 > T
 287 0.77 0.15 -0.04 -0.04
 288 0.76 1.02 -0.05 -0.06
 289 -0.10 -0.10 1.17 -0.03
 290 -0.13 -0.12 0.80 1.27
 291 """
 292     f = open(matrix_filename, 'w')
 293     f.write(contents)
 294     f.close()
 295
 296 def make_matrix_dir_rta160(bustard_dir):
 297     """
 298     Create several matrix files in <bustard_dir>/Matrix/
 299     """
 300     destdir = os.path.join(bustard_dir, 'Matrix')
 301     if not os.path.isdir(destdir):
 302         os.mkdir(destdir)
 303
 304     source = os.path.join(TESTDATA_DIR, '61MMFAAXX_4_1_matrix.txt')
 305     lane_fragments = [ "_%d" % (l,) for l in LANE_LIST]
 306     for fragment in lane_fragments:
 307         destination = os.path.join(destdir, 's%s_1_matrix.txt' % ( fragment, ))
 308         shutil.copy(source, destination)
 309
 310 def make_matrix_dir_rta_1_10(bustard_dir):
 311     make_matrix_dir_rta160(bustard_dir)
 312
 313 def make_matrix_dir_rta_1_12(bustard_dir):
 314     make_matrix_dir_rta160(bustard_dir)
 315
 316 def make_phasing_dir(bustard_dir):
 317     """
 318     Create several phasing files in <bustard_dir>/Phasing/
 319
 320     from pipeline 1.4
 321     """
 322     destdir = os.path.join(bustard_dir, 'Phasing')
 323     if not os.path.isdir(destdir):
 324         os.mkdir(destdir)
 325
 326     source = os.path.join(TESTDATA_DIR, '42BRJAAXX_8_01_phasing.xml')
 327     for lane in LANE_LIST:
 328         destination = os.path.join(destdir, 's_%d_01_phasing.xml' % ( lane, ))
 329         shutil.copy(source, destination)
 330
 331 def make_phasing_params(bustard_dir):
 332     for lane in LANE_LIST:
 333         pathname = os.path.join(bustard_dir, 'params%d.xml' % (lane))
 334         f = open(pathname, 'w')
 335         f.write("""<Parameters>
 336   <Phasing>0.009900</Phasing>
 337   <Prephasing>0.003500</Prephasing>
 338 </Parameters>
 339 """)
 340         f.close()
 341
 342 def make_gerald_config_026(gerald_dir):
 343     source = os.path.join(TESTDATA_DIR, 'gerald_config_0.2.6.xml')
 344     destination = os.path.join(gerald_dir, 'config.xml')
 345     shutil.copy(source, destination)
 346
 347 def make_gerald_config_100(gerald_dir):
 348     source = os.path.join(TESTDATA_DIR, 'gerald_config_1.0.xml')
 349     destination = os.path.join(gerald_dir, 'config.xml')
 350     shutil.copy(source, destination)
 351
 352 def make_gerald_config_1_7(gerald_dir):
 353     """CASAVA 1.7 gerald config"""
 354     source = os.path.join(TESTDATA_DIR, 'gerald_config_1.7.xml')
 355     destination = os.path.join(gerald_dir, 'config.xml')
 356     shutil.copy(source, destination)
 357
 358 def make_summary_htm_100(gerald_dir):
 359     source = os.path.join(TESTDATA_DIR, 'Summary-pipeline100.htm')
 360     destination = os.path.join(gerald_dir, 'Summary.htm')
 361     shutil.copy(source, destination)
 362
 363 def make_summary_htm_110(gerald_dir):
 364     source = os.path.join(TESTDATA_DIR, 'Summary-pipeline110.htm')
 365     destination = os.path.join(gerald_dir, 'Summary.htm')
 366     shutil.copy(source, destination)
 367
 368 def make_summary_paired_htm(gerald_dir):
 369     source = os.path.join(TESTDATA_DIR, 'Summary-paired-pipeline110.htm')
 370     destination = os.path.join(gerald_dir, 'Summary.htm')
 371     shutil.copy(source, destination)
 372
 373 def make_summary_ipar130_htm(gerald_dir):
 374     source = os.path.join(TESTDATA_DIR, 'Summary-ipar130.htm')
 375     destination = os.path.join(gerald_dir, 'Summary.htm')
 376     shutil.copy(source, destination)
 377
 378 def make_summary_rta160_xml(gerald_dir):
 379     source = os.path.join(TESTDATA_DIR, 'Summary-rta160.xml')
 380     destination = os.path.join(gerald_dir, 'Summary.xml')
 381     shutil.copy(source, destination)
 382
 383
 384 def make_summary_casava1_7_xml(gerald_dir):
 385     source = os.path.join(TESTDATA_DIR, 'Summary-casava1.7.xml')
 386     destination = os.path.join(gerald_dir, 'Summary.xml')
 387     shutil.copy(source, destination)
 388
 389 def make_status_rta1_12(datadir):
 390     sourcedir = os.path.join(TESTDATA_DIR, '1_12')
 391     status_htm = os.path.join(sourcedir, 'Status.htm')
 392     destination = os.path.join(datadir, 'Status.htm')
 393     shutil.copy(status_htm, destination)
 394
 395     status_dir = os.path.join(datadir, 'Status_Files')
 396     status_source_dir = os.path.join(sourcedir, 'Status_Files')
 397     shutil.copytree(status_source_dir, status_dir)
 398
 399     report_source_dir = os.path.join(sourcedir, 'reports')
 400     report_dir = os.path.join(datadir, 'reports')
 401     shutil.copytree(report_source_dir, report_dir)
 402
 403 def make_eland_results(gerald_dir):
 404     eland_result = """>HWI-EAS229_24_207BTAAXX:1:7:599:759    ACATAGNCACAGACATAAACATAGACATAGAC U0      1       1       3       chrUextra.fa    28189829        R       D.
 405 >HWI-EAS229_24_207BTAAXX:1:7:205:842    AAACAANNCTCCCAAACACGTAAACTGGAAAA  U1      0       1       0       chr2L.fa        8796855 R       DD      24T
 406 >HWI-EAS229_24_207BTAAXX:1:7:776:582    AGCTCANCCGATCGAAAACCTCNCCAAGCAAT        NM      0       0       0
 407 >HWI-EAS229_24_207BTAAXX:1:7:205:842    AAACAANNCTCCCAAACACGTAAACTGGAAAA        U1      0       1       0       Lambda.fa        8796855 R       DD      24T
 408 """
 409     for i in LANE_LIST:
 410         pathname = os.path.join(gerald_dir,
 411                                 's_%d_eland_result.txt' % (i,))
 412         f = open(pathname, 'w')
 413         f.write(eland_result)
 414         f.close()
 415
 416 def make_eland_multi(gerald_dir, paired=False, lane_list=LANE_LIST):
 417     eland_multi = [""">HWI-EAS229_60_30DP9AAXX:1:1:1221:788   AAGATATCTACGACGTGGTATGGCGGTGTCTGGTCGT      NM
 418 >HWI-EAS229_60_30DP9AAXX:1:1:931:747    AAAAAAGCAAATTTCATTCACATGTTCTGTGTTCATA   1:0:2   chr5.fa:55269838R0
 419 >HWI-EAS229_60_30DP9AAXX:1:1:1121:379   AGAAGAGACATTAAGAGTTCCTGAAATTTATATCTGG   2:1:0   chr16.fa:46189180R1,chr7.fa:122968519R0,chr8.fa:48197174F0
 420 >HWI-EAS229_60_30DP9AAXX:1:1:892:1155   ACATTCTCCTTTCCTTCTGAAGTTTTTACGATTCTTT   0:9:10  chr10.fa:114298201F1,chr12.fa:8125072F1,19500297F2,42341293R2,chr13.fa:27688155R2,95069772R1,chr15.fa:51016475F2,chr16.fa:27052155F2,chr1.fa:192426217R2,chr21.fa:23685310R2,chr2.fa:106680068F1,chr3.fa:185226695F2,chr4.fa:106626808R2,chr5.fa:14704894F1,43530779F1,126543189F2,chr6.fa:74284101F1,chr7.fa:22516603F1,chr9.fa:134886204R
 421 >HWI-EAS229_60_30DP9AAXX:1:1:931:747    AAAAAAGCAAATTTCATTCACATGTTCTGTGTTCATA   1:0:0   spike.fa/sample1:55269838R0
 422 >HWI-EAS229_60_30DP9AAXX:1:1:931:747    AAAAAAGCAAATTTCATTCACATGTTCTGTGTTCATA   1:0:0   spike.fa/sample2:55269838R0
 423 """, """>HWI-EAS229_60_30DP9AAXX:1:1:1221:788   AAGATATCTACGACGTGGTATGGCGGTGTCTGGTCGT      NM
 424 >HWI-EAS229_60_30DP9AAXX:1:1:1221:788   NNNNNNNNNNNNNNGTGGTATGGCGGTGTCTGGTCGT     QC
 425 >HWI-EAS229_60_30DP9AAXX:1:1:931:747    AAAAAAGCAAATTTCATTCACATGTTCTGTGTTCATA   1:0:2   chr5.fa:55269838R0
 426 >HWI-EAS229_60_30DP9AAXX:1:1:1121:379   AGAAGAGACATTAAGAGTTCCTGAAATTTATATCTGG   2:1:0   chr16.fa:46189180R1,chr7.fa:122968519R0,chr8.fa:48197174F0,chr7.fa:22516603F1,chr9.fa:134886204R
 427 >HWI-EAS229_60_30DP9AAXX:1:1:892:1155   ACATTCTCCTTTCCTTCTGAAGTTTTTACGATTCTTT   0:9:10  chr10.fa:114298201F1,chr12.fa:8125072F1,19500297F2,42341293R2,chr13.fa:27688155R2,95069772R1,chr15.fa:51016475F2,chr16.fa:27052155F2,chr1.fa:192426217R2,chr21.fa:23685310R2,chr2.fa:106680068F1,chr3.fa:185226695F2,chr4.fa:106626808R2,chr5.fa:14704894F1,43530779F1,126543189F2,chr6.fa:74284101F1
 428 >HWI-EAS229_60_30DP9AAXX:1:1:931:747    AAAAAAGCAAATTTCATTCACATGTTCTGTGTTCATA   1:0:0   spike.fa/sample1:55269838R0
 429 >HWI-EAS229_60_30DP9AAXX:1:1:931:747    AAAAAAGCAAATTTCATTCACATGTTCTGTGTTCATA   1:0:0   spike.fa/sample2:55269838R0
 430 """]
 431     if paired:
 432         for e in [1,2]:
 433             for i in lane_list:
 434                 pathname = os.path.join(gerald_dir,
 435                                         's_%d_%d_eland_multi.txt' % (i,e))
 436                 f = open(pathname, 'w')
 437                 f.write(eland_multi[e-1])
 438                 f.close()
 439     else:
 440         for i in lane_list:
 441             pathname = os.path.join(gerald_dir,
 442                                     's_%d_eland_multi.txt' % (i,))
 443             f = open(pathname, 'w')
 444             f.write(eland_multi[0])
 445             f.close()
 446
 447 def make_eland_export(gerald_dir, paired=False, lane_list=LANE_LIST):
 448     source = os.path.join(TESTDATA_DIR, 'casava_1.7_export.txt')
 449
 450     for i in lane_list:
 451         destination = os.path.join(gerald_dir,
 452                                    's_%d_export.txt' % (i,))
 453         shutil.copy(source, destination)
 454
 455
 456 def make_scarf(gerald_dir, lane_list=LANE_LIST):
 457     seq = """HWI-EAS229_92_30VNBAAXX:1:1:0:161:NCAATTACACGACGCTAGCCCTAAAGCTATTTCGAGG:E[aaaabb^a\a_^^a[S`ba_WZUXaaaaaaUKPER
 458 HWI-EAS229_92_30VNBAAXX:1:1:0:447:NAGATGCGCATTTGAAGTAGGAGCAAAAGATCAAGGT:EUabaab^baabaaaaaaaa^^Uaaaaa\aaaa__`a
 459 HWI-EAS229_92_30VNBAAXX:1:1:0:1210:NATAGCCTCTATAGAAGCCACTATTATTTTTTTCTTA:EUa`]`baaaaa^XQU^a`S``S_`J_aaaaaabb^V
 460 HWI-EAS229_92_30VNBAAXX:1:1:0:1867:NTGGAGCAGATATAAAAACAGATGGTGACGTTGAAGT:E[^UaaaUaba^aaa^aa^XV\baaLaLaaaaQVXV^
 461 HWI-EAS229_92_30VNBAAXX:1:1:0:1898:NAGCTCGTGTCGTGAGATGTTAGGTTAAGTCCTGCAA:EK_aaaaaaaaaaaUZaaZaXM[aaaXSM\aaZ]URE
 462 """
 463     for l in lane_list:
 464         pathname = os.path.join(gerald_dir, 's_%d_sequence.txt' %(l,))
 465         f = open(pathname,'w')
 466         f.write(seq)
 467         f.close()
 468
 469 def make_fastq(gerald_dir, lane_list=LANE_LIST):
 470     seq = """@HWI-EAS229:1:2:182:712#0/1
 471 AAAAAAAAAAAAAAAAAAAAANAAAAAAAAAAAAAAA
 472 +HWI-EAS229:1:2:182:712#0/1
 473 \\bab_bbaabbababbaaa]]D]bb_baabbab\baa
 474 @HWI-EAS229:1:2:198:621#0/1
 475 CCCCCCCCCCCCCCCCCCCCCNCCCCCCCCCCCCCCC
 476 +HWI-EAS229:1:2:198:621#0/1
 477 [aaaaaaa`_`aaaaaaa[`ZDZaaaaaaaaaaaaaa
 478 @HWI-EAS229:1:2:209:1321#0/1
 479 AAAAAAAAAAAAAAAAAAAAANAAAAAAAAAAAAAAA
 480 +HWI-EAS229:1:2:209:1321#0/1
 481 _bbbbbaaababaabbbbab]D]aaaaaaaaaaaaaa
 482 """
 483     for l in lane_list:
 484         pathname = os.path.join(gerald_dir, 's_%d_sequence.txt' %(l,))
 485         f = open(pathname,'w')
 486         f.write(seq)
 487         f.close()
 488
 489 UNALIGNED_READS = [1,2]
 490 UNALIGNED_SAMPLES = [ (1, UNALIGNED_READS, '11111', None, None),
 491                       (2, UNALIGNED_READS, '11112', None, None),
 492                       (3, UNALIGNED_READS, '11113', 1, 'ATCACG'),
 493                       (3, UNALIGNED_READS, '11113', 2, 'CGATGT'),
 494                       (3, UNALIGNED_READS, '11113', 3, 'TTAGGC'),
 495                       (4, UNALIGNED_READS, '11114', 6, 'GCCAAT'),
 496                       (5, UNALIGNED_READS, '11115', 1, 'ATCACG'),
 497                       (5, UNALIGNED_READS, '11116', 7, 'ACTTGA'),
 498                       (5, UNALIGNED_READS, '11117', 9, 'GATCAG'),
 499                       (6, UNALIGNED_READS, '11118', 1, 'ATCACG'),
 500                       (7, UNALIGNED_READS, '11119', 2, 'CGATGT'),
 501                       (8, UNALIGNED_READS, '11120', 3, 'TTAGGC'),
 502                       (1, UNALIGNED_READS, None, None, None),
 503                       (2, UNALIGNED_READS, None, None, None),
 504                       (3, UNALIGNED_READS, None, None, None),
 505                       (4, UNALIGNED_READS, None, None, None),
 506                       (5, UNALIGNED_READS, None, None, None)]
 507
 508
 509 def make_aligned_eland_export(aligned_dir, flowcell_id):
 510     summary_source = os.path.join(TESTDATA_DIR, 'sample_summary_1_12.htm')
 511     for lane, read, project_id, index_id, index_seq in UNALIGNED_SAMPLES:
 512         paths = DemultiplexedPaths(aligned_dir,
 513                                    flowcell_id,
 514                                    lane,
 515                                    project_id,
 516                                    index_id,
 517                                    index_seq)
 518         paths.make_sample_dirs()
 519         paths.make_summary_dirs()
 520         summary_dest = os.path.join(paths.summary_dir, 'Sample_Summary.htm')
 521         shutil.copy(summary_source, summary_dest)
 522
 523         body = get_aligned_sample_export(lane, index_seq)
 524         for split in ['001','002']:
 525             for read in UNALIGNED_READS:
 526                 suffix = 'R{0}_{1}_export.txt.gz'.format(read, split)
 527                 pathname = paths.make_test_filename(suffix)
 528                 stream = gzip.open(pathname, 'w')
 529                 stream.write(body)
 530                 stream.close()
 531
 532
 533 def make_unaligned_fastqs_1_12(unaligned_dir, flowcell_id):
 534     """Create a default mix of unaligned sample files
 535     """
 536     for lane, read, name, index_id, index in UNALIGNED_SAMPLES:
 537         make_unaligned_fastq_sample_1_12(unaligned_dir,
 538                                          flowcell_id,
 539                                          lane,
 540                                          read,
 541                                          name,
 542                                          index_id,
 543                                          index)
 544
 545 def make_unaligned_fastq_sample_1_12(unaligned_dir,
 546                                      flowcell_id,
 547                                      lane,
 548                                      reads,
 549                                      project_id,
 550                                      index_id=None,
 551                                      index_seq=None):
 552
 553     paths = DemultiplexedPaths(unaligned_dir,
 554                                flowcell_id,
 555                                lane,
 556                                project_id,
 557                                index_id,
 558                                index_seq)
 559     paths.make_sample_dirs()
 560
 561     sample_seq = get_unaligned_sample_fastq_data(flowcell_id, lane, index_seq)
 562     for split in ['001','002']:
 563         for read in reads:
 564             suffix = 'R{0}_{1}.fastq.gz'.format(read, split)
 565             pathname = paths.make_test_filename(suffix)
 566             stream = gzip.open(pathname, 'w')
 567             stream.write(sample_seq)
 568             stream.close()
 569
 570     sheetname = os.path.join(paths.sample_dir, 'SampleSheet.csv')
 571     stream = open(sheetname, 'w')
 572     stream.write('FCID,Lane,SampleID,SampleRef,Index,Description,Control,Recipe,Operator,SampleProject'+os.linesep)
 573     template = '{flowcell},{lane},{id},mm9,{index},Sample #{id},N,PR_indexing,Operator,{sample_project}'+os.linesep
 574     stream.write(template.format(flowcell=flowcell_id,
 575                                  lane=lane,
 576                                  id=paths.sample_id,
 577                                  index=paths.index_seq,
 578                                  sample_project=paths.sample_project))
 579     stream.close()
 580
 581
 582 class DemultiplexedPaths(object):
 583     def __init__(self, basedir, flowcell_id, lane, project_id, index_id, index_seq):
 584         if lane not in LANE_LIST:
 585             raise ValueError("Invalid lane ID: {0}".format(lane))
 586         self.basedir = basedir
 587         self.flowcell_id = flowcell_id
 588         self.lane = lane
 589
 590         if project_id is None:
 591             # undetermined
 592             self.index_seq = ''
 593             self.sample_id = 'lane{0}'.format(lane)
 594             self.sample_project = 'Undetermined_indices'
 595             self.rootname = 'lane{lane}_Undetermined_L00{lane}_'.format(
 596                 lane=lane)
 597             self.project_dir = 'Undetermined_indices'
 598             self.sample_dir = 'Sample_lane{lane}'.format(lane=lane)
 599         elif index_seq is None:
 600             self.index_seq = ''
 601             self.sample_id = project_id
 602             self.sample_project = '{project_id}'.format(project_id=project_id)
 603             self.rootname = '{project_id}_NoIndex_L00{lane}_'.format(
 604                 project_id=project_id,
 605                 lane=lane)
 606             self.project_dir = 'Project_' + self.sample_project
 607             self.sample_dir = 'Sample_{project_id}'.format(
 608                 project_id=project_id)
 609         else:
 610             self.index_seq = index_seq
 611             self.sample_id = project_id
 612             self.sample_project = '{project_id}_Index{index_id}'.format(
 613                 project_id=project_id,
 614                 index_id=index_id)
 615             self.rootname = '{project_id}_{index}_L00{lane}_'.format(
 616                 project_id=project_id,
 617                 index=index_seq,
 618                 lane=lane)
 619             self.project_dir = 'Project_' + self.sample_project
 620             self.sample_dir = 'Sample_{project_id}'.format(
 621                 project_id=project_id)
 622
 623         self.project_dir = os.path.join(self.basedir, self.project_dir)
 624         self.sample_dir = os.path.join(self.project_dir, self.sample_dir)
 625         self.summary_dir = 'Summary_Stats_{0}'.format(self.flowcell_id)
 626         self.summary_dir = os.path.join(self.project_dir, self.summary_dir)
 627
 628
 629     def make_sample_dirs(self):
 630         if not os.path.isdir(self.project_dir):
 631             os.mkdir(self.project_dir)
 632         if not os.path.isdir(self.sample_dir):
 633             os.mkdir(self.sample_dir)
 634
 635     def make_summary_dirs(self):
 636         if not os.path.isdir(self.summary_dir):
 637             os.mkdir(self.summary_dir)
 638
 639     def make_test_filename(self, suffix):
 640         filename = self.rootname + suffix
 641         pathname = os.path.join(self.sample_dir, filename)
 642         return pathname
 643
 644     def dump(self):
 645         print ('index seq: {0}'.format(self.index_seq))
 646
 647         print ('project dir: {0}'.format(self.project_dir))
 648         print ('sample dir: {0}'.format(self.sample_dir))
 649         print ('rootname: {0}'.format(self.rootname))
 650         print ('path: {0}'.format(
 651             os.path.join(self.project_dir,
 652                          self.sample_dir,
 653                          self.rootname+'R1_001.fastq.gz')))
 654
 655
 656 def get_unaligned_sample_fastq_data(flowcell_id, lane, index_seq):
 657     seq = """@HWI-ST0787:101:{flowcell}:{lane}:1101:2416:3469 1:Y:0:{index}
 658 TCCTTCATTCCACCGGAGTCTGTGGAATTCTCGGGTGCCAAGGAACTCCA
 659 +
 660 CCCFFFFFHHHHHJJJJJJJJJIJJJJJJJJJJJJJJJJIIJJIIJJJJJ
 661 @HWI-ST0787:101:{flowcell}:{lane}:1101:2677:3293 1:Y:0:{index}
 662 TGGAAATCCATTGGGGTTTCCCCTGGAATTCTCGGGTGCCAAGGAACTCC
 663 +
 664 @CCFF3BDHHHHHIIIIIHHIIIDIIIGIIIEGIIIIIIIIIIIIIIIHH
 665 @HWI-ST0787:101:{flowcell}:{lane}:1101:2616:3297 1:Y:0:{index}
 666 TAATACTGCCGGGTAATGATGGCTGGAATTCTCGGGTGCCAAGGAACTCC
 667 +
 668 CCCFFFFFHHHHHCGHJJJJJJJJJJJJJJJJJIIJJJJJJJJJIHJJJI
 669 @HWI-ST0787:101:{flowcell}:{lane}:1101:2545:3319 1:N:0:{index}
 670 TCCTTCATTCCACCGGAGTCTGCTGGAATTCTCGGGTGCCAAGGAACTCC
 671 +
 672 CCCFFFFFHHHFHJGIGHIJHIIGHIGIGIGEHFIJJJIHIJHJIIJJIH
 673 """.format(flowcell=flowcell_id, lane=lane, index=index_seq)
 674     return seq
 675
 676 def get_aligned_sample_export(lane, index_seq):
 677     body = """HWI-ST0787\t102\t{lane}\t1101\t1207\t1993\t{index}\t1\tAANGGATTCGATCCGGCTTAAGAGATGAAAACCGAAAGGGCCGACCGAA\taaBS`ccceg[`ae[dRR_[[SPPPP__ececfYYWaegh^\\ZLLY\\X`\tNM\t\t\t\t\t\t
 678 HWI-ST0787\t102\t{lane}\t1101\t1478\t1997\t{index}\t1\tCAAGAACCCCGGGGGGGGGGGGGCAGAGAGGGGGAATTTTTTTTTTGTT\tBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\tNM\t\t\t\t\t\t\t\t\t\t\tN
 679 HWI-ST0787\t102\t{lane}\t1101\t1625\t1994\t{index}\t1\tAANAATGCTACAGAGACAAAACAAAACTGATATGAAAGTTGAGAATAAA\tB^BS\cccgegg[Q[QQQ[`egdgffbeggfgh^^YcfgfhXaHY^O^c\tchr9.fa\t67717938\tR\t99\t72
 680 HWI-ST0787\t102\t{lane}\t1101\t1625\t1994\t{index}\t1\tAANAATGCTACAGAGACAAAACAAAACTGATATGAAAGTTGAGAATAAA\tB^BS\cccgegg[Q[QQQ[`egdgffbeggfgh^^YcfgfhXaHY^O^c\t3:4:3\t\t\t\t\t\t\t\t\t\t\tY
 681 """.format(lane=lane, index=index_seq)
 682     return body
 683
 684 def ls_tree(root):
 685     for dirpath, dirnames, filenames in os.walk(root):
 686         for filename in filenames:
 687             print os.path.join(dirpath, filename)
 688
 689
 690 class BaseCallInfo(object):
 691     """Provide customization for how to setup the base call mock data
 692     """
 693     def __init__(self, qseq_file, tile_list, basecall_summary):
 694         self.qseq_file = qseq_file
 695         self.tile_list = tile_list
 696         self.basecall_summary = basecall_summary
 697
 698 # First generation HiSeq Flowcell
 699 ABXX_BASE_CALL_INFO = BaseCallInfo(
 700     qseq_file='AA01CCABXX_8_2_2207_qseq.txt',
 701     tile_list = HISEQ_TILE_LIST,
 702     basecall_summary = 'AA01CCABXX_BustardSummary.xml')