experiments/models.py

   1 from __future__ import absolute_import, print_function, unicode_literals
   2
   3 import datetime
   4 import glob
   5 import logging
   6 import os
   7 import re
   8 import uuid
   9
  10 from django.conf import settings
  11 from django.core import urlresolvers
  12 from django.utils import timezone
  13 from django.db import models
  14 from django.db.models.signals import pre_save
  15
  16 from samples.models import Library, HTSUser
  17 from htsworkflow.util.conversion import parse_flowcell_id
  18 from htsworkflow.pipelines import runfolder
  19
  20 import pytz
  21
  22 LOGGER = logging.getLogger(__name__)
  23 default_pM = 5
  24 try:
  25     default_pM = int(settings.DEFAULT_PM)
  26 except AttributeError as e:
  27     LOGGER.error("invalid value for default_pm")
  28
  29 # how many days to wait before trying to re-import a runfolder
  30 RESCAN_DELAY = 1
  31 try:
  32     RESCAN_DELAY = int(settings.RESCAN_DELAY)
  33 except (ValueError, AttributeError):
  34     LOGGER.error("Missing or invalid settings.RESCAN_DELAY, "\
  35                  "defaulting to %s" % (RESCAN_DELAY,))
  36
  37 RUN_STATUS_CHOICES = (
  38     (0, 'Sequencer running'),  # Solexa Data Pipeline Not Yet Started'),
  39     (1, 'Data Pipeline Started'),
  40     (2, 'Data Pipeline Interrupted'),
  41     (3, 'Data Pipeline Finished'),
  42     (4, 'Collect Results Started'),
  43     (5, 'Collect Results Finished'),
  44     (6, 'QC Started'),
  45     (7, 'QC Finished'),
  46     (255, 'DONE'),
  47   )
  48 RUN_STATUS_REVERSE_MAP = dict(((v, k) for k, v in RUN_STATUS_CHOICES))
  49
  50
  51 class ClusterStation(models.Model):
  52     """List of cluster stations"""
  53     name = models.CharField(max_length=50, unique=True)
  54     isdefault = models.BooleanField(default=False, null=False)
  55
  56     class Meta:
  57         ordering = ["-isdefault", "name"]
  58
  59     def __str__(self):
  60         return str(self.name)
  61
  62     @staticmethod
  63     def update_isdefault(sender, instance, **kwargs):
  64         """Clear default if needed
  65         """
  66         if instance.isdefault:
  67             for c in ClusterStation.objects.filter(isdefault=True).all():
  68                 if c.id != instance.id:
  69                     c.isdefault = False
  70                     c.save()
  71
  72 def cluster_station_default():
  73     d = ClusterStation.objects.filter(isdefault=True).all()
  74     if len(d) > 0:
  75         return d[0]
  76     d = ClusterStation.objects.order_by('-id').all()
  77     if len(d) > 0:
  78         return d[0]
  79     return None
  80
  81 pre_save.connect(ClusterStation.update_isdefault, sender=ClusterStation)
  82
  83 class Sequencer(models.Model):
  84     """Sequencers we've owned
  85     """
  86     name = models.CharField(max_length=50, db_index=True)
  87     instrument_name = models.CharField(max_length=50, db_index=True)
  88     serial_number = models.CharField(max_length=50, db_index=True)
  89     model = models.CharField(max_length=255)
  90     active = models.BooleanField(default=True, null=False)
  91     isdefault = models.BooleanField(default=False, null=False)
  92     comment = models.CharField(max_length=255)
  93
  94     class Meta:
  95         ordering = ["-isdefault", "-active", "name"]
  96
  97     def __str__(self):
  98         name = [str(self.name)]
  99         if self.instrument_name is not None:
 100             name.append("(%s)" % (str(self.instrument_name),))
 101         return " ".join(name)
 102
 103     def get_absolute_url(self):
 104         return urlresolvers.reverse('sequencer',
 105                                     kwargs={'sequencer_id': self.id})
 106
 107     @staticmethod
 108     def update_isdefault(sender, instance, **kwargs):
 109         """Clear default if needed
 110         """
 111         if instance.isdefault:
 112             for s in Sequencer.objects.filter(isdefault=True).all():
 113                 if s.id != instance.id:
 114                     s.isdefault = False
 115                     s.save()
 116
 117 pre_save.connect(Sequencer.update_isdefault, sender=Sequencer)
 118
 119 def sequencer_default():
 120     d = Sequencer.objects.filter(isdefault=True).all()
 121     if len(d) > 0:
 122         return d[0]
 123     d = Sequencer.objects.order_by('active', '-id').all()
 124     if len(d) > 0:
 125         return d[0]
 126     return None
 127
 128
 129 class FlowCell(models.Model):
 130     flowcell_id = models.CharField(max_length=20, unique=True, db_index=True)
 131     run_date = models.DateTimeField()
 132     advanced_run = models.BooleanField(default=False)
 133     paired_end = models.BooleanField(default=False)
 134     read_length = models.IntegerField(default=32)  # Stanford is currenlty 25
 135     control_lane = models.IntegerField(choices=[(1, 1),
 136                                                 (2, 2),
 137                                                 (3, 3),
 138                                                 (4, 4),
 139                                                 (5, 5),
 140                                                 (6, 6),
 141                                                 (7, 7),
 142                                                 (8, 8),
 143                                                 (0, 'All Lanes')],
 144                                        null=True,
 145                                        blank=True)
 146
 147     cluster_station = models.ForeignKey(ClusterStation,
 148                                         default=cluster_station_default)
 149     sequencer = models.ForeignKey(Sequencer, default=sequencer_default)
 150
 151     notes = models.TextField(blank=True)
 152
 153     def __str__(self):
 154         return str(self.flowcell_id)
 155
 156     def Lanes(self):
 157         html = ['<table>']
 158         for lane in self.lane_set.order_by('lane_number'):
 159             cluster_estimate = lane.cluster_estimate
 160             if cluster_estimate is not None:
 161                 cluster_estimate = "%s k" % ((int(cluster_estimate) / 1000), )
 162             else:
 163                 cluster_estimate = 'None'
 164             library_id = lane.library_id
 165             library = lane.library
 166             element = '<tr><td>%d</td>'\
 167                       '<td><a href="%s">%s</a></td><td>%s</td></tr>'
 168             html.append(element % (lane.lane_number,
 169                                    library.get_admin_url(),
 170                                    library,
 171                                    cluster_estimate))
 172         html.append('</table>')
 173         return "\n".join(html)
 174     Lanes.allow_tags = True
 175
 176     class Meta:
 177         ordering = ["-run_date"]
 178
 179     def get_admin_url(self):
 180         # that's the django way... except it didn't work
 181         return urlresolvers.reverse('admin:experiments_flowcell_change',
 182                                     args=(self.id,))
 183
 184     def flowcell_type(self):
 185         """Convert our boolean 'is paired' flag to a name
 186         """
 187         if self.paired_end:
 188             return "Paired"
 189         else:
 190             return "Single"
 191
 192     def get_absolute_url(self):
 193         flowcell_id, status = parse_flowcell_id(self.flowcell_id)
 194         return urlresolvers.reverse('flowcell_detail', args=[str(flowcell_id)])
 195
 196     def get_raw_data_directory(self):
 197         """Return location of where the raw data is stored"""
 198         flowcell_id, status = parse_flowcell_id(self.flowcell_id)
 199
 200         return os.path.join(settings.RESULT_HOME_DIR, flowcell_id)
 201
 202     def update_sequencing_runs(self):
 203         result_root = self.get_raw_data_directory()
 204         LOGGER.debug("Update data runs flowcell root: %s" % (result_root,))
 205         if result_root is None:
 206             return
 207
 208         run_xml_re = re.compile(glob.fnmatch.translate('run*.xml'))
 209
 210         for dirpath, dirnames, filenames in os.walk(result_root):
 211             for filename in filenames:
 212                 if run_xml_re.match(filename):
 213                     # we have a run directory
 214                     relative_pathname = get_relative_pathname(dirpath)
 215                     self.import_sequencing_run(relative_pathname, filename)
 216
 217     def import_sequencing_run(self, relative_pathname, run_xml_name, force=False):
 218         """Given a result directory import files"""
 219         now = timezone.now()
 220         run_dir = get_absolute_pathname(relative_pathname)
 221         run_xml_path = os.path.join(run_dir, run_xml_name)
 222
 223         runs = SequencingRun.objects.filter(result_dir = relative_pathname)
 224         if len(runs) == 0:
 225             run = SequencingRun()
 226             created = True
 227         elif len(runs) > 1:
 228             raise RuntimeError("Too many data runs for %s" % (
 229                 relative_pathname,))
 230         else:
 231             run = runs[0]
 232             created = False
 233
 234         if created or force or (now-run.last_update_time).days > RESCAN_DELAY:
 235             LOGGER.debug("Importing run from %s" % (relative_pathname,))
 236             run_xml_data = runfolder.load_pipeline_run_xml(run_xml_path)
 237             run.flowcell = self
 238             run.status = RUN_STATUS_REVERSE_MAP['DONE']
 239             run.result_dir = relative_pathname
 240             run.runfolder_name = run_xml_data.runfolder_name
 241             run.cycle_start = run_xml_data.image_analysis.start
 242             run.cycle_stop = run_xml_data.image_analysis.stop
 243             naive_run_start_time = datetime.datetime.fromordinal(run_xml_data.image_analysis.date.toordinal())
 244             run.run_start_time = pytz.timezone(settings.TIME_ZONE).localize(naive_run_start_time)
 245             run.image_software = run_xml_data.image_analysis.software
 246             run.image_version = run_xml_data.image_analysis.version
 247             run.basecall_software = run_xml_data.bustard.software
 248             run.basecall_version = run_xml_data.bustard.version
 249             # we're frequently not running alignments
 250             if run_xml_data.gerald:
 251                 run.alignment_software = run_xml_data.gerald.software
 252                 run.alignment_version = run_xml_data.gerald.version
 253
 254             run.last_update_time = timezone.now()
 255             run.save()
 256
 257             run.update_result_files()
 258
 259
 260 # FIXME: should we automatically update sequencing run?
 261 #        Or should we expect someone to call update_sequencing_runs?
 262 #def update_flowcell_sequencingruns(sender, instance, *args, **kwargs):
 263 #    """Update our sequencing rungs
 264 #    """
 265 #    if not os.path.exists(settings.RESULT_HOME_DIR):
 266 #       return
 267 #
 268 #    instance.update_sequencing_runs()
 269 #post_init.connect(update_flowcell_sequencingruns, sender=FlowCell)
 270
 271 LANE_STATUS_CODES = [(0, 'Failed'),
 272                      (1, 'Marginal'),
 273                      (2, 'Good'),
 274                      (100, 'Not run')]
 275 LANE_STATUS_MAP = dict((int(k), v) for k, v in LANE_STATUS_CODES)
 276 LANE_STATUS_MAP[None] = "Unknown"
 277
 278
 279 def is_valid_lane(value):
 280     if value >= 1 and value <= 8:
 281         return True
 282     else:
 283         return False
 284
 285
 286 class Lane(models.Model):
 287     class Meta:
 288         ordering = ['-flowcell__run_date', '-library__id']
 289
 290     flowcell = models.ForeignKey(FlowCell)
 291     lane_number = models.IntegerField()
 292     library = models.ForeignKey(Library)
 293     pM = models.DecimalField(max_digits=5,
 294                              decimal_places=2,
 295                              blank=False,
 296                              null=False,
 297                              default=default_pM)
 298     cluster_estimate = models.IntegerField(blank=True, null=True)
 299     status = models.IntegerField(choices=LANE_STATUS_CODES,
 300                                  null=True,
 301                                  blank=True)
 302     comment = models.TextField(null=True, blank=True)
 303
 304     def get_absolute_url(self):
 305         return urlresolvers.reverse('flowcell_lane_detail',
 306                                     kwargs={'lane_pk': str(self.id)})
 307
 308     def __str__(self):
 309         return self.flowcell.flowcell_id + ':' + str(self.lane_number)
 310
 311
 312 class SequencingRun(models.Model):
 313     flowcell = models.ForeignKey(FlowCell, verbose_name="Flowcell Id")
 314     runfolder_name = models.CharField(max_length=50)
 315     result_dir = models.CharField(max_length=255)
 316     last_update_time = models.DateTimeField()
 317     run_start_time = models.DateTimeField()
 318     cycle_start = models.IntegerField(null=True, blank=True)
 319     cycle_stop = models.IntegerField(null=True, blank=True)
 320     run_status = models.IntegerField(choices=RUN_STATUS_CHOICES,
 321                                      null=True, blank=True)
 322     image_software = models.CharField(max_length=50)
 323     image_version = models.CharField(max_length=50)
 324     basecall_software = models.CharField(max_length=50)
 325     basecall_version = models.CharField(max_length=50)
 326     alignment_software = models.CharField(max_length=50)
 327     alignment_version = models.CharField(max_length=50)
 328     comment = models.TextField(blank=True)
 329
 330     def update_result_files(self):
 331         abs_result_dir = get_absolute_pathname(self.result_dir)
 332
 333         for dirname, dirnames, filenames in os.walk(abs_result_dir):
 334             for filename in filenames:
 335                 pathname = os.path.join(dirname, filename)
 336                 relative_pathname = get_relative_pathname(pathname)
 337                 datafiles = self.datafile_set.filter(
 338                     sequencing_run=self,
 339                     relative_pathname=relative_pathname)
 340                 if len(datafiles) > 0:
 341                     continue
 342
 343                 metadata = find_file_type_metadata_from_filename(filename)
 344                 if metadata is not None:
 345                     metadata['filename'] = filename
 346                     newfile = DataFile()
 347                     newfile.sequencing_run = self
 348                     newfile.file_type = metadata['file_type']
 349                     newfile.relative_pathname = relative_pathname
 350
 351                     lane_number = metadata.get('lane', None)
 352                     if lane_number is not None:
 353                         lane = self.flowcell.lane_set.get(
 354                             lane_number=lane_number)
 355                         newfile.library = lane.library
 356
 357                     newfile.save()
 358                     self.datafile_set.add(newfile)
 359
 360         self.last_update_time = timezone.now()
 361
 362     def lane_files(self):
 363         lanes = {}
 364
 365         for datafile in self.datafile_set.all():
 366             metadata = datafile.attributes
 367             if metadata is not None:
 368                 lane = metadata.get('lane', None)
 369                 if lane is not None:
 370                     lane_file_set = lanes.setdefault(lane, {})
 371                     normalized_name = datafile.file_type.normalized_name
 372                     lane_file_set[normalized_name] = datafile
 373         return lanes
 374
 375     def ivc_plots(self, lane):
 376         ivc_name = ['IVC All', 'IVC Call',
 377                     'IVC Percent Base', 'IVC Percent All', 'IVC Percent Call']
 378
 379         plots = {}
 380         for rel_filename, metadata in self.get_result_files():
 381             if metadata.file_type.name in ivc_name:
 382                 plots[metadata.file_type.name] = (rel_filename, metadata)
 383
 384
 385 class FileType(models.Model):
 386     """Represent potential file types
 387
 388     regex is a pattern used to detect if a filename matches this type
 389     data run currently assumes that there may be a (?P<lane>) and
 390     (?P<end>) pattern in the regular expression.
 391     """
 392     name = models.CharField(max_length=50)
 393     mimetype = models.CharField(max_length=50, null=True, blank=True)
 394     # regular expression from glob.fnmatch.translate
 395     regex = models.TextField(null=True, blank=True)
 396
 397     def parse_filename(self, pathname):
 398         """Does filename match our pattern?
 399
 400         Returns None if not, or dictionary of match variables if we do.
 401         """
 402         path, filename = os.path.split(pathname)
 403         if len(self.regex) > 0:
 404             match = re.match(self.regex, filename)
 405             if match is not None:
 406                 # These are (?P<>) names we know about from our
 407                 # default regexes.
 408                 results = match.groupdict()
 409
 410                 # convert int parameters
 411                 for attribute_name in ['lane', 'end']:
 412                     value = results.get(attribute_name, None)
 413                     if value is not None:
 414                         results[attribute_name] = int(value)
 415
 416                 return results
 417
 418     def _get_normalized_name(self):
 419         """Crush data file name into identifier friendly name"""
 420         return self.name.replace(' ', '_').lower()
 421     normalized_name = property(_get_normalized_name)
 422
 423     def __str__(self):
 424         return self.name
 425
 426
 427 def str_uuid():
 428     """Helper function to set default UUID in DataFile"""
 429     return str(uuid.uuid1())
 430
 431
 432 class DataFile(models.Model):
 433     """Store map from random ID to filename"""
 434     random_key = models.CharField(max_length=64,
 435                                   db_index=True,
 436                                   default=str_uuid)
 437     sequencing_run = models.ForeignKey(SequencingRun, db_index=True, null=True)
 438     library = models.ForeignKey(Library, db_index=True, null=True, blank=True)
 439     file_type = models.ForeignKey(FileType)
 440     relative_pathname = models.CharField(max_length=255, db_index=True)
 441
 442     def _get_attributes(self):
 443         return self.file_type.parse_filename(self.relative_pathname)
 444     attributes = property(_get_attributes)
 445
 446     def _get_pathname(self):
 447         return get_absolute_pathname(self.relative_pathname)
 448     pathname = property(_get_pathname)
 449
 450     def get_absolute_url(self):
 451         return urlresolvers.reverse('read_result_file', (), {'key': self.random_key})
 452
 453
 454 def find_file_type_metadata_from_filename(pathname):
 455     path, filename = os.path.split(pathname)
 456     result = None
 457     for file_type in FileType.objects.all():
 458         result = file_type.parse_filename(filename)
 459         if result is not None:
 460             result['file_type'] = file_type
 461             return result
 462
 463     return None
 464
 465
 466 def get_relative_pathname(abspath):
 467     """Strip off the result home directory from a path
 468     """
 469     result_home_dir = os.path.join(settings.RESULT_HOME_DIR, '')
 470     relative_pathname = abspath.replace(result_home_dir, '')
 471     return relative_pathname
 472
 473
 474 def get_absolute_pathname(relative_pathname):
 475     """Attach relative path to  results home directory"""
 476     return os.path.join(settings.RESULT_HOME_DIR, relative_pathname)