4a42caae4ba8021d124648c75bd25bedd5893f2b
[htsworkflow.git] / experiments / models.py
1 from __future__ import absolute_import, print_function, unicode_literals
2
3 import datetime
4 import glob
5 import logging
6 import os
7 import re
8 import uuid
9
10 from django.conf import settings
11 from django.core import urlresolvers
12 from django.utils import timezone
13 from django.db import models
14 from django.db.models.signals import pre_save
15
16 from samples.models import Library, HTSUser
17 from htsworkflow.util.conversion import parse_flowcell_id
18 from htsworkflow.pipelines import runfolder
19
20 import pytz
21
22 LOGGER = logging.getLogger(__name__)
23 default_pM = 5
24 try:
25     default_pM = int(settings.DEFAULT_PM)
26 except AttributeError as e:
27     LOGGER.error("invalid value for default_pm")
28
29 # how many days to wait before trying to re-import a runfolder
30 RESCAN_DELAY = 1
31 try:
32     RESCAN_DELAY = int(settings.RESCAN_DELAY)
33 except (ValueError, AttributeError):
34     LOGGER.error("Missing or invalid settings.RESCAN_DELAY, "\
35                  "defaulting to %s" % (RESCAN_DELAY,))
36
37 RUN_STATUS_CHOICES = (
38     (0, 'Sequencer running'),  # Solexa Data Pipeline Not Yet Started'),
39     (1, 'Data Pipeline Started'),
40     (2, 'Data Pipeline Interrupted'),
41     (3, 'Data Pipeline Finished'),
42     (4, 'Collect Results Started'),
43     (5, 'Collect Results Finished'),
44     (6, 'QC Started'),
45     (7, 'QC Finished'),
46     (255, 'DONE'),
47   )
48 RUN_STATUS_REVERSE_MAP = dict(((v, k) for k, v in RUN_STATUS_CHOICES))
49
50
51 class ClusterStation(models.Model):
52     """List of cluster stations"""
53     name = models.CharField(max_length=50, unique=True)
54     isdefault = models.BooleanField(default=False, null=False)
55
56     class Meta:
57         ordering = ["-isdefault", "name"]
58
59     def __str__(self):
60         return str(self.name)
61
62     @staticmethod
63     def update_isdefault(sender, instance, **kwargs):
64         """Clear default if needed
65         """
66         if instance.isdefault:
67             for c in ClusterStation.objects.filter(isdefault=True).all():
68                 if c.id != instance.id:
69                     c.isdefault = False
70                     c.save()
71
72 def cluster_station_default():
73     d = ClusterStation.objects.filter(isdefault=True).all()
74     if len(d) > 0:
75         return d[0]
76     d = ClusterStation.objects.order_by('-id').all()
77     if len(d) > 0:
78         return d[0]
79     return None
80
81 pre_save.connect(ClusterStation.update_isdefault, sender=ClusterStation)
82
83 class Sequencer(models.Model):
84     """Sequencers we've owned
85     """
86     name = models.CharField(max_length=50, db_index=True)
87     instrument_name = models.CharField(max_length=50, db_index=True)
88     serial_number = models.CharField(max_length=50, db_index=True)
89     model = models.CharField(max_length=255)
90     active = models.BooleanField(default=True, null=False)
91     isdefault = models.BooleanField(default=False, null=False)
92     comment = models.CharField(max_length=255)
93
94     class Meta:
95         ordering = ["-isdefault", "-active", "name"]
96
97     def __str__(self):
98         name = [str(self.name)]
99         if self.instrument_name is not None:
100             name.append("(%s)" % (str(self.instrument_name),))
101         return " ".join(name)
102
103     def get_absolute_url(self):
104         return urlresolvers.reverse('sequencer',
105                                     kwargs={'sequencer_id': self.id})
106
107     @staticmethod
108     def update_isdefault(sender, instance, **kwargs):
109         """Clear default if needed
110         """
111         if instance.isdefault:
112             for s in Sequencer.objects.filter(isdefault=True).all():
113                 if s.id != instance.id:
114                     s.isdefault = False
115                     s.save()
116
117 pre_save.connect(Sequencer.update_isdefault, sender=Sequencer)
118
119 def sequencer_default():
120     d = Sequencer.objects.filter(isdefault=True).all()
121     if len(d) > 0:
122         return d[0]
123     d = Sequencer.objects.order_by('active', '-id').all()
124     if len(d) > 0:
125         return d[0]
126     return None
127
128
129 class FlowCell(models.Model):
130     flowcell_id = models.CharField(max_length=20, unique=True, db_index=True)
131     run_date = models.DateTimeField()
132     advanced_run = models.BooleanField(default=False)
133     paired_end = models.BooleanField(default=False)
134     read_length = models.IntegerField(default=32)  # Stanford is currenlty 25
135     control_lane = models.IntegerField(choices=[(1, 1),
136                                                 (2, 2),
137                                                 (3, 3),
138                                                 (4, 4),
139                                                 (5, 5),
140                                                 (6, 6),
141                                                 (7, 7),
142                                                 (8, 8),
143                                                 (0, 'All Lanes')],
144                                        null=True,
145                                        blank=True)
146
147     cluster_station = models.ForeignKey(ClusterStation,
148                                         default=cluster_station_default)
149     sequencer = models.ForeignKey(Sequencer, default=sequencer_default)
150
151     notes = models.TextField(blank=True)
152
153     def __str__(self):
154         return str(self.flowcell_id)
155
156     def Lanes(self):
157         html = ['<table>']
158         for lane in self.lane_set.order_by('lane_number'):
159             cluster_estimate = lane.cluster_estimate
160             if cluster_estimate is not None:
161                 cluster_estimate = "%s k" % ((int(cluster_estimate) / 1000), )
162             else:
163                 cluster_estimate = 'None'
164             library_id = lane.library_id
165             library = lane.library
166             element = '<tr><td>%d</td>'\
167                       '<td><a href="%s">%s</a></td><td>%s</td></tr>'
168             html.append(element % (lane.lane_number,
169                                    library.get_admin_url(),
170                                    library,
171                                    cluster_estimate))
172         html.append('</table>')
173         return "\n".join(html)
174     Lanes.allow_tags = True
175
176     class Meta:
177         ordering = ["-run_date"]
178
179     def get_admin_url(self):
180         # that's the django way... except it didn't work
181         return urlresolvers.reverse('admin:experiments_flowcell_change',
182                                     args=(self.id,))
183
184     def flowcell_type(self):
185         """Convert our boolean 'is paired' flag to a name
186         """
187         if self.paired_end:
188             return "Paired"
189         else:
190             return "Single"
191
192     def get_absolute_url(self):
193         flowcell_id, status = parse_flowcell_id(self.flowcell_id)
194         return urlresolvers.reverse('flowcell_detail', args=[str(flowcell_id)])
195
196     def get_raw_data_directory(self):
197         """Return location of where the raw data is stored"""
198         flowcell_id, status = parse_flowcell_id(self.flowcell_id)
199
200         return os.path.join(settings.RESULT_HOME_DIR, flowcell_id)
201
202     def update_sequencing_runs(self):
203         result_root = self.get_raw_data_directory()
204         LOGGER.debug("Update data runs flowcell root: %s" % (result_root,))
205         if result_root is None:
206             return
207
208         run_xml_re = re.compile(glob.fnmatch.translate('run*.xml'))
209
210         for dirpath, dirnames, filenames in os.walk(result_root):
211             for filename in filenames:
212                 if run_xml_re.match(filename):
213                     # we have a run directory
214                     relative_pathname = get_relative_pathname(dirpath)
215                     self.import_sequencing_run(relative_pathname, filename)
216
217     def import_sequencing_run(self, relative_pathname, run_xml_name, force=False):
218         """Given a result directory import files"""
219         now = timezone.now()
220         run_dir = get_absolute_pathname(relative_pathname)
221         run_xml_path = os.path.join(run_dir, run_xml_name)
222
223         runs = SequencingRun.objects.filter(result_dir = relative_pathname)
224         if len(runs) == 0:
225             run = SequencingRun()
226             created = True
227         elif len(runs) > 1:
228             raise RuntimeError("Too many data runs for %s" % (
229                 relative_pathname,))
230         else:
231             run = runs[0]
232             created = False
233
234         if created or force or (now-run.last_update_time).days > RESCAN_DELAY:
235             LOGGER.debug("Importing run from %s" % (relative_pathname,))
236             run_xml_data = runfolder.load_pipeline_run_xml(run_xml_path)
237             run.flowcell = self
238             run.status = RUN_STATUS_REVERSE_MAP['DONE']
239             run.result_dir = relative_pathname
240             run.runfolder_name = run_xml_data.runfolder_name
241             run.cycle_start = run_xml_data.image_analysis.start
242             run.cycle_stop = run_xml_data.image_analysis.stop
243             naive_run_start_time = datetime.datetime.fromordinal(run_xml_data.image_analysis.date.toordinal())
244             run.run_start_time = pytz.timezone(settings.TIME_ZONE).localize(naive_run_start_time)
245             run.image_software = run_xml_data.image_analysis.software
246             run.image_version = run_xml_data.image_analysis.version
247             run.basecall_software = run_xml_data.bustard.software
248             run.basecall_version = run_xml_data.bustard.version
249             # we're frequently not running alignments
250             if run_xml_data.gerald:
251                 run.alignment_software = run_xml_data.gerald.software
252                 run.alignment_version = run_xml_data.gerald.version
253
254             run.last_update_time = timezone.now()
255             run.save()
256
257             run.update_result_files()
258
259
260 # FIXME: should we automatically update sequencing run?
261 #        Or should we expect someone to call update_sequencing_runs?
262 #def update_flowcell_sequencingruns(sender, instance, *args, **kwargs):
263 #    """Update our sequencing rungs
264 #    """
265 #    if not os.path.exists(settings.RESULT_HOME_DIR):
266 #       return
267 #
268 #    instance.update_sequencing_runs()
269 #post_init.connect(update_flowcell_sequencingruns, sender=FlowCell)
270
271 LANE_STATUS_CODES = [(0, 'Failed'),
272                      (1, 'Marginal'),
273                      (2, 'Good'),
274                      (100, 'Not run')]
275 LANE_STATUS_MAP = dict((int(k), v) for k, v in LANE_STATUS_CODES)
276 LANE_STATUS_MAP[None] = "Unknown"
277
278
279 def is_valid_lane(value):
280     if value >= 1 and value <= 8:
281         return True
282     else:
283         return False
284
285
286 class Lane(models.Model):
287     flowcell = models.ForeignKey(FlowCell)
288     lane_number = models.IntegerField()
289     library = models.ForeignKey(Library)
290     pM = models.DecimalField(max_digits=5,
291                              decimal_places=2,
292                              blank=False,
293                              null=False,
294                              default=default_pM)
295     cluster_estimate = models.IntegerField(blank=True, null=True)
296     status = models.IntegerField(choices=LANE_STATUS_CODES,
297                                  null=True,
298                                  blank=True)
299     comment = models.TextField(null=True, blank=True)
300
301     def get_absolute_url(self):
302         return urlresolvers.reverse('flowcell_lane_detail',
303                                     kwargs={'lane_pk': str(self.id)})
304
305     def __str__(self):
306         return self.flowcell.flowcell_id + ':' + str(self.lane_number)
307
308
309 class SequencingRun(models.Model):
310     flowcell = models.ForeignKey(FlowCell, verbose_name="Flowcell Id")
311     runfolder_name = models.CharField(max_length=50)
312     result_dir = models.CharField(max_length=255)
313     last_update_time = models.DateTimeField()
314     run_start_time = models.DateTimeField()
315     cycle_start = models.IntegerField(null=True, blank=True)
316     cycle_stop = models.IntegerField(null=True, blank=True)
317     run_status = models.IntegerField(choices=RUN_STATUS_CHOICES,
318                                      null=True, blank=True)
319     image_software = models.CharField(max_length=50)
320     image_version = models.CharField(max_length=50)
321     basecall_software = models.CharField(max_length=50)
322     basecall_version = models.CharField(max_length=50)
323     alignment_software = models.CharField(max_length=50)
324     alignment_version = models.CharField(max_length=50)
325     comment = models.TextField(blank=True)
326
327     def update_result_files(self):
328         abs_result_dir = get_absolute_pathname(self.result_dir)
329
330         for dirname, dirnames, filenames in os.walk(abs_result_dir):
331             for filename in filenames:
332                 pathname = os.path.join(dirname, filename)
333                 relative_pathname = get_relative_pathname(pathname)
334                 datafiles = self.datafile_set.filter(
335                     sequencing_run=self,
336                     relative_pathname=relative_pathname)
337                 if len(datafiles) > 0:
338                     continue
339
340                 metadata = find_file_type_metadata_from_filename(filename)
341                 if metadata is not None:
342                     metadata['filename'] = filename
343                     newfile = DataFile()
344                     newfile.sequencing_run = self
345                     newfile.file_type = metadata['file_type']
346                     newfile.relative_pathname = relative_pathname
347
348                     lane_number = metadata.get('lane', None)
349                     if lane_number is not None:
350                         lane = self.flowcell.lane_set.get(
351                             lane_number=lane_number)
352                         newfile.library = lane.library
353
354                     self.datafile_set.add(newfile)
355
356         self.last_update_time = timezone.now()
357
358     def lane_files(self):
359         lanes = {}
360
361         for datafile in self.datafile_set.all():
362             metadata = datafile.attributes
363             if metadata is not None:
364                 lane = metadata.get('lane', None)
365                 if lane is not None:
366                     lane_file_set = lanes.setdefault(lane, {})
367                     normalized_name = datafile.file_type.normalized_name
368                     lane_file_set[normalized_name] = datafile
369         return lanes
370
371     def ivc_plots(self, lane):
372         ivc_name = ['IVC All', 'IVC Call',
373                     'IVC Percent Base', 'IVC Percent All', 'IVC Percent Call']
374
375         plots = {}
376         for rel_filename, metadata in self.get_result_files():
377             if metadata.file_type.name in ivc_name:
378                 plots[metadata.file_type.name] = (rel_filename, metadata)
379
380
381 class FileType(models.Model):
382     """Represent potential file types
383
384     regex is a pattern used to detect if a filename matches this type
385     data run currently assumes that there may be a (?P<lane>) and
386     (?P<end>) pattern in the regular expression.
387     """
388     name = models.CharField(max_length=50)
389     mimetype = models.CharField(max_length=50, null=True, blank=True)
390     # regular expression from glob.fnmatch.translate
391     regex = models.TextField(null=True, blank=True)
392
393     def parse_filename(self, pathname):
394         """Does filename match our pattern?
395
396         Returns None if not, or dictionary of match variables if we do.
397         """
398         path, filename = os.path.split(pathname)
399         if len(self.regex) > 0:
400             match = re.match(self.regex, filename)
401             if match is not None:
402                 # These are (?P<>) names we know about from our
403                 # default regexes.
404                 results = match.groupdict()
405
406                 # convert int parameters
407                 for attribute_name in ['lane', 'end']:
408                     value = results.get(attribute_name, None)
409                     if value is not None:
410                         results[attribute_name] = int(value)
411
412                 return results
413
414     def _get_normalized_name(self):
415         """Crush data file name into identifier friendly name"""
416         return self.name.replace(' ', '_').lower()
417     normalized_name = property(_get_normalized_name)
418
419     def __str__(self):
420         return self.name
421
422
423 def str_uuid():
424     """Helper function to set default UUID in DataFile"""
425     return str(uuid.uuid1())
426
427
428 class DataFile(models.Model):
429     """Store map from random ID to filename"""
430     random_key = models.CharField(max_length=64,
431                                   db_index=True,
432                                   default=str_uuid)
433     sequencing_run = models.ForeignKey(SequencingRun, db_index=True, null=True)
434     library = models.ForeignKey(Library, db_index=True, null=True, blank=True)
435     file_type = models.ForeignKey(FileType)
436     relative_pathname = models.CharField(max_length=255, db_index=True)
437
438     def _get_attributes(self):
439         return self.file_type.parse_filename(self.relative_pathname)
440     attributes = property(_get_attributes)
441
442     def _get_pathname(self):
443         return get_absolute_pathname(self.relative_pathname)
444     pathname = property(_get_pathname)
445
446     def get_absolute_url(self):
447         return urlresolvers.reverse('read_result_file', (), {'key': self.random_key})
448
449
450 def find_file_type_metadata_from_filename(pathname):
451     path, filename = os.path.split(pathname)
452     result = None
453     for file_type in FileType.objects.all():
454         result = file_type.parse_filename(filename)
455         if result is not None:
456             result['file_type'] = file_type
457             return result
458
459     return None
460
461
462 def get_relative_pathname(abspath):
463     """Strip off the result home directory from a path
464     """
465     result_home_dir = os.path.join(settings.RESULT_HOME_DIR, '')
466     relative_pathname = abspath.replace(result_home_dir, '')
467     return relative_pathname
468
469
470 def get_absolute_pathname(relative_pathname):
471     """Attach relative path to  results home directory"""
472     return os.path.join(settings.RESULT_HOME_DIR, relative_pathname)