Merge branch 'master' of mus.cacr.caltech.edu:htsworkflow
[htsworkflow.git] / htsworkflow / frontend / samples / views.py
1 # Create your views here.
2 import StringIO
3 import logging
4 import os
5 import sys
6
7 try:
8     import json
9 except ImportError, e:
10     import simplejson as json
11
12 from htsworkflow.frontend.auth import require_api_key
13 from htsworkflow.frontend.experiments.models import FlowCell, Lane, LANE_STATUS_MAP
14 from htsworkflow.frontend.samples.changelist import ChangeList
15 from htsworkflow.frontend.samples.models import Library, Species, HTSUser
16 from htsworkflow.frontend.samples.results import get_flowcell_result_dict
17 from htsworkflow.frontend.bcmagic.forms import BarcodeMagicForm
18 from htsworkflow.pipelines.runfolder import load_pipeline_run_xml
19 from htsworkflow.pipelines import runfolder
20 from htsworkflow.pipelines.eland import ResultLane
21 from htsworkflow.util.conversion import unicode_or_none, parse_flowcell_id
22 from htsworkflow.util import makebed
23 from htsworkflow.util import opener
24
25
26 from django.core.exceptions import ObjectDoesNotExist
27 from django.http import HttpResponse, HttpResponseRedirect, Http404
28 from django.shortcuts import render_to_response, get_object_or_404
29 from django.template import RequestContext
30 from django.template.loader import get_template
31 from django.contrib.auth.decorators import login_required
32 from django.conf import settings
33
34 LANE_LIST = [1,2,3,4,5,6,7,8]
35 SAMPLES_CONTEXT_DEFAULTS = {
36     'app_name': 'Flowcell/Library Tracker',
37     'bcmagic': BarcodeMagicForm()
38 }
39
40 def count_lanes(lane_set):
41     single = 0
42     paired = 1
43     short_read = 0
44     medium_read = 1
45     long_read = 2
46     counts = [[0,0,0,],[0,0,0]]
47     
48     for lane in lane_set.all():
49         if lane.flowcell.paired_end:
50             lane_type = paired
51         else:
52             lane_type = single
53         if lane.flowcell.read_length < 40:
54             read_type = short_read
55         elif lane.flowcell.read_length < 100:
56             read_type = medium_read
57         else:
58             read_type = long_read
59         counts[lane_type][read_type] += 1
60         
61     return counts
62
63 def create_library_context(cl):
64     """
65      Create a list of libraries that includes how many lanes were run
66     """
67     records = []
68     #for lib in library_items.object_list:
69     for lib in cl.result_list:
70        summary = {}
71        summary['library'] = lib
72        summary['library_id'] = lib.id
73        summary['library_name'] = lib.library_name
74        summary['species_name' ] = lib.library_species.scientific_name
75        if lib.amplified_from_sample is not None:
76            summary['amplified_from'] = lib.amplified_from_sample.id
77        else:
78            summary['amplified_from'] = ''
79        lanes_run = count_lanes(lib.lane_set)
80        # suppress zeros
81        for row in xrange(len(lanes_run)):
82            for col in xrange(len(lanes_run[row])):
83                if lanes_run[row][col] == 0:
84                    lanes_run[row][col] = ''
85        summary['lanes_run'] = lanes_run
86        summary['is_archived'] = lib.is_archived()
87        records.append(summary)
88     cl.result_count = unicode(cl.paginator._count)
89     return {'library_list': records }
90
91 def library(request):
92     # build changelist
93     fcl = ChangeList(request, Library,
94         list_filter=['affiliations', 'library_species'],
95         search_fields=['id', 'library_name', 'amplified_from_sample__id'],
96         list_per_page=200,
97         queryset=Library.objects.filter(hidden__exact=0)
98     )
99
100     context = { 'cl': fcl, 'title': 'Library Index'}
101     context.update(create_library_context(fcl))
102     t = get_template('samples/library_index.html')
103     c = RequestContext(request, context)
104     return HttpResponse( t.render(c) )
105     
106
107 def library_to_flowcells(request, lib_id):
108     """
109     Display information about all the flowcells a library has been run on.
110     """
111     
112     try:
113       lib = Library.objects.get(id=lib_id)
114     except:
115       return HttpResponse("Library %s does not exist" % (lib_id))
116    
117     flowcell_list = []
118     flowcell_run_results = {} # aka flowcells we're looking at
119     for lane in lib.lane_set.all():
120         fc = lane.flowcell
121         flowcell_id, id = parse_flowcell_id(fc.flowcell_id)
122         if flowcell_id not in flowcell_run_results:
123             flowcell_run_results[flowcell_id] = get_flowcell_result_dict(flowcell_id)
124         flowcell_list.append((fc.flowcell_id, lane.lane_number))
125
126     flowcell_list.sort()
127     lane_summary_list = []
128     eland_results = []
129     for fc, lane_number in flowcell_list:
130         lane_summary, err_list = _summary_stats(fc, lane_number)
131         lane_summary_list.extend(lane_summary)
132         
133         eland_results.extend(_make_eland_results(fc, lane_number, flowcell_run_results))
134
135     context = {
136         'page_name': 'Library Details',
137         'lib': lib,
138         'eland_results': eland_results,
139         'lane_summary_list': lane_summary_list,
140     }
141     context.update(SAMPLES_CONTEXT_DEFAULTS)
142
143     return render_to_response(
144         'samples/library_detail.html',
145         context,
146         context_instance = RequestContext(request))
147
148 def lanes_for(request, username=None):
149     """
150     Generate a report of recent activity for a user
151     """
152     query = {}
153     if username is not None:
154         user = HTSUser.objects.get(username=username)
155         query.update({'library__affiliations__users__id':user.id})
156     fcl = ChangeList(request, Lane,
157         list_filter=[],
158         search_fields=['flowcell__flowcell_id', 'library__id', 'library__library_name'],
159         list_per_page=200,
160         queryset=Lane.objects.filter(**query)
161     )
162
163     context = { 'lanes': fcl, 'title': 'Lane Index'}
164
165     return render_to_response(
166         'samples/lanes_for.html',
167         context,
168         context_instance = RequestContext(request)
169     )
170           
171     
172 def summaryhtm_fc_cnm(request, flowcell_id, cnm):
173     """
174     returns a Summary.htm file if it exists.
175     """
176     fc_id, status = parse_flowcell_id(flowcell_id)
177     d = get_flowcell_result_dict(fc_id)
178     
179     if d is None:
180         return HttpResponse('<b>Results for Flowcell %s not found.</b>' % (fc_id))
181     
182     if cnm not in d:
183         return HttpResponse('<b>Results for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
184     
185     summary_filepath = d[cnm]['summary']
186     
187     if summary_filepath is None:
188         return HttpResponse('<b>Summary.htm for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
189     
190     f = open(summary_filepath, 'r')
191     
192     return HttpResponse(f)
193
194
195 def result_fc_cnm_eland_lane(request, flowcell_id, cnm, lane):
196     """
197     returns an eland_file upon calling.
198     """
199     fc_id, status = parse_flowcell_id(flowcell_id)
200     d = get_flowcell_result_dict(fc_id)
201     
202     if d is None:
203         return HttpResponse('<b>Results for Flowcell %s not found.</b>' % (fc_id))
204     
205     if cnm not in d:
206         return HttpResponse('<b>Results for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
207     
208     erd = d[cnm]['eland_results']
209     lane = int(lane)
210     
211     if lane not in erd:
212         return HttpResponse('<b>Results for Flowcell %s; %s; lane %s not found.</b>' % (fc_id, cnm, lane))
213     
214     filepath = erd[lane]
215     
216     #f = opener.autoopen(filepath, 'r')
217     # return HttpResponse(f, mimetype="application/x-elandresult")
218
219     f = open(filepath, 'r')
220     return HttpResponse(f, mimetype='application/x-bzip2')
221     
222
223
224 def bedfile_fc_cnm_eland_lane_ucsc(request, fc_id, cnm, lane):
225     """
226     returns a bed file for a given flowcell, CN-M (i.e. C1-33), and lane (ucsc compatible)
227     """
228     return bedfile_fc_cnm_eland_lane(request, fc_id, cnm, lane, ucsc_compatible=True)
229
230
231 def bedfile_fc_cnm_eland_lane(request, flowcell_id, cnm, lane, ucsc_compatible=False):
232     """
233     returns a bed file for a given flowcell, CN-M (i.e. C1-33), and lane
234     """
235     fc_id, status = parse_flowcell_id(flowcell_id)
236     d = get_flowcell_result_dict(fc_id)
237     
238     if d is None:
239         return HttpResponse('<b>Results for Flowcell %s not found.</b>' % (fc_id))
240     
241     if cnm not in d:
242         return HttpResponse('<b>Results for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
243     
244     erd = d[cnm]['eland_results']
245     lane = int(lane)
246     
247     if lane not in erd:
248         return HttpResponse('<b>Results for Flowcell %s; %s; lane %s not found.</b>' % (fc_id, cnm, lane))
249     
250     filepath = erd[lane]
251     
252     # Eland result file
253     fi = opener.autoopen(filepath, 'r')
254     # output memory file
255     
256     name, description = makebed.make_description( fc_id, lane )
257     
258     bedgen = makebed.make_bed_from_eland_generator(fi, name, description)
259     
260     if ucsc_compatible:
261         return HttpResponse(bedgen)
262     else:
263         return HttpResponse(bedgen, mimetype="application/x-bedfile")
264
265
266 def _summary_stats(flowcell_id, lane_id):
267     """
268     Return the summary statistics for a given flowcell, lane, and end.
269     """
270     fc_id, status = parse_flowcell_id(flowcell_id)
271     fc_result_dict = get_flowcell_result_dict(fc_id)
272
273     summary_list = []
274     err_list = []
275     
276     if fc_result_dict is None:
277         err_list.append('Results for Flowcell %s not found.' % (fc_id))
278         return (summary_list, err_list)
279
280     for cycle_width in fc_result_dict:
281         xmlpath = fc_result_dict[cycle_width]['run_xml']
282         
283         if xmlpath is None:
284             err_list.append('Run xml for Flowcell %s(%s) not found.' % (fc_id, cycle_width))
285             continue
286         
287         run = load_pipeline_run_xml(xmlpath)
288         gerald_summary = run.gerald.summary.lane_results
289         for end in range(len(gerald_summary)):
290             end_summary = run.gerald.eland_results.results[end]
291             if end_summary.has_key(lane_id):
292                 eland_summary = run.gerald.eland_results.results[end][lane_id]
293             else:
294                 eland_summary = ResultLane(lane_id=lane_id, end=end)
295             # add information to lane_summary
296             eland_summary.flowcell_id = flowcell_id
297             if len(gerald_summary) > end and gerald_summary[end].has_key(lane_id):
298                 eland_summary.clusters = gerald_summary[end][lane_id].cluster
299             else:
300                 eland_summary.clusters = None
301             eland_summary.cycle_width = cycle_width
302             if hasattr(eland_summary, 'genome_map'):
303                 eland_summary.summarized_reads = runfolder.summarize_mapped_reads( 
304                                                    eland_summary.genome_map, 
305                                                    eland_summary.mapped_reads)
306
307             # grab some more information out of the flowcell db
308             flowcell = FlowCell.objects.get(flowcell_id=flowcell_id)
309             #pm_field = 'lane_%d_pM' % (lane_id)
310             lane_obj = flowcell.lane_set.get(lane_number=lane_id)
311             eland_summary.flowcell = flowcell
312             eland_summary.lane = lane_obj
313
314             summary_list.append(eland_summary)
315
316         #except Exception, e:
317         #    summary_list.append("Summary report needs to be updated.")
318         #    logging.error("Exception: " + str(e))
319     
320     return (summary_list, err_list)
321
322     
323 def get_eland_result_type(pathname):
324     """
325     Guess the eland result file type from the filename
326     """
327     path, filename = os.path.split(pathname)
328     if 'extended' in filename:
329         return 'extended'
330     elif 'multi' in filename:
331         return 'multi'
332     elif 'result' in filename:
333         return 'result'
334     else:
335         return 'unknown'
336
337 def _make_eland_results(flowcell_id, lane_number, interesting_flowcells):
338     fc_id, status = parse_flowcell_id(flowcell_id)
339     cur_fc = interesting_flowcells.get(fc_id, None)
340     if cur_fc is None:
341       return []
342
343     flowcell = FlowCell.objects.get(flowcell_id=flowcell_id)
344     lane = flowcell.lane_set.get(lane_number=lane_number)
345     # Loop throw storage devices if a result has been archived
346     storage_id_list = []
347     if cur_fc is not None:
348         for lts in flowcell.longtermstorage_set.all():
349             for sd in lts.storage_devices.all():
350                 # Use barcode_id if it exists
351                 if sd.barcode_id is not None and sd.barcode_id != '':
352                     storage_id_list.append(sd.barcode_id)
353                 # Otherwise use UUID
354                 else:
355                     storage_id_list.append(sd.uuid)
356         
357     # Formatting for template use
358     if len(storage_id_list) == 0:
359         storage_ids = None
360     else:
361         storage_ids = ', '.join([ '<a href="/inventory/%s/">%s</a>' % (s,s) for s in storage_id_list ])
362
363     results = []
364     for cycle in cur_fc.keys():
365         result_path = cur_fc[cycle]['eland_results'].get(lane, None)
366         result_link = make_result_link(fc_id, cycle, lane, result_path)
367         results.append({'flowcell_id': fc_id,
368                         'flowcell': flowcell,
369                         'run_date': flowcell.run_date,
370                         'cycle': cycle, 
371                         'lane': lane, 
372                         'summary_url': make_summary_url(flowcell_id, cycle),
373                         'result_url': result_link[0],
374                         'result_label': result_link[1],
375                         'bed_url': result_link[2],
376                         'storage_ids': storage_ids
377         })
378     return results
379
380 def make_summary_url(flowcell_id, cycle_name):
381     url = '/results/%s/%s/summary/' % (flowcell_id, cycle_name)
382     return url
383
384 def make_result_link(flowcell_id, cycle_name, lane, eland_result_path):
385     if eland_result_path is None:
386         return ("", "", "")
387
388     result_type = get_eland_result_type(eland_result_path)
389     result_url = '/results/%s/%s/eland_result/%s' % (flowcell_id, cycle_name, lane)
390     result_label = 'eland %s' % (result_type,)
391     bed_url = None
392     if result_type == 'result':
393        bed_url_pattern = '/results/%s/%s/bedfile/%s'
394        bed_url = bed_url_pattern % (flowcell_id, cycle_name, lane)
395     
396     return (result_url, result_label, bed_url)
397
398 def _files(flowcell_id, lane):
399     """
400     Sets up available files for download
401     """
402     lane = int(lane)
403
404     flowcell_id, id = parse_flowcell_id(flowcell_id)
405     d = get_flowcell_result_dict(flowcell_id)
406     
407     if d is None:
408         return ''
409     
410     output = []
411     
412     # c_name == 'CN-M' (i.e. C1-33)
413     for c_name in d:
414         
415         if d[c_name]['summary'] is not None:
416             output.append('<a href="/results/%s/%s/summary/">summary(%s)</a>' \
417                           % (flowcell_id, c_name, c_name))
418         
419         erd = d[c_name]['eland_results']
420         if lane in erd:
421             result_type = get_eland_result_type(erd[lane])
422             result_url_pattern = '<a href="/results/%s/%s/eland_result/%s">eland %s(%s)</a>'
423             output.append(result_url_pattern % (flowcell_id, c_name, lane, result_type, c_name))
424             if result_type == 'result':
425                 bed_url_pattern = '<a href="/results/%s/%s/bedfile/%s">bedfile(%s)</a>'
426                 output.append(bed_url_pattern % (flowcell_id, c_name, lane, c_name))
427     
428     if len(output) == 0:
429         return ''
430     
431     return '(' + '|'.join(output) + ')'
432
433 def library_id_to_admin_url(request, lib_id):
434     lib = Library.objects.get(id=lib_id)
435     return HttpResponseRedirect('/admin/samples/library/%s' % (lib.id,))
436
437 def library_dict(library_id):
438     """
439     Given a library id construct a dictionary containing important information
440     return None if nothing was found
441     """
442     try:
443         lib = Library.objects.get(id = library_id)
444     except Library.DoesNotExist, e:
445         return None
446
447     #lane_info = lane_information(lib.lane_set)
448     lane_info = []
449     for lane in lib.lane_set.all():
450         lane_info.append( {'flowcell':lane.flowcell.flowcell_id,
451                            'lane_number': lane.lane_number,
452                            'paired_end': lane.flowcell.paired_end,
453                            'read_length': lane.flowcell.read_length,
454                            'status_code': lane.status,
455                            'status': LANE_STATUS_MAP[lane.status]} )
456         
457     info = {
458         # 'affiliations'?
459         # 'aligned_reads': lib.aligned_reads,
460         #'amplified_into_sample': lib.amplified_into_sample, # into is a colleciton...
461         #'amplified_from_sample_id': lib.amplified_from_sample, 
462         #'antibody_name': lib.antibody_name(), # we have no antibodies.
463         'antibody_id': lib.antibody_id,
464         'cell_line_id': lib.cell_line_id,
465         'cell_line': unicode_or_none(lib.cell_line),
466         'experiment_type': lib.experiment_type.name,
467         'experiment_type_id': lib.experiment_type_id,
468         'gel_cut_size': lib.gel_cut_size,
469         'hidden': lib.hidden,
470         'id': lib.id,
471         'insert_size': lib.insert_size,
472         'lane_set': lane_info,
473         'library_id': lib.id,
474         'library_name': lib.library_name,
475         'library_species': lib.library_species.scientific_name,
476         'library_species_id': lib.library_species_id,
477         #'library_type': lib.library_type.name,
478         'library_type_id': lib.library_type_id,
479         'made_for': lib.made_for,
480         'made_by': lib.made_by,
481         'notes': lib.notes,
482         'replicate': lib.replicate,
483         'stopping_point': lib.stopping_point,
484         'successful_pM': unicode_or_none(lib.successful_pM),
485         'undiluted_concentration': unicode_or_none(lib.undiluted_concentration)
486         }
487     if lib.library_type_id is None:
488         info['library_type'] = None
489     else:
490         info['library_type'] = lib.library_type.name
491     return info
492
493 def library_json(request, library_id):
494     """
495     Return a json formatted library dictionary
496     """
497     require_api_key(request)
498     # what validation should we do on library_id?
499     
500     lib = library_dict(library_id)
501     if lib is None:
502         raise Http404
503
504     lib_json = json.dumps(lib)
505     return HttpResponse(lib_json, mimetype='application/json')
506
507 def species_json(request, species_id):
508     """
509     Return information about a species.
510     """
511     raise Http404
512
513 def species(request, species_id):
514     species = get_object_or_404(Species, id=species_id)
515     
516     context = RequestContext(request,
517                              { 'species': species })
518
519     return render_to_response("samples/species_detail.html", context)
520
521 @login_required
522 def user_profile(request):
523     """
524     Information about the user
525     """
526     context = {
527                 'page_name': 'User Profile',
528                 'media': '',
529                 #'bcmagic': BarcodeMagicForm(),
530                 #'select': 'settings',
531             }
532     context.update(SAMPLES_CONTEXT_DEFAULTS)
533     return render_to_response('registration/profile.html', context,
534                               context_instance=RequestContext(request))
535