htsworkflow/frontend/samples/views.py

   1 # Create your views here.
   2 import StringIO
   3 import logging
   4 import os
   5 import sys
   6
   7 try:
   8     import json
   9 except ImportError, e:
  10     import simplejson as json
  11
  12 from htsworkflow.frontend.auth import require_api_key
  13 from htsworkflow.frontend.experiments.models import FlowCell
  14 from htsworkflow.frontend.samples.changelist import ChangeList
  15 from htsworkflow.frontend.samples.models import Library
  16 from htsworkflow.frontend.samples.results import get_flowcell_result_dict, parse_flowcell_id
  17 from htsworkflow.frontend.bcmagic.forms import BarcodeMagicForm
  18 from htsworkflow.pipelines.runfolder import load_pipeline_run_xml
  19 from htsworkflow.pipelines import runfolder
  20 from htsworkflow.pipelines.eland import ResultLane
  21 from htsworkflow.frontend import settings
  22 from htsworkflow.util import makebed
  23 from htsworkflow.util import opener
  24
  25 from django.core.exceptions import ObjectDoesNotExist
  26 from django.http import HttpResponse, HttpResponseRedirect, Http404
  27 from django.shortcuts import render_to_response
  28 from django.template import RequestContext
  29 from django.template.loader import get_template
  30 from django.contrib.auth.decorators import login_required
  31
  32 LANE_LIST = [1,2,3,4,5,6,7,8]
  33 SAMPLES_CONTEXT_DEFAULTS = {
  34     'app_name': 'Flowcell/Library Tracker',
  35     'bcmagic': BarcodeMagicForm()
  36 }
  37
  38 def create_library_context(cl):
  39     """
  40     Create a list of libraries that includes how many lanes were run
  41     """
  42     records = []
  43     #for lib in library_items.object_list:
  44     for lib in cl.result_list:
  45        summary = {}
  46        summary['library_id'] = lib.id
  47        summary['library_name'] = lib.library_name
  48        summary['species_name' ] = lib.library_species.scientific_name
  49        if lib.amplified_from_sample is not None:
  50            summary['amplified_from'] = lib.amplified_from_sample.id
  51        else:
  52            summary['amplified_from'] = ''
  53        lanes_run = 0
  54        #for lane_id in LANE_LIST:
  55        #    lane = getattr(lib, 'lane_%d_library' % (lane_id,))
  56        #    lanes_run += len( lane.all() )
  57        lanes_run = lib.lane_set.count()
  58        summary['lanes_run'] = lanes_run
  59        summary['is_archived'] = lib.is_archived()
  60        records.append(summary)
  61     cl.result_count = unicode(cl.paginator._count) + u" libraries"
  62     return {'library_list': records }
  63
  64 def library(request):
  65    # build changelist
  66     fcl = ChangeList(request, Library,
  67         list_filter=['affiliations', 'library_species'],
  68         search_fields=['id', 'library_name', 'amplified_from_sample__id'],
  69         list_per_page=200,
  70         queryset=Library.objects.filter(hidden__exact=0)
  71     )
  72
  73     context = { 'cl': fcl, 'title': 'Library Index'}
  74     context.update(create_library_context(fcl))
  75     t = get_template('samples/library_index.html')
  76     c = RequestContext(request, context)
  77
  78     app_context = {
  79         'page_name': 'Library Index',
  80         'east_region_config_div': 'changelist-filter',
  81         'body': t.render(c)
  82     }
  83     app_context.update(SAMPLES_CONTEXT_DEFAULTS)
  84
  85     app_t = get_template('flowcell_libraries_app.html')
  86     app_c = RequestContext(request, app_context)
  87     return HttpResponse( app_t.render(app_c) )
  88
  89 def library_to_flowcells(request, lib_id):
  90     """
  91     Display information about all the flowcells a library has been run on.
  92     """
  93
  94     try:
  95       lib = Library.objects.get(id=lib_id)
  96     except:
  97       return HttpResponse("Library %s does not exist" % (lib_id))
  98
  99     flowcell_list = []
 100     flowcell_run_results = {} # aka flowcells we're looking at
 101     for lane in lib.lane_set.all():
 102         fc = lane.flowcell
 103         flowcell_id, id = parse_flowcell_id(fc.flowcell_id)
 104         if flowcell_id not in flowcell_run_results:
 105             flowcell_run_results[flowcell_id] = get_flowcell_result_dict(flowcell_id)
 106         flowcell_list.append((fc.flowcell_id, lane.lane_number))
 107
 108     flowcell_list.sort()
 109     lane_summary_list = []
 110     eland_results = []
 111     for fc, lane_number in flowcell_list:
 112         lane_summary, err_list = _summary_stats(fc, lane_number)
 113
 114         eland_results.extend(_make_eland_results(fc, lane_number, flowcell_run_results))
 115         lane_summary_list.extend(lane_summary)
 116
 117     context = {
 118         'page_name': 'Library Details',
 119         'lib': lib,
 120         'eland_results': eland_results,
 121         'lane_summary_list': lane_summary_list,
 122     }
 123     context.update(SAMPLES_CONTEXT_DEFAULTS)
 124
 125     return render_to_response(
 126         'samples/library_detail.html',
 127         context,
 128         context_instance = RequestContext(request))
 129
 130 def summaryhtm_fc_cnm(request, flowcell_id, cnm):
 131     """
 132     returns a Summary.htm file if it exists.
 133     """
 134     fc_id, status = parse_flowcell_id(flowcell_id)
 135     d = get_flowcell_result_dict(fc_id)
 136
 137     if d is None:
 138         return HttpResponse('<b>Results for Flowcell %s not found.</b>' % (fc_id))
 139
 140     if cnm not in d:
 141         return HttpResponse('<b>Results for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
 142
 143     summary_filepath = d[cnm]['summary']
 144
 145     if summary_filepath is None:
 146         return HttpResponse('<b>Summary.htm for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
 147
 148     f = open(summary_filepath, 'r')
 149
 150     return HttpResponse(f)
 151
 152
 153 def result_fc_cnm_eland_lane(request, flowcell_id, cnm, lane):
 154     """
 155     returns an eland_file upon calling.
 156     """
 157     fc_id, status = parse_flowcell_id(flowcell_id)
 158     d = get_flowcell_result_dict(fc_id)
 159
 160     if d is None:
 161         return HttpResponse('<b>Results for Flowcell %s not found.</b>' % (fc_id))
 162
 163     if cnm not in d:
 164         return HttpResponse('<b>Results for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
 165
 166     erd = d[cnm]['eland_results']
 167     lane = int(lane)
 168
 169     if lane not in erd:
 170         return HttpResponse('<b>Results for Flowcell %s; %s; lane %s not found.</b>' % (fc_id, cnm, lane))
 171
 172     filepath = erd[lane]
 173
 174     #f = opener.autoopen(filepath, 'r')
 175     # return HttpResponse(f, mimetype="application/x-elandresult")
 176
 177     f = open(filepath, 'r')
 178     return HttpResponse(f, mimetype='application/x-bzip2')
 179
 180
 181
 182 def bedfile_fc_cnm_eland_lane_ucsc(request, fc_id, cnm, lane):
 183     """
 184     returns a bed file for a given flowcell, CN-M (i.e. C1-33), and lane (ucsc compatible)
 185     """
 186     return bedfile_fc_cnm_eland_lane(request, fc_id, cnm, lane, ucsc_compatible=True)
 187
 188
 189 def bedfile_fc_cnm_eland_lane(request, flowcell_id, cnm, lane, ucsc_compatible=False):
 190     """
 191     returns a bed file for a given flowcell, CN-M (i.e. C1-33), and lane
 192     """
 193     fc_id, status = parse_flowcell_id(flowcell_id)
 194     d = get_flowcell_result_dict(fc_id)
 195
 196     if d is None:
 197         return HttpResponse('<b>Results for Flowcell %s not found.</b>' % (fc_id))
 198
 199     if cnm not in d:
 200         return HttpResponse('<b>Results for Flowcell %s; %s not found.</b>' % (fc_id, cnm))
 201
 202     erd = d[cnm]['eland_results']
 203     lane = int(lane)
 204
 205     if lane not in erd:
 206         return HttpResponse('<b>Results for Flowcell %s; %s; lane %s not found.</b>' % (fc_id, cnm, lane))
 207
 208     filepath = erd[lane]
 209
 210     # Eland result file
 211     fi = opener.autoopen(filepath, 'r')
 212     # output memory file
 213
 214     name, description = makebed.make_description( fc_id, lane )
 215
 216     bedgen = makebed.make_bed_from_eland_generator(fi, name, description)
 217
 218     if ucsc_compatible:
 219         return HttpResponse(bedgen)
 220     else:
 221         return HttpResponse(bedgen, mimetype="application/x-bedfile")
 222
 223
 224 def _summary_stats(flowcell_id, lane_id):
 225     """
 226     Return the summary statistics for a given flowcell, lane, and end.
 227     """
 228     fc_id, status = parse_flowcell_id(flowcell_id)
 229     fc_result_dict = get_flowcell_result_dict(fc_id)
 230
 231     summary_list = []
 232     err_list = []
 233
 234     if fc_result_dict is None:
 235         err_list.append('Results for Flowcell %s not found.' % (fc_id))
 236         return (summary_list, err_list)
 237
 238     for cycle_width in fc_result_dict:
 239         xmlpath = fc_result_dict[cycle_width]['run_xml']
 240
 241         if xmlpath is None:
 242             err_list.append('Run xml for Flowcell %s(%s) not found.' % (fc_id, cycle_width))
 243             continue
 244
 245         run = load_pipeline_run_xml(xmlpath)
 246         gerald_summary = run.gerald.summary.lane_results
 247         for end in range(len(gerald_summary)):
 248             end_summary = run.gerald.eland_results.results[end]
 249             if end_summary.has_key(lane_id):
 250                 eland_summary = run.gerald.eland_results.results[end][lane_id]
 251             else:
 252                 eland_summary = ResultLane(lane_id=lane_id, end=end)
 253             # add information to lane_summary
 254             eland_summary.flowcell_id = flowcell_id
 255             if len(gerald_summary) > end and gerald_summary[end].has_key(lane_id):
 256                 eland_summary.clusters = gerald_summary[end][lane_id].cluster
 257             else:
 258                 eland_summary.clusters = None
 259             eland_summary.cycle_width = cycle_width
 260             if hasattr(eland_summary, 'genome_map'):
 261                 eland_summary.summarized_reads = runfolder.summarize_mapped_reads(
 262                                                    eland_summary.genome_map,
 263                                                    eland_summary.mapped_reads)
 264
 265             # grab some more information out of the flowcell db
 266             flowcell = FlowCell.objects.get(flowcell_id=flowcell_id)
 267             #pm_field = 'lane_%d_pM' % (lane_id)
 268             lane_obj = flowcell.lane_set.get(lane_number=lane_id)
 269             eland_summary.successful_pm = lane_obj.pM
 270
 271             summary_list.append(eland_summary)
 272
 273         #except Exception, e:
 274         #    summary_list.append("Summary report needs to be updated.")
 275         #    logging.error("Exception: " + str(e))
 276
 277     return (summary_list, err_list)
 278
 279 def _summary_stats_old(flowcell_id, lane):
 280     """
 281     return a dictionary of summary stats for a given flowcell_id & lane.
 282     """
 283     fc_id, status = parse_flowcell_id(flowcell_id)
 284     fc_result_dict = get_flowcell_result_dict(fc_id)
 285
 286     dict_list = []
 287     err_list = []
 288     summary_list = []
 289
 290     if fc_result_dict is None:
 291         err_list.append('Results for Flowcell %s not found.' % (fc_id))
 292         return (dict_list, err_list, summary_list)
 293
 294     for cnm in fc_result_dict:
 295
 296         xmlpath = fc_result_dict[cnm]['run_xml']
 297
 298         if xmlpath is None:
 299             err_list.append('Run xml for Flowcell %s(%s) not found.' % (fc_id, cnm))
 300             continue
 301
 302         tree = ElementTree.parse(xmlpath).getroot()
 303         results = runfolder.PipelineRun(pathname='', xml=tree)
 304         try:
 305             lane_report = runfolder.summarize_lane(results.gerald, lane)
 306             summary_list.append(os.linesep.join(lane_report))
 307         except Exception, e:
 308             summary_list.append("Summary report needs to be updated.")
 309             logging.error("Exception: " + str(e))
 310
 311         print >>sys.stderr, "----------------------------------"
 312         print >>sys.stderr, "-- DOES NOT SUPPORT PAIRED END ---"
 313         print >>sys.stderr, "----------------------------------"
 314         lane_results = results.gerald.summary[0][lane]
 315         lrs = lane_results
 316
 317         d = {}
 318
 319         d['average_alignment_score'] = lrs.average_alignment_score
 320         d['average_first_cycle_intensity'] = lrs.average_first_cycle_intensity
 321         d['cluster'] = lrs.cluster
 322         d['lane'] = lrs.lane
 323         d['flowcell'] = flowcell_id
 324         d['cnm'] = cnm
 325         d['percent_error_rate'] = lrs.percent_error_rate
 326         d['percent_intensity_after_20_cycles'] = lrs.percent_intensity_after_20_cycles
 327         d['percent_pass_filter_align'] = lrs.percent_pass_filter_align
 328         d['percent_pass_filter_clusters'] = lrs.percent_pass_filter_clusters
 329
 330         #FIXME: function finished, but need to take advantage of
 331         #   may need to take in a list of lanes so we only have to
 332         #   load the xml file once per flowcell rather than once
 333         #   per lane.
 334         dict_list.append(d)
 335
 336     return (dict_list, err_list, summary_list)
 337
 338
 339 def get_eland_result_type(pathname):
 340     """
 341     Guess the eland result file type from the filename
 342     """
 343     path, filename = os.path.split(pathname)
 344     if 'extended' in filename:
 345         return 'extended'
 346     elif 'multi' in filename:
 347         return 'multi'
 348     elif 'result' in filename:
 349         return 'result'
 350     else:
 351         return 'unknown'
 352
 353 def _make_eland_results(flowcell_id, lane, interesting_flowcells):
 354     fc_id, status = parse_flowcell_id(flowcell_id)
 355     cur_fc = interesting_flowcells.get(fc_id, None)
 356     if cur_fc is None:
 357       return []
 358
 359     flowcell = FlowCell.objects.get(flowcell_id=flowcell_id)
 360     # Loop throw storage devices if a result has been archived
 361     storage_id_list = []
 362     if cur_fc is not None:
 363         for lts in flowcell.longtermstorage_set.all():
 364             for sd in lts.storage_devices.all():
 365                 # Use barcode_id if it exists
 366                 if sd.barcode_id is not None and sd.barcode_id != '':
 367                     storage_id_list.append(sd.barcode_id)
 368                 # Otherwise use UUID
 369                 else:
 370                     storage_id_list.append(sd.uuid)
 371
 372     # Formatting for template use
 373     if len(storage_id_list) == 0:
 374         storage_ids = None
 375     else:
 376         storage_ids = ', '.join([ '<a href="/inventory/%s/">%s</a>' % (s,s) for s in storage_id_list ])
 377
 378     results = []
 379     for cycle in cur_fc.keys():
 380         result_path = cur_fc[cycle]['eland_results'].get(lane, None)
 381         result_link = make_result_link(fc_id, cycle, lane, result_path)
 382         results.append({'flowcell_id': fc_id,
 383                         'run_date': flowcell.run_date,
 384                         'cycle': cycle,
 385                         'lane': lane,
 386                         'summary_url': make_summary_url(flowcell_id, cycle),
 387                         'result_url': result_link[0],
 388                         'result_label': result_link[1],
 389                         'bed_url': result_link[2],
 390                         'storage_ids': storage_ids
 391         })
 392     return results
 393
 394 def make_summary_url(flowcell_id, cycle_name):
 395     url = '/results/%s/%s/summary/' % (flowcell_id, cycle_name)
 396     return url
 397
 398 def make_result_link(flowcell_id, cycle_name, lane, eland_result_path):
 399     if eland_result_path is None:
 400         return ("", "", "")
 401
 402     result_type = get_eland_result_type(eland_result_path)
 403     result_url = '/results/%s/%s/eland_result/%s' % (flowcell_id, cycle_name, lane)
 404     result_label = 'eland %s' % (result_type,)
 405     bed_url = None
 406     if result_type == 'result':
 407        bed_url_pattern = '/results/%s/%s/bedfile/%s'
 408        bed_url = bed_url_pattern % (flowcell_id, cycle_name, lane)
 409
 410     return (result_url, result_label, bed_url)
 411
 412 def _files(flowcell_id, lane):
 413     """
 414     Sets up available files for download
 415     """
 416     lane = int(lane)
 417
 418     flowcell_id, id = parse_flowcell_id(flowcell_id)
 419     d = get_flowcell_result_dict(flowcell_id)
 420
 421     if d is None:
 422         return ''
 423
 424     output = []
 425
 426     # c_name == 'CN-M' (i.e. C1-33)
 427     for c_name in d:
 428
 429         if d[c_name]['summary'] is not None:
 430             output.append('<a href="/results/%s/%s/summary/">summary(%s)</a>' \
 431                           % (flowcell_id, c_name, c_name))
 432
 433         erd = d[c_name]['eland_results']
 434         if lane in erd:
 435             result_type = get_eland_result_type(erd[lane])
 436             result_url_pattern = '<a href="/results/%s/%s/eland_result/%s">eland %s(%s)</a>'
 437             output.append(result_url_pattern % (flowcell_id, c_name, lane, result_type, c_name))
 438             if result_type == 'result':
 439                 bed_url_pattern = '<a href="/results/%s/%s/bedfile/%s">bedfile(%s)</a>'
 440                 output.append(bed_url_pattern % (flowcell_id, c_name, lane, c_name))
 441
 442     if len(output) == 0:
 443         return ''
 444
 445     return '(' + '|'.join(output) + ')'
 446
 447 def library_id_to_admin_url(request, lib_id):
 448     lib = Library.objects.get(id=lib_id)
 449     return HttpResponseRedirect('/admin/samples/library/%s' % (lib.id,))
 450
 451 def library_dict(library_id):
 452     """
 453     Given a library id construct a dictionary containing important information
 454     return None if nothing was found
 455     """
 456     try:
 457         lib = Library.objects.get(id = library_id)
 458     except Library.DoesNotExist, e:
 459         return None
 460
 461     #lane_info = lane_information(lib.lane_set)
 462     lane_info = []
 463     for lane in lib.lane_set.all():
 464         lane_info.append( {'flowcell':lane.flowcell.flowcell_id,
 465                            'lane_number': lane.lane_number} )
 466
 467     info = {
 468         # 'affiliations'?
 469         # 'aligned_reads': lib.aligned_reads,
 470         #'amplified_into_sample': lib.amplified_into_sample, # into is a colleciton...
 471         #'amplified_from_sample_id': lib.amplified_from_sample,
 472         #'antibody_name': lib.antibody_name(), # we have no antibodies.
 473         'antibody_id': lib.antibody_id,
 474         'avg_lib_size': lib.avg_lib_size,
 475         'cell_line': lib.cell_line.cellline_name,
 476         'cell_line_id': lib.cell_line_id,
 477         'experiment_type': lib.experiment_type.name,
 478         'experiment_type_id': lib.experiment_type_id,
 479         'id': lib.id,
 480         'lane_set': lane_info,
 481         'library_id': lib.id,
 482         'library_name': lib.library_name,
 483         'library_species': lib.library_species.scientific_name,
 484         'library_species_id': lib.library_species_id,
 485         #'library_type': lib.library_type.name,
 486         'library_type_id': lib.library_type_id,
 487         'made_for': lib.made_for,
 488         'made_by': lib.made_by,
 489         'notes': lib.notes,
 490         'replicate': lib.replicate,
 491         'stopping_point': lib.stopping_point,
 492         'successful_pM': unicode(lib.successful_pM),
 493         'undiluted_concentration': unicode(lib.undiluted_concentration)
 494         }
 495     if lib.library_type_id is None:
 496         info['library_type'] = None
 497     else:
 498         info['library_type'] = lib.library_type.name
 499     return info
 500
 501 def library_json(request, library_id):
 502     """
 503     Return a json formatted library dictionary
 504     """
 505     require_api_key(request)
 506     # what validation should we do on library_id?
 507
 508     lib = library_dict(library_id)
 509     if lib is None:
 510         raise Http404
 511
 512     lib_json = json.dumps(lib)
 513     return HttpResponse(lib_json, mimetype='application/json')
 514
 515 def species_json(request, species_id):
 516     """
 517     Return information about a species.
 518     """
 519     raise Http404
 520
 521 @login_required
 522 def user_profile(request):
 523     """
 524     Information about the user
 525     """
 526     context = {
 527                 'page_name': 'User Profile',
 528                 'media': '',
 529                 #'bcmagic': BarcodeMagicForm(),
 530                 #'select': 'settings',
 531             }
 532     context.update(SAMPLES_CONTEXT_DEFAULTS)
 533     return render_to_response('registration/profile.html', context,
 534                               context_instance=RequestContext(request))
 535