1 from __future__ import absolute_import, print_function, unicode_literals
3 # some core functions of the exp tracker module
4 from datetime import datetime, timedelta
8 from django.contrib.auth.decorators import login_required
9 from django.views.decorators.csrf import csrf_exempt
10 from django.core.exceptions import ObjectDoesNotExist
11 from django.core.mail import send_mail, mail_admins
12 from django.http import HttpResponse, Http404, JsonResponse
13 from django.conf import settings
14 from django.utils import timezone
16 from htsworkflow.auth import require_api_key
17 from .models import FlowCell, DataRun, Lane, LANE_STATUS_MAP
18 from samples.models import Library, MultiplexIndex, HTSUser
20 def flowcell_information(flowcell_id):
22 Return a dictionary describing a flowcell
25 fc = FlowCell.objects.get(flowcell_id__startswith=flowcell_id)
26 except FlowCell.DoesNotExist as e:
30 for lane in fc.lane_set.all():
32 'cluster_estimate': lane.cluster_estimate,
33 'comment': lane.comment,
34 'experiment_type': lane.library.experiment_type.name,
35 'experiment_type_id': lane.library.experiment_type_id,
36 'flowcell': lane.flowcell.flowcell_id,
37 'lane_number': lane.lane_number,
38 'library_name': lane.library.library_name,
39 'library_id': lane.library.id,
40 'library_species': lane.library.library_species.scientific_name,
42 'read_length': lane.flowcell.read_length,
43 'status_code': lane.status,
44 'status': LANE_STATUS_MAP[lane.status]
46 sequences = lane.library.index_sequences()
47 if sequences is not None:
48 lane_item['index_sequence'] = sequences
50 lane_set.setdefault(lane.lane_number,[]).append(lane_item)
52 if fc.control_lane is None:
55 control_lane = int(fc.control_lane)
58 'advanced_run': fc.advanced_run,
59 'cluster_station_id': fc.cluster_station_id,
60 'cluster_station': fc.cluster_station.name,
61 'control_lane': control_lane,
62 # 'datarun_set': how should this be represented?,
63 'flowcell_id': fc.flowcell_id,
67 'paired_end': fc.paired_end,
68 'read_length': fc.read_length,
69 'run_date': fc.run_date.isoformat(),
70 'sequencer_id': fc.sequencer_id,
71 'sequencer': fc.sequencer.name,
77 def flowcell_json(request, fc_id):
79 Return a JSON blob containing enough information to generate a config file.
81 require_api_key(request)
83 fc_dict = flowcell_information(fc_id)
88 return JsonResponse({'result': fc_dict})
90 def lanes_for(username=None):
92 Given a user id try to return recent lanes as a list of dictionaries
95 if username is not None:
96 user = HTSUser.objects.get(username=username)
97 query.update({'library__affiliations__users__id': user.id})
99 lanes = Lane.objects.filter(**query).order_by('-flowcell__run_date')
103 affiliations = l.library.affiliations.all()
104 affiliations_list = [(a.id, a.name) for a in affiliations]
105 result.append({ 'flowcell': l.flowcell.flowcell_id,
106 'run_date': l.flowcell.run_date.isoformat(),
107 'lane_number': l.lane_number,
108 'library': l.library.id,
109 'library_name': l.library.library_name,
110 'comment': l.comment,
111 'affiliations': affiliations_list})
115 def lanes_for_json(request, username):
117 Format lanes for a user
119 require_api_key(request)
122 result = lanes_for(username)
123 except ObjectDoesNotExist as e:
126 #convert query set to python structure
128 return JsonResponse({'result': result})
131 def updStatus(request):
135 UpdatedStatus = 'unknown'
137 runfolder = 'unknown'
138 ClIP = request.META['REMOTE_ADDR']
140 if hasattr(request, 'user'):
143 #Check access permission
144 if not (user.is_superuser and ClIP in settings.ALLOWED_IPS):
145 return HttpResponse("%s access denied from %s." % (user, ClIP))
147 # ~~~~~~Parameters for the job ~~~~
148 if 'fcid' in request.REQUEST:
149 fcid = request.REQUEST['fcid']
151 return HttpResponse('missing fcid')
153 if 'runf' in request.REQUEST:
154 runfolder = request.REQUEST['runf']
156 return HttpResponse('missing runf')
159 if 'updst' in request.REQUEST:
160 UpdatedStatus = request.REQUEST['updst']
162 return HttpResponse('missing status')
164 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
166 # Update Data Run status in DB
167 # Try get rec. If not found return 'entry not found + <fcid><runfolder>', if found try update and return updated
169 rec = DataRun.objects.get(run_folder=runfolder)
170 rec.run_status = UpdatedStatus
172 #if there's a message update that too
173 mytimestamp = timezone.now().__str__()
174 mytimestamp = re.sub(pattern=":[^:]*$",repl="",string=mytimestamp)
175 if 'msg' in request.REQUEST:
176 rec.run_note += ", "+request.REQUEST['msg']+" ("+mytimestamp+")"
178 if UpdatedStatus == '1':
179 rec.run_note = "Started ("+mytimestamp+")"
182 output = "Hello "+settings.ALLOWED_IPS[ClIP]+". Updated to:'"+DataRun.RUN_STATUS_CHOICES[int(UpdatedStatus)][1].__str__()+"'"
183 except ObjectDoesNotExist:
184 output = "entry not found: "+fcid+", "+runfolder
187 #Notify researcher by email
189 #send_mail('Exp Tracker', 'Data Run Status '+output, 'rrauch@stanford.edu', ['rrrami@gmail.com'], fail_silently=False)
190 #mail_admins("test subject", "testing , testing", fail_silently=False)
191 # gives error: (49, "Can't assign requested address")
192 return HttpResponse(output)
194 def generateConfile(request,fcid):
196 #ClIP = request.META['REMOTE_ADDR']
197 #if (ClIP in settings.ALLOWED_IPS): granted = True
199 #if not granted: return HttpResponse("access denied.")
201 config = ['READ_LENGTH 25']
202 config += ['ANALYSIS eland']
203 config += ['GENOME_FILE all_chr.fa']
204 config += ['ELAND_MULTIPLE_INSTANCES 8']
205 genome_dir = 'GENOME_DIR /Volumes/Genomes/'
206 eland_genome = 'ELAND_GENOME /Volumes/Genomes/'
209 fc = FlowCell.objects.get(flowcell_id=fcid)
210 for lane in fc.lane_set.all():
211 config += [ str(lane.lane_number) +":" + \
212 genome_dir + lane.library.library_species.scientific_name ]
213 config += [ str(lane.lane_number) +":" + \
214 eland_genome + lane.library.library_species.scientific_name ]
216 except ObjectDoesNotExist:
217 config = 'Entry not found for fcid = '+fcid
219 return os.linesep.join(config)
223 ClIP = req.META['REMOTE_ADDR']
224 if (ClIP in settings.ALLOWED_IPS): granted = True
226 if not granted: return HttpResponse("access denied. IP: "+ClIP)
229 cnfgfile = 'Nothing found'
230 runfolder = 'unknown'
231 request = req.REQUEST
232 if 'fcid' in request:
233 fcid = request['fcid']
234 if 'runf' in request:
235 runfolder = request['runf']
237 rec = DataRun.objects.get(run_folder=runfolder) #,flowcell_id=fcid)
238 cnfgfile = rec.config_params
239 #match_str = re.compile(r"READ_LENGTH.+$")
240 match_str = re.compile('^READ_LENGTH.+')
241 if not match_str.search(cnfgfile):
242 cnfgfile = generateConfile(request,fcid)
243 if match_str.search(cnfgfile):
244 rec = DataRun.objects.get(run_folder=runfolder) #,flowcell_id=fcid)
245 rec.config_params = cnfgfile
248 cnfgfile = 'Failed generating config params for RunFolder = '+runfolder +', Flowcell id = '+ fcid+ ' Config Text:\n'+cnfgfile
250 except ObjectDoesNotExist:
251 cnfgfile = 'Entry not found for RunFolder = '+runfolder
253 return HttpResponse(cnfgfile, content_type='text/plain')
255 def getLaneLibs(req):
257 ClIP = req.META['REMOTE_ADDR']
258 if (ClIP in settings.ALLOWED_IPS): granted = True
260 if not granted: return HttpResponse("access denied.")
262 request = req.REQUEST
265 if 'fcid' in request:
266 fcid = request['fcid']
268 rec = FlowCell.objects.get(flowcell_id=fcid)
270 year = datetime.today().year.__str__()
271 year = replace(year,'20','')
272 month = datetime.today().month
273 if month < 10: month = "0"+month.__str__()
274 else: month = month.__str__()
275 day = datetime.today().day
276 if day < 10: day = "0"+day.__str__()
277 else: day = day.__str__()
278 mydate = year+month+day
279 outputfile = '<?xml version="1.0" ?>'
280 outputfile += '\n<SolexaResult Date="'+mydate+'" Flowcell="'+fcid+'" Client="'+settings.ALLOWED_IPS[ClIP]+'">'
281 outputfile += '\n<Lane Index="1" Name="'+rec.lane_1_library.library_name+'" Library="'+rec.lane_1_library.id+'" Genome="'+rec.lane_1_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
282 outputfile += '\n<Lane Index="2" Name="'+rec.lane_2_library.library_name+'" Library="'+rec.lane_2_library.id+'" Genome="'+rec.lane_2_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
283 outputfile += '\n<Lane Index="3" Name="'+rec.lane_3_library.library_name+'" Library="'+rec.lane_3_library.id+'" Genome="'+rec.lane_3_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
284 outputfile += '\n<Lane Index="4" Name="'+rec.lane_4_library.library_name+'" Library="'+rec.lane_4_library.id+'" Genome="'+rec.lane_4_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
285 outputfile += '\n<Lane Index="5" Name="'+rec.lane_5_library.library_name+'" Library="'+rec.lane_5_library.id+'" Genome="'+rec.lane_5_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
286 outputfile += '\n<Lane Index="6" Name="'+rec.lane_6_library.library_name+'" Library="'+rec.lane_6_library.id+'" Genome="'+rec.lane_6_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
287 outputfile += '\n<Lane Index="7" Name="'+rec.lane_7_library.library_name+'" Library="'+rec.lane_7_library.id+'" Genome="'+rec.lane_7_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
288 outputfile += '\n<Lane Index="8" Name="'+rec.lane_8_library.library_name+'" Library="'+rec.lane_8_library.id+'" Genome="'+rec.lane_8_library.library_species.use_genome_build+'" PrimerName="" PrimerSeq=""/>'
289 outputfile += '\n</SolexaResult>'
290 except ObjectDoesNotExist:
291 outputfile = 'Flowcell entry not found for: '+fcid
292 else: outputfile = 'Missing input: flowcell id'
294 return HttpResponse(outputfile, content_type='text/plain')
296 def estimateFlowcellDuration(flowcell):
298 Attempt to estimate how long it will take to run a flowcell
301 # (3600 seconds * 1.5 hours per cycle )
302 sequencing_seconds_per_cycle= 3600 * 1.5
303 # 800 is a rough guess
304 pipeline_seconds_per_cycle = 800
306 cycles = flowcell.read_length
307 if flowcell.paired_end:
309 sequencing_time = timedelta(0, cycles * sequencing_seconds_per_cycle)
310 analysis_time = timedelta(0, cycles * pipeline_seconds_per_cycle)
311 estimate_mid = sequencing_time + analysis_time
315 def estimateFlowcellTimeRemaining(flowcell):
316 estimate_mid = estimateFlowcellDuration(flowcell)
318 # offset for how long we've been running
319 running_time = timezone.now() - flowcell.run_date
320 estimate_mid -= running_time
324 def roundToDays(estimate):
326 Given a time estimate round up and down in days
329 estimate_low = timedelta(estimate.days, 0)
330 # floor estimate_mid and add a day
331 estimate_high = timedelta(estimate.days+1, 0)
333 return (estimate_low, estimate_high)
336 def makeUserLaneMap(flowcell):
338 Given a flowcell return a mapping of users interested in
339 the libraries on those lanes.
343 for lane in flowcell.lane_set.all():
344 for affiliation in lane.library.affiliations.all():
345 for user in affiliation.users.all():
346 users.setdefault(user,[]).append(lane)
350 def getUsersForFlowcell(flowcell):
353 for lane in flowcell.lane_set.all():
354 for affiliation in lane.library.affiliations.all():
355 for user in affiliation.users.all():
360 def makeUserLibraryMap(libraries):
362 Given an interable set of libraries return a mapping or
363 users interested in those libraries.
367 for library in libraries:
368 for affiliation in library.affiliations.all():
369 for user in affiliation.users.all():
370 users.setdefault(user,[]).append(library)
374 def makeAffiliationLaneMap(flowcell):
377 for lane in flowcell.lane_set.all():
378 for affiliation in lane.library.affiliations.all():
379 affs.setdefault(affiliation,[]).append(lane)
383 def makeEmailLaneMap(flowcell):
385 Create a list of email addresses and the lanes associated with those users.
387 The email addresses can come from both the "users" table and the "affiliations" table.
390 for lane in flowcell.lane_set.all():
391 for affiliation in lane.library.affiliations.all():
392 if affiliation.email is not None and len(affiliation.email) > 0:
393 emails.setdefault(affiliation.email,set()).add(lane)
394 for user in affiliation.users.all():
395 if user.email is not None and len(user.email) > 0:
396 emails.setdefault(user.email,set()).add(lane)