X-Git-Url: http://woldlab.caltech.edu/gitweb/?a=blobdiff_plain;f=htsworkflow%2Fsubmission%2Fsubmission.py;h=897053fbc148477c108d232635958db45488dfc5;hb=1000ba383f47a9c3ab422f4552b3725038c96541;hp=e4ce90c73b073287913ec2f76f5f1d5f0d9bd887;hpb=697a1fe031741e6d7614127a2f16e69027578e10;p=htsworkflow.git diff --git a/htsworkflow/submission/submission.py b/htsworkflow/submission/submission.py index e4ce90c..897053f 100644 --- a/htsworkflow/submission/submission.py +++ b/htsworkflow/submission/submission.py @@ -8,32 +8,32 @@ import RDF from htsworkflow.util.rdfhelp import \ blankOrUri, \ - dafTermOntology, \ dump_model, \ + fromTypedNode, \ get_model, \ - libraryOntology, \ - owlNS, \ - rdfNS, \ - submissionLog, \ - submissionOntology, \ - toTypedNode, \ - fromTypedNode + strip_namespace, \ + toTypedNode +from htsworkflow.util.rdfns import * from htsworkflow.util.hashfile import make_md5sum - +from htsworkflow.submission.fastqname import FastqName from htsworkflow.submission.daf import \ MetadataLookupException, \ + ModelException, \ get_submission_uri +from django.conf import settings +from django.template import Context, Template, loader + LOGGER = logging.getLogger(__name__) class Submission(object): - def __init__(self, name, model): + def __init__(self, name, model, host): self.name = name self.model = model self.submissionSet = get_submission_uri(self.name) self.submissionSetNS = RDF.NS(str(self.submissionSet) + '#') - self.libraryNS = RDF.NS('http://jumpgate.caltech.edu/library/') + self.libraryNS = RDF.NS('{0}/library/'.format(host)) self.__view_map = None @@ -57,7 +57,15 @@ class Submission(object): submission_files = os.listdir(analysis_dir) for filename in submission_files: - self.construct_file_attributes(analysis_dir, libNode, filename) + pathname = os.path.abspath(os.path.join(analysis_dir, filename)) + self.construct_file_attributes(analysis_dir, libNode, pathname) + + def analysis_nodes(self, result_map): + """Return an iterable of analysis nodes + """ + for result_dir in result_map.values(): + an_analysis = self.get_submission_node(result_dir) + yield an_analysis def construct_file_attributes(self, analysis_dir, libNode, pathname): """Looking for the best extension @@ -83,7 +91,7 @@ class Submission(object): rdfNS['type']) if file_classification is None: errmsg = 'Could not find class for {0}' - logger.warning(errmsg.format(str(file_type))) + LOGGER.warning(errmsg.format(str(file_type))) return self.model.add_statement( @@ -113,20 +121,28 @@ class Submission(object): an_analysis)) # add file specific information - fileNode = self.link_file_to_classes(filename, - an_analysis, - an_analysis_uri, - analysis_dir) + fileNode = self.make_file_node(pathname, an_analysis) self.add_md5s(filename, fileNode, analysis_dir) + self.add_fastq_metadata(filename, fileNode) + self.add_label(file_type, fileNode, libNode) self.model.add_statement( RDF.Statement(fileNode, rdfNS['type'], file_type)) + self.model.add_statement( + RDF.Statement(fileNode, + libraryOntology['library'], + libNode)) + LOGGER.debug("Done.") - def link_file_to_classes(self, filename, submissionNode, submission_uri, analysis_dir): + def make_file_node(self, pathname, submissionNode): + """Create file node and attach it to its submission. + """ # add file specific information - fileNode = RDF.Node(RDF.Uri(submission_uri + '/' + filename)) + path, filename = os.path.split(pathname) + pathname = os.path.abspath(pathname) + fileNode = RDF.Node(RDF.Uri('file://'+ pathname)) self.model.add_statement( RDF.Statement(submissionNode, dafTermOntology['has_file'], @@ -135,6 +151,10 @@ class Submission(object): RDF.Statement(fileNode, dafTermOntology['filename'], filename)) + self.model.add_statement( + RDF.Statement(fileNode, + dafTermOntology['relative_path'], + os.path.relpath(pathname))) return fileNode def add_md5s(self, filename, fileNode, analysis_dir): @@ -148,14 +168,86 @@ class Submission(object): self.model.add_statement( RDF.Statement(fileNode, dafTermOntology['md5sum'], md5)) + def add_fastq_metadata(self, filename, fileNode): + # How should I detect if this is actually a fastq file? + try: + fqname = FastqName(filename=filename) + except ValueError: + # currently its just ignore it if the fastq name parser fails + return + + terms = [('flowcell', libraryOntology['flowcell_id']), + ('lib_id', libraryOntology['library_id']), + ('lane', libraryOntology['lane_number']), + ('read', libraryOntology['read']), + ('cycle', libraryOntology['read_length'])] + for file_term, model_term in terms: + value = fqname.get(file_term) + if value is not None: + s = RDF.Statement(fileNode, model_term, toTypedNode(value)) + self.model.append(s) + + def add_label(self, file_type, file_node, lib_node): + """Add rdfs:label to a file node + """ + #template_term = libraryOntology['label_template'] + template_term = libraryOntology['label_template'] + label_template = self.model.get_target(file_type, template_term) + if label_template: + template = loader.get_template('submission_view_rdfs_label_metadata.sparql') + context = Context({ + 'library': str(lib_node.uri), + }) + for r in self.execute_query(template, context): + context = Context(r) + label = Template(label_template).render(context) + s = RDF.Statement(file_node, rdfsNS['label'], unicode(label)) + self.model.append(s) + def _add_library_details_to_model(self, libNode): + # attributes that can have multiple values + set_attributes = set((libraryOntology['has_lane'], + libraryOntology['has_mappings'], + dafTermOntology['has_file'])) parser = RDF.Parser(name='rdfa') - new_statements = parser.parse_as_stream(libNode.uri) + try: + new_statements = parser.parse_as_stream(libNode.uri) + except RDF.RedlandError as e: + LOGGER.error(e) + return + LOGGER.debug("Scanning %s", str(libNode.uri)) + toadd = [] for s in new_statements: + # always add "collections" + if s.predicate in set_attributes: + toadd.append(s) + continue # don't override things we already have in the model targets = list(self.model.get_targets(s.subject, s.predicate)) if len(targets) == 0: - self.model.append(s) + toadd.append(s) + + for s in toadd: + self.model.append(s) + + self._add_lane_details(libNode) + + def _add_lane_details(self, libNode): + """Import lane details + """ + query = RDF.Statement(libNode, libraryOntology['has_lane'], None) + lanes = [] + for lane_stmt in self.model.find_statements(query): + lanes.append(lane_stmt.object) + + parser = RDF.Parser(name='rdfa') + for lane in lanes: + LOGGER.debug("Importing %s" % (lane.uri,)) + try: + parser.parse_into_model(self.model, lane.uri) + except RDF.RedlandError, e: + LOGGER.error("Error accessing %s" % (lane.uri,)) + raise e def find_best_match(self, filename): @@ -256,8 +348,11 @@ class Submission(object): 'Small RNA (non-multiplexed)',] paired = ['Barcoded Illumina', 'Multiplexing', + 'NEBNext Multiplexed', + 'NEBNext Small RNA', 'Nextera', - 'Paired End (non-multiplexed)',] + 'Paired End (non-multiplexed)', + 'Dual Index Illumina',] if library_type in single: return False elif library_type in paired: @@ -275,6 +370,27 @@ class Submission(object): query = RDF.SPARQLQuery(str(formatted_query)) rdfstream = query.execute(self.model) results = [] - for r in rdfstream: - results.append(r) + for record in rdfstream: + d = {} + for key, value in record.items(): + d[key] = fromTypedNode(value) + results.append(d) return results + + +def list_submissions(model): + """Return generator of submissions in this model. + """ + query_body = """ + PREFIX subns: + + select distinct ?submission + where { ?submission subns:has_submission ?library_dir } + """ + query = RDF.SPARQLQuery(query_body) + rdfstream = query.execute(model) + for row in rdfstream: + s = strip_namespace(submissionLog, row['submission']) + if s[-1] in ['#', '/', '?']: + s = s[:-1] + yield s