X-Git-Url: http://woldlab.caltech.edu/gitweb/?a=blobdiff_plain;f=htsworkflow%2Fsubmission%2Fsubmission.py;h=897053fbc148477c108d232635958db45488dfc5;hb=1000ba383f47a9c3ab422f4552b3725038c96541;hp=01bdb17228689281b3f33e8aaaaa3af5ccda6d18;hpb=ab3052aec5e16ebd3905ac297b958fb10a730f54;p=htsworkflow.git diff --git a/htsworkflow/submission/submission.py b/htsworkflow/submission/submission.py index 01bdb17..897053f 100644 --- a/htsworkflow/submission/submission.py +++ b/htsworkflow/submission/submission.py @@ -8,22 +8,22 @@ import RDF from htsworkflow.util.rdfhelp import \ blankOrUri, \ - dafTermOntology, \ dump_model, \ + fromTypedNode, \ get_model, \ - libraryOntology, \ - owlNS, \ - rdfNS, \ - submissionLog, \ - submissionOntology, \ - toTypedNode, \ - fromTypedNode + strip_namespace, \ + toTypedNode +from htsworkflow.util.rdfns import * from htsworkflow.util.hashfile import make_md5sum - +from htsworkflow.submission.fastqname import FastqName from htsworkflow.submission.daf import \ MetadataLookupException, \ + ModelException, \ get_submission_uri +from django.conf import settings +from django.template import Context, Template, loader + LOGGER = logging.getLogger(__name__) class Submission(object): @@ -60,6 +60,13 @@ class Submission(object): pathname = os.path.abspath(os.path.join(analysis_dir, filename)) self.construct_file_attributes(analysis_dir, libNode, pathname) + def analysis_nodes(self, result_map): + """Return an iterable of analysis nodes + """ + for result_dir in result_map.values(): + an_analysis = self.get_submission_node(result_dir) + yield an_analysis + def construct_file_attributes(self, analysis_dir, libNode, pathname): """Looking for the best extension The 'best' is the longest match @@ -114,21 +121,28 @@ class Submission(object): an_analysis)) # add file specific information - fileNode = self.link_file_to_classes(pathname, - an_analysis, - an_analysis_uri, - analysis_dir) + fileNode = self.make_file_node(pathname, an_analysis) self.add_md5s(filename, fileNode, analysis_dir) + self.add_fastq_metadata(filename, fileNode) + self.add_label(file_type, fileNode, libNode) self.model.add_statement( RDF.Statement(fileNode, rdfNS['type'], file_type)) + self.model.add_statement( + RDF.Statement(fileNode, + libraryOntology['library'], + libNode)) + LOGGER.debug("Done.") - def link_file_to_classes(self, pathname, submissionNode, submission_uri, analysis_dir): + def make_file_node(self, pathname, submissionNode): + """Create file node and attach it to its submission. + """ # add file specific information path, filename = os.path.split(pathname) - fileNode = RDF.Node(RDF.Uri('file://'+ os.path.abspath(pathname))) + pathname = os.path.abspath(pathname) + fileNode = RDF.Node(RDF.Uri('file://'+ pathname)) self.model.add_statement( RDF.Statement(submissionNode, dafTermOntology['has_file'], @@ -137,6 +151,10 @@ class Submission(object): RDF.Statement(fileNode, dafTermOntology['filename'], filename)) + self.model.add_statement( + RDF.Statement(fileNode, + dafTermOntology['relative_path'], + os.path.relpath(pathname))) return fileNode def add_md5s(self, filename, fileNode, analysis_dir): @@ -150,13 +168,54 @@ class Submission(object): self.model.add_statement( RDF.Statement(fileNode, dafTermOntology['md5sum'], md5)) + def add_fastq_metadata(self, filename, fileNode): + # How should I detect if this is actually a fastq file? + try: + fqname = FastqName(filename=filename) + except ValueError: + # currently its just ignore it if the fastq name parser fails + return + + terms = [('flowcell', libraryOntology['flowcell_id']), + ('lib_id', libraryOntology['library_id']), + ('lane', libraryOntology['lane_number']), + ('read', libraryOntology['read']), + ('cycle', libraryOntology['read_length'])] + for file_term, model_term in terms: + value = fqname.get(file_term) + if value is not None: + s = RDF.Statement(fileNode, model_term, toTypedNode(value)) + self.model.append(s) + + def add_label(self, file_type, file_node, lib_node): + """Add rdfs:label to a file node + """ + #template_term = libraryOntology['label_template'] + template_term = libraryOntology['label_template'] + label_template = self.model.get_target(file_type, template_term) + if label_template: + template = loader.get_template('submission_view_rdfs_label_metadata.sparql') + context = Context({ + 'library': str(lib_node.uri), + }) + for r in self.execute_query(template, context): + context = Context(r) + label = Template(label_template).render(context) + s = RDF.Statement(file_node, rdfsNS['label'], unicode(label)) + self.model.append(s) + def _add_library_details_to_model(self, libNode): # attributes that can have multiple values set_attributes = set((libraryOntology['has_lane'], libraryOntology['has_mappings'], dafTermOntology['has_file'])) parser = RDF.Parser(name='rdfa') - new_statements = parser.parse_as_stream(libNode.uri) + try: + new_statements = parser.parse_as_stream(libNode.uri) + except RDF.RedlandError as e: + LOGGER.error(e) + return + LOGGER.debug("Scanning %s", str(libNode.uri)) toadd = [] for s in new_statements: # always add "collections" @@ -289,8 +348,11 @@ class Submission(object): 'Small RNA (non-multiplexed)',] paired = ['Barcoded Illumina', 'Multiplexing', + 'NEBNext Multiplexed', + 'NEBNext Small RNA', 'Nextera', - 'Paired End (non-multiplexed)',] + 'Paired End (non-multiplexed)', + 'Dual Index Illumina',] if library_type in single: return False elif library_type in paired: @@ -314,3 +376,21 @@ class Submission(object): d[key] = fromTypedNode(value) results.append(d) return results + + +def list_submissions(model): + """Return generator of submissions in this model. + """ + query_body = """ + PREFIX subns: + + select distinct ?submission + where { ?submission subns:has_submission ?library_dir } + """ + query = RDF.SPARQLQuery(query_body) + rdfstream = query.execute(model) + for row in rdfstream: + s = strip_namespace(submissionLog, row['submission']) + if s[-1] in ['#', '/', '?']: + s = s[:-1] + yield s