Add option to get_model to disable RDF graph context.

[htsworkflow.git] / htsworkflow / util / rdfhelp.py

diff --git a/htsworkflow/util/rdfhelp.py b/htsworkflow/util/rdfhelp.py
index 93b7ada645e876834319236359b3121c73f94c74..48294416abf1def5a9bc1c3d77b0a4eaa1a607b0 100644 (file)
--- a/htsworkflow/util/rdfhelp.py
+++ b/htsworkflow/util/rdfhelp.py
@@ -7,7 +7,9 @@ from urlparse import urlparse, urlunparse
 from urllib2 import urlopen
 import logging
 import os
+import sys
 import types
+from pkg_resources import resource_listdir, resource_string
 
 import lxml.html
 import lxml.html.clean
@@ -83,7 +85,7 @@ def blankOrUri(value=None):
     return node
 
 
-def toTypedNode(value):
+def toTypedNode(value, language="en"):
     """Convert a python variable to a RDF Node with its closest xsd type
     """
     if type(value) == types.BooleanType:
@@ -111,15 +113,17 @@ def toTypedNode(value):
     if value_type is not None:
         node = RDF.Node(literal=value, datatype=value_type)
     else:
-        node = RDF.Node(literal=unicode(value).encode('utf-8'))
+        node = RDF.Node(literal=unicode(value).encode('utf-8'), language=language)
     return node
 
 
 def fromTypedNode(node):
     """Convert a typed RDF Node to its closest python equivalent
     """
-    if node is None:
-        return None
+    if not isinstance(node, RDF.Node):
+        return node
+    if node.is_resource():
+        return node
 
     value_type = get_node_type(node)
     literal = node.literal_value['string']
@@ -210,7 +214,7 @@ def simplify_uri(uri):
                 return element
     raise ValueError("Unable to simplify %s" % (uri,))
 
-def stripNamespace(namespace, term):
+def strip_namespace(namespace, term):
     """Remove the namespace portion of a term
 
     returns None if they aren't in common
@@ -228,15 +232,17 @@ def stripNamespace(namespace, term):
     return term_s.replace(namespace._prefix, "")
 
 
-def get_model(model_name=None, directory=None):
+def get_model(model_name=None, directory=None, use_contexts=True):
     if directory is None:
         directory = os.getcwd()
 
+    contexts = 'yes' if use_contexts else 'no'
+
     if model_name is None:
-        storage = RDF.MemoryStorage(options_string="contexts='yes'")
+        storage = RDF.MemoryStorage(options_string="contexts='{}'".format(contexts))
         logger.info("Using RDF Memory model")
     else:
-        options = "contexts='yes',hash-type='bdb',dir='{0}'".format(directory)
+        options = "contexts='{0}',hash-type='bdb',dir='{1}'".format(contexts, directory)
         storage = RDF.HashStorage(model_name,
                       options=options)
         logger.info("Using {0} with options {1}".format(model_name, options))
@@ -267,15 +273,20 @@ def load_into_model(model, parser_name, path, ns=None):
 
     statements = []
     retries = 3
+    succeeded = False
     while retries > 0:
         try:
             retries -= 1
             statements = rdf_parser.parse_as_stream(url, ns)
             retries = 0
+            succeeded = True
         except RDF.RedlandError, e:
             errmsg = "RDF.RedlandError: {0} {1} tries remaining"
             logger.error(errmsg.format(str(e), retries))
 
+    if not succeeded:
+        logger.warn("Unable to download %s", url)
+
     for s in statements:
         conditionally_add_statement(model, s, ns)
 
@@ -319,17 +330,24 @@ def add_default_schemas(model, schema_path=None):
     or in the list of directories provided in schema_path
     """
 
-    if schema_path is None:
-        path, _ = os.path.split(__file__)
-        schema_path = [os.path.join(path, 'schemas')]
-    elif type(schema_path) in types.StringTypes:
-        schema_path = [schema_path]
+    schemas = resource_listdir(__name__, 'schemas')
+    for s in schemas:
+        schema = resource_string(__name__,  'schemas/' + s)
+        namespace = 'file://localhost/htsworkflow/schemas/'+s
+        add_schema(model, schema, namespace)
 
-    for p in schema_path:
-        for f in glob(os.path.join(p, '*.turtle')):
-            add_schema(model, f)
+    if schema_path:
+        if type(schema_path) in types.StringTypes:
+            schema_path = [schema_path]
 
-def add_schema(model, filename):
+        for path in schema_path:
+            for pathname in glob(os.path.join(path, '*.turtle')):
+                url = 'file://' + os.path.splitext(pathname)[0]
+                stream = open(pathname, 'r')
+                add_schema(model, stream, url)
+                stream.close()
+
+def add_schema(model, schema, url):
     """Add a schema to a model.
 
     Main difference from 'load_into_model' is it tags it with
@@ -337,8 +355,7 @@ def add_schema(model, filename):
     """
     parser = RDF.Parser(name='turtle')
     context = RDF.Node(RDF.Uri(SCHEMAS_URL))
-    url = 'file://' + filename
-    for s in parser.parse_as_stream(url):
+    for s in parser.parse_string_as_stream(schema, url):
         try:
             model.append(s, context)
         except RDF.RedlandError as e:
@@ -413,12 +430,23 @@ def get_serializer(name='turtle'):
     writer.set_namespace('wot', wotNS._prefix)
 
     # should these be here, kind of specific to an application
-    writer.set_namespace('libraryOntology', libraryOntology._prefix)
+    writer.set_namespace('htswlib', libraryOntology._prefix)
     writer.set_namespace('ucscSubmission', submissionOntology._prefix)
     writer.set_namespace('ucscDaf', dafTermOntology._prefix)
+    writer.set_namespace('geoSoft', geoSoftNS._prefix)
+    writer.set_namespace('encode3', encode3NS._prefix)
     return writer
 
+def get_turtle_header():
+    """Return a turtle header with our typical namespaces
+    """
+    serializer = get_serializer()
+    empty = get_model()
+    return serializer.serialize_model_to_string(empty)
 
-def dump_model(model):
+def dump_model(model, destination=None):
+    if destination is None:
+        destination = sys.stdout
     serializer = get_serializer()
-    print serializer.serialize_model_to_string(model)
+    destination.write(serializer.serialize_model_to_string(model))
+    destination.write(os.linesep)