Add a way to report what url was failing

[htsworkflow.git] / htsworkflow / util / rdfhelp.py

diff --git a/htsworkflow/util/rdfhelp.py b/htsworkflow/util/rdfhelp.py
index fda8772f858601a0f488c61248da3d39d67c7be3..ac5f6ccd1056a561418cd9b17e155ee936097630 100644 (file)
--- a/htsworkflow/util/rdfhelp.py
+++ b/htsworkflow/util/rdfhelp.py
@@ -7,7 +7,9 @@ from urlparse import urlparse, urlunparse
 from urllib2 import urlopen
 import logging
 import os
+import sys
 import types
+from pkg_resources import resource_listdir, resource_string
 
 import lxml.html
 import lxml.html.clean
@@ -83,7 +85,7 @@ def blankOrUri(value=None):
     return node
 
 
-def toTypedNode(value):
+def toTypedNode(value, language="en"):
     """Convert a python variable to a RDF Node with its closest xsd type
     """
     if type(value) == types.BooleanType:
@@ -111,15 +113,17 @@ def toTypedNode(value):
     if value_type is not None:
         node = RDF.Node(literal=value, datatype=value_type)
     else:
-        node = RDF.Node(literal=unicode(value).encode('utf-8'))
+        node = RDF.Node(literal=unicode(value).encode('utf-8'), language=language)
     return node
 
 
 def fromTypedNode(node):
     """Convert a typed RDF Node to its closest python equivalent
     """
-    if node is None:
-        return None
+    if not isinstance(node, RDF.Node):
+        return node
+    if node.is_resource():
+        return node
 
     value_type = get_node_type(node)
     literal = node.literal_value['string']
@@ -258,23 +262,29 @@ def load_into_model(model, parser_name, path, ns=None):
     if len(url_parts[0]) == 0 or url_parts[0] == 'file':
         url_parts[0] = 'file'
         url_parts[2] = os.path.abspath(url_parts[2])
-        if parser_name is None or parser_name == 'guess':
-            parser_name = guess_parser_by_extension(path)
+    if parser_name is None or parser_name == 'guess':
+        parser_name = guess_parser_by_extension(path)
     url = urlunparse(url_parts)
     logger.info("Opening {0} with parser {1}".format(url, parser_name))
 
     rdf_parser = RDF.Parser(name=parser_name)
 
+    statements = []
     retries = 3
+    succeeded = False
     while retries > 0:
         try:
             retries -= 1
             statements = rdf_parser.parse_as_stream(url, ns)
             retries = 0
+            succeeded = True
         except RDF.RedlandError, e:
             errmsg = "RDF.RedlandError: {0} {1} tries remaining"
-            logger.error(errmsg.format(str(e), tries))
-
+            logger.error(errmsg.format(str(e), retries))
+            
+    if not succeeded:
+        logger.warn("Unable to download %s", url)
+        
     for s in statements:
         conditionally_add_statement(model, s, ns)
 
@@ -318,17 +328,24 @@ def add_default_schemas(model, schema_path=None):
     or in the list of directories provided in schema_path
     """
 
-    if schema_path is None:
-        path, _ = os.path.split(__file__)
-        schema_path = [os.path.join(path, 'schemas')]
-    elif type(schema_path) in types.StringTypes:
-        schema_path = [schema_path]
+    schemas = resource_listdir(__name__, 'schemas')
+    for s in schemas:
+        schema = resource_string(__name__,  'schemas/' + s)
+        namespace = 'file://localhost/htsworkflow/schemas/'+s
+        add_schema(model, schema, namespace)
 
-    for p in schema_path:
-        for f in glob(os.path.join(p, '*.turtle')):
-            add_schema(model, f)
+    if schema_path:
+        if type(schema_path) in types.StringTypes:
+            schema_path = [schema_path]
 
-def add_schema(model, filename):
+        for path in schema_path:
+            for pathname in glob(os.path.join(path, '*.turtle')):
+                url = 'file://' + os.path.splitext(pathname)[0]
+                stream = open(pathname, 'r')
+                add_schema(model, stream, url)
+                stream.close()
+
+def add_schema(model, schema, url):
     """Add a schema to a model.
 
     Main difference from 'load_into_model' is it tags it with
@@ -336,8 +353,7 @@ def add_schema(model, filename):
     """
     parser = RDF.Parser(name='turtle')
     context = RDF.Node(RDF.Uri(SCHEMAS_URL))
-    url = 'file://' + filename
-    for s in parser.parse_as_stream(url):
+    for s in parser.parse_string_as_stream(schema, url):
         try:
             model.append(s, context)
         except RDF.RedlandError as e:
@@ -384,16 +400,16 @@ def guess_parser(content_type, pathname):
         return 'turtle'
     elif content_type in ('text/html',):
         return 'rdfa'
-    elif content_type is None:
+    elif content_type is None or content_type in ('text/plain',):
         return guess_parser_by_extension(pathname)
 
 def guess_parser_by_extension(pathname):
     _, ext = os.path.splitext(pathname)
     if ext in ('.xml', '.rdf'):
         return 'rdfxml'
-    elif ext in ('.html'):
+    elif ext in ('.html',):
         return 'rdfa'
-    elif ext in ('.turtle'):
+    elif ext in ('.turtle',):
         return 'turtle'
     return 'guess'
 
@@ -412,12 +428,23 @@ def get_serializer(name='turtle'):
     writer.set_namespace('wot', wotNS._prefix)
 
     # should these be here, kind of specific to an application
-    writer.set_namespace('libraryOntology', libraryOntology._prefix)
+    writer.set_namespace('htswlib', libraryOntology._prefix)
     writer.set_namespace('ucscSubmission', submissionOntology._prefix)
     writer.set_namespace('ucscDaf', dafTermOntology._prefix)
+    writer.set_namespace('geoSoft', geoSoftNS._prefix)
+    writer.set_namespace('encode3', encode3NS._prefix)
     return writer
 
+def get_turtle_header():
+    """Return a turtle header with our typical namespaces
+    """
+    serializer = get_serializer()
+    empty = get_model()
+    return serializer.serialize_model_to_string(empty)
 
-def dump_model(model):
+def dump_model(model, destination=None):
+    if destination is None:
+        destination = sys.stdout
     serializer = get_serializer()
-    print serializer.serialize_model_to_string(model)
+    destination.write(serializer.serialize_model_to_string(model))
+    destination.write(os.linesep)