"""
import collections
from datetime import datetime
+from glob import glob
from urlparse import urlparse, urlunparse
from urllib2 import urlopen
import logging
import os
+import sys
import types
+from pkg_resources import resource_listdir, resource_string
import lxml.html
import lxml.html.clean
logger = logging.getLogger(__name__)
-# standard ontology namespaces
-owlNS = RDF.NS('http://www.w3.org/2002/07/owl#')
-dublinCoreNS = RDF.NS("http://purl.org/dc/elements/1.1/")
-rdfNS = RDF.NS("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
-rdfsNS = RDF.NS("http://www.w3.org/2000/01/rdf-schema#")
-xsdNS = RDF.NS("http://www.w3.org/2001/XMLSchema#")
-
-# internal ontologies
-submissionOntology = RDF.NS(
- "http://jumpgate.caltech.edu/wiki/UcscSubmissionOntology#")
-dafTermOntology = RDF.NS("http://jumpgate.caltech.edu/wiki/UcscDaf#")
-libraryOntology = RDF.NS("http://jumpgate.caltech.edu/wiki/LibraryOntology#")
-inventoryOntology = RDF.NS(
- "http://jumpgate.caltech.edu/wiki/InventoryOntology#")
-submissionLog = RDF.NS("http://jumpgate.caltech.edu/wiki/SubmissionsLog/")
-geoSoftNS = RDF.NS('http://www.ncbi.nlm.nih.gov/geo/info/soft2.html#')
+from htsworkflow.util.rdfns import *
+
+SCHEMAS_URL='http://jumpgate.caltech.edu/phony/schemas'
+INFERENCE_URL='http://jumpgate.caltech.edu/phony/inference'
ISOFORMAT_MS = "%Y-%m-%dT%H:%M:%S.%f"
ISOFORMAT_SHORT = "%Y-%m-%dT%H:%M:%S"
-
def sparql_query(model, query_filename, output_format='text'):
"""Execute sparql query from file
"""
return node
-def toTypedNode(value):
+def toTypedNode(value, language="en"):
"""Convert a python variable to a RDF Node with its closest xsd type
"""
if type(value) == types.BooleanType:
if value_type is not None:
node = RDF.Node(literal=value, datatype=value_type)
else:
- node = RDF.Node(literal=unicode(value).encode('utf-8'))
+ node = RDF.Node(literal=unicode(value).encode('utf-8'), language=language)
return node
def fromTypedNode(node):
"""Convert a typed RDF Node to its closest python equivalent
"""
- if node is None:
- return None
+ if not isinstance(node, RDF.Node):
+ return node
+ if node.is_resource():
+ return node
value_type = get_node_type(node)
literal = node.literal_value['string']
return element
raise ValueError("Unable to simplify %s" % (uri,))
-def stripNamespace(namespace, term):
+def strip_namespace(namespace, term):
"""Remove the namespace portion of a term
returns None if they aren't in common
directory = os.getcwd()
if model_name is None:
- storage = RDF.MemoryStorage()
+ storage = RDF.MemoryStorage(options_string="contexts='yes'")
logger.info("Using RDF Memory model")
else:
- options = "hash-type='bdb',dir='{0}'".format(directory)
+ options = "contexts='yes',hash-type='bdb',dir='{0}'".format(directory)
storage = RDF.HashStorage(model_name,
options=options)
logger.info("Using {0} with options {1}".format(model_name, options))
statements = []
retries = 3
+ succeeded = False
while retries > 0:
try:
retries -= 1
statements = rdf_parser.parse_as_stream(url, ns)
retries = 0
+ succeeded = True
except RDF.RedlandError, e:
errmsg = "RDF.RedlandError: {0} {1} tries remaining"
logger.error(errmsg.format(str(e), retries))
-
+
+ if not succeeded:
+ logger.warn("Unable to download %s", url)
+
for s in statements:
conditionally_add_statement(model, s, ns)
for s in rdf_parser.parse_string_as_stream(data, ns):
conditionally_add_statement(model, s, ns)
+
def fixup_namespace(ns):
if ns is None:
ns = RDF.Uri("http://localhost/")
raise ValueError(errmsg.format(str(type(ns))))
return ns
+
def conditionally_add_statement(model, s, ns):
imports = owlNS['imports']
if s.predicate == imports:
s.object = sanitize_literal(s.object)
model.add_statement(s)
+
+def add_default_schemas(model, schema_path=None):
+ """Add default schemas to a model
+ Looks for turtle files in either htsworkflow/util/schemas
+ or in the list of directories provided in schema_path
+ """
+
+ schemas = resource_listdir(__name__, 'schemas')
+ for s in schemas:
+ schema = resource_string(__name__, 'schemas/' + s)
+ namespace = 'file://localhost/htsworkflow/schemas/'+s
+ add_schema(model, schema, namespace)
+
+ if schema_path:
+ if type(schema_path) in types.StringTypes:
+ schema_path = [schema_path]
+
+ for path in schema_path:
+ for pathname in glob(os.path.join(path, '*.turtle')):
+ url = 'file://' + os.path.splitext(pathname)[0]
+ stream = open(pathname, 'r')
+ add_schema(model, stream, url)
+ stream.close()
+
+def add_schema(model, schema, url):
+ """Add a schema to a model.
+
+ Main difference from 'load_into_model' is it tags it with
+ a RDFlib context so I can remove them later.
+ """
+ parser = RDF.Parser(name='turtle')
+ context = RDF.Node(RDF.Uri(SCHEMAS_URL))
+ for s in parser.parse_string_as_stream(schema, url):
+ try:
+ model.append(s, context)
+ except RDF.RedlandError as e:
+ logger.error("%s with %s", str(e), str(s))
+
+
+def remove_schemas(model):
+ """Remove statements labeled with our schema context"""
+ context = RDF.Node(RDF.Uri(SCHEMAS_URL))
+ model.context_remove_statements(context)
+
+
def sanitize_literal(node):
"""Clean up a literal string
"""
"""
writer = RDF.Serializer(name=name)
# really standard stuff
- writer.set_namespace('owl', owlNS._prefix)
writer.set_namespace('rdf', rdfNS._prefix)
writer.set_namespace('rdfs', rdfsNS._prefix)
+ writer.set_namespace('owl', owlNS._prefix)
+ writer.set_namespace('dc', dcNS._prefix)
+ writer.set_namespace('xml', xmlNS._prefix)
writer.set_namespace('xsd', xsdNS._prefix)
+ writer.set_namespace('vs', vsNS._prefix)
+ writer.set_namespace('wot', wotNS._prefix)
# should these be here, kind of specific to an application
- writer.set_namespace('libraryOntology', libraryOntology._prefix)
+ writer.set_namespace('htswlib', libraryOntology._prefix)
writer.set_namespace('ucscSubmission', submissionOntology._prefix)
writer.set_namespace('ucscDaf', dafTermOntology._prefix)
+ writer.set_namespace('geoSoft', geoSoftNS._prefix)
+ writer.set_namespace('encode3', encode3NS._prefix)
return writer
+def get_turtle_header():
+ """Return a turtle header with our typical namespaces
+ """
+ serializer = get_serializer()
+ empty = get_model()
+ return serializer.serialize_model_to_string(empty)
-def dump_model(model):
+def dump_model(model, destination=None):
+ if destination is None:
+ destination = sys.stdout
serializer = get_serializer()
- print serializer.serialize_model_to_string(model)
+ destination.write(serializer.serialize_model_to_string(model))
+ destination.write(os.linesep)