1 """Helper features for working with librdf
4 from datetime import datetime
5 from urlparse import urlparse, urlunparse
6 from urllib2 import urlopen
12 import lxml.html.clean
15 logger = logging.getLogger(__name__)
17 # standard ontology namespaces
18 owlNS = RDF.NS('http://www.w3.org/2002/07/owl#')
19 dublinCoreNS = RDF.NS("http://purl.org/dc/elements/1.1/")
20 rdfNS = RDF.NS("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
21 rdfsNS = RDF.NS("http://www.w3.org/2000/01/rdf-schema#")
22 xsdNS = RDF.NS("http://www.w3.org/2001/XMLSchema#")
25 submissionOntology = RDF.NS(
26 "http://jumpgate.caltech.edu/wiki/UcscSubmissionOntology#")
27 dafTermOntology = RDF.NS("http://jumpgate.caltech.edu/wiki/UcscDaf#")
28 libraryOntology = RDF.NS("http://jumpgate.caltech.edu/wiki/LibraryOntology#")
29 inventoryOntology = RDF.NS(
30 "http://jumpgate.caltech.edu/wiki/InventoryOntology#")
31 submissionLog = RDF.NS("http://jumpgate.caltech.edu/wiki/SubmissionsLog/")
32 geoSoftNS = RDF.NS('http://www.ncbi.nlm.nih.gov/geo/info/soft2.html#')
34 ISOFORMAT_MS = "%Y-%m-%dT%H:%M:%S.%f"
35 ISOFORMAT_SHORT = "%Y-%m-%dT%H:%M:%S"
38 def sparql_query(model, query_filename, output_format='text'):
39 """Execute sparql query from file
41 logger.info("Opening: %s" % (query_filename,))
42 query_body = open(query_filename, 'r').read()
43 query = RDF.SPARQLQuery(query_body)
44 results = query.execute(model)
45 if output_format == 'html':
46 html_query_results(results)
48 display_query_results(results)
51 def display_query_results(results):
52 """A very simple display of sparql query results showing name value pairs
55 for k, v in row.items()[::-1]:
56 print "{0}: {1}".format(k, v)
59 def html_query_results(result_stream):
60 from django.conf import settings
61 from django.template import Context, loader
63 # I did this because I couldn't figure out how to
64 # get simplify_rdf into the django template as a filter
65 class Simplified(object):
66 def __init__(self, value):
67 self.simple = simplify_rdf(value)
68 if value.is_resource():
73 template = loader.get_template('rdf_report.html')
75 for row in result_stream:
76 new_row = collections.OrderedDict()
78 for k,v in row.items():
79 new_row[k] = Simplified(v)
80 results.append(new_row)
81 context = Context({'results': results,})
82 print template.render(context)
84 def blankOrUri(value=None):
85 """Return a blank node for None or a resource node for strings.
90 elif type(value) in types.StringTypes:
91 node = RDF.Node(uri_string=value)
92 elif isinstance(value, RDF.Node):
98 def toTypedNode(value):
99 """Convert a python variable to a RDF Node with its closest xsd type
101 if type(value) == types.BooleanType:
102 value_type = xsdNS['boolean'].uri
107 elif type(value) in (types.IntType, types.LongType):
108 value_type = xsdNS['decimal'].uri
109 value = unicode(value)
110 elif type(value) == types.FloatType:
111 value_type = xsdNS['float'].uri
112 value = unicode(value)
113 elif isinstance(value, datetime):
114 value_type = xsdNS['dateTime'].uri
115 if value.microsecond == 0:
116 value = value.strftime(ISOFORMAT_SHORT)
118 value = value.strftime(ISOFORMAT_MS)
121 value = unicode(value)
123 if value_type is not None:
124 node = RDF.Node(literal=value, datatype=value_type)
126 node = RDF.Node(literal=unicode(value).encode('utf-8'))
130 def fromTypedNode(node):
131 """Convert a typed RDF Node to its closest python equivalent
136 value_type = get_node_type(node)
137 literal = node.literal_value['string']
138 literal_lower = literal.lower()
140 if value_type == 'boolean':
141 if literal_lower in ('1', 'yes', 'true'):
143 elif literal_lower in ('0', 'no', 'false'):
146 raise ValueError("Unrecognized boolean %s" % (literal,))
147 elif value_type == 'integer':
149 elif value_type == 'decimal' and literal.find('.') == -1:
151 elif value_type in ('decimal', 'float', 'double'):
152 return float(literal)
153 elif value_type in ('string'):
155 elif value_type in ('dateTime'):
157 return datetime.strptime(literal, ISOFORMAT_MS)
158 except ValueError, _:
159 return datetime.strptime(literal, ISOFORMAT_SHORT)
163 def get_node_type(node):
164 """Return just the base name of a XSD datatype:
165 e.g. http://www.w3.org/2001/XMLSchema#integer -> integer
167 # chop off xml schema declaration
168 value_type = node.literal_value['datatype']
169 if value_type is None:
172 value_type = str(value_type)
173 return value_type.replace(str(xsdNS[''].uri), '')
176 def simplify_rdf(value):
177 """Return a short name for a RDF object
178 e.g. The last part of a URI or an untyped string.
180 if isinstance(value, RDF.Node):
181 if value.is_resource():
182 name = simplify_uri(str(value.uri))
183 elif value.is_blank():
186 name = value.literal_value['string']
187 elif isinstance(value, RDF.Uri):
188 name = split_uri(str(value))
194 def simplify_uri(uri):
195 """Split off the end of a uri
197 >>> simplify_uri('http://asdf.org/foo/bar')
199 >>> simplify_uri('http://asdf.org/foo/bar#bleem')
201 >>> simplify_uri('http://asdf.org/foo/bar/')
203 >>> simplify_uri('http://asdf.org/foo/bar?was=foo')
206 if isinstance(uri, RDF.Node):
207 if uri.is_resource():
210 raise ValueError("Can't simplify an RDF literal")
211 if isinstance(uri, RDF.Uri):
214 parsed = urlparse(uri)
215 if len(parsed.query) > 0:
217 elif len(parsed.fragment) > 0:
218 return parsed.fragment
219 elif len(parsed.path) > 0:
220 for element in reversed(parsed.path.split('/')):
223 raise ValueError("Unable to simplify %s" % (uri,))
225 def stripNamespace(namespace, term):
226 """Remove the namespace portion of a term
228 returns None if they aren't in common
230 if isinstance(term, RDF.Node):
231 if term.is_resource():
234 raise ValueError("This works on resources")
235 elif not isinstance(term, RDF.Uri):
236 raise ValueError("This works on resources")
238 if not term_s.startswith(namespace._prefix):
240 return term_s.replace(namespace._prefix, "")
243 def get_model(model_name=None, directory=None):
244 if directory is None:
245 directory = os.getcwd()
247 if model_name is None:
248 storage = RDF.MemoryStorage()
249 logger.info("Using RDF Memory model")
251 options = "hash-type='bdb',dir='{0}'".format(directory)
252 storage = RDF.HashStorage(model_name,
254 logger.info("Using {0} with options {1}".format(model_name, options))
255 model = RDF.Model(storage)
259 def load_into_model(model, parser_name, path, ns=None):
260 if type(ns) in types.StringTypes:
263 if isinstance(path, RDF.Node):
264 if path.is_resource():
267 raise ValueError("url to load can't be a RDF literal")
269 url_parts = list(urlparse(path))
270 if len(url_parts[0]) == 0 or url_parts[0] == 'file':
271 url_parts[0] = 'file'
272 url_parts[2] = os.path.abspath(url_parts[2])
273 if parser_name is None or parser_name == 'guess':
274 parser_name = guess_parser_by_extension(path)
275 url = urlunparse(url_parts)
276 logger.info("Opening {0} with parser {1}".format(url, parser_name))
278 rdf_parser = RDF.Parser(name=parser_name)
279 for s in rdf_parser.parse_as_stream(url, ns):
280 conditionally_add_statement(model, s, ns)
282 def load_string_into_model(model, parser_name, data, ns=None):
283 ns = fixup_namespace(ns)
284 logger.debug("load_string_into_model parser={0}, len={1}".format(
285 parser_name, len(data)))
286 rdf_parser = RDF.Parser(name=parser_name)
288 for s in rdf_parser.parse_string_as_stream(data, ns):
289 conditionally_add_statement(model, s, ns)
291 def fixup_namespace(ns):
293 ns = RDF.Uri("http://localhost/")
294 elif type(ns) in types.StringTypes:
296 elif not(isinstance(ns, RDF.Uri)):
297 errmsg = "Namespace should be string or uri not {0}"
298 raise ValueError(errmsg.format(str(type(ns))))
301 def conditionally_add_statement(model, s, ns):
302 imports = owlNS['imports']
303 if s.predicate == imports:
305 logger.info("Importing %s" % (obj,))
306 load_into_model(model, None, obj, ns)
307 if s.object.is_literal():
308 value_type = get_node_type(s.object)
309 if value_type == 'string':
310 s.object = sanitize_literal(s.object)
311 model.add_statement(s)
313 def sanitize_literal(node):
314 """Clean up a literal string
316 if not isinstance(node, RDF.Node):
317 raise ValueError("sanitize_literal only works on RDF.Nodes")
319 s = node.literal_value['string']
321 element = lxml.html.fromstring(s)
322 cleaner = lxml.html.clean.Cleaner(page_structure=False)
323 element = cleaner.clean_html(element)
324 text = lxml.html.tostring(element)
328 args = {'literal': text[p_len:-slash_p_len]}
330 args = {'literal': ''}
331 datatype = node.literal_value['datatype']
332 if datatype is not None:
333 args['datatype'] = datatype
334 language = node.literal_value['language']
335 if language is not None:
336 args['language'] = language
337 return RDF.Node(**args)
340 def guess_parser(content_type, pathname):
341 if content_type in ('application/rdf+xml',):
343 elif content_type in ('application/x-turtle',):
345 elif content_type in ('text/html',):
347 elif content_type is None:
348 return guess_parser_by_extension(pathname)
350 def guess_parser_by_extension(pathname):
351 _, ext = os.path.splitext(pathname)
352 if ext in ('.xml', '.rdf'):
354 elif ext in ('.html'):
356 elif ext in ('.turtle'):
360 def get_serializer(name='turtle'):
361 """Return a serializer with our standard prefixes loaded
363 writer = RDF.Serializer(name=name)
364 # really standard stuff
365 writer.set_namespace('owl', owlNS._prefix)
366 writer.set_namespace('rdf', rdfNS._prefix)
367 writer.set_namespace('rdfs', rdfsNS._prefix)
368 writer.set_namespace('xsd', xsdNS._prefix)
370 # should these be here, kind of specific to an application
371 writer.set_namespace('libraryOntology', libraryOntology._prefix)
372 writer.set_namespace('ucscSubmission', submissionOntology._prefix)
373 writer.set_namespace('ucscDaf', dafTermOntology._prefix)
377 def dump_model(model):
378 serializer = get_serializer()
379 print serializer.serialize_model_to_string(model)