1 """Helper features for working with librdf
4 from datetime import datetime
5 from urlparse import urlparse, urlunparse
6 from urllib2 import urlopen
12 import lxml.html.clean
15 logger = logging.getLogger(__name__)
17 # standard ontology namespaces
18 owlNS = RDF.NS('http://www.w3.org/2002/07/owl#')
19 dublinCoreNS = RDF.NS("http://purl.org/dc/elements/1.1/")
20 rdfNS = RDF.NS("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
21 rdfsNS = RDF.NS("http://www.w3.org/2000/01/rdf-schema#")
22 xsdNS = RDF.NS("http://www.w3.org/2001/XMLSchema#")
25 submissionOntology = RDF.NS(
26 "http://jumpgate.caltech.edu/wiki/UcscSubmissionOntology#")
27 dafTermOntology = RDF.NS("http://jumpgate.caltech.edu/wiki/UcscDaf#")
28 libraryOntology = RDF.NS("http://jumpgate.caltech.edu/wiki/LibraryOntology#")
29 inventoryOntology = RDF.NS(
30 "http://jumpgate.caltech.edu/wiki/InventoryOntology#")
31 submissionLog = RDF.NS("http://jumpgate.caltech.edu/wiki/SubmissionsLog/")
32 geoSoftNS = RDF.NS('http://www.ncbi.nlm.nih.gov/geo/info/soft2.html#')
34 ISOFORMAT_MS = "%Y-%m-%dT%H:%M:%S.%f"
35 ISOFORMAT_SHORT = "%Y-%m-%dT%H:%M:%S"
38 def sparql_query(model, query_filename, output_format='text'):
39 """Execute sparql query from file
41 logger.info("Opening: %s" % (query_filename,))
42 query_body = open(query_filename, 'r').read()
43 query = RDF.SPARQLQuery(query_body)
44 results = query.execute(model)
45 if output_format == 'html':
46 html_query_results(results)
48 display_query_results(results)
51 def display_query_results(results):
52 """A very simple display of sparql query results showing name value pairs
55 for k, v in row.items()[::-1]:
56 print "{0}: {1}".format(k, v)
59 def html_query_results(result_stream):
60 from django.conf import settings
61 from django.template import Context, loader
63 # I did this because I couldn't figure out how to
64 # get simplify_rdf into the django template as a filter
65 class Simplified(object):
66 def __init__(self, value):
67 self.simple = simplify_rdf(value)
68 if value.is_resource():
73 template = loader.get_template('rdf_report.html')
75 for row in result_stream:
76 new_row = collections.OrderedDict()
78 for k,v in row.items():
79 new_row[k] = Simplified(v)
80 results.append(new_row)
81 context = Context({'results': results,})
82 print template.render(context)
84 def blankOrUri(value=None):
85 """Return a blank node for None or a resource node for strings.
90 elif type(value) in types.StringTypes:
91 node = RDF.Node(uri_string=value)
92 elif isinstance(value, RDF.Node):
98 def toTypedNode(value):
99 """Convert a python variable to a RDF Node with its closest xsd type
101 if type(value) == types.BooleanType:
102 value_type = xsdNS['boolean'].uri
107 elif type(value) in (types.IntType, types.LongType):
108 value_type = xsdNS['decimal'].uri
109 value = unicode(value)
110 elif type(value) == types.FloatType:
111 value_type = xsdNS['float'].uri
112 value = unicode(value)
113 elif isinstance(value, datetime):
114 value_type = xsdNS['dateTime'].uri
115 if value.microsecond == 0:
116 value = value.strftime(ISOFORMAT_SHORT)
118 value = value.strftime(ISOFORMAT_MS)
121 value = unicode(value)
123 if value_type is not None:
124 node = RDF.Node(literal=value, datatype=value_type)
126 node = RDF.Node(literal=unicode(value).encode('utf-8'))
130 def fromTypedNode(node):
131 """Convert a typed RDF Node to its closest python equivalent
136 value_type = get_node_type(node)
137 literal = node.literal_value['string']
138 literal_lower = literal.lower()
140 if value_type == 'boolean':
141 if literal_lower in ('1', 'yes', 'true'):
143 elif literal_lower in ('0', 'no', 'false'):
146 raise ValueError("Unrecognized boolean %s" % (literal,))
147 elif value_type == 'integer':
149 elif value_type == 'decimal' and literal.find('.') == -1:
151 elif value_type in ('decimal', 'float', 'double'):
152 return float(literal)
153 elif value_type in ('string'):
155 elif value_type in ('dateTime'):
157 return datetime.strptime(literal, ISOFORMAT_MS)
158 except ValueError, _:
159 return datetime.strptime(literal, ISOFORMAT_SHORT)
163 def get_node_type(node):
164 """Return just the base name of a XSD datatype:
165 e.g. http://www.w3.org/2001/XMLSchema#integer -> integer
167 # chop off xml schema declaration
168 value_type = node.literal_value['datatype']
169 if value_type is None:
172 value_type = str(value_type)
173 return value_type.replace(str(xsdNS[''].uri), '')
176 def simplify_rdf(value):
177 """Return a short name for a RDF object
178 e.g. The last part of a URI or an untyped string.
180 if isinstance(value, RDF.Node):
181 if value.is_resource():
182 name = simplify_uri(str(value.uri))
183 elif value.is_blank():
186 name = value.literal_value['string']
187 elif isinstance(value, RDF.Uri):
188 name = split_uri(str(value))
194 def simplify_uri(uri):
195 """Split off the end of a uri
197 >>> simplify_uri('http://asdf.org/foo/bar')
199 >>> simplify_uri('http://asdf.org/foo/bar#bleem')
201 >>> simplify_uri('http://asdf.org/foo/bar/')
203 >>> simplify_uri('http://asdf.org/foo/bar?was=foo')
206 parsed = urlparse(uri)
207 if len(parsed.query) > 0:
209 elif len(parsed.fragment) > 0:
210 return parsed.fragment
211 elif len(parsed.path) > 0:
212 for element in reversed(parsed.path.split('/')):
215 raise ValueError("Unable to simplify %s" % (uri,))
217 def simplifyUri(namespace, term):
218 """Remove the namespace portion of a term
220 returns None if they aren't in common
222 if isinstance(term, RDF.Node):
223 if term.is_resource():
226 raise ValueError("This works on resources")
227 elif not isinstance(term, RDF.Uri):
228 raise ValueError("This works on resources")
230 if not term_s.startswith(namespace._prefix):
232 return term_s.replace(namespace._prefix, "")
235 def get_model(model_name=None, directory=None):
236 if directory is None:
237 directory = os.getcwd()
239 if model_name is None:
240 storage = RDF.MemoryStorage()
241 logger.info("Using RDF Memory model")
243 options = "hash-type='bdb',dir='{0}'".format(directory)
244 storage = RDF.HashStorage(model_name,
246 logger.info("Using {0} with options {1}".format(model_name, options))
247 model = RDF.Model(storage)
251 def load_into_model(model, parser_name, path, ns=None):
252 url_parts = list(urlparse(path))
253 if len(url_parts[0]) == 0:
254 url_parts[0] = 'file'
255 url_parts[2] = os.path.abspath(url_parts[2])
256 url = urlunparse(url_parts)
257 logger.info("Opening %s" % (url,))
259 logger.debug("request status: %s" % (req.code,))
260 if parser_name is None:
261 content_type = req.headers.get('Content-Type', None)
262 parser_name = guess_parser(content_type, path)
263 logger.debug("Guessed parser: %s" % (parser_name,))
265 load_string_into_model(model, parser_name, data, ns)
268 def load_string_into_model(model, parser_name, data, ns=None):
270 ns = RDF.NS("http://localhost/")
271 imports = owlNS['imports']
272 rdf_parser = RDF.Parser(name=parser_name)
273 for s in rdf_parser.parse_string_as_stream(data, ns):
274 if s.predicate == imports:
276 logger.info("Importing %s" % (obj,))
277 load_into_model(model, None, obj, ns)
278 if s.object.is_literal():
279 value_type = get_node_type(s.object)
280 if value_type == 'string':
281 s.object = sanitize_literal(s.object)
282 model.add_statement(s)
285 def sanitize_literal(node):
286 """Clean up a literal string
288 if not isinstance(node, RDF.Node):
289 raise ValueError("sanitize_literal only works on RDF.Nodes")
291 element = lxml.html.fromstring(node.literal_value['string'])
292 cleaner = lxml.html.clean.Cleaner(page_structure=False)
293 element = cleaner.clean_html(element)
294 text = lxml.html.tostring(element)
298 args = {'literal': text[p_len:-slash_p_len]}
299 datatype = node.literal_value['datatype']
300 if datatype is not None:
301 args['datatype'] = datatype
302 language = node.literal_value['language']
303 if language is not None:
304 args['language'] = language
305 return RDF.Node(**args)
308 def guess_parser(content_type, pathname):
309 if content_type in ('application/rdf+xml'):
311 elif content_type in ('application/x-turtle'):
313 elif content_type in ('text/html'):
315 elif content_type is None:
316 _, ext = os.path.splitext(pathname)
317 if ext in ('xml', 'rdf'):
319 elif ext in ('html'):
321 elif ext in ('turtle'):
325 def get_serializer(name='turtle'):
326 """Return a serializer with our standard prefixes loaded
328 writer = RDF.Serializer(name=name)
329 # really standard stuff
330 writer.set_namespace('owl', owlNS._prefix)
331 writer.set_namespace('rdf', rdfNS._prefix)
332 writer.set_namespace('rdfs', rdfsNS._prefix)
333 writer.set_namespace('xsd', xsdNS._prefix)
335 # should these be here, kind of specific to an application
336 writer.set_namespace('libraryOntology', libraryOntology._prefix)
337 writer.set_namespace('ucscSubmission', submissionOntology._prefix)
338 writer.set_namespace('ucscDaf', dafTermOntology._prefix)
342 def dump_model(model):
343 serializer = get_serializer()
344 print serializer.serialize_model_to_string(model)