Use django.utils.encode smart_text or smart_str for intelligent unicode handling

some things like writing files or talking to redland are
quite picky about whether or not they're unicode or ascii strings.

diff --git a/experiments/test_experiments.py b/experiments/test_experiments.py
index a353f56408c9253bc6cee91d61174ea37f899f19..50246b2227158ed44ec081757201d52cb62254d3 100644 (file)
--- a/experiments/test_experiments.py
+++ b/experiments/test_experiments.py
@@ -19,6 +19,7 @@ from django.test import TestCase
 from django.test.utils import setup_test_environment, teardown_test_environment
 from django.db import connection
 from django.conf import settings
+from django.utils.encoding import smart_text
 
 from .models import ClusterStation, cluster_station_default, \
     DataRun, Sequencer, FlowCell, FileType
@@ -133,7 +134,7 @@ class ExperimentsTestCases(TestCase):
 
             response = self.client.get('/experiments/config/%s/json' % (fc_id,), apidata)
             # strptime isoformat string = '%Y-%m-%dT%H:%M:%S'
-            fc_json = json.loads(response.content)['result']
+            fc_json = json.loads(smart_text(response.content))['result']
             self.assertEqual(fc_json['flowcell_id'], fc_id)
             self.assertEqual(fc_json['sequencer'], fc_django.sequencer.name)
             self.assertEqual(fc_json['read_length'], fc_django.read_length)
@@ -175,7 +176,7 @@ class ExperimentsTestCases(TestCase):
         """
         response = self.client.get('/experiments/config/FC12150/json', apidata)
         self.assertEqual(response.status_code, 200)
-        flowcell = json.loads(response.content)['result']
+        flowcell = json.loads(smart_text(response.content))['result']
 
         # library id is 12150 + lane number (1-8), so 12153
         lane_contents = flowcell['lane_set']['3']
@@ -184,7 +185,7 @@ class ExperimentsTestCases(TestCase):
 
         response = self.client.get('/samples/library/12153/json', apidata)
         self.assertEqual(response.status_code, 200)
-        library_12153 = json.loads(response.content)['result']
+        library_12153 = json.loads(smart_text(response.content))['result']
 
         self.assertEqual(library_12153['library_id'], '12153')
 
@@ -266,7 +267,7 @@ class ExperimentsTestCases(TestCase):
         self.assertEqual(len(lanes), 8)
 
         response = self.client.get('/experiments/lanes_for/%s/json' % (user,), apidata)
-        lanes_json = json.loads(response.content)['result']
+        lanes_json = json.loads(smart_text(response.content))['result']
         self.assertEqual(len(lanes), len(lanes_json))
         for i in range(len(lanes)):
             self.assertEqual(lanes[i]['comment'], lanes_json[i]['comment'])
@@ -392,7 +393,7 @@ class ExperimentsTestCases(TestCase):
         if status is not None: self.assertTrue(status)
 
         ns = urljoin('http://localhost', url)
-        load_string_into_model(model, 'rdfa', response.content, ns=ns)
+        load_string_into_model(model, 'rdfa', smart_text(response.content), ns=ns)
         body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         prefix libns: <http://jumpgate.caltech.edu/wiki/LibraryOntology#>
 
@@ -460,8 +461,8 @@ class TestEmailNotify(TestCase):
         response = self.client.get(self.url)
         self.assertEqual(response.status_code, 200)
 
-        self.assertTrue(self.affiliation.email in response.content)
-        self.assertTrue(self.library.library_name in response.content)
+        self.assertTrue(self.affiliation.email in smart_text(response.content))
+        self.assertTrue(self.library.library_name in smart_text(response.content))
 
         response = self.client.get(self.url, {'send':'1','bcc':'on'})
         self.assertEqual(response.status_code, 200)
@@ -481,9 +482,10 @@ class TestEmailNotify(TestCase):
         response = self.client.get(self.url)
         self.assertEqual(response.status_code, 200)
         #print("email navigation content:", response.content)
-        self.assertTrue(re.search(self.fc.flowcell_id, response.content))
+        self.assertTrue(re.search(self.fc.flowcell_id, smart_text(response.content)))
         # require that navigation back to the admin page exists
-        self.assertTrue(re.search('<a href="{}">[^<]+</a>'.format(admin_url), response.content))
+        self.assertTrue(re.search('<a href="{}">[^<]+</a>'.format(admin_url),
+                                  smart_text(response.content)))
 
 def multi_lane_to_dict(lane):
     """Convert a list of lane entries into a dictionary indexed by library ID
@@ -554,7 +556,7 @@ class TestSequencer(TestCase):
         status = validate_xhtml(response.content)
         if status is not None: self.assertTrue(status)
 
-        load_string_into_model(model, 'rdfa', response.content)
+        load_string_into_model(model, 'rdfa', smart_text(response.content))
 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)
@@ -572,11 +574,12 @@ class TestSequencer(TestCase):
 
         url = '/lane/{}'.format(self.lane.id)
         response = self.client.get(url)
+        rdfbody = smart_text(response.content)
         self.assertEqual(response.status_code, 200)
-        status = validate_xhtml(response.content)
+        status = validate_xhtml(rdfbody)
         if status is not None: self.assertTrue(status)
 
-        load_string_into_model(model, 'rdfa', response.content)
+        load_string_into_model(model, 'rdfa', rdfbody)
 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)

diff --git a/htsworkflow/pipelines/test/test_retrieve_config.py b/htsworkflow/pipelines/test/test_retrieve_config.py
index 1b412c7d98832760cce6a9c4482638fe8b531508..ffb63dc0ac05afe3c191a8d37e653a8dbe03c8af 100644 (file)
--- a/htsworkflow/pipelines/test/test_retrieve_config.py
+++ b/htsworkflow/pipelines/test/test_retrieve_config.py
@@ -9,6 +9,7 @@ except ImportError as e:
     import simplejson as json
 
 from django.test import TestCase
+from django.utils.encoding import smart_text
 
 from samples.samples_factory import LibraryFactory, LibraryTypeFactory, \
     MultiplexIndexFactory
@@ -33,7 +34,7 @@ class RetrieveTestCases(TestCase):
         flowcell_request = self.client.get('/experiments/config/FC12150/json',
                                            apidata)
         self.failUnlessEqual(flowcell_request.status_code, 200)
-        flowcell_info = json.loads(flowcell_request.content)['result']
+        flowcell_info = json.loads(smart_text(flowcell_request.content))['result']
 
         options = getCombinedOptions(['-f','FC12150','-g',os.getcwd()])
         genome_map = {library.library_species.scientific_name: '/tmp/build' }
@@ -71,7 +72,7 @@ class RetrieveTestCases(TestCase):
         url = '/experiments/config/%s/json' % (fcid,)
         flowcell_request = self.client.get(url, apidata)
         self.failUnlessEqual(flowcell_request.status_code, 200)
-        flowcell_info = json.loads(flowcell_request.content)['result']
+        flowcell_info = json.loads(smart_text(flowcell_request.content))['result']
 
         options = getCombinedOptions(['-f',fcid,'-g',os.getcwd(),])
 

diff --git a/htsworkflow/submission/condorfastq.py b/htsworkflow/submission/condorfastq.py
index 8f9fccb7b897a677d02971fc45c791488b6237db..705d47e0412587b287010985c90b2469f325cca2 100644 (file)
--- a/htsworkflow/submission/condorfastq.py
+++ b/htsworkflow/submission/condorfastq.py
@@ -22,6 +22,7 @@ from htsworkflow.util.conversion import parse_flowcell_id
 
 from django.conf import settings
 from django.template import Context, loader
+from django.utils.encoding import smart_str
 
 import RDF
 
@@ -257,7 +258,7 @@ WHERE {
         """Add link between target pathname and the 'lane' that produced it
         (note lane objects are now post demultiplexing.)
         """
-        target_uri = 'file://' + target.encode('utf-8')
+        target_uri = 'file://' + smart_str(target)
         target_node = RDF.Node(RDF.Uri(target_uri))
         source_stmt = RDF.Statement(target_node, dcNS['source'], seq.filenode)
         self.model.add_statement(source_stmt)

diff --git a/htsworkflow/util/hashfile.py b/htsworkflow/util/hashfile.py
index fcfb7a386edbb1453d85275167be641ab358b236..97628832d9408f007ab9e145b285d40e2b0dda6d 100644 (file)
--- a/htsworkflow/util/hashfile.py
+++ b/htsworkflow/util/hashfile.py
@@ -4,6 +4,8 @@ import logging
 import os
 from subprocess import Popen, PIPE
 
+from django.utils.encoding import smart_text
+
 logger = logging.getLogger(__name__)
 
 def make_md5sum(filename):

diff --git a/inventory/test_inventory.py b/inventory/test_inventory.py
index 6a5c59552e1c92ebcf3f833ed21e30379064d512..43d091101fd016b8ab1fbbf39936f3b97d59336a 100644 (file)
--- a/inventory/test_inventory.py
+++ b/inventory/test_inventory.py
@@ -10,6 +10,7 @@ from django.conf import settings
 
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
+from django.utils.encoding import smart_text
 
 from .models import Item, Vendor
 from .inventory_factory import ItemFactory, LongTermStorageFactory
@@ -37,7 +38,7 @@ class InventoryTestCase(TestCase):
         self.failUnlessEqual(response.status_code, 200)
 
         model = get_model()
-        load_string_into_model(model, 'rdfa', response.content, url)
+        load_string_into_model(model, 'rdfa', smart_text(response.content), url)
 
         itemNode = RDF.Node(RDF.Uri(url))
         item_type = fromTypedNode(

diff --git a/samples/test_samples.py b/samples/test_samples.py
index 4d14538c3d1c17780d67161ab0b55aba13a2e7ad..df98418aed4afad2e5fb18797d7417a4ae75a96a 100644 (file)
--- a/samples/test_samples.py
+++ b/samples/test_samples.py
@@ -6,6 +6,7 @@ import json
 
 from django.test import TestCase, RequestFactory
 from django.conf import settings
+from django.utils.encoding import smart_text, smart_str
 
 from .models import Affiliation, ExperimentType, Species, Library
 from .views import library_dict, library_json, library
@@ -52,7 +53,7 @@ class SampleWebTestCase(TestCase):
         lib_dict = library_dict(library.id)
         url = '/samples/library/%s/json' % (library.id,)
         lib_response = self.client.get(url, apidata)
-        lib_json = json.loads(str(lib_response.content))['result']
+        lib_json = json.loads(smart_text(lib_response.content))['result']
 
         for d in [lib_dict, lib_json]:
             # amplified_from_sample is a link to the library table,
@@ -150,7 +151,7 @@ class SampleWebTestCase(TestCase):
 
         response = self.client.get(library.get_absolute_url())
         self.assertEqual(response.status_code, 200)
-        content = response.content
+        content = smart_text(response.content)
         load_string_into_model(model, 'rdfa', content)
 
         body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
@@ -199,7 +200,7 @@ class SampleWebTestCase(TestCase):
 
         response = self.client.get('/library/')
         self.assertEqual(response.status_code, 200)
-        load_string_into_model(model, 'rdfa', response.content)
+        load_string_into_model(model, 'rdfa', smart_text(response.content))
 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)
@@ -298,12 +299,13 @@ class TestRDFaLibrary(TestCase):
         ## request = self.request.get(url)
         ## lib_response = library(request)
         lib_response = self.client.get(url)
-        with open('/tmp/body.html', 'w') as outstream:
-            outstream.write(lib_response.content)
-        self.failIfEqual(len(lib_response.content), 0)
+        lib_body = smart_str(lib_response.content)
+        self.failIfEqual(len(lib_body), 0)
+        with open('/tmp/body.html', 'wt') as outstream:
+            outstream.write(lib_body)
 
         parser.parse_string_into_model(model,
-                                       lib_response.content,
+                                       lib_body,
                                        'http://localhost'+url)
         # help debugging rdf errrors
         #with open('/tmp/test.ttl', 'w') as outstream: