Use django.utils.encode smart_text or smart_str for intelligent unicode handling

some things like writing files or talking to redland are
quite picky about whether or not they're unicode or ascii strings.

diff --git a/experiments/test_experiments.py b/experiments/test_experiments.py
index a353f56408c9253bc6cee91d61174ea37f899f19..50246b2227158ed44ec081757201d52cb62254d3 100644 (file)
--- a/experiments/test_experiments.py
+++ b/experiments/test_experiments.py
@@ -19,6 +19,7 @@ from django.test import TestCase
 from django.test.utils import setup_test_environment, teardown_test_environment
 from django.db import connection
 from django.conf import settings
 from django.test.utils import setup_test_environment, teardown_test_environment
 from django.db import connection
 from django.conf import settings

+from django.utils.encoding import smart_text

 
 from .models import ClusterStation, cluster_station_default, \
     DataRun, Sequencer, FlowCell, FileType
 
 from .models import ClusterStation, cluster_station_default, \
     DataRun, Sequencer, FlowCell, FileType


@@ -133,7 +134,7 @@ class ExperimentsTestCases(TestCase):
 
             response = self.client.get('/experiments/config/%s/json' % (fc_id,), apidata)
             # strptime isoformat string = '%Y-%m-%dT%H:%M:%S'
 
             response = self.client.get('/experiments/config/%s/json' % (fc_id,), apidata)
             # strptime isoformat string = '%Y-%m-%dT%H:%M:%S'

-            fc_json = json.loads(response.content)['result']
+            fc_json = json.loads(smart_text(response.content))['result']

             self.assertEqual(fc_json['flowcell_id'], fc_id)
             self.assertEqual(fc_json['sequencer'], fc_django.sequencer.name)
             self.assertEqual(fc_json['read_length'], fc_django.read_length)
             self.assertEqual(fc_json['flowcell_id'], fc_id)
             self.assertEqual(fc_json['sequencer'], fc_django.sequencer.name)
             self.assertEqual(fc_json['read_length'], fc_django.read_length)


@@ -175,7 +176,7 @@ class ExperimentsTestCases(TestCase):
         """
         response = self.client.get('/experiments/config/FC12150/json', apidata)
         self.assertEqual(response.status_code, 200)
         """
         response = self.client.get('/experiments/config/FC12150/json', apidata)
         self.assertEqual(response.status_code, 200)

-        flowcell = json.loads(response.content)['result']
+        flowcell = json.loads(smart_text(response.content))['result']

 
         # library id is 12150 + lane number (1-8), so 12153
         lane_contents = flowcell['lane_set']['3']
 
         # library id is 12150 + lane number (1-8), so 12153
         lane_contents = flowcell['lane_set']['3']


@@ -184,7 +185,7 @@ class ExperimentsTestCases(TestCase):
 
         response = self.client.get('/samples/library/12153/json', apidata)
         self.assertEqual(response.status_code, 200)
 
         response = self.client.get('/samples/library/12153/json', apidata)
         self.assertEqual(response.status_code, 200)

-        library_12153 = json.loads(response.content)['result']
+        library_12153 = json.loads(smart_text(response.content))['result']

 
         self.assertEqual(library_12153['library_id'], '12153')
 
 
         self.assertEqual(library_12153['library_id'], '12153')
 


@@ -266,7 +267,7 @@ class ExperimentsTestCases(TestCase):
         self.assertEqual(len(lanes), 8)
 
         response = self.client.get('/experiments/lanes_for/%s/json' % (user,), apidata)
         self.assertEqual(len(lanes), 8)
 
         response = self.client.get('/experiments/lanes_for/%s/json' % (user,), apidata)

-        lanes_json = json.loads(response.content)['result']
+        lanes_json = json.loads(smart_text(response.content))['result']

         self.assertEqual(len(lanes), len(lanes_json))
         for i in range(len(lanes)):
             self.assertEqual(lanes[i]['comment'], lanes_json[i]['comment'])
         self.assertEqual(len(lanes), len(lanes_json))
         for i in range(len(lanes)):
             self.assertEqual(lanes[i]['comment'], lanes_json[i]['comment'])


@@ -392,7 +393,7 @@ class ExperimentsTestCases(TestCase):
         if status is not None: self.assertTrue(status)
 
         ns = urljoin('http://localhost', url)
         if status is not None: self.assertTrue(status)
 
         ns = urljoin('http://localhost', url)

-        load_string_into_model(model, 'rdfa', response.content, ns=ns)
+        load_string_into_model(model, 'rdfa', smart_text(response.content), ns=ns)

         body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         prefix libns: <http://jumpgate.caltech.edu/wiki/LibraryOntology#>
 
         body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         prefix libns: <http://jumpgate.caltech.edu/wiki/LibraryOntology#>
 


@@ -460,8 +461,8 @@ class TestEmailNotify(TestCase):
         response = self.client.get(self.url)
         self.assertEqual(response.status_code, 200)
 
         response = self.client.get(self.url)
         self.assertEqual(response.status_code, 200)
 

-        self.assertTrue(self.affiliation.email in response.content)
-        self.assertTrue(self.library.library_name in response.content)
+        self.assertTrue(self.affiliation.email in smart_text(response.content))
+        self.assertTrue(self.library.library_name in smart_text(response.content))

 
         response = self.client.get(self.url, {'send':'1','bcc':'on'})
         self.assertEqual(response.status_code, 200)
 
         response = self.client.get(self.url, {'send':'1','bcc':'on'})
         self.assertEqual(response.status_code, 200)


@@ -481,9 +482,10 @@ class TestEmailNotify(TestCase):
         response = self.client.get(self.url)
         self.assertEqual(response.status_code, 200)
         #print("email navigation content:", response.content)
         response = self.client.get(self.url)
         self.assertEqual(response.status_code, 200)
         #print("email navigation content:", response.content)

-        self.assertTrue(re.search(self.fc.flowcell_id, response.content))
+        self.assertTrue(re.search(self.fc.flowcell_id, smart_text(response.content)))

         # require that navigation back to the admin page exists
         # require that navigation back to the admin page exists

-        self.assertTrue(re.search('<a href="{}">[^<]+</a>'.format(admin_url), response.content))
+        self.assertTrue(re.search('<a href="{}">[^<]+</a>'.format(admin_url),
+                                  smart_text(response.content)))

 
 def multi_lane_to_dict(lane):
     """Convert a list of lane entries into a dictionary indexed by library ID
 
 def multi_lane_to_dict(lane):
     """Convert a list of lane entries into a dictionary indexed by library ID


@@ -554,7 +556,7 @@ class TestSequencer(TestCase):
         status = validate_xhtml(response.content)
         if status is not None: self.assertTrue(status)
 
         status = validate_xhtml(response.content)
         if status is not None: self.assertTrue(status)
 

-        load_string_into_model(model, 'rdfa', response.content)
+        load_string_into_model(model, 'rdfa', smart_text(response.content))

 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)
 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)


@@ -572,11 +574,12 @@ class TestSequencer(TestCase):
 
         url = '/lane/{}'.format(self.lane.id)
         response = self.client.get(url)
 
         url = '/lane/{}'.format(self.lane.id)
         response = self.client.get(url)

+        rdfbody = smart_text(response.content)

         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.status_code, 200)

-        status = validate_xhtml(response.content)
+        status = validate_xhtml(rdfbody)

         if status is not None: self.assertTrue(status)
 
         if status is not None: self.assertTrue(status)
 

-        load_string_into_model(model, 'rdfa', response.content)
+        load_string_into_model(model, 'rdfa', rdfbody)

 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)
 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)

diff --git a/htsworkflow/pipelines/test/test_retrieve_config.py b/htsworkflow/pipelines/test/test_retrieve_config.py
index 1b412c7d98832760cce6a9c4482638fe8b531508..ffb63dc0ac05afe3c191a8d37e653a8dbe03c8af 100644 (file)
--- a/htsworkflow/pipelines/test/test_retrieve_config.py
+++ b/htsworkflow/pipelines/test/test_retrieve_config.py
@@ -9,6 +9,7 @@ except ImportError as e:
     import simplejson as json
 
 from django.test import TestCase
     import simplejson as json
 
 from django.test import TestCase

+from django.utils.encoding import smart_text

 
 from samples.samples_factory import LibraryFactory, LibraryTypeFactory, \
     MultiplexIndexFactory
 
 from samples.samples_factory import LibraryFactory, LibraryTypeFactory, \
     MultiplexIndexFactory


@@ -33,7 +34,7 @@ class RetrieveTestCases(TestCase):
         flowcell_request = self.client.get('/experiments/config/FC12150/json',
                                            apidata)
         self.failUnlessEqual(flowcell_request.status_code, 200)
         flowcell_request = self.client.get('/experiments/config/FC12150/json',
                                            apidata)
         self.failUnlessEqual(flowcell_request.status_code, 200)

-        flowcell_info = json.loads(flowcell_request.content)['result']
+        flowcell_info = json.loads(smart_text(flowcell_request.content))['result']

 
         options = getCombinedOptions(['-f','FC12150','-g',os.getcwd()])
         genome_map = {library.library_species.scientific_name: '/tmp/build' }
 
         options = getCombinedOptions(['-f','FC12150','-g',os.getcwd()])
         genome_map = {library.library_species.scientific_name: '/tmp/build' }


@@ -71,7 +72,7 @@ class RetrieveTestCases(TestCase):
         url = '/experiments/config/%s/json' % (fcid,)
         flowcell_request = self.client.get(url, apidata)
         self.failUnlessEqual(flowcell_request.status_code, 200)
         url = '/experiments/config/%s/json' % (fcid,)
         flowcell_request = self.client.get(url, apidata)
         self.failUnlessEqual(flowcell_request.status_code, 200)

-        flowcell_info = json.loads(flowcell_request.content)['result']
+        flowcell_info = json.loads(smart_text(flowcell_request.content))['result']

 
         options = getCombinedOptions(['-f',fcid,'-g',os.getcwd(),])
 
 
         options = getCombinedOptions(['-f',fcid,'-g',os.getcwd(),])
 

diff --git a/htsworkflow/submission/condorfastq.py b/htsworkflow/submission/condorfastq.py
index 8f9fccb7b897a677d02971fc45c791488b6237db..705d47e0412587b287010985c90b2469f325cca2 100644 (file)
--- a/htsworkflow/submission/condorfastq.py
+++ b/htsworkflow/submission/condorfastq.py
@@ -22,6 +22,7 @@ from htsworkflow.util.conversion import parse_flowcell_id
 
 from django.conf import settings
 from django.template import Context, loader
 
 from django.conf import settings
 from django.template import Context, loader

+from django.utils.encoding import smart_str

 
 import RDF
 
 
 import RDF
 


@@ -257,7 +258,7 @@ WHERE {
         """Add link between target pathname and the 'lane' that produced it
         (note lane objects are now post demultiplexing.)
         """
         """Add link between target pathname and the 'lane' that produced it
         (note lane objects are now post demultiplexing.)
         """

-        target_uri = 'file://' + target.encode('utf-8')
+        target_uri = 'file://' + smart_str(target)

         target_node = RDF.Node(RDF.Uri(target_uri))
         source_stmt = RDF.Statement(target_node, dcNS['source'], seq.filenode)
         self.model.add_statement(source_stmt)
         target_node = RDF.Node(RDF.Uri(target_uri))
         source_stmt = RDF.Statement(target_node, dcNS['source'], seq.filenode)
         self.model.add_statement(source_stmt)

diff --git a/htsworkflow/util/hashfile.py b/htsworkflow/util/hashfile.py
index fcfb7a386edbb1453d85275167be641ab358b236..97628832d9408f007ab9e145b285d40e2b0dda6d 100644 (file)
--- a/htsworkflow/util/hashfile.py
+++ b/htsworkflow/util/hashfile.py
@@ -4,6 +4,8 @@ import logging
 import os
 from subprocess import Popen, PIPE
 
 import os
 from subprocess import Popen, PIPE
 

+from django.utils.encoding import smart_text
+

 logger = logging.getLogger(__name__)
 
 def make_md5sum(filename):
 logger = logging.getLogger(__name__)
 
 def make_md5sum(filename):

diff --git a/inventory/test_inventory.py b/inventory/test_inventory.py
index 6a5c59552e1c92ebcf3f833ed21e30379064d512..43d091101fd016b8ab1fbbf39936f3b97d59336a 100644 (file)
--- a/inventory/test_inventory.py
+++ b/inventory/test_inventory.py
@@ -10,6 +10,7 @@ from django.conf import settings
 
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse

+from django.utils.encoding import smart_text

 
 from .models import Item, Vendor
 from .inventory_factory import ItemFactory, LongTermStorageFactory
 
 from .models import Item, Vendor
 from .inventory_factory import ItemFactory, LongTermStorageFactory


@@ -37,7 +38,7 @@ class InventoryTestCase(TestCase):
         self.failUnlessEqual(response.status_code, 200)
 
         model = get_model()
         self.failUnlessEqual(response.status_code, 200)
 
         model = get_model()

-        load_string_into_model(model, 'rdfa', response.content, url)
+        load_string_into_model(model, 'rdfa', smart_text(response.content), url)

 
         itemNode = RDF.Node(RDF.Uri(url))
         item_type = fromTypedNode(
 
         itemNode = RDF.Node(RDF.Uri(url))
         item_type = fromTypedNode(

diff --git a/samples/test_samples.py b/samples/test_samples.py
index 4d14538c3d1c17780d67161ab0b55aba13a2e7ad..df98418aed4afad2e5fb18797d7417a4ae75a96a 100644 (file)
--- a/samples/test_samples.py
+++ b/samples/test_samples.py
@@ -6,6 +6,7 @@ import json
 
 from django.test import TestCase, RequestFactory
 from django.conf import settings
 
 from django.test import TestCase, RequestFactory
 from django.conf import settings

+from django.utils.encoding import smart_text, smart_str

 
 from .models import Affiliation, ExperimentType, Species, Library
 from .views import library_dict, library_json, library
 
 from .models import Affiliation, ExperimentType, Species, Library
 from .views import library_dict, library_json, library


@@ -52,7 +53,7 @@ class SampleWebTestCase(TestCase):
         lib_dict = library_dict(library.id)
         url = '/samples/library/%s/json' % (library.id,)
         lib_response = self.client.get(url, apidata)
         lib_dict = library_dict(library.id)
         url = '/samples/library/%s/json' % (library.id,)
         lib_response = self.client.get(url, apidata)

-        lib_json = json.loads(str(lib_response.content))['result']
+        lib_json = json.loads(smart_text(lib_response.content))['result']

 
         for d in [lib_dict, lib_json]:
             # amplified_from_sample is a link to the library table,
 
         for d in [lib_dict, lib_json]:
             # amplified_from_sample is a link to the library table,


@@ -150,7 +151,7 @@ class SampleWebTestCase(TestCase):
 
         response = self.client.get(library.get_absolute_url())
         self.assertEqual(response.status_code, 200)
 
         response = self.client.get(library.get_absolute_url())
         self.assertEqual(response.status_code, 200)

-        content = response.content
+        content = smart_text(response.content)

         load_string_into_model(model, 'rdfa', content)
 
         body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         load_string_into_model(model, 'rdfa', content)
 
         body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>


@@ -199,7 +200,7 @@ class SampleWebTestCase(TestCase):
 
         response = self.client.get('/library/')
         self.assertEqual(response.status_code, 200)
 
         response = self.client.get('/library/')
         self.assertEqual(response.status_code, 200)

-        load_string_into_model(model, 'rdfa', response.content)
+        load_string_into_model(model, 'rdfa', smart_text(response.content))

 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)
 
         errmsgs = list(inference.run_validation())
         self.assertEqual(len(errmsgs), 0)


@@ -298,12 +299,13 @@ class TestRDFaLibrary(TestCase):
         ## request = self.request.get(url)
         ## lib_response = library(request)
         lib_response = self.client.get(url)
         ## request = self.request.get(url)
         ## lib_response = library(request)
         lib_response = self.client.get(url)

-        with open('/tmp/body.html', 'w') as outstream:
-            outstream.write(lib_response.content)
-        self.failIfEqual(len(lib_response.content), 0)
+        lib_body = smart_str(lib_response.content)
+        self.failIfEqual(len(lib_body), 0)
+        with open('/tmp/body.html', 'wt') as outstream:
+            outstream.write(lib_body)

 
         parser.parse_string_into_model(model,
 
         parser.parse_string_into_model(model,

-                                       lib_response.content,
+                                       lib_body,

                                        'http://localhost'+url)
         # help debugging rdf errrors
         #with open('/tmp/test.ttl', 'w') as outstream:
                                        'http://localhost'+url)
         # help debugging rdf errrors
         #with open('/tmp/test.ttl', 'w') as outstream: