From 26168c507d67cb3f1f342ce6ddd2f4fd4bf16dba Mon Sep 17 00:00:00 2001 From: Diane Trout Date: Thu, 19 Mar 2015 14:48:46 -0700 Subject: [PATCH] Use django.utils.encode smart_text or smart_str for intelligent unicode handling some things like writing files or talking to redland are quite picky about whether or not they're unicode or ascii strings. --- experiments/test_experiments.py | 27 ++++++++++--------- .../pipelines/test/test_retrieve_config.py | 5 ++-- htsworkflow/submission/condorfastq.py | 3 ++- htsworkflow/util/hashfile.py | 2 ++ inventory/test_inventory.py | 3 ++- samples/test_samples.py | 16 ++++++----- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/experiments/test_experiments.py b/experiments/test_experiments.py index a353f56..50246b2 100644 --- a/experiments/test_experiments.py +++ b/experiments/test_experiments.py @@ -19,6 +19,7 @@ from django.test import TestCase from django.test.utils import setup_test_environment, teardown_test_environment from django.db import connection from django.conf import settings +from django.utils.encoding import smart_text from .models import ClusterStation, cluster_station_default, \ DataRun, Sequencer, FlowCell, FileType @@ -133,7 +134,7 @@ class ExperimentsTestCases(TestCase): response = self.client.get('/experiments/config/%s/json' % (fc_id,), apidata) # strptime isoformat string = '%Y-%m-%dT%H:%M:%S' - fc_json = json.loads(response.content)['result'] + fc_json = json.loads(smart_text(response.content))['result'] self.assertEqual(fc_json['flowcell_id'], fc_id) self.assertEqual(fc_json['sequencer'], fc_django.sequencer.name) self.assertEqual(fc_json['read_length'], fc_django.read_length) @@ -175,7 +176,7 @@ class ExperimentsTestCases(TestCase): """ response = self.client.get('/experiments/config/FC12150/json', apidata) self.assertEqual(response.status_code, 200) - flowcell = json.loads(response.content)['result'] + flowcell = json.loads(smart_text(response.content))['result'] # library id is 12150 + lane number (1-8), so 12153 lane_contents = flowcell['lane_set']['3'] @@ -184,7 +185,7 @@ class ExperimentsTestCases(TestCase): response = self.client.get('/samples/library/12153/json', apidata) self.assertEqual(response.status_code, 200) - library_12153 = json.loads(response.content)['result'] + library_12153 = json.loads(smart_text(response.content))['result'] self.assertEqual(library_12153['library_id'], '12153') @@ -266,7 +267,7 @@ class ExperimentsTestCases(TestCase): self.assertEqual(len(lanes), 8) response = self.client.get('/experiments/lanes_for/%s/json' % (user,), apidata) - lanes_json = json.loads(response.content)['result'] + lanes_json = json.loads(smart_text(response.content))['result'] self.assertEqual(len(lanes), len(lanes_json)) for i in range(len(lanes)): self.assertEqual(lanes[i]['comment'], lanes_json[i]['comment']) @@ -392,7 +393,7 @@ class ExperimentsTestCases(TestCase): if status is not None: self.assertTrue(status) ns = urljoin('http://localhost', url) - load_string_into_model(model, 'rdfa', response.content, ns=ns) + load_string_into_model(model, 'rdfa', smart_text(response.content), ns=ns) body = """prefix rdf: prefix libns: @@ -460,8 +461,8 @@ class TestEmailNotify(TestCase): response = self.client.get(self.url) self.assertEqual(response.status_code, 200) - self.assertTrue(self.affiliation.email in response.content) - self.assertTrue(self.library.library_name in response.content) + self.assertTrue(self.affiliation.email in smart_text(response.content)) + self.assertTrue(self.library.library_name in smart_text(response.content)) response = self.client.get(self.url, {'send':'1','bcc':'on'}) self.assertEqual(response.status_code, 200) @@ -481,9 +482,10 @@ class TestEmailNotify(TestCase): response = self.client.get(self.url) self.assertEqual(response.status_code, 200) #print("email navigation content:", response.content) - self.assertTrue(re.search(self.fc.flowcell_id, response.content)) + self.assertTrue(re.search(self.fc.flowcell_id, smart_text(response.content))) # require that navigation back to the admin page exists - self.assertTrue(re.search('[^<]+'.format(admin_url), response.content)) + self.assertTrue(re.search('[^<]+'.format(admin_url), + smart_text(response.content))) def multi_lane_to_dict(lane): """Convert a list of lane entries into a dictionary indexed by library ID @@ -554,7 +556,7 @@ class TestSequencer(TestCase): status = validate_xhtml(response.content) if status is not None: self.assertTrue(status) - load_string_into_model(model, 'rdfa', response.content) + load_string_into_model(model, 'rdfa', smart_text(response.content)) errmsgs = list(inference.run_validation()) self.assertEqual(len(errmsgs), 0) @@ -572,11 +574,12 @@ class TestSequencer(TestCase): url = '/lane/{}'.format(self.lane.id) response = self.client.get(url) + rdfbody = smart_text(response.content) self.assertEqual(response.status_code, 200) - status = validate_xhtml(response.content) + status = validate_xhtml(rdfbody) if status is not None: self.assertTrue(status) - load_string_into_model(model, 'rdfa', response.content) + load_string_into_model(model, 'rdfa', rdfbody) errmsgs = list(inference.run_validation()) self.assertEqual(len(errmsgs), 0) diff --git a/htsworkflow/pipelines/test/test_retrieve_config.py b/htsworkflow/pipelines/test/test_retrieve_config.py index 1b412c7..ffb63dc 100644 --- a/htsworkflow/pipelines/test/test_retrieve_config.py +++ b/htsworkflow/pipelines/test/test_retrieve_config.py @@ -9,6 +9,7 @@ except ImportError as e: import simplejson as json from django.test import TestCase +from django.utils.encoding import smart_text from samples.samples_factory import LibraryFactory, LibraryTypeFactory, \ MultiplexIndexFactory @@ -33,7 +34,7 @@ class RetrieveTestCases(TestCase): flowcell_request = self.client.get('/experiments/config/FC12150/json', apidata) self.failUnlessEqual(flowcell_request.status_code, 200) - flowcell_info = json.loads(flowcell_request.content)['result'] + flowcell_info = json.loads(smart_text(flowcell_request.content))['result'] options = getCombinedOptions(['-f','FC12150','-g',os.getcwd()]) genome_map = {library.library_species.scientific_name: '/tmp/build' } @@ -71,7 +72,7 @@ class RetrieveTestCases(TestCase): url = '/experiments/config/%s/json' % (fcid,) flowcell_request = self.client.get(url, apidata) self.failUnlessEqual(flowcell_request.status_code, 200) - flowcell_info = json.loads(flowcell_request.content)['result'] + flowcell_info = json.loads(smart_text(flowcell_request.content))['result'] options = getCombinedOptions(['-f',fcid,'-g',os.getcwd(),]) diff --git a/htsworkflow/submission/condorfastq.py b/htsworkflow/submission/condorfastq.py index 8f9fccb..705d47e 100644 --- a/htsworkflow/submission/condorfastq.py +++ b/htsworkflow/submission/condorfastq.py @@ -22,6 +22,7 @@ from htsworkflow.util.conversion import parse_flowcell_id from django.conf import settings from django.template import Context, loader +from django.utils.encoding import smart_str import RDF @@ -257,7 +258,7 @@ WHERE { """Add link between target pathname and the 'lane' that produced it (note lane objects are now post demultiplexing.) """ - target_uri = 'file://' + target.encode('utf-8') + target_uri = 'file://' + smart_str(target) target_node = RDF.Node(RDF.Uri(target_uri)) source_stmt = RDF.Statement(target_node, dcNS['source'], seq.filenode) self.model.add_statement(source_stmt) diff --git a/htsworkflow/util/hashfile.py b/htsworkflow/util/hashfile.py index fcfb7a3..9762883 100644 --- a/htsworkflow/util/hashfile.py +++ b/htsworkflow/util/hashfile.py @@ -4,6 +4,8 @@ import logging import os from subprocess import Popen, PIPE +from django.utils.encoding import smart_text + logger = logging.getLogger(__name__) def make_md5sum(filename): diff --git a/inventory/test_inventory.py b/inventory/test_inventory.py index 6a5c595..43d0911 100644 --- a/inventory/test_inventory.py +++ b/inventory/test_inventory.py @@ -10,6 +10,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.core.urlresolvers import reverse +from django.utils.encoding import smart_text from .models import Item, Vendor from .inventory_factory import ItemFactory, LongTermStorageFactory @@ -37,7 +38,7 @@ class InventoryTestCase(TestCase): self.failUnlessEqual(response.status_code, 200) model = get_model() - load_string_into_model(model, 'rdfa', response.content, url) + load_string_into_model(model, 'rdfa', smart_text(response.content), url) itemNode = RDF.Node(RDF.Uri(url)) item_type = fromTypedNode( diff --git a/samples/test_samples.py b/samples/test_samples.py index 4d14538..df98418 100644 --- a/samples/test_samples.py +++ b/samples/test_samples.py @@ -6,6 +6,7 @@ import json from django.test import TestCase, RequestFactory from django.conf import settings +from django.utils.encoding import smart_text, smart_str from .models import Affiliation, ExperimentType, Species, Library from .views import library_dict, library_json, library @@ -52,7 +53,7 @@ class SampleWebTestCase(TestCase): lib_dict = library_dict(library.id) url = '/samples/library/%s/json' % (library.id,) lib_response = self.client.get(url, apidata) - lib_json = json.loads(str(lib_response.content))['result'] + lib_json = json.loads(smart_text(lib_response.content))['result'] for d in [lib_dict, lib_json]: # amplified_from_sample is a link to the library table, @@ -150,7 +151,7 @@ class SampleWebTestCase(TestCase): response = self.client.get(library.get_absolute_url()) self.assertEqual(response.status_code, 200) - content = response.content + content = smart_text(response.content) load_string_into_model(model, 'rdfa', content) body = """prefix rdf: @@ -199,7 +200,7 @@ class SampleWebTestCase(TestCase): response = self.client.get('/library/') self.assertEqual(response.status_code, 200) - load_string_into_model(model, 'rdfa', response.content) + load_string_into_model(model, 'rdfa', smart_text(response.content)) errmsgs = list(inference.run_validation()) self.assertEqual(len(errmsgs), 0) @@ -298,12 +299,13 @@ class TestRDFaLibrary(TestCase): ## request = self.request.get(url) ## lib_response = library(request) lib_response = self.client.get(url) - with open('/tmp/body.html', 'w') as outstream: - outstream.write(lib_response.content) - self.failIfEqual(len(lib_response.content), 0) + lib_body = smart_str(lib_response.content) + self.failIfEqual(len(lib_body), 0) + with open('/tmp/body.html', 'wt') as outstream: + outstream.write(lib_body) parser.parse_string_into_model(model, - lib_response.content, + lib_body, 'http://localhost'+url) # help debugging rdf errrors #with open('/tmp/test.ttl', 'w') as outstream: -- 2.30.2