From: Diane Trout Date: Thu, 29 Jan 2015 21:39:32 +0000 (-0800) Subject: use six.moves to work around urllib / urllib2 / urlparse to urllib 2to3 cleanup X-Git-Url: http://woldlab.caltech.edu/gitweb/?p=htsworkflow.git;a=commitdiff_plain;h=5804fa5655649233edfbf7f20d4bee23908232bf use six.moves to work around urllib / urllib2 / urlparse to urllib 2to3 cleanup --- diff --git a/encode_submission/encode3.py b/encode_submission/encode3.py index 29352d5..82e0401 100644 --- a/encode_submission/encode3.py +++ b/encode_submission/encode3.py @@ -18,9 +18,6 @@ import stat import sys import time import types -import urllib -import urllib2 -import urlparse from zipfile import ZipFile import RDF diff --git a/encode_submission/encode_find.py b/encode_submission/encode_find.py index 870a585..dd02f31 100644 --- a/encode_submission/encode_find.py +++ b/encode_submission/encode_find.py @@ -18,8 +18,7 @@ import re # redland rdf lib import RDF import sys -import urllib -import urlparse +from six.moves import urllib if not 'DJANGO_SETTINGS_MODULE' in os.environ: os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings' @@ -483,7 +482,7 @@ def reload_libraries(model, library_list): load_library_detail(model, library_urn) def user_library_id_to_library_urn(library_id): - split_url = urlparse.urlsplit(library_id) + split_url = urllib.parse.urlsplit(library_id) if len(split_url.scheme) == 0: return LIBRARY_NS[library_id] else: @@ -645,7 +644,7 @@ def login(cookie=None): response, content = http.request(LOGIN_URL, 'POST', headers=headers, - body=urllib.urlencode(credentials)) + body=urllib.parse.urlencode(credentials)) LOGGER.debug("Login to {0}, status {1}".format(LOGIN_URL, response['status'])) diff --git a/encode_submission/geo_gather.py b/encode_submission/geo_gather.py index 53e2d44..46d7635 100644 --- a/encode_submission/geo_gather.py +++ b/encode_submission/geo_gather.py @@ -16,9 +16,6 @@ import stat import sys import time import types -import urllib -import urllib2 -import urlparse from zipfile import ZipFile import RDF diff --git a/encode_submission/ucsc_gather.py b/encode_submission/ucsc_gather.py index 6961fa9..4fa8164 100644 --- a/encode_submission/ucsc_gather.py +++ b/encode_submission/ucsc_gather.py @@ -16,9 +16,6 @@ import stat import sys import time import types -import urllib -import urllib2 -import urlparse from zipfile import ZipFile import RDF diff --git a/htsworkflow/automation/copier.py b/htsworkflow/automation/copier.py index 1dc719d..1ba0ab2 100644 --- a/htsworkflow/automation/copier.py +++ b/htsworkflow/automation/copier.py @@ -9,7 +9,7 @@ import subprocess import sys import time import traceback -import urlparse +from six.moves import urllib from benderjab import rpc @@ -289,13 +289,13 @@ class CopierBot(rpc.XmlRpcBot): return reply def validate_url(self, url): - user_url = urlparse.urlsplit(url) + user_url = urllib.parse.urlsplit(url) user_scheme = user_url[0] user_netloc = user_url[1] user_path = user_url[2] for source in self.sources: - source_url = urlparse.urlsplit(source) + source_url = urllib.parse.urlsplit(source) source_scheme = source_url[0] source_netloc = source_url[1] source_path = source_url[2] diff --git a/htsworkflow/pipelines/sequences.py b/htsworkflow/pipelines/sequences.py index 2aba709..cc6f8ba 100644 --- a/htsworkflow/pipelines/sequences.py +++ b/htsworkflow/pipelines/sequences.py @@ -7,7 +7,7 @@ import os import types import re import sys -from urlparse import urljoin, urlparse +from six.moves.urllib.parse import urljoin, urlparse import RDF from htsworkflow.util.rdfhelp import libraryOntology as libNS diff --git a/htsworkflow/submission/condorfastq.py b/htsworkflow/submission/condorfastq.py index 37d60ed..8f9fccb 100644 --- a/htsworkflow/submission/condorfastq.py +++ b/htsworkflow/submission/condorfastq.py @@ -5,7 +5,7 @@ import os from pprint import pformat,pprint import sys import types -from urlparse import urljoin, urlparse +from six.moves.urllib.parse import urljoin, urlparse from htsworkflow.pipelines.sequences import scan_for_sequences, \ update_model_sequence_library diff --git a/htsworkflow/submission/daf.py b/htsworkflow/submission/daf.py index 35b9a6c..9c4b30e 100644 --- a/htsworkflow/submission/daf.py +++ b/htsworkflow/submission/daf.py @@ -7,7 +7,7 @@ import re import string from six.moves import StringIO import types -import urlparse +from six.moves import urllib import RDF from htsworkflow.util.rdfhelp import \ @@ -232,7 +232,7 @@ def submission_uri_to_string(submission_uri): def get_view_namespace(submission_uri): submission_uri = submission_uri_to_string(submission_uri) - view_uri = urlparse.urljoin(submission_uri, 'view/') + view_uri = urllib.parse.urljoin(submission_uri, 'view/') viewNS = RDF.NS(view_uri) return viewNS diff --git a/htsworkflow/submission/encoded.py b/htsworkflow/submission/encoded.py index 29f6ecc..334a03f 100644 --- a/htsworkflow/submission/encoded.py +++ b/htsworkflow/submission/encoded.py @@ -12,7 +12,7 @@ import jsonschema import os import requests import types -from urlparse import urljoin, urlparse, urlunparse +from six.moves.urllib.parse import urljoin, urlparse, urlunparse LOGGER = logging.getLogger(__name__) diff --git a/htsworkflow/submission/ucsc.py b/htsworkflow/submission/ucsc.py index ff71736..f668a2c 100644 --- a/htsworkflow/submission/ucsc.py +++ b/htsworkflow/submission/ucsc.py @@ -1,8 +1,7 @@ """Utilities for extracting information from the ENCODE DCC """ import logging -import urlparse -import urllib2 +from six.moves import urllib LOGGER = logging.getLogger(__name__) @@ -21,7 +20,7 @@ def ddf_download_url(submission_id): 'http://encodesubmit.ucsc.edu/pipeline/download_ddf/1234' """ fragment = 'download_ddf/%s' % (submission_id,) - return urlparse.urljoin(UCSCEncodePipeline, fragment) + return urllib.parse.urljoin(UCSCEncodePipeline, fragment) def daf_download_url(submission_id): @@ -31,7 +30,7 @@ def daf_download_url(submission_id): 'http://encodesubmit.ucsc.edu/pipeline/download_daf/1234' """ fragment = 'download_daf/%s' % (submission_id,) - return urlparse.urljoin(UCSCEncodePipeline, fragment) + return urllib.parse.urljoin(UCSCEncodePipeline, fragment) def submission_view_url(submission_id): @@ -41,7 +40,7 @@ def submission_view_url(submission_id): 'http://encodesubmit.ucsc.edu/pipeline/show/1234' """ fragment = 'show/%s' % (submission_id,) - return urlparse.urljoin(UCSCEncodePipeline, fragment) + return urllib.parse.urljoin(UCSCEncodePipeline, fragment) def get_encodedcc_file_index(genome, composite): @@ -58,10 +57,10 @@ def get_encodedcc_file_index(genome, composite): request_url = base_url + 'files.txt' try: - request = urllib2.urlopen(request_url) + request = urllib.request.urlopen(request_url) file_index = parse_ucsc_file_index(request, base_url) return file_index - except urllib2.HTTPError as e: + except urllib.request.HTTPError as e: err = e pass diff --git a/htsworkflow/util/rdfhelp.py b/htsworkflow/util/rdfhelp.py index 8fa4d0b..89db483 100644 --- a/htsworkflow/util/rdfhelp.py +++ b/htsworkflow/util/rdfhelp.py @@ -5,8 +5,7 @@ from __future__ import print_function import collections from datetime import datetime from glob import glob -from urlparse import urlparse, urlunparse -from urllib2 import urlopen +from six.moves import urllib import logging import os import sys @@ -205,7 +204,7 @@ def simplify_uri(uri): if isinstance(uri, RDF.Uri): uri = str(uri) - parsed = urlparse(uri) + parsed = urllib.parse.urlparse(uri) if len(parsed.query) > 0: return parsed.query elif len(parsed.fragment) > 0: @@ -262,13 +261,13 @@ def load_into_model(model, parser_name, path, ns=None): else: raise ValueError("url to load can't be a RDF literal") - url_parts = list(urlparse(path)) + url_parts = list(urllib.parse.urlparse(path)) if len(url_parts[0]) == 0 or url_parts[0] == 'file': url_parts[0] = 'file' url_parts[2] = os.path.abspath(url_parts[2]) if parser_name is None or parser_name == 'guess': parser_name = guess_parser_by_extension(path) - url = urlunparse(url_parts) + url = urllib.parse.urlunparse(url_parts) logger.info("Opening {0} with parser {1}".format(url, parser_name)) rdf_parser = RDF.Parser(name=parser_name) diff --git a/scripts/htsw-record-runfolder b/scripts/htsw-record-runfolder index 674cd52..dfd0ae1 100755 --- a/scripts/htsw-record-runfolder +++ b/scripts/htsw-record-runfolder @@ -4,8 +4,7 @@ from optparse import OptionParser import os import re import sys -import urllib2 -import urlparse +from six.moves import urllib from django.conf import settings @@ -42,12 +41,12 @@ def update_db(root_url, flowcells, serial, debug=False): Creates link between flowcell and storage device over http """ for fc in flowcells: - url = urlparse.urljoin(root_url, '%s/%s/' % (fc, serial)) + url = urllib.parse.urljoin(root_url, '%s/%s/' % (fc, serial)) - req = urllib2.Request(url) + req = urllib.request.Request(url) try: response = urllib2.urlopen(req) - except urllib2.URLError, e: + except urllib.request.HTTPError, e: print 'ERROR - HTTP OUTPUT (Return Code: %s); use -v/--verbose for more details.' % (e.code) if debug: print e.read()