--- /dev/null
+#!/usr/bin/env python
+
+from ConfigParser import SafeConfigParser
+
+import logging
+import os
+from optparse import OptionParser
+import stat
+import shelve
+
+from htsworkflow.util import api
+from htsworkflow.pipelines.sequences import scan_for_sequences
+
+def build_flowcell_db(fcdb_filename, sequences, baseurl, apiid, apikey):
+ """
+ compare our flowcell database with our list of sequences and return
+ a fully populated database
+ """
+ fcdb = shelve.open(fcdb_filename)
+ libdb = {}
+ apidata = {'apiid': apiid, 'apikey': apikey}
+ for seq in sequences:
+ flowcell = seq.flowcell
+ flowcell_info = None
+
+ # get info about flowcell from server or shelf
+ if not fcdb.has_key(flowcell):
+ url = api.flowcell_url(baseurl, flowcell)
+ flowcell_info = api.retrieve_info(url, apidata)
+ if flowcell_info is not None:
+ fcdb[flowcell] = flowcell_info
+ else:
+ flowcell_info = fcdb[flowcell]
+
+ # make library id db
+ if flowcell_info is not None:
+ seq_library_id = flowcell_info['lane_set'][unicode(seq.lane)]['library_id']
+ libdb.setdefault(seq_library_id, []).append(seq)
+
+ fcdb.sync()
+ return fcdb, libdb
+
+def carefully_make_hardlink(source, destination, dry_run=False):
+ """
+ Make a hard link, failing if a different link already exists
+
+ Checking to see if the link already exists and is
+ the same as the link we want to make.
+ If the link already exists and is different, throw an error.
+
+ If we didn't update anything return 0, if we did update
+ return 1.
+ """
+ logging.debug("CHECKING: %s -> %s", source, destination)
+
+ if not os.path.exists(source):
+ logging.warning("%s doesn't exist", source)
+ return 0
+
+ if os.path.exists(destination):
+ if os.path.samefile(source, destination):
+ logging.debug('SAME: %s -> %s' % (source, destination))
+ return 0
+ else:
+ logging.error('%s and %s are different files, skipping' % \
+ (source, destination))
+ return 0
+ logging.debug('Linking: %s -> %s' % (source, destination))
+
+ # we would do something by this part
+ if dry_run: return 1
+
+ os.link(source, destination)
+ os.chmod(destination,
+ stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH )
+ return 1
+
+def make_library_links(root, library_db, dry_run=False):
+ """
+ Make a tree of sequencer roots organized by library id
+
+ Root is the root of the library tree
+ library_db is a dictionary of SequenceFiles organized by library id
+ """
+ count = 0
+ root = os.path.abspath(root)
+ for lib_id, sequences in library_db.items():
+ target_dir = os.path.join(root, lib_id)
+ if not os.path.exists(target_dir):
+ logging.info("mkdir %s" % (target_dir,))
+ if not dry_run:
+ os.mkdir(target_dir)
+
+ for s in sequences:
+ count += carefully_make_hardlink(s.path,
+ s.make_target_name(target_dir),
+ dry_run=dry_run)
+ return count
+
+def configure_logging(opts):
+ # setup logging
+ level = logging.WARN
+ if opts.verbose:
+ level = logging.INFO
+ if opts.debug:
+ level = logging.DEBUG
+ logging.basicConfig(level=level)
+
+
+def configure_opts(opts):
+ """
+ Load in options from config file
+ """
+ SECTION_NAME = 'sequence_archive'
+ ARCHIVE_OPT = 'sequence_archive'
+ CACHE_OPT = 'cache'
+ HOST_OPT = 'host'
+ APIID_OPT = 'apiid'
+ APIKEY_OPT = 'apikey'
+
+ # figure out what config file to read
+ config_path = [os.path.expanduser('~/.htsworkflow.ini'),
+ '/etc/htsworkflow.ini']
+ if opts.config is not None:
+ config_path = [opts.config]
+ # parse options from config file
+ config_file = SafeConfigParser()
+ config_file.read(config_path)
+
+ # load defaults from config file if not overriden by the command line
+ if opts.cache is None:
+ if config_file.has_option(SECTION_NAME, CACHE_OPT):
+ opts.cache = config_file.get(FRONTEND_NAME, CACHE_OPT)
+ else:
+ opts.cache = os.path.expanduser('~/.flowcelldb.shelve')
+
+ if opts.sequence_archive is None and \
+ config_file.has_option(SECTION_NAME, ARCHIVE_OPT):
+ opts.sequence_archive = config_file.get(SECTION_NAME, ARCHIVE_OPT)
+ opts.sequence_archive = os.path.expanduser(opts.sequence_archive)
+
+ opts.sequence_archive = os.path.abspath(opts.sequence_archive)
+ opts.library_tree = os.path.join(opts.sequence_archive, 'libraries')
+ opts.flowcells = os.path.join(opts.sequence_archive, 'flowcells')
+ opts.srfs = os.path.join(opts.sequence_archive, 'srfs')
+
+ if opts.host is None and config_file.has_option(SECTION_NAME, HOST_OPT):
+ opts.host = config_file.get(SECTION_NAME, HOST_OPT)
+
+ if opts.apiid is None and config_file.has_option(SECTION_NAME, APIID_OPT):
+ opts.apiid = config_file.get(SECTION_NAME, APIID_OPT)
+
+ if opts.apikey is None and config_file.has_option(SECTION_NAME, APIKEY_OPT):
+ opts.apikey = config_file.get(SECTION_NAME, APIKEY_OPT)
+
+ return opts
+
+def make_parser():
+ """
+ Make parser
+ """
+ parser = OptionParser()
+ parser.add_option('-c', '--config', default=None,
+ help='path to a configuration file containing a '
+ 'sequence archive section')
+ parser.add_option('--cache', default=None,
+ help="default flowcell cache")
+
+ parser.add_option('--host', default=None,
+ help="specify http://host for quering flowcell information")
+ parser.add_option('--apiid', default=None,
+ help="API ID to use when retriving information")
+ parser.add_option("--apikey", default=None,
+ help="API Key for when retriving information")
+
+ parser.add_option('-a', '--sequence-archive', default=None,
+ help='path to where the sequence archive lives')
+
+ parser.add_option('-v', '--verbose', action='store_true', default=False,
+ help='be more verbose')
+ parser.add_option('-d', '--debug', action='store_true', default=False,
+ help='report everything')
+
+ parser.add_option("--dry-run", dest="dry_run", action="store_true",
+ default=False,
+ help="Don't modify the filesystem")
+ return parser
+
+def main(cmdline=None):
+ parser = make_parser()
+ opts, args = parser.parse_args(cmdline)
+
+ configure_logging(opts)
+ opts = configure_opts(opts)
+
+ # complain if critical things are missing
+ if opts.cache is None:
+ parser.error('Need location of htsworkflow frontend database')
+
+ if opts.sequence_archive is None:
+ parser.error('Need the root path for the sequence archive')
+
+ seq_dirs = [ opts.flowcells, opts.srfs ]
+ if len(args) > 0:
+ seq_dirs = [os.path.abspath(f) for f in args]
+
+ seqs = scan_for_sequences(seq_dirs)
+ fcdb, libdb = build_flowcell_db(opts.cache, seqs, opts.host, opts.apiid, opts.apikey)
+ updates = make_library_links(opts.library_tree, libdb, dry_run=opts.dry_run)
+
+ logging.warn("%s flowcells in database" % (len(fcdb),))
+ logging.warn("found %s sequence files" % (len(seqs),))
+ logging.warn("%s libraries being checked" % (len(libdb),))
+ logging.warn("%s sequence files were linked" % (updates,))
+
+ return 0
+
+if __name__ == "__main__":
+ main()
projects / htsworkflow.git / blobdiff