X-Git-Url: http://woldlab.caltech.edu/gitweb/?p=tabix.git;a=blobdiff_plain;f=index.c;h=b3479fb7b8846f8506b8de98cea97df3fb17ea48;hp=cc564981ee5696f0d3350298312287c516946f85;hb=36e5c2e93cc97d22f8b33768602d511697fa1adf;hpb=bc08e6c2f34c1367d0fe75dd19cedccfab881229

diff --git a/index.c b/index.c
index cc56498..b3479fb 100644
--- a/index.c
+++ b/index.c
@@ -1,5 +1,6 @@
 #include <ctype.h>
 #include <assert.h>
+#include <sys/stat.h>
 #include "khash.h"
 #include "ksort.h"
 #include "kstring.h"
@@ -41,6 +42,15 @@ struct __ti_index_t {
 	ti_lidx_t *index2;
 };
 
+struct __ti_iter_t {
+	int from_first; // read from the first record; no random access
+	int tid, beg, end, n_off, i, finished;
+	uint64_t curr_off;
+	kstring_t str;
+	const ti_index_t *idx;
+	pair64_t *off;
+};
+
 typedef struct {
 	int tid, beg, end, bin;
 } ti_intv_t;
@@ -158,9 +168,12 @@ static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
 				intv->tid = get_tid(idx, str->s + b);
 				if (i != str->l) str->s[i] = '\t';
 			} else if (id == idx->conf.bc) {
-				// here ->beg is 1-based. it will be changed to 0-based at the end of this routine.
-				intv->beg = strtol(str->s + b, &s, 0);
-				if (idx->conf.preset&TI_FLAG_UCSC) ++intv->beg;
+				// here ->beg is 0-based.
+				intv->beg = intv->end = strtol(str->s + b, &s, 0);
+				if (!(idx->conf.preset&TI_FLAG_UCSC)) --intv->beg;
+				else ++intv->end;
+				if (intv->beg < 0) intv->beg = 0;
+				if (intv->end < 1) intv->end = 1;
 			} else {
 				if ((idx->conf.preset&0xffff) == TI_PRESET_GENERIC) {
 					if (id == idx->conf.ec) intv->end = strtol(str->s + b, &s, 0);
@@ -174,7 +187,8 @@ static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
 							if (op == 'M' || op == 'D' || op == 'N') l += x;
 							s = t + 1;
 						}
-						intv->end = intv->beg + l - 1;
+						if (l == 0) l = 1;
+						intv->end = intv->beg + l;
 					}
 				} else if ((idx->conf.preset&0xffff) == TI_PRESET_VCF) {
 					// FIXME: the following is NOT tested and is likely to be buggy
@@ -188,7 +202,7 @@ static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
 								s = t + 1;
 							} else ++s;
 						}
-						intv->end = intv->beg + max - 1;
+						intv->end = intv->beg + max;
 					}
 				}
 			}
@@ -197,7 +211,7 @@ static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
 		}
 	}
 	if (intv->tid < 0 || intv->beg < 0 || intv->end < 0) return -1;
-	intv->bin = ti_reg2bin(intv->beg-1, intv->end);
+	intv->bin = ti_reg2bin(intv->beg, intv->end);
 	return 0;
 }
 
@@ -236,9 +250,13 @@ static inline void insert_offset2(ti_lidx_t *index2, int _beg, int _end, uint64_
 		index2->offset = (uint64_t*)realloc(index2->offset, index2->m * 8);
 		memset(index2->offset + old_m, 0, 8 * (index2->m - old_m));
 	}
-	for (i = beg + 1; i <= end; ++i)
-		if (index2->offset[i] == 0) index2->offset[i] = offset;
-	index2->n = end + 1;
+	if (beg == end) {
+		if (index2->offset[beg] == 0) index2->offset[beg] = offset;
+	} else {
+		for (i = beg; i <= end; ++i)
+			if (index2->offset[i] == 0) index2->offset[i] = offset;
+	}
+	if (index2->n < end + 1) index2->n = end + 1;
 }
 
 static void merge_chunks(ti_index_t *idx)
@@ -262,6 +280,17 @@ static void merge_chunks(ti_index_t *idx)
 	} // ~for(i)
 }
 
+static void fill_missing(ti_index_t *idx)
+{
+	int i, j;
+	for (i = 0; i < idx->n; ++i) {
+		ti_lidx_t *idx2 = &idx->index2[i];
+		for (j = 1; j < idx2->n; ++j)
+			if (idx2->offset[j] == 0)
+				idx2->offset[j] = idx2->offset[j-1];
+	}
+}
+
 ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
 {
 	int ret;
@@ -294,10 +323,10 @@ ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
 			last_tid = intv.tid;
 			last_bin = 0xffffffffu;
 		} else if (last_coor > intv.beg) {
-			fprintf(stderr, "[ti_index_core] the file is not sorted at line %llu\n", (unsigned long long)lineno);
+			fprintf(stderr, "[ti_index_core] the file out of order at line %llu\n", (unsigned long long)lineno);
 			exit(1);
 		}
-		if (intv.bin < 4681) insert_offset2(&idx->index2[intv.tid], intv.beg, intv.end, last_off);
+		insert_offset2(&idx->index2[intv.tid], intv.beg, intv.end, last_off);
 		if (intv.bin != last_bin) { // then possibly write the binning index
 			if (save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record
 				insert_offset(idx->index[save_tid], save_bin, save_off, last_off);
@@ -316,6 +345,7 @@ ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
 	}
 	if (save_tid >= 0) insert_offset(idx->index[save_tid], save_bin, save_off, bgzf_tell(fp));
 	merge_chunks(idx);
+	fill_missing(idx);
 
 	free(str->s); free(str);
 	return idx;
@@ -523,22 +553,10 @@ static ti_index_t *ti_index_load_core(BGZF *fp)
 	return idx;
 }
 
-ti_index_t *ti_index_load_local(const char *_fn)
+ti_index_t *ti_index_load_local(const char *fnidx)
 {
 	BGZF *fp;
-	char *fnidx, *fn;
-
-	if (strstr(_fn, "ftp://") == _fn || strstr(_fn, "http://") == _fn) {
-		const char *p;
-		int l = strlen(_fn);
-		for (p = _fn + l - 1; p >= _fn; --p)
-			if (*p == '/') break;
-		fn = strdup(p + 1);
-	} else fn = strdup(_fn);
-	fnidx = (char*)calloc(strlen(fn) + 5, 1);
-	strcpy(fnidx, fn); strcat(fnidx, ".tbi");
 	fp = bgzf_open(fnidx, "r");
-	free(fnidx); free(fn);
 	if (fp) {
 		ti_index_t *idx = ti_index_load_core(fp);
 		bgzf_close(fp);
@@ -584,17 +602,48 @@ static void download_from_remote(const char *url)
 }
 #endif
 
+static char *get_local_version(const char *fn)
+{
+    struct stat sbuf;
+	char *fnidx = (char*)calloc(strlen(fn) + 5, 1);
+	strcat(strcpy(fnidx, fn), ".tbi");
+	if ((strstr(fnidx, "ftp://") == fnidx || strstr(fnidx, "http://") == fnidx)) {
+		char *p, *url;
+		int l = strlen(fnidx);
+		for (p = fnidx + l - 1; p >= fnidx; --p)
+			if (*p == '/') break;
+		url = fnidx; fnidx = strdup(p + 1);
+		if (stat(fnidx, &sbuf) == 0) {
+			free(url);
+			return fnidx;
+		}
+		fprintf(stderr, "[%s] downloading the index file...\n", __func__);
+		download_from_remote(url);
+		free(url);
+	}
+    if (stat(fnidx, &sbuf) == 0) return fnidx;
+	free(fnidx); return 0;
+}
+
+const char **ti_seqname(const ti_index_t *idx, int *n)
+{
+	const char **names;
+	khint_t k;
+	*n = idx->n;
+	names = calloc(idx->n, sizeof(void*));
+	for (k = kh_begin(idx->tname); k < kh_end(idx->tname); ++k)
+		if (kh_exist(idx->tname, k))
+			names[kh_val(idx->tname, k)] = kh_key(idx->tname, k);
+	return names;
+}
+
 ti_index_t *ti_index_load(const char *fn)
 {
 	ti_index_t *idx;
-	idx = ti_index_load_local(fn);
-	if (idx == 0 && (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn)) {
-		char *fnidx = calloc(strlen(fn) + 5, 1);
-		strcat(strcpy(fnidx, fn), ".tbi");
-		fprintf(stderr, "[ti_index_load] attempting to download the remote index file.\n");
-		download_from_remote(fnidx);
-		idx = ti_index_load_local(fn);
-	}
+    char *fname = get_local_version(fn);
+	if (fname == 0) return 0;
+	idx = ti_index_load_local(fname);
+    free(fname);
 	if (idx == 0) fprintf(stderr, "[ti_index_load] fail to load BAM index.\n");
 	return idx;
 }
@@ -636,12 +685,19 @@ int ti_index_build(const char *fn, const ti_conf_t *conf)
  * parse a region in the format chr:beg-end *
  ********************************************/
 
-int ti_parse_region(ti_index_t *idx, const char *str, int *tid, int *begin, int *end)
+int ti_get_tid(const ti_index_t *idx, const char *name)
+{
+	khiter_t iter;
+	const khash_t(s) *h = idx->tname;
+	iter = kh_get(s, h, name); /* get the tid */
+	if (iter == kh_end(h)) return -1;
+	return kh_value(h, iter);
+}
+
+int ti_parse_region(const ti_index_t *idx, const char *str, int *tid, int *begin, int *end)
 {
 	char *s, *p;
 	int i, l, k;
-	khiter_t iter;
-	khash_t(s) *h = idx->tname;
 	l = strlen(str);
 	p = s = (char*)malloc(l+1);
 	/* squeeze out "," */
@@ -650,12 +706,10 @@ int ti_parse_region(ti_index_t *idx, const char *str, int *tid, int *begin, int
 	s[k] = 0;
 	for (i = 0; i != k; ++i) if (s[i] == ':') break;
 	s[i] = 0;
-	iter = kh_get(s, h, s); /* get the tid */
-	if (iter == kh_end(h)) { // name not found
-		*tid = -1; free(s);
+	if ((*tid = ti_get_tid(idx, s)) < 0) {
+		free(s);
 		return -1;
 	}
-	*tid = kh_value(h, iter);
 	if (i == k) { /* dump the whole sequence */
 		*begin = 0; *end = 1<<29; free(s);
 		return 0;
@@ -681,6 +735,8 @@ int ti_parse_region(ti_index_t *idx, const char *str, int *tid, int *begin, int
 static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[MAX_BIN])
 {
 	int i = 0, k;
+	if (beg >= end) return 0;
+	if (end >= 1u<<29) end = 1u<<29;
 	--end;
 	list[i++] = 0;
 	for (k =    1 + (beg>>26); k <=    1 + (end>>26); ++k) list[i++] = k;
@@ -691,13 +747,15 @@ static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[MAX_BIN])
 	return i;
 }
 
-static inline int is_overlap(int32_t beg, int32_t end, int rbeg, int rend)
+ti_iter_t ti_iter_first()
 {
-	return (rend > beg && rbeg < end);
+	ti_iter_t iter;
+	iter = calloc(1, sizeof(struct __ti_iter_t));
+	iter->from_first = 1;
+	return iter;
 }
 
-// ti_fetch helper function retrieves 
-pair64_t *get_chunk_coordinates(const ti_index_t *idx, int tid, int beg, int end, int* cnt_off)
+ti_iter_t ti_iter_query(const ti_index_t *idx, int tid, int beg, int end)
 {
 	uint16_t *bins;
 	int i, n_bins, n_off;
@@ -705,17 +763,34 @@ pair64_t *get_chunk_coordinates(const ti_index_t *idx, int tid, int beg, int end
 	khint_t k;
 	khash_t(i) *index;
 	uint64_t min_off;
-
+	ti_iter_t iter = 0;
+
+	if (beg < 0) beg = 0;
+	if (end < beg) return 0;
+	// initialize the iterator
+	iter = calloc(1, sizeof(struct __ti_iter_t));
+	iter->idx = idx; iter->tid = tid; iter->beg = beg; iter->end = end; iter->i = -1;
+	// random access
 	bins = (uint16_t*)calloc(MAX_BIN, 2);
 	n_bins = reg2bins(beg, end, bins);
 	index = idx->index[tid];
-	min_off = (beg>>TAD_LIDX_SHIFT >= idx->index2[tid].n)? 0 : idx->index2[tid].offset[beg>>TAD_LIDX_SHIFT];
+	if (idx->index2[tid].n > 0) {
+		min_off = (beg>>TAD_LIDX_SHIFT >= idx->index2[tid].n)? idx->index2[tid].offset[idx->index2[tid].n-1]
+			: idx->index2[tid].offset[beg>>TAD_LIDX_SHIFT];
+		if (min_off == 0) { // improvement for index files built by tabix prior to 0.1.4
+			int n = beg>>TAD_LIDX_SHIFT;
+			if (n > idx->index2[tid].n) n = idx->index2[tid].n;
+			for (i = n - 1; i >= 0; --i)
+				if (idx->index2[tid].offset[i] != 0) break;
+			if (i >= 0) min_off = idx->index2[tid].offset[i];
+		}
+	} else min_off = 0; // tabix 0.1.2 may produce such index files
 	for (i = n_off = 0; i < n_bins; ++i) {
 		if ((k = kh_get(i, index, bins[i])) != kh_end(index))
 			n_off += kh_value(index, k).n;
 	}
 	if (n_off == 0) {
-		free(bins); return 0;
+		free(bins); return iter;
 	}
 	off = (pair64_t*)calloc(n_off, 16);
 	for (i = n_off = 0; i < n_bins; ++i) {
@@ -746,44 +821,125 @@ pair64_t *get_chunk_coordinates(const ti_index_t *idx, int tid, int beg, int end
 			n_off = l + 1;
 		}
 	}
-	*cnt_off = n_off;
-	return off;
+	iter->n_off = n_off; iter->off = off;
+	return iter;
 }
 
-int ti_fetch(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end, void *data, ti_fetch_f func)
+const char *ti_iter_read(BGZF *fp, ti_iter_t iter, int *len)
 {
-	int n_off;
-	pair64_t *off = get_chunk_coordinates(idx, tid, beg, end, &n_off);
-	if (off == 0) return 0;
-	{
-		// retrive alignments
-		uint64_t curr_off;
-		int i, ret, n_seeks;
-		kstring_t *str = calloc(1, sizeof(kstring_t));
-		n_seeks = 0; i = -1; curr_off = 0;
-		for (;;) {
-			if (curr_off == 0 || curr_off >= off[i].v) { // then jump to the next chunk
-				if (i == n_off - 1) break; // no more chunks
-				if (i >= 0) assert(curr_off == off[i].v); // otherwise bug
-				if (i < 0 || off[i].v != off[i+1].u) { // not adjacent chunks; then seek
-					bgzf_seek(fp, off[i+1].u, SEEK_SET);
-					curr_off = bgzf_tell(fp);
-					++n_seeks;
-				}
-				++i;
+	if (iter->finished) return 0;
+	if (iter->from_first) {
+		int ret;
+		if ((ret = ti_readline(fp, &iter->str)) < 0) {
+			iter->finished = 1;
+			return 0;
+		} else {
+			if (len) *len = iter->str.l;
+			return iter->str.s;
+		}
+	}
+	if (iter->n_off == 0) return 0;
+	while (1) {
+		int ret;
+		if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk
+			if (iter->i == iter->n_off - 1) break; // no more chunks
+			if (iter->i >= 0) assert(iter->curr_off == iter->off[iter->i].v); // otherwise bug
+			if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek
+				bgzf_seek(fp, iter->off[iter->i+1].u, SEEK_SET);
+				iter->curr_off = bgzf_tell(fp);
 			}
-			if ((ret = ti_readline(fp, str)) >= 0) {
-				ti_intv_t intv;
-				curr_off = bgzf_tell(fp);
-				if (str->s[0] == idx->conf.meta_char) continue;
-				get_intv((ti_index_t*)idx, str, &intv);
-				if (intv.tid != tid || intv.beg >= end) break; // no need to proceed
-				else if (is_overlap(beg, end, intv.beg, intv.end)) func(str->l, str->s, data);
-			} else break; // end of file
+			++iter->i;
 		}
-//		fprintf(stderr, "[ti_fetch] # seek calls: %d\n", n_seeks);
-		free(str->s); free(str);
+		if ((ret = ti_readline(fp, &iter->str)) >= 0) {
+			ti_intv_t intv;
+			iter->curr_off = bgzf_tell(fp);
+			if (iter->str.s[0] == iter->idx->conf.meta_char) continue;
+			get_intv((ti_index_t*)iter->idx, &iter->str, &intv);
+			if (intv.tid != iter->tid || intv.beg >= iter->end) break; // no need to proceed
+			else if (intv.end > iter->beg && iter->end > intv.beg) {
+				if (len) *len = iter->str.l;
+				return iter->str.s;
+			}
+		} else break; // end of file
+	}
+	iter->finished = 1;
+	return 0;
+}
+
+void ti_iter_destroy(ti_iter_t iter)
+{
+	if (iter) {
+		free(iter->str.s); free(iter->off);
+		free(iter);
+	}
+}
+
+int ti_fetch(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end, void *data, ti_fetch_f func)
+{
+	ti_iter_t iter;
+	const char *s;
+	int len;
+	iter = ti_iter_query(idx, tid, beg, end);
+	while ((s = ti_iter_read(fp, iter, &len)) != 0)
+		func(len, s, data);
+	ti_iter_destroy(iter);
+	return 0;
+}
+
+/*******************
+ * High-level APIs *
+ *******************/
+
+tabix_t *ti_open(const char *fn, const char *fnidx)
+{
+	tabix_t *t;
+	BGZF *fp;
+	if ((fp = bgzf_open(fn, "r")) == 0) return 0;
+	t = calloc(1, sizeof(tabix_t));
+	t->fn = strdup(fn);
+	if (fnidx) t->fnidx = strdup(fnidx);
+	t->fp = fp;
+	return t;
+}
+
+void ti_close(tabix_t *t)
+{
+	if (t) {
+		bgzf_close(t->fp);
+		if (t->idx) ti_index_destroy(t->idx);
+		free(t->fn); free(t->fnidx);
+		free(t);
+	}
+}
+
+int ti_lazy_index_load(tabix_t *t)
+{
+	if (t->idx == 0) { // load index
+		if (t->fnidx) t->idx = ti_index_load_local(t->fnidx);
+		else t->idx = ti_index_load(t->fn);
+		if (t->idx == 0) return -1; // fail to load index
 	}
-	free(off);
 	return 0;
 }
+
+ti_iter_t ti_queryi(tabix_t *t, int tid, int beg, int end)
+{
+	if (tid < 0) return ti_iter_first();
+	if (ti_lazy_index_load(t) != 0) return 0;
+	return ti_iter_query(t->idx, tid, beg, end);	
+}
+
+ti_iter_t ti_query(tabix_t *t, const char *name, int beg, int end)
+{
+	int tid;
+	if (name == 0) return ti_iter_first();
+	// then need to load the index
+	if (ti_lazy_index_load(t) != 0) return 0;
+	if ((tid = ti_get_tid(t->idx, name)) < 0) return 0;
+	return ti_iter_query(t->idx, tid, beg, end);
+}
+
+const char *ti_read(tabix_t *t, ti_iter_t iter, int *len)
+{
+	return ti_iter_read(t->fp, iter, len);
+}