X-Git-Url: http://woldlab.caltech.edu/gitweb/?p=tabix.git;a=blobdiff_plain;f=index.c;h=15cd65e01b379e5ed3daf2ea9aa427996faa6176;hp=61d7d6d6fdc370d09d9b4b916e629cf48e1aa8c2;hb=HEAD;hpb=d1fcb739b58e050328dcfc47c3af235bc0e41455

diff --git a/index.c b/index.c
index 61d7d6d..15cd65e 100644
--- a/index.c
+++ b/index.c
@@ -19,7 +19,7 @@ typedef struct {
 } pair64_t;
 
 #define pair64_lt(a,b) ((a).u < (b).u)
-KSORT_INIT(off, pair64_t, pair64_lt)
+KSORT_INIT(offt, pair64_t, pair64_lt)
 
 typedef struct {
 	uint32_t m, n;
@@ -47,7 +47,6 @@ struct __ti_iter_t {
 	int tid, beg, end, n_off, i, finished;
 	uint64_t curr_off;
 	kstring_t str;
-	BGZF *fp;
 	const ti_index_t *idx;
 	pair64_t *off;
 };
@@ -84,38 +83,7 @@ int ti_readline(BGZF *fp, kstring_t *str)
  * commented out above. */
 int ti_readline(BGZF *fp, kstring_t *str)
 {
-	int l, state = 0;
-	unsigned char *buf = (unsigned char*)fp->uncompressed_block;
-	str->l = 0;
-	do {
-		if (fp->block_offset >= fp->block_length) {
-			if (bgzf_read_block(fp) != 0) { state = -2; break; }
-			if (fp->block_length == 0) { state = -1; break; }
-		}
-		for (l = fp->block_offset; l < fp->block_length && buf[l] != '\n'; ++l);
-		if (l < fp->block_length) state = 1;
-		l -= fp->block_offset;
-		if (str->l + l + 1 >= str->m) {
-			str->m = str->l + l + 2;
-			kroundup32(str->m);
-			str->s = (char*)realloc(str->s, str->m);
-		}
-		memcpy(str->s + str->l, buf + fp->block_offset, l);
-		str->l += l;
-		fp->block_offset += l + 1;
-		if (fp->block_offset >= fp->block_length) {
-#ifdef _USE_KNETFILE
-			fp->block_address = knet_tell(fp->x.fpr);
-#else
-			fp->block_address = ftello(fp->file);
-#endif
-			fp->block_offset = 0;
-			fp->block_length = 0;
-		} 
-	} while (state == 0);
-	if (str->l == 0 && state < 0) return state;
-	str->s[str->l] = 0;
-	return str->l;
+	return bgzf_getline(fp, '\n', str);
 }
 
 /*************************************
@@ -157,32 +125,31 @@ static int get_tid(ti_index_t *idx, const char *ss)
 	return tid;
 }
 
-static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
+int ti_get_intv(const ti_conf_t *conf, int len, char *line, ti_interval_t *intv)
 {
-	int i, b = 0, id = 1;
+	int i, b = 0, id = 1, ncols = 0;
 	char *s;
-	intv->tid = intv->beg = intv->end = intv->bin = -1;
-	for (i = 0; i <= str->l; ++i) {
-		if (str->s[i] == '\t' || str->s[i] == 0) {
-			if (id == idx->conf.sc) {
-				str->s[i] = 0;
-				intv->tid = get_tid(idx, str->s + b);
-				if (i != str->l) str->s[i] = '\t';
-			} else if (id == idx->conf.bc) {
+	intv->ss = intv->se = 0; intv->beg = intv->end = -1;
+	for (i = 0; i <= len; ++i) {
+		if (line[i] == '\t' || line[i] == 0) {
+            ++ncols;
+			if (id == conf->sc) {
+				intv->ss = line + b; intv->se = line + i;
+			} else if (id == conf->bc) {
 				// here ->beg is 0-based.
-				intv->beg = intv->end = strtol(str->s + b, &s, 0);
-				if (!(idx->conf.preset&TI_FLAG_UCSC)) --intv->beg;
+				intv->beg = intv->end = strtol(line + b, &s, 0);
+				if (!(conf->preset&TI_FLAG_UCSC)) --intv->beg;
 				else ++intv->end;
 				if (intv->beg < 0) intv->beg = 0;
 				if (intv->end < 1) intv->end = 1;
 			} else {
-				if ((idx->conf.preset&0xffff) == TI_PRESET_GENERIC) {
-					if (id == idx->conf.ec) intv->end = strtol(str->s + b, &s, 0);
-				} else if ((idx->conf.preset&0xffff) == TI_PRESET_SAM) {
+				if ((conf->preset&0xffff) == TI_PRESET_GENERIC) {
+					if (id == conf->ec) intv->end = strtol(line + b, &s, 0);
+				} else if ((conf->preset&0xffff) == TI_PRESET_SAM) {
 					if (id == 6) { // CIGAR
 						int l = 0, op;
 						char *t;
-						for (s = str->s + b; s < str->s + i;) {
+						for (s = line + b; s < line + i;) {
 							long x = strtol(s, &t, 10);
 							op = toupper(*t);
 							if (op == 'M' || op == 'D' || op == 'N') l += x;
@@ -191,19 +158,21 @@ static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
 						if (l == 0) l = 1;
 						intv->end = intv->beg + l;
 					}
-				} else if ((idx->conf.preset&0xffff) == TI_PRESET_VCF) {
+				} else if ((conf->preset&0xffff) == TI_PRESET_VCF) {
 					// FIXME: the following is NOT tested and is likely to be buggy
-					if (id == 5) { // ALT
-						char *t;
-						int max = 1;
-						for (s = str->s + b; s < str->s + i;) {
-							if (s[i] == 'D') {
-								long x = strtol(s + 1, &t, 10);
-								if (x > max) max = x;
-								s = t + 1;
-							} else ++s;
+					if (id == 4) {
+						if (b < i) intv->end = intv->beg + (i - b);
+					} else if (id == 8) { // look for "END="
+						int c = line[i];
+						line[i] = 0;
+						s = strstr(line + b, "END=");
+						if (s == line + b) s += 4;
+						else if (s) {
+							s = strstr(line + b, ";END=");
+							if (s) s += 5;
 						}
-						intv->end = intv->beg + max;
+						if (s) intv->end = strtol(s, &s, 0);
+						line[i] = c;
 					}
 				}
 			}
@@ -211,11 +180,33 @@ static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
 			++id;
 		}
 	}
-	if (intv->tid < 0 || intv->beg < 0 || intv->end < 0) return -1;
-	intv->bin = ti_reg2bin(intv->beg, intv->end);
+/*
+	if (ncols < conf->sc || ncols < conf->bc || ncols < conf->ec) {
+		if (ncols == 1) fprintf(stderr,"[get_intv] Is the file tab-delimited? The line has %d field only: %s\n", ncols, line);
+		else fprintf(stderr,"[get_intv] The line has %d field(s) only: %s\n", ncols, line);
+		exit(1);
+	}
+*/
+	if (intv->ss == 0 || intv->se == 0 || intv->beg < 0 || intv->end < 0) return -1;
 	return 0;
 }
 
+static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
+{
+	ti_interval_t x;
+	intv->tid = intv->beg = intv->end = intv->bin = -1;
+	if (ti_get_intv(&idx->conf, str->l, str->s, &x) == 0) {
+		int c = *x.se;
+		*x.se = '\0'; intv->tid = get_tid(idx, x.ss); *x.se = c;
+		intv->beg = x.beg; intv->end = x.end;
+		intv->bin = ti_reg2bin(intv->beg, intv->end);
+		return (intv->tid >= 0 && intv->beg >= 0 && intv->end >= 0)? 0 : -1;
+	} else {
+		fprintf(stderr, "[%s] the following line cannot be parsed and skipped: %s\n", __func__, str->s);
+		return -1;
+	}
+}
+
 /************
  * indexing *
  ************/
@@ -239,7 +230,7 @@ static inline void insert_offset(khash_t(i) *h, int bin, uint64_t beg, uint64_t
 	l->list[l->n].u = beg; l->list[l->n++].v = end;
 }
 
-static inline void insert_offset2(ti_lidx_t *index2, int _beg, int _end, uint64_t offset)
+static inline uint64_t insert_offset2(ti_lidx_t *index2, int _beg, int _end, uint64_t offset)
 {
 	int i, beg, end;
 	beg = _beg >> TAD_LIDX_SHIFT;
@@ -258,6 +249,7 @@ static inline void insert_offset2(ti_lidx_t *index2, int _beg, int _end, uint64_
 			if (index2->offset[i] == 0) index2->offset[i] = offset;
 	}
 	if (index2->n < end + 1) index2->n = end + 1;
+	return (uint64_t)beg<<32 | end;
 }
 
 static void merge_chunks(ti_index_t *idx)
@@ -298,7 +290,7 @@ ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
 	ti_index_t *idx;
 	uint32_t last_bin, save_bin;
 	int32_t last_coor, last_tid, save_tid;
-	uint64_t save_off, last_off, lineno = 0;
+	uint64_t save_off, last_off, lineno = 0, offset0 = (uint64_t)-1, tmp;
 	kstring_t *str;
 
 	str = calloc(1, sizeof(kstring_t));
@@ -320,14 +312,25 @@ ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
 			continue;
 		}
 		get_intv(idx, str, &intv);
+        if ( intv.beg<0 || intv.end<0 )
+        {
+            fprintf(stderr,"[ti_index_core] the indexes overlap or are out of bounds\n");
+            exit(1);
+        }
 		if (last_tid != intv.tid) { // change of chromosomes
+            if (last_tid>intv.tid )
+            {
+                fprintf(stderr,"[ti_index_core] the chromosome blocks not continuous at line %llu, is the file sorted? [pos %d]\n",(unsigned long long)lineno,intv.beg+1);
+                exit(1);
+            }
 			last_tid = intv.tid;
 			last_bin = 0xffffffffu;
 		} else if (last_coor > intv.beg) {
 			fprintf(stderr, "[ti_index_core] the file out of order at line %llu\n", (unsigned long long)lineno);
 			exit(1);
 		}
-		insert_offset2(&idx->index2[intv.tid], intv.beg, intv.end, last_off);
+		tmp = insert_offset2(&idx->index2[intv.tid], intv.beg, intv.end, last_off);
+		if (last_off == 0) offset0 = tmp;
 		if (intv.bin != last_bin) { // then possibly write the binning index
 			if (save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record
 				insert_offset(idx->index[save_tid], save_bin, save_off, last_off);
@@ -347,6 +350,10 @@ ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
 	if (save_tid >= 0) insert_offset(idx->index[save_tid], save_bin, save_off, bgzf_tell(fp));
 	merge_chunks(idx);
 	fill_missing(idx);
+	if (offset0 != (uint64_t)-1 && idx->n && idx->index2[0].offset) {
+		int i, beg = offset0>>32, end = offset0&0xffffffffu;
+		for (i = beg; i <= end; ++i) idx->index2[0].offset[i] = 0;
+	}
 
 	free(str->s); free(str);
 	return idx;
@@ -644,8 +651,8 @@ ti_index_t *ti_index_load(const char *fn)
     char *fname = get_local_version(fn);
 	if (fname == 0) return 0;
 	idx = ti_index_load_local(fname);
+	if (idx == 0) fprintf(stderr, "[ti_index_load] fail to load the index: %s\n", fname);
     free(fname);
-	if (idx == 0) fprintf(stderr, "[ti_index_load] fail to load BAM index.\n");
 	return idx;
 }
 
@@ -655,7 +662,7 @@ int ti_index_build2(const char *fn, const ti_conf_t *conf, const char *_fnidx)
 	BGZF *fp, *fpidx;
 	ti_index_t *idx;
 	if ((fp = bgzf_open(fn, "r")) == 0) {
-		fprintf(stderr, "[ti_index_build2] fail to open the BAM file.\n");
+		fprintf(stderr, "[ti_index_build2] fail to open the file: %s\n", fn);
 		return -1;
 	}
 	idx = ti_index_core(fp, conf);
@@ -686,12 +693,19 @@ int ti_index_build(const char *fn, const ti_conf_t *conf)
  * parse a region in the format chr:beg-end *
  ********************************************/
 
-int ti_parse_region(ti_index_t *idx, const char *str, int *tid, int *begin, int *end)
+int ti_get_tid(const ti_index_t *idx, const char *name)
+{
+	khiter_t iter;
+	const khash_t(s) *h = idx->tname;
+	iter = kh_get(s, h, name); /* get the tid */
+	if (iter == kh_end(h)) return -1;
+	return kh_value(h, iter);
+}
+
+int ti_parse_region(const ti_index_t *idx, const char *str, int *tid, int *begin, int *end)
 {
 	char *s, *p;
 	int i, l, k;
-	khiter_t iter;
-	khash_t(s) *h = idx->tname;
 	l = strlen(str);
 	p = s = (char*)malloc(l+1);
 	/* squeeze out "," */
@@ -700,12 +714,10 @@ int ti_parse_region(ti_index_t *idx, const char *str, int *tid, int *begin, int
 	s[k] = 0;
 	for (i = 0; i != k; ++i) if (s[i] == ':') break;
 	s[i] = 0;
-	iter = kh_get(s, h, s); /* get the tid */
-	if (iter == kh_end(h)) { // name not found
-		*tid = -1; free(s);
+	if ((*tid = ti_get_tid(idx, s)) < 0) {
+		free(s);
 		return -1;
 	}
-	*tid = kh_value(h, iter);
 	if (i == k) { /* dump the whole sequence */
 		*begin = 0; *end = 1<<29; free(s);
 		return 0;
@@ -731,6 +743,8 @@ int ti_parse_region(ti_index_t *idx, const char *str, int *tid, int *begin, int
 static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[MAX_BIN])
 {
 	int i = 0, k;
+	if (beg >= end) return 0;
+	if (end >= 1u<<29) end = 1u<<29;
 	--end;
 	list[i++] = 0;
 	for (k =    1 + (beg>>26); k <=    1 + (end>>26); ++k) list[i++] = k;
@@ -741,15 +755,15 @@ static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[MAX_BIN])
 	return i;
 }
 
-ti_iter_t ti_first(BGZF *fp)
+ti_iter_t ti_iter_first()
 {
 	ti_iter_t iter;
 	iter = calloc(1, sizeof(struct __ti_iter_t));
-	iter->fp = fp; iter->from_first = 1;
+	iter->from_first = 1;
 	return iter;
 }
 
-ti_iter_t ti_query(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end)
+ti_iter_t ti_iter_query(const ti_index_t *idx, int tid, int beg, int end)
 {
 	uint16_t *bins;
 	int i, n_bins, n_off;
@@ -759,10 +773,11 @@ ti_iter_t ti_query(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end)
 	uint64_t min_off;
 	ti_iter_t iter = 0;
 
+	if (beg < 0) beg = 0;
+	if (end < beg) return 0;
 	// initialize the iterator
 	iter = calloc(1, sizeof(struct __ti_iter_t));
-	iter->fp = fp; iter->idx = idx;
-	iter->tid = tid; iter->beg = beg; iter->end = end; iter->i = -1;
+	iter->idx = idx; iter->tid = tid; iter->beg = beg; iter->end = end; iter->i = -1;
 	// random access
 	bins = (uint16_t*)calloc(MAX_BIN, 2);
 	n_bins = reg2bins(beg, end, bins);
@@ -794,10 +809,13 @@ ti_iter_t ti_query(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end)
 				if (p->list[j].v > min_off) off[n_off++] = p->list[j];
 		}
 	}
+	if (n_off == 0) {
+		free(bins); free(off); return iter;
+	}
 	free(bins);
 	{
 		int l;
-		ks_introsort(off, n_off, off);
+		ks_introsort(offt, n_off, off);
 		// resolve completely contained adjacent blocks
 		for (i = 1, l = 0; i < n_off; ++i)
 			if (off[l].v < off[i].v)
@@ -818,12 +836,12 @@ ti_iter_t ti_query(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end)
 	return iter;
 }
 
-const char *ti_iter_read(ti_iter_t iter, int *len)
+const char *ti_iter_read(BGZF *fp, ti_iter_t iter, int *len)
 {
 	if (iter->finished) return 0;
 	if (iter->from_first) {
 		int ret;
-		if ((ret = ti_readline(iter->fp, &iter->str)) < 0) {
+		if ((ret = ti_readline(fp, &iter->str)) < 0) {
 			iter->finished = 1;
 			return 0;
 		} else {
@@ -838,14 +856,14 @@ const char *ti_iter_read(ti_iter_t iter, int *len)
 			if (iter->i == iter->n_off - 1) break; // no more chunks
 			if (iter->i >= 0) assert(iter->curr_off == iter->off[iter->i].v); // otherwise bug
 			if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek
-				bgzf_seek(iter->fp, iter->off[iter->i+1].u, SEEK_SET);
-				iter->curr_off = bgzf_tell(iter->fp);
+				bgzf_seek(fp, iter->off[iter->i+1].u, SEEK_SET);
+				iter->curr_off = bgzf_tell(fp);
 			}
 			++iter->i;
 		}
-		if ((ret = ti_readline(iter->fp, &iter->str)) >= 0) {
+		if ((ret = ti_readline(fp, &iter->str)) >= 0) {
 			ti_intv_t intv;
-			iter->curr_off = bgzf_tell(iter->fp);
+			iter->curr_off = bgzf_tell(fp);
 			if (iter->str.s[0] == iter->idx->conf.meta_char) continue;
 			get_intv((ti_index_t*)iter->idx, &iter->str, &intv);
 			if (intv.tid != iter->tid || intv.beg >= iter->end) break; // no need to proceed
@@ -872,9 +890,78 @@ int ti_fetch(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end, void *d
 	ti_iter_t iter;
 	const char *s;
 	int len;
-	iter = ti_query(fp, idx, tid, beg, end);
-	while ((s = ti_iter_read(iter, &len)) != 0)
+	iter = ti_iter_query(idx, tid, beg, end);
+	while ((s = ti_iter_read(fp, iter, &len)) != 0)
 		func(len, s, data);
 	ti_iter_destroy(iter);
 	return 0;
 }
+
+const ti_conf_t *ti_get_conf(ti_index_t *idx) { return idx? &idx->conf : 0; }
+
+/*******************
+ * High-level APIs *
+ *******************/
+
+tabix_t *ti_open(const char *fn, const char *fnidx)
+{
+	tabix_t *t;
+	BGZF *fp;
+	if ((fp = bgzf_open(fn, "r")) == 0) return 0;
+	t = calloc(1, sizeof(tabix_t));
+	t->fn = strdup(fn);
+	if (fnidx) t->fnidx = strdup(fnidx);
+	t->fp = fp;
+	return t;
+}
+
+void ti_close(tabix_t *t)
+{
+	if (t) {
+		bgzf_close(t->fp);
+		if (t->idx) ti_index_destroy(t->idx);
+		free(t->fn); free(t->fnidx);
+		free(t);
+	}
+}
+
+int ti_lazy_index_load(tabix_t *t)
+{
+	if (t->idx == 0) { // load index
+		if (t->fnidx) t->idx = ti_index_load_local(t->fnidx);
+		else t->idx = ti_index_load(t->fn);
+		if (t->idx == 0) return -1; // fail to load index
+	}
+	return 0;
+}
+
+ti_iter_t ti_queryi(tabix_t *t, int tid, int beg, int end)
+{
+	if (tid < 0) return ti_iter_first();
+	if (ti_lazy_index_load(t) != 0) return 0;
+	return ti_iter_query(t->idx, tid, beg, end);	
+}
+
+ti_iter_t ti_querys(tabix_t *t, const char *reg)
+{
+	int tid, beg, end;
+	if (reg == 0) return ti_iter_first();
+	if (ti_lazy_index_load(t) != 0) return 0;
+	if (ti_parse_region(t->idx, reg, &tid, &beg, &end) < 0) return 0;
+	return ti_iter_query(t->idx, tid, beg, end);
+}
+
+ti_iter_t ti_query(tabix_t *t, const char *name, int beg, int end)
+{
+	int tid;
+	if (name == 0) return ti_iter_first();
+	// then need to load the index
+	if (ti_lazy_index_load(t) != 0) return 0;
+	if ((tid = ti_get_tid(t->idx, name)) < 0) return 0;
+	return ti_iter_query(t->idx, tid, beg, end);
+}
+
+const char *ti_read(tabix_t *t, ti_iter_t iter, int *len)
+{
+	return ti_iter_read(t->fp, iter, len);
+}