Imported Upstream version 0.1.18
[samtools.git] / razf.c
1 /*
2  * RAZF : Random Access compressed(Z) File
3  * Version: 1.0
4  * Release Date: 2008-10-27
5  *
6  * Copyright 2008, Jue Ruan <ruanjue@gmail.com>, Heng Li <lh3@sanger.ac.uk>
7  *
8  * All rights reserved.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  */
31
32 #ifndef _NO_RAZF
33
34 #include <fcntl.h>
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <unistd.h>
39 #include "razf.h"
40
41
42 #if ZLIB_VERNUM < 0x1221
43 struct _gz_header_s {
44     int     text;
45     uLong   time;
46     int     xflags;
47     int     os;
48     Bytef   *extra;
49     uInt    extra_len;
50     uInt    extra_max;
51     Bytef   *name;
52     uInt    name_max;
53     Bytef   *comment;
54     uInt    comm_max;
55     int     hcrc;
56     int     done;
57 };
58 #warning "zlib < 1.2.2.1; RAZF writing is disabled."
59 #endif
60
61 #define DEF_MEM_LEVEL 8
62
63 static inline uint32_t byte_swap_4(uint32_t v){
64         v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
65         return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
66 }
67
68 static inline uint64_t byte_swap_8(uint64_t v){
69         v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
70         v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
71         return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
72 }
73
74 static inline int is_big_endian(){
75         int x = 0x01;
76         char *c = (char*)&x;
77         return (c[0] != 0x01);
78 }
79
80 #ifndef _RZ_READONLY
81 static void add_zindex(RAZF *rz, int64_t in, int64_t out){
82         if(rz->index->size == rz->index->cap){
83                 rz->index->cap = rz->index->cap * 1.5 + 2;
84                 rz->index->cell_offsets = realloc(rz->index->cell_offsets, sizeof(int) * rz->index->cap);
85                 rz->index->bin_offsets  = realloc(rz->index->bin_offsets, sizeof(int64_t) * (rz->index->cap/RZ_BIN_SIZE + 1));
86         }
87         if(rz->index->size % RZ_BIN_SIZE == 0) rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE] = out;
88         rz->index->cell_offsets[rz->index->size] = out - rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE];
89         rz->index->size ++;
90 }
91
92 static void save_zindex(RAZF *rz, int fd){
93         int32_t i, v32;
94         int is_be;
95         is_be = is_big_endian();
96         if(is_be) write(fd, &rz->index->size, sizeof(int));
97         else {
98                 v32 = byte_swap_4((uint32_t)rz->index->size);
99                 write(fd, &v32, sizeof(uint32_t));
100         }
101         v32 = rz->index->size / RZ_BIN_SIZE + 1;
102         if(!is_be){
103                 for(i=0;i<v32;i++) rz->index->bin_offsets[i]  = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
104                 for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
105         }
106         write(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
107         write(fd, rz->index->cell_offsets, sizeof(int32_t) * rz->index->size);
108 }
109 #endif
110
111 #ifdef _USE_KNETFILE
112 static void load_zindex(RAZF *rz, knetFile *fp){
113 #else
114 static void load_zindex(RAZF *rz, int fd){
115 #endif
116         int32_t i, v32;
117         int is_be;
118         if(!rz->load_index) return;
119         if(rz->index == NULL) rz->index = malloc(sizeof(ZBlockIndex));
120         is_be = is_big_endian();
121 #ifdef _USE_KNETFILE
122         knet_read(fp, &rz->index->size, sizeof(int));
123 #else
124         read(fd, &rz->index->size, sizeof(int));
125 #endif
126         if(!is_be) rz->index->size = byte_swap_4((uint32_t)rz->index->size);
127         rz->index->cap = rz->index->size;
128         v32 = rz->index->size / RZ_BIN_SIZE + 1;
129         rz->index->bin_offsets  = malloc(sizeof(int64_t) * v32);
130 #ifdef _USE_KNETFILE
131         knet_read(fp, rz->index->bin_offsets, sizeof(int64_t) * v32);
132 #else
133         read(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
134 #endif
135         rz->index->cell_offsets = malloc(sizeof(int) * rz->index->size);
136 #ifdef _USE_KNETFILE
137         knet_read(fp, rz->index->cell_offsets, sizeof(int) * rz->index->size);
138 #else
139         read(fd, rz->index->cell_offsets, sizeof(int) * rz->index->size);
140 #endif
141         if(!is_be){
142                 for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
143                 for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
144         }
145 }
146
147 #ifdef _RZ_READONLY
148 static RAZF* razf_open_w(int fd)
149 {
150         fprintf(stderr, "[razf_open_w] Writing is not available with zlib ver < 1.2.2.1\n");
151         return 0;
152 }
153 #else
154 static RAZF* razf_open_w(int fd){
155         RAZF *rz;
156 #ifdef _WIN32
157         setmode(fd, O_BINARY);
158 #endif
159         rz = calloc(1, sizeof(RAZF));
160         rz->mode = 'w';
161 #ifdef _USE_KNETFILE
162     rz->x.fpw = fd;
163 #else
164         rz->filedes = fd;
165 #endif
166         rz->stream = calloc(sizeof(z_stream), 1);
167         rz->inbuf  = malloc(RZ_BUFFER_SIZE);
168         rz->outbuf = malloc(RZ_BUFFER_SIZE);
169         rz->index = calloc(sizeof(ZBlockIndex), 1);
170         deflateInit2(rz->stream, RZ_COMPRESS_LEVEL, Z_DEFLATED, WINDOW_BITS + 16, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
171         rz->stream->avail_out = RZ_BUFFER_SIZE;
172         rz->stream->next_out  = rz->outbuf;
173         rz->header = calloc(sizeof(gz_header), 1);
174         rz->header->os    = 0x03; //Unix
175         rz->header->text  = 0;
176         rz->header->time  = 0;
177         rz->header->extra = malloc(7);
178         strncpy((char*)rz->header->extra, "RAZF", 4);
179         rz->header->extra[4] = 1; // obsolete field
180         // block size = RZ_BLOCK_SIZE, Big-Endian
181         rz->header->extra[5] = RZ_BLOCK_SIZE >> 8;
182         rz->header->extra[6] = RZ_BLOCK_SIZE & 0xFF;
183         rz->header->extra_len = 7;
184         rz->header->name = rz->header->comment  = 0;
185         rz->header->hcrc = 0;
186         deflateSetHeader(rz->stream, rz->header);
187         rz->block_pos = rz->block_off = 0;
188         return rz;
189 }
190
191 static void _razf_write(RAZF* rz, const void *data, int size){
192         int tout;
193         rz->stream->avail_in = size;
194         rz->stream->next_in  = (void*)data;
195         while(1){
196                 tout = rz->stream->avail_out;
197                 deflate(rz->stream, Z_NO_FLUSH);
198                 rz->out += tout - rz->stream->avail_out;
199                 if(rz->stream->avail_out) break;
200 #ifdef _USE_KNETFILE
201                 write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
202 #else
203                 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
204 #endif
205                 rz->stream->avail_out = RZ_BUFFER_SIZE;
206                 rz->stream->next_out  = rz->outbuf;
207                 if(rz->stream->avail_in == 0) break;
208         };
209         rz->in += size - rz->stream->avail_in;
210         rz->block_off += size - rz->stream->avail_in;
211 }
212
213 static void razf_flush(RAZF *rz){
214         uint32_t tout;
215         if(rz->buf_len){
216                 _razf_write(rz, rz->inbuf, rz->buf_len);
217                 rz->buf_off = rz->buf_len = 0;
218         }
219         if(rz->stream->avail_out){
220 #ifdef _USE_KNETFILE    
221                 write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
222 #else        
223                 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
224 #endif
225                 rz->stream->avail_out = RZ_BUFFER_SIZE;
226                 rz->stream->next_out  = rz->outbuf;
227         }
228         while(1){
229                 tout = rz->stream->avail_out;
230                 deflate(rz->stream, Z_FULL_FLUSH);
231                 rz->out += tout - rz->stream->avail_out;
232                 if(rz->stream->avail_out == 0){
233 #ifdef _USE_KNETFILE    
234                         write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
235 #else            
236                         write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
237 #endif
238                         rz->stream->avail_out = RZ_BUFFER_SIZE;
239                         rz->stream->next_out  = rz->outbuf;
240                 } else break;
241         }
242         rz->block_pos = rz->out;
243         rz->block_off = 0;
244 }
245
246 static void razf_end_flush(RAZF *rz){
247         uint32_t tout;
248         if(rz->buf_len){
249                 _razf_write(rz, rz->inbuf, rz->buf_len);
250                 rz->buf_off = rz->buf_len = 0;
251         }
252         while(1){
253                 tout = rz->stream->avail_out;
254                 deflate(rz->stream, Z_FINISH);
255                 rz->out += tout - rz->stream->avail_out;
256                 if(rz->stream->avail_out < RZ_BUFFER_SIZE){
257 #ifdef _USE_KNETFILE        
258                         write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
259 #else            
260                         write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
261 #endif
262                         rz->stream->avail_out = RZ_BUFFER_SIZE;
263                         rz->stream->next_out  = rz->outbuf;
264                 } else break;
265         }
266 }
267
268 static void _razf_buffered_write(RAZF *rz, const void *data, int size){
269         int i, n;
270         while(1){
271                 if(rz->buf_len == RZ_BUFFER_SIZE){
272                         _razf_write(rz, rz->inbuf, rz->buf_len);
273                         rz->buf_len = 0;
274                 }
275                 if(size + rz->buf_len < RZ_BUFFER_SIZE){
276                         for(i=0;i<size;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
277                         rz->buf_len += size;
278                         return;
279                 } else {
280                         n = RZ_BUFFER_SIZE - rz->buf_len;
281                         for(i=0;i<n;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
282                         size -= n;
283                         data += n;
284                         rz->buf_len += n;
285                 }
286         }
287 }
288
289 int razf_write(RAZF* rz, const void *data, int size){
290         int ori_size, n;
291         int64_t next_block;
292         ori_size = size;
293         next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
294         while(rz->in + rz->buf_len + size >= next_block){
295                 n = next_block - rz->in - rz->buf_len;
296                 _razf_buffered_write(rz, data, n);
297                 data += n;
298                 size -= n;
299                 razf_flush(rz);
300                 add_zindex(rz, rz->in, rz->out);
301                 next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
302         }
303         _razf_buffered_write(rz, data, size);
304         return ori_size;
305 }
306 #endif
307
308 /* gzip flag byte */
309 #define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
310 #define HEAD_CRC     0x02 /* bit 1 set: header CRC present */
311 #define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
312 #define ORIG_NAME    0x08 /* bit 3 set: original file name present */
313 #define COMMENT      0x10 /* bit 4 set: file comment present */
314 #define RESERVED     0xE0 /* bits 5..7: reserved */
315
316 static int _read_gz_header(unsigned char *data, int size, int *extra_off, int *extra_len){
317         int method, flags, n, len;
318         if(size < 2) return 0;
319         if(data[0] != 0x1f || data[1] != 0x8b) return 0;
320         if(size < 4) return 0;
321         method = data[2];
322         flags  = data[3];
323         if(method != Z_DEFLATED || (flags & RESERVED)) return 0;
324         n = 4 + 6; // Skip 6 bytes
325         *extra_off = n + 2;
326         *extra_len = 0;
327         if(flags & EXTRA_FIELD){
328                 if(size < n + 2) return 0;
329                 len = ((int)data[n + 1] << 8) | data[n];
330                 n += 2;
331                 *extra_off = n;
332                 while(len){
333                         if(n >= size) return 0;
334                         n ++;
335                         len --;
336                 }
337                 *extra_len = n - (*extra_off);
338         }
339         if(flags & ORIG_NAME) while(n < size && data[n++]);
340         if(flags & COMMENT) while(n < size && data[n++]);
341         if(flags & HEAD_CRC){
342                 if(n + 2 > size) return 0;
343                 n += 2;
344         }
345         return n;
346 }
347
348 #ifdef _USE_KNETFILE
349 static RAZF* razf_open_r(knetFile *fp, int _load_index){
350 #else
351 static RAZF* razf_open_r(int fd, int _load_index){
352 #endif
353         RAZF *rz;
354         int ext_off, ext_len;
355         int n, is_be, ret;
356         int64_t end;
357         unsigned char c[] = "RAZF";
358         rz = calloc(1, sizeof(RAZF));
359         rz->mode = 'r';
360 #ifdef _USE_KNETFILE
361     rz->x.fpr = fp;
362 #else
363 #ifdef _WIN32
364         setmode(fd, O_BINARY);
365 #endif
366         rz->filedes = fd;
367 #endif
368         rz->stream = calloc(sizeof(z_stream), 1);
369         rz->inbuf  = malloc(RZ_BUFFER_SIZE);
370         rz->outbuf = malloc(RZ_BUFFER_SIZE);
371         rz->end = rz->src_end = 0x7FFFFFFFFFFFFFFFLL;
372 #ifdef _USE_KNETFILE
373     n = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE);
374 #else
375         n = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
376 #endif
377         ret = _read_gz_header(rz->inbuf, n, &ext_off, &ext_len);
378         if(ret == 0){
379                 PLAIN_FILE:
380                 rz->in = n;
381                 rz->file_type = FILE_TYPE_PLAIN;
382                 memcpy(rz->outbuf, rz->inbuf, n);
383                 rz->buf_len = n;
384                 free(rz->stream);
385                 rz->stream = NULL;
386                 return rz;
387         }
388         rz->header_size = ret;
389         ret = inflateInit2(rz->stream, -WINDOW_BITS);
390         if(ret != Z_OK){ inflateEnd(rz->stream); goto PLAIN_FILE;}
391         rz->stream->avail_in = n - rz->header_size;
392         rz->stream->next_in  = rz->inbuf + rz->header_size;
393         rz->stream->avail_out = RZ_BUFFER_SIZE;
394         rz->stream->next_out  = rz->outbuf;
395         rz->file_type = FILE_TYPE_GZ;
396         rz->in = rz->header_size;
397         rz->block_pos = rz->header_size;
398         rz->next_block_pos = rz->header_size;
399         rz->block_off = 0;
400         if(ext_len < 7 || memcmp(rz->inbuf + ext_off, c, 4) != 0) return rz;
401         if(((((unsigned char*)rz->inbuf)[ext_off + 5] << 8) | ((unsigned char*)rz->inbuf)[ext_off + 6]) != RZ_BLOCK_SIZE){
402                 fprintf(stderr, " -- WARNING: RZ_BLOCK_SIZE is not %d, treat source as gz file.  in %s -- %s:%d --\n", RZ_BLOCK_SIZE, __FUNCTION__, __FILE__, __LINE__);
403                 return rz;
404         }
405         rz->load_index = _load_index;
406         rz->file_type = FILE_TYPE_RZ;
407 #ifdef _USE_KNETFILE
408         if(knet_seek(fp, -16, SEEK_END) == -1){
409 #else
410         if(lseek(fd, -16, SEEK_END) == -1){
411 #endif
412                 UNSEEKABLE:
413                 rz->seekable = 0;
414                 rz->index = NULL;
415                 rz->src_end = rz->end = 0x7FFFFFFFFFFFFFFFLL;
416         } else {
417                 is_be = is_big_endian();
418                 rz->seekable = 1;
419 #ifdef _USE_KNETFILE
420         knet_read(fp, &end, sizeof(int64_t));
421 #else
422                 read(fd, &end, sizeof(int64_t));
423 #endif        
424                 if(!is_be) rz->src_end = (int64_t)byte_swap_8((uint64_t)end);
425                 else rz->src_end = end;
426
427 #ifdef _USE_KNETFILE
428                 knet_read(fp, &end, sizeof(int64_t));
429 #else
430                 read(fd, &end, sizeof(int64_t));
431 #endif        
432                 if(!is_be) rz->end = (int64_t)byte_swap_8((uint64_t)end);
433                 else rz->end = end;
434                 if(n > rz->end){
435                         rz->stream->avail_in -= n - rz->end;
436                         n = rz->end;
437                 }
438                 if(rz->end > rz->src_end){
439 #ifdef _USE_KNETFILE
440             knet_seek(fp, rz->in, SEEK_SET);
441 #else
442                         lseek(fd, rz->in, SEEK_SET);
443 #endif
444                         goto UNSEEKABLE;
445                 }
446 #ifdef _USE_KNETFILE
447         knet_seek(fp, rz->end, SEEK_SET);
448                 if(knet_tell(fp) != rz->end){
449                         knet_seek(fp, rz->in, SEEK_SET);
450 #else
451                 if(lseek(fd, rz->end, SEEK_SET) != rz->end){
452                         lseek(fd, rz->in, SEEK_SET);
453 #endif
454                         goto UNSEEKABLE;
455                 }
456 #ifdef _USE_KNETFILE
457                 load_zindex(rz, fp);
458                 knet_seek(fp, n, SEEK_SET);
459 #else
460                 load_zindex(rz, fd);
461                 lseek(fd, n, SEEK_SET);
462 #endif
463         }
464         return rz;
465 }
466
467 #ifdef _USE_KNETFILE
468 RAZF* razf_dopen(int fd, const char *mode){
469     if (strstr(mode, "r")) fprintf(stderr,"[razf_dopen] implement me\n");
470     else if(strstr(mode, "w")) return razf_open_w(fd);
471         return NULL;
472 }
473
474 RAZF* razf_dopen2(int fd, const char *mode)
475 {
476     fprintf(stderr,"[razf_dopen2] implement me\n");
477     return NULL;
478 }
479 #else
480 RAZF* razf_dopen(int fd, const char *mode){
481         if(strstr(mode, "r")) return razf_open_r(fd, 1);
482         else if(strstr(mode, "w")) return razf_open_w(fd);
483         else return NULL;
484 }
485
486 RAZF* razf_dopen2(int fd, const char *mode)
487 {
488         if(strstr(mode, "r")) return razf_open_r(fd, 0);
489         else if(strstr(mode, "w")) return razf_open_w(fd);
490         else return NULL;
491 }
492 #endif
493
494 static inline RAZF* _razf_open(const char *filename, const char *mode, int _load_index){
495         int fd;
496         RAZF *rz;
497         if(strstr(mode, "r")){
498 #ifdef _USE_KNETFILE
499         knetFile *fd = knet_open(filename, "r");
500         if (fd == 0) {
501             fprintf(stderr, "[_razf_open] fail to open %s\n", filename);
502             return NULL;
503         }
504 #else
505 #ifdef _WIN32
506                 fd = open(filename, O_RDONLY | O_BINARY);
507 #else
508                 fd = open(filename, O_RDONLY);
509 #endif
510 #endif
511                 if(fd < 0) return NULL;
512                 rz = razf_open_r(fd, _load_index);
513         } else if(strstr(mode, "w")){
514 #ifdef _WIN32
515                 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
516 #else
517                 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666);
518 #endif
519                 if(fd < 0) return NULL;
520                 rz = razf_open_w(fd);
521         } else return NULL;
522         return rz;
523 }
524
525 RAZF* razf_open(const char *filename, const char *mode){
526         return _razf_open(filename, mode, 1);
527 }
528
529 RAZF* razf_open2(const char *filename, const char *mode){
530         return _razf_open(filename, mode, 0);
531 }
532
533 int razf_get_data_size(RAZF *rz, int64_t *u_size, int64_t *c_size){
534         int64_t n;
535         if(rz->mode != 'r' && rz->mode != 'R') return 0;
536         switch(rz->file_type){
537                 case FILE_TYPE_PLAIN:
538                         if(rz->end == 0x7fffffffffffffffLL){
539 #ifdef _USE_KNETFILE
540                                 if(knet_seek(rz->x.fpr, 0, SEEK_CUR) == -1) return 0;
541                 n = knet_tell(rz->x.fpr);
542                                 knet_seek(rz->x.fpr, 0, SEEK_END);
543                 rz->end = knet_tell(rz->x.fpr);
544                                 knet_seek(rz->x.fpr, n, SEEK_SET);
545 #else
546                                 if((n = lseek(rz->filedes, 0, SEEK_CUR)) == -1) return 0;
547                                 rz->end = lseek(rz->filedes, 0, SEEK_END);
548                                 lseek(rz->filedes, n, SEEK_SET);
549 #endif                
550                         }
551                         *u_size = *c_size = rz->end;
552                         return 1;
553                 case FILE_TYPE_GZ:
554                         return 0;
555                 case FILE_TYPE_RZ:
556                         if(rz->src_end == rz->end) return 0;
557                         *u_size = rz->src_end;
558                         *c_size = rz->end;
559                         return 1;
560                 default:
561                         return 0;
562         }
563 }
564
565 static int _razf_read(RAZF* rz, void *data, int size){
566         int ret, tin;
567         if(rz->z_eof || rz->z_err) return 0;
568         if (rz->file_type == FILE_TYPE_PLAIN) {
569 #ifdef _USE_KNETFILE
570                 ret = knet_read(rz->x.fpr, data, size);
571 #else
572                 ret = read(rz->filedes, data, size);
573 #endif        
574                 if (ret == 0) rz->z_eof = 1;
575                 return ret;
576         }
577         rz->stream->avail_out = size;
578         rz->stream->next_out  = data;
579         while(rz->stream->avail_out){
580                 if(rz->stream->avail_in == 0){
581                         if(rz->in >= rz->end){ rz->z_eof = 1; break; }
582                         if(rz->end - rz->in < RZ_BUFFER_SIZE){
583 #ifdef _USE_KNETFILE
584                                 rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, rz->end -rz->in);
585 #else
586                                 rz->stream->avail_in = read(rz->filedes, rz->inbuf, rz->end -rz->in);
587 #endif        
588                         } else {
589 #ifdef _USE_KNETFILE
590                                 rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE);
591 #else
592                                 rz->stream->avail_in = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
593 #endif        
594                         }
595                         if(rz->stream->avail_in == 0){
596                                 rz->z_eof = 1;
597                                 break;
598                         }
599                         rz->stream->next_in = rz->inbuf;
600                 }
601                 tin = rz->stream->avail_in;
602                 ret = inflate(rz->stream, Z_BLOCK);
603                 rz->in += tin - rz->stream->avail_in;
604                 if(ret == Z_NEED_DICT || ret == Z_MEM_ERROR || ret == Z_DATA_ERROR){
605                         fprintf(stderr, "[_razf_read] inflate error: %d %s (at %s:%d)\n", ret, rz->stream->msg ? rz->stream->msg : "", __FILE__, __LINE__);
606                         rz->z_err = 1;
607                         break;
608                 }
609                 if(ret == Z_STREAM_END){
610                         rz->z_eof = 1;
611                         break;
612                 }
613                 if ((rz->stream->data_type&128) && !(rz->stream->data_type&64)){
614                         rz->buf_flush = 1;
615                         rz->next_block_pos = rz->in;
616                         break;
617                 }
618         }
619         return size - rz->stream->avail_out;
620 }
621
622 int razf_read(RAZF *rz, void *data, int size){
623         int ori_size, i;
624         ori_size = size;
625         while(size > 0){
626                 if(rz->buf_len){
627                         if(size < rz->buf_len){
628                                 for(i=0;i<size;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
629                                 rz->buf_off += size;
630                                 rz->buf_len -= size;
631                                 data += size;
632                                 rz->block_off += size;
633                                 size = 0;
634                                 break;
635                         } else {
636                                 for(i=0;i<rz->buf_len;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
637                                 data += rz->buf_len;
638                                 size -= rz->buf_len;
639                                 rz->block_off += rz->buf_len;
640                                 rz->buf_off = 0;
641                                 rz->buf_len = 0;
642                                 if(rz->buf_flush){
643                                         rz->block_pos = rz->next_block_pos;
644                                         rz->block_off = 0;
645                                         rz->buf_flush = 0;
646                                 }
647                         }
648                 } else if(rz->buf_flush){
649                         rz->block_pos = rz->next_block_pos;
650                         rz->block_off = 0;
651                         rz->buf_flush = 0;
652                 }
653                 if(rz->buf_flush) continue;
654                 rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
655                 if(rz->z_eof && rz->buf_len == 0) break;
656         }
657         rz->out += ori_size - size;
658         return ori_size - size;
659 }
660
661 int razf_skip(RAZF* rz, int size){
662         int ori_size;
663         ori_size = size;
664         while(size > 0){
665                 if(rz->buf_len){
666                         if(size < rz->buf_len){
667                                 rz->buf_off += size;
668                                 rz->buf_len -= size;
669                                 rz->block_off += size;
670                                 size = 0;
671                                 break;
672                         } else {
673                                 size -= rz->buf_len;
674                                 rz->buf_off = 0;
675                                 rz->buf_len = 0;
676                                 rz->block_off += rz->buf_len;
677                                 if(rz->buf_flush){
678                                         rz->block_pos = rz->next_block_pos;
679                                         rz->block_off = 0;
680                                         rz->buf_flush = 0;
681                                 }
682                         }
683                 } else if(rz->buf_flush){
684                         rz->block_pos = rz->next_block_pos;
685                         rz->block_off = 0;
686                         rz->buf_flush = 0;
687                 }
688                 if(rz->buf_flush) continue;
689                 rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
690                 if(rz->z_eof || rz->z_err) break;
691         }
692         rz->out += ori_size - size;
693         return ori_size - size;
694 }
695
696 static void _razf_reset_read(RAZF *rz, int64_t in, int64_t out){
697 #ifdef _USE_KNETFILE
698         knet_seek(rz->x.fpr, in, SEEK_SET);
699 #else
700         lseek(rz->filedes, in, SEEK_SET);
701 #endif
702         rz->in  = in;
703         rz->out = out;
704         rz->block_pos = in;
705         rz->next_block_pos = in;
706         rz->block_off = 0;
707         rz->buf_flush = 0;
708         rz->z_eof = rz->z_err = 0;
709         inflateReset(rz->stream);
710         rz->stream->avail_in = 0;
711         rz->buf_off = rz->buf_len = 0;
712 }
713
714 int64_t razf_jump(RAZF *rz, int64_t block_start, int block_offset){
715         int64_t pos;
716         rz->z_eof = 0;
717         if(rz->file_type == FILE_TYPE_PLAIN){
718                 rz->buf_off = rz->buf_len = 0;
719                 pos = block_start + block_offset;
720 #ifdef _USE_KNETFILE
721                 knet_seek(rz->x.fpr, pos, SEEK_SET);
722         pos = knet_tell(rz->x.fpr);
723 #else
724                 pos = lseek(rz->filedes, pos, SEEK_SET);
725 #endif
726                 rz->out = rz->in = pos;
727                 return pos;
728         }
729         if(block_start == rz->block_pos && block_offset >= rz->block_off) {
730                 block_offset -= rz->block_off;
731                 goto SKIP; // Needn't reset inflate
732         }
733         if(block_start  == 0) block_start = rz->header_size; // Automaticly revist wrong block_start
734         _razf_reset_read(rz, block_start, 0);
735         SKIP:
736         if(block_offset) razf_skip(rz, block_offset);
737         return rz->block_off;
738 }
739
740 int64_t razf_seek(RAZF* rz, int64_t pos, int where){
741         int64_t idx;
742         int64_t seek_pos, new_out;
743         rz->z_eof = 0;
744         if (where == SEEK_CUR) pos += rz->out;
745         else if (where == SEEK_END) pos += rz->src_end;
746         if(rz->file_type == FILE_TYPE_PLAIN){
747 #ifdef _USE_KNETFILE
748                 knet_seek(rz->x.fpr, pos, SEEK_SET);
749         seek_pos = knet_tell(rz->x.fpr);
750 #else
751                 seek_pos = lseek(rz->filedes, pos, SEEK_SET);
752 #endif
753                 rz->buf_off = rz->buf_len = 0;
754                 rz->out = rz->in = seek_pos;
755                 return seek_pos;
756         } else if(rz->file_type == FILE_TYPE_GZ){
757                 if(pos >= rz->out) goto SKIP;
758                 return rz->out;
759         }
760         if(pos == rz->out) return pos;
761         if(pos > rz->src_end) return rz->out;
762         if(!rz->seekable || !rz->load_index){
763                 if(pos >= rz->out) goto SKIP;
764         }
765         idx = pos / RZ_BLOCK_SIZE - 1;
766         seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
767         new_out  = (idx + 1) * RZ_BLOCK_SIZE;
768         if(pos > rz->out && new_out <= rz->out) goto SKIP;
769         _razf_reset_read(rz, seek_pos, new_out);
770         SKIP:
771         razf_skip(rz, (int)(pos - rz->out));
772         return rz->out;
773 }
774
775 uint64_t razf_tell2(RAZF *rz)
776 {
777         /*
778         if (rz->load_index) {
779                 int64_t idx, seek_pos;
780                 idx = rz->out / RZ_BLOCK_SIZE - 1;
781                 seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
782                 if (seek_pos != rz->block_pos || rz->out%RZ_BLOCK_SIZE != rz->block_off)
783                         fprintf(stderr, "[razf_tell2] inconsistent block offset: (%lld, %lld) != (%lld, %lld)\n",
784                                         (long long)seek_pos, (long long)rz->out%RZ_BLOCK_SIZE, (long long)rz->block_pos, (long long) rz->block_off);
785         }
786         */
787         return (uint64_t)rz->block_pos<<16 | (rz->block_off&0xffff);
788 }
789
790 int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where)
791 {
792         if (where != SEEK_SET) return -1;
793         return razf_jump(rz, voffset>>16, voffset&0xffff);
794 }
795
796 void razf_close(RAZF *rz){
797         if(rz->mode == 'w'){
798 #ifndef _RZ_READONLY
799                 razf_end_flush(rz);
800                 deflateEnd(rz->stream);
801 #ifdef _USE_KNETFILE
802                 save_zindex(rz, rz->x.fpw);
803                 if(is_big_endian()){
804                         write(rz->x.fpw, &rz->in, sizeof(int64_t));
805                         write(rz->x.fpw, &rz->out, sizeof(int64_t));
806                 } else {
807                         uint64_t v64 = byte_swap_8((uint64_t)rz->in);
808                         write(rz->x.fpw, &v64, sizeof(int64_t));
809                         v64 = byte_swap_8((uint64_t)rz->out);
810                         write(rz->x.fpw, &v64, sizeof(int64_t));
811                 }
812 #else
813                 save_zindex(rz, rz->filedes);
814                 if(is_big_endian()){
815                         write(rz->filedes, &rz->in, sizeof(int64_t));
816                         write(rz->filedes, &rz->out, sizeof(int64_t));
817                 } else {
818                         uint64_t v64 = byte_swap_8((uint64_t)rz->in);
819                         write(rz->filedes, &v64, sizeof(int64_t));
820                         v64 = byte_swap_8((uint64_t)rz->out);
821                         write(rz->filedes, &v64, sizeof(int64_t));
822                 }
823 #endif
824 #endif
825         } else if(rz->mode == 'r'){
826                 if(rz->stream) inflateEnd(rz->stream);
827         }
828         if(rz->inbuf) free(rz->inbuf);
829         if(rz->outbuf) free(rz->outbuf);
830         if(rz->header){
831                 free(rz->header->extra);
832                 free(rz->header->name);
833                 free(rz->header->comment);
834                 free(rz->header);
835         }
836         if(rz->index){
837                 free(rz->index->bin_offsets);
838                 free(rz->index->cell_offsets);
839                 free(rz->index);
840         }
841         free(rz->stream);
842 #ifdef _USE_KNETFILE
843     if (rz->mode == 'r')
844         knet_close(rz->x.fpr);
845     if (rz->mode == 'w')
846         close(rz->x.fpw);
847 #else
848         close(rz->filedes);
849 #endif
850         free(rz);
851 }
852
853 #endif