diff --git a/samtools-0.1.13/bam.c b/samtools-0.1.13/bam.c deleted file mode 100644 --- a/samtools-0.1.13/bam.c +++ /dev/null @@ -1,332 +0,0 @@ -#include -#include -#include -#include -#include "bam.h" -#include "bam_endian.h" -#include "kstring.h" -#include "sam_header.h" - -int bam_is_be = 0; -char *bam_flag2char_table = "pPuUrR12sfd\0\0\0\0\0"; - -/************************** - * CIGAR related routines * - **************************/ - -uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar) -{ - uint32_t k, end; - end = c->pos; - for (k = 0; k < c->n_cigar; ++k) { - int op = cigar[k] & BAM_CIGAR_MASK; - if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP) - end += cigar[k] >> BAM_CIGAR_SHIFT; - } - return end; -} - -int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar) -{ - uint32_t k; - int32_t l = 0; - for (k = 0; k < c->n_cigar; ++k) { - int op = cigar[k] & BAM_CIGAR_MASK; - if (op == BAM_CMATCH || op == BAM_CINS || op == BAM_CSOFT_CLIP) - l += cigar[k] >> BAM_CIGAR_SHIFT; - } - return l; -} - -/******************** - * BAM I/O routines * - ********************/ - -bam_header_t *bam_header_init() -{ - bam_is_be = bam_is_big_endian(); - return (bam_header_t*)calloc(1, sizeof(bam_header_t)); -} - -void bam_header_destroy(bam_header_t *header) -{ - int32_t i; - extern void bam_destroy_header_hash(bam_header_t *header); - if (header == 0) return; - if (header->target_name) { - for (i = 0; i < header->n_targets; ++i) - free(header->target_name[i]); - free(header->target_name); - free(header->target_len); - } - free(header->text); - if (header->dict) sam_header_free(header->dict); - if (header->rg2lib) sam_tbl_destroy(header->rg2lib); - bam_destroy_header_hash(header); - free(header); -} - -bam_header_t *bam_header_read(bamFile fp) -{ - bam_header_t *header; - char buf[4]; - int magic_len; - int32_t i = 1, name_len; - // check EOF - i = bgzf_check_EOF(fp); - if (i < 0) { - // If the file is a pipe, checking the EOF marker will *always* fail - // with ESPIPE. Suppress the error message in this case. - if (errno != ESPIPE) perror("[bam_header_read] bgzf_check_EOF"); - } - else if (i == 0) fprintf(stderr, "[bam_header_read] EOF marker is absent. The input is probably truncated.\n"); - // read "BAM1" - magic_len = bam_read(fp, buf, 4); - if (magic_len != 4 || strncmp(buf, "BAM\001", 4) != 0) { - fprintf(stderr, "[bam_header_read] invalid BAM binary header (this is not a BAM file).\n"); - return 0; - } - header = bam_header_init(); - // read plain text and the number of reference sequences - bam_read(fp, &header->l_text, 4); - if (bam_is_be) bam_swap_endian_4p(&header->l_text); - header->text = (char*)calloc(header->l_text + 1, 1); - bam_read(fp, header->text, header->l_text); - bam_read(fp, &header->n_targets, 4); - if (bam_is_be) bam_swap_endian_4p(&header->n_targets); - // read reference sequence names and lengths - header->target_name = (char**)calloc(header->n_targets, sizeof(char*)); - header->target_len = (uint32_t*)calloc(header->n_targets, 4); - for (i = 0; i != header->n_targets; ++i) { - bam_read(fp, &name_len, 4); - if (bam_is_be) bam_swap_endian_4p(&name_len); - header->target_name[i] = (char*)calloc(name_len, 1); - bam_read(fp, header->target_name[i], name_len); - bam_read(fp, &header->target_len[i], 4); - if (bam_is_be) bam_swap_endian_4p(&header->target_len[i]); - } - return header; -} - -int bam_header_write(bamFile fp, const bam_header_t *header) -{ - char buf[4]; - int32_t i, name_len, x; - // write "BAM1" - strncpy(buf, "BAM\001", 4); - bam_write(fp, buf, 4); - // write plain text and the number of reference sequences - if (bam_is_be) { - x = bam_swap_endian_4(header->l_text); - bam_write(fp, &x, 4); - if (header->l_text) bam_write(fp, header->text, header->l_text); - x = bam_swap_endian_4(header->n_targets); - bam_write(fp, &x, 4); - } else { - bam_write(fp, &header->l_text, 4); - if (header->l_text) bam_write(fp, header->text, header->l_text); - bam_write(fp, &header->n_targets, 4); - } - // write sequence names and lengths - for (i = 0; i != header->n_targets; ++i) { - char *p = header->target_name[i]; - name_len = strlen(p) + 1; - if (bam_is_be) { - x = bam_swap_endian_4(name_len); - bam_write(fp, &x, 4); - } else bam_write(fp, &name_len, 4); - bam_write(fp, p, name_len); - if (bam_is_be) { - x = bam_swap_endian_4(header->target_len[i]); - bam_write(fp, &x, 4); - } else bam_write(fp, &header->target_len[i], 4); - } - bgzf_flush(fp); - return 0; -} - -static void swap_endian_data(const bam1_core_t *c, int data_len, uint8_t *data) -{ - uint8_t *s; - uint32_t i, *cigar = (uint32_t*)(data + c->l_qname); - s = data + c->n_cigar*4 + c->l_qname + c->l_qseq + (c->l_qseq + 1)/2; - for (i = 0; i < c->n_cigar; ++i) bam_swap_endian_4p(&cigar[i]); - while (s < data + data_len) { - uint8_t type; - s += 2; // skip key - type = toupper(*s); ++s; // skip type - if (type == 'C' || type == 'A') ++s; - else if (type == 'S') { bam_swap_endian_2p(s); s += 2; } - else if (type == 'I' || type == 'F') { bam_swap_endian_4p(s); s += 4; } - else if (type == 'D') { bam_swap_endian_8p(s); s += 8; } - else if (type == 'Z' || type == 'H') { while (*s) ++s; ++s; } - } -} - -int bam_read1(bamFile fp, bam1_t *b) -{ - bam1_core_t *c = &b->core; - int32_t block_len, ret, i; - uint32_t x[8]; - - assert(BAM_CORE_SIZE == 32); - if ((ret = bam_read(fp, &block_len, 4)) != 4) { - if (ret == 0) return -1; // normal end-of-file - else return -2; // truncated - } - if (bam_read(fp, x, BAM_CORE_SIZE) != BAM_CORE_SIZE) return -3; - if (bam_is_be) { - bam_swap_endian_4p(&block_len); - for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i); - } - c->tid = x[0]; c->pos = x[1]; - c->bin = x[2]>>16; c->qual = x[2]>>8&0xff; c->l_qname = x[2]&0xff; - c->flag = x[3]>>16; c->n_cigar = x[3]&0xffff; - c->l_qseq = x[4]; - c->mtid = x[5]; c->mpos = x[6]; c->isize = x[7]; - b->data_len = block_len - BAM_CORE_SIZE; - if (b->m_data < b->data_len) { - b->m_data = b->data_len; - kroundup32(b->m_data); - b->data = (uint8_t*)realloc(b->data, b->m_data); - } - if (bam_read(fp, b->data, b->data_len) != b->data_len) return -4; - b->l_aux = b->data_len - c->n_cigar * 4 - c->l_qname - c->l_qseq - (c->l_qseq+1)/2; - if (bam_is_be) swap_endian_data(c, b->data_len, b->data); - return 4 + block_len; -} - -inline int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data) -{ - uint32_t x[8], block_len = data_len + BAM_CORE_SIZE, y; - int i; - assert(BAM_CORE_SIZE == 32); - x[0] = c->tid; - x[1] = c->pos; - x[2] = (uint32_t)c->bin<<16 | c->qual<<8 | c->l_qname; - x[3] = (uint32_t)c->flag<<16 | c->n_cigar; - x[4] = c->l_qseq; - x[5] = c->mtid; - x[6] = c->mpos; - x[7] = c->isize; - bgzf_flush_try(fp, 4 + block_len); - if (bam_is_be) { - for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i); - y = block_len; - bam_write(fp, bam_swap_endian_4p(&y), 4); - swap_endian_data(c, data_len, data); - } else bam_write(fp, &block_len, 4); - bam_write(fp, x, BAM_CORE_SIZE); - bam_write(fp, data, data_len); - if (bam_is_be) swap_endian_data(c, data_len, data); - return 4 + block_len; -} - -int bam_write1(bamFile fp, const bam1_t *b) -{ - return bam_write1_core(fp, &b->core, b->data_len, b->data); -} - -char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of) -{ - uint8_t *s = bam1_seq(b), *t = bam1_qual(b); - int i; - const bam1_core_t *c = &b->core; - kstring_t str; - str.l = str.m = 0; str.s = 0; - - kputsn(bam1_qname(b), c->l_qname-1, &str); kputc('\t', &str); - if (of == BAM_OFDEC) { kputw(c->flag, &str); kputc('\t', &str); } - else if (of == BAM_OFHEX) ksprintf(&str, "0x%x\t", c->flag); - else { // BAM_OFSTR - for (i = 0; i < 16; ++i) - if ((c->flag & 1<tid < 0) kputsn("*\t", 2, &str); - else { - if (header) kputs(header->target_name[c->tid] , &str); - else kputw(c->tid, &str); - kputc('\t', &str); - } - kputw(c->pos + 1, &str); kputc('\t', &str); kputw(c->qual, &str); kputc('\t', &str); - if (c->n_cigar == 0) kputc('*', &str); - else { - for (i = 0; i < c->n_cigar; ++i) { - kputw(bam1_cigar(b)[i]>>BAM_CIGAR_SHIFT, &str); - kputc("MIDNSHP"[bam1_cigar(b)[i]&BAM_CIGAR_MASK], &str); - } - } - kputc('\t', &str); - if (c->mtid < 0) kputsn("*\t", 2, &str); - else if (c->mtid == c->tid) kputsn("=\t", 2, &str); - else { - if (header) kputs(header->target_name[c->mtid], &str); - else kputw(c->mtid, &str); - kputc('\t', &str); - } - kputw(c->mpos + 1, &str); kputc('\t', &str); kputw(c->isize, &str); kputc('\t', &str); - if (c->l_qseq) { - for (i = 0; i < c->l_qseq; ++i) kputc(bam_nt16_rev_table[bam1_seqi(s, i)], &str); - kputc('\t', &str); - if (t[0] == 0xff) kputc('*', &str); - else for (i = 0; i < c->l_qseq; ++i) kputc(t[i] + 33, &str); - } else kputsn("*\t*", 3, &str); - s = bam1_aux(b); - while (s < b->data + b->data_len) { - uint8_t type, key[2]; - key[0] = s[0]; key[1] = s[1]; - s += 2; type = *s; ++s; - kputc('\t', &str); kputsn((char*)key, 2, &str); kputc(':', &str); - if (type == 'A') { kputsn("A:", 2, &str); kputc(*s, &str); ++s; } - else if (type == 'C') { kputsn("i:", 2, &str); kputw(*s, &str); ++s; } - else if (type == 'c') { kputsn("i:", 2, &str); kputw(*(int8_t*)s, &str); ++s; } - else if (type == 'S') { kputsn("i:", 2, &str); kputw(*(uint16_t*)s, &str); s += 2; } - else if (type == 's') { kputsn("i:", 2, &str); kputw(*(int16_t*)s, &str); s += 2; } - else if (type == 'I') { kputsn("i:", 2, &str); kputuw(*(uint32_t*)s, &str); s += 4; } - else if (type == 'i') { kputsn("i:", 2, &str); kputw(*(int32_t*)s, &str); s += 4; } - else if (type == 'f') { ksprintf(&str, "f:%g", *(float*)s); s += 4; } - else if (type == 'd') { ksprintf(&str, "d:%lg", *(double*)s); s += 8; } - else if (type == 'Z' || type == 'H') { kputc(type, &str); kputc(':', &str); while (*s) kputc(*s++, &str); ++s; } - } - return str.s; -} - -char *bam_format1(const bam_header_t *header, const bam1_t *b) -{ - return bam_format1_core(header, b, BAM_OFDEC); -} - -void bam_view1(const bam_header_t *header, const bam1_t *b) -{ - char *s = bam_format1(header, b); - puts(s); - free(s); -} - -int bam_validate1(const bam_header_t *header, const bam1_t *b) -{ - char *s; - - if (b->core.tid < -1 || b->core.mtid < -1) return 0; - if (header && (b->core.tid >= header->n_targets || b->core.mtid >= header->n_targets)) return 0; - - if (b->data_len < b->core.l_qname) return 0; - s = memchr(bam1_qname(b), '\0', b->core.l_qname); - if (s != &bam1_qname(b)[b->core.l_qname-1]) return 0; - - // FIXME: Other fields could also be checked, especially the auxiliary data - - return 1; -} - -// FIXME: we should also check the LB tag associated with each alignment -const char *bam_get_library(bam_header_t *h, const bam1_t *b) -{ - const uint8_t *rg; - if (h->dict == 0) h->dict = sam_header_parse2(h->text); - if (h->rg2lib == 0) h->rg2lib = sam_header2tbl(h->dict, "RG", "ID", "LB"); - rg = bam_aux_get(b, "RG"); - return (rg == 0)? 0 : sam_tbl_get(h->rg2lib, (const char*)(rg + 1)); -} diff --git a/samtools-0.1.13/bam.h b/samtools-0.1.13/bam.h deleted file mode 100644 --- a/samtools-0.1.13/bam.h +++ /dev/null @@ -1,743 +0,0 @@ -/* The MIT License - - Copyright (c) 2008-2010 Genome Research Ltd (GRL). - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -/* Contact: Heng Li */ - -#ifndef BAM_BAM_H -#define BAM_BAM_H - -/*! - @header - - BAM library provides I/O and various operations on manipulating files - in the BAM (Binary Alignment/Mapping) or SAM (Sequence Alignment/Map) - format. It now supports importing from or exporting to TAM, sorting, - merging, generating pileup, and quickly retrieval of reads overlapped - with a specified region. - - @copyright Genome Research Ltd. - */ - -#define BAM_VERSION "0.1.13 (r926:134)" - -#include -#include -#include -#include - -#ifndef BAM_LITE -#define BAM_VIRTUAL_OFFSET16 -#include "bgzf.h" -/*! @abstract BAM file handler */ -typedef BGZF *bamFile; -#define bam_open(fn, mode) bgzf_open(fn, mode) -#define bam_dopen(fd, mode) bgzf_fdopen(fd, mode) -#define bam_close(fp) bgzf_close(fp) -#define bam_read(fp, buf, size) bgzf_read(fp, buf, size) -#define bam_write(fp, buf, size) bgzf_write(fp, buf, size) -#define bam_tell(fp) bgzf_tell(fp) -#define bam_seek(fp, pos, dir) bgzf_seek(fp, pos, dir) -#else -#define BAM_TRUE_OFFSET -#include -typedef gzFile bamFile; -#define bam_open(fn, mode) gzopen(fn, mode) -#define bam_dopen(fd, mode) gzdopen(fd, mode) -#define bam_close(fp) gzclose(fp) -#define bam_read(fp, buf, size) gzread(fp, buf, size) -/* no bam_write/bam_tell/bam_seek() here */ -#endif - -/*! @typedef - @abstract Structure for the alignment header. - @field n_targets number of reference sequences - @field target_name names of the reference sequences - @field target_len lengths of the referene sequences - @field dict header dictionary - @field hash hash table for fast name lookup - @field rg2lib hash table for @RG-ID -> LB lookup - @field l_text length of the plain text in the header - @field text plain text - - @discussion Field hash points to null by default. It is a private - member. - */ -typedef struct { - int32_t n_targets; - char **target_name; - uint32_t *target_len; - void *dict, *hash, *rg2lib; - size_t l_text, n_text; - char *text; -} bam_header_t; - -/*! @abstract the read is paired in sequencing, no matter whether it is mapped in a pair */ -#define BAM_FPAIRED 1 -/*! @abstract the read is mapped in a proper pair */ -#define BAM_FPROPER_PAIR 2 -/*! @abstract the read itself is unmapped; conflictive with BAM_FPROPER_PAIR */ -#define BAM_FUNMAP 4 -/*! @abstract the mate is unmapped */ -#define BAM_FMUNMAP 8 -/*! @abstract the read is mapped to the reverse strand */ -#define BAM_FREVERSE 16 -/*! @abstract the mate is mapped to the reverse strand */ -#define BAM_FMREVERSE 32 -/*! @abstract this is read1 */ -#define BAM_FREAD1 64 -/*! @abstract this is read2 */ -#define BAM_FREAD2 128 -/*! @abstract not primary alignment */ -#define BAM_FSECONDARY 256 -/*! @abstract QC failure */ -#define BAM_FQCFAIL 512 -/*! @abstract optical or PCR duplicate */ -#define BAM_FDUP 1024 - -#define BAM_OFDEC 0 -#define BAM_OFHEX 1 -#define BAM_OFSTR 2 - -/*! @abstract defautl mask for pileup */ -#define BAM_DEF_MASK (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP) - -#define BAM_CORE_SIZE sizeof(bam1_core_t) - -/** - * Describing how CIGAR operation/length is packed in a 32-bit integer. - */ -#define BAM_CIGAR_SHIFT 4 -#define BAM_CIGAR_MASK ((1 << BAM_CIGAR_SHIFT) - 1) - -/* - CIGAR operations. - */ -/*! @abstract CIGAR: match */ -#define BAM_CMATCH 0 -/*! @abstract CIGAR: insertion to the reference */ -#define BAM_CINS 1 -/*! @abstract CIGAR: deletion from the reference */ -#define BAM_CDEL 2 -/*! @abstract CIGAR: skip on the reference (e.g. spliced alignment) */ -#define BAM_CREF_SKIP 3 -/*! @abstract CIGAR: clip on the read with clipped sequence present in qseq */ -#define BAM_CSOFT_CLIP 4 -/*! @abstract CIGAR: clip on the read with clipped sequence trimmed off */ -#define BAM_CHARD_CLIP 5 -/*! @abstract CIGAR: padding */ -#define BAM_CPAD 6 - -/*! @typedef - @abstract Structure for core alignment information. - @field tid chromosome ID, defined by bam_header_t - @field pos 0-based leftmost coordinate - @field strand strand; 0 for forward and 1 otherwise - @field bin bin calculated by bam_reg2bin() - @field qual mapping quality - @field l_qname length of the query name - @field flag bitwise flag - @field n_cigar number of CIGAR operations - @field l_qseq length of the query sequence (read) - */ -typedef struct { - int32_t tid; - int32_t pos; - uint32_t bin:16, qual:8, l_qname:8; - uint32_t flag:16, n_cigar:16; - int32_t l_qseq; - int32_t mtid; - int32_t mpos; - int32_t isize; -} bam1_core_t; - -/*! @typedef - @abstract Structure for one alignment. - @field core core information about the alignment - @field l_aux length of auxiliary data - @field data_len current length of bam1_t::data - @field m_data maximum length of bam1_t::data - @field data all variable-length data, concatenated; structure: cigar-qname-seq-qual-aux - - @discussion Notes: - - 1. qname is zero tailing and core.l_qname includes the tailing '\0'. - 2. l_qseq is calculated from the total length of an alignment block - on reading or from CIGAR. - */ -typedef struct { - bam1_core_t core; - int l_aux, data_len, m_data; - uint8_t *data; -} bam1_t; - -typedef struct __bam_iter_t *bam_iter_t; - -#define bam1_strand(b) (((b)->core.flag&BAM_FREVERSE) != 0) -#define bam1_mstrand(b) (((b)->core.flag&BAM_FMREVERSE) != 0) - -/*! @function - @abstract Get the CIGAR array - @param b pointer to an alignment - @return pointer to the CIGAR array - - @discussion In the CIGAR array, each element is a 32-bit integer. The - lower 4 bits gives a CIGAR operation and the higher 28 bits keep the - length of a CIGAR. - */ -#define bam1_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname)) - -/*! @function - @abstract Get the name of the query - @param b pointer to an alignment - @return pointer to the name string, null terminated - */ -#define bam1_qname(b) ((char*)((b)->data)) - -/*! @function - @abstract Get query sequence - @param b pointer to an alignment - @return pointer to sequence - - @discussion Each base is encoded in 4 bits: 1 for A, 2 for C, 4 for G, - 8 for T and 15 for N. Two bases are packed in one byte with the base - at the higher 4 bits having smaller coordinate on the read. It is - recommended to use bam1_seqi() macro to get the base. - */ -#define bam1_seq(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname) - -/*! @function - @abstract Get query quality - @param b pointer to an alignment - @return pointer to quality string - */ -#define bam1_qual(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1)) - -/*! @function - @abstract Get a base on read - @param s Query sequence returned by bam1_seq() - @param i The i-th position, 0-based - @return 4-bit integer representing the base. - */ -#define bam1_seqi(s, i) ((s)[(i)/2] >> 4*(1-(i)%2) & 0xf) - -/*! @function - @abstract Get query sequence and quality - @param b pointer to an alignment - @return pointer to the concatenated auxiliary data - */ -#define bam1_aux(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (b)->core.l_qseq + ((b)->core.l_qseq + 1)/2) - -#ifndef kroundup32 -/*! @function - @abstract Round an integer to the next closest power-2 integer. - @param x integer to be rounded (in place) - @discussion x will be modified. - */ -#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) -#endif - -/*! - @abstract Whether the machine is big-endian; modified only in - bam_header_init(). - */ -extern int bam_is_be; - -/*! @abstract Table for converting a nucleotide character to the 4-bit encoding. */ -extern unsigned char bam_nt16_table[256]; - -/*! @abstract Table for converting a 4-bit encoded nucleotide to a letter. */ -extern char *bam_nt16_rev_table; - -extern char bam_nt16_nt4_table[]; - -#ifdef __cplusplus -extern "C" { -#endif - - /********************* - * Low-level SAM I/O * - *********************/ - - /*! @abstract TAM file handler */ - typedef struct __tamFile_t *tamFile; - - /*! - @abstract Open a SAM file for reading, either uncompressed or compressed by gzip/zlib. - @param fn SAM file name - @return SAM file handler - */ - tamFile sam_open(const char *fn); - - /*! - @abstract Close a SAM file handler - @param fp SAM file handler - */ - void sam_close(tamFile fp); - - /*! - @abstract Read one alignment from a SAM file handler - @param fp SAM file handler - @param header header information (ordered names of chromosomes) - @param b read alignment; all members in b will be updated - @return 0 if successful; otherwise negative - */ - int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b); - - /*! - @abstract Read header information from a TAB-delimited list file. - @param fn_list file name for the list - @return a pointer to the header structure - - @discussion Each line in this file consists of chromosome name and - the length of chromosome. - */ - bam_header_t *sam_header_read2(const char *fn_list); - - /*! - @abstract Read header from a SAM file (if present) - @param fp SAM file handler - @return pointer to header struct; 0 if no @SQ lines available - */ - bam_header_t *sam_header_read(tamFile fp); - - /*! - @abstract Parse @SQ lines a update a header struct - @param h pointer to the header struct to be updated - @return number of target sequences - - @discussion bam_header_t::{n_targets,target_len,target_name} will - be destroyed in the first place. - */ - int sam_header_parse(bam_header_t *h); - int32_t bam_get_tid(const bam_header_t *header, const char *seq_name); - - /*! - @abstract Parse @RG lines a update a header struct - @param h pointer to the header struct to be updated - @return number of @RG lines - - @discussion bam_header_t::rg2lib will be destroyed in the first - place. - */ - int sam_header_parse_rg(bam_header_t *h); - -#define sam_write1(header, b) bam_view1(header, b) - - - /******************************** - * APIs for string dictionaries * - ********************************/ - - int bam_strmap_put(void *strmap, const char *rg, const char *lib); - const char *bam_strmap_get(const void *strmap, const char *rg); - void *bam_strmap_dup(const void*); - void *bam_strmap_init(); - void bam_strmap_destroy(void *strmap); - - - /********************* - * Low-level BAM I/O * - *********************/ - - /*! - @abstract Initialize a header structure. - @return the pointer to the header structure - - @discussion This function also modifies the global variable - bam_is_be. - */ - bam_header_t *bam_header_init(); - - /*! - @abstract Destroy a header structure. - @param header pointer to the header - */ - void bam_header_destroy(bam_header_t *header); - - /*! - @abstract Read a header structure from BAM. - @param fp BAM file handler, opened by bam_open() - @return pointer to the header structure - - @discussion The file position indicator must be placed at the - beginning of the file. Upon success, the position indicator will - be set at the start of the first alignment. - */ - bam_header_t *bam_header_read(bamFile fp); - - /*! - @abstract Write a header structure to BAM. - @param fp BAM file handler - @param header pointer to the header structure - @return always 0 currently - */ - int bam_header_write(bamFile fp, const bam_header_t *header); - - /*! - @abstract Read an alignment from BAM. - @param fp BAM file handler - @param b read alignment; all members are updated. - @return number of bytes read from the file - - @discussion The file position indicator must be - placed right before an alignment. Upon success, this function - will set the position indicator to the start of the next - alignment. This function is not affected by the machine - endianness. - */ - int bam_read1(bamFile fp, bam1_t *b); - - /*! - @abstract Write an alignment to BAM. - @param fp BAM file handler - @param c pointer to the bam1_core_t structure - @param data_len total length of variable size data related to - the alignment - @param data pointer to the concatenated data - @return number of bytes written to the file - - @discussion This function is not affected by the machine - endianness. - */ - int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data); - - /*! - @abstract Write an alignment to BAM. - @param fp BAM file handler - @param b alignment to write - @return number of bytes written to the file - - @abstract It is equivalent to: - bam_write1_core(fp, &b->core, b->data_len, b->data) - */ - int bam_write1(bamFile fp, const bam1_t *b); - - /*! @function - @abstract Initiate a pointer to bam1_t struct - */ -#define bam_init1() ((bam1_t*)calloc(1, sizeof(bam1_t))) - - /*! @function - @abstract Free the memory allocated for an alignment. - @param b pointer to an alignment - */ -#define bam_destroy1(b) do { \ - if (b) { free((b)->data); free(b); } \ - } while (0) - - /*! - @abstract Format a BAM record in the SAM format - @param header pointer to the header structure - @param b alignment to print - @return a pointer to the SAM string - */ - char *bam_format1(const bam_header_t *header, const bam1_t *b); - - char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of); - - /*! - @abstract Check whether a BAM record is plausibly valid - @param header associated header structure, or NULL if unavailable - @param b alignment to validate - @return 0 if the alignment is invalid; non-zero otherwise - - @discussion Simple consistency check of some of the fields of the - alignment record. If the header is provided, several additional checks - are made. Not all fields are checked, so a non-zero result is not a - guarantee that the record is valid. However it is usually good enough - to detect when bam_seek() has been called with a virtual file offset - that is not the offset of an alignment record. - */ - int bam_validate1(const bam_header_t *header, const bam1_t *b); - - const char *bam_get_library(bam_header_t *header, const bam1_t *b); - - - /*************** - * pileup APIs * - ***************/ - - /*! @typedef - @abstract Structure for one alignment covering the pileup position. - @field b pointer to the alignment - @field qpos position of the read base at the pileup site, 0-based - @field indel indel length; 0 for no indel, positive for ins and negative for del - @field is_del 1 iff the base on the padded read is a deletion - @field level the level of the read in the "viewer" mode - - @discussion See also bam_plbuf_push() and bam_lplbuf_push(). The - difference between the two functions is that the former does not - set bam_pileup1_t::level, while the later does. Level helps the - implementation of alignment viewers, but calculating this has some - overhead. - */ - typedef struct { - bam1_t *b; - int32_t qpos; - int indel, level; - uint32_t is_del:1, is_head:1, is_tail:1, is_refskip:1, aux:28; - } bam_pileup1_t; - - typedef int (*bam_plp_auto_f)(void *data, bam1_t *b); - - struct __bam_plp_t; - typedef struct __bam_plp_t *bam_plp_t; - - bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data); - int bam_plp_push(bam_plp_t iter, const bam1_t *b); - const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp); - const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp); - void bam_plp_set_mask(bam_plp_t iter, int mask); - void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt); - void bam_plp_reset(bam_plp_t iter); - void bam_plp_destroy(bam_plp_t iter); - - struct __bam_mplp_t; - typedef struct __bam_mplp_t *bam_mplp_t; - - bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data); - void bam_mplp_destroy(bam_mplp_t iter); - void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt); - int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp); - - /*! @typedef - @abstract Type of function to be called by bam_plbuf_push(). - @param tid chromosome ID as is defined in the header - @param pos start coordinate of the alignment, 0-based - @param n number of elements in pl array - @param pl array of alignments - @param data user provided data - @discussion See also bam_plbuf_push(), bam_plbuf_init() and bam_pileup1_t. - */ - typedef int (*bam_pileup_f)(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data); - - typedef struct { - bam_plp_t iter; - bam_pileup_f func; - void *data; - } bam_plbuf_t; - - void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask); - void bam_plbuf_reset(bam_plbuf_t *buf); - bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data); - void bam_plbuf_destroy(bam_plbuf_t *buf); - int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf); - - int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data); - - struct __bam_lplbuf_t; - typedef struct __bam_lplbuf_t bam_lplbuf_t; - - void bam_lplbuf_reset(bam_lplbuf_t *buf); - - /*! @abstract bam_plbuf_init() equivalent with level calculated. */ - bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data); - - /*! @abstract bam_plbuf_destroy() equivalent with level calculated. */ - void bam_lplbuf_destroy(bam_lplbuf_t *tv); - - /*! @abstract bam_plbuf_push() equivalent with level calculated. */ - int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *buf); - - - /********************* - * BAM indexing APIs * - *********************/ - - struct __bam_index_t; - typedef struct __bam_index_t bam_index_t; - - /*! - @abstract Build index for a BAM file. - @discussion Index file "fn.bai" will be created. - @param fn name of the BAM file - @return always 0 currently - */ - int bam_index_build(const char *fn); - - /*! - @abstract Load index from file "fn.bai". - @param fn name of the BAM file (NOT the index file) - @return pointer to the index structure - */ - bam_index_t *bam_index_load(const char *fn); - - /*! - @abstract Destroy an index structure. - @param idx pointer to the index structure - */ - void bam_index_destroy(bam_index_t *idx); - - /*! @typedef - @abstract Type of function to be called by bam_fetch(). - @param b the alignment - @param data user provided data - */ - typedef int (*bam_fetch_f)(const bam1_t *b, void *data); - - /*! - @abstract Retrieve the alignments that are overlapped with the - specified region. - - @discussion A user defined function will be called for each - retrieved alignment ordered by its start position. - - @param fp BAM file handler - @param idx pointer to the alignment index - @param tid chromosome ID as is defined in the header - @param beg start coordinate, 0-based - @param end end coordinate, 0-based - @param data user provided data (will be transferred to func) - @param func user defined function - */ - int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func); - - bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end); - int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b); - void bam_iter_destroy(bam_iter_t iter); - - /*! - @abstract Parse a region in the format: "chr2:100,000-200,000". - @discussion bam_header_t::hash will be initialized if empty. - @param header pointer to the header structure - @param str string to be parsed - @param ref_id the returned chromosome ID - @param begin the returned start coordinate - @param end the returned end coordinate - @return 0 on success; -1 on failure - */ - int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *begin, int *end); - - - /************************** - * APIs for optional tags * - **************************/ - - /*! - @abstract Retrieve data of a tag - @param b pointer to an alignment struct - @param tag two-character tag to be retrieved - - @return pointer to the type and data. The first character is the - type that can be 'iIsScCdfAZH'. - - @discussion Use bam_aux2?() series to convert the returned data to - the corresponding type. - */ - uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]); - - int32_t bam_aux2i(const uint8_t *s); - float bam_aux2f(const uint8_t *s); - double bam_aux2d(const uint8_t *s); - char bam_aux2A(const uint8_t *s); - char *bam_aux2Z(const uint8_t *s); - - int bam_aux_del(bam1_t *b, uint8_t *s); - void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data); - uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2]); // an alias of bam_aux_get() - - - /***************** - * Miscellaneous * - *****************/ - - /*! - @abstract Calculate the rightmost coordinate of an alignment on the - reference genome. - - @param c pointer to the bam1_core_t structure - @param cigar the corresponding CIGAR array (from bam1_t::cigar) - @return the rightmost coordinate, 0-based - */ - uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar); - - /*! - @abstract Calculate the length of the query sequence from CIGAR. - @param c pointer to the bam1_core_t structure - @param cigar the corresponding CIGAR array (from bam1_t::cigar) - @return length of the query sequence - */ - int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar); - -#ifdef __cplusplus -} -#endif - -/*! - @abstract Calculate the minimum bin that contains a region [beg,end). - @param beg start of the region, 0-based - @param end end of the region, 0-based - @return bin - */ -static inline int bam_reg2bin(uint32_t beg, uint32_t end) -{ - --end; - if (beg>>14 == end>>14) return 4681 + (beg>>14); - if (beg>>17 == end>>17) return 585 + (beg>>17); - if (beg>>20 == end>>20) return 73 + (beg>>20); - if (beg>>23 == end>>23) return 9 + (beg>>23); - if (beg>>26 == end>>26) return 1 + (beg>>26); - return 0; -} - -/*! - @abstract Copy an alignment - @param bdst destination alignment struct - @param bsrc source alignment struct - @return pointer to the destination alignment struct - */ -static inline bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc) -{ - uint8_t *data = bdst->data; - int m_data = bdst->m_data; // backup data and m_data - if (m_data < bsrc->data_len) { // double the capacity - m_data = bsrc->data_len; kroundup32(m_data); - data = (uint8_t*)realloc(data, m_data); - } - memcpy(data, bsrc->data, bsrc->data_len); // copy var-len data - *bdst = *bsrc; // copy the rest - // restore the backup - bdst->m_data = m_data; - bdst->data = data; - return bdst; -} - -/*! - @abstract Duplicate an alignment - @param src source alignment struct - @return pointer to the destination alignment struct - */ -static inline bam1_t *bam_dup1(const bam1_t *src) -{ - bam1_t *b; - b = bam_init1(); - *b = *src; - b->m_data = b->data_len; - b->data = (uint8_t*)calloc(b->data_len, 1); - memcpy(b->data, src->data, b->data_len); - return b; -} - -#endif diff --git a/samtools-0.1.13/bam_aux.c b/samtools-0.1.13/bam_aux.c deleted file mode 100644 --- a/samtools-0.1.13/bam_aux.c +++ /dev/null @@ -1,189 +0,0 @@ -#include -#include "bam.h" -#include "khash.h" -typedef char *str_p; -KHASH_MAP_INIT_STR(s, int) -KHASH_MAP_INIT_STR(r2l, str_p) - -void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data) -{ - int ori_len = b->data_len; - b->data_len += 3 + len; - b->l_aux += 3 + len; - if (b->m_data < b->data_len) { - b->m_data = b->data_len; - kroundup32(b->m_data); - b->data = (uint8_t*)realloc(b->data, b->m_data); - } - b->data[ori_len] = tag[0]; b->data[ori_len + 1] = tag[1]; - b->data[ori_len + 2] = type; - memcpy(b->data + ori_len + 3, data, len); -} - -uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2]) -{ - return bam_aux_get(b, tag); -} - -#define __skip_tag(s) do { \ - int type = toupper(*(s)); \ - ++(s); \ - if (type == 'C' || type == 'A') ++(s); \ - else if (type == 'S') (s) += 2; \ - else if (type == 'I' || type == 'F') (s) += 4; \ - else if (type == 'D') (s) += 8; \ - else if (type == 'Z' || type == 'H') { while (*(s)) ++(s); ++(s); } \ - } while (0) - -uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]) -{ - uint8_t *s; - int y = tag[0]<<8 | tag[1]; - s = bam1_aux(b); - while (s < b->data + b->data_len) { - int x = (int)s[0]<<8 | s[1]; - s += 2; - if (x == y) return s; - __skip_tag(s); - } - return 0; -} -// s MUST BE returned by bam_aux_get() -int bam_aux_del(bam1_t *b, uint8_t *s) -{ - uint8_t *p, *aux; - aux = bam1_aux(b); - p = s - 2; - __skip_tag(s); - memmove(p, s, b->l_aux - (s - aux)); - b->data_len -= s - p; - b->l_aux -= s - p; - return 0; -} - -void bam_init_header_hash(bam_header_t *header) -{ - if (header->hash == 0) { - int ret, i; - khiter_t iter; - khash_t(s) *h; - header->hash = h = kh_init(s); - for (i = 0; i < header->n_targets; ++i) { - iter = kh_put(s, h, header->target_name[i], &ret); - kh_value(h, iter) = i; - } - } -} - -void bam_destroy_header_hash(bam_header_t *header) -{ - if (header->hash) - kh_destroy(s, (khash_t(s)*)header->hash); -} - -int32_t bam_get_tid(const bam_header_t *header, const char *seq_name) -{ - khint_t k; - khash_t(s) *h = (khash_t(s)*)header->hash; - k = kh_get(s, h, seq_name); - return k == kh_end(h)? -1 : kh_value(h, k); -} - -int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *begin, int *end) -{ - char *s, *p; - int i, l, k; - khiter_t iter; - khash_t(s) *h; - - bam_init_header_hash(header); - h = (khash_t(s)*)header->hash; - - l = strlen(str); - p = s = (char*)malloc(l+1); - /* squeeze out "," */ - for (i = k = 0; i != l; ++i) - if (str[i] != ',' && !isspace(str[i])) s[k++] = str[i]; - s[k] = 0; - for (i = 0; i != k; ++i) if (s[i] == ':') break; - s[i] = 0; - iter = kh_get(s, h, s); /* get the ref_id */ - if (iter == kh_end(h)) { // name not found - *ref_id = -1; free(s); - return -1; - } - *ref_id = kh_value(h, iter); - if (i == k) { /* dump the whole sequence */ - *begin = 0; *end = 1<<29; free(s); - return 0; - } - for (p = s + i + 1; i != k; ++i) if (s[i] == '-') break; - *begin = atoi(p); - if (i < k) { - p = s + i + 1; - *end = atoi(p); - } else *end = 1<<29; - if (*begin > 0) --*begin; - free(s); - if (*begin > *end) { - fprintf(stderr, "[bam_parse_region] invalid region.\n"); - return -1; - } - return 0; -} - -int32_t bam_aux2i(const uint8_t *s) -{ - int type; - if (s == 0) return 0; - type = *s++; - if (type == 'c') return (int32_t)*(int8_t*)s; - else if (type == 'C') return (int32_t)*(uint8_t*)s; - else if (type == 's') return (int32_t)*(int16_t*)s; - else if (type == 'S') return (int32_t)*(uint16_t*)s; - else if (type == 'i' || type == 'I') return *(int32_t*)s; - else return 0; -} - -float bam_aux2f(const uint8_t *s) -{ - int type; - type = *s++; - if (s == 0) return 0.0; - if (type == 'f') return *(float*)s; - else return 0.0; -} - -double bam_aux2d(const uint8_t *s) -{ - int type; - type = *s++; - if (s == 0) return 0.0; - if (type == 'd') return *(double*)s; - else return 0.0; -} - -char bam_aux2A(const uint8_t *s) -{ - int type; - type = *s++; - if (s == 0) return 0; - if (type == 'A') return *(char*)s; - else return 0; -} - -char *bam_aux2Z(const uint8_t *s) -{ - int type; - type = *s++; - if (s == 0) return 0; - if (type == 'Z' || type == 'H') return (char*)s; - else return 0; -} - -#ifdef _WIN32 -double drand48() -{ - return (double)rand() / RAND_MAX; -} -#endif diff --git a/samtools-0.1.13/bam_endian.h b/samtools-0.1.13/bam_endian.h deleted file mode 100644 --- a/samtools-0.1.13/bam_endian.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef BAM_ENDIAN_H -#define BAM_ENDIAN_H - -#include - -static inline int bam_is_big_endian() -{ - long one= 1; - return !(*((char *)(&one))); -} -static inline uint16_t bam_swap_endian_2(uint16_t v) -{ - return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8)); -} -static inline void *bam_swap_endian_2p(void *x) -{ - *(uint16_t*)x = bam_swap_endian_2(*(uint16_t*)x); - return x; -} -static inline uint32_t bam_swap_endian_4(uint32_t v) -{ - v = ((v & 0x0000FFFFU) << 16) | (v >> 16); - return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8); -} -static inline void *bam_swap_endian_4p(void *x) -{ - *(uint32_t*)x = bam_swap_endian_4(*(uint32_t*)x); - return x; -} -static inline uint64_t bam_swap_endian_8(uint64_t v) -{ - v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32); - v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16); - return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8); -} -static inline void *bam_swap_endian_8p(void *x) -{ - *(uint64_t*)x = bam_swap_endian_8(*(uint64_t*)x); - return x; -} - -#endif diff --git a/samtools-0.1.13/bam_import.c b/samtools-0.1.13/bam_import.c deleted file mode 100644 --- a/samtools-0.1.13/bam_import.c +++ /dev/null @@ -1,459 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef _WIN32 -#include -#endif -#include "kstring.h" -#include "bam.h" -#include "sam_header.h" -#include "kseq.h" -#include "khash.h" - -KSTREAM_INIT(gzFile, gzread, 8192) -KHASH_MAP_INIT_STR(ref, uint64_t) - -void bam_init_header_hash(bam_header_t *header); -void bam_destroy_header_hash(bam_header_t *header); -int32_t bam_get_tid(const bam_header_t *header, const char *seq_name); - -unsigned char bam_nt16_table[256] = { - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 1, 2, 4, 8, 15,15,15,15, 15,15,15,15, 15, 0 /*=*/,15,15, - 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15, - 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15, - 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15, - 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, - 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15 -}; - -unsigned short bam_char2flag_table[256] = { - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,BAM_FREAD1,BAM_FREAD2,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - BAM_FPROPER_PAIR,0,BAM_FMREVERSE,0, 0,BAM_FMUNMAP,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, BAM_FDUP,0,BAM_FQCFAIL,0, 0,0,0,0, 0,0,0,0, - BAM_FPAIRED,0,BAM_FREVERSE,BAM_FSECONDARY, 0,BAM_FUNMAP,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 -}; - -char *bam_nt16_rev_table = "=ACMGRSVTWYHKDBN"; - -struct __tamFile_t { - gzFile fp; - kstream_t *ks; - kstring_t *str; - uint64_t n_lines; - int is_first; -}; - -char **__bam_get_lines(const char *fn, int *_n) // for bam_plcmd.c only -{ - char **list = 0, *s; - int n = 0, dret, m = 0; - gzFile fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); - kstream_t *ks; - kstring_t *str; - str = (kstring_t*)calloc(1, sizeof(kstring_t)); - ks = ks_init(fp); - while (ks_getuntil(ks, '\n', str, &dret) > 0) { - if (n == m) { - m = m? m << 1 : 16; - list = (char**)realloc(list, m * sizeof(char*)); - } - if (str->s[str->l-1] == '\r') - str->s[--str->l] = '\0'; - s = list[n++] = (char*)calloc(str->l + 1, 1); - strcpy(s, str->s); - } - ks_destroy(ks); - gzclose(fp); - free(str->s); free(str); - *_n = n; - return list; -} - -static bam_header_t *hash2header(const kh_ref_t *hash) -{ - bam_header_t *header; - khiter_t k; - header = bam_header_init(); - header->n_targets = kh_size(hash); - header->target_name = (char**)calloc(kh_size(hash), sizeof(char*)); - header->target_len = (uint32_t*)calloc(kh_size(hash), 4); - for (k = kh_begin(hash); k != kh_end(hash); ++k) { - if (kh_exist(hash, k)) { - int i = (int)kh_value(hash, k); - header->target_name[i] = (char*)kh_key(hash, k); - header->target_len[i] = kh_value(hash, k)>>32; - } - } - bam_init_header_hash(header); - return header; -} -bam_header_t *sam_header_read2(const char *fn) -{ - bam_header_t *header; - int c, dret, ret, error = 0; - gzFile fp; - kstream_t *ks; - kstring_t *str; - kh_ref_t *hash; - khiter_t k; - if (fn == 0) return 0; - fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); - if (fp == 0) return 0; - hash = kh_init(ref); - ks = ks_init(fp); - str = (kstring_t*)calloc(1, sizeof(kstring_t)); - while (ks_getuntil(ks, 0, str, &dret) > 0) { - char *s = strdup(str->s); - int len, i; - i = kh_size(hash); - ks_getuntil(ks, 0, str, &dret); - len = atoi(str->s); - k = kh_put(ref, hash, s, &ret); - if (ret == 0) { - fprintf(stderr, "[sam_header_read2] duplicated sequence name: %s\n", s); - error = 1; - } - kh_value(hash, k) = (uint64_t)len<<32 | i; - if (dret != '\n') - while ((c = ks_getc(ks)) != '\n' && c != -1); - } - ks_destroy(ks); - gzclose(fp); - free(str->s); free(str); - fprintf(stderr, "[sam_header_read2] %d sequences loaded.\n", kh_size(hash)); - if (error) return 0; - header = hash2header(hash); - kh_destroy(ref, hash); - return header; -} -static inline uint8_t *alloc_data(bam1_t *b, int size) -{ - if (b->m_data < size) { - b->m_data = size; - kroundup32(b->m_data); - b->data = (uint8_t*)realloc(b->data, b->m_data); - } - return b->data; -} -static inline void parse_error(int64_t n_lines, const char * __restrict msg) -{ - fprintf(stderr, "Parse error at line %lld: %s\n", (long long)n_lines, msg); - abort(); -} -static inline void append_text(bam_header_t *header, kstring_t *str) -{ - size_t x = header->l_text, y = header->l_text + str->l + 2; // 2 = 1 byte dret + 1 byte null - kroundup32(x); kroundup32(y); - if (x < y) - { - header->n_text = y; - header->text = (char*)realloc(header->text, y); - if ( !header->text ) - { - fprintf(stderr,"realloc failed to alloc %ld bytes\n", y); - abort(); - } - } - // Sanity check - if ( header->l_text+str->l+1 >= header->n_text ) - { - fprintf(stderr,"append_text FIXME: %ld>=%ld, x=%ld,y=%ld\n", header->l_text+str->l+1,header->n_text,x,y); - abort(); - } - strncpy(header->text + header->l_text, str->s, str->l+1); // we cannot use strcpy() here. - header->l_text += str->l + 1; - header->text[header->l_text] = 0; -} - -int sam_header_parse(bam_header_t *h) -{ - char **tmp; - int i; - free(h->target_len); free(h->target_name); - h->n_targets = 0; h->target_len = 0; h->target_name = 0; - if (h->l_text < 3) return 0; - if (h->dict == 0) h->dict = sam_header_parse2(h->text); - tmp = sam_header2list(h->dict, "SQ", "SN", &h->n_targets); - if (h->n_targets == 0) return 0; - h->target_name = calloc(h->n_targets, sizeof(void*)); - for (i = 0; i < h->n_targets; ++i) - h->target_name[i] = strdup(tmp[i]); - free(tmp); - tmp = sam_header2list(h->dict, "SQ", "LN", &h->n_targets); - h->target_len = calloc(h->n_targets, 4); - for (i = 0; i < h->n_targets; ++i) - h->target_len[i] = atoi(tmp[i]); - free(tmp); - return h->n_targets; -} - -bam_header_t *sam_header_read(tamFile fp) -{ - int ret, dret; - bam_header_t *header = bam_header_init(); - kstring_t *str = fp->str; - while ((ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret)) >= 0 && str->s[0] == '@') { // skip header - str->s[str->l] = dret; // note that str->s is NOT null terminated!! - append_text(header, str); - if (dret != '\n') { - ret = ks_getuntil(fp->ks, '\n', str, &dret); - str->s[str->l] = '\n'; // NOT null terminated!! - append_text(header, str); - } - ++fp->n_lines; - } - sam_header_parse(header); - bam_init_header_hash(header); - fp->is_first = 1; - return header; -} - -int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) -{ - int ret, doff, doff0, dret, z = 0; - bam1_core_t *c = &b->core; - kstring_t *str = fp->str; - kstream_t *ks = fp->ks; - - if (fp->is_first) { - fp->is_first = 0; - ret = str->l; - } else { - do { // special consideration for empty lines - ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret); - if (ret >= 0) z += str->l + 1; - } while (ret == 0); - } - if (ret < 0) return -1; - ++fp->n_lines; - doff = 0; - - { // name - c->l_qname = strlen(str->s) + 1; - memcpy(alloc_data(b, doff + c->l_qname) + doff, str->s, c->l_qname); - doff += c->l_qname; - } - { // flag - long flag; - char *s; - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; - flag = strtol((char*)str->s, &s, 0); - if (*s) { // not the end of the string - flag = 0; - for (s = str->s; *s; ++s) - flag |= bam_char2flag_table[(int)*s]; - } - c->flag = flag; - } - { // tid, pos, qual - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->tid = bam_get_tid(header, str->s); - if (c->tid < 0 && strcmp(str->s, "*")) { - if (header->n_targets == 0) { - fprintf(stderr, "[sam_read1] missing header? Abort!\n"); - exit(1); - } else fprintf(stderr, "[sam_read1] reference '%s' is recognized as '*'.\n", str->s); - } - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->pos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->qual = isdigit(str->s[0])? atoi(str->s) : 0; - if (ret < 0) return -2; - } - { // cigar - char *s, *t; - int i, op; - long x; - c->n_cigar = 0; - if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -3; - z += str->l + 1; - if (str->s[0] != '*') { - for (s = str->s; *s; ++s) { - if (isalpha(*s)) ++c->n_cigar; - else if (!isdigit(*s)) parse_error(fp->n_lines, "invalid CIGAR character"); - } - b->data = alloc_data(b, doff + c->n_cigar * 4); - for (i = 0, s = str->s; i != c->n_cigar; ++i) { - x = strtol(s, &t, 10); - op = toupper(*t); - if (op == 'M' || op == '=' || op == 'X') op = BAM_CMATCH; - else if (op == 'I') op = BAM_CINS; - else if (op == 'D') op = BAM_CDEL; - else if (op == 'N') op = BAM_CREF_SKIP; - else if (op == 'S') op = BAM_CSOFT_CLIP; - else if (op == 'H') op = BAM_CHARD_CLIP; - else if (op == 'P') op = BAM_CPAD; - else parse_error(fp->n_lines, "invalid CIGAR operation"); - s = t + 1; - bam1_cigar(b)[i] = x << BAM_CIGAR_SHIFT | op; - } - if (*s) parse_error(fp->n_lines, "unmatched CIGAR operation"); - c->bin = bam_reg2bin(c->pos, bam_calend(c, bam1_cigar(b))); - doff += c->n_cigar * 4; - } else { - if (!(c->flag&BAM_FUNMAP)) { - fprintf(stderr, "Parse warning at line %lld: mapped sequence without CIGAR\n", (long long)fp->n_lines); - c->flag |= BAM_FUNMAP; - } - c->bin = bam_reg2bin(c->pos, c->pos + 1); - } - } - { // mtid, mpos, isize - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; - c->mtid = strcmp(str->s, "=")? bam_get_tid(header, str->s) : c->tid; - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; - c->mpos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; - ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; - c->isize = (str->s[0] == '-' || isdigit(str->s[0]))? atoi(str->s) : 0; - if (ret < 0) return -4; - } - { // seq and qual - int i; - uint8_t *p = 0; - if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -5; // seq - z += str->l + 1; - if (strcmp(str->s, "*")) { - c->l_qseq = strlen(str->s); - if (c->n_cigar && c->l_qseq != (int32_t)bam_cigar2qlen(c, bam1_cigar(b))) - parse_error(fp->n_lines, "CIGAR and sequence length are inconsistent"); - p = (uint8_t*)alloc_data(b, doff + c->l_qseq + (c->l_qseq+1)/2) + doff; - memset(p, 0, (c->l_qseq+1)/2); - for (i = 0; i < c->l_qseq; ++i) - p[i/2] |= bam_nt16_table[(int)str->s[i]] << 4*(1-i%2); - } else c->l_qseq = 0; - if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -6; // qual - z += str->l + 1; - if (strcmp(str->s, "*") && c->l_qseq != strlen(str->s)) - parse_error(fp->n_lines, "sequence and quality are inconsistent"); - p += (c->l_qseq+1)/2; - if (strcmp(str->s, "*") == 0) for (i = 0; i < c->l_qseq; ++i) p[i] = 0xff; - else for (i = 0; i < c->l_qseq; ++i) p[i] = str->s[i] - 33; - doff += c->l_qseq + (c->l_qseq+1)/2; - } - doff0 = doff; - if (dret != '\n' && dret != '\r') { // aux - while (ks_getuntil(ks, KS_SEP_TAB, str, &dret) >= 0) { - uint8_t *s, type, key[2]; - z += str->l + 1; - if (str->l < 6 || str->s[2] != ':' || str->s[4] != ':') - parse_error(fp->n_lines, "missing colon in auxiliary data"); - key[0] = str->s[0]; key[1] = str->s[1]; - type = str->s[3]; - s = alloc_data(b, doff + 3) + doff; - s[0] = key[0]; s[1] = key[1]; s += 2; doff += 2; - if (type == 'A' || type == 'a' || type == 'c' || type == 'C') { // c and C for backward compatibility - s = alloc_data(b, doff + 2) + doff; - *s++ = 'A'; *s = str->s[5]; - doff += 2; - } else if (type == 'I' || type == 'i') { - long long x; - s = alloc_data(b, doff + 5) + doff; - x = (long long)atoll(str->s + 5); - if (x < 0) { - if (x >= -127) { - *s++ = 'c'; *(int8_t*)s = (int8_t)x; - s += 1; doff += 2; - } else if (x >= -32767) { - *s++ = 's'; *(int16_t*)s = (int16_t)x; - s += 2; doff += 3; - } else { - *s++ = 'i'; *(int32_t*)s = (int32_t)x; - s += 4; doff += 5; - if (x < -2147483648ll) - fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.", - (long long)fp->n_lines, x); - } - } else { - if (x <= 255) { - *s++ = 'C'; *s++ = (uint8_t)x; - doff += 2; - } else if (x <= 65535) { - *s++ = 'S'; *(uint16_t*)s = (uint16_t)x; - s += 2; doff += 3; - } else { - *s++ = 'I'; *(uint32_t*)s = (uint32_t)x; - s += 4; doff += 5; - if (x > 4294967295ll) - fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.", - (long long)fp->n_lines, x); - } - } - } else if (type == 'f') { - s = alloc_data(b, doff + 5) + doff; - *s++ = 'f'; - *(float*)s = (float)atof(str->s + 5); - s += 4; doff += 5; - } else if (type == 'd') { - s = alloc_data(b, doff + 9) + doff; - *s++ = 'd'; - *(float*)s = (float)atof(str->s + 9); - s += 8; doff += 9; - } else if (type == 'Z' || type == 'H') { - int size = 1 + (str->l - 5) + 1; - if (type == 'H') { // check whether the hex string is valid - int i; - if ((str->l - 5) % 2 == 1) parse_error(fp->n_lines, "length of the hex string not even"); - for (i = 0; i < str->l - 5; ++i) { - int c = toupper(str->s[5 + i]); - if (!((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F'))) - parse_error(fp->n_lines, "invalid hex character"); - } - } - s = alloc_data(b, doff + size) + doff; - *s++ = type; - memcpy(s, str->s + 5, str->l - 5); - s[str->l - 5] = 0; - doff += size; - } else parse_error(fp->n_lines, "unrecognized type"); - if (dret == '\n' || dret == '\r') break; - } - } - b->l_aux = doff - doff0; - b->data_len = doff; - return z; -} - -tamFile sam_open(const char *fn) -{ - tamFile fp; - gzFile gzfp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "rb") : gzopen(fn, "rb"); - if (gzfp == 0) return 0; - fp = (tamFile)calloc(1, sizeof(struct __tamFile_t)); - fp->str = (kstring_t*)calloc(1, sizeof(kstring_t)); - fp->fp = gzfp; - fp->ks = ks_init(fp->fp); - return fp; -} - -void sam_close(tamFile fp) -{ - if (fp) { - ks_destroy(fp->ks); - gzclose(fp->fp); - free(fp->str->s); free(fp->str); - free(fp); - } -} diff --git a/samtools-0.1.13/bam_index.c b/samtools-0.1.13/bam_index.c deleted file mode 100644 --- a/samtools-0.1.13/bam_index.c +++ /dev/null @@ -1,711 +0,0 @@ -#include -#include -#include "bam.h" -#include "khash.h" -#include "ksort.h" -#include "bam_endian.h" -#ifdef _USE_KNETFILE -#include "knetfile.h" -#endif - -/*! - @header - - Alignment indexing. Before indexing, BAM must be sorted based on the - leftmost coordinate of alignments. In indexing, BAM uses two indices: - a UCSC binning index and a simple linear index. The binning index is - efficient for alignments spanning long distance, while the auxiliary - linear index helps to reduce unnecessary seek calls especially for - short alignments. - - The UCSC binning scheme was suggested by Richard Durbin and Lincoln - Stein and is explained by Kent et al. (2002). In this scheme, each bin - represents a contiguous genomic region which can be fully contained in - another bin; each alignment is associated with a bin which represents - the smallest region containing the entire alignment. The binning - scheme is essentially another representation of R-tree. A distinct bin - uniquely corresponds to a distinct internal node in a R-tree. Bin A is - a child of Bin B if region A is contained in B. - - In BAM, each bin may span 2^29, 2^26, 2^23, 2^20, 2^17 or 2^14 bp. Bin - 0 spans a 512Mbp region, bins 1-8 span 64Mbp, 9-72 8Mbp, 73-584 1Mbp, - 585-4680 128Kbp and bins 4681-37449 span 16Kbp regions. If we want to - find the alignments overlapped with a region [rbeg,rend), we need to - calculate the list of bins that may be overlapped the region and test - the alignments in the bins to confirm the overlaps. If the specified - region is short, typically only a few alignments in six bins need to - be retrieved. The overlapping alignments can be quickly fetched. - - */ - -#define BAM_MIN_CHUNK_GAP 32768 -// 1<<14 is the size of minimum bin. -#define BAM_LIDX_SHIFT 14 - -#define BAM_MAX_BIN 37450 // =(8^6-1)/7+1 - -typedef struct { - uint64_t u, v; -} pair64_t; - -#define pair64_lt(a,b) ((a).u < (b).u) -KSORT_INIT(off, pair64_t, pair64_lt) - -typedef struct { - uint32_t m, n; - pair64_t *list; -} bam_binlist_t; - -typedef struct { - int32_t n, m; - uint64_t *offset; -} bam_lidx_t; - -KHASH_MAP_INIT_INT(i, bam_binlist_t) - -struct __bam_index_t { - int32_t n; - uint64_t n_no_coor; // unmapped reads without coordinate - khash_t(i) **index; - bam_lidx_t *index2; -}; - -// requirement: len <= LEN_MASK -static inline void insert_offset(khash_t(i) *h, int bin, uint64_t beg, uint64_t end) -{ - khint_t k; - bam_binlist_t *l; - int ret; - k = kh_put(i, h, bin, &ret); - l = &kh_value(h, k); - if (ret) { // not present - l->m = 1; l->n = 0; - l->list = (pair64_t*)calloc(l->m, 16); - } - if (l->n == l->m) { - l->m <<= 1; - l->list = (pair64_t*)realloc(l->list, l->m * 16); - } - l->list[l->n].u = beg; l->list[l->n++].v = end; -} - -static inline void insert_offset2(bam_lidx_t *index2, bam1_t *b, uint64_t offset) -{ - int i, beg, end; - beg = b->core.pos >> BAM_LIDX_SHIFT; - end = (bam_calend(&b->core, bam1_cigar(b)) - 1) >> BAM_LIDX_SHIFT; - if (index2->m < end + 1) { - int old_m = index2->m; - index2->m = end + 1; - kroundup32(index2->m); - index2->offset = (uint64_t*)realloc(index2->offset, index2->m * 8); - memset(index2->offset + old_m, 0, 8 * (index2->m - old_m)); - } - if (beg == end) { - if (index2->offset[beg] == 0) index2->offset[beg] = offset; - } else { - for (i = beg; i <= end; ++i) - if (index2->offset[i] == 0) index2->offset[i] = offset; - } - index2->n = end + 1; -} - -static void merge_chunks(bam_index_t *idx) -{ -#if defined(BAM_TRUE_OFFSET) || defined(BAM_VIRTUAL_OFFSET16) - khash_t(i) *index; - int i, l, m; - khint_t k; - for (i = 0; i < idx->n; ++i) { - index = idx->index[i]; - for (k = kh_begin(index); k != kh_end(index); ++k) { - bam_binlist_t *p; - if (!kh_exist(index, k) || kh_key(index, k) == BAM_MAX_BIN) continue; - p = &kh_value(index, k); - m = 0; - for (l = 1; l < p->n; ++l) { -#ifdef BAM_TRUE_OFFSET - if (p->list[m].v + BAM_MIN_CHUNK_GAP > p->list[l].u) p->list[m].v = p->list[l].v; -#else - if (p->list[m].v>>16 == p->list[l].u>>16) p->list[m].v = p->list[l].v; -#endif - else p->list[++m] = p->list[l]; - } // ~for(l) - p->n = m + 1; - } // ~for(k) - } // ~for(i) -#endif // defined(BAM_TRUE_OFFSET) || defined(BAM_BGZF) -} - -static void fill_missing(bam_index_t *idx) -{ - int i, j; - for (i = 0; i < idx->n; ++i) { - bam_lidx_t *idx2 = &idx->index2[i]; - for (j = 1; j < idx2->n; ++j) - if (idx2->offset[j] == 0) - idx2->offset[j] = idx2->offset[j-1]; - } -} - -bam_index_t *bam_index_core(bamFile fp) -{ - bam1_t *b; - bam_header_t *h; - int i, ret; - bam_index_t *idx; - uint32_t last_bin, save_bin; - int32_t last_coor, last_tid, save_tid; - bam1_core_t *c; - uint64_t save_off, last_off, n_mapped, n_unmapped, off_beg, off_end, n_no_coor; - - idx = (bam_index_t*)calloc(1, sizeof(bam_index_t)); - b = (bam1_t*)calloc(1, sizeof(bam1_t)); - h = bam_header_read(fp); - c = &b->core; - - idx->n = h->n_targets; - bam_header_destroy(h); - idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*)); - for (i = 0; i < idx->n; ++i) idx->index[i] = kh_init(i); - idx->index2 = (bam_lidx_t*)calloc(idx->n, sizeof(bam_lidx_t)); - - save_bin = save_tid = last_tid = last_bin = 0xffffffffu; - save_off = last_off = bam_tell(fp); last_coor = 0xffffffffu; - n_mapped = n_unmapped = n_no_coor = off_end = 0; - off_beg = off_end = bam_tell(fp); - while ((ret = bam_read1(fp, b)) >= 0) { - if (c->tid < 0) ++n_no_coor; - if (last_tid != c->tid) { // change of chromosomes - last_tid = c->tid; - last_bin = 0xffffffffu; - } else if (last_coor > c->pos) { - fprintf(stderr, "[bam_index_core] the alignment is not sorted (%s): %u > %u in %d-th chr\n", - bam1_qname(b), last_coor, c->pos, c->tid+1); - exit(1); - } - if (c->tid >= 0) insert_offset2(&idx->index2[b->core.tid], b, last_off); - if (c->bin != last_bin) { // then possibly write the binning index - if (save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record - insert_offset(idx->index[save_tid], save_bin, save_off, last_off); - if (last_bin == 0xffffffffu && save_tid != 0xffffffffu) { // write the meta element - off_end = last_off; - insert_offset(idx->index[save_tid], BAM_MAX_BIN, off_beg, off_end); - insert_offset(idx->index[save_tid], BAM_MAX_BIN, n_mapped, n_unmapped); - n_mapped = n_unmapped = 0; - off_beg = off_end; - } - save_off = last_off; - save_bin = last_bin = c->bin; - save_tid = c->tid; - if (save_tid < 0) break; - } - if (bam_tell(fp) <= last_off) { - fprintf(stderr, "[bam_index_core] bug in BGZF/RAZF: %llx < %llx\n", - (unsigned long long)bam_tell(fp), (unsigned long long)last_off); - exit(1); - } - if (c->flag & BAM_FUNMAP) ++n_unmapped; - else ++n_mapped; - last_off = bam_tell(fp); - last_coor = b->core.pos; - } - if (save_tid >= 0) { - insert_offset(idx->index[save_tid], save_bin, save_off, bam_tell(fp)); - insert_offset(idx->index[save_tid], BAM_MAX_BIN, off_beg, bam_tell(fp)); - insert_offset(idx->index[save_tid], BAM_MAX_BIN, n_mapped, n_unmapped); - } - merge_chunks(idx); - fill_missing(idx); - if (ret >= 0) { - while ((ret = bam_read1(fp, b)) >= 0) { - ++n_no_coor; - if (c->tid >= 0 && n_no_coor) { - fprintf(stderr, "[bam_index_core] the alignment is not sorted: reads without coordinates prior to reads with coordinates.\n"); - exit(1); - } - } - } - if (ret < -1) fprintf(stderr, "[bam_index_core] truncated file? Continue anyway. (%d)\n", ret); - free(b->data); free(b); - idx->n_no_coor = n_no_coor; - return idx; -} - -void bam_index_destroy(bam_index_t *idx) -{ - khint_t k; - int i; - if (idx == 0) return; - for (i = 0; i < idx->n; ++i) { - khash_t(i) *index = idx->index[i]; - bam_lidx_t *index2 = idx->index2 + i; - for (k = kh_begin(index); k != kh_end(index); ++k) { - if (kh_exist(index, k)) - free(kh_value(index, k).list); - } - kh_destroy(i, index); - free(index2->offset); - } - free(idx->index); free(idx->index2); - free(idx); -} - -void bam_index_save(const bam_index_t *idx, FILE *fp) -{ - int32_t i, size; - khint_t k; - fwrite("BAI\1", 1, 4, fp); - if (bam_is_be) { - uint32_t x = idx->n; - fwrite(bam_swap_endian_4p(&x), 4, 1, fp); - } else fwrite(&idx->n, 4, 1, fp); - for (i = 0; i < idx->n; ++i) { - khash_t(i) *index = idx->index[i]; - bam_lidx_t *index2 = idx->index2 + i; - // write binning index - size = kh_size(index); - if (bam_is_be) { // big endian - uint32_t x = size; - fwrite(bam_swap_endian_4p(&x), 4, 1, fp); - } else fwrite(&size, 4, 1, fp); - for (k = kh_begin(index); k != kh_end(index); ++k) { - if (kh_exist(index, k)) { - bam_binlist_t *p = &kh_value(index, k); - if (bam_is_be) { // big endian - uint32_t x; - x = kh_key(index, k); fwrite(bam_swap_endian_4p(&x), 4, 1, fp); - x = p->n; fwrite(bam_swap_endian_4p(&x), 4, 1, fp); - for (x = 0; (int)x < p->n; ++x) { - bam_swap_endian_8p(&p->list[x].u); - bam_swap_endian_8p(&p->list[x].v); - } - fwrite(p->list, 16, p->n, fp); - for (x = 0; (int)x < p->n; ++x) { - bam_swap_endian_8p(&p->list[x].u); - bam_swap_endian_8p(&p->list[x].v); - } - } else { - fwrite(&kh_key(index, k), 4, 1, fp); - fwrite(&p->n, 4, 1, fp); - fwrite(p->list, 16, p->n, fp); - } - } - } - // write linear index (index2) - if (bam_is_be) { - int x = index2->n; - fwrite(bam_swap_endian_4p(&x), 4, 1, fp); - } else fwrite(&index2->n, 4, 1, fp); - if (bam_is_be) { // big endian - int x; - for (x = 0; (int)x < index2->n; ++x) - bam_swap_endian_8p(&index2->offset[x]); - fwrite(index2->offset, 8, index2->n, fp); - for (x = 0; (int)x < index2->n; ++x) - bam_swap_endian_8p(&index2->offset[x]); - } else fwrite(index2->offset, 8, index2->n, fp); - } - { // write the number of reads coor-less records. - uint64_t x = idx->n_no_coor; - if (bam_is_be) bam_swap_endian_8p(&x); - fwrite(&x, 8, 1, fp); - } - fflush(fp); -} - -static bam_index_t *bam_index_load_core(FILE *fp) -{ - int i; - char magic[4]; - bam_index_t *idx; - if (fp == 0) { - fprintf(stderr, "[bam_index_load_core] fail to load index.\n"); - return 0; - } - fread(magic, 1, 4, fp); - if (strncmp(magic, "BAI\1", 4)) { - fprintf(stderr, "[bam_index_load] wrong magic number.\n"); - fclose(fp); - return 0; - } - idx = (bam_index_t*)calloc(1, sizeof(bam_index_t)); - fread(&idx->n, 4, 1, fp); - if (bam_is_be) bam_swap_endian_4p(&idx->n); - idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*)); - idx->index2 = (bam_lidx_t*)calloc(idx->n, sizeof(bam_lidx_t)); - for (i = 0; i < idx->n; ++i) { - khash_t(i) *index; - bam_lidx_t *index2 = idx->index2 + i; - uint32_t key, size; - khint_t k; - int j, ret; - bam_binlist_t *p; - index = idx->index[i] = kh_init(i); - // load binning index - fread(&size, 4, 1, fp); - if (bam_is_be) bam_swap_endian_4p(&size); - for (j = 0; j < (int)size; ++j) { - fread(&key, 4, 1, fp); - if (bam_is_be) bam_swap_endian_4p(&key); - k = kh_put(i, index, key, &ret); - p = &kh_value(index, k); - fread(&p->n, 4, 1, fp); - if (bam_is_be) bam_swap_endian_4p(&p->n); - p->m = p->n; - p->list = (pair64_t*)malloc(p->m * 16); - fread(p->list, 16, p->n, fp); - if (bam_is_be) { - int x; - for (x = 0; x < p->n; ++x) { - bam_swap_endian_8p(&p->list[x].u); - bam_swap_endian_8p(&p->list[x].v); - } - } - } - // load linear index - fread(&index2->n, 4, 1, fp); - if (bam_is_be) bam_swap_endian_4p(&index2->n); - index2->m = index2->n; - index2->offset = (uint64_t*)calloc(index2->m, 8); - fread(index2->offset, index2->n, 8, fp); - if (bam_is_be) - for (j = 0; j < index2->n; ++j) bam_swap_endian_8p(&index2->offset[j]); - } - if (fread(&idx->n_no_coor, 8, 1, fp) == 0) idx->n_no_coor = 0; - if (bam_is_be) bam_swap_endian_8p(&idx->n_no_coor); - return idx; -} - -bam_index_t *bam_index_load_local(const char *_fn) -{ - FILE *fp; - char *fnidx, *fn; - - if (strstr(_fn, "ftp://") == _fn || strstr(_fn, "http://") == _fn) { - const char *p; - int l = strlen(_fn); - for (p = _fn + l - 1; p >= _fn; --p) - if (*p == '/') break; - fn = strdup(p + 1); - } else fn = strdup(_fn); - fnidx = (char*)calloc(strlen(fn) + 5, 1); - strcpy(fnidx, fn); strcat(fnidx, ".bai"); - fp = fopen(fnidx, "rb"); - if (fp == 0) { // try "{base}.bai" - char *s = strstr(fn, "bam"); - if (s == fn + strlen(fn) - 3) { - strcpy(fnidx, fn); - fnidx[strlen(fn)-1] = 'i'; - fp = fopen(fnidx, "rb"); - } - } - free(fnidx); free(fn); - if (fp) { - bam_index_t *idx = bam_index_load_core(fp); - fclose(fp); - return idx; - } else return 0; -} - -#ifdef _USE_KNETFILE -static void download_from_remote(const char *url) -{ - const int buf_size = 1 * 1024 * 1024; - char *fn; - FILE *fp; - uint8_t *buf; - knetFile *fp_remote; - int l; - if (strstr(url, "ftp://") != url && strstr(url, "http://") != url) return; - l = strlen(url); - for (fn = (char*)url + l - 1; fn >= url; --fn) - if (*fn == '/') break; - ++fn; // fn now points to the file name - fp_remote = knet_open(url, "r"); - if (fp_remote == 0) { - fprintf(stderr, "[download_from_remote] fail to open remote file.\n"); - return; - } - if ((fp = fopen(fn, "wb")) == 0) { - fprintf(stderr, "[download_from_remote] fail to create file in the working directory.\n"); - knet_close(fp_remote); - return; - } - buf = (uint8_t*)calloc(buf_size, 1); - while ((l = knet_read(fp_remote, buf, buf_size)) != 0) - fwrite(buf, 1, l, fp); - free(buf); - fclose(fp); - knet_close(fp_remote); -} -#else -static void download_from_remote(const char *url) -{ - return; -} -#endif - -bam_index_t *bam_index_load(const char *fn) -{ - bam_index_t *idx; - idx = bam_index_load_local(fn); - if (idx == 0 && (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn)) { - char *fnidx = calloc(strlen(fn) + 5, 1); - strcat(strcpy(fnidx, fn), ".bai"); - fprintf(stderr, "[bam_index_load] attempting to download the remote index file.\n"); - download_from_remote(fnidx); - idx = bam_index_load_local(fn); - } - if (idx == 0) fprintf(stderr, "[bam_index_load] fail to load BAM index.\n"); - return idx; -} - -int bam_index_build2(const char *fn, const char *_fnidx) -{ - char *fnidx; - FILE *fpidx; - bamFile fp; - bam_index_t *idx; - if ((fp = bam_open(fn, "r")) == 0) { - fprintf(stderr, "[bam_index_build2] fail to open the BAM file.\n"); - return -1; - } - idx = bam_index_core(fp); - bam_close(fp); - if (_fnidx == 0) { - fnidx = (char*)calloc(strlen(fn) + 5, 1); - strcpy(fnidx, fn); strcat(fnidx, ".bai"); - } else fnidx = strdup(_fnidx); - fpidx = fopen(fnidx, "wb"); - if (fpidx == 0) { - fprintf(stderr, "[bam_index_build2] fail to create the index file.\n"); - free(fnidx); - return -1; - } - bam_index_save(idx, fpidx); - bam_index_destroy(idx); - fclose(fpidx); - free(fnidx); - return 0; -} - -int bam_index_build(const char *fn) -{ - return bam_index_build2(fn, 0); -} - -int bam_index(int argc, char *argv[]) -{ - if (argc < 2) { - fprintf(stderr, "Usage: samtools index [out.index]\n"); - return 1; - } - if (argc >= 3) bam_index_build2(argv[1], argv[2]); - else bam_index_build(argv[1]); - return 0; -} - -int bam_idxstats(int argc, char *argv[]) -{ - bam_index_t *idx; - bam_header_t *header; - bamFile fp; - int i; - if (argc < 2) { - fprintf(stderr, "Usage: samtools idxstats \n"); - return 1; - } - fp = bam_open(argv[1], "r"); - if (fp == 0) { fprintf(stderr, "[%s] fail to open BAM.\n", __func__); return 1; } - header = bam_header_read(fp); - bam_close(fp); - idx = bam_index_load(argv[1]); - if (idx == 0) { fprintf(stderr, "[%s] fail to load the index.\n", __func__); return 1; } - for (i = 0; i < idx->n; ++i) { - khint_t k; - khash_t(i) *h = idx->index[i]; - printf("%s\t%d", header->target_name[i], header->target_len[i]); - k = kh_get(i, h, BAM_MAX_BIN); - if (k != kh_end(h)) - printf("\t%llu\t%llu", (long long)kh_val(h, k).list[1].u, (long long)kh_val(h, k).list[1].v); - else printf("\t0\t0"); - putchar('\n'); - } - printf("*\t0\t0\t%llu\n", (long long)idx->n_no_coor); - bam_header_destroy(header); - bam_index_destroy(idx); - return 0; -} - -static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[BAM_MAX_BIN]) -{ - int i = 0, k; - if (beg >= end) return 0; - if (end >= 1u<<29) end = 1u<<29; - --end; - list[i++] = 0; - for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list[i++] = k; - for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list[i++] = k; - for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list[i++] = k; - for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list[i++] = k; - for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list[i++] = k; - return i; -} - -static inline int is_overlap(uint32_t beg, uint32_t end, const bam1_t *b) -{ - uint32_t rbeg = b->core.pos; - uint32_t rend = b->core.n_cigar? bam_calend(&b->core, bam1_cigar(b)) : b->core.pos + 1; - return (rend > beg && rbeg < end); -} - -struct __bam_iter_t { - int from_first; // read from the first record; no random access - int tid, beg, end, n_off, i, finished; - uint64_t curr_off; - pair64_t *off; -}; - -// bam_fetch helper function retrieves -bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end) -{ - uint16_t *bins; - int i, n_bins, n_off; - pair64_t *off; - khint_t k; - khash_t(i) *index; - uint64_t min_off; - bam_iter_t iter = 0; - - if (beg < 0) beg = 0; - if (end < beg) return 0; - // initialize iter - iter = calloc(1, sizeof(struct __bam_iter_t)); - iter->tid = tid, iter->beg = beg, iter->end = end; iter->i = -1; - // - bins = (uint16_t*)calloc(BAM_MAX_BIN, 2); - n_bins = reg2bins(beg, end, bins); - index = idx->index[tid]; - if (idx->index2[tid].n > 0) { - min_off = (beg>>BAM_LIDX_SHIFT >= idx->index2[tid].n)? idx->index2[tid].offset[idx->index2[tid].n-1] - : idx->index2[tid].offset[beg>>BAM_LIDX_SHIFT]; - if (min_off == 0) { // improvement for index files built by tabix prior to 0.1.4 - int n = beg>>BAM_LIDX_SHIFT; - if (n > idx->index2[tid].n) n = idx->index2[tid].n; - for (i = n - 1; i >= 0; --i) - if (idx->index2[tid].offset[i] != 0) break; - if (i >= 0) min_off = idx->index2[tid].offset[i]; - } - } else min_off = 0; // tabix 0.1.2 may produce such index files - for (i = n_off = 0; i < n_bins; ++i) { - if ((k = kh_get(i, index, bins[i])) != kh_end(index)) - n_off += kh_value(index, k).n; - } - if (n_off == 0) { - free(bins); return iter; - } - off = (pair64_t*)calloc(n_off, 16); - for (i = n_off = 0; i < n_bins; ++i) { - if ((k = kh_get(i, index, bins[i])) != kh_end(index)) { - int j; - bam_binlist_t *p = &kh_value(index, k); - for (j = 0; j < p->n; ++j) - if (p->list[j].v > min_off) off[n_off++] = p->list[j]; - } - } - free(bins); - if (n_off == 0) { - free(off); return iter; - } - { - bam1_t *b = (bam1_t*)calloc(1, sizeof(bam1_t)); - int l; - ks_introsort(off, n_off, off); - // resolve completely contained adjacent blocks - for (i = 1, l = 0; i < n_off; ++i) - if (off[l].v < off[i].v) - off[++l] = off[i]; - n_off = l + 1; - // resolve overlaps between adjacent blocks; this may happen due to the merge in indexing - for (i = 1; i < n_off; ++i) - if (off[i-1].v >= off[i].u) off[i-1].v = off[i].u; - { // merge adjacent blocks -#if defined(BAM_TRUE_OFFSET) || defined(BAM_VIRTUAL_OFFSET16) - for (i = 1, l = 0; i < n_off; ++i) { -#ifdef BAM_TRUE_OFFSET - if (off[l].v + BAM_MIN_CHUNK_GAP > off[i].u) off[l].v = off[i].v; -#else - if (off[l].v>>16 == off[i].u>>16) off[l].v = off[i].v; -#endif - else off[++l] = off[i]; - } - n_off = l + 1; -#endif - } - bam_destroy1(b); - } - iter->n_off = n_off; iter->off = off; - return iter; -} - -pair64_t *get_chunk_coordinates(const bam_index_t *idx, int tid, int beg, int end, int *cnt_off) -{ // for pysam compatibility - bam_iter_t iter; - pair64_t *off; - iter = bam_iter_query(idx, tid, beg, end); - off = iter->off; *cnt_off = iter->n_off; - free(iter); - return off; -} - -void bam_iter_destroy(bam_iter_t iter) -{ - if (iter) { free(iter->off); free(iter); } -} - -int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b) -{ - int ret; - if (iter && iter->finished) return -1; - if (iter == 0 || iter->from_first) { - ret = bam_read1(fp, b); - if (ret < 0 && iter) iter->finished = 1; - return ret; - } - if (iter->off == 0) return -1; - for (;;) { - if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk - if (iter->i == iter->n_off - 1) { ret = -1; break; } // no more chunks - if (iter->i >= 0) assert(iter->curr_off == iter->off[iter->i].v); // otherwise bug - if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek - bam_seek(fp, iter->off[iter->i+1].u, SEEK_SET); - iter->curr_off = bam_tell(fp); - } - ++iter->i; - } - if ((ret = bam_read1(fp, b)) >= 0) { - iter->curr_off = bam_tell(fp); - if (b->core.tid != iter->tid || b->core.pos >= iter->end) { // no need to proceed - ret = bam_validate1(NULL, b)? -1 : -5; // determine whether end of region or error - break; - } - else if (is_overlap(iter->beg, iter->end, b)) return ret; - } else break; // end of file or error - } - iter->finished = 1; - return ret; -} - -int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func) -{ - int ret; - bam_iter_t iter; - bam1_t *b; - b = bam_init1(); - iter = bam_iter_query(idx, tid, beg, end); - while ((ret = bam_iter_read(fp, iter, b)) >= 0) func(b, data); - bam_iter_destroy(iter); - bam_destroy1(b); - return (ret == -1)? 0 : ret; -} diff --git a/samtools-0.1.13/bam_lpileup.c b/samtools-0.1.13/bam_lpileup.c deleted file mode 100644 --- a/samtools-0.1.13/bam_lpileup.c +++ /dev/null @@ -1,198 +0,0 @@ -#include -#include -#include -#include "bam.h" -#include "ksort.h" - -#define TV_GAP 2 - -typedef struct __freenode_t { - uint32_t level:28, cnt:4; - struct __freenode_t *next; -} freenode_t, *freenode_p; - -#define freenode_lt(a,b) ((a)->cnt < (b)->cnt || ((a)->cnt == (b)->cnt && (a)->level < (b)->level)) -KSORT_INIT(node, freenode_p, freenode_lt) - -/* Memory pool, similar to the one in bam_pileup.c */ -typedef struct { - int cnt, n, max; - freenode_t **buf; -} mempool_t; - -static mempool_t *mp_init() -{ - return (mempool_t*)calloc(1, sizeof(mempool_t)); -} -static void mp_destroy(mempool_t *mp) -{ - int k; - for (k = 0; k < mp->n; ++k) free(mp->buf[k]); - free(mp->buf); free(mp); -} -static inline freenode_t *mp_alloc(mempool_t *mp) -{ - ++mp->cnt; - if (mp->n == 0) return (freenode_t*)calloc(1, sizeof(freenode_t)); - else return mp->buf[--mp->n]; -} -static inline void mp_free(mempool_t *mp, freenode_t *p) -{ - --mp->cnt; p->next = 0; p->cnt = TV_GAP; - if (mp->n == mp->max) { - mp->max = mp->max? mp->max<<1 : 256; - mp->buf = (freenode_t**)realloc(mp->buf, sizeof(freenode_t*) * mp->max); - } - mp->buf[mp->n++] = p; -} - -/* core part */ -struct __bam_lplbuf_t { - int max, n_cur, n_pre; - int max_level, *cur_level, *pre_level; - mempool_t *mp; - freenode_t **aux, *head, *tail; - int n_nodes, m_aux; - bam_pileup_f func; - void *user_data; - bam_plbuf_t *plbuf; -}; - -void bam_lplbuf_reset(bam_lplbuf_t *buf) -{ - freenode_t *p, *q; - bam_plbuf_reset(buf->plbuf); - for (p = buf->head; p->next;) { - q = p->next; - mp_free(buf->mp, p); - p = q; - } - buf->head = buf->tail; - buf->max_level = 0; - buf->n_cur = buf->n_pre = 0; - buf->n_nodes = 0; -} - -static int tview_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data) -{ - bam_lplbuf_t *tv = (bam_lplbuf_t*)data; - freenode_t *p; - int i, l, max_level; - // allocate memory if necessary - if (tv->max < n) { // enlarge - tv->max = n; - kroundup32(tv->max); - tv->cur_level = (int*)realloc(tv->cur_level, sizeof(int) * tv->max); - tv->pre_level = (int*)realloc(tv->pre_level, sizeof(int) * tv->max); - } - tv->n_cur = n; - // update cnt - for (p = tv->head; p->next; p = p->next) - if (p->cnt > 0) --p->cnt; - // calculate cur_level[] - max_level = 0; - for (i = l = 0; i < n; ++i) { - const bam_pileup1_t *p = pl + i; - if (p->is_head) { - if (tv->head->next && tv->head->cnt == 0) { // then take a free slot - freenode_t *p = tv->head->next; - tv->cur_level[i] = tv->head->level; - mp_free(tv->mp, tv->head); - tv->head = p; - --tv->n_nodes; - } else tv->cur_level[i] = ++tv->max_level; - } else { - tv->cur_level[i] = tv->pre_level[l++]; - if (p->is_tail) { // then return a free slot - tv->tail->level = tv->cur_level[i]; - tv->tail->next = mp_alloc(tv->mp); - tv->tail = tv->tail->next; - ++tv->n_nodes; - } - } - if (tv->cur_level[i] > max_level) max_level = tv->cur_level[i]; - ((bam_pileup1_t*)p)->level = tv->cur_level[i]; - } - assert(l == tv->n_pre); - tv->func(tid, pos, n, pl, tv->user_data); - // sort the linked list - if (tv->n_nodes) { - freenode_t *q; - if (tv->n_nodes + 1 > tv->m_aux) { // enlarge - tv->m_aux = tv->n_nodes + 1; - kroundup32(tv->m_aux); - tv->aux = (freenode_t**)realloc(tv->aux, sizeof(void*) * tv->m_aux); - } - for (p = tv->head, i = l = 0; p->next;) { - if (p->level > max_level) { // then discard this entry - q = p->next; - mp_free(tv->mp, p); - p = q; - } else { - tv->aux[i++] = p; - p = p->next; - } - } - tv->aux[i] = tv->tail; // add a proper tail for the loop below - tv->n_nodes = i; - if (tv->n_nodes) { - ks_introsort(node, tv->n_nodes, tv->aux); - for (i = 0; i < tv->n_nodes; ++i) tv->aux[i]->next = tv->aux[i+1]; - tv->head = tv->aux[0]; - } else tv->head = tv->tail; - } - // clean up - tv->max_level = max_level; - memcpy(tv->pre_level, tv->cur_level, tv->n_cur * 4); - // squeeze out terminated levels - for (i = l = 0; i < n; ++i) { - const bam_pileup1_t *p = pl + i; - if (!p->is_tail) - tv->pre_level[l++] = tv->pre_level[i]; - } - tv->n_pre = l; -/* - fprintf(stderr, "%d\t", pos+1); - for (i = 0; i < n; ++i) { - const bam_pileup1_t *p = pl + i; - if (p->is_head) fprintf(stderr, "^"); - if (p->is_tail) fprintf(stderr, "$"); - fprintf(stderr, "%d,", p->level); - } - fprintf(stderr, "\n"); -*/ - return 0; -} - -bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data) -{ - bam_lplbuf_t *tv; - tv = (bam_lplbuf_t*)calloc(1, sizeof(bam_lplbuf_t)); - tv->mp = mp_init(); - tv->head = tv->tail = mp_alloc(tv->mp); - tv->func = func; - tv->user_data = data; - tv->plbuf = bam_plbuf_init(tview_func, tv); - return (bam_lplbuf_t*)tv; -} - -void bam_lplbuf_destroy(bam_lplbuf_t *tv) -{ - freenode_t *p, *q; - free(tv->cur_level); free(tv->pre_level); - bam_plbuf_destroy(tv->plbuf); - free(tv->aux); - for (p = tv->head; p->next;) { - q = p->next; - mp_free(tv->mp, p); p = q; - } - mp_free(tv->mp, p); - assert(tv->mp->cnt == 0); - mp_destroy(tv->mp); - free(tv); -} - -int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *tv) -{ - return bam_plbuf_push(b, tv->plbuf); -} diff --git a/samtools-0.1.13/bam_maqcns.c b/samtools-0.1.13/bam_maqcns.c deleted file mode 100644 --- a/samtools-0.1.13/bam_maqcns.c +++ /dev/null @@ -1,626 +0,0 @@ -#include -#include -#include "bam.h" -#include "bam_maqcns.h" -#include "ksort.h" -#include "errmod.h" -#include "kaln.h" -KSORT_INIT_GENERIC(uint32_t) - -#define INDEL_WINDOW_SIZE 50 -#define INDEL_EXT_DEP 0.9 - -typedef struct __bmc_aux_t { - int max; - uint32_t *info; - uint16_t *info16; - errmod_t *em; -} bmc_aux_t; - -typedef struct { - float esum[4], fsum[4]; - uint32_t c[4]; -} glf_call_aux_t; - -/* - P() = \theta \sum_{i=1}^{N-1} 1/i - P(D|) = \sum_{k=1}^{N-1} p_k 1/2 [(k/N)^n_2(1-k/N)^n_1 + (k/N)^n1(1-k/N)^n_2] - p_k = 1/k / \sum_{i=1}^{N-1} 1/i - */ -static void cal_het(bam_maqcns_t *aa) -{ - int k, n1, n2; - double sum_harmo; // harmonic sum - double poly_rate; - - free(aa->lhet); - aa->lhet = (double*)calloc(256 * 256, sizeof(double)); - sum_harmo = 0.0; - for (k = 1; k <= aa->n_hap - 1; ++k) - sum_harmo += 1.0 / k; - for (n1 = 0; n1 < 256; ++n1) { - for (n2 = 0; n2 < 256; ++n2) { - long double sum = 0.0; - double lC = aa->errmod == BAM_ERRMOD_SOAP? 0 : lgamma(n1+n2+1) - lgamma(n1+1) - lgamma(n2+1); - for (k = 1; k <= aa->n_hap - 1; ++k) { - double pk = 1.0 / k / sum_harmo; - double log1 = log((double)k/aa->n_hap); - double log2 = log(1.0 - (double)k/aa->n_hap); - sum += pk * 0.5 * (expl(log1*n2) * expl(log2*n1) + expl(log1*n1) * expl(log2*n2)); - } - aa->lhet[n1<<8|n2] = lC + logl(sum); - } - } - poly_rate = aa->het_rate * sum_harmo; - aa->q_r = -4.343 * log(2.0 * poly_rate / (1.0 - poly_rate)); -} - -/** initialize the helper structure */ -static void cal_coef(bam_maqcns_t *aa) -{ - int k, n, q; - long double sum_a[257], b[256], q_c[256], tmp[256], fk2[256]; - double *lC; - - if (aa->errmod == BAM_ERRMOD_MAQ2) return; // no need to do the following - // aa->lhet will be allocated and initialized - free(aa->fk); free(aa->coef); - aa->coef = 0; - aa->fk = (double*)calloc(256, sizeof(double)); - aa->fk[0] = fk2[0] = 1.0; - for (n = 1; n != 256; ++n) { - aa->fk[n] = pow(aa->theta, n) * (1.0 - aa->eta) + aa->eta; - fk2[n] = aa->fk[n>>1]; // this is an approximation, assuming reads equally likely come from both strands - } - if (aa->errmod == BAM_ERRMOD_SOAP) return; - aa->coef = (double*)calloc(256*256*64, sizeof(double)); - lC = (double*)calloc(256 * 256, sizeof(double)); - for (n = 1; n != 256; ++n) - for (k = 1; k <= n; ++k) - lC[n<<8|k] = lgamma(n+1) - lgamma(k+1) - lgamma(n-k+1); - for (q = 1; q != 64; ++q) { - double e = pow(10.0, -q/10.0); - double le = log(e); - double le1 = log(1.0-e); - for (n = 1; n != 256; ++n) { - double *coef = aa->coef + (q<<16|n<<8); - sum_a[n+1] = 0.0; - for (k = n; k >= 0; --k) { // a_k = \sum_{i=k}^n C^n_k \epsilon^k (1-\epsilon)^{n-k} - sum_a[k] = sum_a[k+1] + expl(lC[n<<8|k] + k*le + (n-k)*le1); - b[k] = sum_a[k+1] / sum_a[k]; - if (b[k] > 0.99) b[k] = 0.99; - } - for (k = 0; k != n; ++k) // log(\bar\beta_{nk}(\bar\epsilon)^{f_k}) - q_c[k] = -4.343 * fk2[k] * logl(b[k] / e); - for (k = 1; k != n; ++k) q_c[k] += q_c[k-1]; // \prod_{i=0}^k c_i - for (k = 0; k <= n; ++k) { // powl() in 64-bit mode seems broken on my Mac OS X 10.4.9 - tmp[k] = -4.343 * logl(1.0 - expl(fk2[k] * logl(b[k]))); - coef[k] = (k? q_c[k-1] : 0) + tmp[k]; // this is the final c_{nk} - } - } - } - free(lC); -} - -bam_maqcns_t *bam_maqcns_init() -{ - bam_maqcns_t *bm; - bm = (bam_maqcns_t*)calloc(1, sizeof(bam_maqcns_t)); - bm->aux = (bmc_aux_t*)calloc(1, sizeof(bmc_aux_t)); - bm->het_rate = 0.001; - bm->theta = 0.83f; - bm->n_hap = 2; - bm->eta = 0.03; - bm->cap_mapQ = 60; - bm->min_baseQ = 13; - return bm; -} - -void bam_maqcns_prepare(bam_maqcns_t *bm) -{ - if (bm->errmod == BAM_ERRMOD_MAQ2) bm->aux->em = errmod_init(1. - bm->theta); - cal_coef(bm); cal_het(bm); -} - -void bam_maqcns_destroy(bam_maqcns_t *bm) -{ - if (bm == 0) return; - free(bm->lhet); free(bm->fk); free(bm->coef); free(bm->aux->info); free(bm->aux->info16); - if (bm->aux->em) errmod_destroy(bm->aux->em); - free(bm->aux); free(bm); -} - -glf1_t *bam_maqcns_glfgen(int _n, const bam_pileup1_t *pl, uint8_t ref_base, bam_maqcns_t *bm) -{ - glf_call_aux_t *b = 0; - int i, j, k, w[8], c, n; - glf1_t *g = (glf1_t*)calloc(1, sizeof(glf1_t)); - float p[16], min_p = 1e30; - uint64_t rms; - - g->ref_base = ref_base; - if (_n == 0) return g; - - // construct aux array - if (bm->aux->max < _n) { - bm->aux->max = _n; - kroundup32(bm->aux->max); - bm->aux->info = (uint32_t*)realloc(bm->aux->info, 4 * bm->aux->max); - bm->aux->info16 = (uint16_t*)realloc(bm->aux->info16, 2 * bm->aux->max); - } - for (i = n = 0, rms = 0; i < _n; ++i) { - const bam_pileup1_t *p = pl + i; - uint32_t q, x = 0, qq; - uint16_t y = 0; - if (p->is_del || p->is_refskip || (p->b->core.flag&BAM_FUNMAP)) continue; - q = (uint32_t)bam1_qual(p->b)[p->qpos]; - if (q < bm->min_baseQ) continue; - x |= (uint32_t)bam1_strand(p->b) << 18 | q << 8 | p->b->core.qual; - y |= bam1_strand(p->b)<<4; - if (p->b->core.qual < q) q = p->b->core.qual; - c = p->b->core.qual < bm->cap_mapQ? p->b->core.qual : bm->cap_mapQ; - rms += c * c; - x |= q << 24; - y |= q << 5; - qq = bam1_seqi(bam1_seq(p->b), p->qpos); - q = bam_nt16_nt4_table[qq? qq : ref_base]; - if (!p->is_del && !p->is_refskip && q < 4) x |= 1 << 21 | q << 16, y |= q; - bm->aux->info16[n] = y; - bm->aux->info[n++] = x; - } - rms = (uint8_t)(sqrt((double)rms / n) + .499); - if (bm->errmod == BAM_ERRMOD_MAQ2) { - errmod_cal(bm->aux->em, n, 4, bm->aux->info16, p); - goto goto_glf; - } - ks_introsort(uint32_t, n, bm->aux->info); - // generate esum and fsum - b = (glf_call_aux_t*)calloc(1, sizeof(glf_call_aux_t)); - for (k = 0; k != 8; ++k) w[k] = 0; - for (j = n - 1; j >= 0; --j) { // calculate esum and fsum - uint32_t info = bm->aux->info[j]; - if (info>>24 < 4 && (info>>8&0x3f) != 0) info = 4<<24 | (info&0xffffff); - k = info>>16&7; - if (info>>24 > 0) { - b->esum[k&3] += bm->fk[w[k]] * (info>>24); - b->fsum[k&3] += bm->fk[w[k]]; - if (w[k] < 0xff) ++w[k]; - ++b->c[k&3]; - } - } - // rescale ->c[] - for (j = c = 0; j != 4; ++j) c += b->c[j]; - if (c > 255) { - for (j = 0; j != 4; ++j) b->c[j] = (int)(254.0 * b->c[j] / c + 0.5); - for (j = c = 0; j != 4; ++j) c += b->c[j]; - } - if (bm->errmod == BAM_ERRMOD_MAQ) { - // generate likelihood - for (j = 0; j != 4; ++j) { - // homozygous - float tmp1, tmp3; - int tmp2, bar_e; - for (k = 0, tmp1 = tmp3 = 0.0, tmp2 = 0; k != 4; ++k) { - if (j == k) continue; - tmp1 += b->esum[k]; tmp2 += b->c[k]; tmp3 += b->fsum[k]; - } - if (tmp2) { - bar_e = (int)(tmp1 / tmp3 + 0.5); - if (bar_e < 4) bar_e = 4; // should not happen - if (bar_e > 63) bar_e = 63; - p[j<<2|j] = tmp1 + bm->coef[bar_e<<16|c<<8|tmp2]; - } else p[j<<2|j] = 0.0; // all the bases are j - // heterozygous - for (k = j + 1; k < 4; ++k) { - for (i = 0, tmp2 = 0, tmp1 = tmp3 = 0.0; i != 4; ++i) { - if (i == j || i == k) continue; - tmp1 += b->esum[i]; tmp2 += b->c[i]; tmp3 += b->fsum[i]; - } - if (tmp2) { - bar_e = (int)(tmp1 / tmp3 + 0.5); - if (bar_e < 4) bar_e = 4; - if (bar_e > 63) bar_e = 63; - p[j<<2|k] = p[k<<2|j] = -4.343 * bm->lhet[b->c[j]<<8|b->c[k]] + tmp1 + bm->coef[bar_e<<16|c<<8|tmp2]; - } else p[j<<2|k] = p[k<<2|j] = -4.343 * bm->lhet[b->c[j]<<8|b->c[k]]; // all the bases are either j or k - } - // - for (k = 0; k != 4; ++k) - if (p[j<<2|k] < 0.0) p[j<<2|k] = 0.0; - } - - { // fix p[k<<2|k] - float max1, max2, min1, min2; - int max_k, min_k; - max_k = min_k = -1; - max1 = max2 = -1.0; min1 = min2 = 1e30; - for (k = 0; k < 4; ++k) { - if (b->esum[k] > max1) { - max2 = max1; max1 = b->esum[k]; max_k = k; - } else if (b->esum[k] > max2) max2 = b->esum[k]; - } - for (k = 0; k < 4; ++k) { - if (p[k<<2|k] < min1) { - min2 = min1; min1 = p[k<<2|k]; min_k = k; - } else if (p[k<<2|k] < min2) min2 = p[k<<2|k]; - } - if (max1 > max2 && (min_k != max_k || min1 + 1.0 > min2)) - p[max_k<<2|max_k] = min1 > 1.0? min1 - 1.0 : 0.0; - } - } else if (bm->errmod == BAM_ERRMOD_SOAP) { // apply the SOAP model - // generate likelihood - for (j = 0; j != 4; ++j) { - float tmp; - // homozygous - for (k = 0, tmp = 0.0; k != 4; ++k) - if (j != k) tmp += b->esum[k]; - p[j<<2|j] = tmp; - // heterozygous - for (k = j + 1; k < 4; ++k) { - for (i = 0, tmp = 0.0; i != 4; ++i) - if (i != j && i != k) tmp += b->esum[i]; - p[j<<2|k] = p[k<<2|j] = -4.343 * bm->lhet[b->c[j]<<8|b->c[k]] + tmp; - } - } - } - -goto_glf: - // convert necessary information to glf1_t - g->ref_base = ref_base; g->max_mapQ = rms; - g->depth = n > 16777215? 16777215 : n; - for (j = 0; j != 4; ++j) - for (k = j; k < 4; ++k) - if (p[j<<2|k] < min_p) min_p = p[j<<2|k]; - g->min_lk = min_p > 255.0? 255 : (int)(min_p + 0.5); - for (j = c = 0; j != 4; ++j) - for (k = j; k < 4; ++k) - g->lk[c++] = p[j<<2|k]-min_p > 255.0? 255 : (int)(p[j<<2|k]-min_p + 0.5); - - free(b); - return g; -} - -uint32_t glf2cns(const glf1_t *g, int q_r) -{ - int i, j, k, p[10], ref4; - uint32_t x = 0; - ref4 = bam_nt16_nt4_table[g->ref_base]; - for (i = k = 0; i < 4; ++i) - for (j = i; j < 4; ++j) { - int prior = (i == ref4 && j == ref4? 0 : i == ref4 || j == ref4? q_r : q_r + 3); - p[k] = (g->lk[k] + prior)<<4 | i<<2 | j; - ++k; - } - for (i = 1; i < 10; ++i) // insertion sort - for (j = i; j > 0 && p[j] < p[j-1]; --j) - k = p[j], p[j] = p[j-1], p[j-1] = k; - x = (1u<<(p[0]&3) | 1u<<(p[0]>>2&3)) << 28; // the best genotype - x |= (uint32_t)g->max_mapQ << 16; // rms mapQ - x |= ((p[1]>>4) - (p[0]>>4) < 256? (p[1]>>4) - (p[0]>>4) : 255) << 8; // consensus Q - for (k = 0; k < 10; ++k) - if ((p[k]&0xf) == (ref4<<2|ref4)) break; - if (k == 10) k = 9; - x |= (p[k]>>4) - (p[0]>>4) < 256? (p[k]>>4) - (p[0]>>4) : 255; // snp Q - return x; -} - -uint32_t bam_maqcns_call(int n, const bam_pileup1_t *pl, bam_maqcns_t *bm) -{ - glf1_t *g; - uint32_t x; - if (n) { - g = bam_maqcns_glfgen(n, pl, 0xf, bm); - x = g->depth == 0? (0xfU<<28 | 0xfU<<24) : glf2cns(g, (int)(bm->q_r + 0.5)); - free(g); - } else x = 0xfU<<28 | 0xfU<<24; - return x; -} - -/************** *****************/ - -bam_maqindel_opt_t *bam_maqindel_opt_init() -{ - bam_maqindel_opt_t *mi = (bam_maqindel_opt_t*)calloc(1, sizeof(bam_maqindel_opt_t)); - mi->q_indel = 40; - mi->r_indel = 0.00015; - mi->r_snp = 0.001; - // - mi->mm_penalty = 3; - mi->indel_err = 4; - mi->ambi_thres = 10; - return mi; -} - -void bam_maqindel_ret_destroy(bam_maqindel_ret_t *mir) -{ - if (mir == 0) return; - free(mir->s[0]); free(mir->s[1]); free(mir); -} - -int bam_tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, int32_t tpos, int is_left, int32_t *_tpos) -{ - int k, x = c->pos, y = 0, last_y = 0; - *_tpos = c->pos; - for (k = 0; k < c->n_cigar; ++k) { - int op = cigar[k] & BAM_CIGAR_MASK; - int l = cigar[k] >> BAM_CIGAR_SHIFT; - if (op == BAM_CMATCH) { - if (c->pos > tpos) return y; - if (x + l > tpos) { - *_tpos = tpos; - return y + (tpos - x); - } - x += l; y += l; - last_y = y; - } else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l; - else if (op == BAM_CDEL || op == BAM_CREF_SKIP) { - if (x + l > tpos) { - *_tpos = is_left? x : x + l; - return y; - } - x += l; - } - } - *_tpos = x; - return last_y; -} - -#define MINUS_CONST 0x10000000 - -bam_maqindel_ret_t *bam_maqindel(int n, int pos, const bam_maqindel_opt_t *mi, const bam_pileup1_t *pl, const char *ref, - int _n_types, int *_types) -{ - int i, j, n_types, *types, left, right, max_rd_len = 0; - bam_maqindel_ret_t *ret = 0; - // if there is no proposed indel, check if there is an indel from the alignment - if (_n_types == 0) { - for (i = 0; i < n; ++i) { - const bam_pileup1_t *p = pl + i; - if (!(p->b->core.flag&BAM_FUNMAP) && p->indel != 0) break; - } - if (i == n) return 0; // no indel - } - { // calculate how many types of indels are available (set n_types and types) - int m; - uint32_t *aux; - aux = (uint32_t*)calloc(n + _n_types + 1, 4); - m = 0; - aux[m++] = MINUS_CONST; // zero indel is always a type - for (i = 0; i < n; ++i) { - const bam_pileup1_t *p = pl + i; - if (!(p->b->core.flag&BAM_FUNMAP) && p->indel != 0) - aux[m++] = MINUS_CONST + p->indel; - j = bam_cigar2qlen(&p->b->core, bam1_cigar(p->b)); - if (j > max_rd_len) max_rd_len = j; - } - if (_n_types) // then also add this to aux[] - for (i = 0; i < _n_types; ++i) - if (_types[i]) aux[m++] = MINUS_CONST + _types[i]; - ks_introsort(uint32_t, m, aux); - // squeeze out identical types - for (i = 1, n_types = 1; i < m; ++i) - if (aux[i] != aux[i-1]) ++n_types; - types = (int*)calloc(n_types, sizeof(int)); - j = 0; - types[j++] = aux[0] - MINUS_CONST; - for (i = 1; i < m; ++i) { - if (aux[i] != aux[i-1]) - types[j++] = aux[i] - MINUS_CONST; - } - free(aux); - } - { // calculate left and right boundary - left = pos > INDEL_WINDOW_SIZE? pos - INDEL_WINDOW_SIZE : 0; - right = pos + INDEL_WINDOW_SIZE; - if (types[0] < 0) right -= types[0]; - // in case the alignments stand out the reference - for (i = pos; i < right; ++i) - if (ref[i] == 0) break; - right = i; - } - { // the core part - char *ref2, *rs, *inscns = 0; - int qr_snp, k, l, *score, *pscore, max_ins = types[n_types-1]; - qr_snp = (int)(-4.343 * log(mi->r_snp) + .499); - if (max_ins > 0) { // get the consensus of inserted sequences - int *inscns_aux = (int*)calloc(4 * n_types * max_ins, sizeof(int)); - // count occurrences - for (i = 0; i < n_types; ++i) { - if (types[i] <= 0) continue; // not insertion - for (j = 0; j < n; ++j) { - const bam_pileup1_t *p = pl + j; - if (!(p->b->core.flag&BAM_FUNMAP) && p->indel == types[i]) { - for (k = 1; k <= p->indel; ++k) { - int c = bam_nt16_nt4_table[bam1_seqi(bam1_seq(p->b), p->qpos + k)]; - if (c < 4) ++inscns_aux[i*max_ins*4 + (k-1)*4 + c]; - } - } - } - } - // construct the consensus of inserted sequence - inscns = (char*)calloc(n_types * max_ins, sizeof(char)); - for (i = 0; i < n_types; ++i) { - for (j = 0; j < types[i]; ++j) { - int max = 0, max_k = -1, *ia = inscns_aux + i*max_ins*4 + j*4; - for (k = 0; k < 4; ++k) { - if (ia[k] > max) { - max = ia[k]; - max_k = k; - } - } - inscns[i*max_ins + j] = max? 1<= 0? -types[0] : -types[0] + types[i]; - for (jj = 0; jj < tmp && j < right && ref[j]; ++jj, ++j) - ref2[k++] = 4; - } - for (; j < right && ref[j]; ++j) - ref2[k++] = bam_nt16_nt4_table[bam_nt16_table[(int)ref[j]]]; - if (j < right) right = j; - // calculate score for each read - for (j = 0; j < n; ++j) { - const bam_pileup1_t *p = pl + j; - int qbeg, qend, tbeg, tend; - if (p->b->core.flag & BAM_FUNMAP) continue; - qbeg = bam_tpos2qpos(&p->b->core, bam1_cigar(p->b), left, 0, &tbeg); - qend = bam_tpos2qpos(&p->b->core, bam1_cigar(p->b), right, 1, &tend); - assert(tbeg >= left); - for (l = qbeg; l < qend; ++l) - rs[l - qbeg] = bam_nt16_nt4_table[bam1_seqi(bam1_seq(p->b), l)]; - { - int x, y, n_acigar, ps; - uint32_t *acigar; - ps = 0; - if (tend - tbeg + types[i] <= 0) { - score[i*n+j] = -(1<<20); - pscore[i*n+j] = 1<<20; - continue; - } - acigar = ka_global_core((uint8_t*)ref2 + tbeg - left, tend - tbeg + types[i], (uint8_t*)rs, qend - qbeg, &ap, &score[i*n+j], &n_acigar); - x = tbeg - left; y = 0; - for (l = 0; l < n_acigar; ++l) { - int op = acigar[l]&0xf; - int len = acigar[l]>>4; - if (op == BAM_CMATCH) { - int k; - for (k = 0; k < len; ++k) - if (ref2[x+k] != rs[y+k] && ref2[x+k] < 4) - ps += bam1_qual(p->b)[y+k] < qr_snp? bam1_qual(p->b)[y+k] : qr_snp; - x += len; y += len; - } else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) { - if (op == BAM_CINS && l > 0 && l < n_acigar - 1) ps += mi->q_indel * len; - y += len; - } else if (op == BAM_CDEL) { - if (l > 0 && l < n_acigar - 1) ps += mi->q_indel * len; - x += len; - } - } - pscore[i*n+j] = ps; - /*if (1) { // for debugging only - fprintf(stderr, "id=%d, pos=%d, type=%d, j=%d, score=%d, psore=%d, %d, %d, %d, %d, %d, ", - j, pos+1, types[i], j, score[i*n+j], pscore[i*n+j], tbeg, tend, qbeg, qend, mi->q_indel); - for (l = 0; l < n_acigar; ++l) fprintf(stderr, "%d%c", acigar[l]>>4, "MIDS"[acigar[l]&0xf]); - fprintf(stderr, "\n"); - for (l = 0; l < tend - tbeg + types[i]; ++l) fputc("ACGTN"[ref2[l+tbeg-left]], stderr); - fputc('\n', stderr); - for (l = 0; l < qend - qbeg; ++l) fputc("ACGTN"[rs[l]], stderr); - fputc('\n', stderr); - }*/ - free(acigar); - } - } - } - { // get final result - int *sum, max1, max2, max1_i, max2_i; - // pick up the best two score - sum = (int*)calloc(n_types, sizeof(int)); - for (i = 0; i < n_types; ++i) - for (j = 0; j < n; ++j) - sum[i] += -pscore[i*n+j]; - max1 = max2 = -0x7fffffff; max1_i = max2_i = -1; - for (i = 0; i < n_types; ++i) { - if (sum[i] > max1) { - max2 = max1; max2_i = max1_i; max1 = sum[i]; max1_i = i; - } else if (sum[i] > max2) { - max2 = sum[i]; max2_i = i; - } - } - free(sum); - // write ret - ret = (bam_maqindel_ret_t*)calloc(1, sizeof(bam_maqindel_ret_t)); - ret->indel1 = types[max1_i]; ret->indel2 = types[max2_i]; - ret->s[0] = (char*)calloc(abs(ret->indel1) + 2, 1); - ret->s[1] = (char*)calloc(abs(ret->indel2) + 2, 1); - // write indel sequence - if (ret->indel1 > 0) { - ret->s[0][0] = '+'; - for (k = 0; k < ret->indel1; ++k) - ret->s[0][k+1] = bam_nt16_rev_table[(int)inscns[max1_i*max_ins + k]]; - } else if (ret->indel1 < 0) { - ret->s[0][0] = '-'; - for (k = 0; k < -ret->indel1 && ref[pos + k + 1]; ++k) - ret->s[0][k+1] = ref[pos + k + 1]; - } else ret->s[0][0] = '*'; - if (ret->indel2 > 0) { - ret->s[1][0] = '+'; - for (k = 0; k < ret->indel2; ++k) - ret->s[1][k+1] = bam_nt16_rev_table[(int)inscns[max2_i*max_ins + k]]; - } else if (ret->indel2 < 0) { - ret->s[1][0] = '-'; - for (k = 0; k < -ret->indel2 && ref[pos + k + 1]; ++k) - ret->s[1][k+1] = ref[pos + k + 1]; - } else ret->s[1][0] = '*'; - // write count - for (i = 0; i < n; ++i) { - const bam_pileup1_t *p = pl + i; - if (p->indel == ret->indel1) ++ret->cnt1; - else if (p->indel == ret->indel2) ++ret->cnt2; - else ++ret->cnt_anti; - } - { // write gl[] - int tmp, seq_err = 0; - double x = 1.0; - tmp = max1_i - max2_i; - if (tmp < 0) tmp = -tmp; - for (j = 0; j < tmp + 1; ++j) x *= INDEL_EXT_DEP; - seq_err = mi->q_indel * (1.0 - x) / (1.0 - INDEL_EXT_DEP); - ret->gl[0] = ret->gl[1] = 0; - for (j = 0; j < n; ++j) { - int s1 = pscore[max1_i*n + j], s2 = pscore[max2_i*n + j]; - //fprintf(stderr, "id=%d, %d, %d, %d, %d, %d\n", j, pl[j].b->core.pos+1, types[max1_i], types[max2_i], s1, s2); - if (s1 > s2) ret->gl[0] += s1 - s2 < seq_err? s1 - s2 : seq_err; - else ret->gl[1] += s2 - s1 < seq_err? s2 - s1 : seq_err; - } - } - // write cnt_ref and cnt_ambi - if (max1_i != 0 && max2_i != 0) { - for (j = 0; j < n; ++j) { - int diff1 = score[j] - score[max1_i * n + j]; - int diff2 = score[j] - score[max2_i * n + j]; - if (diff1 > 0 && diff2 > 0) ++ret->cnt_ref; - else if (diff1 == 0 || diff2 == 0) ++ret->cnt_ambi; - } - } - } - free(score); free(pscore); free(ref2); free(rs); free(inscns); - } - { // call genotype - int q[3], qr_indel = (int)(-4.343 * log(mi->r_indel) + 0.5); - int min1, min2, min1_i; - q[0] = ret->gl[0] + (ret->s[0][0] != '*'? 0 : 0) * qr_indel; - q[1] = ret->gl[1] + (ret->s[1][0] != '*'? 0 : 0) * qr_indel; - q[2] = n * 3 + (ret->s[0][0] == '*' || ret->s[1][0] == '*'? 1 : 1) * qr_indel; - min1 = min2 = 0x7fffffff; min1_i = -1; - for (i = 0; i < 3; ++i) { - if (q[i] < min1) { - min2 = min1; min1 = q[i]; min1_i = i; - } else if (q[i] < min2) min2 = q[i]; - } - ret->gt = min1_i; - ret->q_cns = min2 - min1; - // set q_ref - if (ret->gt < 2) ret->q_ref = (ret->s[ret->gt][0] == '*')? 0 : q[1-ret->gt] - q[ret->gt] - qr_indel - 3; - else ret->q_ref = (ret->s[0][0] == '*')? q[0] - q[2] : q[1] - q[2]; - if (ret->q_ref < 0) ret->q_ref = 0; - } - free(types); - return ret; -} diff --git a/samtools-0.1.13/bam_maqcns.h b/samtools-0.1.13/bam_maqcns.h deleted file mode 100644 --- a/samtools-0.1.13/bam_maqcns.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef BAM_MAQCNS_H -#define BAM_MAQCNS_H - -#include "glf.h" - -#define BAM_ERRMOD_MAQ2 0 -#define BAM_ERRMOD_MAQ 1 -#define BAM_ERRMOD_SOAP 2 - -struct __bmc_aux_t; - -typedef struct { - float het_rate, theta; - int n_hap, cap_mapQ, errmod, min_baseQ; - - float eta, q_r; - double *fk, *coef; - double *lhet; - struct __bmc_aux_t *aux; -} bam_maqcns_t; - -typedef struct { - int q_indel; // indel sequencing error, phred scaled - float r_indel; // indel prior - float r_snp; // snp prior - // hidden parameters, unchangeable from command line - int mm_penalty, indel_err, ambi_thres; -} bam_maqindel_opt_t; - -typedef struct { - int indel1, indel2; - int cnt1, cnt2, cnt_anti; - int cnt_ref, cnt_ambi; - char *s[2]; - // - int gt, gl[2]; - int q_cns, q_ref; -} bam_maqindel_ret_t; - -#ifdef __cplusplus -extern "C" { -#endif - - bam_maqcns_t *bam_maqcns_init(); - void bam_maqcns_prepare(bam_maqcns_t *bm); - void bam_maqcns_destroy(bam_maqcns_t *bm); - glf1_t *bam_maqcns_glfgen(int n, const bam_pileup1_t *pl, uint8_t ref_base, bam_maqcns_t *bm); - uint32_t bam_maqcns_call(int n, const bam_pileup1_t *pl, bam_maqcns_t *bm); - // return: cns<<28 | cns2<<24 | mapQ<<16 | cnsQ<<8 | cnsQ2 - uint32_t glf2cns(const glf1_t *g, int q_r); - - bam_maqindel_opt_t *bam_maqindel_opt_init(); - bam_maqindel_ret_t *bam_maqindel(int n, int pos, const bam_maqindel_opt_t *mi, const bam_pileup1_t *pl, const char *ref, - int _n_types, int *_types); - void bam_maqindel_ret_destroy(bam_maqindel_ret_t*); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.13/bam_md.c b/samtools-0.1.13/bam_md.c deleted file mode 100644 --- a/samtools-0.1.13/bam_md.c +++ /dev/null @@ -1,357 +0,0 @@ -#include -#include -#include -#include -#include -#include "faidx.h" -#include "sam.h" -#include "kstring.h" -#include "kaln.h" -#include "kprobaln.h" - -char bam_nt16_nt4_table[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 }; - -void bam_fillmd1_core(bam1_t *b, char *ref, int is_equal, int max_nm) -{ - uint8_t *seq = bam1_seq(b); - uint32_t *cigar = bam1_cigar(b); - bam1_core_t *c = &b->core; - int i, x, y, u = 0; - kstring_t *str; - uint8_t *old_md, *old_nm; - int32_t old_nm_i = -1, nm = 0; - - str = (kstring_t*)calloc(1, sizeof(kstring_t)); - for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { - int j, l = cigar[i]>>4, op = cigar[i]&0xf; - if (op == BAM_CMATCH) { - for (j = 0; j < l; ++j) { - int z = y + j; - int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; - if (ref[x+j] == 0) break; // out of boundary - if ((c1 == c2 && c1 != 15 && c2 != 15) || c1 == 0) { // a match - if (is_equal) seq[z/2] &= (z&1)? 0xf0 : 0x0f; - ++u; - } else { - ksprintf(str, "%d", u); - kputc(ref[x+j], str); - u = 0; ++nm; - } - } - if (j < l) break; - x += l; y += l; - } else if (op == BAM_CDEL) { - ksprintf(str, "%d", u); - kputc('^', str); - for (j = 0; j < l; ++j) { - if (ref[x+j] == 0) break; - kputc(ref[x+j], str); - } - u = 0; - if (j < l) break; - x += l; nm += l; - } else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) { - y += l; - if (op == BAM_CINS) nm += l; - } else if (op == BAM_CREF_SKIP) { - x += l; - } - } - ksprintf(str, "%d", u); - // apply max_nm - if (max_nm > 0 && nm >= max_nm) { - for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { - int j, l = cigar[i]>>4, op = cigar[i]&0xf; - if (op == BAM_CMATCH) { - for (j = 0; j < l; ++j) { - int z = y + j; - int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; - if (ref[x+j] == 0) break; // out of boundary - if ((c1 == c2 && c1 != 15 && c2 != 15) || c1 == 0) { // a match - seq[z/2] |= (z&1)? 0x0f : 0xf0; - bam1_qual(b)[z] = 0; - } - } - if (j < l) break; - x += l; y += l; - } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) x += l; - else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l; - } - } - // update NM - old_nm = bam_aux_get(b, "NM"); - if (c->flag & BAM_FUNMAP) return; - if (old_nm) old_nm_i = bam_aux2i(old_nm); - if (!old_nm) bam_aux_append(b, "NM", 'i', 4, (uint8_t*)&nm); - else if (nm != old_nm_i) { - fprintf(stderr, "[bam_fillmd1] different NM for read '%s': %d -> %d\n", bam1_qname(b), old_nm_i, nm); - bam_aux_del(b, old_nm); - bam_aux_append(b, "NM", 'i', 4, (uint8_t*)&nm); - } - // update MD - old_md = bam_aux_get(b, "MD"); - if (!old_md) bam_aux_append(b, "MD", 'Z', str->l + 1, (uint8_t*)str->s); - else { - int is_diff = 0; - if (strlen((char*)old_md+1) == str->l) { - for (i = 0; i < str->l; ++i) - if (toupper(old_md[i+1]) != toupper(str->s[i])) - break; - if (i < str->l) is_diff = 1; - } else is_diff = 1; - if (is_diff) { - fprintf(stderr, "[bam_fillmd1] different MD for read '%s': '%s' -> '%s'\n", bam1_qname(b), old_md+1, str->s); - bam_aux_del(b, old_md); - bam_aux_append(b, "MD", 'Z', str->l + 1, (uint8_t*)str->s); - } - } - free(str->s); free(str); -} - -void bam_fillmd1(bam1_t *b, char *ref, int is_equal) -{ - bam_fillmd1_core(b, ref, is_equal, 0); -} - -int bam_cap_mapQ(bam1_t *b, char *ref, int thres) -{ - uint8_t *seq = bam1_seq(b), *qual = bam1_qual(b); - uint32_t *cigar = bam1_cigar(b); - bam1_core_t *c = &b->core; - int i, x, y, mm, q, len, clip_l, clip_q; - double t; - if (thres < 0) thres = 40; // set the default - mm = q = len = clip_l = clip_q = 0; - for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { - int j, l = cigar[i]>>4, op = cigar[i]&0xf; - if (op == BAM_CMATCH) { - for (j = 0; j < l; ++j) { - int z = y + j; - int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; - if (ref[x+j] == 0) break; // out of boundary - if (c2 != 15 && c1 != 15 && qual[z] >= 13) { // not ambiguous - ++len; - if (c1 && c1 != c2 && qual[z] >= 13) { // mismatch - ++mm; - q += qual[z] > 33? 33 : qual[z]; - } - } - } - if (j < l) break; - x += l; y += l; len += l; - } else if (op == BAM_CDEL) { - for (j = 0; j < l; ++j) - if (ref[x+j] == 0) break; - if (j < l) break; - x += l; - } else if (op == BAM_CSOFT_CLIP) { - for (j = 0; j < l; ++j) clip_q += qual[y+j]; - clip_l += l; - y += l; - } else if (op == BAM_CHARD_CLIP) { - clip_q += 13 * l; - clip_l += l; - } else if (op == BAM_CINS) y += l; - else if (op == BAM_CREF_SKIP) x += l; - } - for (i = 0, t = 1; i < mm; ++i) - t *= (double)len / (i+1); - t = q - 4.343 * log(t) + clip_q / 5.; - if (t > thres) return -1; - if (t < 0) t = 0; - t = sqrt((thres - t) / thres) * thres; -// fprintf(stderr, "%s %lf %d\n", bam1_qname(b), t, q); - return (int)(t + .499); -} - -int bam_prob_realn_core(bam1_t *b, const char *ref, int flag) -{ - int k, i, bw, x, y, yb, ye, xb, xe, apply_baq = flag&1, extend_baq = flag>>1&1; - uint32_t *cigar = bam1_cigar(b); - bam1_core_t *c = &b->core; - kpa_par_t conf = kpa_par_def; - uint8_t *bq = 0, *zq = 0, *qual = bam1_qual(b); - if ((c->flag & BAM_FUNMAP) || b->core.l_qseq == 0) return -1; // do nothing - // test if BQ or ZQ is present - if ((bq = bam_aux_get(b, "BQ")) != 0) ++bq; - if ((zq = bam_aux_get(b, "ZQ")) != 0 && *zq == 'Z') ++zq; - if (bq && zq) { // remove the ZQ tag - bam_aux_del(b, zq-1); - zq = 0; - } - if (bq || zq) { - if ((apply_baq && zq) || (!apply_baq && bq)) return -3; // in both cases, do nothing - if (bq && apply_baq) { // then convert BQ to ZQ - for (i = 0; i < c->l_qseq; ++i) - qual[i] = qual[i] + 64 < bq[i]? 0 : qual[i] - ((int)bq[i] - 64); - *(bq - 3) = 'Z'; - } else if (zq && !apply_baq) { // then convert ZQ to BQ - for (i = 0; i < c->l_qseq; ++i) - qual[i] += (int)zq[i] - 64; - *(zq - 3) = 'B'; - } - return 0; - } - // find the start and end of the alignment - x = c->pos, y = 0, yb = ye = xb = xe = -1; - for (k = 0; k < c->n_cigar; ++k) { - int op, l; - op = cigar[k]&0xf; l = cigar[k]>>4; - if (op == BAM_CMATCH) { - if (yb < 0) yb = y; - if (xb < 0) xb = x; - ye = y + l; xe = x + l; - x += l; y += l; - } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l; - else if (op == BAM_CDEL) x += l; - else if (op == BAM_CREF_SKIP) return -1; // do nothing if there is a reference skip - } - // set bandwidth and the start and the end - bw = 7; - if (abs((xe - xb) - (ye - yb)) > bw) - bw = abs((xe - xb) - (ye - yb)) + 3; - conf.bw = bw; - xb -= yb + bw/2; if (xb < 0) xb = 0; - xe += c->l_qseq - ye + bw/2; - if (xe - xb - c->l_qseq > bw) - xb += (xe - xb - c->l_qseq - bw) / 2, xe -= (xe - xb - c->l_qseq - bw) / 2; - { // glocal - uint8_t *s, *r, *q, *seq = bam1_seq(b), *bq; - int *state; - bq = calloc(c->l_qseq + 1, 1); - memcpy(bq, qual, c->l_qseq); - s = calloc(c->l_qseq, 1); - for (i = 0; i < c->l_qseq; ++i) s[i] = bam_nt16_nt4_table[bam1_seqi(seq, i)]; - r = calloc(xe - xb, 1); - for (i = xb; i < xe; ++i) { - if (ref[i] == 0) { xe = i; break; } - r[i-xb] = bam_nt16_nt4_table[bam_nt16_table[(int)ref[i]]]; - } - state = calloc(c->l_qseq, sizeof(int)); - q = calloc(c->l_qseq, 1); - kpa_glocal(r, xe-xb, s, c->l_qseq, qual, &conf, state, q); - if (!extend_baq) { // in this block, bq[] is capped by base quality qual[] - for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) { - int op = cigar[k]&0xf, l = cigar[k]>>4; - if (op == BAM_CMATCH) { - for (i = y; i < y + l; ++i) { - if ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y)) bq[i] = 0; - else bq[i] = bq[i] < q[i]? bq[i] : q[i]; - } - x += l; y += l; - } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l; - else if (op == BAM_CDEL) x += l; - } - for (i = 0; i < c->l_qseq; ++i) bq[i] = qual[i] - bq[i] + 64; // finalize BQ - } else { // in this block, bq[] is BAQ that can be larger than qual[] (different from the above!) - uint8_t *left, *rght; - left = calloc(c->l_qseq, 1); rght = calloc(c->l_qseq, 1); - for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) { - int op = cigar[k]&0xf, l = cigar[k]>>4; - if (op == BAM_CMATCH) { - for (i = y; i < y + l; ++i) - bq[i] = ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y))? 0 : q[i]; - for (left[y] = bq[y], i = y + 1; i < y + l; ++i) - left[i] = bq[i] > left[i-1]? bq[i] : left[i-1]; - for (rght[y+l-1] = bq[y+l-1], i = y + l - 2; i >= y; --i) - rght[i] = bq[i] > rght[i+1]? bq[i] : rght[i+1]; - for (i = y; i < y + l; ++i) - bq[i] = left[i] < rght[i]? left[i] : rght[i]; - x += l; y += l; - } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l; - else if (op == BAM_CDEL) x += l; - } - for (i = 0; i < c->l_qseq; ++i) bq[i] = 64 + (qual[i] <= bq[i]? 0 : qual[i] - bq[i]); // finalize BQ - free(left); free(rght); - } - if (apply_baq) { - for (i = 0; i < c->l_qseq; ++i) qual[i] -= bq[i] - 64; // modify qual - bam_aux_append(b, "ZQ", 'Z', c->l_qseq + 1, bq); - } else bam_aux_append(b, "BQ", 'Z', c->l_qseq + 1, bq); - free(bq); free(s); free(r); free(q); free(state); - } - return 0; -} - -int bam_prob_realn(bam1_t *b, const char *ref) -{ - return bam_prob_realn_core(b, ref, 1); -} - -int bam_fillmd(int argc, char *argv[]) -{ - int c, is_equal, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm, is_realn, capQ, baq_flag; - samfile_t *fp, *fpout = 0; - faidx_t *fai; - char *ref = 0, mode_w[8], mode_r[8]; - bam1_t *b; - - is_equal = is_bam_out = is_sam_in = is_uncompressed = is_realn = max_nm = capQ = baq_flag = 0; - mode_w[0] = mode_r[0] = 0; - strcpy(mode_r, "r"); strcpy(mode_w, "w"); - while ((c = getopt(argc, argv, "EreubSC:n:A")) >= 0) { - switch (c) { - case 'r': is_realn = 1; break; - case 'e': is_equal = 1; break; - case 'b': is_bam_out = 1; break; - case 'u': is_uncompressed = is_bam_out = 1; break; - case 'S': is_sam_in = 1; break; - case 'n': max_nm = atoi(optarg); break; - case 'C': capQ = atoi(optarg); break; - case 'A': baq_flag |= 1; break; - case 'E': baq_flag |= 2; break; - default: fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n", c); return 1; - } - } - if (!is_sam_in) strcat(mode_r, "b"); - if (is_bam_out) strcat(mode_w, "b"); - else strcat(mode_w, "h"); - if (is_uncompressed) strcat(mode_w, "u"); - if (optind + 1 >= argc) { - fprintf(stderr, "\n"); - fprintf(stderr, "Usage: samtools fillmd [-eubrS] \n\n"); - fprintf(stderr, "Options: -e change identical bases to '='\n"); - fprintf(stderr, " -u uncompressed BAM output (for piping)\n"); - fprintf(stderr, " -b compressed BAM output\n"); - fprintf(stderr, " -S the input is SAM with header\n"); - fprintf(stderr, " -A modify the quality string\n"); - fprintf(stderr, " -r compute the BQ tag (without -A) or cap baseQ by BAQ (with -A)\n"); - fprintf(stderr, " -E extended BAQ for better sensitivity but lower specificity\n\n"); - return 1; - } - fp = samopen(argv[optind], mode_r, 0); - if (fp == 0) return 1; - if (is_sam_in && (fp->header == 0 || fp->header->n_targets == 0)) { - fprintf(stderr, "[bam_fillmd] input SAM does not have header. Abort!\n"); - return 1; - } - fpout = samopen("-", mode_w, fp->header); - fai = fai_load(argv[optind+1]); - - b = bam_init1(); - while ((ret = samread(fp, b)) >= 0) { - if (b->core.tid >= 0) { - if (tid != b->core.tid) { - free(ref); - ref = fai_fetch(fai, fp->header->target_name[b->core.tid], &len); - tid = b->core.tid; - if (ref == 0) - fprintf(stderr, "[bam_fillmd] fail to find sequence '%s' in the reference.\n", - fp->header->target_name[tid]); - } - if (is_realn) bam_prob_realn_core(b, ref, baq_flag); - if (capQ > 10) { - int q = bam_cap_mapQ(b, ref, capQ); - if (b->core.qual > q) b->core.qual = q; - } - if (ref) bam_fillmd1_core(b, ref, is_equal, max_nm); - } - samwrite(fpout, b); - } - bam_destroy1(b); - - free(ref); - fai_destroy(fai); - samclose(fp); samclose(fpout); - return 0; -} diff --git a/samtools-0.1.13/bam_pileup.c b/samtools-0.1.13/bam_pileup.c deleted file mode 100644 --- a/samtools-0.1.13/bam_pileup.c +++ /dev/null @@ -1,437 +0,0 @@ -#include -#include -#include -#include -#include "sam.h" - -typedef struct { - int k, x, y, end; -} cstate_t; - -static cstate_t g_cstate_null = { -1, 0, 0, 0 }; - -typedef struct __linkbuf_t { - bam1_t b; - uint32_t beg, end; - cstate_t s; - struct __linkbuf_t *next; -} lbnode_t; - -/* --- BEGIN: Memory pool */ - -typedef struct { - int cnt, n, max; - lbnode_t **buf; -} mempool_t; - -static mempool_t *mp_init() -{ - mempool_t *mp; - mp = (mempool_t*)calloc(1, sizeof(mempool_t)); - return mp; -} -static void mp_destroy(mempool_t *mp) -{ - int k; - for (k = 0; k < mp->n; ++k) { - free(mp->buf[k]->b.data); - free(mp->buf[k]); - } - free(mp->buf); - free(mp); -} -static inline lbnode_t *mp_alloc(mempool_t *mp) -{ - ++mp->cnt; - if (mp->n == 0) return (lbnode_t*)calloc(1, sizeof(lbnode_t)); - else return mp->buf[--mp->n]; -} -static inline void mp_free(mempool_t *mp, lbnode_t *p) -{ - --mp->cnt; p->next = 0; // clear lbnode_t::next here - if (mp->n == mp->max) { - mp->max = mp->max? mp->max<<1 : 256; - mp->buf = (lbnode_t**)realloc(mp->buf, sizeof(lbnode_t*) * mp->max); - } - mp->buf[mp->n++] = p; -} - -/* --- END: Memory pool */ - -/* --- BEGIN: Auxiliary functions */ - -/* s->k: the index of the CIGAR operator that has just been processed. - s->x: the reference coordinate of the start of s->k - s->y: the query coordiante of the start of s->k - */ -static inline int resolve_cigar2(bam_pileup1_t *p, uint32_t pos, cstate_t *s) -{ -#define _cop(c) ((c)&BAM_CIGAR_MASK) -#define _cln(c) ((c)>>BAM_CIGAR_SHIFT) - - bam1_t *b = p->b; - bam1_core_t *c = &b->core; - uint32_t *cigar = bam1_cigar(b); - int k, is_head = 0; - // determine the current CIGAR operation -// fprintf(stderr, "%s\tpos=%d\tend=%d\t(%d,%d,%d)\n", bam1_qname(b), pos, s->end, s->k, s->x, s->y); - if (s->k == -1) { // never processed - is_head = 1; - if (c->n_cigar == 1) { // just one operation, save a loop - if (_cop(cigar[0]) == BAM_CMATCH) s->k = 0, s->x = c->pos, s->y = 0; - } else { // find the first match or deletion - for (k = 0, s->x = c->pos, s->y = 0; k < c->n_cigar; ++k) { - int op = _cop(cigar[k]); - int l = _cln(cigar[k]); - if (op == BAM_CMATCH || op == BAM_CDEL) break; - else if (op == BAM_CREF_SKIP) s->x += l; - else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l; - } - assert(k < c->n_cigar); - s->k = k; - } - } else { // the read has been processed before - int op, l = _cln(cigar[s->k]); - if (pos - s->x >= l) { // jump to the next operation - assert(s->k < c->n_cigar); // otherwise a bug: this function should not be called in this case - op = _cop(cigar[s->k+1]); - if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP) { // jump to the next without a loop - if (_cop(cigar[s->k]) == BAM_CMATCH) s->y += l; - s->x += l; - ++s->k; - } else { // find the next M/D/N - if (_cop(cigar[s->k]) == BAM_CMATCH) s->y += l; - s->x += l; - for (k = s->k + 1; k < c->n_cigar; ++k) { - op = _cop(cigar[k]), l = _cln(cigar[k]); - if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP) break; - else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l; - } - s->k = k; - } - assert(s->k < c->n_cigar); // otherwise a bug - } // else, do nothing - } - { // collect pileup information - int op, l; - op = _cop(cigar[s->k]); l = _cln(cigar[s->k]); - p->is_del = p->indel = p->is_refskip = 0; - if (s->x + l - 1 == pos && s->k + 1 < c->n_cigar) { // peek the next operation - int op2 = _cop(cigar[s->k+1]); - int l2 = _cln(cigar[s->k+1]); - if (op2 == BAM_CDEL) p->indel = -(int)l2; - else if (op2 == BAM_CINS) p->indel = l2; - else if (op2 == BAM_CPAD && s->k + 2 < c->n_cigar) { // no working for adjacent padding - int l3 = 0; - for (k = s->k + 2; k < c->n_cigar; ++k) { - op2 = _cop(cigar[k]); l2 = _cln(cigar[k]); - if (op2 == BAM_CINS) l3 += l2; - else if (op2 == BAM_CDEL || op2 == BAM_CMATCH || op2 == BAM_CREF_SKIP) break; - } - if (l3 > 0) p->indel = l3; - } - } - if (op == BAM_CMATCH) { - p->qpos = s->y + (pos - s->x); - } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) { - p->is_del = 1; p->qpos = s->y; // FIXME: distinguish D and N!!!!! - p->is_refskip = (op == BAM_CREF_SKIP); - } // cannot be other operations; otherwise a bug - p->is_head = (pos == c->pos); p->is_tail = (pos == s->end); - } - return 1; -} - -/* --- END: Auxiliary functions */ - -/******************* - * pileup iterator * - *******************/ - -struct __bam_plp_t { - mempool_t *mp; - lbnode_t *head, *tail, *dummy; - int32_t tid, pos, max_tid, max_pos; - int is_eof, flag_mask, max_plp, error, maxcnt; - bam_pileup1_t *plp; - // for the "auto" interface only - bam1_t *b; - bam_plp_auto_f func; - void *data; -}; - -bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data) -{ - bam_plp_t iter; - iter = calloc(1, sizeof(struct __bam_plp_t)); - iter->mp = mp_init(); - iter->head = iter->tail = mp_alloc(iter->mp); - iter->dummy = mp_alloc(iter->mp); - iter->max_tid = iter->max_pos = -1; - iter->flag_mask = BAM_DEF_MASK; - iter->maxcnt = 8000; - if (func) { - iter->func = func; - iter->data = data; - iter->b = bam_init1(); - } - return iter; -} - -void bam_plp_destroy(bam_plp_t iter) -{ - mp_free(iter->mp, iter->dummy); - mp_free(iter->mp, iter->head); - if (iter->mp->cnt != 0) - fprintf(stderr, "[bam_plp_destroy] memory leak: %d. Continue anyway.\n", iter->mp->cnt); - mp_destroy(iter->mp); - if (iter->b) bam_destroy1(iter->b); - free(iter->plp); - free(iter); -} - -const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp) -{ - if (iter->error) { *_n_plp = -1; return 0; } - *_n_plp = 0; - if (iter->is_eof && iter->head->next == 0) return 0; - while (iter->is_eof || iter->max_tid > iter->tid || (iter->max_tid == iter->tid && iter->max_pos > iter->pos)) { - int n_plp = 0; - lbnode_t *p, *q; - // write iter->plp at iter->pos - iter->dummy->next = iter->head; - for (p = iter->head, q = iter->dummy; p->next; q = p, p = p->next) { - if (p->b.core.tid < iter->tid || (p->b.core.tid == iter->tid && p->end <= iter->pos)) { // then remove - q->next = p->next; mp_free(iter->mp, p); p = q; - } else if (p->b.core.tid == iter->tid && p->beg <= iter->pos) { // here: p->end > pos; then add to pileup - if (n_plp == iter->max_plp) { // then double the capacity - iter->max_plp = iter->max_plp? iter->max_plp<<1 : 256; - iter->plp = (bam_pileup1_t*)realloc(iter->plp, sizeof(bam_pileup1_t) * iter->max_plp); - } - iter->plp[n_plp].b = &p->b; - if (resolve_cigar2(iter->plp + n_plp, iter->pos, &p->s)) ++n_plp; // actually always true... - } - } - iter->head = iter->dummy->next; // dummy->next may be changed - *_n_plp = n_plp; *_tid = iter->tid; *_pos = iter->pos; - // update iter->tid and iter->pos - if (iter->head->next) { - if (iter->tid > iter->head->b.core.tid) { - fprintf(stderr, "[%s] unsorted input. Pileup aborts.\n", __func__); - iter->error = 1; - *_n_plp = -1; - return 0; - } - } - if (iter->tid < iter->head->b.core.tid) { // come to a new reference sequence - iter->tid = iter->head->b.core.tid; iter->pos = iter->head->beg; // jump to the next reference - } else if (iter->pos < iter->head->beg) { // here: tid == head->b.core.tid - iter->pos = iter->head->beg; // jump to the next position - } else ++iter->pos; // scan contiguously - // return - if (n_plp) return iter->plp; - if (iter->is_eof && iter->head->next == 0) break; - } - return 0; -} - -int bam_plp_push(bam_plp_t iter, const bam1_t *b) -{ - if (iter->error) return -1; - if (b) { - if (b->core.tid < 0) return 0; - if (b->core.flag & iter->flag_mask) return 0; - if (iter->tid == b->core.tid && iter->pos == b->core.pos && iter->mp->cnt > iter->maxcnt) return 0; - bam_copy1(&iter->tail->b, b); - iter->tail->beg = b->core.pos; iter->tail->end = bam_calend(&b->core, bam1_cigar(b)); - iter->tail->s = g_cstate_null; iter->tail->s.end = iter->tail->end - 1; // initialize cstate_t - if (b->core.tid < iter->max_tid) { - fprintf(stderr, "[bam_pileup_core] the input is not sorted (chromosomes out of order)\n"); - iter->error = 1; - return -1; - } - if ((b->core.tid == iter->max_tid) && (iter->tail->beg < iter->max_pos)) { - fprintf(stderr, "[bam_pileup_core] the input is not sorted (reads out of order)\n"); - iter->error = 1; - return -1; - } - iter->max_tid = b->core.tid; iter->max_pos = iter->tail->beg; - if (iter->tail->end > iter->pos || iter->tail->b.core.tid > iter->tid) { - iter->tail->next = mp_alloc(iter->mp); - iter->tail = iter->tail->next; - } - } else iter->is_eof = 1; - return 0; -} - -const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp) -{ - const bam_pileup1_t *plp; - if (iter->func == 0 || iter->error) { *_n_plp = -1; return 0; } - if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp; - else { // no pileup line can be obtained; read alignments - *_n_plp = 0; - if (iter->is_eof) return 0; - while (iter->func(iter->data, iter->b) >= 0) { - if (bam_plp_push(iter, iter->b) < 0) { - *_n_plp = -1; - return 0; - } - if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp; - // otherwise no pileup line can be returned; read the next alignment. - } - bam_plp_push(iter, 0); - if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp; - return 0; - } -} - -void bam_plp_reset(bam_plp_t iter) -{ - lbnode_t *p, *q; - iter->max_tid = iter->max_pos = -1; - iter->tid = iter->pos = 0; - iter->is_eof = 0; - for (p = iter->head; p->next;) { - q = p->next; - mp_free(iter->mp, p); - p = q; - } - iter->head = iter->tail; -} - -void bam_plp_set_mask(bam_plp_t iter, int mask) -{ - iter->flag_mask = mask < 0? BAM_DEF_MASK : (BAM_FUNMAP | mask); -} - -void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt) -{ - iter->maxcnt = maxcnt; -} - -/***************** - * callback APIs * - *****************/ - -int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data) -{ - bam_plbuf_t *buf; - int ret; - bam1_t *b; - b = bam_init1(); - buf = bam_plbuf_init(func, func_data); - bam_plbuf_set_mask(buf, mask); - while ((ret = bam_read1(fp, b)) >= 0) - bam_plbuf_push(b, buf); - bam_plbuf_push(0, buf); - bam_plbuf_destroy(buf); - bam_destroy1(b); - return 0; -} - -void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask) -{ - bam_plp_set_mask(buf->iter, mask); -} - -void bam_plbuf_reset(bam_plbuf_t *buf) -{ - bam_plp_reset(buf->iter); -} - -bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data) -{ - bam_plbuf_t *buf; - buf = calloc(1, sizeof(bam_plbuf_t)); - buf->iter = bam_plp_init(0, 0); - buf->func = func; - buf->data = data; - return buf; -} - -void bam_plbuf_destroy(bam_plbuf_t *buf) -{ - bam_plp_destroy(buf->iter); - free(buf); -} - -int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf) -{ - int ret, n_plp, tid, pos; - const bam_pileup1_t *plp; - ret = bam_plp_push(buf->iter, b); - if (ret < 0) return ret; - while ((plp = bam_plp_next(buf->iter, &tid, &pos, &n_plp)) != 0) - buf->func(tid, pos, n_plp, plp, buf->data); - return 0; -} - -/*********** - * mpileup * - ***********/ - -struct __bam_mplp_t { - int n; - uint64_t min, *pos; - bam_plp_t *iter; - int *n_plp; - const bam_pileup1_t **plp; -}; - -bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data) -{ - int i; - bam_mplp_t iter; - iter = calloc(1, sizeof(struct __bam_mplp_t)); - iter->pos = calloc(n, 8); - iter->n_plp = calloc(n, sizeof(int)); - iter->plp = calloc(n, sizeof(void*)); - iter->iter = calloc(n, sizeof(void*)); - iter->n = n; - iter->min = (uint64_t)-1; - for (i = 0; i < n; ++i) { - iter->iter[i] = bam_plp_init(func, data[i]); - iter->pos[i] = iter->min; - } - return iter; -} - -void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt) -{ - int i; - for (i = 0; i < iter->n; ++i) - iter->iter[i]->maxcnt = maxcnt; -} - -void bam_mplp_destroy(bam_mplp_t iter) -{ - int i; - for (i = 0; i < iter->n; ++i) bam_plp_destroy(iter->iter[i]); - free(iter->iter); free(iter->pos); free(iter->n_plp); free(iter->plp); - free(iter); -} - -int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp) -{ - int i, ret = 0; - uint64_t new_min = (uint64_t)-1; - for (i = 0; i < iter->n; ++i) { - if (iter->pos[i] == iter->min) { - int tid, pos; - iter->plp[i] = bam_plp_auto(iter->iter[i], &tid, &pos, &iter->n_plp[i]); - iter->pos[i] = (uint64_t)tid<<32 | pos; - } - if (iter->plp[i] && iter->pos[i] < new_min) new_min = iter->pos[i]; - } - iter->min = new_min; - if (new_min == (uint64_t)-1) return 0; - *_tid = new_min>>32; *_pos = (uint32_t)new_min; - for (i = 0; i < iter->n; ++i) { - if (iter->pos[i] == iter->min) { // FIXME: valgrind reports "uninitialised value(s) at this line" - n_plp[i] = iter->n_plp[i], plp[i] = iter->plp[i]; - ++ret; - } else n_plp[i] = 0, plp[i] = 0; - } - return ret; -} diff --git a/samtools-0.1.13/bam_reheader.c b/samtools-0.1.13/bam_reheader.c deleted file mode 100644 --- a/samtools-0.1.13/bam_reheader.c +++ /dev/null @@ -1,61 +0,0 @@ -#include -#include -#include "bgzf.h" -#include "bam.h" - -#define BUF_SIZE 0x10000 - -int bam_reheader(BGZF *in, const bam_header_t *h, int fd) -{ - BGZF *fp; - bam_header_t *old; - int len; - uint8_t *buf; - if (in->open_mode != 'r') return -1; - buf = malloc(BUF_SIZE); - old = bam_header_read(in); - fp = bgzf_fdopen(fd, "w"); - bam_header_write(fp, h); - if (in->block_offset < in->block_length) { - bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset); - bgzf_flush(fp); - } -#ifdef _USE_KNETFILE - while ((len = knet_read(in->x.fpr, buf, BUF_SIZE)) > 0) - fwrite(buf, 1, len, fp->x.fpw); -#else - while (!feof(in->file) && (len = fread(buf, 1, BUF_SIZE, in->file)) > 0) - fwrite(buf, 1, len, fp->file); -#endif - free(buf); - fp->block_offset = in->block_offset = 0; - bgzf_close(fp); - return 0; -} - -int main_reheader(int argc, char *argv[]) -{ - bam_header_t *h; - BGZF *in; - if (argc != 3) { - fprintf(stderr, "Usage: samtools reheader \n"); - return 1; - } - { // read the header - tamFile fph = sam_open(argv[1]); - if (fph == 0) { - fprintf(stderr, "[%s] fail to read the header from %s.\n", __func__, argv[1]); - return 1; - } - h = sam_header_read(fph); - sam_close(fph); - } - in = strcmp(argv[2], "-")? bam_open(argv[2], "r") : bam_dopen(fileno(stdin), "r"); - if (in == 0) { - fprintf(stderr, "[%s] fail to open file %s.\n", __func__, argv[2]); - return 1; - } - bam_reheader(in, h, fileno(stdout)); - bgzf_close(in); - return 0; -} diff --git a/samtools-0.1.13/bam_sort.c b/samtools-0.1.13/bam_sort.c deleted file mode 100644 --- a/samtools-0.1.13/bam_sort.c +++ /dev/null @@ -1,421 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "bam.h" -#include "ksort.h" - -static int g_is_by_qname = 0; - -static inline int strnum_cmp(const char *a, const char *b) -{ - char *pa, *pb; - pa = (char*)a; pb = (char*)b; - while (*pa && *pb) { - if (isdigit(*pa) && isdigit(*pb)) { - long ai, bi; - ai = strtol(pa, &pa, 10); - bi = strtol(pb, &pb, 10); - if (ai != bi) return aibi? 1 : 0; - } else { - if (*pa != *pb) break; - ++pa; ++pb; - } - } - if (*pa == *pb) - return (pa-a) < (pb-b)? -1 : (pa-a) > (pb-b)? 1 : 0; - return *pa<*pb? -1 : *pa>*pb? 1 : 0; -} - -#define HEAP_EMPTY 0xffffffffffffffffull - -typedef struct { - int i; - uint64_t pos, idx; - bam1_t *b; -} heap1_t; - -#define __pos_cmp(a, b) ((a).pos > (b).pos || ((a).pos == (b).pos && ((a).i > (b).i || ((a).i == (b).i && (a).idx > (b).idx)))) - -static inline int heap_lt(const heap1_t a, const heap1_t b) -{ - if (g_is_by_qname) { - int t; - if (a.b == 0 || b.b == 0) return a.b == 0? 1 : 0; - t = strnum_cmp(bam1_qname(a.b), bam1_qname(b.b)); - return (t > 0 || (t == 0 && __pos_cmp(a, b))); - } else return __pos_cmp(a, b); -} - -KSORT_INIT(heap, heap1_t, heap_lt) - -static void swap_header_targets(bam_header_t *h1, bam_header_t *h2) -{ - bam_header_t t; - t.n_targets = h1->n_targets, h1->n_targets = h2->n_targets, h2->n_targets = t.n_targets; - t.target_name = h1->target_name, h1->target_name = h2->target_name, h2->target_name = t.target_name; - t.target_len = h1->target_len, h1->target_len = h2->target_len, h2->target_len = t.target_len; -} - -static void swap_header_text(bam_header_t *h1, bam_header_t *h2) -{ - int tempi; - char *temps; - tempi = h1->l_text, h1->l_text = h2->l_text, h2->l_text = tempi; - temps = h1->text, h1->text = h2->text, h2->text = temps; -} - -#define MERGE_RG 1 -#define MERGE_UNCOMP 2 - -/*! - @abstract Merge multiple sorted BAM. - @param is_by_qname whether to sort by query name - @param out output BAM file name - @param headers name of SAM file from which to copy '@' header lines, - or NULL to copy them from the first file to be merged - @param n number of files to be merged - @param fn names of files to be merged - - @discussion Padding information may NOT correctly maintained. This - function is NOT thread safe. - */ -int bam_merge_core(int by_qname, const char *out, const char *headers, int n, char * const *fn, - int flag, const char *reg) -{ - bamFile fpout, *fp; - heap1_t *heap; - bam_header_t *hout = 0; - bam_header_t *hheaders = NULL; - int i, j, *RG_len = 0; - uint64_t idx = 0; - char **RG = 0; - bam_iter_t *iter = 0; - - if (headers) { - tamFile fpheaders = sam_open(headers); - if (fpheaders == 0) { - const char *message = strerror(errno); - fprintf(stderr, "[bam_merge_core] cannot open '%s': %s\n", headers, message); - return -1; - } - hheaders = sam_header_read(fpheaders); - sam_close(fpheaders); - } - - g_is_by_qname = by_qname; - fp = (bamFile*)calloc(n, sizeof(bamFile)); - heap = (heap1_t*)calloc(n, sizeof(heap1_t)); - iter = (bam_iter_t*)calloc(n, sizeof(bam_iter_t)); - // prepare RG tag - if (flag & MERGE_RG) { - RG = (char**)calloc(n, sizeof(void*)); - RG_len = (int*)calloc(n, sizeof(int)); - for (i = 0; i != n; ++i) { - int l = strlen(fn[i]); - const char *s = fn[i]; - if (l > 4 && strcmp(s + l - 4, ".bam") == 0) l -= 4; - for (j = l - 1; j >= 0; --j) if (s[j] == '/') break; - ++j; l -= j; - RG[i] = calloc(l + 1, 1); - RG_len[i] = l; - strncpy(RG[i], s + j, l); - } - } - // read the first - for (i = 0; i != n; ++i) { - bam_header_t *hin; - fp[i] = bam_open(fn[i], "r"); - if (fp[i] == 0) { - int j; - fprintf(stderr, "[bam_merge_core] fail to open file %s\n", fn[i]); - for (j = 0; j < i; ++j) bam_close(fp[j]); - free(fp); free(heap); - // FIXME: possible memory leak - return -1; - } - hin = bam_header_read(fp[i]); - if (i == 0) { // the first BAM - hout = hin; - } else { // validate multiple baf - int min_n_targets = hout->n_targets; - if (hin->n_targets < min_n_targets) min_n_targets = hin->n_targets; - - for (j = 0; j < min_n_targets; ++j) - if (strcmp(hout->target_name[j], hin->target_name[j]) != 0) { - fprintf(stderr, "[bam_merge_core] different target sequence name: '%s' != '%s' in file '%s'\n", - hout->target_name[j], hin->target_name[j], fn[i]); - return -1; - } - - // If this input file has additional target reference sequences, - // add them to the headers to be output - if (hin->n_targets > hout->n_targets) { - swap_header_targets(hout, hin); - // FIXME Possibly we should also create @SQ text headers - // for the newly added reference sequences - } - - bam_header_destroy(hin); - } - } - - if (hheaders) { - // If the text headers to be swapped in include any @SQ headers, - // check that they are consistent with the existing binary list - // of reference information. - if (hheaders->n_targets > 0) { - if (hout->n_targets != hheaders->n_targets) { - fprintf(stderr, "[bam_merge_core] number of @SQ headers in '%s' differs from number of target sequences\n", headers); - if (!reg) return -1; - } - for (j = 0; j < hout->n_targets; ++j) - if (strcmp(hout->target_name[j], hheaders->target_name[j]) != 0) { - fprintf(stderr, "[bam_merge_core] @SQ header '%s' in '%s' differs from target sequence\n", hheaders->target_name[j], headers); - if (!reg) return -1; - } - } - - swap_header_text(hout, hheaders); - bam_header_destroy(hheaders); - } - - if (reg) { - int tid, beg, end; - if (bam_parse_region(hout, reg, &tid, &beg, &end) < 0) { - fprintf(stderr, "[%s] Malformated region string or undefined reference name\n", __func__); - return -1; - } - for (i = 0; i < n; ++i) { - bam_index_t *idx; - idx = bam_index_load(fn[i]); - iter[i] = bam_iter_query(idx, tid, beg, end); - bam_index_destroy(idx); - } - } - - for (i = 0; i < n; ++i) { - heap1_t *h = heap + i; - h->i = i; - h->b = (bam1_t*)calloc(1, sizeof(bam1_t)); - if (bam_iter_read(fp[i], iter[i], h->b) >= 0) { - h->pos = ((uint64_t)h->b->core.tid<<32) | (uint32_t)h->b->core.pos<<1 | bam1_strand(h->b); - h->idx = idx++; - } - else h->pos = HEAP_EMPTY; - } - if (flag & MERGE_UNCOMP) { - fpout = strcmp(out, "-")? bam_open(out, "wu") : bam_dopen(fileno(stdout), "wu"); - } else { - fpout = strcmp(out, "-")? bam_open(out, "w") : bam_dopen(fileno(stdout), "w"); - } - if (fpout == 0) { - fprintf(stderr, "[%s] fail to create the output file.\n", __func__); - return -1; - } - bam_header_write(fpout, hout); - bam_header_destroy(hout); - - ks_heapmake(heap, n, heap); - while (heap->pos != HEAP_EMPTY) { - bam1_t *b = heap->b; - if (flag & MERGE_RG) { - uint8_t *rg = bam_aux_get(b, "RG"); - if (rg) bam_aux_del(b, rg); - bam_aux_append(b, "RG", 'Z', RG_len[heap->i] + 1, (uint8_t*)RG[heap->i]); - } - bam_write1_core(fpout, &b->core, b->data_len, b->data); - if ((j = bam_iter_read(fp[heap->i], iter[heap->i], b)) >= 0) { - heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)b->core.pos<<1 | bam1_strand(b); - heap->idx = idx++; - } else if (j == -1) { - heap->pos = HEAP_EMPTY; - free(heap->b->data); free(heap->b); - heap->b = 0; - } else fprintf(stderr, "[bam_merge_core] '%s' is truncated. Continue anyway.\n", fn[heap->i]); - ks_heapadjust(heap, 0, n, heap); - } - - if (flag & MERGE_RG) { - for (i = 0; i != n; ++i) free(RG[i]); - free(RG); free(RG_len); - } - for (i = 0; i != n; ++i) { - bam_iter_destroy(iter[i]); - bam_close(fp[i]); - } - bam_close(fpout); - free(fp); free(heap); free(iter); - return 0; -} - -int bam_merge(int argc, char *argv[]) -{ - int c, is_by_qname = 0, flag = 0, ret = 0; - char *fn_headers = NULL, *reg = 0; - - while ((c = getopt(argc, argv, "h:nruR:")) >= 0) { - switch (c) { - case 'r': flag |= MERGE_RG; break; - case 'h': fn_headers = strdup(optarg); break; - case 'n': is_by_qname = 1; break; - case 'u': flag |= MERGE_UNCOMP; break; - case 'R': reg = strdup(optarg); break; - } - } - if (optind + 2 >= argc) { - fprintf(stderr, "\n"); - fprintf(stderr, "Usage: samtools merge [-nr] [-h inh.sam] [...]\n\n"); - fprintf(stderr, "Options: -n sort by read names\n"); - fprintf(stderr, " -r attach RG tag (inferred from file names)\n"); - fprintf(stderr, " -u uncompressed BAM output\n"); - fprintf(stderr, " -R STR merge file in the specified region STR [all]\n"); - fprintf(stderr, " -h FILE copy the header in FILE to [in1.bam]\n\n"); - fprintf(stderr, "Note: Samtools' merge does not reconstruct the @RG dictionary in the header. Users\n"); - fprintf(stderr, " must provide the correct header with -h, or uses Picard which properly maintains\n"); - fprintf(stderr, " the header dictionary in merging.\n\n"); - return 1; - } - if (bam_merge_core(is_by_qname, argv[optind], fn_headers, argc - optind - 1, argv + optind + 1, flag, reg) < 0) ret = 1; - free(reg); - free(fn_headers); - return ret; -} - -typedef bam1_t *bam1_p; - -static inline int bam1_lt(const bam1_p a, const bam1_p b) -{ - if (g_is_by_qname) { - int t = strnum_cmp(bam1_qname(a), bam1_qname(b)); - return (t < 0 || (t == 0 && (((uint64_t)a->core.tid<<32|a->core.pos) < ((uint64_t)b->core.tid<<32|b->core.pos)))); - } else return (((uint64_t)a->core.tid<<32|a->core.pos) < ((uint64_t)b->core.tid<<32|b->core.pos)); -} -KSORT_INIT(sort, bam1_p, bam1_lt) - -static void sort_blocks(int n, int k, bam1_p *buf, const char *prefix, const bam_header_t *h, int is_stdout) -{ - char *name; - int i; - bamFile fp; - ks_mergesort(sort, k, buf, 0); - name = (char*)calloc(strlen(prefix) + 20, 1); - if (n >= 0) sprintf(name, "%s.%.4d.bam", prefix, n); - else sprintf(name, "%s.bam", prefix); - fp = is_stdout? bam_dopen(fileno(stdout), "w") : bam_open(name, "w"); - if (fp == 0) { - fprintf(stderr, "[sort_blocks] fail to create file %s.\n", name); - free(name); - // FIXME: possible memory leak - return; - } - free(name); - bam_header_write(fp, h); - for (i = 0; i < k; ++i) - bam_write1_core(fp, &buf[i]->core, buf[i]->data_len, buf[i]->data); - bam_close(fp); -} - -/*! - @abstract Sort an unsorted BAM file based on the chromosome order - and the leftmost position of an alignment - - @param is_by_qname whether to sort by query name - @param fn name of the file to be sorted - @param prefix prefix of the output and the temporary files; upon - sucessess, prefix.bam will be written. - @param max_mem approxiate maximum memory (very inaccurate) - - @discussion It may create multiple temporary subalignment files - and then merge them by calling bam_merge_core(). This function is - NOT thread safe. - */ -void bam_sort_core_ext(int is_by_qname, const char *fn, const char *prefix, size_t max_mem, int is_stdout) -{ - int n, ret, k, i; - size_t mem; - bam_header_t *header; - bamFile fp; - bam1_t *b, **buf; - - g_is_by_qname = is_by_qname; - n = k = 0; mem = 0; - fp = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r"); - if (fp == 0) { - fprintf(stderr, "[bam_sort_core] fail to open file %s\n", fn); - return; - } - header = bam_header_read(fp); - buf = (bam1_t**)calloc(max_mem / BAM_CORE_SIZE, sizeof(bam1_t*)); - // write sub files - for (;;) { - if (buf[k] == 0) buf[k] = (bam1_t*)calloc(1, sizeof(bam1_t)); - b = buf[k]; - if ((ret = bam_read1(fp, b)) < 0) break; - mem += ret; - ++k; - if (mem >= max_mem) { - sort_blocks(n++, k, buf, prefix, header, 0); - mem = 0; k = 0; - } - } - if (ret != -1) - fprintf(stderr, "[bam_sort_core] truncated file. Continue anyway.\n"); - if (n == 0) sort_blocks(-1, k, buf, prefix, header, is_stdout); - else { // then merge - char **fns, *fnout; - fprintf(stderr, "[bam_sort_core] merging from %d files...\n", n+1); - sort_blocks(n++, k, buf, prefix, header, 0); - fnout = (char*)calloc(strlen(prefix) + 20, 1); - if (is_stdout) sprintf(fnout, "-"); - else sprintf(fnout, "%s.bam", prefix); - fns = (char**)calloc(n, sizeof(char*)); - for (i = 0; i < n; ++i) { - fns[i] = (char*)calloc(strlen(prefix) + 20, 1); - sprintf(fns[i], "%s.%.4d.bam", prefix, i); - } - bam_merge_core(is_by_qname, fnout, 0, n, fns, 0, 0); - free(fnout); - for (i = 0; i < n; ++i) { - unlink(fns[i]); - free(fns[i]); - } - free(fns); - } - for (k = 0; k < max_mem / BAM_CORE_SIZE; ++k) { - if (buf[k]) { - free(buf[k]->data); - free(buf[k]); - } - } - free(buf); - bam_header_destroy(header); - bam_close(fp); -} - -void bam_sort_core(int is_by_qname, const char *fn, const char *prefix, size_t max_mem) -{ - bam_sort_core_ext(is_by_qname, fn, prefix, max_mem, 0); -} - -int bam_sort(int argc, char *argv[]) -{ - size_t max_mem = 500000000; - int c, is_by_qname = 0, is_stdout = 0; - while ((c = getopt(argc, argv, "nom:")) >= 0) { - switch (c) { - case 'o': is_stdout = 1; break; - case 'n': is_by_qname = 1; break; - case 'm': max_mem = atol(optarg); break; - } - } - if (optind + 2 > argc) { - fprintf(stderr, "Usage: samtools sort [-on] [-m ] \n"); - return 1; - } - bam_sort_core_ext(is_by_qname, argv[optind], argv[optind+1], max_mem, is_stdout); - return 0; -} diff --git a/samtools-0.1.13/bgzf.c b/samtools-0.1.13/bgzf.c deleted file mode 100644 --- a/samtools-0.1.13/bgzf.c +++ /dev/null @@ -1,671 +0,0 @@ -/* The MIT License - - Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -/* - 2009-06-29 by lh3: cache recent uncompressed blocks. - 2009-06-25 by lh3: optionally use my knetfile library to access file on a FTP. - 2009-06-12 by lh3: support a mode string like "wu" where 'u' for uncompressed output */ - -#include -#include -#include -#include -#include -#include -#include -#include "bgzf.h" - -#include "khash.h" -typedef struct { - int size; - uint8_t *block; - int64_t end_offset; -} cache_t; -KHASH_MAP_INIT_INT64(cache, cache_t) - -#if defined(_WIN32) || defined(_MSC_VER) -#define ftello(fp) ftell(fp) -#define fseeko(fp, offset, whence) fseek(fp, offset, whence) -#else -extern off_t ftello(FILE *stream); -extern int fseeko(FILE *stream, off_t offset, int whence); -#endif - -typedef int8_t bgzf_byte_t; - -static const int DEFAULT_BLOCK_SIZE = 64 * 1024; -static const int MAX_BLOCK_SIZE = 64 * 1024; - -static const int BLOCK_HEADER_LENGTH = 18; -static const int BLOCK_FOOTER_LENGTH = 8; - -static const int GZIP_ID1 = 31; -static const int GZIP_ID2 = 139; -static const int CM_DEFLATE = 8; -static const int FLG_FEXTRA = 4; -static const int OS_UNKNOWN = 255; -static const int BGZF_ID1 = 66; // 'B' -static const int BGZF_ID2 = 67; // 'C' -static const int BGZF_LEN = 2; -static const int BGZF_XLEN = 6; // BGZF_LEN+4 - -static const int GZIP_WINDOW_BITS = -15; // no zlib header -static const int Z_DEFAULT_MEM_LEVEL = 8; - - -inline -void -packInt16(uint8_t* buffer, uint16_t value) -{ - buffer[0] = value; - buffer[1] = value >> 8; -} - -inline -int -unpackInt16(const uint8_t* buffer) -{ - return (buffer[0] | (buffer[1] << 8)); -} - -inline -void -packInt32(uint8_t* buffer, uint32_t value) -{ - buffer[0] = value; - buffer[1] = value >> 8; - buffer[2] = value >> 16; - buffer[3] = value >> 24; -} - -static inline -int -bgzf_min(int x, int y) -{ - return (x < y) ? x : y; -} - -static -void -report_error(BGZF* fp, const char* message) { - fp->error = message; -} - -static BGZF *bgzf_read_init() -{ - BGZF *fp; - fp = calloc(1, sizeof(BGZF)); - fp->uncompressed_block_size = MAX_BLOCK_SIZE; - fp->uncompressed_block = malloc(MAX_BLOCK_SIZE); - fp->compressed_block_size = MAX_BLOCK_SIZE; - fp->compressed_block = malloc(MAX_BLOCK_SIZE); - fp->cache_size = 0; - fp->cache = kh_init(cache); - return fp; -} - -static -BGZF* -open_read(int fd) -{ -#ifdef _USE_KNETFILE - knetFile *file = knet_dopen(fd, "r"); -#else - FILE* file = fdopen(fd, "r"); -#endif - BGZF* fp; - if (file == 0) return 0; - fp = bgzf_read_init(); - fp->file_descriptor = fd; - fp->open_mode = 'r'; -#ifdef _USE_KNETFILE - fp->x.fpr = file; -#else - fp->file = file; -#endif - return fp; -} - -static -BGZF* -open_write(int fd, bool is_uncompressed) -{ - FILE* file = fdopen(fd, "w"); - BGZF* fp; - if (file == 0) return 0; - fp = malloc(sizeof(BGZF)); - fp->file_descriptor = fd; - fp->open_mode = 'w'; - fp->owned_file = 0; fp->is_uncompressed = is_uncompressed; -#ifdef _USE_KNETFILE - fp->x.fpw = file; -#else - fp->file = file; -#endif - fp->uncompressed_block_size = DEFAULT_BLOCK_SIZE; - fp->uncompressed_block = NULL; - fp->compressed_block_size = MAX_BLOCK_SIZE; - fp->compressed_block = malloc(MAX_BLOCK_SIZE); - fp->block_address = 0; - fp->block_offset = 0; - fp->block_length = 0; - fp->error = NULL; - return fp; -} - -BGZF* -bgzf_open(const char* __restrict path, const char* __restrict mode) -{ - BGZF* fp = NULL; - if (strchr(mode, 'r') || strchr(mode, 'R')) { /* The reading mode is preferred. */ -#ifdef _USE_KNETFILE - knetFile *file = knet_open(path, mode); - if (file == 0) return 0; - fp = bgzf_read_init(); - fp->file_descriptor = -1; - fp->open_mode = 'r'; - fp->x.fpr = file; -#else - int fd, oflag = O_RDONLY; -#ifdef _WIN32 - oflag |= O_BINARY; -#endif - fd = open(path, oflag); - if (fd == -1) return 0; - fp = open_read(fd); -#endif - } else if (strchr(mode, 'w') || strchr(mode, 'W')) { - int fd, oflag = O_WRONLY | O_CREAT | O_TRUNC; -#ifdef _WIN32 - oflag |= O_BINARY; -#endif - fd = open(path, oflag, 0666); - if (fd == -1) return 0; - fp = open_write(fd, strchr(mode, 'u')? 1 : 0); - } - if (fp != NULL) fp->owned_file = 1; - return fp; -} - -BGZF* -bgzf_fdopen(int fd, const char * __restrict mode) -{ - if (fd == -1) return 0; - if (mode[0] == 'r' || mode[0] == 'R') { - return open_read(fd); - } else if (mode[0] == 'w' || mode[0] == 'W') { - return open_write(fd, strstr(mode, "u")? 1 : 0); - } else { - return NULL; - } -} - -static -int -deflate_block(BGZF* fp, int block_length) -{ - // Deflate the block in fp->uncompressed_block into fp->compressed_block. - // Also adds an extra field that stores the compressed block length. - - bgzf_byte_t* buffer = fp->compressed_block; - int buffer_size = fp->compressed_block_size; - - // Init gzip header - buffer[0] = GZIP_ID1; - buffer[1] = GZIP_ID2; - buffer[2] = CM_DEFLATE; - buffer[3] = FLG_FEXTRA; - buffer[4] = 0; // mtime - buffer[5] = 0; - buffer[6] = 0; - buffer[7] = 0; - buffer[8] = 0; - buffer[9] = OS_UNKNOWN; - buffer[10] = BGZF_XLEN; - buffer[11] = 0; - buffer[12] = BGZF_ID1; - buffer[13] = BGZF_ID2; - buffer[14] = BGZF_LEN; - buffer[15] = 0; - buffer[16] = 0; // placeholder for block length - buffer[17] = 0; - - // loop to retry for blocks that do not compress enough - int input_length = block_length; - int compressed_length = 0; - while (1) { - int compress_level = fp->is_uncompressed? 0 : Z_DEFAULT_COMPRESSION; - z_stream zs; - zs.zalloc = NULL; - zs.zfree = NULL; - zs.next_in = fp->uncompressed_block; - zs.avail_in = input_length; - zs.next_out = (void*)&buffer[BLOCK_HEADER_LENGTH]; - zs.avail_out = buffer_size - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH; - - int status = deflateInit2(&zs, compress_level, Z_DEFLATED, - GZIP_WINDOW_BITS, Z_DEFAULT_MEM_LEVEL, Z_DEFAULT_STRATEGY); - if (status != Z_OK) { - report_error(fp, "deflate init failed"); - return -1; - } - status = deflate(&zs, Z_FINISH); - if (status != Z_STREAM_END) { - deflateEnd(&zs); - if (status == Z_OK) { - // Not enough space in buffer. - // Can happen in the rare case the input doesn't compress enough. - // Reduce the amount of input until it fits. - input_length -= 1024; - if (input_length <= 0) { - // should never happen - report_error(fp, "input reduction failed"); - return -1; - } - continue; - } - report_error(fp, "deflate failed"); - return -1; - } - status = deflateEnd(&zs); - if (status != Z_OK) { - report_error(fp, "deflate end failed"); - return -1; - } - compressed_length = zs.total_out; - compressed_length += BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH; - if (compressed_length > MAX_BLOCK_SIZE) { - // should never happen - report_error(fp, "deflate overflow"); - return -1; - } - break; - } - - packInt16((uint8_t*)&buffer[16], compressed_length-1); - uint32_t crc = crc32(0L, NULL, 0L); - crc = crc32(crc, fp->uncompressed_block, input_length); - packInt32((uint8_t*)&buffer[compressed_length-8], crc); - packInt32((uint8_t*)&buffer[compressed_length-4], input_length); - - int remaining = block_length - input_length; - if (remaining > 0) { - if (remaining > input_length) { - // should never happen (check so we can use memcpy) - report_error(fp, "remainder too large"); - return -1; - } - memcpy(fp->uncompressed_block, - fp->uncompressed_block + input_length, - remaining); - } - fp->block_offset = remaining; - return compressed_length; -} - -static -int -inflate_block(BGZF* fp, int block_length) -{ - // Inflate the block in fp->compressed_block into fp->uncompressed_block - - z_stream zs; - zs.zalloc = NULL; - zs.zfree = NULL; - zs.next_in = fp->compressed_block + 18; - zs.avail_in = block_length - 16; - zs.next_out = fp->uncompressed_block; - zs.avail_out = fp->uncompressed_block_size; - - int status = inflateInit2(&zs, GZIP_WINDOW_BITS); - if (status != Z_OK) { - report_error(fp, "inflate init failed"); - return -1; - } - status = inflate(&zs, Z_FINISH); - if (status != Z_STREAM_END) { - inflateEnd(&zs); - report_error(fp, "inflate failed"); - return -1; - } - status = inflateEnd(&zs); - if (status != Z_OK) { - report_error(fp, "inflate failed"); - return -1; - } - return zs.total_out; -} - -static -int -check_header(const bgzf_byte_t* header) -{ - return (header[0] == GZIP_ID1 && - header[1] == (bgzf_byte_t) GZIP_ID2 && - header[2] == Z_DEFLATED && - (header[3] & FLG_FEXTRA) != 0 && - unpackInt16((uint8_t*)&header[10]) == BGZF_XLEN && - header[12] == BGZF_ID1 && - header[13] == BGZF_ID2 && - unpackInt16((uint8_t*)&header[14]) == BGZF_LEN); -} - -static void free_cache(BGZF *fp) -{ - khint_t k; - khash_t(cache) *h = (khash_t(cache)*)fp->cache; - if (fp->open_mode != 'r') return; - for (k = kh_begin(h); k < kh_end(h); ++k) - if (kh_exist(h, k)) free(kh_val(h, k).block); - kh_destroy(cache, h); -} - -static int load_block_from_cache(BGZF *fp, int64_t block_address) -{ - khint_t k; - cache_t *p; - khash_t(cache) *h = (khash_t(cache)*)fp->cache; - k = kh_get(cache, h, block_address); - if (k == kh_end(h)) return 0; - p = &kh_val(h, k); - if (fp->block_length != 0) fp->block_offset = 0; - fp->block_address = block_address; - fp->block_length = p->size; - memcpy(fp->uncompressed_block, p->block, MAX_BLOCK_SIZE); -#ifdef _USE_KNETFILE - knet_seek(fp->x.fpr, p->end_offset, SEEK_SET); -#else - fseeko(fp->file, p->end_offset, SEEK_SET); -#endif - return p->size; -} - -static void cache_block(BGZF *fp, int size) -{ - int ret; - khint_t k; - cache_t *p; - khash_t(cache) *h = (khash_t(cache)*)fp->cache; - if (MAX_BLOCK_SIZE >= fp->cache_size) return; - if ((kh_size(h) + 1) * MAX_BLOCK_SIZE > fp->cache_size) { - /* A better way would be to remove the oldest block in the - * cache, but here we remove a random one for simplicity. This - * should not have a big impact on performance. */ - for (k = kh_begin(h); k < kh_end(h); ++k) - if (kh_exist(h, k)) break; - if (k < kh_end(h)) { - free(kh_val(h, k).block); - kh_del(cache, h, k); - } - } - k = kh_put(cache, h, fp->block_address, &ret); - if (ret == 0) return; // if this happens, a bug! - p = &kh_val(h, k); - p->size = fp->block_length; - p->end_offset = fp->block_address + size; - p->block = malloc(MAX_BLOCK_SIZE); - memcpy(kh_val(h, k).block, fp->uncompressed_block, MAX_BLOCK_SIZE); -} - -int -bgzf_read_block(BGZF* fp) -{ - bgzf_byte_t header[BLOCK_HEADER_LENGTH]; - int count, size = 0; -#ifdef _USE_KNETFILE - int64_t block_address = knet_tell(fp->x.fpr); - if (load_block_from_cache(fp, block_address)) return 0; - count = knet_read(fp->x.fpr, header, sizeof(header)); -#else - int64_t block_address = ftello(fp->file); - if (load_block_from_cache(fp, block_address)) return 0; - count = fread(header, 1, sizeof(header), fp->file); -#endif - if (count == 0) { - fp->block_length = 0; - return 0; - } - size = count; - if (count != sizeof(header)) { - report_error(fp, "read failed"); - return -1; - } - if (!check_header(header)) { - report_error(fp, "invalid block header"); - return -1; - } - int block_length = unpackInt16((uint8_t*)&header[16]) + 1; - bgzf_byte_t* compressed_block = (bgzf_byte_t*) fp->compressed_block; - memcpy(compressed_block, header, BLOCK_HEADER_LENGTH); - int remaining = block_length - BLOCK_HEADER_LENGTH; -#ifdef _USE_KNETFILE - count = knet_read(fp->x.fpr, &compressed_block[BLOCK_HEADER_LENGTH], remaining); -#else - count = fread(&compressed_block[BLOCK_HEADER_LENGTH], 1, remaining, fp->file); -#endif - if (count != remaining) { - report_error(fp, "read failed"); - return -1; - } - size += count; - count = inflate_block(fp, block_length); - if (count < 0) return -1; - if (fp->block_length != 0) { - // Do not reset offset if this read follows a seek. - fp->block_offset = 0; - } - fp->block_address = block_address; - fp->block_length = count; - cache_block(fp, size); - return 0; -} - -int -bgzf_read(BGZF* fp, void* data, int length) -{ - if (length <= 0) { - return 0; - } - if (fp->open_mode != 'r') { - report_error(fp, "file not open for reading"); - return -1; - } - - int bytes_read = 0; - bgzf_byte_t* output = data; - while (bytes_read < length) { - int available = fp->block_length - fp->block_offset; - if (available <= 0) { - if (bgzf_read_block(fp) != 0) { - return -1; - } - available = fp->block_length - fp->block_offset; - if (available <= 0) { - break; - } - } - int copy_length = bgzf_min(length-bytes_read, available); - bgzf_byte_t* buffer = fp->uncompressed_block; - memcpy(output, buffer + fp->block_offset, copy_length); - fp->block_offset += copy_length; - output += copy_length; - bytes_read += copy_length; - } - if (fp->block_offset == fp->block_length) { -#ifdef _USE_KNETFILE - fp->block_address = knet_tell(fp->x.fpr); -#else - fp->block_address = ftello(fp->file); -#endif - fp->block_offset = 0; - fp->block_length = 0; - } - return bytes_read; -} - -int bgzf_flush(BGZF* fp) -{ - while (fp->block_offset > 0) { - int count, block_length; - block_length = deflate_block(fp, fp->block_offset); - if (block_length < 0) return -1; -#ifdef _USE_KNETFILE - count = fwrite(fp->compressed_block, 1, block_length, fp->x.fpw); -#else - count = fwrite(fp->compressed_block, 1, block_length, fp->file); -#endif - if (count != block_length) { - report_error(fp, "write failed"); - return -1; - } - fp->block_address += block_length; - } - return 0; -} - -int bgzf_flush_try(BGZF *fp, int size) -{ - if (fp->block_offset + size > fp->uncompressed_block_size) - return bgzf_flush(fp); - return -1; -} - -int bgzf_write(BGZF* fp, const void* data, int length) -{ - if (fp->open_mode != 'w') { - report_error(fp, "file not open for writing"); - return -1; - } - - if (fp->uncompressed_block == NULL) - fp->uncompressed_block = malloc(fp->uncompressed_block_size); - - const bgzf_byte_t* input = data; - int block_length = fp->uncompressed_block_size; - int bytes_written = 0; - while (bytes_written < length) { - int copy_length = bgzf_min(block_length - fp->block_offset, length - bytes_written); - bgzf_byte_t* buffer = fp->uncompressed_block; - memcpy(buffer + fp->block_offset, input, copy_length); - fp->block_offset += copy_length; - input += copy_length; - bytes_written += copy_length; - if (fp->block_offset == block_length) { - if (bgzf_flush(fp) != 0) { - break; - } - } - } - return bytes_written; -} - -int bgzf_close(BGZF* fp) -{ - if (fp->open_mode == 'w') { - if (bgzf_flush(fp) != 0) return -1; - { // add an empty block - int count, block_length = deflate_block(fp, 0); -#ifdef _USE_KNETFILE - count = fwrite(fp->compressed_block, 1, block_length, fp->x.fpw); -#else - count = fwrite(fp->compressed_block, 1, block_length, fp->file); -#endif - } -#ifdef _USE_KNETFILE - if (fflush(fp->x.fpw) != 0) { -#else - if (fflush(fp->file) != 0) { -#endif - report_error(fp, "flush failed"); - return -1; - } - } - if (fp->owned_file) { -#ifdef _USE_KNETFILE - int ret; - if (fp->open_mode == 'w') ret = fclose(fp->x.fpw); - else ret = knet_close(fp->x.fpr); - if (ret != 0) return -1; -#else - if (fclose(fp->file) != 0) return -1; -#endif - } - free(fp->uncompressed_block); - free(fp->compressed_block); - free_cache(fp); - free(fp); - return 0; -} - -void bgzf_set_cache_size(BGZF *fp, int cache_size) -{ - if (fp) fp->cache_size = cache_size; -} - -int bgzf_check_EOF(BGZF *fp) -{ - static uint8_t magic[28] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0"; - uint8_t buf[28]; - off_t offset; -#ifdef _USE_KNETFILE - offset = knet_tell(fp->x.fpr); - if (knet_seek(fp->x.fpr, -28, SEEK_END) != 0) return -1; - knet_read(fp->x.fpr, buf, 28); - knet_seek(fp->x.fpr, offset, SEEK_SET); -#else - offset = ftello(fp->file); - if (fseeko(fp->file, -28, SEEK_END) != 0) return -1; - fread(buf, 1, 28, fp->file); - fseeko(fp->file, offset, SEEK_SET); -#endif - return (memcmp(magic, buf, 28) == 0)? 1 : 0; -} - -int64_t bgzf_seek(BGZF* fp, int64_t pos, int where) -{ - int block_offset; - int64_t block_address; - - if (fp->open_mode != 'r') { - report_error(fp, "file not open for read"); - return -1; - } - if (where != SEEK_SET) { - report_error(fp, "unimplemented seek option"); - return -1; - } - block_offset = pos & 0xFFFF; - block_address = (pos >> 16) & 0xFFFFFFFFFFFFLL; -#ifdef _USE_KNETFILE - if (knet_seek(fp->x.fpr, block_address, SEEK_SET) != 0) { -#else - if (fseeko(fp->file, block_address, SEEK_SET) != 0) { -#endif - report_error(fp, "seek failed"); - return -1; - } - fp->block_length = 0; // indicates current block is not loaded - fp->block_address = block_address; - fp->block_offset = block_offset; - return 0; -} diff --git a/samtools-0.1.13/bgzf.h b/samtools-0.1.13/bgzf.h deleted file mode 100644 --- a/samtools-0.1.13/bgzf.h +++ /dev/null @@ -1,157 +0,0 @@ -/* The MIT License - - Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#ifndef __BGZF_H -#define __BGZF_H - -#include -#include -#include -#include -#ifdef _USE_KNETFILE -#include "knetfile.h" -#endif - -//typedef int8_t bool; - -typedef struct { - int file_descriptor; - char open_mode; // 'r' or 'w' - bool owned_file, is_uncompressed; -#ifdef _USE_KNETFILE - union { - knetFile *fpr; - FILE *fpw; - } x; -#else - FILE* file; -#endif - int uncompressed_block_size; - int compressed_block_size; - void* uncompressed_block; - void* compressed_block; - int64_t block_address; - int block_length; - int block_offset; - int cache_size; - const char* error; - void *cache; // a pointer to a hash table -} BGZF; - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Open an existing file descriptor for reading or writing. - * Mode must be either "r" or "w". - * A subsequent bgzf_close will not close the file descriptor. - * Returns null on error. - */ -BGZF* bgzf_fdopen(int fd, const char* __restrict mode); - -/* - * Open the specified file for reading or writing. - * Mode must be either "r" or "w". - * Returns null on error. - */ -BGZF* bgzf_open(const char* path, const char* __restrict mode); - -/* - * Close the BGZ file and free all associated resources. - * Does not close the underlying file descriptor if created with bgzf_fdopen. - * Returns zero on success, -1 on error. - */ -int bgzf_close(BGZF* fp); - -/* - * Read up to length bytes from the file storing into data. - * Returns the number of bytes actually read. - * Returns zero on end of file. - * Returns -1 on error. - */ -int bgzf_read(BGZF* fp, void* data, int length); - -/* - * Write length bytes from data to the file. - * Returns the number of bytes written. - * Returns -1 on error. - */ -int bgzf_write(BGZF* fp, const void* data, int length); - -/* - * Return a virtual file pointer to the current location in the file. - * No interpetation of the value should be made, other than a subsequent - * call to bgzf_seek can be used to position the file at the same point. - * Return value is non-negative on success. - * Returns -1 on error. - */ -#define bgzf_tell(fp) ((fp->block_address << 16) | (fp->block_offset & 0xFFFF)) - -/* - * Set the file to read from the location specified by pos, which must - * be a value previously returned by bgzf_tell for this file (but not - * necessarily one returned by this file handle). - * The where argument must be SEEK_SET. - * Seeking on a file opened for write is not supported. - * Returns zero on success, -1 on error. - */ -int64_t bgzf_seek(BGZF* fp, int64_t pos, int where); - -/* - * Set the cache size. Zero to disable. By default, caching is - * disabled. The recommended cache size for frequent random access is - * about 8M bytes. - */ -void bgzf_set_cache_size(BGZF *fp, int cache_size); - -int bgzf_check_EOF(BGZF *fp); -int bgzf_read_block(BGZF* fp); -int bgzf_flush(BGZF* fp); -int bgzf_flush_try(BGZF *fp, int size); - -#ifdef __cplusplus -} -#endif - -static inline int bgzf_getc(BGZF *fp) -{ - int c; - if (fp->block_offset >= fp->block_length) { - if (bgzf_read_block(fp) != 0) return -2; /* error */ - if (fp->block_length == 0) return -1; /* end-of-file */ - } - c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++]; - if (fp->block_offset == fp->block_length) { -#ifdef _USE_KNETFILE - fp->block_address = knet_tell(fp->x.fpr); -#else - fp->block_address = ftello(fp->file); -#endif - fp->block_offset = 0; - fp->block_length = 0; - } - return c; -} - -#endif diff --git a/samtools-0.1.13/errmod.c b/samtools-0.1.13/errmod.c deleted file mode 100644 --- a/samtools-0.1.13/errmod.c +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include "errmod.h" -#include "ksort.h" -KSORT_INIT_GENERIC(uint16_t) - -typedef struct __errmod_coef_t { - double *fk, *beta, *lhet; -} errmod_coef_t; - -typedef struct { - double fsum[16], bsum[16]; - uint32_t c[16]; -} call_aux_t; - -static errmod_coef_t *cal_coef(double depcorr, double eta) -{ - int k, n, q; - long double sum, sum1; - double *lC; - errmod_coef_t *ec; - - ec = calloc(1, sizeof(errmod_coef_t)); - // initialize ->fk - ec->fk = (double*)calloc(256, sizeof(double)); - ec->fk[0] = 1.0; - for (n = 1; n != 256; ++n) - ec->fk[n] = pow(1. - depcorr, n) * (1.0 - eta) + eta; - // initialize ->coef - ec->beta = (double*)calloc(256 * 256 * 64, sizeof(double)); - lC = (double*)calloc(256 * 256, sizeof(double)); - for (n = 1; n != 256; ++n) { - double lgn = lgamma(n+1); - for (k = 1; k <= n; ++k) - lC[n<<8|k] = lgn - lgamma(k+1) - lgamma(n-k+1); - } - for (q = 1; q != 64; ++q) { - double e = pow(10.0, -q/10.0); - double le = log(e); - double le1 = log(1.0 - e); - for (n = 1; n <= 255; ++n) { - double *beta = ec->beta + (q<<16|n<<8); - sum1 = sum = 0.0; - for (k = n; k >= 0; --k, sum1 = sum) { - sum = sum1 + expl(lC[n<<8|k] + k*le + (n-k)*le1); - beta[k] = -10. / M_LN10 * logl(sum1 / sum); - } - } - } - // initialize ->lhet - ec->lhet = (double*)calloc(256 * 256, sizeof(double)); - for (n = 0; n < 256; ++n) - for (k = 0; k < 256; ++k) - ec->lhet[n<<8|k] = lC[n<<8|k] - M_LN2 * n; - free(lC); - return ec; -} - -errmod_t *errmod_init(float depcorr) -{ - errmod_t *em; - em = (errmod_t*)calloc(1, sizeof(errmod_t)); - em->depcorr = depcorr; - em->coef = cal_coef(depcorr, 0.03); - return em; -} - -void errmod_destroy(errmod_t *em) -{ - if (em == 0) return; - free(em->coef->lhet); free(em->coef->fk); free(em->coef->beta); - free(em->coef); free(em); -} -// qual:6, strand:1, base:4 -int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q) -{ - call_aux_t aux; - int i, j, k, w[32]; - - if (m > m) return -1; - memset(q, 0, m * m * sizeof(float)); - if (n == 0) return 0; - // calculate aux.esum and aux.fsum - if (n > 255) { // then sample 255 bases - ks_shuffle(uint16_t, n, bases); - n = 255; - } - ks_introsort(uint16_t, n, bases); - memset(w, 0, 32 * sizeof(int)); - memset(&aux, 0, sizeof(call_aux_t)); - for (j = n - 1; j >= 0; --j) { // calculate esum and fsum - uint16_t b = bases[j]; - int q = b>>5 < 4? 4 : b>>5; - if (q > 63) q = 63; - k = b&0x1f; - aux.fsum[k&0xf] += em->coef->fk[w[k]]; - aux.bsum[k&0xf] += em->coef->fk[w[k]] * em->coef->beta[q<<16|n<<8|aux.c[k&0xf]]; - ++aux.c[k&0xf]; - ++w[k]; - } - // generate likelihood - for (j = 0; j != m; ++j) { - float tmp1, tmp3; - int tmp2, bar_e; - // homozygous - for (k = 0, tmp1 = tmp3 = 0.0, tmp2 = 0; k != m; ++k) { - if (k == j) continue; - tmp1 += aux.bsum[k]; tmp2 += aux.c[k]; tmp3 += aux.fsum[k]; - } - if (tmp2) { - bar_e = (int)(tmp1 / tmp3 + 0.499); - if (bar_e > 63) bar_e = 63; - q[j*m+j] = tmp1; - } - // heterozygous - for (k = j + 1; k < m; ++k) { - int cjk = aux.c[j] + aux.c[k]; - for (i = 0, tmp2 = 0, tmp1 = tmp3 = 0.0; i < m; ++i) { - if (i == j || i == k) continue; - tmp1 += aux.bsum[i]; tmp2 += aux.c[i]; tmp3 += aux.fsum[i]; - } - if (tmp2) { - bar_e = (int)(tmp1 / tmp3 + 0.499); - if (bar_e > 63) bar_e = 63; - q[j*m+k] = q[k*m+j] = -4.343 * em->coef->lhet[cjk<<8|aux.c[k]] + tmp1; - } else q[j*m+k] = q[k*m+j] = -4.343 * em->coef->lhet[cjk<<8|aux.c[k]]; // all the bases are either j or k - } - for (k = 0; k != m; ++k) if (q[j*m+k] < 0.0) q[j*m+k] = 0.0; - } - return 0; -} diff --git a/samtools-0.1.13/errmod.h b/samtools-0.1.13/errmod.h deleted file mode 100644 --- a/samtools-0.1.13/errmod.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef ERRMOD_H -#define ERRMOD_H - -#include - -struct __errmod_coef_t; - -typedef struct { - double depcorr; - struct __errmod_coef_t *coef; -} errmod_t; - -errmod_t *errmod_init(float depcorr); -void errmod_destroy(errmod_t *em); - -/* - n: number of bases - m: maximum base - bases[i]: qual:6, strand:1, base:4 - q[i*m+j]: phred-scaled likelihood of (i,j) - */ -int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q); - -#endif diff --git a/samtools-0.1.13/faidx.c b/samtools-0.1.13/faidx.c deleted file mode 100644 --- a/samtools-0.1.13/faidx.c +++ /dev/null @@ -1,423 +0,0 @@ -#include -#include -#include -#include -#include -#include "faidx.h" -#include "khash.h" - -typedef struct { - uint64_t len:32, line_len:16, line_blen:16; - uint64_t offset; -} faidx1_t; -KHASH_MAP_INIT_STR(s, faidx1_t) - -#ifndef _NO_RAZF -#include "razf.h" -#else -#ifdef _WIN32 -#define ftello(fp) ftell(fp) -#define fseeko(fp, offset, whence) fseek(fp, offset, whence) -#else -extern off_t ftello(FILE *stream); -extern int fseeko(FILE *stream, off_t offset, int whence); -#endif -#define RAZF FILE -#define razf_read(fp, buf, size) fread(buf, 1, size, fp) -#define razf_open(fn, mode) fopen(fn, mode) -#define razf_close(fp) fclose(fp) -#define razf_seek(fp, offset, whence) fseeko(fp, offset, whence) -#define razf_tell(fp) ftello(fp) -#endif -#ifdef _USE_KNETFILE -#include "knetfile.h" -#endif - -struct __faidx_t { - RAZF *rz; - int n, m; - char **name; - khash_t(s) *hash; -}; - -#ifndef kroundup32 -#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) -#endif - -static inline void fai_insert_index(faidx_t *idx, const char *name, int len, int line_len, int line_blen, uint64_t offset) -{ - khint_t k; - int ret; - faidx1_t t; - if (idx->n == idx->m) { - idx->m = idx->m? idx->m<<1 : 16; - idx->name = (char**)realloc(idx->name, sizeof(void*) * idx->m); - } - idx->name[idx->n] = strdup(name); - k = kh_put(s, idx->hash, idx->name[idx->n], &ret); - t.len = len; t.line_len = line_len; t.line_blen = line_blen; t.offset = offset; - kh_value(idx->hash, k) = t; - ++idx->n; -} - -faidx_t *fai_build_core(RAZF *rz) -{ - char c, *name; - int l_name, m_name, ret; - int len, line_len, line_blen, state; - int l1, l2; - faidx_t *idx; - uint64_t offset; - - idx = (faidx_t*)calloc(1, sizeof(faidx_t)); - idx->hash = kh_init(s); - name = 0; l_name = m_name = 0; - len = line_len = line_blen = -1; state = 0; l1 = l2 = -1; offset = 0; - while (razf_read(rz, &c, 1)) { - if (c == '\n') { // an empty line - if (state == 1) { - offset = razf_tell(rz); - continue; - } else if ((state == 0 && len < 0) || state == 2) continue; - } - if (c == '>') { // fasta header - if (len >= 0) - fai_insert_index(idx, name, len, line_len, line_blen, offset); - l_name = 0; - while ((ret = razf_read(rz, &c, 1)) != 0 && !isspace(c)) { - if (m_name < l_name + 2) { - m_name = l_name + 2; - kroundup32(m_name); - name = (char*)realloc(name, m_name); - } - name[l_name++] = c; - } - name[l_name] = '\0'; - if (ret == 0) { - fprintf(stderr, "[fai_build_core] the last entry has no sequence\n"); - free(name); fai_destroy(idx); - return 0; - } - if (c != '\n') while (razf_read(rz, &c, 1) && c != '\n'); - state = 1; len = 0; - offset = razf_tell(rz); - } else { - if (state == 3) { - fprintf(stderr, "[fai_build_core] inlined empty line is not allowed in sequence '%s'.\n", name); - free(name); fai_destroy(idx); - return 0; - } - if (state == 2) state = 3; - l1 = l2 = 0; - do { - ++l1; - if (isgraph(c)) ++l2; - } while ((ret = razf_read(rz, &c, 1)) && c != '\n'); - if (state == 3 && l2) { - fprintf(stderr, "[fai_build_core] different line length in sequence '%s'.\n", name); - free(name); fai_destroy(idx); - return 0; - } - ++l1; len += l2; - if (l2 >= 0x10000) { - fprintf(stderr, "[fai_build_core] line length exceeds 65535 in sequence '%s'.\n", name); - free(name); fai_destroy(idx); - return 0; - } - if (state == 1) line_len = l1, line_blen = l2, state = 0; - else if (state == 0) { - if (l1 != line_len || l2 != line_blen) state = 2; - } - } - } - fai_insert_index(idx, name, len, line_len, line_blen, offset); - free(name); - return idx; -} - -void fai_save(const faidx_t *fai, FILE *fp) -{ - khint_t k; - int i; - for (i = 0; i < fai->n; ++i) { - faidx1_t x; - k = kh_get(s, fai->hash, fai->name[i]); - x = kh_value(fai->hash, k); -#ifdef _WIN32 - fprintf(fp, "%s\t%d\t%ld\t%d\t%d\n", fai->name[i], (int)x.len, (long)x.offset, (int)x.line_blen, (int)x.line_len); -#else - fprintf(fp, "%s\t%d\t%lld\t%d\t%d\n", fai->name[i], (int)x.len, (long long)x.offset, (int)x.line_blen, (int)x.line_len); -#endif - } -} - -faidx_t *fai_read(FILE *fp) -{ - faidx_t *fai; - char *buf, *p; - int len, line_len, line_blen; -#ifdef _WIN32 - long offset; -#else - long long offset; -#endif - fai = (faidx_t*)calloc(1, sizeof(faidx_t)); - fai->hash = kh_init(s); - buf = (char*)calloc(0x10000, 1); - while (!feof(fp) && fgets(buf, 0x10000, fp)) { - for (p = buf; *p && isgraph(*p); ++p); - *p = 0; ++p; -#ifdef _WIN32 - sscanf(p, "%d%ld%d%d", &len, &offset, &line_blen, &line_len); -#else - sscanf(p, "%d%lld%d%d", &len, &offset, &line_blen, &line_len); -#endif - fai_insert_index(fai, buf, len, line_len, line_blen, offset); - } - free(buf); - return fai; -} - -void fai_destroy(faidx_t *fai) -{ - int i; - for (i = 0; i < fai->n; ++i) free(fai->name[i]); - free(fai->name); - kh_destroy(s, fai->hash); - if (fai->rz) razf_close(fai->rz); - free(fai); -} - -int fai_build(const char *fn) -{ - char *str; - RAZF *rz; - FILE *fp; - faidx_t *fai; - str = (char*)calloc(strlen(fn) + 5, 1); - sprintf(str, "%s.fai", fn); - rz = razf_open(fn, "r"); - if (rz == 0) { - fprintf(stderr, "[fai_build] fail to open the FASTA file %s\n",fn); - free(str); - return -1; - } - fai = fai_build_core(rz); - razf_close(rz); - fp = fopen(str, "wb"); - if (fp == 0) { - fprintf(stderr, "[fai_build] fail to write FASTA index %s\n",str); - fai_destroy(fai); free(str); - return -1; - } - fai_save(fai, fp); - fclose(fp); - free(str); - fai_destroy(fai); - return 0; -} - -#ifdef _USE_KNETFILE -FILE *download_and_open(const char *fn) -{ - const int buf_size = 1 * 1024 * 1024; - uint8_t *buf; - FILE *fp; - knetFile *fp_remote; - const char *url = fn; - const char *p; - int l = strlen(fn); - for (p = fn + l - 1; p >= fn; --p) - if (*p == '/') break; - fn = p + 1; - - // First try to open a local copy - fp = fopen(fn, "r"); - if (fp) - return fp; - - // If failed, download from remote and open - fp_remote = knet_open(url, "rb"); - if (fp_remote == 0) { - fprintf(stderr, "[download_from_remote] fail to open remote file %s\n",url); - return NULL; - } - if ((fp = fopen(fn, "wb")) == 0) { - fprintf(stderr, "[download_from_remote] fail to create file in the working directory %s\n",fn); - knet_close(fp_remote); - return NULL; - } - buf = (uint8_t*)calloc(buf_size, 1); - while ((l = knet_read(fp_remote, buf, buf_size)) != 0) - fwrite(buf, 1, l, fp); - free(buf); - fclose(fp); - knet_close(fp_remote); - - return fopen(fn, "r"); -} -#endif - -faidx_t *fai_load(const char *fn) -{ - char *str; - FILE *fp; - faidx_t *fai; - str = (char*)calloc(strlen(fn) + 5, 1); - sprintf(str, "%s.fai", fn); - -#ifdef _USE_KNETFILE - if (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn) - { - fp = download_and_open(str); - if ( !fp ) - { - fprintf(stderr, "[fai_load] failed to open remote FASTA index %s\n", str); - free(str); - return 0; - } - } - else -#endif - fp = fopen(str, "rb"); - if (fp == 0) { - fprintf(stderr, "[fai_load] build FASTA index.\n"); - fai_build(fn); - fp = fopen(str, "rb"); - if (fp == 0) { - fprintf(stderr, "[fai_load] fail to open FASTA index.\n"); - free(str); - return 0; - } - } - - fai = fai_read(fp); - fclose(fp); - - fai->rz = razf_open(fn, "rb"); - free(str); - if (fai->rz == 0) { - fprintf(stderr, "[fai_load] fail to open FASTA file.\n"); - return 0; - } - return fai; -} - -char *fai_fetch(const faidx_t *fai, const char *str, int *len) -{ - char *s, *p, c; - int i, l, k; - khiter_t iter; - faidx1_t val; - khash_t(s) *h; - int beg, end; - - beg = end = -1; - h = fai->hash; - l = strlen(str); - p = s = (char*)malloc(l+1); - /* squeeze out "," */ - for (i = k = 0; i != l; ++i) - if (str[i] != ',' && !isspace(str[i])) s[k++] = str[i]; - s[k] = 0; - for (i = 0; i != k; ++i) if (s[i] == ':') break; - s[i] = 0; - iter = kh_get(s, h, s); /* get the ref_id */ - if (iter == kh_end(h)) { - *len = 0; - free(s); return 0; - } - val = kh_value(h, iter); - if (i == k) { /* dump the whole sequence */ - beg = 0; end = val.len; - } else { - for (p = s + i + 1; i != k; ++i) if (s[i] == '-') break; - beg = atoi(p); - if (i < k) { - p = s + i + 1; - end = atoi(p); - } else end = val.len; - } - if (beg > 0) --beg; - if (beg >= val.len) beg = val.len; - if (end >= val.len) end = val.len; - if (beg > end) beg = end; - free(s); - - // now retrieve the sequence - l = 0; - s = (char*)malloc(end - beg + 2); - razf_seek(fai->rz, val.offset + beg / val.line_blen * val.line_len + beg % val.line_blen, SEEK_SET); - while (razf_read(fai->rz, &c, 1) == 1 && l < end - beg && !fai->rz->z_err) - if (isgraph(c)) s[l++] = c; - s[l] = '\0'; - *len = l; - return s; -} - -int faidx_main(int argc, char *argv[]) -{ - if (argc == 1) { - fprintf(stderr, "Usage: faidx [ [...]]\n"); - return 1; - } else { - if (argc == 2) fai_build(argv[1]); - else { - int i, j, k, l; - char *s; - faidx_t *fai; - fai = fai_load(argv[1]); - if (fai == 0) return 1; - for (i = 2; i != argc; ++i) { - printf(">%s\n", argv[i]); - s = fai_fetch(fai, argv[i], &l); - for (j = 0; j < l; j += 60) { - for (k = 0; k < 60 && k < l - j; ++k) - putchar(s[j + k]); - putchar('\n'); - } - free(s); - } - fai_destroy(fai); - } - } - return 0; -} - -int faidx_fetch_nseq(const faidx_t *fai) -{ - return fai->n; -} - -char *faidx_fetch_seq(const faidx_t *fai, char *c_name, int p_beg_i, int p_end_i, int *len) -{ - int l; - char c; - khiter_t iter; - faidx1_t val; - char *seq=NULL; - - // Adjust position - iter = kh_get(s, fai->hash, c_name); - if(iter == kh_end(fai->hash)) return 0; - val = kh_value(fai->hash, iter); - if(p_end_i < p_beg_i) p_beg_i = p_end_i; - if(p_beg_i < 0) p_beg_i = 0; - else if(val.len <= p_beg_i) p_beg_i = val.len - 1; - if(p_end_i < 0) p_end_i = 0; - else if(val.len <= p_end_i) p_end_i = val.len - 1; - - // Now retrieve the sequence - l = 0; - seq = (char*)malloc(p_end_i - p_beg_i + 2); - razf_seek(fai->rz, val.offset + p_beg_i / val.line_blen * val.line_len + p_beg_i % val.line_blen, SEEK_SET); - while (razf_read(fai->rz, &c, 1) == 1 && l < p_end_i - p_beg_i + 1) - if (isgraph(c)) seq[l++] = c; - seq[l] = '\0'; - *len = l; - return seq; -} - -#ifdef FAIDX_MAIN -int main(int argc, char *argv[]) { return faidx_main(argc, argv); } -#endif diff --git a/samtools-0.1.13/faidx.h b/samtools-0.1.13/faidx.h deleted file mode 100644 --- a/samtools-0.1.13/faidx.h +++ /dev/null @@ -1,103 +0,0 @@ -/* The MIT License - - Copyright (c) 2008 Genome Research Ltd (GRL). - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -/* Contact: Heng Li */ - -#ifndef FAIDX_H -#define FAIDX_H - -/*! - @header - - Index FASTA files and extract subsequence. - - @copyright The Wellcome Trust Sanger Institute. - */ - -struct __faidx_t; -typedef struct __faidx_t faidx_t; - -#ifdef __cplusplus -extern "C" { -#endif - - /*! - @abstract Build index for a FASTA or razip compressed FASTA file. - @param fn FASTA file name - @return 0 on success; or -1 on failure - @discussion File "fn.fai" will be generated. - */ - int fai_build(const char *fn); - - /*! - @abstract Distroy a faidx_t struct. - @param fai Pointer to the struct to be destroyed - */ - void fai_destroy(faidx_t *fai); - - /*! - @abstract Load index from "fn.fai". - @param fn File name of the FASTA file - */ - faidx_t *fai_load(const char *fn); - - /*! - @abstract Fetch the sequence in a region. - @param fai Pointer to the faidx_t struct - @param reg Region in the format "chr2:20,000-30,000" - @param len Length of the region - @return Pointer to the sequence; null on failure - - @discussion The returned sequence is allocated by malloc family - and should be destroyed by end users by calling free() on it. - */ - char *fai_fetch(const faidx_t *fai, const char *reg, int *len); - - /*! - @abstract Fetch the number of sequences. - @param fai Pointer to the faidx_t struct - @return The number of sequences - */ - int faidx_fetch_nseq(const faidx_t *fai); - - /*! - @abstract Fetch the sequence in a region. - @param fai Pointer to the faidx_t struct - @param c_name Region name - @param p_beg_i Beginning position number (zero-based) - @param p_end_i End position number (zero-based) - @param len Length of the region - @return Pointer to the sequence; null on failure - - @discussion The returned sequence is allocated by malloc family - and should be destroyed by end users by calling free() on it. - */ - char *faidx_fetch_seq(const faidx_t *fai, char *c_name, int p_beg_i, int p_end_i, int *len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.13/glf.c b/samtools-0.1.13/glf.c deleted file mode 100644 --- a/samtools-0.1.13/glf.c +++ /dev/null @@ -1,236 +0,0 @@ -#include -#include -#include "glf.h" - -#ifdef _NO_BGZF -// then alias bgzf_*() functions -#endif - -static int glf3_is_BE = 0; - -static inline uint32_t bam_swap_endian_4(uint32_t v) -{ - v = ((v & 0x0000FFFFU) << 16) | (v >> 16); - return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8); -} - -static inline uint16_t bam_swap_endian_2(uint16_t v) -{ - return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8)); -} - -static inline int bam_is_big_endian() -{ - long one= 1; - return !(*((char *)(&one))); -} - -glf3_header_t *glf3_header_init() -{ - glf3_is_BE = bam_is_big_endian(); - return (glf3_header_t*)calloc(1, sizeof(glf3_header_t)); -} - -glf3_header_t *glf3_header_read(glfFile fp) -{ - glf3_header_t *h; - char magic[4]; - h = glf3_header_init(); - bgzf_read(fp, magic, 4); - if (strncmp(magic, "GLF\3", 4)) { - fprintf(stderr, "[glf3_header_read] invalid magic.\n"); - glf3_header_destroy(h); - return 0; - } - bgzf_read(fp, &h->l_text, 4); - if (glf3_is_BE) h->l_text = bam_swap_endian_4(h->l_text); - if (h->l_text) { - h->text = (uint8_t*)calloc(h->l_text + 1, 1); - bgzf_read(fp, h->text, h->l_text); - } - return h; -} - -void glf3_header_write(glfFile fp, const glf3_header_t *h) -{ - int32_t x; - bgzf_write(fp, "GLF\3", 4); - x = glf3_is_BE? bam_swap_endian_4(h->l_text) : h->l_text; - bgzf_write(fp, &x, 4); - if (h->l_text) bgzf_write(fp, h->text, h->l_text); -} - -void glf3_header_destroy(glf3_header_t *h) -{ - free(h->text); - free(h); -} - -char *glf3_ref_read(glfFile fp, int *len) -{ - int32_t n, x; - char *str; - *len = 0; - if (bgzf_read(fp, &n, 4) != 4) return 0; - if (glf3_is_BE) n = bam_swap_endian_4(n); - if (n < 0) { - fprintf(stderr, "[glf3_ref_read] invalid reference name length: %d.\n", n); - return 0; - } - str = (char*)calloc(n + 1, 1); // not necesarily n+1 in fact - x = bgzf_read(fp, str, n); - x += bgzf_read(fp, len, 4); - if (x != n + 4) { - free(str); *len = -1; return 0; // truncated - } - if (glf3_is_BE) *len = bam_swap_endian_4(*len); - return str; -} - -void glf3_ref_write(glfFile fp, const char *str, int len) -{ - int32_t m, n = strlen(str) + 1; - m = glf3_is_BE? bam_swap_endian_4(n) : n; - bgzf_write(fp, &m, 4); - bgzf_write(fp, str, n); - if (glf3_is_BE) len = bam_swap_endian_4(len); - bgzf_write(fp, &len, 4); -} - -void glf3_view1(const char *ref_name, const glf3_t *g3, int pos) -{ - int j; - if (g3->rtype == GLF3_RTYPE_END) return; - printf("%s\t%d\t%c\t%d\t%d\t%d", ref_name, pos + 1, - g3->rtype == GLF3_RTYPE_INDEL? '*' : "XACMGRSVTWYHKDBN"[g3->ref_base], - g3->depth, g3->rms_mapQ, g3->min_lk); - if (g3->rtype == GLF3_RTYPE_SUB) - for (j = 0; j != 10; ++j) printf("\t%d", g3->lk[j]); - else { - printf("\t%d\t%d\t%d\t%d\t%d\t%s\t%s\t", g3->lk[0], g3->lk[1], g3->lk[2], g3->indel_len[0], g3->indel_len[1], - g3->indel_len[0]? g3->indel_seq[0] : "*", g3->indel_len[1]? g3->indel_seq[1] : "*"); - } - printf("\n"); -} - -int glf3_write1(glfFile fp, const glf3_t *g3) -{ - int r; - uint8_t c; - uint32_t y[2]; - c = g3->rtype<<4 | g3->ref_base; - r = bgzf_write(fp, &c, 1); - if (g3->rtype == GLF3_RTYPE_END) return r; - y[0] = g3->offset; - y[1] = g3->min_lk<<24 | g3->depth; - if (glf3_is_BE) { - y[0] = bam_swap_endian_4(y[0]); - y[1] = bam_swap_endian_4(y[1]); - } - r += bgzf_write(fp, y, 8); - r += bgzf_write(fp, &g3->rms_mapQ, 1); - if (g3->rtype == GLF3_RTYPE_SUB) r += bgzf_write(fp, g3->lk, 10); - else { - int16_t x[2]; - r += bgzf_write(fp, g3->lk, 3); - x[0] = glf3_is_BE? bam_swap_endian_2(g3->indel_len[0]) : g3->indel_len[0]; - x[1] = glf3_is_BE? bam_swap_endian_2(g3->indel_len[1]) : g3->indel_len[1]; - r += bgzf_write(fp, x, 4); - if (g3->indel_len[0]) r += bgzf_write(fp, g3->indel_seq[0], abs(g3->indel_len[0])); - if (g3->indel_len[1]) r += bgzf_write(fp, g3->indel_seq[1], abs(g3->indel_len[1])); - } - return r; -} - -#ifndef kv_roundup32 -#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) -#endif - -int glf3_read1(glfFile fp, glf3_t *g3) -{ - int r; - uint8_t c; - uint32_t y[2]; - r = bgzf_read(fp, &c, 1); - if (r == 0) return 0; - g3->ref_base = c & 0xf; - g3->rtype = c>>4; - if (g3->rtype == GLF3_RTYPE_END) return r; - r += bgzf_read(fp, y, 8); - if (glf3_is_BE) { - y[0] = bam_swap_endian_4(y[0]); - y[1] = bam_swap_endian_4(y[1]); - } - g3->offset = y[0]; - g3->min_lk = y[1]>>24; - g3->depth = y[1]<<8>>8; - r += bgzf_read(fp, &g3->rms_mapQ, 1); - if (g3->rtype == GLF3_RTYPE_SUB) r += bgzf_read(fp, g3->lk, 10); - else { - int16_t x[2], max; - r += bgzf_read(fp, g3->lk, 3); - r += bgzf_read(fp, x, 4); - if (glf3_is_BE) { - x[0] = bam_swap_endian_2(x[0]); - x[1] = bam_swap_endian_2(x[1]); - } - g3->indel_len[0] = x[0]; - g3->indel_len[1] = x[1]; - x[0] = abs(x[0]); x[1] = abs(x[1]); - max = (x[0] > x[1]? x[0] : x[1]) + 1; - if (g3->max_len < max) { - g3->max_len = max; - kv_roundup32(g3->max_len); - g3->indel_seq[0] = (char*)realloc(g3->indel_seq[0], g3->max_len); - g3->indel_seq[1] = (char*)realloc(g3->indel_seq[1], g3->max_len); - } - r += bgzf_read(fp, g3->indel_seq[0], x[0]); - r += bgzf_read(fp, g3->indel_seq[1], x[1]); - g3->indel_seq[0][x[0]] = g3->indel_seq[1][x[1]] = 0; - } - return r; -} - -void glf3_view(glfFile fp) -{ - glf3_header_t *h; - char *name; - glf3_t *g3; - int len; - h = glf3_header_read(fp); - g3 = glf3_init1(); - while ((name = glf3_ref_read(fp, &len)) != 0) { - int pos = 0; - while (glf3_read1(fp, g3) && g3->rtype != GLF3_RTYPE_END) { - pos += g3->offset; - glf3_view1(name, g3, pos); - } - free(name); - } - glf3_header_destroy(h); - glf3_destroy1(g3); -} - -int glf3_view_main(int argc, char *argv[]) -{ - glfFile fp; - if (argc == 1) { - fprintf(stderr, "Usage: glfview \n"); - return 1; - } - fp = (strcmp(argv[1], "-") == 0)? bgzf_fdopen(fileno(stdin), "r") : bgzf_open(argv[1], "r"); - if (fp == 0) { - fprintf(stderr, "Fail to open file '%s'\n", argv[1]); - return 1; - } - glf3_view(fp); - bgzf_close(fp); - return 0; -} - -#ifdef GLFVIEW_MAIN -int main(int argc, char *argv[]) -{ - return glf3_view_main(argc, argv); -} -#endif diff --git a/samtools-0.1.13/glf.h b/samtools-0.1.13/glf.h deleted file mode 100644 --- a/samtools-0.1.13/glf.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef GLF_H_ -#define GLF_H_ - -typedef struct { - unsigned char ref_base:4, dummy:4; /** "XACMGRSVTWYHKDBN"[ref_base] gives the reference base */ - unsigned char max_mapQ; /** maximum mapping quality */ - unsigned char lk[10]; /** log likelihood ratio, capped at 255 */ - unsigned min_lk:8, depth:24; /** minimum lk capped at 255, and the number of mapped reads */ -} glf1_t; - -#include -#include "bgzf.h" -typedef BGZF *glfFile; - -#define GLF3_RTYPE_END 0 -#define GLF3_RTYPE_SUB 1 -#define GLF3_RTYPE_INDEL 2 - -typedef struct { - uint8_t ref_base:4, rtype:4; /** "XACMGRSVTWYHKDBN"[ref_base] gives the reference base */ - uint8_t rms_mapQ; /** RMS mapping quality */ - uint8_t lk[10]; /** log likelihood ratio, capped at 255 */ - uint32_t min_lk:8, depth:24; /** minimum lk capped at 255, and the number of mapped reads */ - int32_t offset; /** the first base in a chromosome has offset zero. */ - // for indel (lkHom1, lkHom2 and lkHet are the first three elements in lk[10]) - int16_t indel_len[2]; - int32_t max_len; // maximum indel len; will be modified by glf3_read1() - char *indel_seq[2]; -} glf3_t; - -typedef struct { - int32_t l_text; - uint8_t *text; -} glf3_header_t; - -#ifdef __cplusplus -extern "C" { -#endif - -#define glf3_init1() ((glf3_t*)calloc(1, sizeof(glf3_t))) -#define glf3_destroy1(g3) do { free((g3)->indel_seq[0]); free((g3)->indel_seq[1]); free(g3); } while (0) - - glf3_header_t *glf3_header_init(); - glf3_header_t *glf3_header_read(glfFile fp); - void glf3_header_write(glfFile fp, const glf3_header_t *h); - void glf3_header_destroy(glf3_header_t *h); - char *glf3_ref_read(glfFile fp, int *len); - void glf3_ref_write(glfFile fp, const char *name, int len); - int glf3_write1(glfFile fp, const glf3_t *g3); - int glf3_read1(glfFile fp, glf3_t *g3); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.13/kaln.c b/samtools-0.1.13/kaln.c deleted file mode 100644 --- a/samtools-0.1.13/kaln.c +++ /dev/null @@ -1,486 +0,0 @@ -/* The MIT License - - Copyright (c) 2003-2006, 2008, 2009, by Heng Li - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#include -#include -#include -#include -#include -#include "kaln.h" - -#define FROM_M 0 -#define FROM_I 1 -#define FROM_D 2 - -typedef struct { - int i, j; - unsigned char ctype; -} path_t; - -int aln_sm_blosum62[] = { -/* A R N D C Q E G H I L K M F P S T W Y V * X */ - 4,-1,-2,-2, 0,-1,-1, 0,-2,-1,-1,-1,-1,-2,-1, 1, 0,-3,-2, 0,-4, 0, - -1, 5, 0,-2,-3, 1, 0,-2, 0,-3,-2, 2,-1,-3,-2,-1,-1,-3,-2,-3,-4,-1, - -2, 0, 6, 1,-3, 0, 0, 0, 1,-3,-3, 0,-2,-3,-2, 1, 0,-4,-2,-3,-4,-1, - -2,-2, 1, 6,-3, 0, 2,-1,-1,-3,-4,-1,-3,-3,-1, 0,-1,-4,-3,-3,-4,-1, - 0,-3,-3,-3, 9,-3,-4,-3,-3,-1,-1,-3,-1,-2,-3,-1,-1,-2,-2,-1,-4,-2, - -1, 1, 0, 0,-3, 5, 2,-2, 0,-3,-2, 1, 0,-3,-1, 0,-1,-2,-1,-2,-4,-1, - -1, 0, 0, 2,-4, 2, 5,-2, 0,-3,-3, 1,-2,-3,-1, 0,-1,-3,-2,-2,-4,-1, - 0,-2, 0,-1,-3,-2,-2, 6,-2,-4,-4,-2,-3,-3,-2, 0,-2,-2,-3,-3,-4,-1, - -2, 0, 1,-1,-3, 0, 0,-2, 8,-3,-3,-1,-2,-1,-2,-1,-2,-2, 2,-3,-4,-1, - -1,-3,-3,-3,-1,-3,-3,-4,-3, 4, 2,-3, 1, 0,-3,-2,-1,-3,-1, 3,-4,-1, - -1,-2,-3,-4,-1,-2,-3,-4,-3, 2, 4,-2, 2, 0,-3,-2,-1,-2,-1, 1,-4,-1, - -1, 2, 0,-1,-3, 1, 1,-2,-1,-3,-2, 5,-1,-3,-1, 0,-1,-3,-2,-2,-4,-1, - -1,-1,-2,-3,-1, 0,-2,-3,-2, 1, 2,-1, 5, 0,-2,-1,-1,-1,-1, 1,-4,-1, - -2,-3,-3,-3,-2,-3,-3,-3,-1, 0, 0,-3, 0, 6,-4,-2,-2, 1, 3,-1,-4,-1, - -1,-2,-2,-1,-3,-1,-1,-2,-2,-3,-3,-1,-2,-4, 7,-1,-1,-4,-3,-2,-4,-2, - 1,-1, 1, 0,-1, 0, 0, 0,-1,-2,-2, 0,-1,-2,-1, 4, 1,-3,-2,-2,-4, 0, - 0,-1, 0,-1,-1,-1,-1,-2,-2,-1,-1,-1,-1,-2,-1, 1, 5,-2,-2, 0,-4, 0, - -3,-3,-4,-4,-2,-2,-3,-2,-2,-3,-2,-3,-1, 1,-4,-3,-2,11, 2,-3,-4,-2, - -2,-2,-2,-3,-2,-1,-2,-3, 2,-1,-1,-2,-1, 3,-3,-2,-2, 2, 7,-1,-4,-1, - 0,-3,-3,-3,-1,-2,-2,-3,-3, 3, 1,-2, 1,-1,-2,-2, 0,-3,-1, 4,-4,-1, - -4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4, 1,-4, - 0,-1,-1,-1,-2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-2, 0, 0,-2,-1,-1,-4,-1 -}; - -int aln_sm_blast[] = { - 1, -3, -3, -3, -2, - -3, 1, -3, -3, -2, - -3, -3, 1, -3, -2, - -3, -3, -3, 1, -2, - -2, -2, -2, -2, -2 -}; - -int aln_sm_qual[] = { - 0, -23, -23, -23, 0, - -23, 0, -23, -23, 0, - -23, -23, 0, -23, 0, - -23, -23, -23, 0, 0, - 0, 0, 0, 0, 0 -}; - -ka_param_t ka_param_blast = { 5, 2, 5, 2, aln_sm_blast, 5, 50 }; -ka_param_t ka_param_aa2aa = { 10, 2, 10, 2, aln_sm_blosum62, 22, 50 }; - -ka_param2_t ka_param2_qual = { 37, 11, 37, 11, 37, 11, 0, 0, aln_sm_qual, 5, 50 }; - -static uint32_t *ka_path2cigar32(const path_t *path, int path_len, int *n_cigar) -{ - int i, n; - uint32_t *cigar; - unsigned char last_type; - - if (path_len == 0 || path == 0) { - *n_cigar = 0; - return 0; - } - - last_type = path->ctype; - for (i = n = 1; i < path_len; ++i) { - if (last_type != path[i].ctype) ++n; - last_type = path[i].ctype; - } - *n_cigar = n; - cigar = (uint32_t*)calloc(*n_cigar, 4); - - cigar[0] = 1u << 4 | path[path_len-1].ctype; - last_type = path[path_len-1].ctype; - for (i = path_len - 2, n = 0; i >= 0; --i) { - if (path[i].ctype == last_type) cigar[n] += 1u << 4; - else { - cigar[++n] = 1u << 4 | path[i].ctype; - last_type = path[i].ctype; - } - } - - return cigar; -} - -/***************************/ -/* START OF common_align.c */ -/***************************/ - -#define SET_INF(s) (s).M = (s).I = (s).D = MINOR_INF; - -#define set_M(MM, cur, p, sc) \ -{ \ - if ((p)->M >= (p)->I) { \ - if ((p)->M >= (p)->D) { \ - (MM) = (p)->M + (sc); (cur)->Mt = FROM_M; \ - } else { \ - (MM) = (p)->D + (sc); (cur)->Mt = FROM_D; \ - } \ - } else { \ - if ((p)->I > (p)->D) { \ - (MM) = (p)->I + (sc); (cur)->Mt = FROM_I; \ - } else { \ - (MM) = (p)->D + (sc); (cur)->Mt = FROM_D; \ - } \ - } \ -} -#define set_I(II, cur, p) \ -{ \ - if ((p)->M - gap_open > (p)->I) { \ - (cur)->It = FROM_M; \ - (II) = (p)->M - gap_open - gap_ext; \ - } else { \ - (cur)->It = FROM_I; \ - (II) = (p)->I - gap_ext; \ - } \ -} -#define set_end_I(II, cur, p) \ -{ \ - if (gap_end_ext >= 0) { \ - if ((p)->M - gap_end_open > (p)->I) { \ - (cur)->It = FROM_M; \ - (II) = (p)->M - gap_end_open - gap_end_ext; \ - } else { \ - (cur)->It = FROM_I; \ - (II) = (p)->I - gap_end_ext; \ - } \ - } else set_I(II, cur, p); \ -} -#define set_D(DD, cur, p) \ -{ \ - if ((p)->M - gap_open > (p)->D) { \ - (cur)->Dt = FROM_M; \ - (DD) = (p)->M - gap_open - gap_ext; \ - } else { \ - (cur)->Dt = FROM_D; \ - (DD) = (p)->D - gap_ext; \ - } \ -} -#define set_end_D(DD, cur, p) \ -{ \ - if (gap_end_ext >= 0) { \ - if ((p)->M - gap_end_open > (p)->D) { \ - (cur)->Dt = FROM_M; \ - (DD) = (p)->M - gap_end_open - gap_end_ext; \ - } else { \ - (cur)->Dt = FROM_D; \ - (DD) = (p)->D - gap_end_ext; \ - } \ - } else set_D(DD, cur, p); \ -} - -typedef struct { - uint8_t Mt:3, It:2, Dt:3; -} dpcell_t; - -typedef struct { - int M, I, D; -} dpscore_t; - -/*************************** - * banded global alignment * - ***************************/ -uint32_t *ka_global_core(uint8_t *seq1, int len1, uint8_t *seq2, int len2, const ka_param_t *ap, int *_score, int *n_cigar) -{ - int i, j; - dpcell_t **dpcell, *q; - dpscore_t *curr, *last, *s; - int b1, b2, tmp_end; - int *mat, end, max = 0; - uint8_t type, ctype; - uint32_t *cigar = 0; - - int gap_open, gap_ext, gap_end_open, gap_end_ext, b; - int *score_matrix, N_MATRIX_ROW; - - /* initialize some align-related parameters. just for compatibility */ - gap_open = ap->gap_open; - gap_ext = ap->gap_ext; - gap_end_open = ap->gap_end_open; - gap_end_ext = ap->gap_end_ext; - b = ap->band_width; - score_matrix = ap->matrix; - N_MATRIX_ROW = ap->row; - - if (n_cigar) *n_cigar = 0; - if (len1 == 0 || len2 == 0) return 0; - - /* calculate b1 and b2 */ - if (len1 > len2) { - b1 = len1 - len2 + b; - b2 = b; - } else { - b1 = b; - b2 = len2 - len1 + b; - } - if (b1 > len1) b1 = len1; - if (b2 > len2) b2 = len2; - --seq1; --seq2; - - /* allocate memory */ - end = (b1 + b2 <= len1)? (b1 + b2 + 1) : (len1 + 1); - dpcell = (dpcell_t**)malloc(sizeof(dpcell_t*) * (len2 + 1)); - for (j = 0; j <= len2; ++j) - dpcell[j] = (dpcell_t*)malloc(sizeof(dpcell_t) * end); - for (j = b2 + 1; j <= len2; ++j) - dpcell[j] -= j - b2; - curr = (dpscore_t*)malloc(sizeof(dpscore_t) * (len1 + 1)); - last = (dpscore_t*)malloc(sizeof(dpscore_t) * (len1 + 1)); - - /* set first row */ - SET_INF(*curr); curr->M = 0; - for (i = 1, s = curr + 1; i < b1; ++i, ++s) { - SET_INF(*s); - set_end_D(s->D, dpcell[0] + i, s - 1); - } - s = curr; curr = last; last = s; - - /* core dynamic programming, part 1 */ - tmp_end = (b2 < len2)? b2 : len2 - 1; - for (j = 1; j <= tmp_end; ++j) { - q = dpcell[j]; s = curr; SET_INF(*s); - set_end_I(s->I, q, last); - end = (j + b1 <= len1 + 1)? (j + b1 - 1) : len1; - mat = score_matrix + seq2[j] * N_MATRIX_ROW; - ++s; ++q; - for (i = 1; i != end; ++i, ++s, ++q) { - set_M(s->M, q, last + i - 1, mat[seq1[i]]); /* this will change s->M ! */ - set_I(s->I, q, last + i); - set_D(s->D, q, s - 1); - } - set_M(s->M, q, last + i - 1, mat[seq1[i]]); - set_D(s->D, q, s - 1); - if (j + b1 - 1 > len1) { /* bug fixed, 040227 */ - set_end_I(s->I, q, last + i); - } else s->I = MINOR_INF; - s = curr; curr = last; last = s; - } - /* last row for part 1, use set_end_D() instead of set_D() */ - if (j == len2 && b2 != len2 - 1) { - q = dpcell[j]; s = curr; SET_INF(*s); - set_end_I(s->I, q, last); - end = (j + b1 <= len1 + 1)? (j + b1 - 1) : len1; - mat = score_matrix + seq2[j] * N_MATRIX_ROW; - ++s; ++q; - for (i = 1; i != end; ++i, ++s, ++q) { - set_M(s->M, q, last + i - 1, mat[seq1[i]]); /* this will change s->M ! */ - set_I(s->I, q, last + i); - set_end_D(s->D, q, s - 1); - } - set_M(s->M, q, last + i - 1, mat[seq1[i]]); - set_end_D(s->D, q, s - 1); - if (j + b1 - 1 > len1) { /* bug fixed, 040227 */ - set_end_I(s->I, q, last + i); - } else s->I = MINOR_INF; - s = curr; curr = last; last = s; - ++j; - } - - /* core dynamic programming, part 2 */ - for (; j <= len2 - b2 + 1; ++j) { - SET_INF(curr[j - b2]); - mat = score_matrix + seq2[j] * N_MATRIX_ROW; - end = j + b1 - 1; - for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i != end; ++i, ++s, ++q) { - set_M(s->M, q, last + i - 1, mat[seq1[i]]); - set_I(s->I, q, last + i); - set_D(s->D, q, s - 1); - } - set_M(s->M, q, last + i - 1, mat[seq1[i]]); - set_D(s->D, q, s - 1); - s->I = MINOR_INF; - s = curr; curr = last; last = s; - } - - /* core dynamic programming, part 3 */ - for (; j < len2; ++j) { - SET_INF(curr[j - b2]); - mat = score_matrix + seq2[j] * N_MATRIX_ROW; - for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i < len1; ++i, ++s, ++q) { - set_M(s->M, q, last + i - 1, mat[seq1[i]]); - set_I(s->I, q, last + i); - set_D(s->D, q, s - 1); - } - set_M(s->M, q, last + len1 - 1, mat[seq1[i]]); - set_end_I(s->I, q, last + i); - set_D(s->D, q, s - 1); - s = curr; curr = last; last = s; - } - /* last row */ - if (j == len2) { - SET_INF(curr[j - b2]); - mat = score_matrix + seq2[j] * N_MATRIX_ROW; - for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i < len1; ++i, ++s, ++q) { - set_M(s->M, q, last + i - 1, mat[seq1[i]]); - set_I(s->I, q, last + i); - set_end_D(s->D, q, s - 1); - } - set_M(s->M, q, last + len1 - 1, mat[seq1[i]]); - set_end_I(s->I, q, last + i); - set_end_D(s->D, q, s - 1); - s = curr; curr = last; last = s; - } - - *_score = last[len1].M; - if (n_cigar) { /* backtrace */ - path_t *p, *path = (path_t*)malloc(sizeof(path_t) * (len1 + len2 + 2)); - i = len1; j = len2; - q = dpcell[j] + i; - s = last + len1; - max = s->M; type = q->Mt; ctype = FROM_M; - if (s->I > max) { max = s->I; type = q->It; ctype = FROM_I; } - if (s->D > max) { max = s->D; type = q->Dt; ctype = FROM_D; } - - p = path; - p->ctype = ctype; p->i = i; p->j = j; /* bug fixed 040408 */ - ++p; - do { - switch (ctype) { - case FROM_M: --i; --j; break; - case FROM_I: --j; break; - case FROM_D: --i; break; - } - q = dpcell[j] + i; - ctype = type; - switch (type) { - case FROM_M: type = q->Mt; break; - case FROM_I: type = q->It; break; - case FROM_D: type = q->Dt; break; - } - p->ctype = ctype; p->i = i; p->j = j; - ++p; - } while (i || j); - cigar = ka_path2cigar32(path, p - path - 1, n_cigar); - free(path); - } - - /* free memory */ - for (j = b2 + 1; j <= len2; ++j) - dpcell[j] += j - b2; - for (j = 0; j <= len2; ++j) - free(dpcell[j]); - free(dpcell); - free(curr); free(last); - - return cigar; -} - -typedef struct { - int M, I, D; -} score_aux_t; - -#define MINUS_INF -0x40000000 - -// matrix: len2 rows and len1 columns -int ka_global_score(const uint8_t *_seq1, int len1, const uint8_t *_seq2, int len2, const ka_param2_t *ap) -{ - -#define __score_aux(_p, _q0, _sc, _io, _ie, _do, _de) { \ - int t1, t2; \ - score_aux_t *_q; \ - _q = _q0; \ - _p->M = _q->M >= _q->I? _q->M : _q->I; \ - _p->M = _p->M >= _q->D? _p->M : _q->D; \ - _p->M += (_sc); \ - ++_q; t1 = _q->M - _io - _ie; t2 = _q->I - _ie; _p->I = t1 >= t2? t1 : t2; \ - _q = _p-1; t1 = _q->M - _do - _de; t2 = _q->D - _de; _p->D = t1 >= t2? t1 : t2; \ - } - - int i, j, bw, scmat_size = ap->row, *scmat = ap->matrix, ret; - const uint8_t *seq1, *seq2; - score_aux_t *curr, *last, *swap; - bw = abs(len1 - len2) + ap->band_width; - i = len1 > len2? len1 : len2; - if (bw > i + 1) bw = i + 1; - seq1 = _seq1 - 1; seq2 = _seq2 - 1; - curr = calloc(len1 + 2, sizeof(score_aux_t)); - last = calloc(len1 + 2, sizeof(score_aux_t)); - { // the zero-th row - int x, end = len1; - score_aux_t *p; - j = 0; - x = j + bw; end = len1 < x? len1 : x; // band end - p = curr; - p->M = 0; p->I = p->D = MINUS_INF; - for (i = 1, p = &curr[1]; i <= end; ++i, ++p) - p->M = p->I = MINUS_INF, p->D = -(ap->edo + ap->ede * i); - p->M = p->I = p->D = MINUS_INF; - swap = curr; curr = last; last = swap; - } - for (j = 1; j < len2; ++j) { - int x, beg = 0, end = len1, *scrow, col_end; - score_aux_t *p; - x = j - bw; beg = 0 > x? 0 : x; // band start - x = j + bw; end = len1 < x? len1 : x; // band end - if (beg == 0) { // from zero-th column - p = curr; - p->M = p->D = MINUS_INF; p->I = -(ap->eio + ap->eie * j); - ++beg; // then beg = 1 - } - scrow = scmat + seq2[j] * scmat_size; - if (end == len1) col_end = 1, --end; - else col_end = 0; - for (i = beg, p = &curr[beg]; i <= end; ++i, ++p) - __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->iio, ap->iie, ap->ido, ap->ide); - if (col_end) { - __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->eio, ap->eie, ap->ido, ap->ide); - ++p; - } - p->M = p->I = p->D = MINUS_INF; -// for (i = 0; i <= len1; ++i) printf("(%d,%d,%d) ", curr[i].M, curr[i].I, curr[i].D); putchar('\n'); - swap = curr; curr = last; last = swap; - } - { // the last row - int x, beg = 0, *scrow; - score_aux_t *p; - j = len2; - x = j - bw; beg = 0 > x? 0 : x; // band start - if (beg == 0) { // from zero-th column - p = curr; - p->M = p->D = MINUS_INF; p->I = -(ap->eio + ap->eie * j); - ++beg; // then beg = 1 - } - scrow = scmat + seq2[j] * scmat_size; - for (i = beg, p = &curr[beg]; i < len1; ++i, ++p) - __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->iio, ap->iie, ap->edo, ap->ede); - __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->eio, ap->eie, ap->edo, ap->ede); -// for (i = 0; i <= len1; ++i) printf("(%d,%d,%d) ", curr[i].M, curr[i].I, curr[i].D); putchar('\n'); - } - ret = curr[len1].M >= curr[len1].I? curr[len1].M : curr[len1].I; - ret = ret >= curr[len1].D? ret : curr[len1].D; - free(curr); free(last); - return ret; -} - -#ifdef _MAIN -int main(int argc, char *argv[]) -{ -// int len1 = 35, len2 = 35; -// uint8_t *seq1 = (uint8_t*)"\0\0\3\3\2\0\0\0\1\0\2\1\2\1\3\2\3\3\3\0\2\3\2\1\1\3\3\3\2\3\3\1\0\0\1"; -// uint8_t *seq2 = (uint8_t*)"\0\0\3\3\2\0\0\0\1\0\2\1\2\1\3\2\3\3\3\0\2\3\2\1\1\3\3\3\2\3\3\1\0\1\0"; - int len1 = 4, len2 = 4; - uint8_t *seq1 = (uint8_t*)"\1\0\0\1"; - uint8_t *seq2 = (uint8_t*)"\1\0\1\0"; - int sc; -// ka_global_core(seq1, 2, seq2, 1, &ka_param_qual, &sc, 0); - sc = ka_global_score(seq1, len1, seq2, len2, &ka_param2_qual); - printf("%d\n", sc); - return 0; -} -#endif diff --git a/samtools-0.1.13/kaln.h b/samtools-0.1.13/kaln.h deleted file mode 100644 --- a/samtools-0.1.13/kaln.h +++ /dev/null @@ -1,67 +0,0 @@ -/* The MIT License - - Copyright (c) 2003-2006, 2008, 2009 by Heng Li - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#ifndef LH3_KALN_H_ -#define LH3_KALN_H_ - -#include - -#define MINOR_INF -1073741823 - -typedef struct { - int gap_open; - int gap_ext; - int gap_end_open; - int gap_end_ext; - - int *matrix; - int row; - int band_width; -} ka_param_t; - -typedef struct { - int iio, iie, ido, ide; - int eio, eie, edo, ede; - int *matrix; - int row; - int band_width; -} ka_param2_t; - -#ifdef __cplusplus -extern "C" { -#endif - - uint32_t *ka_global_core(uint8_t *seq1, int len1, uint8_t *seq2, int len2, const ka_param_t *ap, - int *_score, int *n_cigar); - int ka_global_score(const uint8_t *_seq1, int len1, const uint8_t *_seq2, int len2, const ka_param2_t *ap); -#ifdef __cplusplus -} -#endif - -extern ka_param_t ka_param_blast; /* = { 5, 2, 5, 2, aln_sm_blast, 5, 50 }; */ -extern ka_param_t ka_param_qual; // only use this for global alignment!!! -extern ka_param2_t ka_param2_qual; // only use this for global alignment!!! - -#endif diff --git a/samtools-0.1.13/khash.h b/samtools-0.1.13/khash.h deleted file mode 100644 --- a/samtools-0.1.13/khash.h +++ /dev/null @@ -1,528 +0,0 @@ -/* The MIT License - - Copyright (c) 2008, 2009, 2011 by Attractive Chaos - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -/* - An example: - -#include "khash.h" -KHASH_MAP_INIT_INT(32, char) -int main() { - int ret, is_missing; - khiter_t k; - khash_t(32) *h = kh_init(32); - k = kh_put(32, h, 5, &ret); - if (!ret) kh_del(32, h, k); - kh_value(h, k) = 10; - k = kh_get(32, h, 10); - is_missing = (k == kh_end(h)); - k = kh_get(32, h, 5); - kh_del(32, h, k); - for (k = kh_begin(h); k != kh_end(h); ++k) - if (kh_exist(h, k)) kh_value(h, k) = 1; - kh_destroy(32, h); - return 0; -} -*/ - -/* - 2011-02-14 (0.2.5): - - * Allow to declare global functions. - - 2009-09-26 (0.2.4): - - * Improve portability - - 2008-09-19 (0.2.3): - - * Corrected the example - * Improved interfaces - - 2008-09-11 (0.2.2): - - * Improved speed a little in kh_put() - - 2008-09-10 (0.2.1): - - * Added kh_clear() - * Fixed a compiling error - - 2008-09-02 (0.2.0): - - * Changed to token concatenation which increases flexibility. - - 2008-08-31 (0.1.2): - - * Fixed a bug in kh_get(), which has not been tested previously. - - 2008-08-31 (0.1.1): - - * Added destructor -*/ - - -#ifndef __AC_KHASH_H -#define __AC_KHASH_H - -/*! - @header - - Generic hash table library. - - @copyright Heng Li - */ - -#define AC_VERSION_KHASH_H "0.2.5" - -#include -#include -#include - -/* compipler specific configuration */ - -#if UINT_MAX == 0xffffffffu -typedef unsigned int khint32_t; -#elif ULONG_MAX == 0xffffffffu -typedef unsigned long khint32_t; -#endif - -#if ULONG_MAX == ULLONG_MAX -typedef unsigned long khint64_t; -#else -typedef unsigned long long khint64_t; -#endif - -#ifdef _MSC_VER -#define inline __inline -#endif - -typedef khint32_t khint_t; -typedef khint_t khiter_t; - -#define __ac_HASH_PRIME_SIZE 32 -static const khint32_t __ac_prime_list[__ac_HASH_PRIME_SIZE] = -{ - 0ul, 3ul, 11ul, 23ul, 53ul, - 97ul, 193ul, 389ul, 769ul, 1543ul, - 3079ul, 6151ul, 12289ul, 24593ul, 49157ul, - 98317ul, 196613ul, 393241ul, 786433ul, 1572869ul, - 3145739ul, 6291469ul, 12582917ul, 25165843ul, 50331653ul, - 100663319ul, 201326611ul, 402653189ul, 805306457ul, 1610612741ul, - 3221225473ul, 4294967291ul -}; - -#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2) -#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1) -#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3) -#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1))) -#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1))) -#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) -#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) - -static const double __ac_HASH_UPPER = 0.77; - -#define KHASH_DECLARE(name, khkey_t, khval_t) \ - typedef struct { \ - khint_t n_buckets, size, n_occupied, upper_bound; \ - khint32_t *flags; \ - khkey_t *keys; \ - khval_t *vals; \ - } kh_##name##_t; \ - extern kh_##name##_t *kh_init_##name(); \ - extern void kh_destroy_##name(kh_##name##_t *h); \ - extern void kh_clear_##name(kh_##name##_t *h); \ - extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \ - extern void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ - extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ - extern void kh_del_##name(kh_##name##_t *h, khint_t x); - -#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ - typedef struct { \ - khint_t n_buckets, size, n_occupied, upper_bound; \ - khint32_t *flags; \ - khkey_t *keys; \ - khval_t *vals; \ - } kh_##name##_t; \ - SCOPE kh_##name##_t *kh_init_##name() { \ - return (kh_##name##_t*)calloc(1, sizeof(kh_##name##_t)); \ - } \ - SCOPE void kh_destroy_##name(kh_##name##_t *h) \ - { \ - if (h) { \ - free(h->keys); free(h->flags); \ - free(h->vals); \ - free(h); \ - } \ - } \ - SCOPE void kh_clear_##name(kh_##name##_t *h) \ - { \ - if (h && h->flags) { \ - memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(khint32_t)); \ - h->size = h->n_occupied = 0; \ - } \ - } \ - SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ - { \ - if (h->n_buckets) { \ - khint_t inc, k, i, last; \ - k = __hash_func(key); i = k % h->n_buckets; \ - inc = 1 + k % (h->n_buckets - 1); last = i; \ - while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ - if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \ - else i += inc; \ - if (i == last) return h->n_buckets; \ - } \ - return __ac_iseither(h->flags, i)? h->n_buckets : i; \ - } else return 0; \ - } \ - SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ - { \ - khint32_t *new_flags = 0; \ - khint_t j = 1; \ - { \ - khint_t t = __ac_HASH_PRIME_SIZE - 1; \ - while (__ac_prime_list[t] > new_n_buckets) --t; \ - new_n_buckets = __ac_prime_list[t+1]; \ - if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; \ - else { \ - new_flags = (khint32_t*)malloc(((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \ - memset(new_flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \ - if (h->n_buckets < new_n_buckets) { \ - h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \ - if (kh_is_map) \ - h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \ - } \ - } \ - } \ - if (j) { \ - for (j = 0; j != h->n_buckets; ++j) { \ - if (__ac_iseither(h->flags, j) == 0) { \ - khkey_t key = h->keys[j]; \ - khval_t val; \ - if (kh_is_map) val = h->vals[j]; \ - __ac_set_isdel_true(h->flags, j); \ - while (1) { \ - khint_t inc, k, i; \ - k = __hash_func(key); \ - i = k % new_n_buckets; \ - inc = 1 + k % (new_n_buckets - 1); \ - while (!__ac_isempty(new_flags, i)) { \ - if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets; \ - else i += inc; \ - } \ - __ac_set_isempty_false(new_flags, i); \ - if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { \ - { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ - if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \ - __ac_set_isdel_true(h->flags, i); \ - } else { \ - h->keys[i] = key; \ - if (kh_is_map) h->vals[i] = val; \ - break; \ - } \ - } \ - } \ - } \ - if (h->n_buckets > new_n_buckets) { \ - h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \ - if (kh_is_map) \ - h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \ - } \ - free(h->flags); \ - h->flags = new_flags; \ - h->n_buckets = new_n_buckets; \ - h->n_occupied = h->size; \ - h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \ - } \ - } \ - SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \ - { \ - khint_t x; \ - if (h->n_occupied >= h->upper_bound) { \ - if (h->n_buckets > (h->size<<1)) kh_resize_##name(h, h->n_buckets - 1); \ - else kh_resize_##name(h, h->n_buckets + 1); \ - } \ - { \ - khint_t inc, k, i, site, last; \ - x = site = h->n_buckets; k = __hash_func(key); i = k % h->n_buckets; \ - if (__ac_isempty(h->flags, i)) x = i; \ - else { \ - inc = 1 + k % (h->n_buckets - 1); last = i; \ - while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ - if (__ac_isdel(h->flags, i)) site = i; \ - if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \ - else i += inc; \ - if (i == last) { x = site; break; } \ - } \ - if (x == h->n_buckets) { \ - if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \ - else x = i; \ - } \ - } \ - } \ - if (__ac_isempty(h->flags, x)) { \ - h->keys[x] = key; \ - __ac_set_isboth_false(h->flags, x); \ - ++h->size; ++h->n_occupied; \ - *ret = 1; \ - } else if (__ac_isdel(h->flags, x)) { \ - h->keys[x] = key; \ - __ac_set_isboth_false(h->flags, x); \ - ++h->size; \ - *ret = 2; \ - } else *ret = 0; \ - return x; \ - } \ - SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \ - { \ - if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \ - __ac_set_isdel_true(h->flags, x); \ - --h->size; \ - } \ - } - -#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ - KHASH_INIT2(name, static inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) - -/* --- BEGIN OF HASH FUNCTIONS --- */ - -/*! @function - @abstract Integer hash function - @param key The integer [khint32_t] - @return The hash value [khint_t] - */ -#define kh_int_hash_func(key) (khint32_t)(key) -/*! @function - @abstract Integer comparison function - */ -#define kh_int_hash_equal(a, b) ((a) == (b)) -/*! @function - @abstract 64-bit integer hash function - @param key The integer [khint64_t] - @return The hash value [khint_t] - */ -#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11) -/*! @function - @abstract 64-bit integer comparison function - */ -#define kh_int64_hash_equal(a, b) ((a) == (b)) -/*! @function - @abstract const char* hash function - @param s Pointer to a null terminated string - @return The hash value - */ -static inline khint_t __ac_X31_hash_string(const char *s) -{ - khint_t h = *s; - if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s; - return h; -} -/*! @function - @abstract Another interface to const char* hash function - @param key Pointer to a null terminated string [const char*] - @return The hash value [khint_t] - */ -#define kh_str_hash_func(key) __ac_X31_hash_string(key) -/*! @function - @abstract Const char* comparison function - */ -#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0) - -/* --- END OF HASH FUNCTIONS --- */ - -/* Other necessary macros... */ - -/*! - @abstract Type of the hash table. - @param name Name of the hash table [symbol] - */ -#define khash_t(name) kh_##name##_t - -/*! @function - @abstract Initiate a hash table. - @param name Name of the hash table [symbol] - @return Pointer to the hash table [khash_t(name)*] - */ -#define kh_init(name) kh_init_##name() - -/*! @function - @abstract Destroy a hash table. - @param name Name of the hash table [symbol] - @param h Pointer to the hash table [khash_t(name)*] - */ -#define kh_destroy(name, h) kh_destroy_##name(h) - -/*! @function - @abstract Reset a hash table without deallocating memory. - @param name Name of the hash table [symbol] - @param h Pointer to the hash table [khash_t(name)*] - */ -#define kh_clear(name, h) kh_clear_##name(h) - -/*! @function - @abstract Resize a hash table. - @param name Name of the hash table [symbol] - @param h Pointer to the hash table [khash_t(name)*] - @param s New size [khint_t] - */ -#define kh_resize(name, h, s) kh_resize_##name(h, s) - -/*! @function - @abstract Insert a key to the hash table. - @param name Name of the hash table [symbol] - @param h Pointer to the hash table [khash_t(name)*] - @param k Key [type of keys] - @param r Extra return code: 0 if the key is present in the hash table; - 1 if the bucket is empty (never used); 2 if the element in - the bucket has been deleted [int*] - @return Iterator to the inserted element [khint_t] - */ -#define kh_put(name, h, k, r) kh_put_##name(h, k, r) - -/*! @function - @abstract Retrieve a key from the hash table. - @param name Name of the hash table [symbol] - @param h Pointer to the hash table [khash_t(name)*] - @param k Key [type of keys] - @return Iterator to the found element, or kh_end(h) is the element is absent [khint_t] - */ -#define kh_get(name, h, k) kh_get_##name(h, k) - -/*! @function - @abstract Remove a key from the hash table. - @param name Name of the hash table [symbol] - @param h Pointer to the hash table [khash_t(name)*] - @param k Iterator to the element to be deleted [khint_t] - */ -#define kh_del(name, h, k) kh_del_##name(h, k) - - -/*! @function - @abstract Test whether a bucket contains data. - @param h Pointer to the hash table [khash_t(name)*] - @param x Iterator to the bucket [khint_t] - @return 1 if containing data; 0 otherwise [int] - */ -#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x))) - -/*! @function - @abstract Get key given an iterator - @param h Pointer to the hash table [khash_t(name)*] - @param x Iterator to the bucket [khint_t] - @return Key [type of keys] - */ -#define kh_key(h, x) ((h)->keys[x]) - -/*! @function - @abstract Get value given an iterator - @param h Pointer to the hash table [khash_t(name)*] - @param x Iterator to the bucket [khint_t] - @return Value [type of values] - @discussion For hash sets, calling this results in segfault. - */ -#define kh_val(h, x) ((h)->vals[x]) - -/*! @function - @abstract Alias of kh_val() - */ -#define kh_value(h, x) ((h)->vals[x]) - -/*! @function - @abstract Get the start iterator - @param h Pointer to the hash table [khash_t(name)*] - @return The start iterator [khint_t] - */ -#define kh_begin(h) (khint_t)(0) - -/*! @function - @abstract Get the end iterator - @param h Pointer to the hash table [khash_t(name)*] - @return The end iterator [khint_t] - */ -#define kh_end(h) ((h)->n_buckets) - -/*! @function - @abstract Get the number of elements in the hash table - @param h Pointer to the hash table [khash_t(name)*] - @return Number of elements in the hash table [khint_t] - */ -#define kh_size(h) ((h)->size) - -/*! @function - @abstract Get the number of buckets in the hash table - @param h Pointer to the hash table [khash_t(name)*] - @return Number of buckets in the hash table [khint_t] - */ -#define kh_n_buckets(h) ((h)->n_buckets) - -/* More conenient interfaces */ - -/*! @function - @abstract Instantiate a hash set containing integer keys - @param name Name of the hash table [symbol] - */ -#define KHASH_SET_INIT_INT(name) \ - KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal) - -/*! @function - @abstract Instantiate a hash map containing integer keys - @param name Name of the hash table [symbol] - @param khval_t Type of values [type] - */ -#define KHASH_MAP_INIT_INT(name, khval_t) \ - KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal) - -/*! @function - @abstract Instantiate a hash map containing 64-bit integer keys - @param name Name of the hash table [symbol] - */ -#define KHASH_SET_INIT_INT64(name) \ - KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal) - -/*! @function - @abstract Instantiate a hash map containing 64-bit integer keys - @param name Name of the hash table [symbol] - @param khval_t Type of values [type] - */ -#define KHASH_MAP_INIT_INT64(name, khval_t) \ - KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal) - -typedef const char *kh_cstr_t; -/*! @function - @abstract Instantiate a hash map containing const char* keys - @param name Name of the hash table [symbol] - */ -#define KHASH_SET_INIT_STR(name) \ - KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal) - -/*! @function - @abstract Instantiate a hash map containing const char* keys - @param name Name of the hash table [symbol] - @param khval_t Type of values [type] - */ -#define KHASH_MAP_INIT_STR(name, khval_t) \ - KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) - -#endif /* __AC_KHASH_H */ diff --git a/samtools-0.1.13/knetfile.c b/samtools-0.1.13/knetfile.c deleted file mode 100644 --- a/samtools-0.1.13/knetfile.c +++ /dev/null @@ -1,632 +0,0 @@ -/* The MIT License - - Copyright (c) 2008 by Genome Research Ltd (GRL). - 2010 by Attractive Chaos - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -/* Probably I will not do socket programming in the next few years and - therefore I decide to heavily annotate this file, for Linux and - Windows as well. -ac */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#include -#include -#endif - -#include "knetfile.h" - -/* In winsock.h, the type of a socket is SOCKET, which is: "typedef - * u_int SOCKET". An invalid SOCKET is: "(SOCKET)(~0)", or signed - * integer -1. In knetfile.c, I use "int" for socket type - * throughout. This should be improved to avoid confusion. - * - * In Linux/Mac, recv() and read() do almost the same thing. You can see - * in the header file that netread() is simply an alias of read(). In - * Windows, however, they are different and using recv() is mandatory. - */ - -/* This function tests if the file handler is ready for reading (or - * writing if is_read==0). */ -static int socket_wait(int fd, int is_read) -{ - fd_set fds, *fdr = 0, *fdw = 0; - struct timeval tv; - int ret; - tv.tv_sec = 5; tv.tv_usec = 0; // 5 seconds time out - FD_ZERO(&fds); - FD_SET(fd, &fds); - if (is_read) fdr = &fds; - else fdw = &fds; - ret = select(fd+1, fdr, fdw, 0, &tv); -#ifndef _WIN32 - if (ret == -1) perror("select"); -#else - if (ret == 0) - fprintf(stderr, "select time-out\n"); - else if (ret == SOCKET_ERROR) - fprintf(stderr, "select: %d\n", WSAGetLastError()); -#endif - return ret; -} - -#ifndef _WIN32 -/* This function does not work with Windows due to the lack of - * getaddrinfo() in winsock. It is addapted from an example in "Beej's - * Guide to Network Programming" (http://beej.us/guide/bgnet/). */ -static int socket_connect(const char *host, const char *port) -{ -#define __err_connect(func) do { perror(func); freeaddrinfo(res); return -1; } while (0) - - int on = 1, fd; - struct linger lng = { 0, 0 }; - struct addrinfo hints, *res = 0; - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - /* In Unix/Mac, getaddrinfo() is the most convenient way to get - * server information. */ - if (getaddrinfo(host, port, &hints, &res) != 0) __err_connect("getaddrinfo"); - if ((fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) __err_connect("socket"); - /* The following two setsockopt() are used by ftplib - * (http://nbpfaus.net/~pfau/ftplib/). I am not sure if they - * necessary. */ - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) __err_connect("setsockopt"); - if (setsockopt(fd, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) __err_connect("setsockopt"); - if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) __err_connect("connect"); - freeaddrinfo(res); - return fd; -} -#else -/* MinGW's printf has problem with "%lld" */ -char *int64tostr(char *buf, int64_t x) -{ - int cnt; - int i = 0; - do { - buf[i++] = '0' + x % 10; - x /= 10; - } while (x); - buf[i] = 0; - for (cnt = i, i = 0; i < cnt/2; ++i) { - int c = buf[i]; buf[i] = buf[cnt-i-1]; buf[cnt-i-1] = c; - } - return buf; -} - -int64_t strtoint64(const char *buf) -{ - int64_t x; - for (x = 0; *buf != '\0'; ++buf) - x = x * 10 + ((int64_t) *buf - 48); - return x; -} -/* In windows, the first thing is to establish the TCP connection. */ -int knet_win32_init() -{ - WSADATA wsaData; - return WSAStartup(MAKEWORD(2, 2), &wsaData); -} -void knet_win32_destroy() -{ - WSACleanup(); -} -/* A slightly modfied version of the following function also works on - * Mac (and presummably Linux). However, this function is not stable on - * my Mac. It sometimes works fine but sometimes does not. Therefore for - * non-Windows OS, I do not use this one. */ -static SOCKET socket_connect(const char *host, const char *port) -{ -#define __err_connect(func) \ - do { \ - fprintf(stderr, "%s: %d\n", func, WSAGetLastError()); \ - return -1; \ - } while (0) - - int on = 1; - SOCKET fd; - struct linger lng = { 0, 0 }; - struct sockaddr_in server; - struct hostent *hp = 0; - // open socket - if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) __err_connect("socket"); - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)) == -1) __err_connect("setsockopt"); - if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char*)&lng, sizeof(lng)) == -1) __err_connect("setsockopt"); - // get host info - if (isalpha(host[0])) hp = gethostbyname(host); - else { - struct in_addr addr; - addr.s_addr = inet_addr(host); - hp = gethostbyaddr((char*)&addr, 4, AF_INET); - } - if (hp == 0) __err_connect("gethost"); - // connect - server.sin_addr.s_addr = *((unsigned long*)hp->h_addr); - server.sin_family= AF_INET; - server.sin_port = htons(atoi(port)); - if (connect(fd, (struct sockaddr*)&server, sizeof(server)) != 0) __err_connect("connect"); - // freehostent(hp); // strangely in MSDN, hp is NOT freed (memory leak?!) - return fd; -} -#endif - -static off_t my_netread(int fd, void *buf, off_t len) -{ - off_t rest = len, curr, l = 0; - /* recv() and read() may not read the required length of data with - * one call. They have to be called repeatedly. */ - while (rest) { - if (socket_wait(fd, 1) <= 0) break; // socket is not ready for reading - curr = netread(fd, buf + l, rest); - /* According to the glibc manual, section 13.2, a zero returned - * value indicates end-of-file (EOF), which should mean that - * read() will not return zero if EOF has not been met but data - * are not immediately available. */ - if (curr == 0) break; - l += curr; rest -= curr; - } - return l; -} - -/************************* - * FTP specific routines * - *************************/ - -static int kftp_get_response(knetFile *ftp) -{ -#ifndef _WIN32 - unsigned char c; -#else - char c; -#endif - int n = 0; - char *p; - if (socket_wait(ftp->ctrl_fd, 1) <= 0) return 0; - while (netread(ftp->ctrl_fd, &c, 1)) { // FIXME: this is *VERY BAD* for unbuffered I/O - //fputc(c, stderr); - if (n >= ftp->max_response) { - ftp->max_response = ftp->max_response? ftp->max_response<<1 : 256; - ftp->response = realloc(ftp->response, ftp->max_response); - } - ftp->response[n++] = c; - if (c == '\n') { - if (n >= 4 && isdigit(ftp->response[0]) && isdigit(ftp->response[1]) && isdigit(ftp->response[2]) - && ftp->response[3] != '-') break; - n = 0; - continue; - } - } - if (n < 2) return -1; - ftp->response[n-2] = 0; - return strtol(ftp->response, &p, 0); -} - -static int kftp_send_cmd(knetFile *ftp, const char *cmd, int is_get) -{ - if (socket_wait(ftp->ctrl_fd, 0) <= 0) return -1; // socket is not ready for writing - netwrite(ftp->ctrl_fd, cmd, strlen(cmd)); - return is_get? kftp_get_response(ftp) : 0; -} - -static int kftp_pasv_prep(knetFile *ftp) -{ - char *p; - int v[6]; - kftp_send_cmd(ftp, "PASV\r\n", 1); - for (p = ftp->response; *p && *p != '('; ++p); - if (*p != '(') return -1; - ++p; - sscanf(p, "%d,%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]); - memcpy(ftp->pasv_ip, v, 4 * sizeof(int)); - ftp->pasv_port = (v[4]<<8&0xff00) + v[5]; - return 0; -} - - -static int kftp_pasv_connect(knetFile *ftp) -{ - char host[80], port[10]; - if (ftp->pasv_port == 0) { - fprintf(stderr, "[kftp_pasv_connect] kftp_pasv_prep() is not called before hand.\n"); - return -1; - } - sprintf(host, "%d.%d.%d.%d", ftp->pasv_ip[0], ftp->pasv_ip[1], ftp->pasv_ip[2], ftp->pasv_ip[3]); - sprintf(port, "%d", ftp->pasv_port); - ftp->fd = socket_connect(host, port); - if (ftp->fd == -1) return -1; - return 0; -} - -int kftp_connect(knetFile *ftp) -{ - ftp->ctrl_fd = socket_connect(ftp->host, ftp->port); - if (ftp->ctrl_fd == -1) return -1; - kftp_get_response(ftp); - kftp_send_cmd(ftp, "USER anonymous\r\n", 1); - kftp_send_cmd(ftp, "PASS kftp@\r\n", 1); - kftp_send_cmd(ftp, "TYPE I\r\n", 1); - return 0; -} - -int kftp_reconnect(knetFile *ftp) -{ - if (ftp->ctrl_fd != -1) { - netclose(ftp->ctrl_fd); - ftp->ctrl_fd = -1; - } - netclose(ftp->fd); - ftp->fd = -1; - return kftp_connect(ftp); -} - -// initialize ->type, ->host, ->retr and ->size -knetFile *kftp_parse_url(const char *fn, const char *mode) -{ - knetFile *fp; - char *p; - int l; - if (strstr(fn, "ftp://") != fn) return 0; - for (p = (char*)fn + 6; *p && *p != '/'; ++p); - if (*p != '/') return 0; - l = p - fn - 6; - fp = calloc(1, sizeof(knetFile)); - fp->type = KNF_TYPE_FTP; - fp->fd = -1; - /* the Linux/Mac version of socket_connect() also recognizes a port - * like "ftp", but the Windows version does not. */ - fp->port = strdup("21"); - fp->host = calloc(l + 1, 1); - if (strchr(mode, 'c')) fp->no_reconnect = 1; - strncpy(fp->host, fn + 6, l); - fp->retr = calloc(strlen(p) + 8, 1); - sprintf(fp->retr, "RETR %s\r\n", p); - fp->size_cmd = calloc(strlen(p) + 8, 1); - sprintf(fp->size_cmd, "SIZE %s\r\n", p); - fp->seek_offset = 0; - return fp; -} -// place ->fd at offset off -int kftp_connect_file(knetFile *fp) -{ - int ret; - long long file_size; - if (fp->fd != -1) { - netclose(fp->fd); - if (fp->no_reconnect) kftp_get_response(fp); - } - kftp_pasv_prep(fp); - kftp_send_cmd(fp, fp->size_cmd, 1); -#ifndef _WIN32 - if ( sscanf(fp->response,"%*d %lld", &file_size) != 1 ) - { - fprintf(stderr,"[kftp_connect_file] %s\n", fp->response); - return -1; - } -#else - const char *p = fp->response; - while (*p != ' ') ++p; - while (*p < '0' || *p > '9') ++p; - file_size = strtoint64(p); -#endif - fp->file_size = file_size; - if (fp->offset>=0) { - char tmp[32]; -#ifndef _WIN32 - sprintf(tmp, "REST %lld\r\n", (long long)fp->offset); -#else - strcpy(tmp, "REST "); - int64tostr(tmp + 5, fp->offset); - strcat(tmp, "\r\n"); -#endif - kftp_send_cmd(fp, tmp, 1); - } - kftp_send_cmd(fp, fp->retr, 0); - kftp_pasv_connect(fp); - ret = kftp_get_response(fp); - if (ret != 150) { - fprintf(stderr, "[kftp_connect_file] %s\n", fp->response); - netclose(fp->fd); - fp->fd = -1; - return -1; - } - fp->is_ready = 1; - return 0; -} - - -/************************** - * HTTP specific routines * - **************************/ - -knetFile *khttp_parse_url(const char *fn, const char *mode) -{ - knetFile *fp; - char *p, *proxy, *q; - int l; - if (strstr(fn, "http://") != fn) return 0; - // set ->http_host - for (p = (char*)fn + 7; *p && *p != '/'; ++p); - l = p - fn - 7; - fp = calloc(1, sizeof(knetFile)); - fp->http_host = calloc(l + 1, 1); - strncpy(fp->http_host, fn + 7, l); - fp->http_host[l] = 0; - for (q = fp->http_host; *q && *q != ':'; ++q); - if (*q == ':') *q++ = 0; - // get http_proxy - proxy = getenv("http_proxy"); - // set ->host, ->port and ->path - if (proxy == 0) { - fp->host = strdup(fp->http_host); // when there is no proxy, server name is identical to http_host name. - fp->port = strdup(*q? q : "80"); - fp->path = strdup(*p? p : "/"); - } else { - fp->host = (strstr(proxy, "http://") == proxy)? strdup(proxy + 7) : strdup(proxy); - for (q = fp->host; *q && *q != ':'; ++q); - if (*q == ':') *q++ = 0; - fp->port = strdup(*q? q : "80"); - fp->path = strdup(fn); - } - fp->type = KNF_TYPE_HTTP; - fp->ctrl_fd = fp->fd = -1; - fp->seek_offset = 0; - return fp; -} - -int khttp_connect_file(knetFile *fp) -{ - int ret, l = 0; - char *buf, *p; - if (fp->fd != -1) netclose(fp->fd); - fp->fd = socket_connect(fp->host, fp->port); - buf = calloc(0x10000, 1); // FIXME: I am lazy... But in principle, 64KB should be large enough. - l += sprintf(buf + l, "GET %s HTTP/1.0\r\nHost: %s\r\n", fp->path, fp->http_host); - l += sprintf(buf + l, "Range: bytes=%lld-\r\n", (long long)fp->offset); - l += sprintf(buf + l, "\r\n"); - netwrite(fp->fd, buf, l); - l = 0; - while (netread(fp->fd, buf + l, 1)) { // read HTTP header; FIXME: bad efficiency - if (buf[l] == '\n' && l >= 3) - if (strncmp(buf + l - 3, "\r\n\r\n", 4) == 0) break; - ++l; - } - buf[l] = 0; - if (l < 14) { // prematured header - netclose(fp->fd); - fp->fd = -1; - return -1; - } - ret = strtol(buf + 8, &p, 0); // HTTP return code - if (ret == 200 && fp->offset>0) { // 200 (complete result); then skip beginning of the file - off_t rest = fp->offset; - while (rest) { - off_t l = rest < 0x10000? rest : 0x10000; - rest -= my_netread(fp->fd, buf, l); - } - } else if (ret != 206 && ret != 200) { - free(buf); - fprintf(stderr, "[khttp_connect_file] fail to open file (HTTP code: %d).\n", ret); - netclose(fp->fd); - fp->fd = -1; - return -1; - } - free(buf); - fp->is_ready = 1; - return 0; -} - -/******************** - * Generic routines * - ********************/ - -knetFile *knet_open(const char *fn, const char *mode) -{ - knetFile *fp = 0; - if (mode[0] != 'r') { - fprintf(stderr, "[kftp_open] only mode \"r\" is supported.\n"); - return 0; - } - if (strstr(fn, "ftp://") == fn) { - fp = kftp_parse_url(fn, mode); - if (fp == 0) return 0; - if (kftp_connect(fp) == -1) { - knet_close(fp); - return 0; - } - kftp_connect_file(fp); - } else if (strstr(fn, "http://") == fn) { - fp = khttp_parse_url(fn, mode); - if (fp == 0) return 0; - khttp_connect_file(fp); - } else { // local file -#ifdef _WIN32 - /* In windows, O_BINARY is necessary. In Linux/Mac, O_BINARY may - * be undefined on some systems, although it is defined on my - * Mac and the Linux I have tested on. */ - int fd = open(fn, O_RDONLY | O_BINARY); -#else - int fd = open(fn, O_RDONLY); -#endif - if (fd == -1) { - perror("open"); - return 0; - } - fp = (knetFile*)calloc(1, sizeof(knetFile)); - fp->type = KNF_TYPE_LOCAL; - fp->fd = fd; - fp->ctrl_fd = -1; - } - if (fp && fp->fd == -1) { - knet_close(fp); - return 0; - } - return fp; -} - -knetFile *knet_dopen(int fd, const char *mode) -{ - knetFile *fp = (knetFile*)calloc(1, sizeof(knetFile)); - fp->type = KNF_TYPE_LOCAL; - fp->fd = fd; - return fp; -} - -off_t knet_read(knetFile *fp, void *buf, off_t len) -{ - off_t l = 0; - if (fp->fd == -1) return 0; - if (fp->type == KNF_TYPE_FTP) { - if (fp->is_ready == 0) { - if (!fp->no_reconnect) kftp_reconnect(fp); - kftp_connect_file(fp); - } - } else if (fp->type == KNF_TYPE_HTTP) { - if (fp->is_ready == 0) - khttp_connect_file(fp); - } - if (fp->type == KNF_TYPE_LOCAL) { // on Windows, the following block is necessary; not on UNIX - off_t rest = len, curr; - while (rest) { - do { - curr = read(fp->fd, buf + l, rest); - } while (curr < 0 && EINTR == errno); - if (curr < 0) return -1; - if (curr == 0) break; - l += curr; rest -= curr; - } - } else l = my_netread(fp->fd, buf, len); - fp->offset += l; - return l; -} - -off_t knet_seek(knetFile *fp, int64_t off, int whence) -{ - if (whence == SEEK_SET && off == fp->offset) return 0; - if (fp->type == KNF_TYPE_LOCAL) { - /* Be aware that lseek() returns the offset after seeking, - * while fseek() returns zero on success. */ - off_t offset = lseek(fp->fd, off, whence); - if (offset == -1) { - // Be silent, it is OK for knet_seek to fail when the file is streamed - // fprintf(stderr,"[knet_seek] %s\n", strerror(errno)); - return -1; - } - fp->offset = offset; - return 0; - } - else if (fp->type == KNF_TYPE_FTP) - { - if (whence==SEEK_CUR) - fp->offset += off; - else if (whence==SEEK_SET) - fp->offset = off; - else if ( whence==SEEK_END) - fp->offset = fp->file_size+off; - fp->is_ready = 0; - return 0; - } - else if (fp->type == KNF_TYPE_HTTP) - { - if (whence == SEEK_END) { // FIXME: can we allow SEEK_END in future? - fprintf(stderr, "[knet_seek] SEEK_END is not supported for HTTP. Offset is unchanged.\n"); - errno = ESPIPE; - return -1; - } - if (whence==SEEK_CUR) - fp->offset += off; - else if (whence==SEEK_SET) - fp->offset = off; - fp->is_ready = 0; - return 0; - } - errno = EINVAL; - fprintf(stderr,"[knet_seek] %s\n", strerror(errno)); - return -1; -} - -int knet_close(knetFile *fp) -{ - if (fp == 0) return 0; - if (fp->ctrl_fd != -1) netclose(fp->ctrl_fd); // FTP specific - if (fp->fd != -1) { - /* On Linux/Mac, netclose() is an alias of close(), but on - * Windows, it is an alias of closesocket(). */ - if (fp->type == KNF_TYPE_LOCAL) close(fp->fd); - else netclose(fp->fd); - } - free(fp->host); free(fp->port); - free(fp->response); free(fp->retr); // FTP specific - free(fp->path); free(fp->http_host); // HTTP specific - free(fp); - return 0; -} - -#ifdef KNETFILE_MAIN -int main(void) -{ - char *buf; - knetFile *fp; - int type = 4, l; -#ifdef _WIN32 - knet_win32_init(); -#endif - buf = calloc(0x100000, 1); - if (type == 0) { - fp = knet_open("knetfile.c", "r"); - knet_seek(fp, 1000, SEEK_SET); - } else if (type == 1) { // NCBI FTP, large file - fp = knet_open("ftp://ftp.ncbi.nih.gov/1000genomes/ftp/data/NA12878/alignment/NA12878.chrom6.SLX.SRP000032.2009_06.bam", "r"); - knet_seek(fp, 2500000000ll, SEEK_SET); - l = knet_read(fp, buf, 255); - } else if (type == 2) { - fp = knet_open("ftp://ftp.sanger.ac.uk/pub4/treefam/tmp/index.shtml", "r"); - knet_seek(fp, 1000, SEEK_SET); - } else if (type == 3) { - fp = knet_open("http://www.sanger.ac.uk/Users/lh3/index.shtml", "r"); - knet_seek(fp, 1000, SEEK_SET); - } else if (type == 4) { - fp = knet_open("http://www.sanger.ac.uk/Users/lh3/ex1.bam", "r"); - knet_read(fp, buf, 10000); - knet_seek(fp, 20000, SEEK_SET); - knet_seek(fp, 10000, SEEK_SET); - l = knet_read(fp, buf+10000, 10000000) + 10000; - } - if (type != 4 && type != 1) { - knet_read(fp, buf, 255); - buf[255] = 0; - printf("%s\n", buf); - } else write(fileno(stdout), buf, l); - knet_close(fp); - free(buf); - return 0; -} -#endif diff --git a/samtools-0.1.13/knetfile.h b/samtools-0.1.13/knetfile.h deleted file mode 100644 --- a/samtools-0.1.13/knetfile.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef KNETFILE_H -#define KNETFILE_H - -#include -#include - -#ifndef _WIN32 -#define netread(fd, ptr, len) read(fd, ptr, len) -#define netwrite(fd, ptr, len) write(fd, ptr, len) -#define netclose(fd) close(fd) -#else -#include -#define netread(fd, ptr, len) recv(fd, ptr, len, 0) -#define netwrite(fd, ptr, len) send(fd, ptr, len, 0) -#define netclose(fd) closesocket(fd) -#endif - -// FIXME: currently I/O is unbuffered - -#define KNF_TYPE_LOCAL 1 -#define KNF_TYPE_FTP 2 -#define KNF_TYPE_HTTP 3 - -typedef struct knetFile_s { - int type, fd; - int64_t offset; - char *host, *port; - - // the following are for FTP only - int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready; - char *response, *retr, *size_cmd; - int64_t seek_offset; // for lazy seek - int64_t file_size; - - // the following are for HTTP only - char *path, *http_host; -} knetFile; - -#define knet_tell(fp) ((fp)->offset) -#define knet_fileno(fp) ((fp)->fd) - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef _WIN32 - int knet_win32_init(); - void knet_win32_destroy(); -#endif - - knetFile *knet_open(const char *fn, const char *mode); - - /* - This only works with local files. - */ - knetFile *knet_dopen(int fd, const char *mode); - - /* - If ->is_ready==0, this routine updates ->fd; otherwise, it simply - reads from ->fd. - */ - off_t knet_read(knetFile *fp, void *buf, off_t len); - - /* - This routine only sets ->offset and ->is_ready=0. It does not - communicate with the FTP server. - */ - off_t knet_seek(knetFile *fp, int64_t off, int whence); - int knet_close(knetFile *fp); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.13/kprobaln.c b/samtools-0.1.13/kprobaln.c deleted file mode 100644 --- a/samtools-0.1.13/kprobaln.c +++ /dev/null @@ -1,278 +0,0 @@ -/* The MIT License - - Copyright (c) 2003-2006, 2008-2010, by Heng Li - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#include -#include -#include -#include -#include -#include "kprobaln.h" - -/***************************************** - * Probabilistic banded glocal alignment * - *****************************************/ - -#define EI .25 -#define EM .33333333333 - -static float g_qual2prob[256]; - -#define set_u(u, b, i, k) { int x=(i)-(b); x=x>0?x:0; (u)=((k)-x+1)*3; } - -kpa_par_t kpa_par_def = { 0.001, 0.1, 10 }; -kpa_par_t kpa_par_alt = { 0.0001, 0.01, 10 }; - -/* - The topology of the profile HMM: - - /\ /\ /\ /\ - I[1] I[k-1] I[k] I[L] - ^ \ \ ^ \ ^ \ \ ^ - | \ \ | \ | \ \ | - M[0] M[1] -> ... -> M[k-1] -> M[k] -> ... -> M[L] M[L+1] - \ \/ \/ \/ / - \ /\ /\ /\ / - -> D[k-1] -> D[k] -> - - M[0] points to every {M,I}[k] and every {M,I}[k] points M[L+1]. - - On input, _ref is the reference sequence and _query is the query - sequence. Both are sequences of 0/1/2/3/4 where 4 stands for an - ambiguous residue. iqual is the base quality. c sets the gap open - probability, gap extension probability and band width. - - On output, state and q are arrays of length l_query. The higher 30 - bits give the reference position the query base is matched to and the - lower two bits can be 0 (an alignment match) or 1 (an - insertion). q[i] gives the phred scaled posterior probability of - state[i] being wrong. - */ -int kpa_glocal(const uint8_t *_ref, int l_ref, const uint8_t *_query, int l_query, const uint8_t *iqual, - const kpa_par_t *c, int *state, uint8_t *q) -{ - double **f, **b = 0, *s, m[9], sI, sM, bI, bM, pb; - float *qual, *_qual; - const uint8_t *ref, *query; - int bw, bw2, i, k, is_diff = 0, is_backward = 1, Pr; - - /*** initialization ***/ - is_backward = state && q? 1 : 0; - ref = _ref - 1; query = _query - 1; // change to 1-based coordinate - bw = l_ref > l_query? l_ref : l_query; - if (bw > c->bw) bw = c->bw; - if (bw < abs(l_ref - l_query)) bw = abs(l_ref - l_query); - bw2 = bw * 2 + 1; - // allocate the forward and backward matrices f[][] and b[][] and the scaling array s[] - f = calloc(l_query+1, sizeof(void*)); - if (is_backward) b = calloc(l_query+1, sizeof(void*)); - for (i = 0; i <= l_query; ++i) { - f[i] = calloc(bw2 * 3 + 6, sizeof(double)); // FIXME: this is over-allocated for very short seqs - if (is_backward) b[i] = calloc(bw2 * 3 + 6, sizeof(double)); - } - s = calloc(l_query+2, sizeof(double)); // s[] is the scaling factor to avoid underflow - // initialize qual - _qual = calloc(l_query, sizeof(float)); - if (g_qual2prob[0] == 0) - for (i = 0; i < 256; ++i) - g_qual2prob[i] = pow(10, -i/10.); - for (i = 0; i < l_query; ++i) _qual[i] = g_qual2prob[iqual? iqual[i] : 30]; - qual = _qual - 1; - // initialize transition probability - sM = sI = 1. / (2 * l_query + 2); // the value here seems not to affect results; FIXME: need proof - m[0*3+0] = (1 - c->d - c->d) * (1 - sM); m[0*3+1] = m[0*3+2] = c->d * (1 - sM); - m[1*3+0] = (1 - c->e) * (1 - sI); m[1*3+1] = c->e * (1 - sI); m[1*3+2] = 0.; - m[2*3+0] = 1 - c->e; m[2*3+1] = 0.; m[2*3+2] = c->e; - bM = (1 - c->d) / l_ref; bI = c->d / l_ref; // (bM+bI)*l_ref==1 - /*** forward ***/ - // f[0] - set_u(k, bw, 0, 0); - f[0][k] = s[0] = 1.; - { // f[1] - double *fi = f[1], sum; - int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1, _beg, _end; - for (k = beg, sum = 0.; k <= end; ++k) { - int u; - double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] * EM; - set_u(u, bw, 1, k); - fi[u+0] = e * bM; fi[u+1] = EI * bI; - sum += fi[u] + fi[u+1]; - } - // rescale - s[1] = sum; - set_u(_beg, bw, 1, beg); set_u(_end, bw, 1, end); _end += 2; - for (k = _beg; k <= _end; ++k) fi[k] /= sum; - } - // f[2..l_query] - for (i = 2; i <= l_query; ++i) { - double *fi = f[i], *fi1 = f[i-1], sum, qli = qual[i]; - int beg = 1, end = l_ref, x, _beg, _end; - uint8_t qyi = query[i]; - x = i - bw; beg = beg > x? beg : x; // band start - x = i + bw; end = end < x? end : x; // band end - for (k = beg, sum = 0.; k <= end; ++k) { - int u, v11, v01, v10; - double e; - e = (ref[k] > 3 || qyi > 3)? 1. : ref[k] == qyi? 1. - qli : qli * EM; - set_u(u, bw, i, k); set_u(v11, bw, i-1, k-1); set_u(v10, bw, i-1, k); set_u(v01, bw, i, k-1); - fi[u+0] = e * (m[0] * fi1[v11+0] + m[3] * fi1[v11+1] + m[6] * fi1[v11+2]); - fi[u+1] = EI * (m[1] * fi1[v10+0] + m[4] * fi1[v10+1]); - fi[u+2] = m[2] * fi[v01+0] + m[8] * fi[v01+2]; - sum += fi[u] + fi[u+1] + fi[u+2]; -// fprintf(stderr, "F (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, fi[u], fi[u+1], fi[u+2]); // DEBUG - } - // rescale - s[i] = sum; - set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2; - for (k = _beg, sum = 1./sum; k <= _end; ++k) fi[k] *= sum; - } - { // f[l_query+1] - double sum; - for (k = 1, sum = 0.; k <= l_ref; ++k) { - int u; - set_u(u, bw, l_query, k); - if (u < 3 || u >= bw2*3+3) continue; - sum += f[l_query][u+0] * sM + f[l_query][u+1] * sI; - } - s[l_query+1] = sum; // the last scaling factor - } - { // compute likelihood - double p = 1., Pr1 = 0.; - for (i = 0; i <= l_query + 1; ++i) { - p *= s[i]; - if (p < 1e-100) Pr += -4.343 * log(p), p = 1.; - } - Pr1 += -4.343 * log(p * l_ref * l_query); - Pr = (int)(Pr1 + .499); - if (!is_backward) { // skip backward and MAP - for (i = 0; i <= l_query; ++i) free(f[i]); - free(f); free(s); free(_qual); - return Pr; - } - } - /*** backward ***/ - // b[l_query] (b[l_query+1][0]=1 and thus \tilde{b}[][]=1/s[l_query+1]; this is where s[l_query+1] comes from) - for (k = 1; k <= l_ref; ++k) { - int u; - double *bi = b[l_query]; - set_u(u, bw, l_query, k); - if (u < 3 || u >= bw2*3+3) continue; - bi[u+0] = sM / s[l_query] / s[l_query+1]; bi[u+1] = sI / s[l_query] / s[l_query+1]; - } - // b[l_query-1..1] - for (i = l_query - 1; i >= 1; --i) { - int beg = 1, end = l_ref, x, _beg, _end; - double *bi = b[i], *bi1 = b[i+1], y = (i > 1), qli1 = qual[i+1]; - uint8_t qyi1 = query[i+1]; - x = i - bw; beg = beg > x? beg : x; - x = i + bw; end = end < x? end : x; - for (k = end; k >= beg; --k) { - int u, v11, v01, v10; - double e; - set_u(u, bw, i, k); set_u(v11, bw, i+1, k+1); set_u(v10, bw, i+1, k); set_u(v01, bw, i, k+1); - e = (k >= l_ref? 0 : (ref[k+1] > 3 || qyi1 > 3)? 1. : ref[k+1] == qyi1? 1. - qli1 : qli1 * EM) * bi1[v11]; - bi[u+0] = e * m[0] + EI * m[1] * bi1[v10+1] + m[2] * bi[v01+2]; // bi1[v11] has been foled into e. - bi[u+1] = e * m[3] + EI * m[4] * bi1[v10+1]; - bi[u+2] = (e * m[6] + m[8] * bi[v01+2]) * y; -// fprintf(stderr, "B (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, bi[u], bi[u+1], bi[u+2]); // DEBUG - } - // rescale - set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2; - for (k = _beg, y = 1./s[i]; k <= _end; ++k) bi[k] *= y; - } - { // b[0] - int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1; - double sum = 0.; - for (k = end; k >= beg; --k) { - int u; - double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] * EM; - set_u(u, bw, 1, k); - if (u < 3 || u >= bw2*3+3) continue; - sum += e * b[1][u+0] * bM + EI * b[1][u+1] * bI; - } - set_u(k, bw, 0, 0); - pb = b[0][k] = sum / s[0]; // if everything works as is expected, pb == 1.0 - } - is_diff = fabs(pb - 1.) > 1e-7? 1 : 0; - /*** MAP ***/ - for (i = 1; i <= l_query; ++i) { - double sum = 0., *fi = f[i], *bi = b[i], max = 0.; - int beg = 1, end = l_ref, x, max_k = -1; - x = i - bw; beg = beg > x? beg : x; - x = i + bw; end = end < x? end : x; - for (k = beg; k <= end; ++k) { - int u; - double z; - set_u(u, bw, i, k); - z = fi[u+0] * bi[u+0]; if (z > max) max = z, max_k = (k-1)<<2 | 0; sum += z; - z = fi[u+1] * bi[u+1]; if (z > max) max = z, max_k = (k-1)<<2 | 1; sum += z; - } - max /= sum; sum *= s[i]; // if everything works as is expected, sum == 1.0 - if (state) state[i-1] = max_k; - if (q) k = (int)(-4.343 * log(1. - max) + .499), q[i-1] = k > 100? 99 : k; -#ifdef _MAIN - fprintf(stderr, "(%.10lg,%.10lg) (%d,%d:%c,%c:%d) %lg\n", pb, sum, i-1, max_k>>2, - "ACGT"[query[i]], "ACGT"[ref[(max_k>>2)+1]], max_k&3, max); // DEBUG -#endif - } - /*** free ***/ - for (i = 0; i <= l_query; ++i) { - free(f[i]); free(b[i]); - } - free(f); free(b); free(s); free(_qual); - return Pr; -} - -#ifdef _MAIN -#include -int main(int argc, char *argv[]) -{ - uint8_t conv[256], *iqual, *ref, *query; - int c, l_ref, l_query, i, q = 30, b = 10, P; - while ((c = getopt(argc, argv, "b:q:")) >= 0) { - switch (c) { - case 'b': b = atoi(optarg); break; - case 'q': q = atoi(optarg); break; - } - } - if (optind + 2 > argc) { - fprintf(stderr, "Usage: %s [-q %d] [-b %d] \n", argv[0], q, b); // example: acttc attc - return 1; - } - memset(conv, 4, 256); - conv['a'] = conv['A'] = 0; conv['c'] = conv['C'] = 1; - conv['g'] = conv['G'] = 2; conv['t'] = conv['T'] = 3; - ref = (uint8_t*)argv[optind]; query = (uint8_t*)argv[optind+1]; - l_ref = strlen((char*)ref); l_query = strlen((char*)query); - for (i = 0; i < l_ref; ++i) ref[i] = conv[ref[i]]; - for (i = 0; i < l_query; ++i) query[i] = conv[query[i]]; - iqual = malloc(l_query); - memset(iqual, q, l_query); - kpa_par_def.bw = b; - P = kpa_glocal(ref, l_ref, query, l_query, iqual, &kpa_par_alt, 0, 0); - fprintf(stderr, "%d\n", P); - free(iqual); - return 0; -} -#endif diff --git a/samtools-0.1.13/kprobaln.h b/samtools-0.1.13/kprobaln.h deleted file mode 100644 --- a/samtools-0.1.13/kprobaln.h +++ /dev/null @@ -1,49 +0,0 @@ -/* The MIT License - - Copyright (c) 2003-2006, 2008, 2009 by Heng Li - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#ifndef LH3_KPROBALN_H_ -#define LH3_KPROBALN_H_ - -#include - -typedef struct { - float d, e; - int bw; -} kpa_par_t; - -#ifdef __cplusplus -extern "C" { -#endif - - int kpa_glocal(const uint8_t *_ref, int l_ref, const uint8_t *_query, int l_query, const uint8_t *iqual, - const kpa_par_t *c, int *state, uint8_t *q); - -#ifdef __cplusplus -} -#endif - -extern kpa_par_t kpa_par_def, kpa_par_alt; - -#endif diff --git a/samtools-0.1.13/kseq.h b/samtools-0.1.13/kseq.h deleted file mode 100644 --- a/samtools-0.1.13/kseq.h +++ /dev/null @@ -1,227 +0,0 @@ -/* The MIT License - - Copyright (c) 2008 Genome Research Ltd (GRL). - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -/* Contact: Heng Li */ - -/* - 2009-07-16 (lh3): in kstream_t, change "char*" to "unsigned char*" - */ - -/* Last Modified: 12APR2009 */ - -#ifndef AC_KSEQ_H -#define AC_KSEQ_H - -#include -#include -#include - -#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r -#define KS_SEP_TAB 1 // isspace() && !' ' -#define KS_SEP_MAX 1 - -#define __KS_TYPE(type_t) \ - typedef struct __kstream_t { \ - unsigned char *buf; \ - int begin, end, is_eof; \ - type_t f; \ - } kstream_t; - -#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end) -#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0) - -#define __KS_BASIC(type_t, __bufsize) \ - static inline kstream_t *ks_init(type_t f) \ - { \ - kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t)); \ - ks->f = f; \ - ks->buf = malloc(__bufsize); \ - return ks; \ - } \ - static inline void ks_destroy(kstream_t *ks) \ - { \ - if (ks) { \ - free(ks->buf); \ - free(ks); \ - } \ - } - -#define __KS_GETC(__read, __bufsize) \ - static inline int ks_getc(kstream_t *ks) \ - { \ - if (ks->is_eof && ks->begin >= ks->end) return -1; \ - if (ks->begin >= ks->end) { \ - ks->begin = 0; \ - ks->end = __read(ks->f, ks->buf, __bufsize); \ - if (ks->end < __bufsize) ks->is_eof = 1; \ - if (ks->end == 0) return -1; \ - } \ - return (int)ks->buf[ks->begin++]; \ - } - -#ifndef KSTRING_T -#define KSTRING_T kstring_t -typedef struct __kstring_t { - size_t l, m; - char *s; -} kstring_t; -#endif - -#ifndef kroundup32 -#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) -#endif - -#define __KS_GETUNTIL(__read, __bufsize) \ - static int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \ - { \ - if (dret) *dret = 0; \ - str->l = 0; \ - if (ks->begin >= ks->end && ks->is_eof) return -1; \ - for (;;) { \ - int i; \ - if (ks->begin >= ks->end) { \ - if (!ks->is_eof) { \ - ks->begin = 0; \ - ks->end = __read(ks->f, ks->buf, __bufsize); \ - if (ks->end < __bufsize) ks->is_eof = 1; \ - if (ks->end == 0) break; \ - } else break; \ - } \ - if (delimiter > KS_SEP_MAX) { \ - for (i = ks->begin; i < ks->end; ++i) \ - if (ks->buf[i] == delimiter) break; \ - } else if (delimiter == KS_SEP_SPACE) { \ - for (i = ks->begin; i < ks->end; ++i) \ - if (isspace(ks->buf[i])) break; \ - } else if (delimiter == KS_SEP_TAB) { \ - for (i = ks->begin; i < ks->end; ++i) \ - if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break; \ - } else i = 0; /* never come to here! */ \ - if (str->m - str->l < i - ks->begin + 1) { \ - str->m = str->l + (i - ks->begin) + 1; \ - kroundup32(str->m); \ - str->s = (char*)realloc(str->s, str->m); \ - } \ - memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin); \ - str->l = str->l + (i - ks->begin); \ - ks->begin = i + 1; \ - if (i < ks->end) { \ - if (dret) *dret = ks->buf[i]; \ - break; \ - } \ - } \ - if (str->l == 0) { \ - str->m = 1; \ - str->s = (char*)calloc(1, 1); \ - } \ - str->s[str->l] = '\0'; \ - return str->l; \ - } - -#define KSTREAM_INIT(type_t, __read, __bufsize) \ - __KS_TYPE(type_t) \ - __KS_BASIC(type_t, __bufsize) \ - __KS_GETC(__read, __bufsize) \ - __KS_GETUNTIL(__read, __bufsize) - -#define __KSEQ_BASIC(type_t) \ - static inline kseq_t *kseq_init(type_t fd) \ - { \ - kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t)); \ - s->f = ks_init(fd); \ - return s; \ - } \ - static inline void kseq_rewind(kseq_t *ks) \ - { \ - ks->last_char = 0; \ - ks->f->is_eof = ks->f->begin = ks->f->end = 0; \ - } \ - static inline void kseq_destroy(kseq_t *ks) \ - { \ - if (!ks) return; \ - free(ks->name.s); free(ks->comment.s); free(ks->seq.s); free(ks->qual.s); \ - ks_destroy(ks->f); \ - free(ks); \ - } - -/* Return value: - >=0 length of the sequence (normal) - -1 end-of-file - -2 truncated quality string - */ -#define __KSEQ_READ \ - static int kseq_read(kseq_t *seq) \ - { \ - int c; \ - kstream_t *ks = seq->f; \ - if (seq->last_char == 0) { /* then jump to the next header line */ \ - while ((c = ks_getc(ks)) != -1 && c != '>' && c != '@'); \ - if (c == -1) return -1; /* end of file */ \ - seq->last_char = c; \ - } /* the first header char has been read */ \ - seq->comment.l = seq->seq.l = seq->qual.l = 0; \ - if (ks_getuntil(ks, 0, &seq->name, &c) < 0) return -1; \ - if (c != '\n') ks_getuntil(ks, '\n', &seq->comment, 0); \ - while ((c = ks_getc(ks)) != -1 && c != '>' && c != '+' && c != '@') { \ - if (isgraph(c)) { /* printable non-space character */ \ - if (seq->seq.l + 1 >= seq->seq.m) { /* double the memory */ \ - seq->seq.m = seq->seq.l + 2; \ - kroundup32(seq->seq.m); /* rounded to next closest 2^k */ \ - seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \ - } \ - seq->seq.s[seq->seq.l++] = (char)c; \ - } \ - } \ - if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */ \ - seq->seq.s[seq->seq.l] = 0; /* null terminated string */ \ - if (c != '+') return seq->seq.l; /* FASTA */ \ - if (seq->qual.m < seq->seq.m) { /* allocate enough memory */ \ - seq->qual.m = seq->seq.m; \ - seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \ - } \ - while ((c = ks_getc(ks)) != -1 && c != '\n'); /* skip the rest of '+' line */ \ - if (c == -1) return -2; /* we should not stop here */ \ - while ((c = ks_getc(ks)) != -1 && seq->qual.l < seq->seq.l) \ - if (c >= 33 && c <= 127) seq->qual.s[seq->qual.l++] = (unsigned char)c; \ - seq->qual.s[seq->qual.l] = 0; /* null terminated string */ \ - seq->last_char = 0; /* we have not come to the next header line */ \ - if (seq->seq.l != seq->qual.l) return -2; /* qual string is shorter than seq string */ \ - return seq->seq.l; \ - } - -#define __KSEQ_TYPE(type_t) \ - typedef struct { \ - kstring_t name, comment, seq, qual; \ - int last_char; \ - kstream_t *f; \ - } kseq_t; - -#define KSEQ_INIT(type_t, __read) \ - KSTREAM_INIT(type_t, __read, 4096) \ - __KSEQ_TYPE(type_t) \ - __KSEQ_BASIC(type_t) \ - __KSEQ_READ - -#endif diff --git a/samtools-0.1.13/ksort.h b/samtools-0.1.13/ksort.h deleted file mode 100644 --- a/samtools-0.1.13/ksort.h +++ /dev/null @@ -1,281 +0,0 @@ -/* The MIT License - - Copyright (c) 2008 Genome Research Ltd (GRL). - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -/* Contact: Heng Li */ - -/* - 2008-11-16 (0.1.4): - - * Fixed a bug in introsort() that happens in rare cases. - - 2008-11-05 (0.1.3): - - * Fixed a bug in introsort() for complex comparisons. - - * Fixed a bug in mergesort(). The previous version is not stable. - - 2008-09-15 (0.1.2): - - * Accelerated introsort. On my Mac (not on another Linux machine), - my implementation is as fast as std::sort on random input. - - * Added combsort and in introsort, switch to combsort if the - recursion is too deep. - - 2008-09-13 (0.1.1): - - * Added k-small algorithm - - 2008-09-05 (0.1.0): - - * Initial version - -*/ - -#ifndef AC_KSORT_H -#define AC_KSORT_H - -#include -#include - -typedef struct { - void *left, *right; - int depth; -} ks_isort_stack_t; - -#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; } - -#define KSORT_INIT(name, type_t, __sort_lt) \ - void ks_mergesort_##name(size_t n, type_t array[], type_t temp[]) \ - { \ - type_t *a2[2], *a, *b; \ - int curr, shift; \ - \ - a2[0] = array; \ - a2[1] = temp? temp : (type_t*)malloc(sizeof(type_t) * n); \ - for (curr = 0, shift = 0; (1ul<> 1) - 1; i != (size_t)(-1); --i) \ - ks_heapadjust_##name(i, lsize, l); \ - } \ - void ks_heapsort_##name(size_t lsize, type_t l[]) \ - { \ - size_t i; \ - for (i = lsize - 1; i > 0; --i) { \ - type_t tmp; \ - tmp = *l; *l = l[i]; l[i] = tmp; ks_heapadjust_##name(0, i, l); \ - } \ - } \ - inline void __ks_insertsort_##name(type_t *s, type_t *t) \ - { \ - type_t *i, *j, swap_tmp; \ - for (i = s + 1; i < t; ++i) \ - for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) { \ - swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp; \ - } \ - } \ - void ks_combsort_##name(size_t n, type_t a[]) \ - { \ - const double shrink_factor = 1.2473309501039786540366528676643; \ - int do_swap; \ - size_t gap = n; \ - type_t tmp, *i, *j; \ - do { \ - if (gap > 2) { \ - gap = (size_t)(gap / shrink_factor); \ - if (gap == 9 || gap == 10) gap = 11; \ - } \ - do_swap = 0; \ - for (i = a; i < a + n - gap; ++i) { \ - j = i + gap; \ - if (__sort_lt(*j, *i)) { \ - tmp = *i; *i = *j; *j = tmp; \ - do_swap = 1; \ - } \ - } \ - } while (do_swap || gap > 2); \ - if (gap != 1) __ks_insertsort_##name(a, a + n); \ - } \ - void ks_introsort_##name(size_t n, type_t a[]) \ - { \ - int d; \ - ks_isort_stack_t *top, *stack; \ - type_t rp, swap_tmp; \ - type_t *s, *t, *i, *j, *k; \ - \ - if (n < 1) return; \ - else if (n == 2) { \ - if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \ - return; \ - } \ - for (d = 2; 1ul<>1) + 1; \ - if (__sort_lt(*k, *i)) { \ - if (__sort_lt(*k, *j)) k = j; \ - } else k = __sort_lt(*j, *i)? i : j; \ - rp = *k; \ - if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; } \ - for (;;) { \ - do ++i; while (__sort_lt(*i, rp)); \ - do --j; while (i <= j && __sort_lt(rp, *j)); \ - if (j <= i) break; \ - swap_tmp = *i; *i = *j; *j = swap_tmp; \ - } \ - swap_tmp = *i; *i = *t; *t = swap_tmp; \ - if (i-s > t-i) { \ - if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \ - s = t-i > 16? i+1 : t; \ - } else { \ - if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \ - t = i-s > 16? i-1 : s; \ - } \ - } else { \ - if (top == stack) { \ - free(stack); \ - __ks_insertsort_##name(a, a+n); \ - return; \ - } else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \ - } \ - } \ - } \ - /* This function is adapted from: http://ndevilla.free.fr/median/ */ \ - /* 0 <= kk < n */ \ - type_t ks_ksmall_##name(size_t n, type_t arr[], size_t kk) \ - { \ - type_t *low, *high, *k, *ll, *hh, *mid; \ - low = arr; high = arr + n - 1; k = arr + kk; \ - for (;;) { \ - if (high <= low) return *k; \ - if (high == low + 1) { \ - if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \ - return *k; \ - } \ - mid = low + (high - low) / 2; \ - if (__sort_lt(*high, *mid)) KSORT_SWAP(type_t, *mid, *high); \ - if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \ - if (__sort_lt(*low, *mid)) KSORT_SWAP(type_t, *mid, *low); \ - KSORT_SWAP(type_t, *mid, *(low+1)); \ - ll = low + 1; hh = high; \ - for (;;) { \ - do ++ll; while (__sort_lt(*ll, *low)); \ - do --hh; while (__sort_lt(*low, *hh)); \ - if (hh < ll) break; \ - KSORT_SWAP(type_t, *ll, *hh); \ - } \ - KSORT_SWAP(type_t, *low, *hh); \ - if (hh <= k) low = ll; \ - if (hh >= k) high = hh - 1; \ - } \ - } \ - void ks_shuffle_##name(size_t n, type_t a[]) \ - { \ - int i, j; \ - for (i = n; i > 1; --i) { \ - type_t tmp; \ - j = (int)(drand48() * i); \ - tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp; \ - } \ - } - -#define ks_mergesort(name, n, a, t) ks_mergesort_##name(n, a, t) -#define ks_introsort(name, n, a) ks_introsort_##name(n, a) -#define ks_combsort(name, n, a) ks_combsort_##name(n, a) -#define ks_heapsort(name, n, a) ks_heapsort_##name(n, a) -#define ks_heapmake(name, n, a) ks_heapmake_##name(n, a) -#define ks_heapadjust(name, i, n, a) ks_heapadjust_##name(i, n, a) -#define ks_ksmall(name, n, a, k) ks_ksmall_##name(n, a, k) -#define ks_shuffle(name, n, a) ks_shuffle_##name(n, a) - -#define ks_lt_generic(a, b) ((a) < (b)) -#define ks_lt_str(a, b) (strcmp((a), (b)) < 0) - -typedef const char *ksstr_t; - -#define KSORT_INIT_GENERIC(type_t) KSORT_INIT(type_t, type_t, ks_lt_generic) -#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str) - -#endif diff --git a/samtools-0.1.13/kstring.c b/samtools-0.1.13/kstring.c deleted file mode 100644 --- a/samtools-0.1.13/kstring.c +++ /dev/null @@ -1,212 +0,0 @@ -#include -#include -#include -#include -#include -#include "kstring.h" - -int ksprintf(kstring_t *s, const char *fmt, ...) -{ - va_list ap; - int l; - va_start(ap, fmt); - l = vsnprintf(s->s + s->l, s->m - s->l, fmt, ap); // This line does not work with glibc 2.0. See `man snprintf'. - va_end(ap); - if (l + 1 > s->m - s->l) { - s->m = s->l + l + 2; - kroundup32(s->m); - s->s = (char*)realloc(s->s, s->m); - va_start(ap, fmt); - l = vsnprintf(s->s + s->l, s->m - s->l, fmt, ap); - } - va_end(ap); - s->l += l; - return l; -} - -char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux) -{ - const char *p, *start; - if (sep) { // set up the table - if (str == 0 && (aux->tab[0]&1)) return 0; // no need to set up if we have finished - aux->finished = 0; - if (sep[1]) { - aux->sep = -1; - aux->tab[0] = aux->tab[1] = aux->tab[2] = aux->tab[3] = 0; - for (p = sep; *p; ++p) aux->tab[*p>>6] |= 1ull<<(*p&0x3f); - } else aux->sep = sep[0]; - } - if (aux->finished) return 0; - else if (str) aux->p = str - 1, aux->finished = 0; - if (aux->sep < 0) { - for (p = start = aux->p + 1; *p; ++p) - if (aux->tab[*p>>6]>>(*p&0x3f)&1) break; - } else { - for (p = start = aux->p + 1; *p; ++p) - if (*p == aux->sep) break; - } - aux->p = p; // end of token - if (*p == 0) aux->finished = 1; // no more tokens - return (char*)start; -} - -// s MUST BE a null terminated string; l = strlen(s) -int ksplit_core(char *s, int delimiter, int *_max, int **_offsets) -{ - int i, n, max, last_char, last_start, *offsets, l; - n = 0; max = *_max; offsets = *_offsets; - l = strlen(s); - -#define __ksplit_aux do { \ - if (_offsets) { \ - s[i] = 0; \ - if (n == max) { \ - max = max? max<<1 : 2; \ - offsets = (int*)realloc(offsets, sizeof(int) * max); \ - } \ - offsets[n++] = last_start; \ - } else ++n; \ - } while (0) - - for (i = 0, last_char = last_start = 0; i <= l; ++i) { - if (delimiter == 0) { - if (isspace(s[i]) || s[i] == 0) { - if (isgraph(last_char)) __ksplit_aux; // the end of a field - } else { - if (isspace(last_char) || last_char == 0) last_start = i; - } - } else { - if (s[i] == delimiter || s[i] == 0) { - if (last_char != 0 && last_char != delimiter) __ksplit_aux; // the end of a field - } else { - if (last_char == delimiter || last_char == 0) last_start = i; - } - } - last_char = s[i]; - } - *_max = max; *_offsets = offsets; - return n; -} - -/********************** - * Boyer-Moore search * - **********************/ - -typedef unsigned char ubyte_t; - -// reference: http://www-igm.univ-mlv.fr/~lecroq/string/node14.html -static int *ksBM_prep(const ubyte_t *pat, int m) -{ - int i, *suff, *prep, *bmGs, *bmBc; - prep = calloc(m + 256, sizeof(int)); - bmGs = prep; bmBc = prep + m; - { // preBmBc() - for (i = 0; i < 256; ++i) bmBc[i] = m; - for (i = 0; i < m - 1; ++i) bmBc[pat[i]] = m - i - 1; - } - suff = calloc(m, sizeof(int)); - { // suffixes() - int f = 0, g; - suff[m - 1] = m; - g = m - 1; - for (i = m - 2; i >= 0; --i) { - if (i > g && suff[i + m - 1 - f] < i - g) - suff[i] = suff[i + m - 1 - f]; - else { - if (i < g) g = i; - f = i; - while (g >= 0 && pat[g] == pat[g + m - 1 - f]) --g; - suff[i] = f - g; - } - } - } - { // preBmGs() - int j = 0; - for (i = 0; i < m; ++i) bmGs[i] = m; - for (i = m - 1; i >= 0; --i) - if (suff[i] == i + 1) - for (; j < m - 1 - i; ++j) - if (bmGs[j] == m) - bmGs[j] = m - 1 - i; - for (i = 0; i <= m - 2; ++i) - bmGs[m - 1 - suff[i]] = m - 1 - i; - } - free(suff); - return prep; -} - -void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep) -{ - int i, j, *prep = 0, *bmGs, *bmBc; - const ubyte_t *str, *pat; - str = (const ubyte_t*)_str; pat = (const ubyte_t*)_pat; - prep = (_prep == 0 || *_prep == 0)? ksBM_prep(pat, m) : *_prep; - if (_prep && *_prep == 0) *_prep = prep; - bmGs = prep; bmBc = prep + m; - j = 0; - while (j <= n - m) { - for (i = m - 1; i >= 0 && pat[i] == str[i+j]; --i); - if (i >= 0) { - int max = bmBc[str[i+j]] - m + 1 + i; - if (max < bmGs[i]) max = bmGs[i]; - j += max; - } else return (void*)(str + j); - } - if (_prep == 0) free(prep); - return 0; -} - -char *kstrstr(const char *str, const char *pat, int **_prep) -{ - return (char*)kmemmem(str, strlen(str), pat, strlen(pat), _prep); -} - -char *kstrnstr(const char *str, const char *pat, int n, int **_prep) -{ - return (char*)kmemmem(str, n, pat, strlen(pat), _prep); -} - -/*********************** - * The main() function * - ***********************/ - -#ifdef KSTRING_MAIN -#include -int main() -{ - kstring_t *s; - int *fields, n, i; - ks_tokaux_t aux; - char *p; - s = (kstring_t*)calloc(1, sizeof(kstring_t)); - // test ksprintf() - ksprintf(s, " abcdefg: %d ", 100); - printf("'%s'\n", s->s); - // test ksplit() - fields = ksplit(s, 0, &n); - for (i = 0; i < n; ++i) - printf("field[%d] = '%s'\n", i, s->s + fields[i]); - // test kstrtok() - s->l = 0; - for (p = kstrtok("ab:cde:fg/hij::k", ":/", &aux); p; p = kstrtok(0, 0, &aux)) { - kputsn(p, aux.p - p, s); - kputc('\n', s); - } - printf("%s", s->s); - // free - free(s->s); free(s); free(fields); - - { - static char *str = "abcdefgcdgcagtcakcdcd"; - static char *pat = "cd"; - char *ret, *s = str; - int *prep = 0; - while ((ret = kstrstr(s, pat, &prep)) != 0) { - printf("match: %s\n", ret); - s = ret + prep[0]; - } - free(prep); - } - return 0; -} -#endif diff --git a/samtools-0.1.13/kstring.h b/samtools-0.1.13/kstring.h deleted file mode 100644 --- a/samtools-0.1.13/kstring.h +++ /dev/null @@ -1,117 +0,0 @@ -#ifndef KSTRING_H -#define KSTRING_H - -#include -#include -#include - -#ifndef kroundup32 -#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) -#endif - -#ifndef KSTRING_T -#define KSTRING_T kstring_t -typedef struct __kstring_t { - size_t l, m; - char *s; -} kstring_t; -#endif - -typedef struct { - uint64_t tab[4]; - int sep, finished; - const char *p; // end of the current token -} ks_tokaux_t; - -#ifdef __cplusplus -extern "C" { -#endif - - int ksprintf(kstring_t *s, const char *fmt, ...); - int ksplit_core(char *s, int delimiter, int *_max, int **_offsets); - char *kstrstr(const char *str, const char *pat, int **_prep); - char *kstrnstr(const char *str, const char *pat, int n, int **_prep); - void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep); - - /* kstrtok() is similar to strtok_r() except that str is not - * modified and both str and sep can be NULL. For efficiency, it is - * actually recommended to set both to NULL in the subsequent calls - * if sep is not changed. */ - char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux); - -#ifdef __cplusplus -} -#endif - -static inline int kputsn(const char *p, int l, kstring_t *s) -{ - if (s->l + l + 1 >= s->m) { - s->m = s->l + l + 2; - kroundup32(s->m); - s->s = (char*)realloc(s->s, s->m); - } - memcpy(s->s + s->l, p, l); - s->l += l; - s->s[s->l] = 0; - return l; -} - -static inline int kputs(const char *p, kstring_t *s) -{ - return kputsn(p, strlen(p), s); -} - -static inline int kputc(int c, kstring_t *s) -{ - if (s->l + 1 >= s->m) { - s->m = s->l + 2; - kroundup32(s->m); - s->s = (char*)realloc(s->s, s->m); - } - s->s[s->l++] = c; - s->s[s->l] = 0; - return c; -} - -static inline int kputw(int c, kstring_t *s) -{ - char buf[16]; - int l, x; - if (c == 0) return kputc('0', s); - for (l = 0, x = c < 0? -c : c; x > 0; x /= 10) buf[l++] = x%10 + '0'; - if (c < 0) buf[l++] = '-'; - if (s->l + l + 1 >= s->m) { - s->m = s->l + l + 2; - kroundup32(s->m); - s->s = (char*)realloc(s->s, s->m); - } - for (x = l - 1; x >= 0; --x) s->s[s->l++] = buf[x]; - s->s[s->l] = 0; - return 0; -} - -static inline int kputuw(unsigned c, kstring_t *s) -{ - char buf[16]; - int l, i; - unsigned x; - if (c == 0) return kputc('0', s); - for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0'; - if (s->l + l + 1 >= s->m) { - s->m = s->l + l + 2; - kroundup32(s->m); - s->s = (char*)realloc(s->s, s->m); - } - for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i]; - s->s[s->l] = 0; - return 0; -} - -static inline int *ksplit(kstring_t *s, int delimiter, int *n) -{ - int max = 0, *offsets = 0; - *n = ksplit_core(s->s, delimiter, &max, &offsets); - return offsets; -} - -#endif diff --git a/samtools-0.1.13/razf.c b/samtools-0.1.13/razf.c deleted file mode 100644 --- a/samtools-0.1.13/razf.c +++ /dev/null @@ -1,853 +0,0 @@ -/* - * RAZF : Random Access compressed(Z) File - * Version: 1.0 - * Release Date: 2008-10-27 - * - * Copyright 2008, Jue Ruan , Heng Li - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef _NO_RAZF - -#include -#include -#include -#include -#include -#include "razf.h" - - -#if ZLIB_VERNUM < 0x1221 -struct _gz_header_s { - int text; - uLong time; - int xflags; - int os; - Bytef *extra; - uInt extra_len; - uInt extra_max; - Bytef *name; - uInt name_max; - Bytef *comment; - uInt comm_max; - int hcrc; - int done; -}; -#warning "zlib < 1.2.2.1; RAZF writing is disabled." -#endif - -#define DEF_MEM_LEVEL 8 - -static inline uint32_t byte_swap_4(uint32_t v){ - v = ((v & 0x0000FFFFU) << 16) | (v >> 16); - return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8); -} - -static inline uint64_t byte_swap_8(uint64_t v){ - v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32); - v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16); - return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8); -} - -static inline int is_big_endian(){ - int x = 0x01; - char *c = (char*)&x; - return (c[0] != 0x01); -} - -#ifndef _RZ_READONLY -static void add_zindex(RAZF *rz, int64_t in, int64_t out){ - if(rz->index->size == rz->index->cap){ - rz->index->cap = rz->index->cap * 1.5 + 2; - rz->index->cell_offsets = realloc(rz->index->cell_offsets, sizeof(int) * rz->index->cap); - rz->index->bin_offsets = realloc(rz->index->bin_offsets, sizeof(int64_t) * (rz->index->cap/RZ_BIN_SIZE + 1)); - } - if(rz->index->size % RZ_BIN_SIZE == 0) rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE] = out; - rz->index->cell_offsets[rz->index->size] = out - rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE]; - rz->index->size ++; -} - -static void save_zindex(RAZF *rz, int fd){ - int32_t i, v32; - int is_be; - is_be = is_big_endian(); - if(is_be) write(fd, &rz->index->size, sizeof(int)); - else { - v32 = byte_swap_4((uint32_t)rz->index->size); - write(fd, &v32, sizeof(uint32_t)); - } - v32 = rz->index->size / RZ_BIN_SIZE + 1; - if(!is_be){ - for(i=0;iindex->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]); - for(i=0;iindex->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]); - } - write(fd, rz->index->bin_offsets, sizeof(int64_t) * v32); - write(fd, rz->index->cell_offsets, sizeof(int32_t) * rz->index->size); -} -#endif - -#ifdef _USE_KNETFILE -static void load_zindex(RAZF *rz, knetFile *fp){ -#else -static void load_zindex(RAZF *rz, int fd){ -#endif - int32_t i, v32; - int is_be; - if(!rz->load_index) return; - if(rz->index == NULL) rz->index = malloc(sizeof(ZBlockIndex)); - is_be = is_big_endian(); -#ifdef _USE_KNETFILE - knet_read(fp, &rz->index->size, sizeof(int)); -#else - read(fd, &rz->index->size, sizeof(int)); -#endif - if(!is_be) rz->index->size = byte_swap_4((uint32_t)rz->index->size); - rz->index->cap = rz->index->size; - v32 = rz->index->size / RZ_BIN_SIZE + 1; - rz->index->bin_offsets = malloc(sizeof(int64_t) * v32); -#ifdef _USE_KNETFILE - knet_read(fp, rz->index->bin_offsets, sizeof(int64_t) * v32); -#else - read(fd, rz->index->bin_offsets, sizeof(int64_t) * v32); -#endif - rz->index->cell_offsets = malloc(sizeof(int) * rz->index->size); -#ifdef _USE_KNETFILE - knet_read(fp, rz->index->cell_offsets, sizeof(int) * rz->index->size); -#else - read(fd, rz->index->cell_offsets, sizeof(int) * rz->index->size); -#endif - if(!is_be){ - for(i=0;iindex->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]); - for(i=0;iindex->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]); - } -} - -#ifdef _RZ_READONLY -static RAZF* razf_open_w(int fd) -{ - fprintf(stderr, "[razf_open_w] Writing is not available with zlib ver < 1.2.2.1\n"); - return 0; -} -#else -static RAZF* razf_open_w(int fd){ - RAZF *rz; -#ifdef _WIN32 - setmode(fd, O_BINARY); -#endif - rz = calloc(1, sizeof(RAZF)); - rz->mode = 'w'; -#ifdef _USE_KNETFILE - rz->x.fpw = fd; -#else - rz->filedes = fd; -#endif - rz->stream = calloc(sizeof(z_stream), 1); - rz->inbuf = malloc(RZ_BUFFER_SIZE); - rz->outbuf = malloc(RZ_BUFFER_SIZE); - rz->index = calloc(sizeof(ZBlockIndex), 1); - deflateInit2(rz->stream, RZ_COMPRESS_LEVEL, Z_DEFLATED, WINDOW_BITS + 16, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); - rz->stream->avail_out = RZ_BUFFER_SIZE; - rz->stream->next_out = rz->outbuf; - rz->header = calloc(sizeof(gz_header), 1); - rz->header->os = 0x03; //Unix - rz->header->text = 0; - rz->header->time = 0; - rz->header->extra = malloc(7); - strncpy((char*)rz->header->extra, "RAZF", 4); - rz->header->extra[4] = 1; // obsolete field - // block size = RZ_BLOCK_SIZE, Big-Endian - rz->header->extra[5] = RZ_BLOCK_SIZE >> 8; - rz->header->extra[6] = RZ_BLOCK_SIZE & 0xFF; - rz->header->extra_len = 7; - rz->header->name = rz->header->comment = 0; - rz->header->hcrc = 0; - deflateSetHeader(rz->stream, rz->header); - rz->block_pos = rz->block_off = 0; - return rz; -} - -static void _razf_write(RAZF* rz, const void *data, int size){ - int tout; - rz->stream->avail_in = size; - rz->stream->next_in = (void*)data; - while(1){ - tout = rz->stream->avail_out; - deflate(rz->stream, Z_NO_FLUSH); - rz->out += tout - rz->stream->avail_out; - if(rz->stream->avail_out) break; -#ifdef _USE_KNETFILE - write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#else - write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#endif - rz->stream->avail_out = RZ_BUFFER_SIZE; - rz->stream->next_out = rz->outbuf; - if(rz->stream->avail_in == 0) break; - }; - rz->in += size - rz->stream->avail_in; - rz->block_off += size - rz->stream->avail_in; -} - -static void razf_flush(RAZF *rz){ - uint32_t tout; - if(rz->buf_len){ - _razf_write(rz, rz->inbuf, rz->buf_len); - rz->buf_off = rz->buf_len = 0; - } - if(rz->stream->avail_out){ -#ifdef _USE_KNETFILE - write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#else - write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#endif - rz->stream->avail_out = RZ_BUFFER_SIZE; - rz->stream->next_out = rz->outbuf; - } - while(1){ - tout = rz->stream->avail_out; - deflate(rz->stream, Z_FULL_FLUSH); - rz->out += tout - rz->stream->avail_out; - if(rz->stream->avail_out == 0){ -#ifdef _USE_KNETFILE - write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#else - write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#endif - rz->stream->avail_out = RZ_BUFFER_SIZE; - rz->stream->next_out = rz->outbuf; - } else break; - } - rz->block_pos = rz->out; - rz->block_off = 0; -} - -static void razf_end_flush(RAZF *rz){ - uint32_t tout; - if(rz->buf_len){ - _razf_write(rz, rz->inbuf, rz->buf_len); - rz->buf_off = rz->buf_len = 0; - } - while(1){ - tout = rz->stream->avail_out; - deflate(rz->stream, Z_FINISH); - rz->out += tout - rz->stream->avail_out; - if(rz->stream->avail_out < RZ_BUFFER_SIZE){ -#ifdef _USE_KNETFILE - write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#else - write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); -#endif - rz->stream->avail_out = RZ_BUFFER_SIZE; - rz->stream->next_out = rz->outbuf; - } else break; - } -} - -static void _razf_buffered_write(RAZF *rz, const void *data, int size){ - int i, n; - while(1){ - if(rz->buf_len == RZ_BUFFER_SIZE){ - _razf_write(rz, rz->inbuf, rz->buf_len); - rz->buf_len = 0; - } - if(size + rz->buf_len < RZ_BUFFER_SIZE){ - for(i=0;iinbuf + rz->buf_len)[i] = ((char*)data)[i]; - rz->buf_len += size; - return; - } else { - n = RZ_BUFFER_SIZE - rz->buf_len; - for(i=0;iinbuf + rz->buf_len)[i] = ((char*)data)[i]; - size -= n; - data += n; - rz->buf_len += n; - } - } -} - -int razf_write(RAZF* rz, const void *data, int size){ - int ori_size, n; - int64_t next_block; - ori_size = size; - next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE; - while(rz->in + rz->buf_len + size >= next_block){ - n = next_block - rz->in - rz->buf_len; - _razf_buffered_write(rz, data, n); - data += n; - size -= n; - razf_flush(rz); - add_zindex(rz, rz->in, rz->out); - next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE; - } - _razf_buffered_write(rz, data, size); - return ori_size; -} -#endif - -/* gzip flag byte */ -#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ -#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ -#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ -#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ -#define COMMENT 0x10 /* bit 4 set: file comment present */ -#define RESERVED 0xE0 /* bits 5..7: reserved */ - -static int _read_gz_header(unsigned char *data, int size, int *extra_off, int *extra_len){ - int method, flags, n, len; - if(size < 2) return 0; - if(data[0] != 0x1f || data[1] != 0x8b) return 0; - if(size < 4) return 0; - method = data[2]; - flags = data[3]; - if(method != Z_DEFLATED || (flags & RESERVED)) return 0; - n = 4 + 6; // Skip 6 bytes - *extra_off = n + 2; - *extra_len = 0; - if(flags & EXTRA_FIELD){ - if(size < n + 2) return 0; - len = ((int)data[n + 1] << 8) | data[n]; - n += 2; - *extra_off = n; - while(len){ - if(n >= size) return 0; - n ++; - len --; - } - *extra_len = n - (*extra_off); - } - if(flags & ORIG_NAME) while(n < size && data[n++]); - if(flags & COMMENT) while(n < size && data[n++]); - if(flags & HEAD_CRC){ - if(n + 2 > size) return 0; - n += 2; - } - return n; -} - -#ifdef _USE_KNETFILE -static RAZF* razf_open_r(knetFile *fp, int _load_index){ -#else -static RAZF* razf_open_r(int fd, int _load_index){ -#endif - RAZF *rz; - int ext_off, ext_len; - int n, is_be, ret; - int64_t end; - unsigned char c[] = "RAZF"; - rz = calloc(1, sizeof(RAZF)); - rz->mode = 'r'; -#ifdef _USE_KNETFILE - rz->x.fpr = fp; -#else -#ifdef _WIN32 - setmode(fd, O_BINARY); -#endif - rz->filedes = fd; -#endif - rz->stream = calloc(sizeof(z_stream), 1); - rz->inbuf = malloc(RZ_BUFFER_SIZE); - rz->outbuf = malloc(RZ_BUFFER_SIZE); - rz->end = rz->src_end = 0x7FFFFFFFFFFFFFFFLL; -#ifdef _USE_KNETFILE - n = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE); -#else - n = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE); -#endif - ret = _read_gz_header(rz->inbuf, n, &ext_off, &ext_len); - if(ret == 0){ - PLAIN_FILE: - rz->in = n; - rz->file_type = FILE_TYPE_PLAIN; - memcpy(rz->outbuf, rz->inbuf, n); - rz->buf_len = n; - free(rz->stream); - rz->stream = NULL; - return rz; - } - rz->header_size = ret; - ret = inflateInit2(rz->stream, -WINDOW_BITS); - if(ret != Z_OK){ inflateEnd(rz->stream); goto PLAIN_FILE;} - rz->stream->avail_in = n - rz->header_size; - rz->stream->next_in = rz->inbuf + rz->header_size; - rz->stream->avail_out = RZ_BUFFER_SIZE; - rz->stream->next_out = rz->outbuf; - rz->file_type = FILE_TYPE_GZ; - rz->in = rz->header_size; - rz->block_pos = rz->header_size; - rz->next_block_pos = rz->header_size; - rz->block_off = 0; - if(ext_len < 7 || memcmp(rz->inbuf + ext_off, c, 4) != 0) return rz; - if(((((unsigned char*)rz->inbuf)[ext_off + 5] << 8) | ((unsigned char*)rz->inbuf)[ext_off + 6]) != RZ_BLOCK_SIZE){ - fprintf(stderr, " -- WARNING: RZ_BLOCK_SIZE is not %d, treat source as gz file. in %s -- %s:%d --\n", RZ_BLOCK_SIZE, __FUNCTION__, __FILE__, __LINE__); - return rz; - } - rz->load_index = _load_index; - rz->file_type = FILE_TYPE_RZ; -#ifdef _USE_KNETFILE - if(knet_seek(fp, -16, SEEK_END) == -1){ -#else - if(lseek(fd, -16, SEEK_END) == -1){ -#endif - UNSEEKABLE: - rz->seekable = 0; - rz->index = NULL; - rz->src_end = rz->end = 0x7FFFFFFFFFFFFFFFLL; - } else { - is_be = is_big_endian(); - rz->seekable = 1; -#ifdef _USE_KNETFILE - knet_read(fp, &end, sizeof(int64_t)); -#else - read(fd, &end, sizeof(int64_t)); -#endif - if(!is_be) rz->src_end = (int64_t)byte_swap_8((uint64_t)end); - else rz->src_end = end; - -#ifdef _USE_KNETFILE - knet_read(fp, &end, sizeof(int64_t)); -#else - read(fd, &end, sizeof(int64_t)); -#endif - if(!is_be) rz->end = (int64_t)byte_swap_8((uint64_t)end); - else rz->end = end; - if(n > rz->end){ - rz->stream->avail_in -= n - rz->end; - n = rz->end; - } - if(rz->end > rz->src_end){ -#ifdef _USE_KNETFILE - knet_seek(fp, rz->in, SEEK_SET); -#else - lseek(fd, rz->in, SEEK_SET); -#endif - goto UNSEEKABLE; - } -#ifdef _USE_KNETFILE - knet_seek(fp, rz->end, SEEK_SET); - if(knet_tell(fp) != rz->end){ - knet_seek(fp, rz->in, SEEK_SET); -#else - if(lseek(fd, rz->end, SEEK_SET) != rz->end){ - lseek(fd, rz->in, SEEK_SET); -#endif - goto UNSEEKABLE; - } -#ifdef _USE_KNETFILE - load_zindex(rz, fp); - knet_seek(fp, n, SEEK_SET); -#else - load_zindex(rz, fd); - lseek(fd, n, SEEK_SET); -#endif - } - return rz; -} - -#ifdef _USE_KNETFILE -RAZF* razf_dopen(int fd, const char *mode){ - if (strstr(mode, "r")) fprintf(stderr,"[razf_dopen] implement me\n"); - else if(strstr(mode, "w")) return razf_open_w(fd); - return NULL; -} - -RAZF* razf_dopen2(int fd, const char *mode) -{ - fprintf(stderr,"[razf_dopen2] implement me\n"); - return NULL; -} -#else -RAZF* razf_dopen(int fd, const char *mode){ - if(strstr(mode, "r")) return razf_open_r(fd, 1); - else if(strstr(mode, "w")) return razf_open_w(fd); - else return NULL; -} - -RAZF* razf_dopen2(int fd, const char *mode) -{ - if(strstr(mode, "r")) return razf_open_r(fd, 0); - else if(strstr(mode, "w")) return razf_open_w(fd); - else return NULL; -} -#endif - -static inline RAZF* _razf_open(const char *filename, const char *mode, int _load_index){ - int fd; - RAZF *rz; - if(strstr(mode, "r")){ -#ifdef _USE_KNETFILE - knetFile *fd = knet_open(filename, "r"); - if (fd == 0) { - fprintf(stderr, "[_razf_open] fail to open %s\n", filename); - return NULL; - } -#else -#ifdef _WIN32 - fd = open(filename, O_RDONLY | O_BINARY); -#else - fd = open(filename, O_RDONLY); -#endif -#endif - if(fd < 0) return NULL; - rz = razf_open_r(fd, _load_index); - } else if(strstr(mode, "w")){ -#ifdef _WIN32 - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666); -#else - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666); -#endif - if(fd < 0) return NULL; - rz = razf_open_w(fd); - } else return NULL; - return rz; -} - -RAZF* razf_open(const char *filename, const char *mode){ - return _razf_open(filename, mode, 1); -} - -RAZF* razf_open2(const char *filename, const char *mode){ - return _razf_open(filename, mode, 0); -} - -int razf_get_data_size(RAZF *rz, int64_t *u_size, int64_t *c_size){ - int64_t n; - if(rz->mode != 'r' && rz->mode != 'R') return 0; - switch(rz->file_type){ - case FILE_TYPE_PLAIN: - if(rz->end == 0x7fffffffffffffffLL){ -#ifdef _USE_KNETFILE - if(knet_seek(rz->x.fpr, 0, SEEK_CUR) == -1) return 0; - n = knet_tell(rz->x.fpr); - knet_seek(rz->x.fpr, 0, SEEK_END); - rz->end = knet_tell(rz->x.fpr); - knet_seek(rz->x.fpr, n, SEEK_SET); -#else - if((n = lseek(rz->filedes, 0, SEEK_CUR)) == -1) return 0; - rz->end = lseek(rz->filedes, 0, SEEK_END); - lseek(rz->filedes, n, SEEK_SET); -#endif - } - *u_size = *c_size = rz->end; - return 1; - case FILE_TYPE_GZ: - return 0; - case FILE_TYPE_RZ: - if(rz->src_end == rz->end) return 0; - *u_size = rz->src_end; - *c_size = rz->end; - return 1; - default: - return 0; - } -} - -static int _razf_read(RAZF* rz, void *data, int size){ - int ret, tin; - if(rz->z_eof || rz->z_err) return 0; - if (rz->file_type == FILE_TYPE_PLAIN) { -#ifdef _USE_KNETFILE - ret = knet_read(rz->x.fpr, data, size); -#else - ret = read(rz->filedes, data, size); -#endif - if (ret == 0) rz->z_eof = 1; - return ret; - } - rz->stream->avail_out = size; - rz->stream->next_out = data; - while(rz->stream->avail_out){ - if(rz->stream->avail_in == 0){ - if(rz->in >= rz->end){ rz->z_eof = 1; break; } - if(rz->end - rz->in < RZ_BUFFER_SIZE){ -#ifdef _USE_KNETFILE - rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, rz->end -rz->in); -#else - rz->stream->avail_in = read(rz->filedes, rz->inbuf, rz->end -rz->in); -#endif - } else { -#ifdef _USE_KNETFILE - rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE); -#else - rz->stream->avail_in = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE); -#endif - } - if(rz->stream->avail_in == 0){ - rz->z_eof = 1; - break; - } - rz->stream->next_in = rz->inbuf; - } - tin = rz->stream->avail_in; - ret = inflate(rz->stream, Z_BLOCK); - rz->in += tin - rz->stream->avail_in; - if(ret == Z_NEED_DICT || ret == Z_MEM_ERROR || ret == Z_DATA_ERROR){ - fprintf(stderr, "[_razf_read] inflate error: %d %s (at %s:%d)\n", ret, rz->stream->msg ? rz->stream->msg : "", __FILE__, __LINE__); - rz->z_err = 1; - break; - } - if(ret == Z_STREAM_END){ - rz->z_eof = 1; - break; - } - if ((rz->stream->data_type&128) && !(rz->stream->data_type&64)){ - rz->buf_flush = 1; - rz->next_block_pos = rz->in; - break; - } - } - return size - rz->stream->avail_out; -} - -int razf_read(RAZF *rz, void *data, int size){ - int ori_size, i; - ori_size = size; - while(size > 0){ - if(rz->buf_len){ - if(size < rz->buf_len){ - for(i=0;ioutbuf + rz->buf_off)[i]; - rz->buf_off += size; - rz->buf_len -= size; - data += size; - rz->block_off += size; - size = 0; - break; - } else { - for(i=0;ibuf_len;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i]; - data += rz->buf_len; - size -= rz->buf_len; - rz->block_off += rz->buf_len; - rz->buf_off = 0; - rz->buf_len = 0; - if(rz->buf_flush){ - rz->block_pos = rz->next_block_pos; - rz->block_off = 0; - rz->buf_flush = 0; - } - } - } else if(rz->buf_flush){ - rz->block_pos = rz->next_block_pos; - rz->block_off = 0; - rz->buf_flush = 0; - } - if(rz->buf_flush) continue; - rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE); - if(rz->z_eof && rz->buf_len == 0) break; - } - rz->out += ori_size - size; - return ori_size - size; -} - -int razf_skip(RAZF* rz, int size){ - int ori_size; - ori_size = size; - while(size > 0){ - if(rz->buf_len){ - if(size < rz->buf_len){ - rz->buf_off += size; - rz->buf_len -= size; - rz->block_off += size; - size = 0; - break; - } else { - size -= rz->buf_len; - rz->buf_off = 0; - rz->buf_len = 0; - rz->block_off += rz->buf_len; - if(rz->buf_flush){ - rz->block_pos = rz->next_block_pos; - rz->block_off = 0; - rz->buf_flush = 0; - } - } - } else if(rz->buf_flush){ - rz->block_pos = rz->next_block_pos; - rz->block_off = 0; - rz->buf_flush = 0; - } - if(rz->buf_flush) continue; - rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE); - if(rz->z_eof || rz->z_err) break; - } - rz->out += ori_size - size; - return ori_size - size; -} - -static void _razf_reset_read(RAZF *rz, int64_t in, int64_t out){ -#ifdef _USE_KNETFILE - knet_seek(rz->x.fpr, in, SEEK_SET); -#else - lseek(rz->filedes, in, SEEK_SET); -#endif - rz->in = in; - rz->out = out; - rz->block_pos = in; - rz->next_block_pos = in; - rz->block_off = 0; - rz->buf_flush = 0; - rz->z_eof = rz->z_err = 0; - inflateReset(rz->stream); - rz->stream->avail_in = 0; - rz->buf_off = rz->buf_len = 0; -} - -int64_t razf_jump(RAZF *rz, int64_t block_start, int block_offset){ - int64_t pos; - rz->z_eof = 0; - if(rz->file_type == FILE_TYPE_PLAIN){ - rz->buf_off = rz->buf_len = 0; - pos = block_start + block_offset; -#ifdef _USE_KNETFILE - knet_seek(rz->x.fpr, pos, SEEK_SET); - pos = knet_tell(rz->x.fpr); -#else - pos = lseek(rz->filedes, pos, SEEK_SET); -#endif - rz->out = rz->in = pos; - return pos; - } - if(block_start == rz->block_pos && block_offset >= rz->block_off) { - block_offset -= rz->block_off; - goto SKIP; // Needn't reset inflate - } - if(block_start == 0) block_start = rz->header_size; // Automaticly revist wrong block_start - _razf_reset_read(rz, block_start, 0); - SKIP: - if(block_offset) razf_skip(rz, block_offset); - return rz->block_off; -} - -int64_t razf_seek(RAZF* rz, int64_t pos, int where){ - int64_t idx; - int64_t seek_pos, new_out; - rz->z_eof = 0; - if (where == SEEK_CUR) pos += rz->out; - else if (where == SEEK_END) pos += rz->src_end; - if(rz->file_type == FILE_TYPE_PLAIN){ -#ifdef _USE_KNETFILE - knet_seek(rz->x.fpr, pos, SEEK_SET); - seek_pos = knet_tell(rz->x.fpr); -#else - seek_pos = lseek(rz->filedes, pos, SEEK_SET); -#endif - rz->buf_off = rz->buf_len = 0; - rz->out = rz->in = seek_pos; - return seek_pos; - } else if(rz->file_type == FILE_TYPE_GZ){ - if(pos >= rz->out) goto SKIP; - return rz->out; - } - if(pos == rz->out) return pos; - if(pos > rz->src_end) return rz->out; - if(!rz->seekable || !rz->load_index){ - if(pos >= rz->out) goto SKIP; - } - idx = pos / RZ_BLOCK_SIZE - 1; - seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]); - new_out = (idx + 1) * RZ_BLOCK_SIZE; - if(pos > rz->out && new_out <= rz->out) goto SKIP; - _razf_reset_read(rz, seek_pos, new_out); - SKIP: - razf_skip(rz, (int)(pos - rz->out)); - return rz->out; -} - -uint64_t razf_tell2(RAZF *rz) -{ - /* - if (rz->load_index) { - int64_t idx, seek_pos; - idx = rz->out / RZ_BLOCK_SIZE - 1; - seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]); - if (seek_pos != rz->block_pos || rz->out%RZ_BLOCK_SIZE != rz->block_off) - fprintf(stderr, "[razf_tell2] inconsistent block offset: (%lld, %lld) != (%lld, %lld)\n", - (long long)seek_pos, (long long)rz->out%RZ_BLOCK_SIZE, (long long)rz->block_pos, (long long) rz->block_off); - } - */ - return (uint64_t)rz->block_pos<<16 | (rz->block_off&0xffff); -} - -int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where) -{ - if (where != SEEK_SET) return -1; - return razf_jump(rz, voffset>>16, voffset&0xffff); -} - -void razf_close(RAZF *rz){ - if(rz->mode == 'w'){ -#ifndef _RZ_READONLY - razf_end_flush(rz); - deflateEnd(rz->stream); -#ifdef _USE_KNETFILE - save_zindex(rz, rz->x.fpw); - if(is_big_endian()){ - write(rz->x.fpw, &rz->in, sizeof(int64_t)); - write(rz->x.fpw, &rz->out, sizeof(int64_t)); - } else { - uint64_t v64 = byte_swap_8((uint64_t)rz->in); - write(rz->x.fpw, &v64, sizeof(int64_t)); - v64 = byte_swap_8((uint64_t)rz->out); - write(rz->x.fpw, &v64, sizeof(int64_t)); - } -#else - save_zindex(rz, rz->filedes); - if(is_big_endian()){ - write(rz->filedes, &rz->in, sizeof(int64_t)); - write(rz->filedes, &rz->out, sizeof(int64_t)); - } else { - uint64_t v64 = byte_swap_8((uint64_t)rz->in); - write(rz->filedes, &v64, sizeof(int64_t)); - v64 = byte_swap_8((uint64_t)rz->out); - write(rz->filedes, &v64, sizeof(int64_t)); - } -#endif -#endif - } else if(rz->mode == 'r'){ - if(rz->stream) inflateEnd(rz->stream); - } - if(rz->inbuf) free(rz->inbuf); - if(rz->outbuf) free(rz->outbuf); - if(rz->header){ - free(rz->header->extra); - free(rz->header->name); - free(rz->header->comment); - free(rz->header); - } - if(rz->index){ - free(rz->index->bin_offsets); - free(rz->index->cell_offsets); - free(rz->index); - } - free(rz->stream); -#ifdef _USE_KNETFILE - if (rz->mode == 'r') - knet_close(rz->x.fpr); - if (rz->mode == 'w') - close(rz->x.fpw); -#else - close(rz->filedes); -#endif - free(rz); -} - -#endif diff --git a/samtools-0.1.13/razf.h b/samtools-0.1.13/razf.h deleted file mode 100644 --- a/samtools-0.1.13/razf.h +++ /dev/null @@ -1,134 +0,0 @@ - /*- - * RAZF : Random Access compressed(Z) File - * Version: 1.0 - * Release Date: 2008-10-27 - * - * Copyright 2008, Jue Ruan , Heng Li - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef __RAZF_RJ_H -#define __RAZF_RJ_H - -#include -#include -#include "zlib.h" - -#ifdef _USE_KNETFILE -#include "knetfile.h" -#endif - -#if ZLIB_VERNUM < 0x1221 -#define _RZ_READONLY -struct _gz_header_s; -typedef struct _gz_header_s _gz_header; -#define gz_header _gz_header -#endif - -#define WINDOW_BITS 15 - -#ifndef RZ_BLOCK_SIZE -#define RZ_BLOCK_SIZE (1<mode from HEAD to TYPE after call inflateReset */ - int buf_off, buf_len; - int z_err, z_eof; - int seekable; - /* Indice where the source is seekable */ - int load_index; - /* set has_index to 0 in mode 'w', then index will be discarded */ -} RAZF; - -#ifdef __cplusplus -extern "C" { -#endif - - RAZF* razf_dopen(int data_fd, const char *mode); - RAZF *razf_open(const char *fn, const char *mode); - int razf_write(RAZF* rz, const void *data, int size); - int razf_read(RAZF* rz, void *data, int size); - int64_t razf_seek(RAZF* rz, int64_t pos, int where); - void razf_close(RAZF* rz); - -#define razf_tell(rz) ((rz)->out) - - RAZF* razf_open2(const char *filename, const char *mode); - RAZF* razf_dopen2(int fd, const char *mode); - uint64_t razf_tell2(RAZF *rz); - int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.13/sam.c b/samtools-0.1.13/sam.c deleted file mode 100644 --- a/samtools-0.1.13/sam.c +++ /dev/null @@ -1,175 +0,0 @@ -#include -#include -#include "faidx.h" -#include "sam.h" - -#define TYPE_BAM 1 -#define TYPE_READ 2 - -bam_header_t *bam_header_dup(const bam_header_t *h0) -{ - bam_header_t *h; - int i; - h = bam_header_init(); - *h = *h0; - h->hash = h->dict = h->rg2lib = 0; - h->text = (char*)calloc(h->l_text + 1, 1); - memcpy(h->text, h0->text, h->l_text); - h->target_len = (uint32_t*)calloc(h->n_targets, 4); - h->target_name = (char**)calloc(h->n_targets, sizeof(void*)); - for (i = 0; i < h->n_targets; ++i) { - h->target_len[i] = h0->target_len[i]; - h->target_name[i] = strdup(h0->target_name[i]); - } - return h; -} -static void append_header_text(bam_header_t *header, char* text, int len) -{ - int x = header->l_text + 1; - int y = header->l_text + len + 1; // 1 byte null - if (text == 0) return; - kroundup32(x); - kroundup32(y); - if (x < y) header->text = (char*)realloc(header->text, y); - strncpy(header->text + header->l_text, text, len); // we cannot use strcpy() here. - header->l_text += len; - header->text[header->l_text] = 0; -} - -samfile_t *samopen(const char *fn, const char *mode, const void *aux) -{ - samfile_t *fp; - fp = (samfile_t*)calloc(1, sizeof(samfile_t)); - if (mode[0] == 'r') { // read - fp->type |= TYPE_READ; - if (mode[1] == 'b') { // binary - fp->type |= TYPE_BAM; - fp->x.bam = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r"); - if (fp->x.bam == 0) goto open_err_ret; - fp->header = bam_header_read(fp->x.bam); - } else { // text - fp->x.tamr = sam_open(fn); - if (fp->x.tamr == 0) goto open_err_ret; - fp->header = sam_header_read(fp->x.tamr); - if (fp->header->n_targets == 0) { // no @SQ fields - if (aux) { // check if aux is present - bam_header_t *textheader = fp->header; - fp->header = sam_header_read2((const char*)aux); - if (fp->header == 0) goto open_err_ret; - append_header_text(fp->header, textheader->text, textheader->l_text); - bam_header_destroy(textheader); - } - if (fp->header->n_targets == 0) - fprintf(stderr, "[samopen] no @SQ lines in the header.\n"); - } else fprintf(stderr, "[samopen] SAM header is present: %d sequences.\n", fp->header->n_targets); - } - } else if (mode[0] == 'w') { // write - fp->header = bam_header_dup((const bam_header_t*)aux); - if (mode[1] == 'b') { // binary - char bmode[3]; - bmode[0] = 'w'; bmode[1] = strstr(mode, "u")? 'u' : 0; bmode[2] = 0; - fp->type |= TYPE_BAM; - fp->x.bam = strcmp(fn, "-")? bam_open(fn, bmode) : bam_dopen(fileno(stdout), bmode); - if (fp->x.bam == 0) goto open_err_ret; - bam_header_write(fp->x.bam, fp->header); - } else { // text - // open file - fp->x.tamw = strcmp(fn, "-")? fopen(fn, "w") : stdout; - if (fp->x.tamr == 0) goto open_err_ret; - if (strstr(mode, "X")) fp->type |= BAM_OFSTR<<2; - else if (strstr(mode, "x")) fp->type |= BAM_OFHEX<<2; - else fp->type |= BAM_OFDEC<<2; - // write header - if (strstr(mode, "h")) { - int i; - bam_header_t *alt; - // parse the header text - alt = bam_header_init(); - alt->l_text = fp->header->l_text; alt->text = fp->header->text; - sam_header_parse(alt); - alt->l_text = 0; alt->text = 0; - // check if there are @SQ lines in the header - fwrite(fp->header->text, 1, fp->header->l_text, fp->x.tamw); - if (alt->n_targets) { // then write the header text without dumping ->target_{name,len} - if (alt->n_targets != fp->header->n_targets) - fprintf(stderr, "[samopen] inconsistent number of target sequences.\n"); - } else { // then dump ->target_{name,len} - for (i = 0; i < fp->header->n_targets; ++i) - fprintf(fp->x.tamw, "@SQ\tSN:%s\tLN:%d\n", fp->header->target_name[i], fp->header->target_len[i]); - } - bam_header_destroy(alt); - } - } - } - return fp; - -open_err_ret: - free(fp); - return 0; -} - -void samclose(samfile_t *fp) -{ - if (fp == 0) return; - if (fp->header) bam_header_destroy(fp->header); - if (fp->type & TYPE_BAM) bam_close(fp->x.bam); - else if (fp->type & TYPE_READ) sam_close(fp->x.tamr); - else fclose(fp->x.tamw); - free(fp); -} - -int samread(samfile_t *fp, bam1_t *b) -{ - if (fp == 0 || !(fp->type & TYPE_READ)) return -1; // not open for reading - if (fp->type & TYPE_BAM) return bam_read1(fp->x.bam, b); - else return sam_read1(fp->x.tamr, fp->header, b); -} - -int samwrite(samfile_t *fp, const bam1_t *b) -{ - if (fp == 0 || (fp->type & TYPE_READ)) return -1; // not open for writing - if (fp->type & TYPE_BAM) return bam_write1(fp->x.bam, b); - else { - char *s = bam_format1_core(fp->header, b, fp->type>>2&3); - int l = strlen(s); - fputs(s, fp->x.tamw); fputc('\n', fp->x.tamw); - free(s); - return l + 1; - } -} - -int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *func_data) -{ - bam_plbuf_t *buf; - int ret; - bam1_t *b; - b = bam_init1(); - buf = bam_plbuf_init(func, func_data); - bam_plbuf_set_mask(buf, mask); - while ((ret = samread(fp, b)) >= 0) - bam_plbuf_push(b, buf); - bam_plbuf_push(0, buf); - bam_plbuf_destroy(buf); - bam_destroy1(b); - return 0; -} - -char *samfaipath(const char *fn_ref) -{ - char *fn_list = 0; - if (fn_ref == 0) return 0; - fn_list = calloc(strlen(fn_ref) + 5, 1); - strcat(strcpy(fn_list, fn_ref), ".fai"); - if (access(fn_list, R_OK) == -1) { // fn_list is unreadable - if (access(fn_ref, R_OK) == -1) { - fprintf(stderr, "[samfaipath] fail to read file %s.\n", fn_ref); - } else { - fprintf(stderr, "[samfaipath] build FASTA index...\n"); - if (fai_build(fn_ref) == -1) { - fprintf(stderr, "[samfaipath] fail to build FASTA index.\n"); - free(fn_list); fn_list = 0; - } - } - } - return fn_list; -} diff --git a/samtools-0.1.13/sam.h b/samtools-0.1.13/sam.h deleted file mode 100644 --- a/samtools-0.1.13/sam.h +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef BAM_SAM_H -#define BAM_SAM_H - -#include "bam.h" - -/*! - @header - - This file provides higher level of I/O routines and unifies the APIs - for SAM and BAM formats. These APIs are more convenient and - recommended. - - @copyright Genome Research Ltd. - */ - -/*! @typedef - @abstract SAM/BAM file handler - @field type type of the handler; bit 1 for BAM, 2 for reading and bit 3-4 for flag format - @field bam BAM file handler; valid if (type&1) == 1 - @field tamr SAM file handler for reading; valid if type == 2 - @field tamw SAM file handler for writing; valid if type == 0 - @field header header struct - */ -typedef struct { - int type; - union { - tamFile tamr; - bamFile bam; - FILE *tamw; - } x; - bam_header_t *header; -} samfile_t; - -#ifdef __cplusplus -extern "C" { -#endif - - /*! - @abstract Open a SAM/BAM file - - @param fn SAM/BAM file name; "-" is recognized as stdin (for - reading) or stdout (for writing). - - @param mode open mode /[rw](b?)(u?)(h?)([xX]?)/: 'r' for reading, - 'w' for writing, 'b' for BAM I/O, 'u' for uncompressed BAM output, - 'h' for outputing header in SAM, 'x' for HEX flag and 'X' for - string flag. If 'b' present, it must immediately follow 'r' or - 'w'. Valid modes are "r", "w", "wh", "wx", "whx", "wX", "whX", - "rb", "wb" and "wbu" exclusively. - - @param aux auxiliary data; if mode[0]=='w', aux points to - bam_header_t; if strcmp(mode, "rb")!=0 and @SQ header lines in SAM - are absent, aux points the file name of the list of the reference; - aux is not used otherwise. If @SQ header lines are present in SAM, - aux is not used, either. - - @return SAM/BAM file handler - */ - samfile_t *samopen(const char *fn, const char *mode, const void *aux); - - /*! - @abstract Close a SAM/BAM handler - @param fp file handler to be closed - */ - void samclose(samfile_t *fp); - - /*! - @abstract Read one alignment - @param fp file handler - @param b alignment - @return bytes read - */ - int samread(samfile_t *fp, bam1_t *b); - - /*! - @abstract Write one alignment - @param fp file handler - @param b alignment - @return bytes written - */ - int samwrite(samfile_t *fp, const bam1_t *b); - - /*! - @abstract Get the pileup for a whole alignment file - @param fp file handler - @param mask mask transferred to bam_plbuf_set_mask() - @param func user defined function called in the pileup process - #param data user provided data for func() - */ - int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *data); - - char *samfaipath(const char *fn_ref); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.13/sam_header.c b/samtools-0.1.13/sam_header.c deleted file mode 100644 --- a/samtools-0.1.13/sam_header.c +++ /dev/null @@ -1,735 +0,0 @@ -#include "sam_header.h" -#include -#include -#include -#include -#include - -#include "khash.h" -KHASH_MAP_INIT_STR(str, const char *) - -struct _HeaderList -{ - struct _HeaderList *last; // Hack: Used and maintained only by list_append_to_end. Maintained in the root node only. - struct _HeaderList *next; - void *data; -}; -typedef struct _HeaderList list_t; -typedef list_t HeaderDict; - -typedef struct -{ - char key[2]; - char *value; -} -HeaderTag; - -typedef struct -{ - char type[2]; - list_t *tags; -} -HeaderLine; - -const char *o_hd_tags[] = {"SO","GO",NULL}; -const char *r_hd_tags[] = {"VN",NULL}; - -const char *o_sq_tags[] = {"AS","M5","UR","SP",NULL}; -const char *r_sq_tags[] = {"SN","LN",NULL}; -const char *u_sq_tags[] = {"SN",NULL}; - -const char *o_rg_tags[] = {"LB","DS","PU","PI","CN","DT","PL",NULL}; -const char *r_rg_tags[] = {"ID",NULL}; -const char *u_rg_tags[] = {"ID",NULL}; - -const char *o_pg_tags[] = {"VN","CL",NULL}; -const char *r_pg_tags[] = {"ID",NULL}; - -const char *types[] = {"HD","SQ","RG","PG","CO",NULL}; -const char **optional_tags[] = {o_hd_tags,o_sq_tags,o_rg_tags,o_pg_tags,NULL,NULL}; -const char **required_tags[] = {r_hd_tags,r_sq_tags,r_rg_tags,r_pg_tags,NULL,NULL}; -const char **unique_tags[] = {NULL, u_sq_tags,u_rg_tags,NULL,NULL,NULL}; - - -static void debug(const char *format, ...) -{ - va_list ap; - va_start(ap, format); - vfprintf(stderr, format, ap); - va_end(ap); -} - -#if 0 -// Replaced by list_append_to_end -static list_t *list_prepend(list_t *root, void *data) -{ - list_t *l = malloc(sizeof(list_t)); - l->next = root; - l->data = data; - return l; -} -#endif - -// Relies on the root->last being correct. Do not use with the other list_* -// routines unless they are fixed to modify root->last as well. -static list_t *list_append_to_end(list_t *root, void *data) -{ - list_t *l = malloc(sizeof(list_t)); - l->last = l; - l->next = NULL; - l->data = data; - - if ( !root ) - return l; - - root->last->next = l; - root->last = l; - return root; -} - -static list_t *list_append(list_t *root, void *data) -{ - list_t *l = root; - while (l && l->next) - l = l->next; - if ( l ) - { - l->next = malloc(sizeof(list_t)); - l = l->next; - } - else - { - l = malloc(sizeof(list_t)); - root = l; - } - l->data = data; - l->next = NULL; - return root; -} - -static void list_free(list_t *root) -{ - list_t *l = root; - while (root) - { - l = root; - root = root->next; - free(l); - } -} - - - -// Look for a tag "XY" in a predefined const char *[] array. -static int tag_exists(const char *tag, const char **tags) -{ - int itag=0; - if ( !tags ) return -1; - while ( tags[itag] ) - { - if ( tags[itag][0]==tag[0] && tags[itag][1]==tag[1] ) return itag; - itag++; - } - return -1; -} - - - -// Mimics the behaviour of getline, except it returns pointer to the next chunk of the text -// or NULL if everything has been read. The lineptr should be freed by the caller. The -// newline character is stripped. -static const char *nextline(char **lineptr, size_t *n, const char *text) -{ - int len; - const char *to = text; - - if ( !*to ) return NULL; - - while ( *to && *to!='\n' && *to!='\r' ) to++; - len = to - text + 1; - - if ( *to ) - { - // Advance the pointer for the next call - if ( *to=='\n' ) to++; - else if ( *to=='\r' && *(to+1)=='\n' ) to+=2; - } - if ( !len ) - return to; - - if ( !*lineptr ) - { - *lineptr = malloc(len); - *n = len; - } - else if ( *nkey[0] = name[0]; - tag->key[1] = name[1]; - tag->value = malloc(len+1); - memcpy(tag->value,value_from,len+1); - tag->value[len] = 0; - return tag; -} - -static HeaderTag *header_line_has_tag(HeaderLine *hline, const char *key) -{ - list_t *tags = hline->tags; - while (tags) - { - HeaderTag *tag = tags->data; - if ( tag->key[0]==key[0] && tag->key[1]==key[1] ) return tag; - tags = tags->next; - } - return NULL; -} - - -// Return codes: -// 0 .. different types or unique tags differ or conflicting tags, cannot be merged -// 1 .. all tags identical -> no need to merge, drop one -// 2 .. the unique tags match and there are some conflicting tags (same tag, different value) -> error, cannot be merged nor duplicated -// 3 .. there are some missing complementary tags and no unique conflict -> can be merged into a single line -static int sam_header_compare_lines(HeaderLine *hline1, HeaderLine *hline2) -{ - HeaderTag *t1, *t2; - - if ( hline1->type[0]!=hline2->type[0] || hline1->type[1]!=hline2->type[1] ) - return 0; - - int itype = tag_exists(hline1->type,types); - if ( itype==-1 ) { - debug("[sam_header_compare_lines] Unknown type [%c%c]\n", hline1->type[0],hline1->type[1]); - return -1; // FIXME (lh3): error; I do not know how this will be handled in Petr's code - } - - if ( unique_tags[itype] ) - { - t1 = header_line_has_tag(hline1,unique_tags[itype][0]); - t2 = header_line_has_tag(hline2,unique_tags[itype][0]); - if ( !t1 || !t2 ) // this should never happen, the unique tags are required - return 2; - - if ( strcmp(t1->value,t2->value) ) - return 0; // the unique tags differ, cannot be merged - } - if ( !required_tags[itype] && !optional_tags[itype] ) - { - t1 = hline1->tags->data; - t2 = hline2->tags->data; - if ( !strcmp(t1->value,t2->value) ) return 1; // identical comments - return 0; - } - - int missing=0, itag=0; - while ( required_tags[itype] && required_tags[itype][itag] ) - { - t1 = header_line_has_tag(hline1,required_tags[itype][itag]); - t2 = header_line_has_tag(hline2,required_tags[itype][itag]); - if ( !t1 && !t2 ) - return 2; // this should never happen - else if ( !t1 || !t2 ) - missing = 1; // there is some tag missing in one of the hlines - else if ( strcmp(t1->value,t2->value) ) - { - if ( unique_tags[itype] ) - return 2; // the lines have a matching unique tag but have a conflicting tag - - return 0; // the lines contain conflicting tags, cannot be merged - } - itag++; - } - itag = 0; - while ( optional_tags[itype] && optional_tags[itype][itag] ) - { - t1 = header_line_has_tag(hline1,optional_tags[itype][itag]); - t2 = header_line_has_tag(hline2,optional_tags[itype][itag]); - if ( !t1 && !t2 ) - { - itag++; - continue; - } - if ( !t1 || !t2 ) - missing = 1; // there is some tag missing in one of the hlines - else if ( strcmp(t1->value,t2->value) ) - { - if ( unique_tags[itype] ) - return 2; // the lines have a matching unique tag but have a conflicting tag - - return 0; // the lines contain conflicting tags, cannot be merged - } - itag++; - } - if ( missing ) return 3; // there are some missing complementary tags with no conflicts, can be merged - return 1; -} - - -static HeaderLine *sam_header_line_clone(const HeaderLine *hline) -{ - list_t *tags; - HeaderLine *out = malloc(sizeof(HeaderLine)); - out->type[0] = hline->type[0]; - out->type[1] = hline->type[1]; - out->tags = NULL; - - tags = hline->tags; - while (tags) - { - HeaderTag *old = tags->data; - - HeaderTag *new = malloc(sizeof(HeaderTag)); - new->key[0] = old->key[0]; - new->key[1] = old->key[1]; - new->value = strdup(old->value); - out->tags = list_append(out->tags, new); - - tags = tags->next; - } - return out; -} - -static int sam_header_line_merge_with(HeaderLine *out_hline, const HeaderLine *tmpl_hline) -{ - list_t *tmpl_tags; - - if ( out_hline->type[0]!=tmpl_hline->type[0] || out_hline->type[1]!=tmpl_hline->type[1] ) - return 0; - - tmpl_tags = tmpl_hline->tags; - while (tmpl_tags) - { - HeaderTag *tmpl_tag = tmpl_tags->data; - HeaderTag *out_tag = header_line_has_tag(out_hline, tmpl_tag->key); - if ( !out_tag ) - { - HeaderTag *tag = malloc(sizeof(HeaderTag)); - tag->key[0] = tmpl_tag->key[0]; - tag->key[1] = tmpl_tag->key[1]; - tag->value = strdup(tmpl_tag->value); - out_hline->tags = list_append(out_hline->tags,tag); - } - tmpl_tags = tmpl_tags->next; - } - return 1; -} - - -static HeaderLine *sam_header_line_parse(const char *headerLine) -{ - HeaderLine *hline; - HeaderTag *tag; - const char *from, *to; - from = headerLine; - - if ( *from != '@' ) { - debug("[sam_header_line_parse] expected '@', got [%s]\n", headerLine); - return 0; - } - to = ++from; - - while (*to && *to!='\t') to++; - if ( to-from != 2 ) { - debug("[sam_header_line_parse] expected '@XY', got [%s]\nHint: The header tags must be tab-separated.\n", headerLine); - return 0; - } - - hline = malloc(sizeof(HeaderLine)); - hline->type[0] = from[0]; - hline->type[1] = from[1]; - hline->tags = NULL; - - int itype = tag_exists(hline->type, types); - - from = to; - while (*to && *to=='\t') to++; - if ( to-from != 1 ) { - debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from)); - return 0; - } - from = to; - while (*from) - { - while (*to && *to!='\t') to++; - - if ( !required_tags[itype] && !optional_tags[itype] ) - { - // CO is a special case, it can contain anything, including tabs - if ( *to ) { to++; continue; } - tag = new_tag(" ",from,to-1); - } - else - tag = new_tag(from,from+3,to-1); - - if ( header_line_has_tag(hline,tag->key) ) - debug("The tag '%c%c' present (at least) twice on line [%s]\n", tag->key[0],tag->key[1], headerLine); - hline->tags = list_append(hline->tags, tag); - - from = to; - while (*to && *to=='\t') to++; - if ( *to && to-from != 1 ) { - debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from)); - return 0; - } - - from = to; - } - return hline; -} - - -// Must be of an existing type, all tags must be recognised and all required tags must be present -static int sam_header_line_validate(HeaderLine *hline) -{ - list_t *tags; - HeaderTag *tag; - int itype, itag; - - // Is the type correct? - itype = tag_exists(hline->type, types); - if ( itype==-1 ) - { - debug("The type [%c%c] not recognised.\n", hline->type[0],hline->type[1]); - return 0; - } - - // Has all required tags? - itag = 0; - while ( required_tags[itype] && required_tags[itype][itag] ) - { - if ( !header_line_has_tag(hline,required_tags[itype][itag]) ) - { - debug("The tag [%c%c] required for [%c%c] not present.\n", required_tags[itype][itag][0],required_tags[itype][itag][1], - hline->type[0],hline->type[1]); - return 0; - } - itag++; - } - - // Are all tags recognised? - tags = hline->tags; - while ( tags ) - { - tag = tags->data; - if ( !tag_exists(tag->key,required_tags[itype]) && !tag_exists(tag->key,optional_tags[itype]) ) - { - debug("Unknown tag [%c%c] for [%c%c].\n", tag->key[0],tag->key[1], hline->type[0],hline->type[1]); - return 0; - } - tags = tags->next; - } - - return 1; -} - - -static void print_header_line(FILE *fp, HeaderLine *hline) -{ - list_t *tags = hline->tags; - HeaderTag *tag; - - fprintf(fp, "@%c%c", hline->type[0],hline->type[1]); - while (tags) - { - tag = tags->data; - - fprintf(fp, "\t"); - if ( tag->key[0]!=' ' || tag->key[1]!=' ' ) - fprintf(fp, "%c%c:", tag->key[0],tag->key[1]); - fprintf(fp, "%s", tag->value); - - tags = tags->next; - } - fprintf(fp,"\n"); -} - - -static void sam_header_line_free(HeaderLine *hline) -{ - list_t *tags = hline->tags; - while (tags) - { - HeaderTag *tag = tags->data; - free(tag->value); - free(tag); - tags = tags->next; - } - list_free(hline->tags); - free(hline); -} - -void sam_header_free(void *_header) -{ - HeaderDict *header = (HeaderDict*)_header; - list_t *hlines = header; - while (hlines) - { - sam_header_line_free(hlines->data); - hlines = hlines->next; - } - list_free(header); -} - -HeaderDict *sam_header_clone(const HeaderDict *dict) -{ - HeaderDict *out = NULL; - while (dict) - { - HeaderLine *hline = dict->data; - out = list_append(out, sam_header_line_clone(hline)); - dict = dict->next; - } - return out; -} - -// Returns a newly allocated string -char *sam_header_write(const void *_header) -{ - const HeaderDict *header = (const HeaderDict*)_header; - char *out = NULL; - int len=0, nout=0; - const list_t *hlines; - - // Calculate the length of the string to allocate - hlines = header; - while (hlines) - { - len += 4; // @XY and \n - - HeaderLine *hline = hlines->data; - list_t *tags = hline->tags; - while (tags) - { - HeaderTag *tag = tags->data; - len += strlen(tag->value) + 1; // \t - if ( tag->key[0]!=' ' || tag->key[1]!=' ' ) - len += strlen(tag->value) + 3; // XY: - tags = tags->next; - } - hlines = hlines->next; - } - - nout = 0; - out = malloc(len+1); - hlines = header; - while (hlines) - { - HeaderLine *hline = hlines->data; - - nout += sprintf(out+nout,"@%c%c",hline->type[0],hline->type[1]); - - list_t *tags = hline->tags; - while (tags) - { - HeaderTag *tag = tags->data; - nout += sprintf(out+nout,"\t"); - if ( tag->key[0]!=' ' || tag->key[1]!=' ' ) - nout += sprintf(out+nout,"%c%c:", tag->key[0],tag->key[1]); - nout += sprintf(out+nout,"%s", tag->value); - tags = tags->next; - } - hlines = hlines->next; - nout += sprintf(out+nout,"\n"); - } - out[len] = 0; - return out; -} - -void *sam_header_parse2(const char *headerText) -{ - list_t *hlines = NULL; - HeaderLine *hline; - const char *text; - char *buf=NULL; - size_t nbuf = 0; - - if ( !headerText ) - return 0; - - text = headerText; - while ( (text=nextline(&buf, &nbuf, text)) ) - { - hline = sam_header_line_parse(buf); - if ( hline && sam_header_line_validate(hline) ) - // With too many (~250,000) reference sequences the header parsing was too slow with list_append. - hlines = list_append_to_end(hlines, hline); - else - { - if (hline) sam_header_line_free(hline); - sam_header_free(hlines); - if ( buf ) free(buf); - return NULL; - } - } - if ( buf ) free(buf); - - return hlines; -} - -void *sam_header2tbl(const void *_dict, char type[2], char key_tag[2], char value_tag[2]) -{ - const HeaderDict *dict = (const HeaderDict*)_dict; - const list_t *l = dict; - khash_t(str) *tbl = kh_init(str); - khiter_t k; - int ret; - - if (_dict == 0) return tbl; // return an empty (not null) hash table - while (l) - { - HeaderLine *hline = l->data; - if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] ) - { - l = l->next; - continue; - } - - HeaderTag *key, *value; - key = header_line_has_tag(hline,key_tag); - value = header_line_has_tag(hline,value_tag); - if ( !key || !value ) - { - l = l->next; - continue; - } - - k = kh_get(str, tbl, key->value); - if ( k != kh_end(tbl) ) - debug("[sam_header_lookup_table] They key %s not unique.\n", key->value); - k = kh_put(str, tbl, key->value, &ret); - kh_value(tbl, k) = value->value; - - l = l->next; - } - return tbl; -} - -char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n) -{ - const HeaderDict *dict = (const HeaderDict*)_dict; - const list_t *l = dict; - int max, n; - char **ret; - - ret = 0; *_n = max = n = 0; - while (l) - { - HeaderLine *hline = l->data; - if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] ) - { - l = l->next; - continue; - } - - HeaderTag *key; - key = header_line_has_tag(hline,key_tag); - if ( !key ) - { - l = l->next; - continue; - } - - if (n == max) { - max = max? max<<1 : 4; - ret = realloc(ret, max * sizeof(void*)); - } - ret[n++] = key->value; - - l = l->next; - } - *_n = n; - return ret; -} - -const char *sam_tbl_get(void *h, const char *key) -{ - khash_t(str) *tbl = (khash_t(str)*)h; - khint_t k; - k = kh_get(str, tbl, key); - return k == kh_end(tbl)? 0 : kh_val(tbl, k); -} - -int sam_tbl_size(void *h) -{ - khash_t(str) *tbl = (khash_t(str)*)h; - return h? kh_size(tbl) : 0; -} - -void sam_tbl_destroy(void *h) -{ - khash_t(str) *tbl = (khash_t(str)*)h; - kh_destroy(str, tbl); -} - -void *sam_header_merge(int n, const void **_dicts) -{ - const HeaderDict **dicts = (const HeaderDict**)_dicts; - HeaderDict *out_dict; - int idict, status; - - if ( n<2 ) return NULL; - - out_dict = sam_header_clone(dicts[0]); - - for (idict=1; idictdata, out_hlines->data); - if ( status==0 ) - { - out_hlines = out_hlines->next; - continue; - } - - if ( status==2 ) - { - print_header_line(stderr,tmpl_hlines->data); - print_header_line(stderr,out_hlines->data); - debug("Conflicting lines, cannot merge the headers.\n"); - return 0; - } - if ( status==3 ) - sam_header_line_merge_with(out_hlines->data, tmpl_hlines->data); - - inserted = 1; - break; - } - if ( !inserted ) - out_dict = list_append(out_dict, sam_header_line_clone(tmpl_hlines->data)); - - tmpl_hlines = tmpl_hlines->next; - } - } - - return out_dict; -} - - diff --git a/samtools-0.1.13/sam_header.h b/samtools-0.1.13/sam_header.h deleted file mode 100644 --- a/samtools-0.1.13/sam_header.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __SAM_HEADER_H__ -#define __SAM_HEADER_H__ - -#ifdef __cplusplus -extern "C" { -#endif - - void *sam_header_parse2(const char *headerText); - void *sam_header_merge(int n, const void **dicts); - void sam_header_free(void *header); - char *sam_header_write(const void *headerDict); // returns a newly allocated string - - char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n); - - void *sam_header2tbl(const void *dict, char type[2], char key_tag[2], char value_tag[2]); - const char *sam_tbl_get(void *h, const char *key); - int sam_tbl_size(void *h); - void sam_tbl_destroy(void *h); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/samtools-0.1.18/bam.c b/samtools-0.1.18/bam.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam.c @@ -0,0 +1,362 @@ +#include +#include +#include +#include +#include "bam.h" +#include "bam_endian.h" +#include "kstring.h" +#include "sam_header.h" + +int bam_is_be = 0, bam_verbose = 2; +char *bam_flag2char_table = "pPuUrR12sfd\0\0\0\0\0"; + +/************************** + * CIGAR related routines * + **************************/ + +uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar) +{ + uint32_t k, end; + end = c->pos; + for (k = 0; k < c->n_cigar; ++k) { + int op = cigar[k] & BAM_CIGAR_MASK; + if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP) + end += cigar[k] >> BAM_CIGAR_SHIFT; + } + return end; +} + +int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar) +{ + uint32_t k; + int32_t l = 0; + for (k = 0; k < c->n_cigar; ++k) { + int op = cigar[k] & BAM_CIGAR_MASK; + if (op == BAM_CMATCH || op == BAM_CINS || op == BAM_CSOFT_CLIP || op == BAM_CEQUAL || op == BAM_CDIFF) + l += cigar[k] >> BAM_CIGAR_SHIFT; + } + return l; +} + +/******************** + * BAM I/O routines * + ********************/ + +bam_header_t *bam_header_init() +{ + bam_is_be = bam_is_big_endian(); + return (bam_header_t*)calloc(1, sizeof(bam_header_t)); +} + +void bam_header_destroy(bam_header_t *header) +{ + int32_t i; + extern void bam_destroy_header_hash(bam_header_t *header); + if (header == 0) return; + if (header->target_name) { + for (i = 0; i < header->n_targets; ++i) + free(header->target_name[i]); + free(header->target_name); + free(header->target_len); + } + free(header->text); + if (header->dict) sam_header_free(header->dict); + if (header->rg2lib) sam_tbl_destroy(header->rg2lib); + bam_destroy_header_hash(header); + free(header); +} + +bam_header_t *bam_header_read(bamFile fp) +{ + bam_header_t *header; + char buf[4]; + int magic_len; + int32_t i = 1, name_len; + // check EOF + i = bgzf_check_EOF(fp); + if (i < 0) { + // If the file is a pipe, checking the EOF marker will *always* fail + // with ESPIPE. Suppress the error message in this case. + if (errno != ESPIPE) perror("[bam_header_read] bgzf_check_EOF"); + } + else if (i == 0) fprintf(stderr, "[bam_header_read] EOF marker is absent. The input is probably truncated.\n"); + // read "BAM1" + magic_len = bam_read(fp, buf, 4); + if (magic_len != 4 || strncmp(buf, "BAM\001", 4) != 0) { + fprintf(stderr, "[bam_header_read] invalid BAM binary header (this is not a BAM file).\n"); + return 0; + } + header = bam_header_init(); + // read plain text and the number of reference sequences + bam_read(fp, &header->l_text, 4); + if (bam_is_be) bam_swap_endian_4p(&header->l_text); + header->text = (char*)calloc(header->l_text + 1, 1); + bam_read(fp, header->text, header->l_text); + bam_read(fp, &header->n_targets, 4); + if (bam_is_be) bam_swap_endian_4p(&header->n_targets); + // read reference sequence names and lengths + header->target_name = (char**)calloc(header->n_targets, sizeof(char*)); + header->target_len = (uint32_t*)calloc(header->n_targets, 4); + for (i = 0; i != header->n_targets; ++i) { + bam_read(fp, &name_len, 4); + if (bam_is_be) bam_swap_endian_4p(&name_len); + header->target_name[i] = (char*)calloc(name_len, 1); + bam_read(fp, header->target_name[i], name_len); + bam_read(fp, &header->target_len[i], 4); + if (bam_is_be) bam_swap_endian_4p(&header->target_len[i]); + } + return header; +} + +int bam_header_write(bamFile fp, const bam_header_t *header) +{ + char buf[4]; + int32_t i, name_len, x; + // write "BAM1" + strncpy(buf, "BAM\001", 4); + bam_write(fp, buf, 4); + // write plain text and the number of reference sequences + if (bam_is_be) { + x = bam_swap_endian_4(header->l_text); + bam_write(fp, &x, 4); + if (header->l_text) bam_write(fp, header->text, header->l_text); + x = bam_swap_endian_4(header->n_targets); + bam_write(fp, &x, 4); + } else { + bam_write(fp, &header->l_text, 4); + if (header->l_text) bam_write(fp, header->text, header->l_text); + bam_write(fp, &header->n_targets, 4); + } + // write sequence names and lengths + for (i = 0; i != header->n_targets; ++i) { + char *p = header->target_name[i]; + name_len = strlen(p) + 1; + if (bam_is_be) { + x = bam_swap_endian_4(name_len); + bam_write(fp, &x, 4); + } else bam_write(fp, &name_len, 4); + bam_write(fp, p, name_len); + if (bam_is_be) { + x = bam_swap_endian_4(header->target_len[i]); + bam_write(fp, &x, 4); + } else bam_write(fp, &header->target_len[i], 4); + } + bgzf_flush(fp); + return 0; +} + +static void swap_endian_data(const bam1_core_t *c, int data_len, uint8_t *data) +{ + uint8_t *s; + uint32_t i, *cigar = (uint32_t*)(data + c->l_qname); + s = data + c->n_cigar*4 + c->l_qname + c->l_qseq + (c->l_qseq + 1)/2; + for (i = 0; i < c->n_cigar; ++i) bam_swap_endian_4p(&cigar[i]); + while (s < data + data_len) { + uint8_t type; + s += 2; // skip key + type = toupper(*s); ++s; // skip type + if (type == 'C' || type == 'A') ++s; + else if (type == 'S') { bam_swap_endian_2p(s); s += 2; } + else if (type == 'I' || type == 'F') { bam_swap_endian_4p(s); s += 4; } + else if (type == 'D') { bam_swap_endian_8p(s); s += 8; } + else if (type == 'Z' || type == 'H') { while (*s) ++s; ++s; } + else if (type == 'B') { + int32_t n, Bsize = bam_aux_type2size(*s); + memcpy(&n, s + 1, 4); + if (1 == Bsize) { + } else if (2 == Bsize) { + for (i = 0; i < n; i += 2) + bam_swap_endian_2p(s + 5 + i); + } else if (4 == Bsize) { + for (i = 0; i < n; i += 4) + bam_swap_endian_4p(s + 5 + i); + } + bam_swap_endian_4p(s+1); + } + } +} + +int bam_read1(bamFile fp, bam1_t *b) +{ + bam1_core_t *c = &b->core; + int32_t block_len, ret, i; + uint32_t x[8]; + + assert(BAM_CORE_SIZE == 32); + if ((ret = bam_read(fp, &block_len, 4)) != 4) { + if (ret == 0) return -1; // normal end-of-file + else return -2; // truncated + } + if (bam_read(fp, x, BAM_CORE_SIZE) != BAM_CORE_SIZE) return -3; + if (bam_is_be) { + bam_swap_endian_4p(&block_len); + for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i); + } + c->tid = x[0]; c->pos = x[1]; + c->bin = x[2]>>16; c->qual = x[2]>>8&0xff; c->l_qname = x[2]&0xff; + c->flag = x[3]>>16; c->n_cigar = x[3]&0xffff; + c->l_qseq = x[4]; + c->mtid = x[5]; c->mpos = x[6]; c->isize = x[7]; + b->data_len = block_len - BAM_CORE_SIZE; + if (b->m_data < b->data_len) { + b->m_data = b->data_len; + kroundup32(b->m_data); + b->data = (uint8_t*)realloc(b->data, b->m_data); + } + if (bam_read(fp, b->data, b->data_len) != b->data_len) return -4; + b->l_aux = b->data_len - c->n_cigar * 4 - c->l_qname - c->l_qseq - (c->l_qseq+1)/2; + if (bam_is_be) swap_endian_data(c, b->data_len, b->data); + return 4 + block_len; +} + +inline int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data) +{ + uint32_t x[8], block_len = data_len + BAM_CORE_SIZE, y; + int i; + assert(BAM_CORE_SIZE == 32); + x[0] = c->tid; + x[1] = c->pos; + x[2] = (uint32_t)c->bin<<16 | c->qual<<8 | c->l_qname; + x[3] = (uint32_t)c->flag<<16 | c->n_cigar; + x[4] = c->l_qseq; + x[5] = c->mtid; + x[6] = c->mpos; + x[7] = c->isize; + bgzf_flush_try(fp, 4 + block_len); + if (bam_is_be) { + for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i); + y = block_len; + bam_write(fp, bam_swap_endian_4p(&y), 4); + swap_endian_data(c, data_len, data); + } else bam_write(fp, &block_len, 4); + bam_write(fp, x, BAM_CORE_SIZE); + bam_write(fp, data, data_len); + if (bam_is_be) swap_endian_data(c, data_len, data); + return 4 + block_len; +} + +int bam_write1(bamFile fp, const bam1_t *b) +{ + return bam_write1_core(fp, &b->core, b->data_len, b->data); +} + +char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of) +{ + uint8_t *s = bam1_seq(b), *t = bam1_qual(b); + int i; + const bam1_core_t *c = &b->core; + kstring_t str; + str.l = str.m = 0; str.s = 0; + + kputsn(bam1_qname(b), c->l_qname-1, &str); kputc('\t', &str); + if (of == BAM_OFDEC) { kputw(c->flag, &str); kputc('\t', &str); } + else if (of == BAM_OFHEX) ksprintf(&str, "0x%x\t", c->flag); + else { // BAM_OFSTR + for (i = 0; i < 16; ++i) + if ((c->flag & 1<tid < 0) kputsn("*\t", 2, &str); + else { + if (header) kputs(header->target_name[c->tid] , &str); + else kputw(c->tid, &str); + kputc('\t', &str); + } + kputw(c->pos + 1, &str); kputc('\t', &str); kputw(c->qual, &str); kputc('\t', &str); + if (c->n_cigar == 0) kputc('*', &str); + else { + for (i = 0; i < c->n_cigar; ++i) { + kputw(bam1_cigar(b)[i]>>BAM_CIGAR_SHIFT, &str); + kputc("MIDNSHP=X"[bam1_cigar(b)[i]&BAM_CIGAR_MASK], &str); + } + } + kputc('\t', &str); + if (c->mtid < 0) kputsn("*\t", 2, &str); + else if (c->mtid == c->tid) kputsn("=\t", 2, &str); + else { + if (header) kputs(header->target_name[c->mtid], &str); + else kputw(c->mtid, &str); + kputc('\t', &str); + } + kputw(c->mpos + 1, &str); kputc('\t', &str); kputw(c->isize, &str); kputc('\t', &str); + if (c->l_qseq) { + for (i = 0; i < c->l_qseq; ++i) kputc(bam_nt16_rev_table[bam1_seqi(s, i)], &str); + kputc('\t', &str); + if (t[0] == 0xff) kputc('*', &str); + else for (i = 0; i < c->l_qseq; ++i) kputc(t[i] + 33, &str); + } else kputsn("*\t*", 3, &str); + s = bam1_aux(b); + while (s < b->data + b->data_len) { + uint8_t type, key[2]; + key[0] = s[0]; key[1] = s[1]; + s += 2; type = *s; ++s; + kputc('\t', &str); kputsn((char*)key, 2, &str); kputc(':', &str); + if (type == 'A') { kputsn("A:", 2, &str); kputc(*s, &str); ++s; } + else if (type == 'C') { kputsn("i:", 2, &str); kputw(*s, &str); ++s; } + else if (type == 'c') { kputsn("i:", 2, &str); kputw(*(int8_t*)s, &str); ++s; } + else if (type == 'S') { kputsn("i:", 2, &str); kputw(*(uint16_t*)s, &str); s += 2; } + else if (type == 's') { kputsn("i:", 2, &str); kputw(*(int16_t*)s, &str); s += 2; } + else if (type == 'I') { kputsn("i:", 2, &str); kputuw(*(uint32_t*)s, &str); s += 4; } + else if (type == 'i') { kputsn("i:", 2, &str); kputw(*(int32_t*)s, &str); s += 4; } + else if (type == 'f') { ksprintf(&str, "f:%g", *(float*)s); s += 4; } + else if (type == 'd') { ksprintf(&str, "d:%lg", *(double*)s); s += 8; } + else if (type == 'Z' || type == 'H') { kputc(type, &str); kputc(':', &str); while (*s) kputc(*s++, &str); ++s; } + else if (type == 'B') { + uint8_t sub_type = *(s++); + int32_t n; + memcpy(&n, s, 4); + s += 4; // no point to the start of the array + kputc(type, &str); kputc(':', &str); kputc(sub_type, &str); // write the typing + for (i = 0; i < n; ++i) { + kputc(',', &str); + if ('c' == sub_type || 'c' == sub_type) { kputw(*(int8_t*)s, &str); ++s; } + else if ('C' == sub_type) { kputw(*(uint8_t*)s, &str); ++s; } + else if ('s' == sub_type) { kputw(*(int16_t*)s, &str); s += 2; } + else if ('S' == sub_type) { kputw(*(uint16_t*)s, &str); s += 2; } + else if ('i' == sub_type) { kputw(*(int32_t*)s, &str); s += 4; } + else if ('I' == sub_type) { kputuw(*(uint32_t*)s, &str); s += 4; } + else if ('f' == sub_type) { ksprintf(&str, "%g", *(float*)s); s += 4; } + } + } + } + return str.s; +} + +char *bam_format1(const bam_header_t *header, const bam1_t *b) +{ + return bam_format1_core(header, b, BAM_OFDEC); +} + +void bam_view1(const bam_header_t *header, const bam1_t *b) +{ + char *s = bam_format1(header, b); + puts(s); + free(s); +} + +int bam_validate1(const bam_header_t *header, const bam1_t *b) +{ + char *s; + + if (b->core.tid < -1 || b->core.mtid < -1) return 0; + if (header && (b->core.tid >= header->n_targets || b->core.mtid >= header->n_targets)) return 0; + + if (b->data_len < b->core.l_qname) return 0; + s = memchr(bam1_qname(b), '\0', b->core.l_qname); + if (s != &bam1_qname(b)[b->core.l_qname-1]) return 0; + + // FIXME: Other fields could also be checked, especially the auxiliary data + + return 1; +} + +// FIXME: we should also check the LB tag associated with each alignment +const char *bam_get_library(bam_header_t *h, const bam1_t *b) +{ + const uint8_t *rg; + if (h->dict == 0) h->dict = sam_header_parse2(h->text); + if (h->rg2lib == 0) h->rg2lib = sam_header2tbl(h->dict, "RG", "ID", "LB"); + rg = bam_aux_get(b, "RG"); + return (rg == 0)? 0 : sam_tbl_get(h->rg2lib, (const char*)(rg + 1)); +} diff --git a/samtools-0.1.18/bam.h b/samtools-0.1.18/bam.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam.h @@ -0,0 +1,763 @@ +/* The MIT License + + Copyright (c) 2008-2010 Genome Research Ltd (GRL). + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* Contact: Heng Li */ + +#ifndef BAM_BAM_H +#define BAM_BAM_H + +/*! + @header + + BAM library provides I/O and various operations on manipulating files + in the BAM (Binary Alignment/Mapping) or SAM (Sequence Alignment/Map) + format. It now supports importing from or exporting to SAM, sorting, + merging, generating pileup, and quickly retrieval of reads overlapped + with a specified region. + + @copyright Genome Research Ltd. + */ + +#define BAM_VERSION "0.1.18 (r982:295)" + +#include +#include +#include +#include + +#ifndef BAM_LITE +#define BAM_VIRTUAL_OFFSET16 +#include "bgzf.h" +/*! @abstract BAM file handler */ +typedef BGZF *bamFile; +#define bam_open(fn, mode) bgzf_open(fn, mode) +#define bam_dopen(fd, mode) bgzf_fdopen(fd, mode) +#define bam_close(fp) bgzf_close(fp) +#define bam_read(fp, buf, size) bgzf_read(fp, buf, size) +#define bam_write(fp, buf, size) bgzf_write(fp, buf, size) +#define bam_tell(fp) bgzf_tell(fp) +#define bam_seek(fp, pos, dir) bgzf_seek(fp, pos, dir) +#else +#define BAM_TRUE_OFFSET +#include +typedef gzFile bamFile; +#define bam_open(fn, mode) gzopen(fn, mode) +#define bam_dopen(fd, mode) gzdopen(fd, mode) +#define bam_close(fp) gzclose(fp) +#define bam_read(fp, buf, size) gzread(fp, buf, size) +/* no bam_write/bam_tell/bam_seek() here */ +#endif + +/*! @typedef + @abstract Structure for the alignment header. + @field n_targets number of reference sequences + @field target_name names of the reference sequences + @field target_len lengths of the referene sequences + @field dict header dictionary + @field hash hash table for fast name lookup + @field rg2lib hash table for @RG-ID -> LB lookup + @field l_text length of the plain text in the header + @field text plain text + + @discussion Field hash points to null by default. It is a private + member. + */ +typedef struct { + int32_t n_targets; + char **target_name; + uint32_t *target_len; + void *dict, *hash, *rg2lib; + size_t l_text, n_text; + char *text; +} bam_header_t; + +/*! @abstract the read is paired in sequencing, no matter whether it is mapped in a pair */ +#define BAM_FPAIRED 1 +/*! @abstract the read is mapped in a proper pair */ +#define BAM_FPROPER_PAIR 2 +/*! @abstract the read itself is unmapped; conflictive with BAM_FPROPER_PAIR */ +#define BAM_FUNMAP 4 +/*! @abstract the mate is unmapped */ +#define BAM_FMUNMAP 8 +/*! @abstract the read is mapped to the reverse strand */ +#define BAM_FREVERSE 16 +/*! @abstract the mate is mapped to the reverse strand */ +#define BAM_FMREVERSE 32 +/*! @abstract this is read1 */ +#define BAM_FREAD1 64 +/*! @abstract this is read2 */ +#define BAM_FREAD2 128 +/*! @abstract not primary alignment */ +#define BAM_FSECONDARY 256 +/*! @abstract QC failure */ +#define BAM_FQCFAIL 512 +/*! @abstract optical or PCR duplicate */ +#define BAM_FDUP 1024 + +#define BAM_OFDEC 0 +#define BAM_OFHEX 1 +#define BAM_OFSTR 2 + +/*! @abstract defautl mask for pileup */ +#define BAM_DEF_MASK (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP) + +#define BAM_CORE_SIZE sizeof(bam1_core_t) + +/** + * Describing how CIGAR operation/length is packed in a 32-bit integer. + */ +#define BAM_CIGAR_SHIFT 4 +#define BAM_CIGAR_MASK ((1 << BAM_CIGAR_SHIFT) - 1) + +/* + CIGAR operations. + */ +/*! @abstract CIGAR: M = match or mismatch*/ +#define BAM_CMATCH 0 +/*! @abstract CIGAR: I = insertion to the reference */ +#define BAM_CINS 1 +/*! @abstract CIGAR: D = deletion from the reference */ +#define BAM_CDEL 2 +/*! @abstract CIGAR: N = skip on the reference (e.g. spliced alignment) */ +#define BAM_CREF_SKIP 3 +/*! @abstract CIGAR: S = clip on the read with clipped sequence + present in qseq */ +#define BAM_CSOFT_CLIP 4 +/*! @abstract CIGAR: H = clip on the read with clipped sequence trimmed off */ +#define BAM_CHARD_CLIP 5 +/*! @abstract CIGAR: P = padding */ +#define BAM_CPAD 6 +/*! @abstract CIGAR: equals = match */ +#define BAM_CEQUAL 7 +/*! @abstract CIGAR: X = mismatch */ +#define BAM_CDIFF 8 + +/*! @typedef + @abstract Structure for core alignment information. + @field tid chromosome ID, defined by bam_header_t + @field pos 0-based leftmost coordinate + @field strand strand; 0 for forward and 1 otherwise + @field bin bin calculated by bam_reg2bin() + @field qual mapping quality + @field l_qname length of the query name + @field flag bitwise flag + @field n_cigar number of CIGAR operations + @field l_qseq length of the query sequence (read) + */ +typedef struct { + int32_t tid; + int32_t pos; + uint32_t bin:16, qual:8, l_qname:8; + uint32_t flag:16, n_cigar:16; + int32_t l_qseq; + int32_t mtid; + int32_t mpos; + int32_t isize; +} bam1_core_t; + +/*! @typedef + @abstract Structure for one alignment. + @field core core information about the alignment + @field l_aux length of auxiliary data + @field data_len current length of bam1_t::data + @field m_data maximum length of bam1_t::data + @field data all variable-length data, concatenated; structure: cigar-qname-seq-qual-aux + + @discussion Notes: + + 1. qname is zero tailing and core.l_qname includes the tailing '\0'. + 2. l_qseq is calculated from the total length of an alignment block + on reading or from CIGAR. + */ +typedef struct { + bam1_core_t core; + int l_aux, data_len, m_data; + uint8_t *data; +} bam1_t; + +typedef struct __bam_iter_t *bam_iter_t; + +#define bam1_strand(b) (((b)->core.flag&BAM_FREVERSE) != 0) +#define bam1_mstrand(b) (((b)->core.flag&BAM_FMREVERSE) != 0) + +/*! @function + @abstract Get the CIGAR array + @param b pointer to an alignment + @return pointer to the CIGAR array + + @discussion In the CIGAR array, each element is a 32-bit integer. The + lower 4 bits gives a CIGAR operation and the higher 28 bits keep the + length of a CIGAR. + */ +#define bam1_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname)) + +/*! @function + @abstract Get the name of the query + @param b pointer to an alignment + @return pointer to the name string, null terminated + */ +#define bam1_qname(b) ((char*)((b)->data)) + +/*! @function + @abstract Get query sequence + @param b pointer to an alignment + @return pointer to sequence + + @discussion Each base is encoded in 4 bits: 1 for A, 2 for C, 4 for G, + 8 for T and 15 for N. Two bases are packed in one byte with the base + at the higher 4 bits having smaller coordinate on the read. It is + recommended to use bam1_seqi() macro to get the base. + */ +#define bam1_seq(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname) + +/*! @function + @abstract Get query quality + @param b pointer to an alignment + @return pointer to quality string + */ +#define bam1_qual(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1)) + +/*! @function + @abstract Get a base on read + @param s Query sequence returned by bam1_seq() + @param i The i-th position, 0-based + @return 4-bit integer representing the base. + */ +#define bam1_seqi(s, i) ((s)[(i)/2] >> 4*(1-(i)%2) & 0xf) + +/*! @function + @abstract Get query sequence and quality + @param b pointer to an alignment + @return pointer to the concatenated auxiliary data + */ +#define bam1_aux(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (b)->core.l_qseq + ((b)->core.l_qseq + 1)/2) + +#ifndef kroundup32 +/*! @function + @abstract Round an integer to the next closest power-2 integer. + @param x integer to be rounded (in place) + @discussion x will be modified. + */ +#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) +#endif + +/*! + @abstract Whether the machine is big-endian; modified only in + bam_header_init(). + */ +extern int bam_is_be; + +/*! + @abstract Verbose level between 0 and 3; 0 is supposed to disable all + debugging information, though this may not have been implemented. + */ +extern int bam_verbose; + +/*! @abstract Table for converting a nucleotide character to the 4-bit encoding. */ +extern unsigned char bam_nt16_table[256]; + +/*! @abstract Table for converting a 4-bit encoded nucleotide to a letter. */ +extern char *bam_nt16_rev_table; + +extern char bam_nt16_nt4_table[]; + +#ifdef __cplusplus +extern "C" { +#endif + + /********************* + * Low-level SAM I/O * + *********************/ + + /*! @abstract TAM file handler */ + typedef struct __tamFile_t *tamFile; + + /*! + @abstract Open a SAM file for reading, either uncompressed or compressed by gzip/zlib. + @param fn SAM file name + @return SAM file handler + */ + tamFile sam_open(const char *fn); + + /*! + @abstract Close a SAM file handler + @param fp SAM file handler + */ + void sam_close(tamFile fp); + + /*! + @abstract Read one alignment from a SAM file handler + @param fp SAM file handler + @param header header information (ordered names of chromosomes) + @param b read alignment; all members in b will be updated + @return 0 if successful; otherwise negative + */ + int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b); + + /*! + @abstract Read header information from a TAB-delimited list file. + @param fn_list file name for the list + @return a pointer to the header structure + + @discussion Each line in this file consists of chromosome name and + the length of chromosome. + */ + bam_header_t *sam_header_read2(const char *fn_list); + + /*! + @abstract Read header from a SAM file (if present) + @param fp SAM file handler + @return pointer to header struct; 0 if no @SQ lines available + */ + bam_header_t *sam_header_read(tamFile fp); + + /*! + @abstract Parse @SQ lines a update a header struct + @param h pointer to the header struct to be updated + @return number of target sequences + + @discussion bam_header_t::{n_targets,target_len,target_name} will + be destroyed in the first place. + */ + int sam_header_parse(bam_header_t *h); + int32_t bam_get_tid(const bam_header_t *header, const char *seq_name); + + /*! + @abstract Parse @RG lines a update a header struct + @param h pointer to the header struct to be updated + @return number of @RG lines + + @discussion bam_header_t::rg2lib will be destroyed in the first + place. + */ + int sam_header_parse_rg(bam_header_t *h); + +#define sam_write1(header, b) bam_view1(header, b) + + + /******************************** + * APIs for string dictionaries * + ********************************/ + + int bam_strmap_put(void *strmap, const char *rg, const char *lib); + const char *bam_strmap_get(const void *strmap, const char *rg); + void *bam_strmap_dup(const void*); + void *bam_strmap_init(); + void bam_strmap_destroy(void *strmap); + + + /********************* + * Low-level BAM I/O * + *********************/ + + /*! + @abstract Initialize a header structure. + @return the pointer to the header structure + + @discussion This function also modifies the global variable + bam_is_be. + */ + bam_header_t *bam_header_init(); + + /*! + @abstract Destroy a header structure. + @param header pointer to the header + */ + void bam_header_destroy(bam_header_t *header); + + /*! + @abstract Read a header structure from BAM. + @param fp BAM file handler, opened by bam_open() + @return pointer to the header structure + + @discussion The file position indicator must be placed at the + beginning of the file. Upon success, the position indicator will + be set at the start of the first alignment. + */ + bam_header_t *bam_header_read(bamFile fp); + + /*! + @abstract Write a header structure to BAM. + @param fp BAM file handler + @param header pointer to the header structure + @return always 0 currently + */ + int bam_header_write(bamFile fp, const bam_header_t *header); + + /*! + @abstract Read an alignment from BAM. + @param fp BAM file handler + @param b read alignment; all members are updated. + @return number of bytes read from the file + + @discussion The file position indicator must be + placed right before an alignment. Upon success, this function + will set the position indicator to the start of the next + alignment. This function is not affected by the machine + endianness. + */ + int bam_read1(bamFile fp, bam1_t *b); + + /*! + @abstract Write an alignment to BAM. + @param fp BAM file handler + @param c pointer to the bam1_core_t structure + @param data_len total length of variable size data related to + the alignment + @param data pointer to the concatenated data + @return number of bytes written to the file + + @discussion This function is not affected by the machine + endianness. + */ + int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data); + + /*! + @abstract Write an alignment to BAM. + @param fp BAM file handler + @param b alignment to write + @return number of bytes written to the file + + @abstract It is equivalent to: + bam_write1_core(fp, &b->core, b->data_len, b->data) + */ + int bam_write1(bamFile fp, const bam1_t *b); + + /*! @function + @abstract Initiate a pointer to bam1_t struct + */ +#define bam_init1() ((bam1_t*)calloc(1, sizeof(bam1_t))) + + /*! @function + @abstract Free the memory allocated for an alignment. + @param b pointer to an alignment + */ +#define bam_destroy1(b) do { \ + if (b) { free((b)->data); free(b); } \ + } while (0) + + /*! + @abstract Format a BAM record in the SAM format + @param header pointer to the header structure + @param b alignment to print + @return a pointer to the SAM string + */ + char *bam_format1(const bam_header_t *header, const bam1_t *b); + + char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of); + + /*! + @abstract Check whether a BAM record is plausibly valid + @param header associated header structure, or NULL if unavailable + @param b alignment to validate + @return 0 if the alignment is invalid; non-zero otherwise + + @discussion Simple consistency check of some of the fields of the + alignment record. If the header is provided, several additional checks + are made. Not all fields are checked, so a non-zero result is not a + guarantee that the record is valid. However it is usually good enough + to detect when bam_seek() has been called with a virtual file offset + that is not the offset of an alignment record. + */ + int bam_validate1(const bam_header_t *header, const bam1_t *b); + + const char *bam_get_library(bam_header_t *header, const bam1_t *b); + + + /*************** + * pileup APIs * + ***************/ + + /*! @typedef + @abstract Structure for one alignment covering the pileup position. + @field b pointer to the alignment + @field qpos position of the read base at the pileup site, 0-based + @field indel indel length; 0 for no indel, positive for ins and negative for del + @field is_del 1 iff the base on the padded read is a deletion + @field level the level of the read in the "viewer" mode + + @discussion See also bam_plbuf_push() and bam_lplbuf_push(). The + difference between the two functions is that the former does not + set bam_pileup1_t::level, while the later does. Level helps the + implementation of alignment viewers, but calculating this has some + overhead. + */ + typedef struct { + bam1_t *b; + int32_t qpos; + int indel, level; + uint32_t is_del:1, is_head:1, is_tail:1, is_refskip:1, aux:28; + } bam_pileup1_t; + + typedef int (*bam_plp_auto_f)(void *data, bam1_t *b); + + struct __bam_plp_t; + typedef struct __bam_plp_t *bam_plp_t; + + bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data); + int bam_plp_push(bam_plp_t iter, const bam1_t *b); + const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp); + const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp); + void bam_plp_set_mask(bam_plp_t iter, int mask); + void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt); + void bam_plp_reset(bam_plp_t iter); + void bam_plp_destroy(bam_plp_t iter); + + struct __bam_mplp_t; + typedef struct __bam_mplp_t *bam_mplp_t; + + bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data); + void bam_mplp_destroy(bam_mplp_t iter); + void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt); + int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp); + + /*! @typedef + @abstract Type of function to be called by bam_plbuf_push(). + @param tid chromosome ID as is defined in the header + @param pos start coordinate of the alignment, 0-based + @param n number of elements in pl array + @param pl array of alignments + @param data user provided data + @discussion See also bam_plbuf_push(), bam_plbuf_init() and bam_pileup1_t. + */ + typedef int (*bam_pileup_f)(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data); + + typedef struct { + bam_plp_t iter; + bam_pileup_f func; + void *data; + } bam_plbuf_t; + + void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask); + void bam_plbuf_reset(bam_plbuf_t *buf); + bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data); + void bam_plbuf_destroy(bam_plbuf_t *buf); + int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf); + + int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data); + + struct __bam_lplbuf_t; + typedef struct __bam_lplbuf_t bam_lplbuf_t; + + void bam_lplbuf_reset(bam_lplbuf_t *buf); + + /*! @abstract bam_plbuf_init() equivalent with level calculated. */ + bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data); + + /*! @abstract bam_plbuf_destroy() equivalent with level calculated. */ + void bam_lplbuf_destroy(bam_lplbuf_t *tv); + + /*! @abstract bam_plbuf_push() equivalent with level calculated. */ + int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *buf); + + + /********************* + * BAM indexing APIs * + *********************/ + + struct __bam_index_t; + typedef struct __bam_index_t bam_index_t; + + /*! + @abstract Build index for a BAM file. + @discussion Index file "fn.bai" will be created. + @param fn name of the BAM file + @return always 0 currently + */ + int bam_index_build(const char *fn); + + /*! + @abstract Load index from file "fn.bai". + @param fn name of the BAM file (NOT the index file) + @return pointer to the index structure + */ + bam_index_t *bam_index_load(const char *fn); + + /*! + @abstract Destroy an index structure. + @param idx pointer to the index structure + */ + void bam_index_destroy(bam_index_t *idx); + + /*! @typedef + @abstract Type of function to be called by bam_fetch(). + @param b the alignment + @param data user provided data + */ + typedef int (*bam_fetch_f)(const bam1_t *b, void *data); + + /*! + @abstract Retrieve the alignments that are overlapped with the + specified region. + + @discussion A user defined function will be called for each + retrieved alignment ordered by its start position. + + @param fp BAM file handler + @param idx pointer to the alignment index + @param tid chromosome ID as is defined in the header + @param beg start coordinate, 0-based + @param end end coordinate, 0-based + @param data user provided data (will be transferred to func) + @param func user defined function + */ + int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func); + + bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end); + int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b); + void bam_iter_destroy(bam_iter_t iter); + + /*! + @abstract Parse a region in the format: "chr2:100,000-200,000". + @discussion bam_header_t::hash will be initialized if empty. + @param header pointer to the header structure + @param str string to be parsed + @param ref_id the returned chromosome ID + @param begin the returned start coordinate + @param end the returned end coordinate + @return 0 on success; -1 on failure + */ + int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *begin, int *end); + + + /************************** + * APIs for optional tags * + **************************/ + + /*! + @abstract Retrieve data of a tag + @param b pointer to an alignment struct + @param tag two-character tag to be retrieved + + @return pointer to the type and data. The first character is the + type that can be 'iIsScCdfAZH'. + + @discussion Use bam_aux2?() series to convert the returned data to + the corresponding type. + */ + uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]); + + int32_t bam_aux2i(const uint8_t *s); + float bam_aux2f(const uint8_t *s); + double bam_aux2d(const uint8_t *s); + char bam_aux2A(const uint8_t *s); + char *bam_aux2Z(const uint8_t *s); + + int bam_aux_del(bam1_t *b, uint8_t *s); + void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data); + uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2]); // an alias of bam_aux_get() + + + /***************** + * Miscellaneous * + *****************/ + + /*! + @abstract Calculate the rightmost coordinate of an alignment on the + reference genome. + + @param c pointer to the bam1_core_t structure + @param cigar the corresponding CIGAR array (from bam1_t::cigar) + @return the rightmost coordinate, 0-based + */ + uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar); + + /*! + @abstract Calculate the length of the query sequence from CIGAR. + @param c pointer to the bam1_core_t structure + @param cigar the corresponding CIGAR array (from bam1_t::cigar) + @return length of the query sequence + */ + int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar); + +#ifdef __cplusplus +} +#endif + +/*! + @abstract Calculate the minimum bin that contains a region [beg,end). + @param beg start of the region, 0-based + @param end end of the region, 0-based + @return bin + */ +static inline int bam_reg2bin(uint32_t beg, uint32_t end) +{ + --end; + if (beg>>14 == end>>14) return 4681 + (beg>>14); + if (beg>>17 == end>>17) return 585 + (beg>>17); + if (beg>>20 == end>>20) return 73 + (beg>>20); + if (beg>>23 == end>>23) return 9 + (beg>>23); + if (beg>>26 == end>>26) return 1 + (beg>>26); + return 0; +} + +/*! + @abstract Copy an alignment + @param bdst destination alignment struct + @param bsrc source alignment struct + @return pointer to the destination alignment struct + */ +static inline bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc) +{ + uint8_t *data = bdst->data; + int m_data = bdst->m_data; // backup data and m_data + if (m_data < bsrc->data_len) { // double the capacity + m_data = bsrc->data_len; kroundup32(m_data); + data = (uint8_t*)realloc(data, m_data); + } + memcpy(data, bsrc->data, bsrc->data_len); // copy var-len data + *bdst = *bsrc; // copy the rest + // restore the backup + bdst->m_data = m_data; + bdst->data = data; + return bdst; +} + +/*! + @abstract Duplicate an alignment + @param src source alignment struct + @return pointer to the destination alignment struct + */ +static inline bam1_t *bam_dup1(const bam1_t *src) +{ + bam1_t *b; + b = bam_init1(); + *b = *src; + b->m_data = b->data_len; + b->data = (uint8_t*)calloc(b->data_len, 1); + memcpy(b->data, src->data, b->data_len); + return b; +} + +static inline int bam_aux_type2size(int x) +{ + if (x == 'C' || x == 'c' || x == 'A') return 1; + else if (x == 'S' || x == 's') return 2; + else if (x == 'I' || x == 'i' || x == 'f') return 4; + else return 0; +} + + +#endif diff --git a/samtools-0.1.18/bam_aux.c b/samtools-0.1.18/bam_aux.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_aux.c @@ -0,0 +1,213 @@ +#include +#include "bam.h" +#include "khash.h" +typedef char *str_p; +KHASH_MAP_INIT_STR(s, int) +KHASH_MAP_INIT_STR(r2l, str_p) + +void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data) +{ + int ori_len = b->data_len; + b->data_len += 3 + len; + b->l_aux += 3 + len; + if (b->m_data < b->data_len) { + b->m_data = b->data_len; + kroundup32(b->m_data); + b->data = (uint8_t*)realloc(b->data, b->m_data); + } + b->data[ori_len] = tag[0]; b->data[ori_len + 1] = tag[1]; + b->data[ori_len + 2] = type; + memcpy(b->data + ori_len + 3, data, len); +} + +uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2]) +{ + return bam_aux_get(b, tag); +} + +#define __skip_tag(s) do { \ + int type = toupper(*(s)); \ + ++(s); \ + if (type == 'Z' || type == 'H') { while (*(s)) ++(s); ++(s); } \ + else if (type == 'B') (s) += 5 + bam_aux_type2size(*(s)) * (*(int32_t*)((s)+1)); \ + else (s) += bam_aux_type2size(type); \ + } while(0) + +uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]) +{ + uint8_t *s; + int y = tag[0]<<8 | tag[1]; + s = bam1_aux(b); + while (s < b->data + b->data_len) { + int x = (int)s[0]<<8 | s[1]; + s += 2; + if (x == y) return s; + __skip_tag(s); + } + return 0; +} +// s MUST BE returned by bam_aux_get() +int bam_aux_del(bam1_t *b, uint8_t *s) +{ + uint8_t *p, *aux; + aux = bam1_aux(b); + p = s - 2; + __skip_tag(s); + memmove(p, s, b->l_aux - (s - aux)); + b->data_len -= s - p; + b->l_aux -= s - p; + return 0; +} + +int bam_aux_drop_other(bam1_t *b, uint8_t *s) +{ + if (s) { + uint8_t *p, *aux; + aux = bam1_aux(b); + p = s - 2; + __skip_tag(s); + memmove(aux, p, s - p); + b->data_len -= b->l_aux - (s - p); + b->l_aux = s - p; + } else { + b->data_len -= b->l_aux; + b->l_aux = 0; + } + return 0; +} + +void bam_init_header_hash(bam_header_t *header) +{ + if (header->hash == 0) { + int ret, i; + khiter_t iter; + khash_t(s) *h; + header->hash = h = kh_init(s); + for (i = 0; i < header->n_targets; ++i) { + iter = kh_put(s, h, header->target_name[i], &ret); + kh_value(h, iter) = i; + } + } +} + +void bam_destroy_header_hash(bam_header_t *header) +{ + if (header->hash) + kh_destroy(s, (khash_t(s)*)header->hash); +} + +int32_t bam_get_tid(const bam_header_t *header, const char *seq_name) +{ + khint_t k; + khash_t(s) *h = (khash_t(s)*)header->hash; + k = kh_get(s, h, seq_name); + return k == kh_end(h)? -1 : kh_value(h, k); +} + +int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *beg, int *end) +{ + char *s; + int i, l, k, name_end; + khiter_t iter; + khash_t(s) *h; + + bam_init_header_hash(header); + h = (khash_t(s)*)header->hash; + + *ref_id = *beg = *end = -1; + name_end = l = strlen(str); + s = (char*)malloc(l+1); + // remove space + for (i = k = 0; i < l; ++i) + if (!isspace(str[i])) s[k++] = str[i]; + s[k] = 0; l = k; + // determine the sequence name + for (i = l - 1; i >= 0; --i) if (s[i] == ':') break; // look for colon from the end + if (i >= 0) name_end = i; + if (name_end < l) { // check if this is really the end + int n_hyphen = 0; + for (i = name_end + 1; i < l; ++i) { + if (s[i] == '-') ++n_hyphen; + else if (!isdigit(s[i]) && s[i] != ',') break; + } + if (i < l || n_hyphen > 1) name_end = l; // malformated region string; then take str as the name + s[name_end] = 0; + iter = kh_get(s, h, s); + if (iter == kh_end(h)) { // cannot find the sequence name + iter = kh_get(s, h, str); // try str as the name + if (iter == kh_end(h)) { + if (bam_verbose >= 2) fprintf(stderr, "[%s] fail to determine the sequence name.\n", __func__); + free(s); return -1; + } else s[name_end] = ':', name_end = l; + } + } else iter = kh_get(s, h, str); + *ref_id = kh_val(h, iter); + // parse the interval + if (name_end < l) { + for (i = k = name_end + 1; i < l; ++i) + if (s[i] != ',') s[k++] = s[i]; + s[k] = 0; + *beg = atoi(s + name_end + 1); + for (i = name_end + 1; i != k; ++i) if (s[i] == '-') break; + *end = i < k? atoi(s + i + 1) : 1<<29; + if (*beg > 0) --*beg; + } else *beg = 0, *end = 1<<29; + free(s); + return *beg <= *end? 0 : -1; +} + +int32_t bam_aux2i(const uint8_t *s) +{ + int type; + if (s == 0) return 0; + type = *s++; + if (type == 'c') return (int32_t)*(int8_t*)s; + else if (type == 'C') return (int32_t)*(uint8_t*)s; + else if (type == 's') return (int32_t)*(int16_t*)s; + else if (type == 'S') return (int32_t)*(uint16_t*)s; + else if (type == 'i' || type == 'I') return *(int32_t*)s; + else return 0; +} + +float bam_aux2f(const uint8_t *s) +{ + int type; + type = *s++; + if (s == 0) return 0.0; + if (type == 'f') return *(float*)s; + else return 0.0; +} + +double bam_aux2d(const uint8_t *s) +{ + int type; + type = *s++; + if (s == 0) return 0.0; + if (type == 'd') return *(double*)s; + else return 0.0; +} + +char bam_aux2A(const uint8_t *s) +{ + int type; + type = *s++; + if (s == 0) return 0; + if (type == 'A') return *(char*)s; + else return 0; +} + +char *bam_aux2Z(const uint8_t *s) +{ + int type; + type = *s++; + if (s == 0) return 0; + if (type == 'Z' || type == 'H') return (char*)s; + else return 0; +} + +#ifdef _WIN32 +double drand48() +{ + return (double)rand() / RAND_MAX; +} +#endif diff --git a/samtools-0.1.18/bam_endian.h b/samtools-0.1.18/bam_endian.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_endian.h @@ -0,0 +1,42 @@ +#ifndef BAM_ENDIAN_H +#define BAM_ENDIAN_H + +#include + +static inline int bam_is_big_endian() +{ + long one= 1; + return !(*((char *)(&one))); +} +static inline uint16_t bam_swap_endian_2(uint16_t v) +{ + return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8)); +} +static inline void *bam_swap_endian_2p(void *x) +{ + *(uint16_t*)x = bam_swap_endian_2(*(uint16_t*)x); + return x; +} +static inline uint32_t bam_swap_endian_4(uint32_t v) +{ + v = ((v & 0x0000FFFFU) << 16) | (v >> 16); + return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8); +} +static inline void *bam_swap_endian_4p(void *x) +{ + *(uint32_t*)x = bam_swap_endian_4(*(uint32_t*)x); + return x; +} +static inline uint64_t bam_swap_endian_8(uint64_t v) +{ + v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32); + v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16); + return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8); +} +static inline void *bam_swap_endian_8p(void *x) +{ + *(uint64_t*)x = bam_swap_endian_8(*(uint64_t*)x); + return x; +} + +#endif diff --git a/samtools-0.1.18/bam_import.c b/samtools-0.1.18/bam_import.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_import.c @@ -0,0 +1,485 @@ +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include "kstring.h" +#include "bam.h" +#include "sam_header.h" +#include "kseq.h" +#include "khash.h" + +KSTREAM_INIT(gzFile, gzread, 16384) +KHASH_MAP_INIT_STR(ref, uint64_t) + +void bam_init_header_hash(bam_header_t *header); +void bam_destroy_header_hash(bam_header_t *header); +int32_t bam_get_tid(const bam_header_t *header, const char *seq_name); + +unsigned char bam_nt16_table[256] = { + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 1, 2, 4, 8, 15,15,15,15, 15,15,15,15, 15, 0 /*=*/,15,15, + 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15, + 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15, + 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15, + 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, + 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15 +}; + +unsigned short bam_char2flag_table[256] = { + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,BAM_FREAD1,BAM_FREAD2,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + BAM_FPROPER_PAIR,0,BAM_FMREVERSE,0, 0,BAM_FMUNMAP,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, BAM_FDUP,0,BAM_FQCFAIL,0, 0,0,0,0, 0,0,0,0, + BAM_FPAIRED,0,BAM_FREVERSE,BAM_FSECONDARY, 0,BAM_FUNMAP,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 +}; + +char *bam_nt16_rev_table = "=ACMGRSVTWYHKDBN"; + +struct __tamFile_t { + gzFile fp; + kstream_t *ks; + kstring_t *str; + uint64_t n_lines; + int is_first; +}; + +char **__bam_get_lines(const char *fn, int *_n) // for bam_plcmd.c only +{ + char **list = 0, *s; + int n = 0, dret, m = 0; + gzFile fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); + kstream_t *ks; + kstring_t *str; + str = (kstring_t*)calloc(1, sizeof(kstring_t)); + ks = ks_init(fp); + while (ks_getuntil(ks, '\n', str, &dret) > 0) { + if (n == m) { + m = m? m << 1 : 16; + list = (char**)realloc(list, m * sizeof(char*)); + } + if (str->s[str->l-1] == '\r') + str->s[--str->l] = '\0'; + s = list[n++] = (char*)calloc(str->l + 1, 1); + strcpy(s, str->s); + } + ks_destroy(ks); + gzclose(fp); + free(str->s); free(str); + *_n = n; + return list; +} + +static bam_header_t *hash2header(const kh_ref_t *hash) +{ + bam_header_t *header; + khiter_t k; + header = bam_header_init(); + header->n_targets = kh_size(hash); + header->target_name = (char**)calloc(kh_size(hash), sizeof(char*)); + header->target_len = (uint32_t*)calloc(kh_size(hash), 4); + for (k = kh_begin(hash); k != kh_end(hash); ++k) { + if (kh_exist(hash, k)) { + int i = (int)kh_value(hash, k); + header->target_name[i] = (char*)kh_key(hash, k); + header->target_len[i] = kh_value(hash, k)>>32; + } + } + bam_init_header_hash(header); + return header; +} +bam_header_t *sam_header_read2(const char *fn) +{ + bam_header_t *header; + int c, dret, ret, error = 0; + gzFile fp; + kstream_t *ks; + kstring_t *str; + kh_ref_t *hash; + khiter_t k; + if (fn == 0) return 0; + fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); + if (fp == 0) return 0; + hash = kh_init(ref); + ks = ks_init(fp); + str = (kstring_t*)calloc(1, sizeof(kstring_t)); + while (ks_getuntil(ks, 0, str, &dret) > 0) { + char *s = strdup(str->s); + int len, i; + i = kh_size(hash); + ks_getuntil(ks, 0, str, &dret); + len = atoi(str->s); + k = kh_put(ref, hash, s, &ret); + if (ret == 0) { + fprintf(stderr, "[sam_header_read2] duplicated sequence name: %s\n", s); + error = 1; + } + kh_value(hash, k) = (uint64_t)len<<32 | i; + if (dret != '\n') + while ((c = ks_getc(ks)) != '\n' && c != -1); + } + ks_destroy(ks); + gzclose(fp); + free(str->s); free(str); + fprintf(stderr, "[sam_header_read2] %d sequences loaded.\n", kh_size(hash)); + if (error) return 0; + header = hash2header(hash); + kh_destroy(ref, hash); + return header; +} +static inline uint8_t *alloc_data(bam1_t *b, int size) +{ + if (b->m_data < size) { + b->m_data = size; + kroundup32(b->m_data); + b->data = (uint8_t*)realloc(b->data, b->m_data); + } + return b->data; +} +static inline void parse_error(int64_t n_lines, const char * __restrict msg) +{ + fprintf(stderr, "Parse error at line %lld: %s\n", (long long)n_lines, msg); + abort(); +} +static inline void append_text(bam_header_t *header, kstring_t *str) +{ + size_t x = header->l_text, y = header->l_text + str->l + 2; // 2 = 1 byte dret + 1 byte null + kroundup32(x); kroundup32(y); + if (x < y) + { + header->n_text = y; + header->text = (char*)realloc(header->text, y); + if ( !header->text ) + { + fprintf(stderr,"realloc failed to alloc %ld bytes\n", y); + abort(); + } + } + // Sanity check + if ( header->l_text+str->l+1 >= header->n_text ) + { + fprintf(stderr,"append_text FIXME: %ld>=%ld, x=%ld,y=%ld\n", header->l_text+str->l+1,header->n_text,x,y); + abort(); + } + strncpy(header->text + header->l_text, str->s, str->l+1); // we cannot use strcpy() here. + header->l_text += str->l + 1; + header->text[header->l_text] = 0; +} + +int sam_header_parse(bam_header_t *h) +{ + char **tmp; + int i; + free(h->target_len); free(h->target_name); + h->n_targets = 0; h->target_len = 0; h->target_name = 0; + if (h->l_text < 3) return 0; + if (h->dict == 0) h->dict = sam_header_parse2(h->text); + tmp = sam_header2list(h->dict, "SQ", "SN", &h->n_targets); + if (h->n_targets == 0) return 0; + h->target_name = calloc(h->n_targets, sizeof(void*)); + for (i = 0; i < h->n_targets; ++i) + h->target_name[i] = strdup(tmp[i]); + free(tmp); + tmp = sam_header2list(h->dict, "SQ", "LN", &h->n_targets); + h->target_len = calloc(h->n_targets, 4); + for (i = 0; i < h->n_targets; ++i) + h->target_len[i] = atoi(tmp[i]); + free(tmp); + return h->n_targets; +} + +bam_header_t *sam_header_read(tamFile fp) +{ + int ret, dret; + bam_header_t *header = bam_header_init(); + kstring_t *str = fp->str; + while ((ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret)) >= 0 && str->s[0] == '@') { // skip header + str->s[str->l] = dret; // note that str->s is NOT null terminated!! + append_text(header, str); + if (dret != '\n') { + ret = ks_getuntil(fp->ks, '\n', str, &dret); + str->s[str->l] = '\n'; // NOT null terminated!! + append_text(header, str); + } + ++fp->n_lines; + } + sam_header_parse(header); + bam_init_header_hash(header); + fp->is_first = 1; + return header; +} + +int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) +{ + int ret, doff, doff0, dret, z = 0; + bam1_core_t *c = &b->core; + kstring_t *str = fp->str; + kstream_t *ks = fp->ks; + + if (fp->is_first) { + fp->is_first = 0; + ret = str->l; + } else { + do { // special consideration for empty lines + ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret); + if (ret >= 0) z += str->l + 1; + } while (ret == 0); + } + if (ret < 0) return -1; + ++fp->n_lines; + doff = 0; + + { // name + c->l_qname = strlen(str->s) + 1; + memcpy(alloc_data(b, doff + c->l_qname) + doff, str->s, c->l_qname); + doff += c->l_qname; + } + { // flag + long flag; + char *s; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + flag = strtol((char*)str->s, &s, 0); + if (*s) { // not the end of the string + flag = 0; + for (s = str->s; *s; ++s) + flag |= bam_char2flag_table[(int)*s]; + } + c->flag = flag; + } + { // tid, pos, qual + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->tid = bam_get_tid(header, str->s); + if (c->tid < 0 && strcmp(str->s, "*")) { + if (header->n_targets == 0) { + fprintf(stderr, "[sam_read1] missing header? Abort!\n"); + exit(1); + } else fprintf(stderr, "[sam_read1] reference '%s' is recognized as '*'.\n", str->s); + } + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->pos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->qual = isdigit(str->s[0])? atoi(str->s) : 0; + if (ret < 0) return -2; + } + { // cigar + char *s, *t; + int i, op; + long x; + c->n_cigar = 0; + if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -3; + z += str->l + 1; + if (str->s[0] != '*') { + for (s = str->s; *s; ++s) { + if ((isalpha(*s)) || (*s=='=')) ++c->n_cigar; + else if (!isdigit(*s)) parse_error(fp->n_lines, "invalid CIGAR character"); + } + b->data = alloc_data(b, doff + c->n_cigar * 4); + for (i = 0, s = str->s; i != c->n_cigar; ++i) { + x = strtol(s, &t, 10); + op = toupper(*t); + if (op == 'M') op = BAM_CMATCH; + else if (op == 'I') op = BAM_CINS; + else if (op == 'D') op = BAM_CDEL; + else if (op == 'N') op = BAM_CREF_SKIP; + else if (op == 'S') op = BAM_CSOFT_CLIP; + else if (op == 'H') op = BAM_CHARD_CLIP; + else if (op == 'P') op = BAM_CPAD; + else if (op == '=') op = BAM_CEQUAL; + else if (op == 'X') op = BAM_CDIFF; + else parse_error(fp->n_lines, "invalid CIGAR operation"); + s = t + 1; + bam1_cigar(b)[i] = x << BAM_CIGAR_SHIFT | op; + } + if (*s) parse_error(fp->n_lines, "unmatched CIGAR operation"); + c->bin = bam_reg2bin(c->pos, bam_calend(c, bam1_cigar(b))); + doff += c->n_cigar * 4; + } else { + if (!(c->flag&BAM_FUNMAP)) { + fprintf(stderr, "Parse warning at line %lld: mapped sequence without CIGAR\n", (long long)fp->n_lines); + c->flag |= BAM_FUNMAP; + } + c->bin = bam_reg2bin(c->pos, c->pos + 1); + } + } + { // mtid, mpos, isize + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + c->mtid = strcmp(str->s, "=")? bam_get_tid(header, str->s) : c->tid; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + c->mpos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + c->isize = (str->s[0] == '-' || isdigit(str->s[0]))? atoi(str->s) : 0; + if (ret < 0) return -4; + } + { // seq and qual + int i; + uint8_t *p = 0; + if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -5; // seq + z += str->l + 1; + if (strcmp(str->s, "*")) { + c->l_qseq = strlen(str->s); + if (c->n_cigar && c->l_qseq != (int32_t)bam_cigar2qlen(c, bam1_cigar(b))) { + fprintf(stderr, "Line %ld, sequence length %i vs %i from CIGAR\n", + (long)fp->n_lines, c->l_qseq, (int32_t)bam_cigar2qlen(c, bam1_cigar(b))); + parse_error(fp->n_lines, "CIGAR and sequence length are inconsistent"); + } + p = (uint8_t*)alloc_data(b, doff + c->l_qseq + (c->l_qseq+1)/2) + doff; + memset(p, 0, (c->l_qseq+1)/2); + for (i = 0; i < c->l_qseq; ++i) + p[i/2] |= bam_nt16_table[(int)str->s[i]] << 4*(1-i%2); + } else c->l_qseq = 0; + if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -6; // qual + z += str->l + 1; + if (strcmp(str->s, "*") && c->l_qseq != strlen(str->s)) + parse_error(fp->n_lines, "sequence and quality are inconsistent"); + p += (c->l_qseq+1)/2; + if (strcmp(str->s, "*") == 0) for (i = 0; i < c->l_qseq; ++i) p[i] = 0xff; + else for (i = 0; i < c->l_qseq; ++i) p[i] = str->s[i] - 33; + doff += c->l_qseq + (c->l_qseq+1)/2; + } + doff0 = doff; + if (dret != '\n' && dret != '\r') { // aux + while (ks_getuntil(ks, KS_SEP_TAB, str, &dret) >= 0) { + uint8_t *s, type, key[2]; + z += str->l + 1; + if (str->l < 6 || str->s[2] != ':' || str->s[4] != ':') + parse_error(fp->n_lines, "missing colon in auxiliary data"); + key[0] = str->s[0]; key[1] = str->s[1]; + type = str->s[3]; + s = alloc_data(b, doff + 3) + doff; + s[0] = key[0]; s[1] = key[1]; s += 2; doff += 2; + if (type == 'A' || type == 'a' || type == 'c' || type == 'C') { // c and C for backward compatibility + s = alloc_data(b, doff + 2) + doff; + *s++ = 'A'; *s = str->s[5]; + doff += 2; + } else if (type == 'I' || type == 'i') { + long long x; + s = alloc_data(b, doff + 5) + doff; + x = (long long)atoll(str->s + 5); + if (x < 0) { + if (x >= -127) { + *s++ = 'c'; *(int8_t*)s = (int8_t)x; + s += 1; doff += 2; + } else if (x >= -32767) { + *s++ = 's'; *(int16_t*)s = (int16_t)x; + s += 2; doff += 3; + } else { + *s++ = 'i'; *(int32_t*)s = (int32_t)x; + s += 4; doff += 5; + if (x < -2147483648ll) + fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.", + (long long)fp->n_lines, x); + } + } else { + if (x <= 255) { + *s++ = 'C'; *s++ = (uint8_t)x; + doff += 2; + } else if (x <= 65535) { + *s++ = 'S'; *(uint16_t*)s = (uint16_t)x; + s += 2; doff += 3; + } else { + *s++ = 'I'; *(uint32_t*)s = (uint32_t)x; + s += 4; doff += 5; + if (x > 4294967295ll) + fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.", + (long long)fp->n_lines, x); + } + } + } else if (type == 'f') { + s = alloc_data(b, doff + 5) + doff; + *s++ = 'f'; + *(float*)s = (float)atof(str->s + 5); + s += 4; doff += 5; + } else if (type == 'd') { + s = alloc_data(b, doff + 9) + doff; + *s++ = 'd'; + *(float*)s = (float)atof(str->s + 9); + s += 8; doff += 9; + } else if (type == 'Z' || type == 'H') { + int size = 1 + (str->l - 5) + 1; + if (type == 'H') { // check whether the hex string is valid + int i; + if ((str->l - 5) % 2 == 1) parse_error(fp->n_lines, "length of the hex string not even"); + for (i = 0; i < str->l - 5; ++i) { + int c = toupper(str->s[5 + i]); + if (!((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F'))) + parse_error(fp->n_lines, "invalid hex character"); + } + } + s = alloc_data(b, doff + size) + doff; + *s++ = type; + memcpy(s, str->s + 5, str->l - 5); + s[str->l - 5] = 0; + doff += size; + } else if (type == 'B') { + int32_t n = 0, Bsize, k = 0, size; + char *p; + if (str->l < 8) parse_error(fp->n_lines, "too few values in aux type B"); + Bsize = bam_aux_type2size(str->s[5]); // the size of each element + for (p = (char*)str->s + 6; *p; ++p) // count the number of elements in the array + if (*p == ',') ++n; + p = str->s + 7; // now p points to the first number in the array + size = 6 + Bsize * n; // total number of bytes allocated to this tag + s = alloc_data(b, doff + 6 * Bsize * n) + doff; // allocate memory + *s++ = 'B'; *s++ = str->s[5]; + memcpy(s, &n, 4); s += 4; // write the number of elements + if (str->s[5] == 'c') while (p < str->s + str->l) ((int8_t*)s)[k++] = (int8_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'C') while (p < str->s + str->l) ((uint8_t*)s)[k++] = (uint8_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 's') while (p < str->s + str->l) ((int16_t*)s)[k++] = (int16_t)strtol(p, &p, 0), ++p; // FIXME: avoid unaligned memory + else if (str->s[5] == 'S') while (p < str->s + str->l) ((uint16_t*)s)[k++] = (uint16_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'i') while (p < str->s + str->l) ((int32_t*)s)[k++] = (int32_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'I') while (p < str->s + str->l) ((uint32_t*)s)[k++] = (uint32_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'f') while (p < str->s + str->l) ((float*)s)[k++] = (float)strtod(p, &p), ++p; + else parse_error(fp->n_lines, "unrecognized array type"); + s += Bsize * n; doff += size; + } else parse_error(fp->n_lines, "unrecognized type"); + if (dret == '\n' || dret == '\r') break; + } + } + b->l_aux = doff - doff0; + b->data_len = doff; + return z; +} + +tamFile sam_open(const char *fn) +{ + tamFile fp; + gzFile gzfp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "rb") : gzopen(fn, "rb"); + if (gzfp == 0) return 0; + fp = (tamFile)calloc(1, sizeof(struct __tamFile_t)); + fp->str = (kstring_t*)calloc(1, sizeof(kstring_t)); + fp->fp = gzfp; + fp->ks = ks_init(fp->fp); + return fp; +} + +void sam_close(tamFile fp) +{ + if (fp) { + ks_destroy(fp->ks); + gzclose(fp->fp); + free(fp->str->s); free(fp->str); + free(fp); + } +} diff --git a/samtools-0.1.18/bam_index.c b/samtools-0.1.18/bam_index.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_index.c @@ -0,0 +1,719 @@ +#include +#include +#include "bam.h" +#include "khash.h" +#include "ksort.h" +#include "bam_endian.h" +#ifdef _USE_KNETFILE +#include "knetfile.h" +#endif + +/*! + @header + + Alignment indexing. Before indexing, BAM must be sorted based on the + leftmost coordinate of alignments. In indexing, BAM uses two indices: + a UCSC binning index and a simple linear index. The binning index is + efficient for alignments spanning long distance, while the auxiliary + linear index helps to reduce unnecessary seek calls especially for + short alignments. + + The UCSC binning scheme was suggested by Richard Durbin and Lincoln + Stein and is explained by Kent et al. (2002). In this scheme, each bin + represents a contiguous genomic region which can be fully contained in + another bin; each alignment is associated with a bin which represents + the smallest region containing the entire alignment. The binning + scheme is essentially another representation of R-tree. A distinct bin + uniquely corresponds to a distinct internal node in a R-tree. Bin A is + a child of Bin B if region A is contained in B. + + In BAM, each bin may span 2^29, 2^26, 2^23, 2^20, 2^17 or 2^14 bp. Bin + 0 spans a 512Mbp region, bins 1-8 span 64Mbp, 9-72 8Mbp, 73-584 1Mbp, + 585-4680 128Kbp and bins 4681-37449 span 16Kbp regions. If we want to + find the alignments overlapped with a region [rbeg,rend), we need to + calculate the list of bins that may be overlapped the region and test + the alignments in the bins to confirm the overlaps. If the specified + region is short, typically only a few alignments in six bins need to + be retrieved. The overlapping alignments can be quickly fetched. + + */ + +#define BAM_MIN_CHUNK_GAP 32768 +// 1<<14 is the size of minimum bin. +#define BAM_LIDX_SHIFT 14 + +#define BAM_MAX_BIN 37450 // =(8^6-1)/7+1 + +typedef struct { + uint64_t u, v; +} pair64_t; + +#define pair64_lt(a,b) ((a).u < (b).u) +KSORT_INIT(off, pair64_t, pair64_lt) + +typedef struct { + uint32_t m, n; + pair64_t *list; +} bam_binlist_t; + +typedef struct { + int32_t n, m; + uint64_t *offset; +} bam_lidx_t; + +KHASH_MAP_INIT_INT(i, bam_binlist_t) + +struct __bam_index_t { + int32_t n; + uint64_t n_no_coor; // unmapped reads without coordinate + khash_t(i) **index; + bam_lidx_t *index2; +}; + +// requirement: len <= LEN_MASK +static inline void insert_offset(khash_t(i) *h, int bin, uint64_t beg, uint64_t end) +{ + khint_t k; + bam_binlist_t *l; + int ret; + k = kh_put(i, h, bin, &ret); + l = &kh_value(h, k); + if (ret) { // not present + l->m = 1; l->n = 0; + l->list = (pair64_t*)calloc(l->m, 16); + } + if (l->n == l->m) { + l->m <<= 1; + l->list = (pair64_t*)realloc(l->list, l->m * 16); + } + l->list[l->n].u = beg; l->list[l->n++].v = end; +} + +static inline void insert_offset2(bam_lidx_t *index2, bam1_t *b, uint64_t offset) +{ + int i, beg, end; + beg = b->core.pos >> BAM_LIDX_SHIFT; + end = (bam_calend(&b->core, bam1_cigar(b)) - 1) >> BAM_LIDX_SHIFT; + if (index2->m < end + 1) { + int old_m = index2->m; + index2->m = end + 1; + kroundup32(index2->m); + index2->offset = (uint64_t*)realloc(index2->offset, index2->m * 8); + memset(index2->offset + old_m, 0, 8 * (index2->m - old_m)); + } + if (beg == end) { + if (index2->offset[beg] == 0) index2->offset[beg] = offset; + } else { + for (i = beg; i <= end; ++i) + if (index2->offset[i] == 0) index2->offset[i] = offset; + } + index2->n = end + 1; +} + +static void merge_chunks(bam_index_t *idx) +{ +#if defined(BAM_TRUE_OFFSET) || defined(BAM_VIRTUAL_OFFSET16) + khash_t(i) *index; + int i, l, m; + khint_t k; + for (i = 0; i < idx->n; ++i) { + index = idx->index[i]; + for (k = kh_begin(index); k != kh_end(index); ++k) { + bam_binlist_t *p; + if (!kh_exist(index, k) || kh_key(index, k) == BAM_MAX_BIN) continue; + p = &kh_value(index, k); + m = 0; + for (l = 1; l < p->n; ++l) { +#ifdef BAM_TRUE_OFFSET + if (p->list[m].v + BAM_MIN_CHUNK_GAP > p->list[l].u) p->list[m].v = p->list[l].v; +#else + if (p->list[m].v>>16 == p->list[l].u>>16) p->list[m].v = p->list[l].v; +#endif + else p->list[++m] = p->list[l]; + } // ~for(l) + p->n = m + 1; + } // ~for(k) + } // ~for(i) +#endif // defined(BAM_TRUE_OFFSET) || defined(BAM_BGZF) +} + +static void fill_missing(bam_index_t *idx) +{ + int i, j; + for (i = 0; i < idx->n; ++i) { + bam_lidx_t *idx2 = &idx->index2[i]; + for (j = 1; j < idx2->n; ++j) + if (idx2->offset[j] == 0) + idx2->offset[j] = idx2->offset[j-1]; + } +} + +bam_index_t *bam_index_core(bamFile fp) +{ + bam1_t *b; + bam_header_t *h; + int i, ret; + bam_index_t *idx; + uint32_t last_bin, save_bin; + int32_t last_coor, last_tid, save_tid; + bam1_core_t *c; + uint64_t save_off, last_off, n_mapped, n_unmapped, off_beg, off_end, n_no_coor; + + idx = (bam_index_t*)calloc(1, sizeof(bam_index_t)); + b = (bam1_t*)calloc(1, sizeof(bam1_t)); + h = bam_header_read(fp); + c = &b->core; + + idx->n = h->n_targets; + bam_header_destroy(h); + idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*)); + for (i = 0; i < idx->n; ++i) idx->index[i] = kh_init(i); + idx->index2 = (bam_lidx_t*)calloc(idx->n, sizeof(bam_lidx_t)); + + save_bin = save_tid = last_tid = last_bin = 0xffffffffu; + save_off = last_off = bam_tell(fp); last_coor = 0xffffffffu; + n_mapped = n_unmapped = n_no_coor = off_end = 0; + off_beg = off_end = bam_tell(fp); + while ((ret = bam_read1(fp, b)) >= 0) { + if (c->tid < 0) ++n_no_coor; + if (last_tid < c->tid || (last_tid >= 0 && c->tid < 0)) { // change of chromosomes + last_tid = c->tid; + last_bin = 0xffffffffu; + } else if ((uint32_t)last_tid > (uint32_t)c->tid) { + fprintf(stderr, "[bam_index_core] the alignment is not sorted (%s): %d-th chr > %d-th chr\n", + bam1_qname(b), last_tid+1, c->tid+1); + return NULL; + } else if ((int32_t)c->tid >= 0 && last_coor > c->pos) { + fprintf(stderr, "[bam_index_core] the alignment is not sorted (%s): %u > %u in %d-th chr\n", + bam1_qname(b), last_coor, c->pos, c->tid+1); + return NULL; + } + if (c->tid >= 0 && !(c->flag & BAM_FUNMAP)) insert_offset2(&idx->index2[b->core.tid], b, last_off); + if (c->bin != last_bin) { // then possibly write the binning index + if (save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record + insert_offset(idx->index[save_tid], save_bin, save_off, last_off); + if (last_bin == 0xffffffffu && save_tid != 0xffffffffu) { // write the meta element + off_end = last_off; + insert_offset(idx->index[save_tid], BAM_MAX_BIN, off_beg, off_end); + insert_offset(idx->index[save_tid], BAM_MAX_BIN, n_mapped, n_unmapped); + n_mapped = n_unmapped = 0; + off_beg = off_end; + } + save_off = last_off; + save_bin = last_bin = c->bin; + save_tid = c->tid; + if (save_tid < 0) break; + } + if (bam_tell(fp) <= last_off) { + fprintf(stderr, "[bam_index_core] bug in BGZF/RAZF: %llx < %llx\n", + (unsigned long long)bam_tell(fp), (unsigned long long)last_off); + return NULL; + } + if (c->flag & BAM_FUNMAP) ++n_unmapped; + else ++n_mapped; + last_off = bam_tell(fp); + last_coor = b->core.pos; + } + if (save_tid >= 0) { + insert_offset(idx->index[save_tid], save_bin, save_off, bam_tell(fp)); + insert_offset(idx->index[save_tid], BAM_MAX_BIN, off_beg, bam_tell(fp)); + insert_offset(idx->index[save_tid], BAM_MAX_BIN, n_mapped, n_unmapped); + } + merge_chunks(idx); + fill_missing(idx); + if (ret >= 0) { + while ((ret = bam_read1(fp, b)) >= 0) { + ++n_no_coor; + if (c->tid >= 0 && n_no_coor) { + fprintf(stderr, "[bam_index_core] the alignment is not sorted: reads without coordinates prior to reads with coordinates.\n"); + return NULL; + } + } + } + if (ret < -1) fprintf(stderr, "[bam_index_core] truncated file? Continue anyway. (%d)\n", ret); + free(b->data); free(b); + idx->n_no_coor = n_no_coor; + return idx; +} + +void bam_index_destroy(bam_index_t *idx) +{ + khint_t k; + int i; + if (idx == 0) return; + for (i = 0; i < idx->n; ++i) { + khash_t(i) *index = idx->index[i]; + bam_lidx_t *index2 = idx->index2 + i; + for (k = kh_begin(index); k != kh_end(index); ++k) { + if (kh_exist(index, k)) + free(kh_value(index, k).list); + } + kh_destroy(i, index); + free(index2->offset); + } + free(idx->index); free(idx->index2); + free(idx); +} + +void bam_index_save(const bam_index_t *idx, FILE *fp) +{ + int32_t i, size; + khint_t k; + fwrite("BAI\1", 1, 4, fp); + if (bam_is_be) { + uint32_t x = idx->n; + fwrite(bam_swap_endian_4p(&x), 4, 1, fp); + } else fwrite(&idx->n, 4, 1, fp); + for (i = 0; i < idx->n; ++i) { + khash_t(i) *index = idx->index[i]; + bam_lidx_t *index2 = idx->index2 + i; + // write binning index + size = kh_size(index); + if (bam_is_be) { // big endian + uint32_t x = size; + fwrite(bam_swap_endian_4p(&x), 4, 1, fp); + } else fwrite(&size, 4, 1, fp); + for (k = kh_begin(index); k != kh_end(index); ++k) { + if (kh_exist(index, k)) { + bam_binlist_t *p = &kh_value(index, k); + if (bam_is_be) { // big endian + uint32_t x; + x = kh_key(index, k); fwrite(bam_swap_endian_4p(&x), 4, 1, fp); + x = p->n; fwrite(bam_swap_endian_4p(&x), 4, 1, fp); + for (x = 0; (int)x < p->n; ++x) { + bam_swap_endian_8p(&p->list[x].u); + bam_swap_endian_8p(&p->list[x].v); + } + fwrite(p->list, 16, p->n, fp); + for (x = 0; (int)x < p->n; ++x) { + bam_swap_endian_8p(&p->list[x].u); + bam_swap_endian_8p(&p->list[x].v); + } + } else { + fwrite(&kh_key(index, k), 4, 1, fp); + fwrite(&p->n, 4, 1, fp); + fwrite(p->list, 16, p->n, fp); + } + } + } + // write linear index (index2) + if (bam_is_be) { + int x = index2->n; + fwrite(bam_swap_endian_4p(&x), 4, 1, fp); + } else fwrite(&index2->n, 4, 1, fp); + if (bam_is_be) { // big endian + int x; + for (x = 0; (int)x < index2->n; ++x) + bam_swap_endian_8p(&index2->offset[x]); + fwrite(index2->offset, 8, index2->n, fp); + for (x = 0; (int)x < index2->n; ++x) + bam_swap_endian_8p(&index2->offset[x]); + } else fwrite(index2->offset, 8, index2->n, fp); + } + { // write the number of reads coor-less records. + uint64_t x = idx->n_no_coor; + if (bam_is_be) bam_swap_endian_8p(&x); + fwrite(&x, 8, 1, fp); + } + fflush(fp); +} + +static bam_index_t *bam_index_load_core(FILE *fp) +{ + int i; + char magic[4]; + bam_index_t *idx; + if (fp == 0) { + fprintf(stderr, "[bam_index_load_core] fail to load index.\n"); + return 0; + } + fread(magic, 1, 4, fp); + if (strncmp(magic, "BAI\1", 4)) { + fprintf(stderr, "[bam_index_load] wrong magic number.\n"); + fclose(fp); + return 0; + } + idx = (bam_index_t*)calloc(1, sizeof(bam_index_t)); + fread(&idx->n, 4, 1, fp); + if (bam_is_be) bam_swap_endian_4p(&idx->n); + idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*)); + idx->index2 = (bam_lidx_t*)calloc(idx->n, sizeof(bam_lidx_t)); + for (i = 0; i < idx->n; ++i) { + khash_t(i) *index; + bam_lidx_t *index2 = idx->index2 + i; + uint32_t key, size; + khint_t k; + int j, ret; + bam_binlist_t *p; + index = idx->index[i] = kh_init(i); + // load binning index + fread(&size, 4, 1, fp); + if (bam_is_be) bam_swap_endian_4p(&size); + for (j = 0; j < (int)size; ++j) { + fread(&key, 4, 1, fp); + if (bam_is_be) bam_swap_endian_4p(&key); + k = kh_put(i, index, key, &ret); + p = &kh_value(index, k); + fread(&p->n, 4, 1, fp); + if (bam_is_be) bam_swap_endian_4p(&p->n); + p->m = p->n; + p->list = (pair64_t*)malloc(p->m * 16); + fread(p->list, 16, p->n, fp); + if (bam_is_be) { + int x; + for (x = 0; x < p->n; ++x) { + bam_swap_endian_8p(&p->list[x].u); + bam_swap_endian_8p(&p->list[x].v); + } + } + } + // load linear index + fread(&index2->n, 4, 1, fp); + if (bam_is_be) bam_swap_endian_4p(&index2->n); + index2->m = index2->n; + index2->offset = (uint64_t*)calloc(index2->m, 8); + fread(index2->offset, index2->n, 8, fp); + if (bam_is_be) + for (j = 0; j < index2->n; ++j) bam_swap_endian_8p(&index2->offset[j]); + } + if (fread(&idx->n_no_coor, 8, 1, fp) == 0) idx->n_no_coor = 0; + if (bam_is_be) bam_swap_endian_8p(&idx->n_no_coor); + return idx; +} + +bam_index_t *bam_index_load_local(const char *_fn) +{ + FILE *fp; + char *fnidx, *fn; + + if (strstr(_fn, "ftp://") == _fn || strstr(_fn, "http://") == _fn) { + const char *p; + int l = strlen(_fn); + for (p = _fn + l - 1; p >= _fn; --p) + if (*p == '/') break; + fn = strdup(p + 1); + } else fn = strdup(_fn); + fnidx = (char*)calloc(strlen(fn) + 5, 1); + strcpy(fnidx, fn); strcat(fnidx, ".bai"); + fp = fopen(fnidx, "rb"); + if (fp == 0) { // try "{base}.bai" + char *s = strstr(fn, "bam"); + if (s == fn + strlen(fn) - 3) { + strcpy(fnidx, fn); + fnidx[strlen(fn)-1] = 'i'; + fp = fopen(fnidx, "rb"); + } + } + free(fnidx); free(fn); + if (fp) { + bam_index_t *idx = bam_index_load_core(fp); + fclose(fp); + return idx; + } else return 0; +} + +#ifdef _USE_KNETFILE +static void download_from_remote(const char *url) +{ + const int buf_size = 1 * 1024 * 1024; + char *fn; + FILE *fp; + uint8_t *buf; + knetFile *fp_remote; + int l; + if (strstr(url, "ftp://") != url && strstr(url, "http://") != url) return; + l = strlen(url); + for (fn = (char*)url + l - 1; fn >= url; --fn) + if (*fn == '/') break; + ++fn; // fn now points to the file name + fp_remote = knet_open(url, "r"); + if (fp_remote == 0) { + fprintf(stderr, "[download_from_remote] fail to open remote file.\n"); + return; + } + if ((fp = fopen(fn, "wb")) == 0) { + fprintf(stderr, "[download_from_remote] fail to create file in the working directory.\n"); + knet_close(fp_remote); + return; + } + buf = (uint8_t*)calloc(buf_size, 1); + while ((l = knet_read(fp_remote, buf, buf_size)) != 0) + fwrite(buf, 1, l, fp); + free(buf); + fclose(fp); + knet_close(fp_remote); +} +#else +static void download_from_remote(const char *url) +{ + return; +} +#endif + +bam_index_t *bam_index_load(const char *fn) +{ + bam_index_t *idx; + idx = bam_index_load_local(fn); + if (idx == 0 && (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn)) { + char *fnidx = calloc(strlen(fn) + 5, 1); + strcat(strcpy(fnidx, fn), ".bai"); + fprintf(stderr, "[bam_index_load] attempting to download the remote index file.\n"); + download_from_remote(fnidx); + idx = bam_index_load_local(fn); + } + if (idx == 0) fprintf(stderr, "[bam_index_load] fail to load BAM index.\n"); + return idx; +} + +int bam_index_build2(const char *fn, const char *_fnidx) +{ + char *fnidx; + FILE *fpidx; + bamFile fp; + bam_index_t *idx; + if ((fp = bam_open(fn, "r")) == 0) { + fprintf(stderr, "[bam_index_build2] fail to open the BAM file.\n"); + return -1; + } + idx = bam_index_core(fp); + bam_close(fp); + if(idx == 0) { + fprintf(stderr, "[bam_index_build2] fail to index the BAM file.\n"); + return -1; + } + if (_fnidx == 0) { + fnidx = (char*)calloc(strlen(fn) + 5, 1); + strcpy(fnidx, fn); strcat(fnidx, ".bai"); + } else fnidx = strdup(_fnidx); + fpidx = fopen(fnidx, "wb"); + if (fpidx == 0) { + fprintf(stderr, "[bam_index_build2] fail to create the index file.\n"); + free(fnidx); + return -1; + } + bam_index_save(idx, fpidx); + bam_index_destroy(idx); + fclose(fpidx); + free(fnidx); + return 0; +} + +int bam_index_build(const char *fn) +{ + return bam_index_build2(fn, 0); +} + +int bam_index(int argc, char *argv[]) +{ + if (argc < 2) { + fprintf(stderr, "Usage: samtools index [out.index]\n"); + return 1; + } + if (argc >= 3) bam_index_build2(argv[1], argv[2]); + else bam_index_build(argv[1]); + return 0; +} + +int bam_idxstats(int argc, char *argv[]) +{ + bam_index_t *idx; + bam_header_t *header; + bamFile fp; + int i; + if (argc < 2) { + fprintf(stderr, "Usage: samtools idxstats \n"); + return 1; + } + fp = bam_open(argv[1], "r"); + if (fp == 0) { fprintf(stderr, "[%s] fail to open BAM.\n", __func__); return 1; } + header = bam_header_read(fp); + bam_close(fp); + idx = bam_index_load(argv[1]); + if (idx == 0) { fprintf(stderr, "[%s] fail to load the index.\n", __func__); return 1; } + for (i = 0; i < idx->n; ++i) { + khint_t k; + khash_t(i) *h = idx->index[i]; + printf("%s\t%d", header->target_name[i], header->target_len[i]); + k = kh_get(i, h, BAM_MAX_BIN); + if (k != kh_end(h)) + printf("\t%llu\t%llu", (long long)kh_val(h, k).list[1].u, (long long)kh_val(h, k).list[1].v); + else printf("\t0\t0"); + putchar('\n'); + } + printf("*\t0\t0\t%llu\n", (long long)idx->n_no_coor); + bam_header_destroy(header); + bam_index_destroy(idx); + return 0; +} + +static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[BAM_MAX_BIN]) +{ + int i = 0, k; + if (beg >= end) return 0; + if (end >= 1u<<29) end = 1u<<29; + --end; + list[i++] = 0; + for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list[i++] = k; + for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list[i++] = k; + for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list[i++] = k; + for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list[i++] = k; + for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list[i++] = k; + return i; +} + +static inline int is_overlap(uint32_t beg, uint32_t end, const bam1_t *b) +{ + uint32_t rbeg = b->core.pos; + uint32_t rend = b->core.n_cigar? bam_calend(&b->core, bam1_cigar(b)) : b->core.pos + 1; + return (rend > beg && rbeg < end); +} + +struct __bam_iter_t { + int from_first; // read from the first record; no random access + int tid, beg, end, n_off, i, finished; + uint64_t curr_off; + pair64_t *off; +}; + +// bam_fetch helper function retrieves +bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end) +{ + uint16_t *bins; + int i, n_bins, n_off; + pair64_t *off; + khint_t k; + khash_t(i) *index; + uint64_t min_off; + bam_iter_t iter = 0; + + if (beg < 0) beg = 0; + if (end < beg) return 0; + // initialize iter + iter = calloc(1, sizeof(struct __bam_iter_t)); + iter->tid = tid, iter->beg = beg, iter->end = end; iter->i = -1; + // + bins = (uint16_t*)calloc(BAM_MAX_BIN, 2); + n_bins = reg2bins(beg, end, bins); + index = idx->index[tid]; + if (idx->index2[tid].n > 0) { + min_off = (beg>>BAM_LIDX_SHIFT >= idx->index2[tid].n)? idx->index2[tid].offset[idx->index2[tid].n-1] + : idx->index2[tid].offset[beg>>BAM_LIDX_SHIFT]; + if (min_off == 0) { // improvement for index files built by tabix prior to 0.1.4 + int n = beg>>BAM_LIDX_SHIFT; + if (n > idx->index2[tid].n) n = idx->index2[tid].n; + for (i = n - 1; i >= 0; --i) + if (idx->index2[tid].offset[i] != 0) break; + if (i >= 0) min_off = idx->index2[tid].offset[i]; + } + } else min_off = 0; // tabix 0.1.2 may produce such index files + for (i = n_off = 0; i < n_bins; ++i) { + if ((k = kh_get(i, index, bins[i])) != kh_end(index)) + n_off += kh_value(index, k).n; + } + if (n_off == 0) { + free(bins); return iter; + } + off = (pair64_t*)calloc(n_off, 16); + for (i = n_off = 0; i < n_bins; ++i) { + if ((k = kh_get(i, index, bins[i])) != kh_end(index)) { + int j; + bam_binlist_t *p = &kh_value(index, k); + for (j = 0; j < p->n; ++j) + if (p->list[j].v > min_off) off[n_off++] = p->list[j]; + } + } + free(bins); + if (n_off == 0) { + free(off); return iter; + } + { + bam1_t *b = (bam1_t*)calloc(1, sizeof(bam1_t)); + int l; + ks_introsort(off, n_off, off); + // resolve completely contained adjacent blocks + for (i = 1, l = 0; i < n_off; ++i) + if (off[l].v < off[i].v) + off[++l] = off[i]; + n_off = l + 1; + // resolve overlaps between adjacent blocks; this may happen due to the merge in indexing + for (i = 1; i < n_off; ++i) + if (off[i-1].v >= off[i].u) off[i-1].v = off[i].u; + { // merge adjacent blocks +#if defined(BAM_TRUE_OFFSET) || defined(BAM_VIRTUAL_OFFSET16) + for (i = 1, l = 0; i < n_off; ++i) { +#ifdef BAM_TRUE_OFFSET + if (off[l].v + BAM_MIN_CHUNK_GAP > off[i].u) off[l].v = off[i].v; +#else + if (off[l].v>>16 == off[i].u>>16) off[l].v = off[i].v; +#endif + else off[++l] = off[i]; + } + n_off = l + 1; +#endif + } + bam_destroy1(b); + } + iter->n_off = n_off; iter->off = off; + return iter; +} + +pair64_t *get_chunk_coordinates(const bam_index_t *idx, int tid, int beg, int end, int *cnt_off) +{ // for pysam compatibility + bam_iter_t iter; + pair64_t *off; + iter = bam_iter_query(idx, tid, beg, end); + off = iter->off; *cnt_off = iter->n_off; + free(iter); + return off; +} + +void bam_iter_destroy(bam_iter_t iter) +{ + if (iter) { free(iter->off); free(iter); } +} + +int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b) +{ + int ret; + if (iter && iter->finished) return -1; + if (iter == 0 || iter->from_first) { + ret = bam_read1(fp, b); + if (ret < 0 && iter) iter->finished = 1; + return ret; + } + if (iter->off == 0) return -1; + for (;;) { + if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk + if (iter->i == iter->n_off - 1) { ret = -1; break; } // no more chunks + if (iter->i >= 0) assert(iter->curr_off == iter->off[iter->i].v); // otherwise bug + if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek + bam_seek(fp, iter->off[iter->i+1].u, SEEK_SET); + iter->curr_off = bam_tell(fp); + } + ++iter->i; + } + if ((ret = bam_read1(fp, b)) >= 0) { + iter->curr_off = bam_tell(fp); + if (b->core.tid != iter->tid || b->core.pos >= iter->end) { // no need to proceed + ret = bam_validate1(NULL, b)? -1 : -5; // determine whether end of region or error + break; + } + else if (is_overlap(iter->beg, iter->end, b)) return ret; + } else break; // end of file or error + } + iter->finished = 1; + return ret; +} + +int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func) +{ + int ret; + bam_iter_t iter; + bam1_t *b; + b = bam_init1(); + iter = bam_iter_query(idx, tid, beg, end); + while ((ret = bam_iter_read(fp, iter, b)) >= 0) func(b, data); + bam_iter_destroy(iter); + bam_destroy1(b); + return (ret == -1)? 0 : ret; +} diff --git a/samtools-0.1.18/bam_lpileup.c b/samtools-0.1.18/bam_lpileup.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_lpileup.c @@ -0,0 +1,198 @@ +#include +#include +#include +#include "bam.h" +#include "ksort.h" + +#define TV_GAP 2 + +typedef struct __freenode_t { + uint32_t level:28, cnt:4; + struct __freenode_t *next; +} freenode_t, *freenode_p; + +#define freenode_lt(a,b) ((a)->cnt < (b)->cnt || ((a)->cnt == (b)->cnt && (a)->level < (b)->level)) +KSORT_INIT(node, freenode_p, freenode_lt) + +/* Memory pool, similar to the one in bam_pileup.c */ +typedef struct { + int cnt, n, max; + freenode_t **buf; +} mempool_t; + +static mempool_t *mp_init() +{ + return (mempool_t*)calloc(1, sizeof(mempool_t)); +} +static void mp_destroy(mempool_t *mp) +{ + int k; + for (k = 0; k < mp->n; ++k) free(mp->buf[k]); + free(mp->buf); free(mp); +} +static inline freenode_t *mp_alloc(mempool_t *mp) +{ + ++mp->cnt; + if (mp->n == 0) return (freenode_t*)calloc(1, sizeof(freenode_t)); + else return mp->buf[--mp->n]; +} +static inline void mp_free(mempool_t *mp, freenode_t *p) +{ + --mp->cnt; p->next = 0; p->cnt = TV_GAP; + if (mp->n == mp->max) { + mp->max = mp->max? mp->max<<1 : 256; + mp->buf = (freenode_t**)realloc(mp->buf, sizeof(freenode_t*) * mp->max); + } + mp->buf[mp->n++] = p; +} + +/* core part */ +struct __bam_lplbuf_t { + int max, n_cur, n_pre; + int max_level, *cur_level, *pre_level; + mempool_t *mp; + freenode_t **aux, *head, *tail; + int n_nodes, m_aux; + bam_pileup_f func; + void *user_data; + bam_plbuf_t *plbuf; +}; + +void bam_lplbuf_reset(bam_lplbuf_t *buf) +{ + freenode_t *p, *q; + bam_plbuf_reset(buf->plbuf); + for (p = buf->head; p->next;) { + q = p->next; + mp_free(buf->mp, p); + p = q; + } + buf->head = buf->tail; + buf->max_level = 0; + buf->n_cur = buf->n_pre = 0; + buf->n_nodes = 0; +} + +static int tview_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data) +{ + bam_lplbuf_t *tv = (bam_lplbuf_t*)data; + freenode_t *p; + int i, l, max_level; + // allocate memory if necessary + if (tv->max < n) { // enlarge + tv->max = n; + kroundup32(tv->max); + tv->cur_level = (int*)realloc(tv->cur_level, sizeof(int) * tv->max); + tv->pre_level = (int*)realloc(tv->pre_level, sizeof(int) * tv->max); + } + tv->n_cur = n; + // update cnt + for (p = tv->head; p->next; p = p->next) + if (p->cnt > 0) --p->cnt; + // calculate cur_level[] + max_level = 0; + for (i = l = 0; i < n; ++i) { + const bam_pileup1_t *p = pl + i; + if (p->is_head) { + if (tv->head->next && tv->head->cnt == 0) { // then take a free slot + freenode_t *p = tv->head->next; + tv->cur_level[i] = tv->head->level; + mp_free(tv->mp, tv->head); + tv->head = p; + --tv->n_nodes; + } else tv->cur_level[i] = ++tv->max_level; + } else { + tv->cur_level[i] = tv->pre_level[l++]; + if (p->is_tail) { // then return a free slot + tv->tail->level = tv->cur_level[i]; + tv->tail->next = mp_alloc(tv->mp); + tv->tail = tv->tail->next; + ++tv->n_nodes; + } + } + if (tv->cur_level[i] > max_level) max_level = tv->cur_level[i]; + ((bam_pileup1_t*)p)->level = tv->cur_level[i]; + } + assert(l == tv->n_pre); + tv->func(tid, pos, n, pl, tv->user_data); + // sort the linked list + if (tv->n_nodes) { + freenode_t *q; + if (tv->n_nodes + 1 > tv->m_aux) { // enlarge + tv->m_aux = tv->n_nodes + 1; + kroundup32(tv->m_aux); + tv->aux = (freenode_t**)realloc(tv->aux, sizeof(void*) * tv->m_aux); + } + for (p = tv->head, i = l = 0; p->next;) { + if (p->level > max_level) { // then discard this entry + q = p->next; + mp_free(tv->mp, p); + p = q; + } else { + tv->aux[i++] = p; + p = p->next; + } + } + tv->aux[i] = tv->tail; // add a proper tail for the loop below + tv->n_nodes = i; + if (tv->n_nodes) { + ks_introsort(node, tv->n_nodes, tv->aux); + for (i = 0; i < tv->n_nodes; ++i) tv->aux[i]->next = tv->aux[i+1]; + tv->head = tv->aux[0]; + } else tv->head = tv->tail; + } + // clean up + tv->max_level = max_level; + memcpy(tv->pre_level, tv->cur_level, tv->n_cur * 4); + // squeeze out terminated levels + for (i = l = 0; i < n; ++i) { + const bam_pileup1_t *p = pl + i; + if (!p->is_tail) + tv->pre_level[l++] = tv->pre_level[i]; + } + tv->n_pre = l; +/* + fprintf(stderr, "%d\t", pos+1); + for (i = 0; i < n; ++i) { + const bam_pileup1_t *p = pl + i; + if (p->is_head) fprintf(stderr, "^"); + if (p->is_tail) fprintf(stderr, "$"); + fprintf(stderr, "%d,", p->level); + } + fprintf(stderr, "\n"); +*/ + return 0; +} + +bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data) +{ + bam_lplbuf_t *tv; + tv = (bam_lplbuf_t*)calloc(1, sizeof(bam_lplbuf_t)); + tv->mp = mp_init(); + tv->head = tv->tail = mp_alloc(tv->mp); + tv->func = func; + tv->user_data = data; + tv->plbuf = bam_plbuf_init(tview_func, tv); + return (bam_lplbuf_t*)tv; +} + +void bam_lplbuf_destroy(bam_lplbuf_t *tv) +{ + freenode_t *p, *q; + free(tv->cur_level); free(tv->pre_level); + bam_plbuf_destroy(tv->plbuf); + free(tv->aux); + for (p = tv->head; p->next;) { + q = p->next; + mp_free(tv->mp, p); p = q; + } + mp_free(tv->mp, p); + assert(tv->mp->cnt == 0); + mp_destroy(tv->mp); + free(tv); +} + +int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *tv) +{ + return bam_plbuf_push(b, tv->plbuf); +} diff --git a/samtools-0.1.18/bam_md.c b/samtools-0.1.18/bam_md.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_md.c @@ -0,0 +1,384 @@ +#include +#include +#include +#include +#include +#include "faidx.h" +#include "sam.h" +#include "kstring.h" +#include "kaln.h" +#include "kprobaln.h" + +#define USE_EQUAL 1 +#define DROP_TAG 2 +#define BIN_QUAL 4 +#define UPDATE_NM 8 +#define UPDATE_MD 16 +#define HASH_QNM 32 + +char bam_nt16_nt4_table[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 }; + +int bam_aux_drop_other(bam1_t *b, uint8_t *s); + +void bam_fillmd1_core(bam1_t *b, char *ref, int flag, int max_nm) +{ + uint8_t *seq = bam1_seq(b); + uint32_t *cigar = bam1_cigar(b); + bam1_core_t *c = &b->core; + int i, x, y, u = 0; + kstring_t *str; + int32_t old_nm_i = -1, nm = 0; + + str = (kstring_t*)calloc(1, sizeof(kstring_t)); + for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { + int j, l = cigar[i]>>4, op = cigar[i]&0xf; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + for (j = 0; j < l; ++j) { + int z = y + j; + int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; + if (ref[x+j] == 0) break; // out of boundary + if ((c1 == c2 && c1 != 15 && c2 != 15) || c1 == 0) { // a match + if (flag&USE_EQUAL) seq[z/2] &= (z&1)? 0xf0 : 0x0f; + ++u; + } else { + kputw(u, str); kputc(ref[x+j], str); + u = 0; ++nm; + } + } + if (j < l) break; + x += l; y += l; + } else if (op == BAM_CDEL) { + kputw(u, str); kputc('^', str); + for (j = 0; j < l; ++j) { + if (ref[x+j] == 0) break; + kputc(ref[x+j], str); + } + u = 0; + if (j < l) break; + x += l; nm += l; + } else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) { + y += l; + if (op == BAM_CINS) nm += l; + } else if (op == BAM_CREF_SKIP) { + x += l; + } + } + kputw(u, str); + // apply max_nm + if (max_nm > 0 && nm >= max_nm) { + for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { + int j, l = cigar[i]>>4, op = cigar[i]&0xf; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + for (j = 0; j < l; ++j) { + int z = y + j; + int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; + if (ref[x+j] == 0) break; // out of boundary + if ((c1 == c2 && c1 != 15 && c2 != 15) || c1 == 0) { // a match + seq[z/2] |= (z&1)? 0x0f : 0xf0; + bam1_qual(b)[z] = 0; + } + } + if (j < l) break; + x += l; y += l; + } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) x += l; + else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l; + } + } + // update NM + if (flag & UPDATE_NM) { + uint8_t *old_nm = bam_aux_get(b, "NM"); + if (c->flag & BAM_FUNMAP) return; + if (old_nm) old_nm_i = bam_aux2i(old_nm); + if (!old_nm) bam_aux_append(b, "NM", 'i', 4, (uint8_t*)&nm); + else if (nm != old_nm_i) { + fprintf(stderr, "[bam_fillmd1] different NM for read '%s': %d -> %d\n", bam1_qname(b), old_nm_i, nm); + bam_aux_del(b, old_nm); + bam_aux_append(b, "NM", 'i', 4, (uint8_t*)&nm); + } + } + // update MD + if (flag & UPDATE_MD) { + uint8_t *old_md = bam_aux_get(b, "MD"); + if (c->flag & BAM_FUNMAP) return; + if (!old_md) bam_aux_append(b, "MD", 'Z', str->l + 1, (uint8_t*)str->s); + else { + int is_diff = 0; + if (strlen((char*)old_md+1) == str->l) { + for (i = 0; i < str->l; ++i) + if (toupper(old_md[i+1]) != toupper(str->s[i])) + break; + if (i < str->l) is_diff = 1; + } else is_diff = 1; + if (is_diff) { + fprintf(stderr, "[bam_fillmd1] different MD for read '%s': '%s' -> '%s'\n", bam1_qname(b), old_md+1, str->s); + bam_aux_del(b, old_md); + bam_aux_append(b, "MD", 'Z', str->l + 1, (uint8_t*)str->s); + } + } + } + // drop all tags but RG + if (flag&DROP_TAG) { + uint8_t *q = bam_aux_get(b, "RG"); + bam_aux_drop_other(b, q); + } + // reduce the resolution of base quality + if (flag&BIN_QUAL) { + uint8_t *qual = bam1_qual(b); + for (i = 0; i < b->core.l_qseq; ++i) + if (qual[i] >= 3) qual[i] = qual[i]/10*10 + 7; + } + free(str->s); free(str); +} + +void bam_fillmd1(bam1_t *b, char *ref, int flag) +{ + bam_fillmd1_core(b, ref, flag, 0); +} + +int bam_cap_mapQ(bam1_t *b, char *ref, int thres) +{ + uint8_t *seq = bam1_seq(b), *qual = bam1_qual(b); + uint32_t *cigar = bam1_cigar(b); + bam1_core_t *c = &b->core; + int i, x, y, mm, q, len, clip_l, clip_q; + double t; + if (thres < 0) thres = 40; // set the default + mm = q = len = clip_l = clip_q = 0; + for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { + int j, l = cigar[i]>>4, op = cigar[i]&0xf; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + for (j = 0; j < l; ++j) { + int z = y + j; + int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; + if (ref[x+j] == 0) break; // out of boundary + if (c2 != 15 && c1 != 15 && qual[z] >= 13) { // not ambiguous + ++len; + if (c1 && c1 != c2 && qual[z] >= 13) { // mismatch + ++mm; + q += qual[z] > 33? 33 : qual[z]; + } + } + } + if (j < l) break; + x += l; y += l; len += l; + } else if (op == BAM_CDEL) { + for (j = 0; j < l; ++j) + if (ref[x+j] == 0) break; + if (j < l) break; + x += l; + } else if (op == BAM_CSOFT_CLIP) { + for (j = 0; j < l; ++j) clip_q += qual[y+j]; + clip_l += l; + y += l; + } else if (op == BAM_CHARD_CLIP) { + clip_q += 13 * l; + clip_l += l; + } else if (op == BAM_CINS) y += l; + else if (op == BAM_CREF_SKIP) x += l; + } + for (i = 0, t = 1; i < mm; ++i) + t *= (double)len / (i+1); + t = q - 4.343 * log(t) + clip_q / 5.; + if (t > thres) return -1; + if (t < 0) t = 0; + t = sqrt((thres - t) / thres) * thres; +// fprintf(stderr, "%s %lf %d\n", bam1_qname(b), t, q); + return (int)(t + .499); +} + +int bam_prob_realn_core(bam1_t *b, const char *ref, int flag) +{ + int k, i, bw, x, y, yb, ye, xb, xe, apply_baq = flag&1, extend_baq = flag>>1&1; + uint32_t *cigar = bam1_cigar(b); + bam1_core_t *c = &b->core; + kpa_par_t conf = kpa_par_def; + uint8_t *bq = 0, *zq = 0, *qual = bam1_qual(b); + if ((c->flag & BAM_FUNMAP) || b->core.l_qseq == 0) return -1; // do nothing + // test if BQ or ZQ is present + if ((bq = bam_aux_get(b, "BQ")) != 0) ++bq; + if ((zq = bam_aux_get(b, "ZQ")) != 0 && *zq == 'Z') ++zq; + if (bq && zq) { // remove the ZQ tag + bam_aux_del(b, zq-1); + zq = 0; + } + if (bq || zq) { + if ((apply_baq && zq) || (!apply_baq && bq)) return -3; // in both cases, do nothing + if (bq && apply_baq) { // then convert BQ to ZQ + for (i = 0; i < c->l_qseq; ++i) + qual[i] = qual[i] + 64 < bq[i]? 0 : qual[i] - ((int)bq[i] - 64); + *(bq - 3) = 'Z'; + } else if (zq && !apply_baq) { // then convert ZQ to BQ + for (i = 0; i < c->l_qseq; ++i) + qual[i] += (int)zq[i] - 64; + *(zq - 3) = 'B'; + } + return 0; + } + // find the start and end of the alignment + x = c->pos, y = 0, yb = ye = xb = xe = -1; + for (k = 0; k < c->n_cigar; ++k) { + int op, l; + op = cigar[k]&0xf; l = cigar[k]>>4; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + if (yb < 0) yb = y; + if (xb < 0) xb = x; + ye = y + l; xe = x + l; + x += l; y += l; + } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l; + else if (op == BAM_CDEL) x += l; + else if (op == BAM_CREF_SKIP) return -1; // do nothing if there is a reference skip + } + // set bandwidth and the start and the end + bw = 7; + if (abs((xe - xb) - (ye - yb)) > bw) + bw = abs((xe - xb) - (ye - yb)) + 3; + conf.bw = bw; + xb -= yb + bw/2; if (xb < 0) xb = 0; + xe += c->l_qseq - ye + bw/2; + if (xe - xb - c->l_qseq > bw) + xb += (xe - xb - c->l_qseq - bw) / 2, xe -= (xe - xb - c->l_qseq - bw) / 2; + { // glocal + uint8_t *s, *r, *q, *seq = bam1_seq(b), *bq; + int *state; + bq = calloc(c->l_qseq + 1, 1); + memcpy(bq, qual, c->l_qseq); + s = calloc(c->l_qseq, 1); + for (i = 0; i < c->l_qseq; ++i) s[i] = bam_nt16_nt4_table[bam1_seqi(seq, i)]; + r = calloc(xe - xb, 1); + for (i = xb; i < xe; ++i) { + if (ref[i] == 0) { xe = i; break; } + r[i-xb] = bam_nt16_nt4_table[bam_nt16_table[(int)ref[i]]]; + } + state = calloc(c->l_qseq, sizeof(int)); + q = calloc(c->l_qseq, 1); + kpa_glocal(r, xe-xb, s, c->l_qseq, qual, &conf, state, q); + if (!extend_baq) { // in this block, bq[] is capped by base quality qual[] + for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) { + int op = cigar[k]&0xf, l = cigar[k]>>4; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + for (i = y; i < y + l; ++i) { + if ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y)) bq[i] = 0; + else bq[i] = bq[i] < q[i]? bq[i] : q[i]; + } + x += l; y += l; + } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l; + else if (op == BAM_CDEL) x += l; + } + for (i = 0; i < c->l_qseq; ++i) bq[i] = qual[i] - bq[i] + 64; // finalize BQ + } else { // in this block, bq[] is BAQ that can be larger than qual[] (different from the above!) + uint8_t *left, *rght; + left = calloc(c->l_qseq, 1); rght = calloc(c->l_qseq, 1); + for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) { + int op = cigar[k]&0xf, l = cigar[k]>>4; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + for (i = y; i < y + l; ++i) + bq[i] = ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y))? 0 : q[i]; + for (left[y] = bq[y], i = y + 1; i < y + l; ++i) + left[i] = bq[i] > left[i-1]? bq[i] : left[i-1]; + for (rght[y+l-1] = bq[y+l-1], i = y + l - 2; i >= y; --i) + rght[i] = bq[i] > rght[i+1]? bq[i] : rght[i+1]; + for (i = y; i < y + l; ++i) + bq[i] = left[i] < rght[i]? left[i] : rght[i]; + x += l; y += l; + } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l; + else if (op == BAM_CDEL) x += l; + } + for (i = 0; i < c->l_qseq; ++i) bq[i] = 64 + (qual[i] <= bq[i]? 0 : qual[i] - bq[i]); // finalize BQ + free(left); free(rght); + } + if (apply_baq) { + for (i = 0; i < c->l_qseq; ++i) qual[i] -= bq[i] - 64; // modify qual + bam_aux_append(b, "ZQ", 'Z', c->l_qseq + 1, bq); + } else bam_aux_append(b, "BQ", 'Z', c->l_qseq + 1, bq); + free(bq); free(s); free(r); free(q); free(state); + } + return 0; +} + +int bam_prob_realn(bam1_t *b, const char *ref) +{ + return bam_prob_realn_core(b, ref, 1); +} + +int bam_fillmd(int argc, char *argv[]) +{ + int c, flt_flag, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm, is_realn, capQ, baq_flag; + samfile_t *fp, *fpout = 0; + faidx_t *fai; + char *ref = 0, mode_w[8], mode_r[8]; + bam1_t *b; + + flt_flag = UPDATE_NM | UPDATE_MD; + is_bam_out = is_sam_in = is_uncompressed = is_realn = max_nm = capQ = baq_flag = 0; + mode_w[0] = mode_r[0] = 0; + strcpy(mode_r, "r"); strcpy(mode_w, "w"); + while ((c = getopt(argc, argv, "EqreuNhbSC:n:Ad")) >= 0) { + switch (c) { + case 'r': is_realn = 1; break; + case 'e': flt_flag |= USE_EQUAL; break; + case 'd': flt_flag |= DROP_TAG; break; + case 'q': flt_flag |= BIN_QUAL; break; + case 'h': flt_flag |= HASH_QNM; break; + case 'N': flt_flag &= ~(UPDATE_MD|UPDATE_NM); break; + case 'b': is_bam_out = 1; break; + case 'u': is_uncompressed = is_bam_out = 1; break; + case 'S': is_sam_in = 1; break; + case 'n': max_nm = atoi(optarg); break; + case 'C': capQ = atoi(optarg); break; + case 'A': baq_flag |= 1; break; + case 'E': baq_flag |= 2; break; + default: fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n", c); return 1; + } + } + if (!is_sam_in) strcat(mode_r, "b"); + if (is_bam_out) strcat(mode_w, "b"); + else strcat(mode_w, "h"); + if (is_uncompressed) strcat(mode_w, "u"); + if (optind + 1 >= argc) { + fprintf(stderr, "\n"); + fprintf(stderr, "Usage: samtools fillmd [-eubrS] \n\n"); + fprintf(stderr, "Options: -e change identical bases to '='\n"); + fprintf(stderr, " -u uncompressed BAM output (for piping)\n"); + fprintf(stderr, " -b compressed BAM output\n"); + fprintf(stderr, " -S the input is SAM with header\n"); + fprintf(stderr, " -A modify the quality string\n"); + fprintf(stderr, " -r compute the BQ tag (without -A) or cap baseQ by BAQ (with -A)\n"); + fprintf(stderr, " -E extended BAQ for better sensitivity but lower specificity\n\n"); + return 1; + } + fp = samopen(argv[optind], mode_r, 0); + if (fp == 0) return 1; + if (is_sam_in && (fp->header == 0 || fp->header->n_targets == 0)) { + fprintf(stderr, "[bam_fillmd] input SAM does not have header. Abort!\n"); + return 1; + } + fpout = samopen("-", mode_w, fp->header); + fai = fai_load(argv[optind+1]); + + b = bam_init1(); + while ((ret = samread(fp, b)) >= 0) { + if (b->core.tid >= 0) { + if (tid != b->core.tid) { + free(ref); + ref = fai_fetch(fai, fp->header->target_name[b->core.tid], &len); + tid = b->core.tid; + if (ref == 0) + fprintf(stderr, "[bam_fillmd] fail to find sequence '%s' in the reference.\n", + fp->header->target_name[tid]); + } + if (is_realn) bam_prob_realn_core(b, ref, baq_flag); + if (capQ > 10) { + int q = bam_cap_mapQ(b, ref, capQ); + if (b->core.qual > q) b->core.qual = q; + } + if (ref) bam_fillmd1_core(b, ref, flt_flag, max_nm); + } + samwrite(fpout, b); + } + bam_destroy1(b); + + free(ref); + fai_destroy(fai); + samclose(fp); samclose(fpout); + return 0; +} diff --git a/samtools-0.1.18/bam_pileup.c b/samtools-0.1.18/bam_pileup.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_pileup.c @@ -0,0 +1,437 @@ +#include +#include +#include +#include +#include "sam.h" + +typedef struct { + int k, x, y, end; +} cstate_t; + +static cstate_t g_cstate_null = { -1, 0, 0, 0 }; + +typedef struct __linkbuf_t { + bam1_t b; + uint32_t beg, end; + cstate_t s; + struct __linkbuf_t *next; +} lbnode_t; + +/* --- BEGIN: Memory pool */ + +typedef struct { + int cnt, n, max; + lbnode_t **buf; +} mempool_t; + +static mempool_t *mp_init() +{ + mempool_t *mp; + mp = (mempool_t*)calloc(1, sizeof(mempool_t)); + return mp; +} +static void mp_destroy(mempool_t *mp) +{ + int k; + for (k = 0; k < mp->n; ++k) { + free(mp->buf[k]->b.data); + free(mp->buf[k]); + } + free(mp->buf); + free(mp); +} +static inline lbnode_t *mp_alloc(mempool_t *mp) +{ + ++mp->cnt; + if (mp->n == 0) return (lbnode_t*)calloc(1, sizeof(lbnode_t)); + else return mp->buf[--mp->n]; +} +static inline void mp_free(mempool_t *mp, lbnode_t *p) +{ + --mp->cnt; p->next = 0; // clear lbnode_t::next here + if (mp->n == mp->max) { + mp->max = mp->max? mp->max<<1 : 256; + mp->buf = (lbnode_t**)realloc(mp->buf, sizeof(lbnode_t*) * mp->max); + } + mp->buf[mp->n++] = p; +} + +/* --- END: Memory pool */ + +/* --- BEGIN: Auxiliary functions */ + +/* s->k: the index of the CIGAR operator that has just been processed. + s->x: the reference coordinate of the start of s->k + s->y: the query coordiante of the start of s->k + */ +static inline int resolve_cigar2(bam_pileup1_t *p, uint32_t pos, cstate_t *s) +{ +#define _cop(c) ((c)&BAM_CIGAR_MASK) +#define _cln(c) ((c)>>BAM_CIGAR_SHIFT) + + bam1_t *b = p->b; + bam1_core_t *c = &b->core; + uint32_t *cigar = bam1_cigar(b); + int k, is_head = 0; + // determine the current CIGAR operation +// fprintf(stderr, "%s\tpos=%d\tend=%d\t(%d,%d,%d)\n", bam1_qname(b), pos, s->end, s->k, s->x, s->y); + if (s->k == -1) { // never processed + is_head = 1; + if (c->n_cigar == 1) { // just one operation, save a loop + if (_cop(cigar[0]) == BAM_CMATCH || _cop(cigar[0]) == BAM_CEQUAL || _cop(cigar[0]) == BAM_CDIFF) s->k = 0, s->x = c->pos, s->y = 0; + } else { // find the first match or deletion + for (k = 0, s->x = c->pos, s->y = 0; k < c->n_cigar; ++k) { + int op = _cop(cigar[k]); + int l = _cln(cigar[k]); + if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CEQUAL || op == BAM_CDIFF) break; + else if (op == BAM_CREF_SKIP) s->x += l; + else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l; + } + assert(k < c->n_cigar); + s->k = k; + } + } else { // the read has been processed before + int op, l = _cln(cigar[s->k]); + if (pos - s->x >= l) { // jump to the next operation + assert(s->k < c->n_cigar); // otherwise a bug: this function should not be called in this case + op = _cop(cigar[s->k+1]); + if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) { // jump to the next without a loop + if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l; + s->x += l; + ++s->k; + } else { // find the next M/D/N/=/X + if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l; + s->x += l; + for (k = s->k + 1; k < c->n_cigar; ++k) { + op = _cop(cigar[k]), l = _cln(cigar[k]); + if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) break; + else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l; + } + s->k = k; + } + assert(s->k < c->n_cigar); // otherwise a bug + } // else, do nothing + } + { // collect pileup information + int op, l; + op = _cop(cigar[s->k]); l = _cln(cigar[s->k]); + p->is_del = p->indel = p->is_refskip = 0; + if (s->x + l - 1 == pos && s->k + 1 < c->n_cigar) { // peek the next operation + int op2 = _cop(cigar[s->k+1]); + int l2 = _cln(cigar[s->k+1]); + if (op2 == BAM_CDEL) p->indel = -(int)l2; + else if (op2 == BAM_CINS) p->indel = l2; + else if (op2 == BAM_CPAD && s->k + 2 < c->n_cigar) { // no working for adjacent padding + int l3 = 0; + for (k = s->k + 2; k < c->n_cigar; ++k) { + op2 = _cop(cigar[k]); l2 = _cln(cigar[k]); + if (op2 == BAM_CINS) l3 += l2; + else if (op2 == BAM_CDEL || op2 == BAM_CMATCH || op2 == BAM_CREF_SKIP || op2 == BAM_CEQUAL || op2 == BAM_CDIFF) break; + } + if (l3 > 0) p->indel = l3; + } + } + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + p->qpos = s->y + (pos - s->x); + } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) { + p->is_del = 1; p->qpos = s->y; // FIXME: distinguish D and N!!!!! + p->is_refskip = (op == BAM_CREF_SKIP); + } // cannot be other operations; otherwise a bug + p->is_head = (pos == c->pos); p->is_tail = (pos == s->end); + } + return 1; +} + +/* --- END: Auxiliary functions */ + +/******************* + * pileup iterator * + *******************/ + +struct __bam_plp_t { + mempool_t *mp; + lbnode_t *head, *tail, *dummy; + int32_t tid, pos, max_tid, max_pos; + int is_eof, flag_mask, max_plp, error, maxcnt; + bam_pileup1_t *plp; + // for the "auto" interface only + bam1_t *b; + bam_plp_auto_f func; + void *data; +}; + +bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data) +{ + bam_plp_t iter; + iter = calloc(1, sizeof(struct __bam_plp_t)); + iter->mp = mp_init(); + iter->head = iter->tail = mp_alloc(iter->mp); + iter->dummy = mp_alloc(iter->mp); + iter->max_tid = iter->max_pos = -1; + iter->flag_mask = BAM_DEF_MASK; + iter->maxcnt = 8000; + if (func) { + iter->func = func; + iter->data = data; + iter->b = bam_init1(); + } + return iter; +} + +void bam_plp_destroy(bam_plp_t iter) +{ + mp_free(iter->mp, iter->dummy); + mp_free(iter->mp, iter->head); + if (iter->mp->cnt != 0) + fprintf(stderr, "[bam_plp_destroy] memory leak: %d. Continue anyway.\n", iter->mp->cnt); + mp_destroy(iter->mp); + if (iter->b) bam_destroy1(iter->b); + free(iter->plp); + free(iter); +} + +const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp) +{ + if (iter->error) { *_n_plp = -1; return 0; } + *_n_plp = 0; + if (iter->is_eof && iter->head->next == 0) return 0; + while (iter->is_eof || iter->max_tid > iter->tid || (iter->max_tid == iter->tid && iter->max_pos > iter->pos)) { + int n_plp = 0; + lbnode_t *p, *q; + // write iter->plp at iter->pos + iter->dummy->next = iter->head; + for (p = iter->head, q = iter->dummy; p->next; q = p, p = p->next) { + if (p->b.core.tid < iter->tid || (p->b.core.tid == iter->tid && p->end <= iter->pos)) { // then remove + q->next = p->next; mp_free(iter->mp, p); p = q; + } else if (p->b.core.tid == iter->tid && p->beg <= iter->pos) { // here: p->end > pos; then add to pileup + if (n_plp == iter->max_plp) { // then double the capacity + iter->max_plp = iter->max_plp? iter->max_plp<<1 : 256; + iter->plp = (bam_pileup1_t*)realloc(iter->plp, sizeof(bam_pileup1_t) * iter->max_plp); + } + iter->plp[n_plp].b = &p->b; + if (resolve_cigar2(iter->plp + n_plp, iter->pos, &p->s)) ++n_plp; // actually always true... + } + } + iter->head = iter->dummy->next; // dummy->next may be changed + *_n_plp = n_plp; *_tid = iter->tid; *_pos = iter->pos; + // update iter->tid and iter->pos + if (iter->head->next) { + if (iter->tid > iter->head->b.core.tid) { + fprintf(stderr, "[%s] unsorted input. Pileup aborts.\n", __func__); + iter->error = 1; + *_n_plp = -1; + return 0; + } + } + if (iter->tid < iter->head->b.core.tid) { // come to a new reference sequence + iter->tid = iter->head->b.core.tid; iter->pos = iter->head->beg; // jump to the next reference + } else if (iter->pos < iter->head->beg) { // here: tid == head->b.core.tid + iter->pos = iter->head->beg; // jump to the next position + } else ++iter->pos; // scan contiguously + // return + if (n_plp) return iter->plp; + if (iter->is_eof && iter->head->next == 0) break; + } + return 0; +} + +int bam_plp_push(bam_plp_t iter, const bam1_t *b) +{ + if (iter->error) return -1; + if (b) { + if (b->core.tid < 0) return 0; + if (b->core.flag & iter->flag_mask) return 0; + if (iter->tid == b->core.tid && iter->pos == b->core.pos && iter->mp->cnt > iter->maxcnt) return 0; + bam_copy1(&iter->tail->b, b); + iter->tail->beg = b->core.pos; iter->tail->end = bam_calend(&b->core, bam1_cigar(b)); + iter->tail->s = g_cstate_null; iter->tail->s.end = iter->tail->end - 1; // initialize cstate_t + if (b->core.tid < iter->max_tid) { + fprintf(stderr, "[bam_pileup_core] the input is not sorted (chromosomes out of order)\n"); + iter->error = 1; + return -1; + } + if ((b->core.tid == iter->max_tid) && (iter->tail->beg < iter->max_pos)) { + fprintf(stderr, "[bam_pileup_core] the input is not sorted (reads out of order)\n"); + iter->error = 1; + return -1; + } + iter->max_tid = b->core.tid; iter->max_pos = iter->tail->beg; + if (iter->tail->end > iter->pos || iter->tail->b.core.tid > iter->tid) { + iter->tail->next = mp_alloc(iter->mp); + iter->tail = iter->tail->next; + } + } else iter->is_eof = 1; + return 0; +} + +const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp) +{ + const bam_pileup1_t *plp; + if (iter->func == 0 || iter->error) { *_n_plp = -1; return 0; } + if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp; + else { // no pileup line can be obtained; read alignments + *_n_plp = 0; + if (iter->is_eof) return 0; + while (iter->func(iter->data, iter->b) >= 0) { + if (bam_plp_push(iter, iter->b) < 0) { + *_n_plp = -1; + return 0; + } + if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp; + // otherwise no pileup line can be returned; read the next alignment. + } + bam_plp_push(iter, 0); + if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp; + return 0; + } +} + +void bam_plp_reset(bam_plp_t iter) +{ + lbnode_t *p, *q; + iter->max_tid = iter->max_pos = -1; + iter->tid = iter->pos = 0; + iter->is_eof = 0; + for (p = iter->head; p->next;) { + q = p->next; + mp_free(iter->mp, p); + p = q; + } + iter->head = iter->tail; +} + +void bam_plp_set_mask(bam_plp_t iter, int mask) +{ + iter->flag_mask = mask < 0? BAM_DEF_MASK : (BAM_FUNMAP | mask); +} + +void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt) +{ + iter->maxcnt = maxcnt; +} + +/***************** + * callback APIs * + *****************/ + +int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data) +{ + bam_plbuf_t *buf; + int ret; + bam1_t *b; + b = bam_init1(); + buf = bam_plbuf_init(func, func_data); + bam_plbuf_set_mask(buf, mask); + while ((ret = bam_read1(fp, b)) >= 0) + bam_plbuf_push(b, buf); + bam_plbuf_push(0, buf); + bam_plbuf_destroy(buf); + bam_destroy1(b); + return 0; +} + +void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask) +{ + bam_plp_set_mask(buf->iter, mask); +} + +void bam_plbuf_reset(bam_plbuf_t *buf) +{ + bam_plp_reset(buf->iter); +} + +bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data) +{ + bam_plbuf_t *buf; + buf = calloc(1, sizeof(bam_plbuf_t)); + buf->iter = bam_plp_init(0, 0); + buf->func = func; + buf->data = data; + return buf; +} + +void bam_plbuf_destroy(bam_plbuf_t *buf) +{ + bam_plp_destroy(buf->iter); + free(buf); +} + +int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf) +{ + int ret, n_plp, tid, pos; + const bam_pileup1_t *plp; + ret = bam_plp_push(buf->iter, b); + if (ret < 0) return ret; + while ((plp = bam_plp_next(buf->iter, &tid, &pos, &n_plp)) != 0) + buf->func(tid, pos, n_plp, plp, buf->data); + return 0; +} + +/*********** + * mpileup * + ***********/ + +struct __bam_mplp_t { + int n; + uint64_t min, *pos; + bam_plp_t *iter; + int *n_plp; + const bam_pileup1_t **plp; +}; + +bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data) +{ + int i; + bam_mplp_t iter; + iter = calloc(1, sizeof(struct __bam_mplp_t)); + iter->pos = calloc(n, 8); + iter->n_plp = calloc(n, sizeof(int)); + iter->plp = calloc(n, sizeof(void*)); + iter->iter = calloc(n, sizeof(void*)); + iter->n = n; + iter->min = (uint64_t)-1; + for (i = 0; i < n; ++i) { + iter->iter[i] = bam_plp_init(func, data[i]); + iter->pos[i] = iter->min; + } + return iter; +} + +void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt) +{ + int i; + for (i = 0; i < iter->n; ++i) + iter->iter[i]->maxcnt = maxcnt; +} + +void bam_mplp_destroy(bam_mplp_t iter) +{ + int i; + for (i = 0; i < iter->n; ++i) bam_plp_destroy(iter->iter[i]); + free(iter->iter); free(iter->pos); free(iter->n_plp); free(iter->plp); + free(iter); +} + +int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp) +{ + int i, ret = 0; + uint64_t new_min = (uint64_t)-1; + for (i = 0; i < iter->n; ++i) { + if (iter->pos[i] == iter->min) { + int tid, pos; + iter->plp[i] = bam_plp_auto(iter->iter[i], &tid, &pos, &iter->n_plp[i]); + iter->pos[i] = (uint64_t)tid<<32 | pos; + } + if (iter->plp[i] && iter->pos[i] < new_min) new_min = iter->pos[i]; + } + iter->min = new_min; + if (new_min == (uint64_t)-1) return 0; + *_tid = new_min>>32; *_pos = (uint32_t)new_min; + for (i = 0; i < iter->n; ++i) { + if (iter->pos[i] == iter->min) { // FIXME: valgrind reports "uninitialised value(s) at this line" + n_plp[i] = iter->n_plp[i], plp[i] = iter->plp[i]; + ++ret; + } else n_plp[i] = 0, plp[i] = 0; + } + return ret; +} diff --git a/samtools-0.1.18/bam_reheader.c b/samtools-0.1.18/bam_reheader.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_reheader.c @@ -0,0 +1,61 @@ +#include +#include +#include "bgzf.h" +#include "bam.h" + +#define BUF_SIZE 0x10000 + +int bam_reheader(BGZF *in, const bam_header_t *h, int fd) +{ + BGZF *fp; + bam_header_t *old; + int len; + uint8_t *buf; + if (in->open_mode != 'r') return -1; + buf = malloc(BUF_SIZE); + old = bam_header_read(in); + fp = bgzf_fdopen(fd, "w"); + bam_header_write(fp, h); + if (in->block_offset < in->block_length) { + bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset); + bgzf_flush(fp); + } +#ifdef _USE_KNETFILE + while ((len = knet_read(in->x.fpr, buf, BUF_SIZE)) > 0) + fwrite(buf, 1, len, fp->x.fpw); +#else + while (!feof(in->file) && (len = fread(buf, 1, BUF_SIZE, in->file)) > 0) + fwrite(buf, 1, len, fp->file); +#endif + free(buf); + fp->block_offset = in->block_offset = 0; + bgzf_close(fp); + return 0; +} + +int main_reheader(int argc, char *argv[]) +{ + bam_header_t *h; + BGZF *in; + if (argc != 3) { + fprintf(stderr, "Usage: samtools reheader \n"); + return 1; + } + { // read the header + tamFile fph = sam_open(argv[1]); + if (fph == 0) { + fprintf(stderr, "[%s] fail to read the header from %s.\n", __func__, argv[1]); + return 1; + } + h = sam_header_read(fph); + sam_close(fph); + } + in = strcmp(argv[2], "-")? bam_open(argv[2], "r") : bam_dopen(fileno(stdin), "r"); + if (in == 0) { + fprintf(stderr, "[%s] fail to open file %s.\n", __func__, argv[2]); + return 1; + } + bam_reheader(in, h, fileno(stdout)); + bgzf_close(in); + return 0; +} diff --git a/samtools-0.1.18/bam_sort.c b/samtools-0.1.18/bam_sort.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bam_sort.c @@ -0,0 +1,438 @@ +#include +#include +#include +#include +#include +#include +#include +#include "bam.h" +#include "ksort.h" + +static int g_is_by_qname = 0; + +static inline int strnum_cmp(const char *a, const char *b) +{ + char *pa, *pb; + pa = (char*)a; pb = (char*)b; + while (*pa && *pb) { + if (isdigit(*pa) && isdigit(*pb)) { + long ai, bi; + ai = strtol(pa, &pa, 10); + bi = strtol(pb, &pb, 10); + if (ai != bi) return aibi? 1 : 0; + } else { + if (*pa != *pb) break; + ++pa; ++pb; + } + } + if (*pa == *pb) + return (pa-a) < (pb-b)? -1 : (pa-a) > (pb-b)? 1 : 0; + return *pa<*pb? -1 : *pa>*pb? 1 : 0; +} + +#define HEAP_EMPTY 0xffffffffffffffffull + +typedef struct { + int i; + uint64_t pos, idx; + bam1_t *b; +} heap1_t; + +#define __pos_cmp(a, b) ((a).pos > (b).pos || ((a).pos == (b).pos && ((a).i > (b).i || ((a).i == (b).i && (a).idx > (b).idx)))) + +static inline int heap_lt(const heap1_t a, const heap1_t b) +{ + if (g_is_by_qname) { + int t; + if (a.b == 0 || b.b == 0) return a.b == 0? 1 : 0; + t = strnum_cmp(bam1_qname(a.b), bam1_qname(b.b)); + return (t > 0 || (t == 0 && __pos_cmp(a, b))); + } else return __pos_cmp(a, b); +} + +KSORT_INIT(heap, heap1_t, heap_lt) + +static void swap_header_targets(bam_header_t *h1, bam_header_t *h2) +{ + bam_header_t t; + t.n_targets = h1->n_targets, h1->n_targets = h2->n_targets, h2->n_targets = t.n_targets; + t.target_name = h1->target_name, h1->target_name = h2->target_name, h2->target_name = t.target_name; + t.target_len = h1->target_len, h1->target_len = h2->target_len, h2->target_len = t.target_len; +} + +static void swap_header_text(bam_header_t *h1, bam_header_t *h2) +{ + int tempi; + char *temps; + tempi = h1->l_text, h1->l_text = h2->l_text, h2->l_text = tempi; + temps = h1->text, h1->text = h2->text, h2->text = temps; +} + +#define MERGE_RG 1 +#define MERGE_UNCOMP 2 +#define MERGE_LEVEL1 4 +#define MERGE_FORCE 8 + +/*! + @abstract Merge multiple sorted BAM. + @param is_by_qname whether to sort by query name + @param out output BAM file name + @param headers name of SAM file from which to copy '@' header lines, + or NULL to copy them from the first file to be merged + @param n number of files to be merged + @param fn names of files to be merged + + @discussion Padding information may NOT correctly maintained. This + function is NOT thread safe. + */ +int bam_merge_core(int by_qname, const char *out, const char *headers, int n, char * const *fn, + int flag, const char *reg) +{ + bamFile fpout, *fp; + heap1_t *heap; + bam_header_t *hout = 0; + bam_header_t *hheaders = NULL; + int i, j, *RG_len = 0; + uint64_t idx = 0; + char **RG = 0; + bam_iter_t *iter = 0; + + if (headers) { + tamFile fpheaders = sam_open(headers); + if (fpheaders == 0) { + const char *message = strerror(errno); + fprintf(stderr, "[bam_merge_core] cannot open '%s': %s\n", headers, message); + return -1; + } + hheaders = sam_header_read(fpheaders); + sam_close(fpheaders); + } + + g_is_by_qname = by_qname; + fp = (bamFile*)calloc(n, sizeof(bamFile)); + heap = (heap1_t*)calloc(n, sizeof(heap1_t)); + iter = (bam_iter_t*)calloc(n, sizeof(bam_iter_t)); + // prepare RG tag + if (flag & MERGE_RG) { + RG = (char**)calloc(n, sizeof(void*)); + RG_len = (int*)calloc(n, sizeof(int)); + for (i = 0; i != n; ++i) { + int l = strlen(fn[i]); + const char *s = fn[i]; + if (l > 4 && strcmp(s + l - 4, ".bam") == 0) l -= 4; + for (j = l - 1; j >= 0; --j) if (s[j] == '/') break; + ++j; l -= j; + RG[i] = calloc(l + 1, 1); + RG_len[i] = l; + strncpy(RG[i], s + j, l); + } + } + // read the first + for (i = 0; i != n; ++i) { + bam_header_t *hin; + fp[i] = bam_open(fn[i], "r"); + if (fp[i] == 0) { + int j; + fprintf(stderr, "[bam_merge_core] fail to open file %s\n", fn[i]); + for (j = 0; j < i; ++j) bam_close(fp[j]); + free(fp); free(heap); + // FIXME: possible memory leak + return -1; + } + hin = bam_header_read(fp[i]); + if (i == 0) { // the first BAM + hout = hin; + } else { // validate multiple baf + int min_n_targets = hout->n_targets; + if (hin->n_targets < min_n_targets) min_n_targets = hin->n_targets; + + for (j = 0; j < min_n_targets; ++j) + if (strcmp(hout->target_name[j], hin->target_name[j]) != 0) { + fprintf(stderr, "[bam_merge_core] different target sequence name: '%s' != '%s' in file '%s'\n", + hout->target_name[j], hin->target_name[j], fn[i]); + return -1; + } + + // If this input file has additional target reference sequences, + // add them to the headers to be output + if (hin->n_targets > hout->n_targets) { + swap_header_targets(hout, hin); + // FIXME Possibly we should also create @SQ text headers + // for the newly added reference sequences + } + + bam_header_destroy(hin); + } + } + + if (hheaders) { + // If the text headers to be swapped in include any @SQ headers, + // check that they are consistent with the existing binary list + // of reference information. + if (hheaders->n_targets > 0) { + if (hout->n_targets != hheaders->n_targets) { + fprintf(stderr, "[bam_merge_core] number of @SQ headers in '%s' differs from number of target sequences\n", headers); + if (!reg) return -1; + } + for (j = 0; j < hout->n_targets; ++j) + if (strcmp(hout->target_name[j], hheaders->target_name[j]) != 0) { + fprintf(stderr, "[bam_merge_core] @SQ header '%s' in '%s' differs from target sequence\n", hheaders->target_name[j], headers); + if (!reg) return -1; + } + } + + swap_header_text(hout, hheaders); + bam_header_destroy(hheaders); + } + + if (reg) { + int tid, beg, end; + if (bam_parse_region(hout, reg, &tid, &beg, &end) < 0) { + fprintf(stderr, "[%s] Malformated region string or undefined reference name\n", __func__); + return -1; + } + for (i = 0; i < n; ++i) { + bam_index_t *idx; + idx = bam_index_load(fn[i]); + iter[i] = bam_iter_query(idx, tid, beg, end); + bam_index_destroy(idx); + } + } + + for (i = 0; i < n; ++i) { + heap1_t *h = heap + i; + h->i = i; + h->b = (bam1_t*)calloc(1, sizeof(bam1_t)); + if (bam_iter_read(fp[i], iter[i], h->b) >= 0) { + h->pos = ((uint64_t)h->b->core.tid<<32) | (uint32_t)((int32_t)h->b->core.pos+1)<<1 | bam1_strand(h->b); + h->idx = idx++; + } + else h->pos = HEAP_EMPTY; + } + if (flag & MERGE_UNCOMP) fpout = strcmp(out, "-")? bam_open(out, "wu") : bam_dopen(fileno(stdout), "wu"); + else if (flag & MERGE_LEVEL1) fpout = strcmp(out, "-")? bam_open(out, "w1") : bam_dopen(fileno(stdout), "w1"); + else fpout = strcmp(out, "-")? bam_open(out, "w") : bam_dopen(fileno(stdout), "w"); + if (fpout == 0) { + fprintf(stderr, "[%s] fail to create the output file.\n", __func__); + return -1; + } + bam_header_write(fpout, hout); + bam_header_destroy(hout); + + ks_heapmake(heap, n, heap); + while (heap->pos != HEAP_EMPTY) { + bam1_t *b = heap->b; + if (flag & MERGE_RG) { + uint8_t *rg = bam_aux_get(b, "RG"); + if (rg) bam_aux_del(b, rg); + bam_aux_append(b, "RG", 'Z', RG_len[heap->i] + 1, (uint8_t*)RG[heap->i]); + } + bam_write1_core(fpout, &b->core, b->data_len, b->data); + if ((j = bam_iter_read(fp[heap->i], iter[heap->i], b)) >= 0) { + heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1)<<1 | bam1_strand(b); + heap->idx = idx++; + } else if (j == -1) { + heap->pos = HEAP_EMPTY; + free(heap->b->data); free(heap->b); + heap->b = 0; + } else fprintf(stderr, "[bam_merge_core] '%s' is truncated. Continue anyway.\n", fn[heap->i]); + ks_heapadjust(heap, 0, n, heap); + } + + if (flag & MERGE_RG) { + for (i = 0; i != n; ++i) free(RG[i]); + free(RG); free(RG_len); + } + for (i = 0; i != n; ++i) { + bam_iter_destroy(iter[i]); + bam_close(fp[i]); + } + bam_close(fpout); + free(fp); free(heap); free(iter); + return 0; +} + +int bam_merge(int argc, char *argv[]) +{ + int c, is_by_qname = 0, flag = 0, ret = 0; + char *fn_headers = NULL, *reg = 0; + + while ((c = getopt(argc, argv, "h:nru1R:f")) >= 0) { + switch (c) { + case 'r': flag |= MERGE_RG; break; + case 'f': flag |= MERGE_FORCE; break; + case 'h': fn_headers = strdup(optarg); break; + case 'n': is_by_qname = 1; break; + case '1': flag |= MERGE_LEVEL1; break; + case 'u': flag |= MERGE_UNCOMP; break; + case 'R': reg = strdup(optarg); break; + } + } + if (optind + 2 >= argc) { + fprintf(stderr, "\n"); + fprintf(stderr, "Usage: samtools merge [-nr] [-h inh.sam] [...]\n\n"); + fprintf(stderr, "Options: -n sort by read names\n"); + fprintf(stderr, " -r attach RG tag (inferred from file names)\n"); + fprintf(stderr, " -u uncompressed BAM output\n"); + fprintf(stderr, " -f overwrite the output BAM if exist\n"); + fprintf(stderr, " -1 compress level 1\n"); + fprintf(stderr, " -R STR merge file in the specified region STR [all]\n"); + fprintf(stderr, " -h FILE copy the header in FILE to [in1.bam]\n\n"); + fprintf(stderr, "Note: Samtools' merge does not reconstruct the @RG dictionary in the header. Users\n"); + fprintf(stderr, " must provide the correct header with -h, or uses Picard which properly maintains\n"); + fprintf(stderr, " the header dictionary in merging.\n\n"); + return 1; + } + if (!(flag & MERGE_FORCE) && strcmp(argv[optind], "-")) { + FILE *fp = fopen(argv[optind], "rb"); + if (fp != NULL) { + fclose(fp); + fprintf(stderr, "[%s] File '%s' exists. Please apply '-f' to overwrite. Abort.\n", __func__, argv[optind]); + return 1; + } + } + if (bam_merge_core(is_by_qname, argv[optind], fn_headers, argc - optind - 1, argv + optind + 1, flag, reg) < 0) ret = 1; + free(reg); + free(fn_headers); + return ret; +} + +typedef bam1_t *bam1_p; + +static inline int bam1_lt(const bam1_p a, const bam1_p b) +{ + if (g_is_by_qname) { + int t = strnum_cmp(bam1_qname(a), bam1_qname(b)); + return (t < 0 || (t == 0 && (((uint64_t)a->core.tid<<32|(a->core.pos+1)) < ((uint64_t)b->core.tid<<32|(b->core.pos+1))))); + } else return (((uint64_t)a->core.tid<<32|(a->core.pos+1)) < ((uint64_t)b->core.tid<<32|(b->core.pos+1))); +} +KSORT_INIT(sort, bam1_p, bam1_lt) + +static void sort_blocks(int n, int k, bam1_p *buf, const char *prefix, const bam_header_t *h, int is_stdout) +{ + char *name, mode[3]; + int i; + bamFile fp; + ks_mergesort(sort, k, buf, 0); + name = (char*)calloc(strlen(prefix) + 20, 1); + if (n >= 0) { + sprintf(name, "%s.%.4d.bam", prefix, n); + strcpy(mode, "w1"); + } else { + sprintf(name, "%s.bam", prefix); + strcpy(mode, "w"); + } + fp = is_stdout? bam_dopen(fileno(stdout), mode) : bam_open(name, mode); + if (fp == 0) { + fprintf(stderr, "[sort_blocks] fail to create file %s.\n", name); + free(name); + // FIXME: possible memory leak + return; + } + free(name); + bam_header_write(fp, h); + for (i = 0; i < k; ++i) + bam_write1_core(fp, &buf[i]->core, buf[i]->data_len, buf[i]->data); + bam_close(fp); +} + +/*! + @abstract Sort an unsorted BAM file based on the chromosome order + and the leftmost position of an alignment + + @param is_by_qname whether to sort by query name + @param fn name of the file to be sorted + @param prefix prefix of the output and the temporary files; upon + sucessess, prefix.bam will be written. + @param max_mem approxiate maximum memory (very inaccurate) + + @discussion It may create multiple temporary subalignment files + and then merge them by calling bam_merge_core(). This function is + NOT thread safe. + */ +void bam_sort_core_ext(int is_by_qname, const char *fn, const char *prefix, size_t max_mem, int is_stdout) +{ + int n, ret, k, i; + size_t mem; + bam_header_t *header; + bamFile fp; + bam1_t *b, **buf; + + g_is_by_qname = is_by_qname; + n = k = 0; mem = 0; + fp = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r"); + if (fp == 0) { + fprintf(stderr, "[bam_sort_core] fail to open file %s\n", fn); + return; + } + header = bam_header_read(fp); + buf = (bam1_t**)calloc(max_mem / BAM_CORE_SIZE, sizeof(bam1_t*)); + // write sub files + for (;;) { + if (buf[k] == 0) buf[k] = (bam1_t*)calloc(1, sizeof(bam1_t)); + b = buf[k]; + if ((ret = bam_read1(fp, b)) < 0) break; + mem += ret; + ++k; + if (mem >= max_mem) { + sort_blocks(n++, k, buf, prefix, header, 0); + mem = 0; k = 0; + } + } + if (ret != -1) + fprintf(stderr, "[bam_sort_core] truncated file. Continue anyway.\n"); + if (n == 0) sort_blocks(-1, k, buf, prefix, header, is_stdout); + else { // then merge + char **fns, *fnout; + fprintf(stderr, "[bam_sort_core] merging from %d files...\n", n+1); + sort_blocks(n++, k, buf, prefix, header, 0); + fnout = (char*)calloc(strlen(prefix) + 20, 1); + if (is_stdout) sprintf(fnout, "-"); + else sprintf(fnout, "%s.bam", prefix); + fns = (char**)calloc(n, sizeof(char*)); + for (i = 0; i < n; ++i) { + fns[i] = (char*)calloc(strlen(prefix) + 20, 1); + sprintf(fns[i], "%s.%.4d.bam", prefix, i); + } + bam_merge_core(is_by_qname, fnout, 0, n, fns, 0, 0); + free(fnout); + for (i = 0; i < n; ++i) { + unlink(fns[i]); + free(fns[i]); + } + free(fns); + } + for (k = 0; k < max_mem / BAM_CORE_SIZE; ++k) { + if (buf[k]) { + free(buf[k]->data); + free(buf[k]); + } + } + free(buf); + bam_header_destroy(header); + bam_close(fp); +} + +void bam_sort_core(int is_by_qname, const char *fn, const char *prefix, size_t max_mem) +{ + bam_sort_core_ext(is_by_qname, fn, prefix, max_mem, 0); +} + +int bam_sort(int argc, char *argv[]) +{ + size_t max_mem = 500000000; + int c, is_by_qname = 0, is_stdout = 0; + while ((c = getopt(argc, argv, "nom:")) >= 0) { + switch (c) { + case 'o': is_stdout = 1; break; + case 'n': is_by_qname = 1; break; + case 'm': max_mem = atol(optarg); break; + } + } + if (optind + 2 > argc) { + fprintf(stderr, "Usage: samtools sort [-on] [-m ] \n"); + return 1; + } + bam_sort_core_ext(is_by_qname, argv[optind], argv[optind+1], max_mem, is_stdout); + return 0; +} diff --git a/samtools-0.1.18/bgzf.c b/samtools-0.1.18/bgzf.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bgzf.c @@ -0,0 +1,714 @@ +/* The MIT License + + Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +/* + 2009-06-29 by lh3: cache recent uncompressed blocks. + 2009-06-25 by lh3: optionally use my knetfile library to access file on a FTP. + 2009-06-12 by lh3: support a mode string like "wu" where 'u' for uncompressed output */ + +#include +#include +#include +#include +#include +#include +#include +#include "bgzf.h" + +#include "khash.h" +typedef struct { + int size; + uint8_t *block; + int64_t end_offset; +} cache_t; +KHASH_MAP_INIT_INT64(cache, cache_t) + +#if defined(_WIN32) || defined(_MSC_VER) +#define ftello(fp) ftell(fp) +#define fseeko(fp, offset, whence) fseek(fp, offset, whence) +#else +extern off_t ftello(FILE *stream); +extern int fseeko(FILE *stream, off_t offset, int whence); +#endif + +typedef int8_t bgzf_byte_t; + +static const int DEFAULT_BLOCK_SIZE = 64 * 1024; +static const int MAX_BLOCK_SIZE = 64 * 1024; + +static const int BLOCK_HEADER_LENGTH = 18; +static const int BLOCK_FOOTER_LENGTH = 8; + +static const int GZIP_ID1 = 31; +static const int GZIP_ID2 = 139; +static const int CM_DEFLATE = 8; +static const int FLG_FEXTRA = 4; +static const int OS_UNKNOWN = 255; +static const int BGZF_ID1 = 66; // 'B' +static const int BGZF_ID2 = 67; // 'C' +static const int BGZF_LEN = 2; +static const int BGZF_XLEN = 6; // BGZF_LEN+4 + +static const int GZIP_WINDOW_BITS = -15; // no zlib header +static const int Z_DEFAULT_MEM_LEVEL = 8; + + +inline +void +packInt16(uint8_t* buffer, uint16_t value) +{ + buffer[0] = value; + buffer[1] = value >> 8; +} + +inline +int +unpackInt16(const uint8_t* buffer) +{ + return (buffer[0] | (buffer[1] << 8)); +} + +inline +void +packInt32(uint8_t* buffer, uint32_t value) +{ + buffer[0] = value; + buffer[1] = value >> 8; + buffer[2] = value >> 16; + buffer[3] = value >> 24; +} + +static inline +int +bgzf_min(int x, int y) +{ + return (x < y) ? x : y; +} + +static +void +report_error(BGZF* fp, const char* message) { + fp->error = message; +} + +int bgzf_check_bgzf(const char *fn) +{ + BGZF *fp; + uint8_t buf[10],magic[10]="\037\213\010\4\0\0\0\0\0\377"; + int n; + + if ((fp = bgzf_open(fn, "r")) == 0) + { + fprintf(stderr, "[bgzf_check_bgzf] failed to open the file: %s\n",fn); + return -1; + } + +#ifdef _USE_KNETFILE + n = knet_read(fp->x.fpr, buf, 10); +#else + n = fread(buf, 1, 10, fp->file); +#endif + bgzf_close(fp); + + if ( n!=10 ) + return -1; + + if ( !memcmp(magic, buf, 10) ) return 1; + return 0; +} + +static BGZF *bgzf_read_init() +{ + BGZF *fp; + fp = calloc(1, sizeof(BGZF)); + fp->uncompressed_block_size = MAX_BLOCK_SIZE; + fp->uncompressed_block = malloc(MAX_BLOCK_SIZE); + fp->compressed_block_size = MAX_BLOCK_SIZE; + fp->compressed_block = malloc(MAX_BLOCK_SIZE); + fp->cache_size = 0; + fp->cache = kh_init(cache); + return fp; +} + +static +BGZF* +open_read(int fd) +{ +#ifdef _USE_KNETFILE + knetFile *file = knet_dopen(fd, "r"); +#else + FILE* file = fdopen(fd, "r"); +#endif + BGZF* fp; + if (file == 0) return 0; + fp = bgzf_read_init(); + fp->file_descriptor = fd; + fp->open_mode = 'r'; +#ifdef _USE_KNETFILE + fp->x.fpr = file; +#else + fp->file = file; +#endif + return fp; +} + +static +BGZF* +open_write(int fd, int compress_level) // compress_level==-1 for the default level +{ + FILE* file = fdopen(fd, "w"); + BGZF* fp; + if (file == 0) return 0; + fp = malloc(sizeof(BGZF)); + fp->file_descriptor = fd; + fp->open_mode = 'w'; + fp->owned_file = 0; + fp->compress_level = compress_level < 0? Z_DEFAULT_COMPRESSION : compress_level; // Z_DEFAULT_COMPRESSION==-1 + if (fp->compress_level > 9) fp->compress_level = Z_DEFAULT_COMPRESSION; +#ifdef _USE_KNETFILE + fp->x.fpw = file; +#else + fp->file = file; +#endif + fp->uncompressed_block_size = DEFAULT_BLOCK_SIZE; + fp->uncompressed_block = NULL; + fp->compressed_block_size = MAX_BLOCK_SIZE; + fp->compressed_block = malloc(MAX_BLOCK_SIZE); + fp->block_address = 0; + fp->block_offset = 0; + fp->block_length = 0; + fp->error = NULL; + return fp; +} + +BGZF* +bgzf_open(const char* __restrict path, const char* __restrict mode) +{ + BGZF* fp = NULL; + if (strchr(mode, 'r') || strchr(mode, 'R')) { /* The reading mode is preferred. */ +#ifdef _USE_KNETFILE + knetFile *file = knet_open(path, mode); + if (file == 0) return 0; + fp = bgzf_read_init(); + fp->file_descriptor = -1; + fp->open_mode = 'r'; + fp->x.fpr = file; +#else + int fd, oflag = O_RDONLY; +#ifdef _WIN32 + oflag |= O_BINARY; +#endif + fd = open(path, oflag); + if (fd == -1) return 0; + fp = open_read(fd); +#endif + } else if (strchr(mode, 'w') || strchr(mode, 'W')) { + int fd, compress_level = -1, oflag = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + oflag |= O_BINARY; +#endif + fd = open(path, oflag, 0666); + if (fd == -1) return 0; + { // set compress_level + int i; + for (i = 0; mode[i]; ++i) + if (mode[i] >= '0' && mode[i] <= '9') break; + if (mode[i]) compress_level = (int)mode[i] - '0'; + if (strchr(mode, 'u')) compress_level = 0; + } + fp = open_write(fd, compress_level); + } + if (fp != NULL) fp->owned_file = 1; + return fp; +} + +BGZF* +bgzf_fdopen(int fd, const char * __restrict mode) +{ + if (fd == -1) return 0; + if (mode[0] == 'r' || mode[0] == 'R') { + return open_read(fd); + } else if (mode[0] == 'w' || mode[0] == 'W') { + int i, compress_level = -1; + for (i = 0; mode[i]; ++i) + if (mode[i] >= '0' && mode[i] <= '9') break; + if (mode[i]) compress_level = (int)mode[i] - '0'; + if (strchr(mode, 'u')) compress_level = 0; + return open_write(fd, compress_level); + } else { + return NULL; + } +} + +static +int +deflate_block(BGZF* fp, int block_length) +{ + // Deflate the block in fp->uncompressed_block into fp->compressed_block. + // Also adds an extra field that stores the compressed block length. + + bgzf_byte_t* buffer = fp->compressed_block; + int buffer_size = fp->compressed_block_size; + + // Init gzip header + buffer[0] = GZIP_ID1; + buffer[1] = GZIP_ID2; + buffer[2] = CM_DEFLATE; + buffer[3] = FLG_FEXTRA; + buffer[4] = 0; // mtime + buffer[5] = 0; + buffer[6] = 0; + buffer[7] = 0; + buffer[8] = 0; + buffer[9] = OS_UNKNOWN; + buffer[10] = BGZF_XLEN; + buffer[11] = 0; + buffer[12] = BGZF_ID1; + buffer[13] = BGZF_ID2; + buffer[14] = BGZF_LEN; + buffer[15] = 0; + buffer[16] = 0; // placeholder for block length + buffer[17] = 0; + + // loop to retry for blocks that do not compress enough + int input_length = block_length; + int compressed_length = 0; + while (1) { + z_stream zs; + zs.zalloc = NULL; + zs.zfree = NULL; + zs.next_in = fp->uncompressed_block; + zs.avail_in = input_length; + zs.next_out = (void*)&buffer[BLOCK_HEADER_LENGTH]; + zs.avail_out = buffer_size - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH; + + int status = deflateInit2(&zs, fp->compress_level, Z_DEFLATED, + GZIP_WINDOW_BITS, Z_DEFAULT_MEM_LEVEL, Z_DEFAULT_STRATEGY); + if (status != Z_OK) { + report_error(fp, "deflate init failed"); + return -1; + } + status = deflate(&zs, Z_FINISH); + if (status != Z_STREAM_END) { + deflateEnd(&zs); + if (status == Z_OK) { + // Not enough space in buffer. + // Can happen in the rare case the input doesn't compress enough. + // Reduce the amount of input until it fits. + input_length -= 1024; + if (input_length <= 0) { + // should never happen + report_error(fp, "input reduction failed"); + return -1; + } + continue; + } + report_error(fp, "deflate failed"); + return -1; + } + status = deflateEnd(&zs); + if (status != Z_OK) { + report_error(fp, "deflate end failed"); + return -1; + } + compressed_length = zs.total_out; + compressed_length += BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH; + if (compressed_length > MAX_BLOCK_SIZE) { + // should never happen + report_error(fp, "deflate overflow"); + return -1; + } + break; + } + + packInt16((uint8_t*)&buffer[16], compressed_length-1); + uint32_t crc = crc32(0L, NULL, 0L); + crc = crc32(crc, fp->uncompressed_block, input_length); + packInt32((uint8_t*)&buffer[compressed_length-8], crc); + packInt32((uint8_t*)&buffer[compressed_length-4], input_length); + + int remaining = block_length - input_length; + if (remaining > 0) { + if (remaining > input_length) { + // should never happen (check so we can use memcpy) + report_error(fp, "remainder too large"); + return -1; + } + memcpy(fp->uncompressed_block, + fp->uncompressed_block + input_length, + remaining); + } + fp->block_offset = remaining; + return compressed_length; +} + +static +int +inflate_block(BGZF* fp, int block_length) +{ + // Inflate the block in fp->compressed_block into fp->uncompressed_block + + z_stream zs; + int status; + zs.zalloc = NULL; + zs.zfree = NULL; + zs.next_in = fp->compressed_block + 18; + zs.avail_in = block_length - 16; + zs.next_out = fp->uncompressed_block; + zs.avail_out = fp->uncompressed_block_size; + + status = inflateInit2(&zs, GZIP_WINDOW_BITS); + if (status != Z_OK) { + report_error(fp, "inflate init failed"); + return -1; + } + status = inflate(&zs, Z_FINISH); + if (status != Z_STREAM_END) { + inflateEnd(&zs); + report_error(fp, "inflate failed"); + return -1; + } + status = inflateEnd(&zs); + if (status != Z_OK) { + report_error(fp, "inflate failed"); + return -1; + } + return zs.total_out; +} + +static +int +check_header(const bgzf_byte_t* header) +{ + return (header[0] == GZIP_ID1 && + header[1] == (bgzf_byte_t) GZIP_ID2 && + header[2] == Z_DEFLATED && + (header[3] & FLG_FEXTRA) != 0 && + unpackInt16((uint8_t*)&header[10]) == BGZF_XLEN && + header[12] == BGZF_ID1 && + header[13] == BGZF_ID2 && + unpackInt16((uint8_t*)&header[14]) == BGZF_LEN); +} + +static void free_cache(BGZF *fp) +{ + khint_t k; + khash_t(cache) *h = (khash_t(cache)*)fp->cache; + if (fp->open_mode != 'r') return; + for (k = kh_begin(h); k < kh_end(h); ++k) + if (kh_exist(h, k)) free(kh_val(h, k).block); + kh_destroy(cache, h); +} + +static int load_block_from_cache(BGZF *fp, int64_t block_address) +{ + khint_t k; + cache_t *p; + khash_t(cache) *h = (khash_t(cache)*)fp->cache; + k = kh_get(cache, h, block_address); + if (k == kh_end(h)) return 0; + p = &kh_val(h, k); + if (fp->block_length != 0) fp->block_offset = 0; + fp->block_address = block_address; + fp->block_length = p->size; + memcpy(fp->uncompressed_block, p->block, MAX_BLOCK_SIZE); +#ifdef _USE_KNETFILE + knet_seek(fp->x.fpr, p->end_offset, SEEK_SET); +#else + fseeko(fp->file, p->end_offset, SEEK_SET); +#endif + return p->size; +} + +static void cache_block(BGZF *fp, int size) +{ + int ret; + khint_t k; + cache_t *p; + khash_t(cache) *h = (khash_t(cache)*)fp->cache; + if (MAX_BLOCK_SIZE >= fp->cache_size) return; + if ((kh_size(h) + 1) * MAX_BLOCK_SIZE > fp->cache_size) { + /* A better way would be to remove the oldest block in the + * cache, but here we remove a random one for simplicity. This + * should not have a big impact on performance. */ + for (k = kh_begin(h); k < kh_end(h); ++k) + if (kh_exist(h, k)) break; + if (k < kh_end(h)) { + free(kh_val(h, k).block); + kh_del(cache, h, k); + } + } + k = kh_put(cache, h, fp->block_address, &ret); + if (ret == 0) return; // if this happens, a bug! + p = &kh_val(h, k); + p->size = fp->block_length; + p->end_offset = fp->block_address + size; + p->block = malloc(MAX_BLOCK_SIZE); + memcpy(kh_val(h, k).block, fp->uncompressed_block, MAX_BLOCK_SIZE); +} + +int +bgzf_read_block(BGZF* fp) +{ + bgzf_byte_t header[BLOCK_HEADER_LENGTH]; + int count, size = 0, block_length, remaining; +#ifdef _USE_KNETFILE + int64_t block_address = knet_tell(fp->x.fpr); + if (load_block_from_cache(fp, block_address)) return 0; + count = knet_read(fp->x.fpr, header, sizeof(header)); +#else + int64_t block_address = ftello(fp->file); + if (load_block_from_cache(fp, block_address)) return 0; + count = fread(header, 1, sizeof(header), fp->file); +#endif + if (count == 0) { + fp->block_length = 0; + return 0; + } + size = count; + if (count != sizeof(header)) { + report_error(fp, "read failed"); + return -1; + } + if (!check_header(header)) { + report_error(fp, "invalid block header"); + return -1; + } + block_length = unpackInt16((uint8_t*)&header[16]) + 1; + bgzf_byte_t* compressed_block = (bgzf_byte_t*) fp->compressed_block; + memcpy(compressed_block, header, BLOCK_HEADER_LENGTH); + remaining = block_length - BLOCK_HEADER_LENGTH; +#ifdef _USE_KNETFILE + count = knet_read(fp->x.fpr, &compressed_block[BLOCK_HEADER_LENGTH], remaining); +#else + count = fread(&compressed_block[BLOCK_HEADER_LENGTH], 1, remaining, fp->file); +#endif + if (count != remaining) { + report_error(fp, "read failed"); + return -1; + } + size += count; + count = inflate_block(fp, block_length); + if (count < 0) return -1; + if (fp->block_length != 0) { + // Do not reset offset if this read follows a seek. + fp->block_offset = 0; + } + fp->block_address = block_address; + fp->block_length = count; + cache_block(fp, size); + return 0; +} + +int +bgzf_read(BGZF* fp, void* data, int length) +{ + if (length <= 0) { + return 0; + } + if (fp->open_mode != 'r') { + report_error(fp, "file not open for reading"); + return -1; + } + + int bytes_read = 0; + bgzf_byte_t* output = data; + while (bytes_read < length) { + int copy_length, available = fp->block_length - fp->block_offset; + bgzf_byte_t *buffer; + if (available <= 0) { + if (bgzf_read_block(fp) != 0) { + return -1; + } + available = fp->block_length - fp->block_offset; + if (available <= 0) { + break; + } + } + copy_length = bgzf_min(length-bytes_read, available); + buffer = fp->uncompressed_block; + memcpy(output, buffer + fp->block_offset, copy_length); + fp->block_offset += copy_length; + output += copy_length; + bytes_read += copy_length; + } + if (fp->block_offset == fp->block_length) { +#ifdef _USE_KNETFILE + fp->block_address = knet_tell(fp->x.fpr); +#else + fp->block_address = ftello(fp->file); +#endif + fp->block_offset = 0; + fp->block_length = 0; + } + return bytes_read; +} + +int bgzf_flush(BGZF* fp) +{ + while (fp->block_offset > 0) { + int count, block_length; + block_length = deflate_block(fp, fp->block_offset); + if (block_length < 0) return -1; +#ifdef _USE_KNETFILE + count = fwrite(fp->compressed_block, 1, block_length, fp->x.fpw); +#else + count = fwrite(fp->compressed_block, 1, block_length, fp->file); +#endif + if (count != block_length) { + report_error(fp, "write failed"); + return -1; + } + fp->block_address += block_length; + } + return 0; +} + +int bgzf_flush_try(BGZF *fp, int size) +{ + if (fp->block_offset + size > fp->uncompressed_block_size) + return bgzf_flush(fp); + return -1; +} + +int bgzf_write(BGZF* fp, const void* data, int length) +{ + const bgzf_byte_t *input = data; + int block_length, bytes_written; + if (fp->open_mode != 'w') { + report_error(fp, "file not open for writing"); + return -1; + } + + if (fp->uncompressed_block == NULL) + fp->uncompressed_block = malloc(fp->uncompressed_block_size); + + input = data; + block_length = fp->uncompressed_block_size; + bytes_written = 0; + while (bytes_written < length) { + int copy_length = bgzf_min(block_length - fp->block_offset, length - bytes_written); + bgzf_byte_t* buffer = fp->uncompressed_block; + memcpy(buffer + fp->block_offset, input, copy_length); + fp->block_offset += copy_length; + input += copy_length; + bytes_written += copy_length; + if (fp->block_offset == block_length) { + if (bgzf_flush(fp) != 0) { + break; + } + } + } + return bytes_written; +} + +int bgzf_close(BGZF* fp) +{ + if (fp->open_mode == 'w') { + if (bgzf_flush(fp) != 0) return -1; + { // add an empty block + int count, block_length = deflate_block(fp, 0); +#ifdef _USE_KNETFILE + count = fwrite(fp->compressed_block, 1, block_length, fp->x.fpw); +#else + count = fwrite(fp->compressed_block, 1, block_length, fp->file); +#endif + } +#ifdef _USE_KNETFILE + if (fflush(fp->x.fpw) != 0) { +#else + if (fflush(fp->file) != 0) { +#endif + report_error(fp, "flush failed"); + return -1; + } + } + if (fp->owned_file) { +#ifdef _USE_KNETFILE + int ret; + if (fp->open_mode == 'w') ret = fclose(fp->x.fpw); + else ret = knet_close(fp->x.fpr); + if (ret != 0) return -1; +#else + if (fclose(fp->file) != 0) return -1; +#endif + } + free(fp->uncompressed_block); + free(fp->compressed_block); + free_cache(fp); + free(fp); + return 0; +} + +void bgzf_set_cache_size(BGZF *fp, int cache_size) +{ + if (fp) fp->cache_size = cache_size; +} + +int bgzf_check_EOF(BGZF *fp) +{ + static uint8_t magic[28] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0"; + uint8_t buf[28]; + off_t offset; +#ifdef _USE_KNETFILE + offset = knet_tell(fp->x.fpr); + if (knet_seek(fp->x.fpr, -28, SEEK_END) != 0) return -1; + knet_read(fp->x.fpr, buf, 28); + knet_seek(fp->x.fpr, offset, SEEK_SET); +#else + offset = ftello(fp->file); + if (fseeko(fp->file, -28, SEEK_END) != 0) return -1; + fread(buf, 1, 28, fp->file); + fseeko(fp->file, offset, SEEK_SET); +#endif + return (memcmp(magic, buf, 28) == 0)? 1 : 0; +} + +int64_t bgzf_seek(BGZF* fp, int64_t pos, int where) +{ + int block_offset; + int64_t block_address; + + if (fp->open_mode != 'r') { + report_error(fp, "file not open for read"); + return -1; + } + if (where != SEEK_SET) { + report_error(fp, "unimplemented seek option"); + return -1; + } + block_offset = pos & 0xFFFF; + block_address = (pos >> 16) & 0xFFFFFFFFFFFFLL; +#ifdef _USE_KNETFILE + if (knet_seek(fp->x.fpr, block_address, SEEK_SET) != 0) { +#else + if (fseeko(fp->file, block_address, SEEK_SET) != 0) { +#endif + report_error(fp, "seek failed"); + return -1; + } + fp->block_length = 0; // indicates current block is not loaded + fp->block_address = block_address; + fp->block_offset = block_offset; + return 0; +} diff --git a/samtools-0.1.18/bgzf.h b/samtools-0.1.18/bgzf.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/bgzf.h @@ -0,0 +1,157 @@ +/* The MIT License + + Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#ifndef __BGZF_H +#define __BGZF_H + +#include +#include +#include +#ifdef _USE_KNETFILE +#include "knetfile.h" +#endif + +//typedef int8_t bool; + +typedef struct { + int file_descriptor; + char open_mode; // 'r' or 'w' + int16_t owned_file, compress_level; +#ifdef _USE_KNETFILE + union { + knetFile *fpr; + FILE *fpw; + } x; +#else + FILE* file; +#endif + int uncompressed_block_size; + int compressed_block_size; + void* uncompressed_block; + void* compressed_block; + int64_t block_address; + int block_length; + int block_offset; + int cache_size; + const char* error; + void *cache; // a pointer to a hash table +} BGZF; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Open an existing file descriptor for reading or writing. + * Mode must be either "r" or "w". + * A subsequent bgzf_close will not close the file descriptor. + * Returns null on error. + */ +BGZF* bgzf_fdopen(int fd, const char* __restrict mode); + +/* + * Open the specified file for reading or writing. + * Mode must be either "r" or "w". + * Returns null on error. + */ +BGZF* bgzf_open(const char* path, const char* __restrict mode); + +/* + * Close the BGZ file and free all associated resources. + * Does not close the underlying file descriptor if created with bgzf_fdopen. + * Returns zero on success, -1 on error. + */ +int bgzf_close(BGZF* fp); + +/* + * Read up to length bytes from the file storing into data. + * Returns the number of bytes actually read. + * Returns zero on end of file. + * Returns -1 on error. + */ +int bgzf_read(BGZF* fp, void* data, int length); + +/* + * Write length bytes from data to the file. + * Returns the number of bytes written. + * Returns -1 on error. + */ +int bgzf_write(BGZF* fp, const void* data, int length); + +/* + * Return a virtual file pointer to the current location in the file. + * No interpetation of the value should be made, other than a subsequent + * call to bgzf_seek can be used to position the file at the same point. + * Return value is non-negative on success. + * Returns -1 on error. + */ +#define bgzf_tell(fp) ((fp->block_address << 16) | (fp->block_offset & 0xFFFF)) + +/* + * Set the file to read from the location specified by pos, which must + * be a value previously returned by bgzf_tell for this file (but not + * necessarily one returned by this file handle). + * The where argument must be SEEK_SET. + * Seeking on a file opened for write is not supported. + * Returns zero on success, -1 on error. + */ +int64_t bgzf_seek(BGZF* fp, int64_t pos, int where); + +/* + * Set the cache size. Zero to disable. By default, caching is + * disabled. The recommended cache size for frequent random access is + * about 8M bytes. + */ +void bgzf_set_cache_size(BGZF *fp, int cache_size); + +int bgzf_check_EOF(BGZF *fp); +int bgzf_read_block(BGZF* fp); +int bgzf_flush(BGZF* fp); +int bgzf_flush_try(BGZF *fp, int size); +int bgzf_check_bgzf(const char *fn); + +#ifdef __cplusplus +} +#endif + +static inline int bgzf_getc(BGZF *fp) +{ + int c; + if (fp->block_offset >= fp->block_length) { + if (bgzf_read_block(fp) != 0) return -2; /* error */ + if (fp->block_length == 0) return -1; /* end-of-file */ + } + c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++]; + if (fp->block_offset == fp->block_length) { +#ifdef _USE_KNETFILE + fp->block_address = knet_tell(fp->x.fpr); +#else + fp->block_address = ftello(fp->file); +#endif + fp->block_offset = 0; + fp->block_length = 0; + } + return c; +} + +#endif diff --git a/samtools-0.1.18/errmod.c b/samtools-0.1.18/errmod.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/errmod.c @@ -0,0 +1,130 @@ +#include +#include "errmod.h" +#include "ksort.h" +KSORT_INIT_GENERIC(uint16_t) + +typedef struct __errmod_coef_t { + double *fk, *beta, *lhet; +} errmod_coef_t; + +typedef struct { + double fsum[16], bsum[16]; + uint32_t c[16]; +} call_aux_t; + +static errmod_coef_t *cal_coef(double depcorr, double eta) +{ + int k, n, q; + long double sum, sum1; + double *lC; + errmod_coef_t *ec; + + ec = calloc(1, sizeof(errmod_coef_t)); + // initialize ->fk + ec->fk = (double*)calloc(256, sizeof(double)); + ec->fk[0] = 1.0; + for (n = 1; n != 256; ++n) + ec->fk[n] = pow(1. - depcorr, n) * (1.0 - eta) + eta; + // initialize ->coef + ec->beta = (double*)calloc(256 * 256 * 64, sizeof(double)); + lC = (double*)calloc(256 * 256, sizeof(double)); + for (n = 1; n != 256; ++n) { + double lgn = lgamma(n+1); + for (k = 1; k <= n; ++k) + lC[n<<8|k] = lgn - lgamma(k+1) - lgamma(n-k+1); + } + for (q = 1; q != 64; ++q) { + double e = pow(10.0, -q/10.0); + double le = log(e); + double le1 = log(1.0 - e); + for (n = 1; n <= 255; ++n) { + double *beta = ec->beta + (q<<16|n<<8); + sum1 = sum = 0.0; + for (k = n; k >= 0; --k, sum1 = sum) { + sum = sum1 + expl(lC[n<<8|k] + k*le + (n-k)*le1); + beta[k] = -10. / M_LN10 * logl(sum1 / sum); + } + } + } + // initialize ->lhet + ec->lhet = (double*)calloc(256 * 256, sizeof(double)); + for (n = 0; n < 256; ++n) + for (k = 0; k < 256; ++k) + ec->lhet[n<<8|k] = lC[n<<8|k] - M_LN2 * n; + free(lC); + return ec; +} + +errmod_t *errmod_init(float depcorr) +{ + errmod_t *em; + em = (errmod_t*)calloc(1, sizeof(errmod_t)); + em->depcorr = depcorr; + em->coef = cal_coef(depcorr, 0.03); + return em; +} + +void errmod_destroy(errmod_t *em) +{ + if (em == 0) return; + free(em->coef->lhet); free(em->coef->fk); free(em->coef->beta); + free(em->coef); free(em); +} +// qual:6, strand:1, base:4 +int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q) +{ + call_aux_t aux; + int i, j, k, w[32]; + + if (m > m) return -1; + memset(q, 0, m * m * sizeof(float)); + if (n == 0) return 0; + // calculate aux.esum and aux.fsum + if (n > 255) { // then sample 255 bases + ks_shuffle(uint16_t, n, bases); + n = 255; + } + ks_introsort(uint16_t, n, bases); + memset(w, 0, 32 * sizeof(int)); + memset(&aux, 0, sizeof(call_aux_t)); + for (j = n - 1; j >= 0; --j) { // calculate esum and fsum + uint16_t b = bases[j]; + int q = b>>5 < 4? 4 : b>>5; + if (q > 63) q = 63; + k = b&0x1f; + aux.fsum[k&0xf] += em->coef->fk[w[k]]; + aux.bsum[k&0xf] += em->coef->fk[w[k]] * em->coef->beta[q<<16|n<<8|aux.c[k&0xf]]; + ++aux.c[k&0xf]; + ++w[k]; + } + // generate likelihood + for (j = 0; j != m; ++j) { + float tmp1, tmp3; + int tmp2, bar_e; + // homozygous + for (k = 0, tmp1 = tmp3 = 0.0, tmp2 = 0; k != m; ++k) { + if (k == j) continue; + tmp1 += aux.bsum[k]; tmp2 += aux.c[k]; tmp3 += aux.fsum[k]; + } + if (tmp2) { + bar_e = (int)(tmp1 / tmp3 + 0.499); + if (bar_e > 63) bar_e = 63; + q[j*m+j] = tmp1; + } + // heterozygous + for (k = j + 1; k < m; ++k) { + int cjk = aux.c[j] + aux.c[k]; + for (i = 0, tmp2 = 0, tmp1 = tmp3 = 0.0; i < m; ++i) { + if (i == j || i == k) continue; + tmp1 += aux.bsum[i]; tmp2 += aux.c[i]; tmp3 += aux.fsum[i]; + } + if (tmp2) { + bar_e = (int)(tmp1 / tmp3 + 0.499); + if (bar_e > 63) bar_e = 63; + q[j*m+k] = q[k*m+j] = -4.343 * em->coef->lhet[cjk<<8|aux.c[k]] + tmp1; + } else q[j*m+k] = q[k*m+j] = -4.343 * em->coef->lhet[cjk<<8|aux.c[k]]; // all the bases are either j or k + } + for (k = 0; k != m; ++k) if (q[j*m+k] < 0.0) q[j*m+k] = 0.0; + } + return 0; +} diff --git a/samtools-0.1.18/errmod.h b/samtools-0.1.18/errmod.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/errmod.h @@ -0,0 +1,24 @@ +#ifndef ERRMOD_H +#define ERRMOD_H + +#include + +struct __errmod_coef_t; + +typedef struct { + double depcorr; + struct __errmod_coef_t *coef; +} errmod_t; + +errmod_t *errmod_init(float depcorr); +void errmod_destroy(errmod_t *em); + +/* + n: number of bases + m: maximum base + bases[i]: qual:6, strand:1, base:4 + q[i*m+j]: phred-scaled likelihood of (i,j) + */ +int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q); + +#endif diff --git a/samtools-0.1.18/faidx.c b/samtools-0.1.18/faidx.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/faidx.c @@ -0,0 +1,432 @@ +#include +#include +#include +#include +#include +#include "faidx.h" +#include "khash.h" + +typedef struct { + int32_t line_len, line_blen; + int64_t len; + uint64_t offset; +} faidx1_t; +KHASH_MAP_INIT_STR(s, faidx1_t) + +#ifndef _NO_RAZF +#include "razf.h" +#else +#ifdef _WIN32 +#define ftello(fp) ftell(fp) +#define fseeko(fp, offset, whence) fseek(fp, offset, whence) +#else +extern off_t ftello(FILE *stream); +extern int fseeko(FILE *stream, off_t offset, int whence); +#endif +#define RAZF FILE +#define razf_read(fp, buf, size) fread(buf, 1, size, fp) +#define razf_open(fn, mode) fopen(fn, mode) +#define razf_close(fp) fclose(fp) +#define razf_seek(fp, offset, whence) fseeko(fp, offset, whence) +#define razf_tell(fp) ftello(fp) +#endif +#ifdef _USE_KNETFILE +#include "knetfile.h" +#endif + +struct __faidx_t { + RAZF *rz; + int n, m; + char **name; + khash_t(s) *hash; +}; + +#ifndef kroundup32 +#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) +#endif + +static inline void fai_insert_index(faidx_t *idx, const char *name, int len, int line_len, int line_blen, uint64_t offset) +{ + khint_t k; + int ret; + faidx1_t t; + if (idx->n == idx->m) { + idx->m = idx->m? idx->m<<1 : 16; + idx->name = (char**)realloc(idx->name, sizeof(void*) * idx->m); + } + idx->name[idx->n] = strdup(name); + k = kh_put(s, idx->hash, idx->name[idx->n], &ret); + t.len = len; t.line_len = line_len; t.line_blen = line_blen; t.offset = offset; + kh_value(idx->hash, k) = t; + ++idx->n; +} + +faidx_t *fai_build_core(RAZF *rz) +{ + char c, *name; + int l_name, m_name, ret; + int line_len, line_blen, state; + int l1, l2; + faidx_t *idx; + uint64_t offset; + int64_t len; + + idx = (faidx_t*)calloc(1, sizeof(faidx_t)); + idx->hash = kh_init(s); + name = 0; l_name = m_name = 0; + len = line_len = line_blen = -1; state = 0; l1 = l2 = -1; offset = 0; + while (razf_read(rz, &c, 1)) { + if (c == '\n') { // an empty line + if (state == 1) { + offset = razf_tell(rz); + continue; + } else if ((state == 0 && len < 0) || state == 2) continue; + } + if (c == '>') { // fasta header + if (len >= 0) + fai_insert_index(idx, name, len, line_len, line_blen, offset); + l_name = 0; + while ((ret = razf_read(rz, &c, 1)) != 0 && !isspace(c)) { + if (m_name < l_name + 2) { + m_name = l_name + 2; + kroundup32(m_name); + name = (char*)realloc(name, m_name); + } + name[l_name++] = c; + } + name[l_name] = '\0'; + if (ret == 0) { + fprintf(stderr, "[fai_build_core] the last entry has no sequence\n"); + free(name); fai_destroy(idx); + return 0; + } + if (c != '\n') while (razf_read(rz, &c, 1) && c != '\n'); + state = 1; len = 0; + offset = razf_tell(rz); + } else { + if (state == 3) { + fprintf(stderr, "[fai_build_core] inlined empty line is not allowed in sequence '%s'.\n", name); + free(name); fai_destroy(idx); + return 0; + } + if (state == 2) state = 3; + l1 = l2 = 0; + do { + ++l1; + if (isgraph(c)) ++l2; + } while ((ret = razf_read(rz, &c, 1)) && c != '\n'); + if (state == 3 && l2) { + fprintf(stderr, "[fai_build_core] different line length in sequence '%s'.\n", name); + free(name); fai_destroy(idx); + return 0; + } + ++l1; len += l2; + if (state == 1) line_len = l1, line_blen = l2, state = 0; + else if (state == 0) { + if (l1 != line_len || l2 != line_blen) state = 2; + } + } + } + fai_insert_index(idx, name, len, line_len, line_blen, offset); + free(name); + return idx; +} + +void fai_save(const faidx_t *fai, FILE *fp) +{ + khint_t k; + int i; + for (i = 0; i < fai->n; ++i) { + faidx1_t x; + k = kh_get(s, fai->hash, fai->name[i]); + x = kh_value(fai->hash, k); +#ifdef _WIN32 + fprintf(fp, "%s\t%d\t%ld\t%d\t%d\n", fai->name[i], (int)x.len, (long)x.offset, (int)x.line_blen, (int)x.line_len); +#else + fprintf(fp, "%s\t%d\t%lld\t%d\t%d\n", fai->name[i], (int)x.len, (long long)x.offset, (int)x.line_blen, (int)x.line_len); +#endif + } +} + +faidx_t *fai_read(FILE *fp) +{ + faidx_t *fai; + char *buf, *p; + int len, line_len, line_blen; +#ifdef _WIN32 + long offset; +#else + long long offset; +#endif + fai = (faidx_t*)calloc(1, sizeof(faidx_t)); + fai->hash = kh_init(s); + buf = (char*)calloc(0x10000, 1); + while (!feof(fp) && fgets(buf, 0x10000, fp)) { + for (p = buf; *p && isgraph(*p); ++p); + *p = 0; ++p; +#ifdef _WIN32 + sscanf(p, "%d%ld%d%d", &len, &offset, &line_blen, &line_len); +#else + sscanf(p, "%d%lld%d%d", &len, &offset, &line_blen, &line_len); +#endif + fai_insert_index(fai, buf, len, line_len, line_blen, offset); + } + free(buf); + return fai; +} + +void fai_destroy(faidx_t *fai) +{ + int i; + for (i = 0; i < fai->n; ++i) free(fai->name[i]); + free(fai->name); + kh_destroy(s, fai->hash); + if (fai->rz) razf_close(fai->rz); + free(fai); +} + +int fai_build(const char *fn) +{ + char *str; + RAZF *rz; + FILE *fp; + faidx_t *fai; + str = (char*)calloc(strlen(fn) + 5, 1); + sprintf(str, "%s.fai", fn); + rz = razf_open(fn, "r"); + if (rz == 0) { + fprintf(stderr, "[fai_build] fail to open the FASTA file %s\n",fn); + free(str); + return -1; + } + fai = fai_build_core(rz); + razf_close(rz); + fp = fopen(str, "wb"); + if (fp == 0) { + fprintf(stderr, "[fai_build] fail to write FASTA index %s\n",str); + fai_destroy(fai); free(str); + return -1; + } + fai_save(fai, fp); + fclose(fp); + free(str); + fai_destroy(fai); + return 0; +} + +#ifdef _USE_KNETFILE +FILE *download_and_open(const char *fn) +{ + const int buf_size = 1 * 1024 * 1024; + uint8_t *buf; + FILE *fp; + knetFile *fp_remote; + const char *url = fn; + const char *p; + int l = strlen(fn); + for (p = fn + l - 1; p >= fn; --p) + if (*p == '/') break; + fn = p + 1; + + // First try to open a local copy + fp = fopen(fn, "r"); + if (fp) + return fp; + + // If failed, download from remote and open + fp_remote = knet_open(url, "rb"); + if (fp_remote == 0) { + fprintf(stderr, "[download_from_remote] fail to open remote file %s\n",url); + return NULL; + } + if ((fp = fopen(fn, "wb")) == 0) { + fprintf(stderr, "[download_from_remote] fail to create file in the working directory %s\n",fn); + knet_close(fp_remote); + return NULL; + } + buf = (uint8_t*)calloc(buf_size, 1); + while ((l = knet_read(fp_remote, buf, buf_size)) != 0) + fwrite(buf, 1, l, fp); + free(buf); + fclose(fp); + knet_close(fp_remote); + + return fopen(fn, "r"); +} +#endif + +faidx_t *fai_load(const char *fn) +{ + char *str; + FILE *fp; + faidx_t *fai; + str = (char*)calloc(strlen(fn) + 5, 1); + sprintf(str, "%s.fai", fn); + +#ifdef _USE_KNETFILE + if (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn) + { + fp = download_and_open(str); + if ( !fp ) + { + fprintf(stderr, "[fai_load] failed to open remote FASTA index %s\n", str); + free(str); + return 0; + } + } + else +#endif + fp = fopen(str, "rb"); + if (fp == 0) { + fprintf(stderr, "[fai_load] build FASTA index.\n"); + fai_build(fn); + fp = fopen(str, "rb"); + if (fp == 0) { + fprintf(stderr, "[fai_load] fail to open FASTA index.\n"); + free(str); + return 0; + } + } + + fai = fai_read(fp); + fclose(fp); + + fai->rz = razf_open(fn, "rb"); + free(str); + if (fai->rz == 0) { + fprintf(stderr, "[fai_load] fail to open FASTA file.\n"); + return 0; + } + return fai; +} + +char *fai_fetch(const faidx_t *fai, const char *str, int *len) +{ + char *s, c; + int i, l, k, name_end; + khiter_t iter; + faidx1_t val; + khash_t(s) *h; + int beg, end; + + beg = end = -1; + h = fai->hash; + name_end = l = strlen(str); + s = (char*)malloc(l+1); + // remove space + for (i = k = 0; i < l; ++i) + if (!isspace(str[i])) s[k++] = str[i]; + s[k] = 0; l = k; + // determine the sequence name + for (i = l - 1; i >= 0; --i) if (s[i] == ':') break; // look for colon from the end + if (i >= 0) name_end = i; + if (name_end < l) { // check if this is really the end + int n_hyphen = 0; + for (i = name_end + 1; i < l; ++i) { + if (s[i] == '-') ++n_hyphen; + else if (!isdigit(s[i]) && s[i] != ',') break; + } + if (i < l || n_hyphen > 1) name_end = l; // malformated region string; then take str as the name + s[name_end] = 0; + iter = kh_get(s, h, s); + if (iter == kh_end(h)) { // cannot find the sequence name + iter = kh_get(s, h, str); // try str as the name + if (iter == kh_end(h)) { + *len = 0; + free(s); return 0; + } else s[name_end] = ':', name_end = l; + } + } else iter = kh_get(s, h, str); + val = kh_value(h, iter); + // parse the interval + if (name_end < l) { + for (i = k = name_end + 1; i < l; ++i) + if (s[i] != ',') s[k++] = s[i]; + s[k] = 0; + beg = atoi(s + name_end + 1); + for (i = name_end + 1; i != k; ++i) if (s[i] == '-') break; + end = i < k? atoi(s + i + 1) : val.len; + if (beg > 0) --beg; + } else beg = 0, end = val.len; + if (beg >= val.len) beg = val.len; + if (end >= val.len) end = val.len; + if (beg > end) beg = end; + free(s); + + // now retrieve the sequence + l = 0; + s = (char*)malloc(end - beg + 2); + razf_seek(fai->rz, val.offset + beg / val.line_blen * val.line_len + beg % val.line_blen, SEEK_SET); + while (razf_read(fai->rz, &c, 1) == 1 && l < end - beg && !fai->rz->z_err) + if (isgraph(c)) s[l++] = c; + s[l] = '\0'; + *len = l; + return s; +} + +int faidx_main(int argc, char *argv[]) +{ + if (argc == 1) { + fprintf(stderr, "Usage: faidx [ [...]]\n"); + return 1; + } else { + if (argc == 2) fai_build(argv[1]); + else { + int i, j, k, l; + char *s; + faidx_t *fai; + fai = fai_load(argv[1]); + if (fai == 0) return 1; + for (i = 2; i != argc; ++i) { + printf(">%s\n", argv[i]); + s = fai_fetch(fai, argv[i], &l); + for (j = 0; j < l; j += 60) { + for (k = 0; k < 60 && k < l - j; ++k) + putchar(s[j + k]); + putchar('\n'); + } + free(s); + } + fai_destroy(fai); + } + } + return 0; +} + +int faidx_fetch_nseq(const faidx_t *fai) +{ + return fai->n; +} + +char *faidx_fetch_seq(const faidx_t *fai, char *c_name, int p_beg_i, int p_end_i, int *len) +{ + int l; + char c; + khiter_t iter; + faidx1_t val; + char *seq=NULL; + + // Adjust position + iter = kh_get(s, fai->hash, c_name); + if(iter == kh_end(fai->hash)) return 0; + val = kh_value(fai->hash, iter); + if(p_end_i < p_beg_i) p_beg_i = p_end_i; + if(p_beg_i < 0) p_beg_i = 0; + else if(val.len <= p_beg_i) p_beg_i = val.len - 1; + if(p_end_i < 0) p_end_i = 0; + else if(val.len <= p_end_i) p_end_i = val.len - 1; + + // Now retrieve the sequence + l = 0; + seq = (char*)malloc(p_end_i - p_beg_i + 2); + razf_seek(fai->rz, val.offset + p_beg_i / val.line_blen * val.line_len + p_beg_i % val.line_blen, SEEK_SET); + while (razf_read(fai->rz, &c, 1) == 1 && l < p_end_i - p_beg_i + 1) + if (isgraph(c)) seq[l++] = c; + seq[l] = '\0'; + *len = l; + return seq; +} + +#ifdef FAIDX_MAIN +int main(int argc, char *argv[]) { return faidx_main(argc, argv); } +#endif diff --git a/samtools-0.1.18/faidx.h b/samtools-0.1.18/faidx.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/faidx.h @@ -0,0 +1,103 @@ +/* The MIT License + + Copyright (c) 2008 Genome Research Ltd (GRL). + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* Contact: Heng Li */ + +#ifndef FAIDX_H +#define FAIDX_H + +/*! + @header + + Index FASTA files and extract subsequence. + + @copyright The Wellcome Trust Sanger Institute. + */ + +struct __faidx_t; +typedef struct __faidx_t faidx_t; + +#ifdef __cplusplus +extern "C" { +#endif + + /*! + @abstract Build index for a FASTA or razip compressed FASTA file. + @param fn FASTA file name + @return 0 on success; or -1 on failure + @discussion File "fn.fai" will be generated. + */ + int fai_build(const char *fn); + + /*! + @abstract Distroy a faidx_t struct. + @param fai Pointer to the struct to be destroyed + */ + void fai_destroy(faidx_t *fai); + + /*! + @abstract Load index from "fn.fai". + @param fn File name of the FASTA file + */ + faidx_t *fai_load(const char *fn); + + /*! + @abstract Fetch the sequence in a region. + @param fai Pointer to the faidx_t struct + @param reg Region in the format "chr2:20,000-30,000" + @param len Length of the region + @return Pointer to the sequence; null on failure + + @discussion The returned sequence is allocated by malloc family + and should be destroyed by end users by calling free() on it. + */ + char *fai_fetch(const faidx_t *fai, const char *reg, int *len); + + /*! + @abstract Fetch the number of sequences. + @param fai Pointer to the faidx_t struct + @return The number of sequences + */ + int faidx_fetch_nseq(const faidx_t *fai); + + /*! + @abstract Fetch the sequence in a region. + @param fai Pointer to the faidx_t struct + @param c_name Region name + @param p_beg_i Beginning position number (zero-based) + @param p_end_i End position number (zero-based) + @param len Length of the region + @return Pointer to the sequence; null on failure + + @discussion The returned sequence is allocated by malloc family + and should be destroyed by end users by calling free() on it. + */ + char *faidx_fetch_seq(const faidx_t *fai, char *c_name, int p_beg_i, int p_end_i, int *len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/samtools-0.1.18/kaln.c b/samtools-0.1.18/kaln.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kaln.c @@ -0,0 +1,486 @@ +/* The MIT License + + Copyright (c) 2003-2006, 2008, 2009, by Heng Li + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include +#include +#include +#include +#include +#include "kaln.h" + +#define FROM_M 0 +#define FROM_I 1 +#define FROM_D 2 + +typedef struct { + int i, j; + unsigned char ctype; +} path_t; + +int aln_sm_blosum62[] = { +/* A R N D C Q E G H I L K M F P S T W Y V * X */ + 4,-1,-2,-2, 0,-1,-1, 0,-2,-1,-1,-1,-1,-2,-1, 1, 0,-3,-2, 0,-4, 0, + -1, 5, 0,-2,-3, 1, 0,-2, 0,-3,-2, 2,-1,-3,-2,-1,-1,-3,-2,-3,-4,-1, + -2, 0, 6, 1,-3, 0, 0, 0, 1,-3,-3, 0,-2,-3,-2, 1, 0,-4,-2,-3,-4,-1, + -2,-2, 1, 6,-3, 0, 2,-1,-1,-3,-4,-1,-3,-3,-1, 0,-1,-4,-3,-3,-4,-1, + 0,-3,-3,-3, 9,-3,-4,-3,-3,-1,-1,-3,-1,-2,-3,-1,-1,-2,-2,-1,-4,-2, + -1, 1, 0, 0,-3, 5, 2,-2, 0,-3,-2, 1, 0,-3,-1, 0,-1,-2,-1,-2,-4,-1, + -1, 0, 0, 2,-4, 2, 5,-2, 0,-3,-3, 1,-2,-3,-1, 0,-1,-3,-2,-2,-4,-1, + 0,-2, 0,-1,-3,-2,-2, 6,-2,-4,-4,-2,-3,-3,-2, 0,-2,-2,-3,-3,-4,-1, + -2, 0, 1,-1,-3, 0, 0,-2, 8,-3,-3,-1,-2,-1,-2,-1,-2,-2, 2,-3,-4,-1, + -1,-3,-3,-3,-1,-3,-3,-4,-3, 4, 2,-3, 1, 0,-3,-2,-1,-3,-1, 3,-4,-1, + -1,-2,-3,-4,-1,-2,-3,-4,-3, 2, 4,-2, 2, 0,-3,-2,-1,-2,-1, 1,-4,-1, + -1, 2, 0,-1,-3, 1, 1,-2,-1,-3,-2, 5,-1,-3,-1, 0,-1,-3,-2,-2,-4,-1, + -1,-1,-2,-3,-1, 0,-2,-3,-2, 1, 2,-1, 5, 0,-2,-1,-1,-1,-1, 1,-4,-1, + -2,-3,-3,-3,-2,-3,-3,-3,-1, 0, 0,-3, 0, 6,-4,-2,-2, 1, 3,-1,-4,-1, + -1,-2,-2,-1,-3,-1,-1,-2,-2,-3,-3,-1,-2,-4, 7,-1,-1,-4,-3,-2,-4,-2, + 1,-1, 1, 0,-1, 0, 0, 0,-1,-2,-2, 0,-1,-2,-1, 4, 1,-3,-2,-2,-4, 0, + 0,-1, 0,-1,-1,-1,-1,-2,-2,-1,-1,-1,-1,-2,-1, 1, 5,-2,-2, 0,-4, 0, + -3,-3,-4,-4,-2,-2,-3,-2,-2,-3,-2,-3,-1, 1,-4,-3,-2,11, 2,-3,-4,-2, + -2,-2,-2,-3,-2,-1,-2,-3, 2,-1,-1,-2,-1, 3,-3,-2,-2, 2, 7,-1,-4,-1, + 0,-3,-3,-3,-1,-2,-2,-3,-3, 3, 1,-2, 1,-1,-2,-2, 0,-3,-1, 4,-4,-1, + -4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4, 1,-4, + 0,-1,-1,-1,-2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-2, 0, 0,-2,-1,-1,-4,-1 +}; + +int aln_sm_blast[] = { + 1, -3, -3, -3, -2, + -3, 1, -3, -3, -2, + -3, -3, 1, -3, -2, + -3, -3, -3, 1, -2, + -2, -2, -2, -2, -2 +}; + +int aln_sm_qual[] = { + 0, -23, -23, -23, 0, + -23, 0, -23, -23, 0, + -23, -23, 0, -23, 0, + -23, -23, -23, 0, 0, + 0, 0, 0, 0, 0 +}; + +ka_param_t ka_param_blast = { 5, 2, 5, 2, aln_sm_blast, 5, 50 }; +ka_param_t ka_param_aa2aa = { 10, 2, 10, 2, aln_sm_blosum62, 22, 50 }; + +ka_param2_t ka_param2_qual = { 37, 11, 37, 11, 37, 11, 0, 0, aln_sm_qual, 5, 50 }; + +static uint32_t *ka_path2cigar32(const path_t *path, int path_len, int *n_cigar) +{ + int i, n; + uint32_t *cigar; + unsigned char last_type; + + if (path_len == 0 || path == 0) { + *n_cigar = 0; + return 0; + } + + last_type = path->ctype; + for (i = n = 1; i < path_len; ++i) { + if (last_type != path[i].ctype) ++n; + last_type = path[i].ctype; + } + *n_cigar = n; + cigar = (uint32_t*)calloc(*n_cigar, 4); + + cigar[0] = 1u << 4 | path[path_len-1].ctype; + last_type = path[path_len-1].ctype; + for (i = path_len - 2, n = 0; i >= 0; --i) { + if (path[i].ctype == last_type) cigar[n] += 1u << 4; + else { + cigar[++n] = 1u << 4 | path[i].ctype; + last_type = path[i].ctype; + } + } + + return cigar; +} + +/***************************/ +/* START OF common_align.c */ +/***************************/ + +#define SET_INF(s) (s).M = (s).I = (s).D = MINOR_INF; + +#define set_M(MM, cur, p, sc) \ +{ \ + if ((p)->M >= (p)->I) { \ + if ((p)->M >= (p)->D) { \ + (MM) = (p)->M + (sc); (cur)->Mt = FROM_M; \ + } else { \ + (MM) = (p)->D + (sc); (cur)->Mt = FROM_D; \ + } \ + } else { \ + if ((p)->I > (p)->D) { \ + (MM) = (p)->I + (sc); (cur)->Mt = FROM_I; \ + } else { \ + (MM) = (p)->D + (sc); (cur)->Mt = FROM_D; \ + } \ + } \ +} +#define set_I(II, cur, p) \ +{ \ + if ((p)->M - gap_open > (p)->I) { \ + (cur)->It = FROM_M; \ + (II) = (p)->M - gap_open - gap_ext; \ + } else { \ + (cur)->It = FROM_I; \ + (II) = (p)->I - gap_ext; \ + } \ +} +#define set_end_I(II, cur, p) \ +{ \ + if (gap_end_ext >= 0) { \ + if ((p)->M - gap_end_open > (p)->I) { \ + (cur)->It = FROM_M; \ + (II) = (p)->M - gap_end_open - gap_end_ext; \ + } else { \ + (cur)->It = FROM_I; \ + (II) = (p)->I - gap_end_ext; \ + } \ + } else set_I(II, cur, p); \ +} +#define set_D(DD, cur, p) \ +{ \ + if ((p)->M - gap_open > (p)->D) { \ + (cur)->Dt = FROM_M; \ + (DD) = (p)->M - gap_open - gap_ext; \ + } else { \ + (cur)->Dt = FROM_D; \ + (DD) = (p)->D - gap_ext; \ + } \ +} +#define set_end_D(DD, cur, p) \ +{ \ + if (gap_end_ext >= 0) { \ + if ((p)->M - gap_end_open > (p)->D) { \ + (cur)->Dt = FROM_M; \ + (DD) = (p)->M - gap_end_open - gap_end_ext; \ + } else { \ + (cur)->Dt = FROM_D; \ + (DD) = (p)->D - gap_end_ext; \ + } \ + } else set_D(DD, cur, p); \ +} + +typedef struct { + uint8_t Mt:3, It:2, Dt:3; +} dpcell_t; + +typedef struct { + int M, I, D; +} dpscore_t; + +/*************************** + * banded global alignment * + ***************************/ +uint32_t *ka_global_core(uint8_t *seq1, int len1, uint8_t *seq2, int len2, const ka_param_t *ap, int *_score, int *n_cigar) +{ + int i, j; + dpcell_t **dpcell, *q; + dpscore_t *curr, *last, *s; + int b1, b2, tmp_end; + int *mat, end, max = 0; + uint8_t type, ctype; + uint32_t *cigar = 0; + + int gap_open, gap_ext, gap_end_open, gap_end_ext, b; + int *score_matrix, N_MATRIX_ROW; + + /* initialize some align-related parameters. just for compatibility */ + gap_open = ap->gap_open; + gap_ext = ap->gap_ext; + gap_end_open = ap->gap_end_open; + gap_end_ext = ap->gap_end_ext; + b = ap->band_width; + score_matrix = ap->matrix; + N_MATRIX_ROW = ap->row; + + if (n_cigar) *n_cigar = 0; + if (len1 == 0 || len2 == 0) return 0; + + /* calculate b1 and b2 */ + if (len1 > len2) { + b1 = len1 - len2 + b; + b2 = b; + } else { + b1 = b; + b2 = len2 - len1 + b; + } + if (b1 > len1) b1 = len1; + if (b2 > len2) b2 = len2; + --seq1; --seq2; + + /* allocate memory */ + end = (b1 + b2 <= len1)? (b1 + b2 + 1) : (len1 + 1); + dpcell = (dpcell_t**)malloc(sizeof(dpcell_t*) * (len2 + 1)); + for (j = 0; j <= len2; ++j) + dpcell[j] = (dpcell_t*)malloc(sizeof(dpcell_t) * end); + for (j = b2 + 1; j <= len2; ++j) + dpcell[j] -= j - b2; + curr = (dpscore_t*)malloc(sizeof(dpscore_t) * (len1 + 1)); + last = (dpscore_t*)malloc(sizeof(dpscore_t) * (len1 + 1)); + + /* set first row */ + SET_INF(*curr); curr->M = 0; + for (i = 1, s = curr + 1; i < b1; ++i, ++s) { + SET_INF(*s); + set_end_D(s->D, dpcell[0] + i, s - 1); + } + s = curr; curr = last; last = s; + + /* core dynamic programming, part 1 */ + tmp_end = (b2 < len2)? b2 : len2 - 1; + for (j = 1; j <= tmp_end; ++j) { + q = dpcell[j]; s = curr; SET_INF(*s); + set_end_I(s->I, q, last); + end = (j + b1 <= len1 + 1)? (j + b1 - 1) : len1; + mat = score_matrix + seq2[j] * N_MATRIX_ROW; + ++s; ++q; + for (i = 1; i != end; ++i, ++s, ++q) { + set_M(s->M, q, last + i - 1, mat[seq1[i]]); /* this will change s->M ! */ + set_I(s->I, q, last + i); + set_D(s->D, q, s - 1); + } + set_M(s->M, q, last + i - 1, mat[seq1[i]]); + set_D(s->D, q, s - 1); + if (j + b1 - 1 > len1) { /* bug fixed, 040227 */ + set_end_I(s->I, q, last + i); + } else s->I = MINOR_INF; + s = curr; curr = last; last = s; + } + /* last row for part 1, use set_end_D() instead of set_D() */ + if (j == len2 && b2 != len2 - 1) { + q = dpcell[j]; s = curr; SET_INF(*s); + set_end_I(s->I, q, last); + end = (j + b1 <= len1 + 1)? (j + b1 - 1) : len1; + mat = score_matrix + seq2[j] * N_MATRIX_ROW; + ++s; ++q; + for (i = 1; i != end; ++i, ++s, ++q) { + set_M(s->M, q, last + i - 1, mat[seq1[i]]); /* this will change s->M ! */ + set_I(s->I, q, last + i); + set_end_D(s->D, q, s - 1); + } + set_M(s->M, q, last + i - 1, mat[seq1[i]]); + set_end_D(s->D, q, s - 1); + if (j + b1 - 1 > len1) { /* bug fixed, 040227 */ + set_end_I(s->I, q, last + i); + } else s->I = MINOR_INF; + s = curr; curr = last; last = s; + ++j; + } + + /* core dynamic programming, part 2 */ + for (; j <= len2 - b2 + 1; ++j) { + SET_INF(curr[j - b2]); + mat = score_matrix + seq2[j] * N_MATRIX_ROW; + end = j + b1 - 1; + for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i != end; ++i, ++s, ++q) { + set_M(s->M, q, last + i - 1, mat[seq1[i]]); + set_I(s->I, q, last + i); + set_D(s->D, q, s - 1); + } + set_M(s->M, q, last + i - 1, mat[seq1[i]]); + set_D(s->D, q, s - 1); + s->I = MINOR_INF; + s = curr; curr = last; last = s; + } + + /* core dynamic programming, part 3 */ + for (; j < len2; ++j) { + SET_INF(curr[j - b2]); + mat = score_matrix + seq2[j] * N_MATRIX_ROW; + for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i < len1; ++i, ++s, ++q) { + set_M(s->M, q, last + i - 1, mat[seq1[i]]); + set_I(s->I, q, last + i); + set_D(s->D, q, s - 1); + } + set_M(s->M, q, last + len1 - 1, mat[seq1[i]]); + set_end_I(s->I, q, last + i); + set_D(s->D, q, s - 1); + s = curr; curr = last; last = s; + } + /* last row */ + if (j == len2) { + SET_INF(curr[j - b2]); + mat = score_matrix + seq2[j] * N_MATRIX_ROW; + for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i < len1; ++i, ++s, ++q) { + set_M(s->M, q, last + i - 1, mat[seq1[i]]); + set_I(s->I, q, last + i); + set_end_D(s->D, q, s - 1); + } + set_M(s->M, q, last + len1 - 1, mat[seq1[i]]); + set_end_I(s->I, q, last + i); + set_end_D(s->D, q, s - 1); + s = curr; curr = last; last = s; + } + + *_score = last[len1].M; + if (n_cigar) { /* backtrace */ + path_t *p, *path = (path_t*)malloc(sizeof(path_t) * (len1 + len2 + 2)); + i = len1; j = len2; + q = dpcell[j] + i; + s = last + len1; + max = s->M; type = q->Mt; ctype = FROM_M; + if (s->I > max) { max = s->I; type = q->It; ctype = FROM_I; } + if (s->D > max) { max = s->D; type = q->Dt; ctype = FROM_D; } + + p = path; + p->ctype = ctype; p->i = i; p->j = j; /* bug fixed 040408 */ + ++p; + do { + switch (ctype) { + case FROM_M: --i; --j; break; + case FROM_I: --j; break; + case FROM_D: --i; break; + } + q = dpcell[j] + i; + ctype = type; + switch (type) { + case FROM_M: type = q->Mt; break; + case FROM_I: type = q->It; break; + case FROM_D: type = q->Dt; break; + } + p->ctype = ctype; p->i = i; p->j = j; + ++p; + } while (i || j); + cigar = ka_path2cigar32(path, p - path - 1, n_cigar); + free(path); + } + + /* free memory */ + for (j = b2 + 1; j <= len2; ++j) + dpcell[j] += j - b2; + for (j = 0; j <= len2; ++j) + free(dpcell[j]); + free(dpcell); + free(curr); free(last); + + return cigar; +} + +typedef struct { + int M, I, D; +} score_aux_t; + +#define MINUS_INF -0x40000000 + +// matrix: len2 rows and len1 columns +int ka_global_score(const uint8_t *_seq1, int len1, const uint8_t *_seq2, int len2, const ka_param2_t *ap) +{ + +#define __score_aux(_p, _q0, _sc, _io, _ie, _do, _de) { \ + int t1, t2; \ + score_aux_t *_q; \ + _q = _q0; \ + _p->M = _q->M >= _q->I? _q->M : _q->I; \ + _p->M = _p->M >= _q->D? _p->M : _q->D; \ + _p->M += (_sc); \ + ++_q; t1 = _q->M - _io - _ie; t2 = _q->I - _ie; _p->I = t1 >= t2? t1 : t2; \ + _q = _p-1; t1 = _q->M - _do - _de; t2 = _q->D - _de; _p->D = t1 >= t2? t1 : t2; \ + } + + int i, j, bw, scmat_size = ap->row, *scmat = ap->matrix, ret; + const uint8_t *seq1, *seq2; + score_aux_t *curr, *last, *swap; + bw = abs(len1 - len2) + ap->band_width; + i = len1 > len2? len1 : len2; + if (bw > i + 1) bw = i + 1; + seq1 = _seq1 - 1; seq2 = _seq2 - 1; + curr = calloc(len1 + 2, sizeof(score_aux_t)); + last = calloc(len1 + 2, sizeof(score_aux_t)); + { // the zero-th row + int x, end = len1; + score_aux_t *p; + j = 0; + x = j + bw; end = len1 < x? len1 : x; // band end + p = curr; + p->M = 0; p->I = p->D = MINUS_INF; + for (i = 1, p = &curr[1]; i <= end; ++i, ++p) + p->M = p->I = MINUS_INF, p->D = -(ap->edo + ap->ede * i); + p->M = p->I = p->D = MINUS_INF; + swap = curr; curr = last; last = swap; + } + for (j = 1; j < len2; ++j) { + int x, beg = 0, end = len1, *scrow, col_end; + score_aux_t *p; + x = j - bw; beg = 0 > x? 0 : x; // band start + x = j + bw; end = len1 < x? len1 : x; // band end + if (beg == 0) { // from zero-th column + p = curr; + p->M = p->D = MINUS_INF; p->I = -(ap->eio + ap->eie * j); + ++beg; // then beg = 1 + } + scrow = scmat + seq2[j] * scmat_size; + if (end == len1) col_end = 1, --end; + else col_end = 0; + for (i = beg, p = &curr[beg]; i <= end; ++i, ++p) + __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->iio, ap->iie, ap->ido, ap->ide); + if (col_end) { + __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->eio, ap->eie, ap->ido, ap->ide); + ++p; + } + p->M = p->I = p->D = MINUS_INF; +// for (i = 0; i <= len1; ++i) printf("(%d,%d,%d) ", curr[i].M, curr[i].I, curr[i].D); putchar('\n'); + swap = curr; curr = last; last = swap; + } + { // the last row + int x, beg = 0, *scrow; + score_aux_t *p; + j = len2; + x = j - bw; beg = 0 > x? 0 : x; // band start + if (beg == 0) { // from zero-th column + p = curr; + p->M = p->D = MINUS_INF; p->I = -(ap->eio + ap->eie * j); + ++beg; // then beg = 1 + } + scrow = scmat + seq2[j] * scmat_size; + for (i = beg, p = &curr[beg]; i < len1; ++i, ++p) + __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->iio, ap->iie, ap->edo, ap->ede); + __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->eio, ap->eie, ap->edo, ap->ede); +// for (i = 0; i <= len1; ++i) printf("(%d,%d,%d) ", curr[i].M, curr[i].I, curr[i].D); putchar('\n'); + } + ret = curr[len1].M >= curr[len1].I? curr[len1].M : curr[len1].I; + ret = ret >= curr[len1].D? ret : curr[len1].D; + free(curr); free(last); + return ret; +} + +#ifdef _MAIN +int main(int argc, char *argv[]) +{ +// int len1 = 35, len2 = 35; +// uint8_t *seq1 = (uint8_t*)"\0\0\3\3\2\0\0\0\1\0\2\1\2\1\3\2\3\3\3\0\2\3\2\1\1\3\3\3\2\3\3\1\0\0\1"; +// uint8_t *seq2 = (uint8_t*)"\0\0\3\3\2\0\0\0\1\0\2\1\2\1\3\2\3\3\3\0\2\3\2\1\1\3\3\3\2\3\3\1\0\1\0"; + int len1 = 4, len2 = 4; + uint8_t *seq1 = (uint8_t*)"\1\0\0\1"; + uint8_t *seq2 = (uint8_t*)"\1\0\1\0"; + int sc; +// ka_global_core(seq1, 2, seq2, 1, &ka_param_qual, &sc, 0); + sc = ka_global_score(seq1, len1, seq2, len2, &ka_param2_qual); + printf("%d\n", sc); + return 0; +} +#endif diff --git a/samtools-0.1.18/kaln.h b/samtools-0.1.18/kaln.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kaln.h @@ -0,0 +1,67 @@ +/* The MIT License + + Copyright (c) 2003-2006, 2008, 2009 by Heng Li + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef LH3_KALN_H_ +#define LH3_KALN_H_ + +#include + +#define MINOR_INF -1073741823 + +typedef struct { + int gap_open; + int gap_ext; + int gap_end_open; + int gap_end_ext; + + int *matrix; + int row; + int band_width; +} ka_param_t; + +typedef struct { + int iio, iie, ido, ide; + int eio, eie, edo, ede; + int *matrix; + int row; + int band_width; +} ka_param2_t; + +#ifdef __cplusplus +extern "C" { +#endif + + uint32_t *ka_global_core(uint8_t *seq1, int len1, uint8_t *seq2, int len2, const ka_param_t *ap, + int *_score, int *n_cigar); + int ka_global_score(const uint8_t *_seq1, int len1, const uint8_t *_seq2, int len2, const ka_param2_t *ap); +#ifdef __cplusplus +} +#endif + +extern ka_param_t ka_param_blast; /* = { 5, 2, 5, 2, aln_sm_blast, 5, 50 }; */ +extern ka_param_t ka_param_qual; // only use this for global alignment!!! +extern ka_param2_t ka_param2_qual; // only use this for global alignment!!! + +#endif diff --git a/samtools-0.1.18/khash.h b/samtools-0.1.18/khash.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/khash.h @@ -0,0 +1,528 @@ +/* The MIT License + + Copyright (c) 2008, 2009, 2011 by Attractive Chaos + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* + An example: + +#include "khash.h" +KHASH_MAP_INIT_INT(32, char) +int main() { + int ret, is_missing; + khiter_t k; + khash_t(32) *h = kh_init(32); + k = kh_put(32, h, 5, &ret); + if (!ret) kh_del(32, h, k); + kh_value(h, k) = 10; + k = kh_get(32, h, 10); + is_missing = (k == kh_end(h)); + k = kh_get(32, h, 5); + kh_del(32, h, k); + for (k = kh_begin(h); k != kh_end(h); ++k) + if (kh_exist(h, k)) kh_value(h, k) = 1; + kh_destroy(32, h); + return 0; +} +*/ + +/* + 2011-02-14 (0.2.5): + + * Allow to declare global functions. + + 2009-09-26 (0.2.4): + + * Improve portability + + 2008-09-19 (0.2.3): + + * Corrected the example + * Improved interfaces + + 2008-09-11 (0.2.2): + + * Improved speed a little in kh_put() + + 2008-09-10 (0.2.1): + + * Added kh_clear() + * Fixed a compiling error + + 2008-09-02 (0.2.0): + + * Changed to token concatenation which increases flexibility. + + 2008-08-31 (0.1.2): + + * Fixed a bug in kh_get(), which has not been tested previously. + + 2008-08-31 (0.1.1): + + * Added destructor +*/ + + +#ifndef __AC_KHASH_H +#define __AC_KHASH_H + +/*! + @header + + Generic hash table library. + + @copyright Heng Li + */ + +#define AC_VERSION_KHASH_H "0.2.5" + +#include +#include +#include + +/* compipler specific configuration */ + +#if UINT_MAX == 0xffffffffu +typedef unsigned int khint32_t; +#elif ULONG_MAX == 0xffffffffu +typedef unsigned long khint32_t; +#endif + +#if ULONG_MAX == ULLONG_MAX +typedef unsigned long khint64_t; +#else +typedef unsigned long long khint64_t; +#endif + +#ifdef _MSC_VER +#define inline __inline +#endif + +typedef khint32_t khint_t; +typedef khint_t khiter_t; + +#define __ac_HASH_PRIME_SIZE 32 +static const khint32_t __ac_prime_list[__ac_HASH_PRIME_SIZE] = +{ + 0ul, 3ul, 11ul, 23ul, 53ul, + 97ul, 193ul, 389ul, 769ul, 1543ul, + 3079ul, 6151ul, 12289ul, 24593ul, 49157ul, + 98317ul, 196613ul, 393241ul, 786433ul, 1572869ul, + 3145739ul, 6291469ul, 12582917ul, 25165843ul, 50331653ul, + 100663319ul, 201326611ul, 402653189ul, 805306457ul, 1610612741ul, + 3221225473ul, 4294967291ul +}; + +#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2) +#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1) +#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3) +#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1))) +#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1))) +#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) +#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) + +static const double __ac_HASH_UPPER = 0.77; + +#define KHASH_DECLARE(name, khkey_t, khval_t) \ + typedef struct { \ + khint_t n_buckets, size, n_occupied, upper_bound; \ + khint32_t *flags; \ + khkey_t *keys; \ + khval_t *vals; \ + } kh_##name##_t; \ + extern kh_##name##_t *kh_init_##name(); \ + extern void kh_destroy_##name(kh_##name##_t *h); \ + extern void kh_clear_##name(kh_##name##_t *h); \ + extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \ + extern void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ + extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ + extern void kh_del_##name(kh_##name##_t *h, khint_t x); + +#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + typedef struct { \ + khint_t n_buckets, size, n_occupied, upper_bound; \ + khint32_t *flags; \ + khkey_t *keys; \ + khval_t *vals; \ + } kh_##name##_t; \ + SCOPE kh_##name##_t *kh_init_##name() { \ + return (kh_##name##_t*)calloc(1, sizeof(kh_##name##_t)); \ + } \ + SCOPE void kh_destroy_##name(kh_##name##_t *h) \ + { \ + if (h) { \ + free(h->keys); free(h->flags); \ + free(h->vals); \ + free(h); \ + } \ + } \ + SCOPE void kh_clear_##name(kh_##name##_t *h) \ + { \ + if (h && h->flags) { \ + memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(khint32_t)); \ + h->size = h->n_occupied = 0; \ + } \ + } \ + SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ + { \ + if (h->n_buckets) { \ + khint_t inc, k, i, last; \ + k = __hash_func(key); i = k % h->n_buckets; \ + inc = 1 + k % (h->n_buckets - 1); last = i; \ + while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ + if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \ + else i += inc; \ + if (i == last) return h->n_buckets; \ + } \ + return __ac_iseither(h->flags, i)? h->n_buckets : i; \ + } else return 0; \ + } \ + SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ + { \ + khint32_t *new_flags = 0; \ + khint_t j = 1; \ + { \ + khint_t t = __ac_HASH_PRIME_SIZE - 1; \ + while (__ac_prime_list[t] > new_n_buckets) --t; \ + new_n_buckets = __ac_prime_list[t+1]; \ + if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; \ + else { \ + new_flags = (khint32_t*)malloc(((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \ + memset(new_flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \ + if (h->n_buckets < new_n_buckets) { \ + h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \ + if (kh_is_map) \ + h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \ + } \ + } \ + } \ + if (j) { \ + for (j = 0; j != h->n_buckets; ++j) { \ + if (__ac_iseither(h->flags, j) == 0) { \ + khkey_t key = h->keys[j]; \ + khval_t val; \ + if (kh_is_map) val = h->vals[j]; \ + __ac_set_isdel_true(h->flags, j); \ + while (1) { \ + khint_t inc, k, i; \ + k = __hash_func(key); \ + i = k % new_n_buckets; \ + inc = 1 + k % (new_n_buckets - 1); \ + while (!__ac_isempty(new_flags, i)) { \ + if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets; \ + else i += inc; \ + } \ + __ac_set_isempty_false(new_flags, i); \ + if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { \ + { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ + if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \ + __ac_set_isdel_true(h->flags, i); \ + } else { \ + h->keys[i] = key; \ + if (kh_is_map) h->vals[i] = val; \ + break; \ + } \ + } \ + } \ + } \ + if (h->n_buckets > new_n_buckets) { \ + h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \ + if (kh_is_map) \ + h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \ + } \ + free(h->flags); \ + h->flags = new_flags; \ + h->n_buckets = new_n_buckets; \ + h->n_occupied = h->size; \ + h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \ + } \ + } \ + SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \ + { \ + khint_t x; \ + if (h->n_occupied >= h->upper_bound) { \ + if (h->n_buckets > (h->size<<1)) kh_resize_##name(h, h->n_buckets - 1); \ + else kh_resize_##name(h, h->n_buckets + 1); \ + } \ + { \ + khint_t inc, k, i, site, last; \ + x = site = h->n_buckets; k = __hash_func(key); i = k % h->n_buckets; \ + if (__ac_isempty(h->flags, i)) x = i; \ + else { \ + inc = 1 + k % (h->n_buckets - 1); last = i; \ + while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ + if (__ac_isdel(h->flags, i)) site = i; \ + if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \ + else i += inc; \ + if (i == last) { x = site; break; } \ + } \ + if (x == h->n_buckets) { \ + if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \ + else x = i; \ + } \ + } \ + } \ + if (__ac_isempty(h->flags, x)) { \ + h->keys[x] = key; \ + __ac_set_isboth_false(h->flags, x); \ + ++h->size; ++h->n_occupied; \ + *ret = 1; \ + } else if (__ac_isdel(h->flags, x)) { \ + h->keys[x] = key; \ + __ac_set_isboth_false(h->flags, x); \ + ++h->size; \ + *ret = 2; \ + } else *ret = 0; \ + return x; \ + } \ + SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \ + { \ + if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \ + __ac_set_isdel_true(h->flags, x); \ + --h->size; \ + } \ + } + +#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + KHASH_INIT2(name, static inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) + +/* --- BEGIN OF HASH FUNCTIONS --- */ + +/*! @function + @abstract Integer hash function + @param key The integer [khint32_t] + @return The hash value [khint_t] + */ +#define kh_int_hash_func(key) (khint32_t)(key) +/*! @function + @abstract Integer comparison function + */ +#define kh_int_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract 64-bit integer hash function + @param key The integer [khint64_t] + @return The hash value [khint_t] + */ +#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11) +/*! @function + @abstract 64-bit integer comparison function + */ +#define kh_int64_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract const char* hash function + @param s Pointer to a null terminated string + @return The hash value + */ +static inline khint_t __ac_X31_hash_string(const char *s) +{ + khint_t h = *s; + if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s; + return h; +} +/*! @function + @abstract Another interface to const char* hash function + @param key Pointer to a null terminated string [const char*] + @return The hash value [khint_t] + */ +#define kh_str_hash_func(key) __ac_X31_hash_string(key) +/*! @function + @abstract Const char* comparison function + */ +#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0) + +/* --- END OF HASH FUNCTIONS --- */ + +/* Other necessary macros... */ + +/*! + @abstract Type of the hash table. + @param name Name of the hash table [symbol] + */ +#define khash_t(name) kh_##name##_t + +/*! @function + @abstract Initiate a hash table. + @param name Name of the hash table [symbol] + @return Pointer to the hash table [khash_t(name)*] + */ +#define kh_init(name) kh_init_##name() + +/*! @function + @abstract Destroy a hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + */ +#define kh_destroy(name, h) kh_destroy_##name(h) + +/*! @function + @abstract Reset a hash table without deallocating memory. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + */ +#define kh_clear(name, h) kh_clear_##name(h) + +/*! @function + @abstract Resize a hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param s New size [khint_t] + */ +#define kh_resize(name, h, s) kh_resize_##name(h, s) + +/*! @function + @abstract Insert a key to the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Key [type of keys] + @param r Extra return code: 0 if the key is present in the hash table; + 1 if the bucket is empty (never used); 2 if the element in + the bucket has been deleted [int*] + @return Iterator to the inserted element [khint_t] + */ +#define kh_put(name, h, k, r) kh_put_##name(h, k, r) + +/*! @function + @abstract Retrieve a key from the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Key [type of keys] + @return Iterator to the found element, or kh_end(h) is the element is absent [khint_t] + */ +#define kh_get(name, h, k) kh_get_##name(h, k) + +/*! @function + @abstract Remove a key from the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Iterator to the element to be deleted [khint_t] + */ +#define kh_del(name, h, k) kh_del_##name(h, k) + + +/*! @function + @abstract Test whether a bucket contains data. + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return 1 if containing data; 0 otherwise [int] + */ +#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x))) + +/*! @function + @abstract Get key given an iterator + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return Key [type of keys] + */ +#define kh_key(h, x) ((h)->keys[x]) + +/*! @function + @abstract Get value given an iterator + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return Value [type of values] + @discussion For hash sets, calling this results in segfault. + */ +#define kh_val(h, x) ((h)->vals[x]) + +/*! @function + @abstract Alias of kh_val() + */ +#define kh_value(h, x) ((h)->vals[x]) + +/*! @function + @abstract Get the start iterator + @param h Pointer to the hash table [khash_t(name)*] + @return The start iterator [khint_t] + */ +#define kh_begin(h) (khint_t)(0) + +/*! @function + @abstract Get the end iterator + @param h Pointer to the hash table [khash_t(name)*] + @return The end iterator [khint_t] + */ +#define kh_end(h) ((h)->n_buckets) + +/*! @function + @abstract Get the number of elements in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @return Number of elements in the hash table [khint_t] + */ +#define kh_size(h) ((h)->size) + +/*! @function + @abstract Get the number of buckets in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @return Number of buckets in the hash table [khint_t] + */ +#define kh_n_buckets(h) ((h)->n_buckets) + +/* More conenient interfaces */ + +/*! @function + @abstract Instantiate a hash set containing integer keys + @param name Name of the hash table [symbol] + */ +#define KHASH_SET_INIT_INT(name) \ + KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing integer keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] + */ +#define KHASH_MAP_INIT_INT(name, khval_t) \ + KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing 64-bit integer keys + @param name Name of the hash table [symbol] + */ +#define KHASH_SET_INIT_INT64(name) \ + KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing 64-bit integer keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] + */ +#define KHASH_MAP_INIT_INT64(name, khval_t) \ + KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal) + +typedef const char *kh_cstr_t; +/*! @function + @abstract Instantiate a hash map containing const char* keys + @param name Name of the hash table [symbol] + */ +#define KHASH_SET_INIT_STR(name) \ + KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing const char* keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] + */ +#define KHASH_MAP_INIT_STR(name, khval_t) \ + KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) + +#endif /* __AC_KHASH_H */ diff --git a/samtools-0.1.18/knetfile.c b/samtools-0.1.18/knetfile.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/knetfile.c @@ -0,0 +1,632 @@ +/* The MIT License + + Copyright (c) 2008 by Genome Research Ltd (GRL). + 2010 by Attractive Chaos + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* Probably I will not do socket programming in the next few years and + therefore I decide to heavily annotate this file, for Linux and + Windows as well. -ac */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#include +#endif + +#include "knetfile.h" + +/* In winsock.h, the type of a socket is SOCKET, which is: "typedef + * u_int SOCKET". An invalid SOCKET is: "(SOCKET)(~0)", or signed + * integer -1. In knetfile.c, I use "int" for socket type + * throughout. This should be improved to avoid confusion. + * + * In Linux/Mac, recv() and read() do almost the same thing. You can see + * in the header file that netread() is simply an alias of read(). In + * Windows, however, they are different and using recv() is mandatory. + */ + +/* This function tests if the file handler is ready for reading (or + * writing if is_read==0). */ +static int socket_wait(int fd, int is_read) +{ + fd_set fds, *fdr = 0, *fdw = 0; + struct timeval tv; + int ret; + tv.tv_sec = 5; tv.tv_usec = 0; // 5 seconds time out + FD_ZERO(&fds); + FD_SET(fd, &fds); + if (is_read) fdr = &fds; + else fdw = &fds; + ret = select(fd+1, fdr, fdw, 0, &tv); +#ifndef _WIN32 + if (ret == -1) perror("select"); +#else + if (ret == 0) + fprintf(stderr, "select time-out\n"); + else if (ret == SOCKET_ERROR) + fprintf(stderr, "select: %d\n", WSAGetLastError()); +#endif + return ret; +} + +#ifndef _WIN32 +/* This function does not work with Windows due to the lack of + * getaddrinfo() in winsock. It is addapted from an example in "Beej's + * Guide to Network Programming" (http://beej.us/guide/bgnet/). */ +static int socket_connect(const char *host, const char *port) +{ +#define __err_connect(func) do { perror(func); freeaddrinfo(res); return -1; } while (0) + + int on = 1, fd; + struct linger lng = { 0, 0 }; + struct addrinfo hints, *res = 0; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + /* In Unix/Mac, getaddrinfo() is the most convenient way to get + * server information. */ + if (getaddrinfo(host, port, &hints, &res) != 0) __err_connect("getaddrinfo"); + if ((fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) __err_connect("socket"); + /* The following two setsockopt() are used by ftplib + * (http://nbpfaus.net/~pfau/ftplib/). I am not sure if they + * necessary. */ + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) __err_connect("setsockopt"); + if (setsockopt(fd, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) __err_connect("setsockopt"); + if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) __err_connect("connect"); + freeaddrinfo(res); + return fd; +} +#else +/* MinGW's printf has problem with "%lld" */ +char *int64tostr(char *buf, int64_t x) +{ + int cnt; + int i = 0; + do { + buf[i++] = '0' + x % 10; + x /= 10; + } while (x); + buf[i] = 0; + for (cnt = i, i = 0; i < cnt/2; ++i) { + int c = buf[i]; buf[i] = buf[cnt-i-1]; buf[cnt-i-1] = c; + } + return buf; +} + +int64_t strtoint64(const char *buf) +{ + int64_t x; + for (x = 0; *buf != '\0'; ++buf) + x = x * 10 + ((int64_t) *buf - 48); + return x; +} +/* In windows, the first thing is to establish the TCP connection. */ +int knet_win32_init() +{ + WSADATA wsaData; + return WSAStartup(MAKEWORD(2, 2), &wsaData); +} +void knet_win32_destroy() +{ + WSACleanup(); +} +/* A slightly modfied version of the following function also works on + * Mac (and presummably Linux). However, this function is not stable on + * my Mac. It sometimes works fine but sometimes does not. Therefore for + * non-Windows OS, I do not use this one. */ +static SOCKET socket_connect(const char *host, const char *port) +{ +#define __err_connect(func) \ + do { \ + fprintf(stderr, "%s: %d\n", func, WSAGetLastError()); \ + return -1; \ + } while (0) + + int on = 1; + SOCKET fd; + struct linger lng = { 0, 0 }; + struct sockaddr_in server; + struct hostent *hp = 0; + // open socket + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) __err_connect("socket"); + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)) == -1) __err_connect("setsockopt"); + if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char*)&lng, sizeof(lng)) == -1) __err_connect("setsockopt"); + // get host info + if (isalpha(host[0])) hp = gethostbyname(host); + else { + struct in_addr addr; + addr.s_addr = inet_addr(host); + hp = gethostbyaddr((char*)&addr, 4, AF_INET); + } + if (hp == 0) __err_connect("gethost"); + // connect + server.sin_addr.s_addr = *((unsigned long*)hp->h_addr); + server.sin_family= AF_INET; + server.sin_port = htons(atoi(port)); + if (connect(fd, (struct sockaddr*)&server, sizeof(server)) != 0) __err_connect("connect"); + // freehostent(hp); // strangely in MSDN, hp is NOT freed (memory leak?!) + return fd; +} +#endif + +static off_t my_netread(int fd, void *buf, off_t len) +{ + off_t rest = len, curr, l = 0; + /* recv() and read() may not read the required length of data with + * one call. They have to be called repeatedly. */ + while (rest) { + if (socket_wait(fd, 1) <= 0) break; // socket is not ready for reading + curr = netread(fd, buf + l, rest); + /* According to the glibc manual, section 13.2, a zero returned + * value indicates end-of-file (EOF), which should mean that + * read() will not return zero if EOF has not been met but data + * are not immediately available. */ + if (curr == 0) break; + l += curr; rest -= curr; + } + return l; +} + +/************************* + * FTP specific routines * + *************************/ + +static int kftp_get_response(knetFile *ftp) +{ +#ifndef _WIN32 + unsigned char c; +#else + char c; +#endif + int n = 0; + char *p; + if (socket_wait(ftp->ctrl_fd, 1) <= 0) return 0; + while (netread(ftp->ctrl_fd, &c, 1)) { // FIXME: this is *VERY BAD* for unbuffered I/O + //fputc(c, stderr); + if (n >= ftp->max_response) { + ftp->max_response = ftp->max_response? ftp->max_response<<1 : 256; + ftp->response = realloc(ftp->response, ftp->max_response); + } + ftp->response[n++] = c; + if (c == '\n') { + if (n >= 4 && isdigit(ftp->response[0]) && isdigit(ftp->response[1]) && isdigit(ftp->response[2]) + && ftp->response[3] != '-') break; + n = 0; + continue; + } + } + if (n < 2) return -1; + ftp->response[n-2] = 0; + return strtol(ftp->response, &p, 0); +} + +static int kftp_send_cmd(knetFile *ftp, const char *cmd, int is_get) +{ + if (socket_wait(ftp->ctrl_fd, 0) <= 0) return -1; // socket is not ready for writing + netwrite(ftp->ctrl_fd, cmd, strlen(cmd)); + return is_get? kftp_get_response(ftp) : 0; +} + +static int kftp_pasv_prep(knetFile *ftp) +{ + char *p; + int v[6]; + kftp_send_cmd(ftp, "PASV\r\n", 1); + for (p = ftp->response; *p && *p != '('; ++p); + if (*p != '(') return -1; + ++p; + sscanf(p, "%d,%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]); + memcpy(ftp->pasv_ip, v, 4 * sizeof(int)); + ftp->pasv_port = (v[4]<<8&0xff00) + v[5]; + return 0; +} + + +static int kftp_pasv_connect(knetFile *ftp) +{ + char host[80], port[10]; + if (ftp->pasv_port == 0) { + fprintf(stderr, "[kftp_pasv_connect] kftp_pasv_prep() is not called before hand.\n"); + return -1; + } + sprintf(host, "%d.%d.%d.%d", ftp->pasv_ip[0], ftp->pasv_ip[1], ftp->pasv_ip[2], ftp->pasv_ip[3]); + sprintf(port, "%d", ftp->pasv_port); + ftp->fd = socket_connect(host, port); + if (ftp->fd == -1) return -1; + return 0; +} + +int kftp_connect(knetFile *ftp) +{ + ftp->ctrl_fd = socket_connect(ftp->host, ftp->port); + if (ftp->ctrl_fd == -1) return -1; + kftp_get_response(ftp); + kftp_send_cmd(ftp, "USER anonymous\r\n", 1); + kftp_send_cmd(ftp, "PASS kftp@\r\n", 1); + kftp_send_cmd(ftp, "TYPE I\r\n", 1); + return 0; +} + +int kftp_reconnect(knetFile *ftp) +{ + if (ftp->ctrl_fd != -1) { + netclose(ftp->ctrl_fd); + ftp->ctrl_fd = -1; + } + netclose(ftp->fd); + ftp->fd = -1; + return kftp_connect(ftp); +} + +// initialize ->type, ->host, ->retr and ->size +knetFile *kftp_parse_url(const char *fn, const char *mode) +{ + knetFile *fp; + char *p; + int l; + if (strstr(fn, "ftp://") != fn) return 0; + for (p = (char*)fn + 6; *p && *p != '/'; ++p); + if (*p != '/') return 0; + l = p - fn - 6; + fp = calloc(1, sizeof(knetFile)); + fp->type = KNF_TYPE_FTP; + fp->fd = -1; + /* the Linux/Mac version of socket_connect() also recognizes a port + * like "ftp", but the Windows version does not. */ + fp->port = strdup("21"); + fp->host = calloc(l + 1, 1); + if (strchr(mode, 'c')) fp->no_reconnect = 1; + strncpy(fp->host, fn + 6, l); + fp->retr = calloc(strlen(p) + 8, 1); + sprintf(fp->retr, "RETR %s\r\n", p); + fp->size_cmd = calloc(strlen(p) + 8, 1); + sprintf(fp->size_cmd, "SIZE %s\r\n", p); + fp->seek_offset = 0; + return fp; +} +// place ->fd at offset off +int kftp_connect_file(knetFile *fp) +{ + int ret; + long long file_size; + if (fp->fd != -1) { + netclose(fp->fd); + if (fp->no_reconnect) kftp_get_response(fp); + } + kftp_pasv_prep(fp); + kftp_send_cmd(fp, fp->size_cmd, 1); +#ifndef _WIN32 + if ( sscanf(fp->response,"%*d %lld", &file_size) != 1 ) + { + fprintf(stderr,"[kftp_connect_file] %s\n", fp->response); + return -1; + } +#else + const char *p = fp->response; + while (*p != ' ') ++p; + while (*p < '0' || *p > '9') ++p; + file_size = strtoint64(p); +#endif + fp->file_size = file_size; + if (fp->offset>=0) { + char tmp[32]; +#ifndef _WIN32 + sprintf(tmp, "REST %lld\r\n", (long long)fp->offset); +#else + strcpy(tmp, "REST "); + int64tostr(tmp + 5, fp->offset); + strcat(tmp, "\r\n"); +#endif + kftp_send_cmd(fp, tmp, 1); + } + kftp_send_cmd(fp, fp->retr, 0); + kftp_pasv_connect(fp); + ret = kftp_get_response(fp); + if (ret != 150) { + fprintf(stderr, "[kftp_connect_file] %s\n", fp->response); + netclose(fp->fd); + fp->fd = -1; + return -1; + } + fp->is_ready = 1; + return 0; +} + + +/************************** + * HTTP specific routines * + **************************/ + +knetFile *khttp_parse_url(const char *fn, const char *mode) +{ + knetFile *fp; + char *p, *proxy, *q; + int l; + if (strstr(fn, "http://") != fn) return 0; + // set ->http_host + for (p = (char*)fn + 7; *p && *p != '/'; ++p); + l = p - fn - 7; + fp = calloc(1, sizeof(knetFile)); + fp->http_host = calloc(l + 1, 1); + strncpy(fp->http_host, fn + 7, l); + fp->http_host[l] = 0; + for (q = fp->http_host; *q && *q != ':'; ++q); + if (*q == ':') *q++ = 0; + // get http_proxy + proxy = getenv("http_proxy"); + // set ->host, ->port and ->path + if (proxy == 0) { + fp->host = strdup(fp->http_host); // when there is no proxy, server name is identical to http_host name. + fp->port = strdup(*q? q : "80"); + fp->path = strdup(*p? p : "/"); + } else { + fp->host = (strstr(proxy, "http://") == proxy)? strdup(proxy + 7) : strdup(proxy); + for (q = fp->host; *q && *q != ':'; ++q); + if (*q == ':') *q++ = 0; + fp->port = strdup(*q? q : "80"); + fp->path = strdup(fn); + } + fp->type = KNF_TYPE_HTTP; + fp->ctrl_fd = fp->fd = -1; + fp->seek_offset = 0; + return fp; +} + +int khttp_connect_file(knetFile *fp) +{ + int ret, l = 0; + char *buf, *p; + if (fp->fd != -1) netclose(fp->fd); + fp->fd = socket_connect(fp->host, fp->port); + buf = calloc(0x10000, 1); // FIXME: I am lazy... But in principle, 64KB should be large enough. + l += sprintf(buf + l, "GET %s HTTP/1.0\r\nHost: %s\r\n", fp->path, fp->http_host); + l += sprintf(buf + l, "Range: bytes=%lld-\r\n", (long long)fp->offset); + l += sprintf(buf + l, "\r\n"); + netwrite(fp->fd, buf, l); + l = 0; + while (netread(fp->fd, buf + l, 1)) { // read HTTP header; FIXME: bad efficiency + if (buf[l] == '\n' && l >= 3) + if (strncmp(buf + l - 3, "\r\n\r\n", 4) == 0) break; + ++l; + } + buf[l] = 0; + if (l < 14) { // prematured header + netclose(fp->fd); + fp->fd = -1; + return -1; + } + ret = strtol(buf + 8, &p, 0); // HTTP return code + if (ret == 200 && fp->offset>0) { // 200 (complete result); then skip beginning of the file + off_t rest = fp->offset; + while (rest) { + off_t l = rest < 0x10000? rest : 0x10000; + rest -= my_netread(fp->fd, buf, l); + } + } else if (ret != 206 && ret != 200) { + free(buf); + fprintf(stderr, "[khttp_connect_file] fail to open file (HTTP code: %d).\n", ret); + netclose(fp->fd); + fp->fd = -1; + return -1; + } + free(buf); + fp->is_ready = 1; + return 0; +} + +/******************** + * Generic routines * + ********************/ + +knetFile *knet_open(const char *fn, const char *mode) +{ + knetFile *fp = 0; + if (mode[0] != 'r') { + fprintf(stderr, "[kftp_open] only mode \"r\" is supported.\n"); + return 0; + } + if (strstr(fn, "ftp://") == fn) { + fp = kftp_parse_url(fn, mode); + if (fp == 0) return 0; + if (kftp_connect(fp) == -1) { + knet_close(fp); + return 0; + } + kftp_connect_file(fp); + } else if (strstr(fn, "http://") == fn) { + fp = khttp_parse_url(fn, mode); + if (fp == 0) return 0; + khttp_connect_file(fp); + } else { // local file +#ifdef _WIN32 + /* In windows, O_BINARY is necessary. In Linux/Mac, O_BINARY may + * be undefined on some systems, although it is defined on my + * Mac and the Linux I have tested on. */ + int fd = open(fn, O_RDONLY | O_BINARY); +#else + int fd = open(fn, O_RDONLY); +#endif + if (fd == -1) { + perror("open"); + return 0; + } + fp = (knetFile*)calloc(1, sizeof(knetFile)); + fp->type = KNF_TYPE_LOCAL; + fp->fd = fd; + fp->ctrl_fd = -1; + } + if (fp && fp->fd == -1) { + knet_close(fp); + return 0; + } + return fp; +} + +knetFile *knet_dopen(int fd, const char *mode) +{ + knetFile *fp = (knetFile*)calloc(1, sizeof(knetFile)); + fp->type = KNF_TYPE_LOCAL; + fp->fd = fd; + return fp; +} + +off_t knet_read(knetFile *fp, void *buf, off_t len) +{ + off_t l = 0; + if (fp->fd == -1) return 0; + if (fp->type == KNF_TYPE_FTP) { + if (fp->is_ready == 0) { + if (!fp->no_reconnect) kftp_reconnect(fp); + kftp_connect_file(fp); + } + } else if (fp->type == KNF_TYPE_HTTP) { + if (fp->is_ready == 0) + khttp_connect_file(fp); + } + if (fp->type == KNF_TYPE_LOCAL) { // on Windows, the following block is necessary; not on UNIX + off_t rest = len, curr; + while (rest) { + do { + curr = read(fp->fd, buf + l, rest); + } while (curr < 0 && EINTR == errno); + if (curr < 0) return -1; + if (curr == 0) break; + l += curr; rest -= curr; + } + } else l = my_netread(fp->fd, buf, len); + fp->offset += l; + return l; +} + +off_t knet_seek(knetFile *fp, int64_t off, int whence) +{ + if (whence == SEEK_SET && off == fp->offset) return 0; + if (fp->type == KNF_TYPE_LOCAL) { + /* Be aware that lseek() returns the offset after seeking, + * while fseek() returns zero on success. */ + off_t offset = lseek(fp->fd, off, whence); + if (offset == -1) { + // Be silent, it is OK for knet_seek to fail when the file is streamed + // fprintf(stderr,"[knet_seek] %s\n", strerror(errno)); + return -1; + } + fp->offset = offset; + return 0; + } + else if (fp->type == KNF_TYPE_FTP) + { + if (whence==SEEK_CUR) + fp->offset += off; + else if (whence==SEEK_SET) + fp->offset = off; + else if ( whence==SEEK_END) + fp->offset = fp->file_size+off; + fp->is_ready = 0; + return 0; + } + else if (fp->type == KNF_TYPE_HTTP) + { + if (whence == SEEK_END) { // FIXME: can we allow SEEK_END in future? + fprintf(stderr, "[knet_seek] SEEK_END is not supported for HTTP. Offset is unchanged.\n"); + errno = ESPIPE; + return -1; + } + if (whence==SEEK_CUR) + fp->offset += off; + else if (whence==SEEK_SET) + fp->offset = off; + fp->is_ready = 0; + return 0; + } + errno = EINVAL; + fprintf(stderr,"[knet_seek] %s\n", strerror(errno)); + return -1; +} + +int knet_close(knetFile *fp) +{ + if (fp == 0) return 0; + if (fp->ctrl_fd != -1) netclose(fp->ctrl_fd); // FTP specific + if (fp->fd != -1) { + /* On Linux/Mac, netclose() is an alias of close(), but on + * Windows, it is an alias of closesocket(). */ + if (fp->type == KNF_TYPE_LOCAL) close(fp->fd); + else netclose(fp->fd); + } + free(fp->host); free(fp->port); + free(fp->response); free(fp->retr); // FTP specific + free(fp->path); free(fp->http_host); // HTTP specific + free(fp); + return 0; +} + +#ifdef KNETFILE_MAIN +int main(void) +{ + char *buf; + knetFile *fp; + int type = 4, l; +#ifdef _WIN32 + knet_win32_init(); +#endif + buf = calloc(0x100000, 1); + if (type == 0) { + fp = knet_open("knetfile.c", "r"); + knet_seek(fp, 1000, SEEK_SET); + } else if (type == 1) { // NCBI FTP, large file + fp = knet_open("ftp://ftp.ncbi.nih.gov/1000genomes/ftp/data/NA12878/alignment/NA12878.chrom6.SLX.SRP000032.2009_06.bam", "r"); + knet_seek(fp, 2500000000ll, SEEK_SET); + l = knet_read(fp, buf, 255); + } else if (type == 2) { + fp = knet_open("ftp://ftp.sanger.ac.uk/pub4/treefam/tmp/index.shtml", "r"); + knet_seek(fp, 1000, SEEK_SET); + } else if (type == 3) { + fp = knet_open("http://www.sanger.ac.uk/Users/lh3/index.shtml", "r"); + knet_seek(fp, 1000, SEEK_SET); + } else if (type == 4) { + fp = knet_open("http://www.sanger.ac.uk/Users/lh3/ex1.bam", "r"); + knet_read(fp, buf, 10000); + knet_seek(fp, 20000, SEEK_SET); + knet_seek(fp, 10000, SEEK_SET); + l = knet_read(fp, buf+10000, 10000000) + 10000; + } + if (type != 4 && type != 1) { + knet_read(fp, buf, 255); + buf[255] = 0; + printf("%s\n", buf); + } else write(fileno(stdout), buf, l); + knet_close(fp); + free(buf); + return 0; +} +#endif diff --git a/samtools-0.1.18/knetfile.h b/samtools-0.1.18/knetfile.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/knetfile.h @@ -0,0 +1,75 @@ +#ifndef KNETFILE_H +#define KNETFILE_H + +#include +#include + +#ifndef _WIN32 +#define netread(fd, ptr, len) read(fd, ptr, len) +#define netwrite(fd, ptr, len) write(fd, ptr, len) +#define netclose(fd) close(fd) +#else +#include +#define netread(fd, ptr, len) recv(fd, ptr, len, 0) +#define netwrite(fd, ptr, len) send(fd, ptr, len, 0) +#define netclose(fd) closesocket(fd) +#endif + +// FIXME: currently I/O is unbuffered + +#define KNF_TYPE_LOCAL 1 +#define KNF_TYPE_FTP 2 +#define KNF_TYPE_HTTP 3 + +typedef struct knetFile_s { + int type, fd; + int64_t offset; + char *host, *port; + + // the following are for FTP only + int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready; + char *response, *retr, *size_cmd; + int64_t seek_offset; // for lazy seek + int64_t file_size; + + // the following are for HTTP only + char *path, *http_host; +} knetFile; + +#define knet_tell(fp) ((fp)->offset) +#define knet_fileno(fp) ((fp)->fd) + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _WIN32 + int knet_win32_init(); + void knet_win32_destroy(); +#endif + + knetFile *knet_open(const char *fn, const char *mode); + + /* + This only works with local files. + */ + knetFile *knet_dopen(int fd, const char *mode); + + /* + If ->is_ready==0, this routine updates ->fd; otherwise, it simply + reads from ->fd. + */ + off_t knet_read(knetFile *fp, void *buf, off_t len); + + /* + This routine only sets ->offset and ->is_ready=0. It does not + communicate with the FTP server. + */ + off_t knet_seek(knetFile *fp, int64_t off, int whence); + int knet_close(knetFile *fp); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/samtools-0.1.18/kprobaln.c b/samtools-0.1.18/kprobaln.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kprobaln.c @@ -0,0 +1,278 @@ +/* The MIT License + + Copyright (c) 2003-2006, 2008-2010, by Heng Li + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include +#include +#include +#include +#include +#include "kprobaln.h" + +/***************************************** + * Probabilistic banded glocal alignment * + *****************************************/ + +#define EI .25 +#define EM .33333333333 + +static float g_qual2prob[256]; + +#define set_u(u, b, i, k) { int x=(i)-(b); x=x>0?x:0; (u)=((k)-x+1)*3; } + +kpa_par_t kpa_par_def = { 0.001, 0.1, 10 }; +kpa_par_t kpa_par_alt = { 0.0001, 0.01, 10 }; + +/* + The topology of the profile HMM: + + /\ /\ /\ /\ + I[1] I[k-1] I[k] I[L] + ^ \ \ ^ \ ^ \ \ ^ + | \ \ | \ | \ \ | + M[0] M[1] -> ... -> M[k-1] -> M[k] -> ... -> M[L] M[L+1] + \ \/ \/ \/ / + \ /\ /\ /\ / + -> D[k-1] -> D[k] -> + + M[0] points to every {M,I}[k] and every {M,I}[k] points M[L+1]. + + On input, _ref is the reference sequence and _query is the query + sequence. Both are sequences of 0/1/2/3/4 where 4 stands for an + ambiguous residue. iqual is the base quality. c sets the gap open + probability, gap extension probability and band width. + + On output, state and q are arrays of length l_query. The higher 30 + bits give the reference position the query base is matched to and the + lower two bits can be 0 (an alignment match) or 1 (an + insertion). q[i] gives the phred scaled posterior probability of + state[i] being wrong. + */ +int kpa_glocal(const uint8_t *_ref, int l_ref, const uint8_t *_query, int l_query, const uint8_t *iqual, + const kpa_par_t *c, int *state, uint8_t *q) +{ + double **f, **b = 0, *s, m[9], sI, sM, bI, bM, pb; + float *qual, *_qual; + const uint8_t *ref, *query; + int bw, bw2, i, k, is_diff = 0, is_backward = 1, Pr; + + /*** initialization ***/ + is_backward = state && q? 1 : 0; + ref = _ref - 1; query = _query - 1; // change to 1-based coordinate + bw = l_ref > l_query? l_ref : l_query; + if (bw > c->bw) bw = c->bw; + if (bw < abs(l_ref - l_query)) bw = abs(l_ref - l_query); + bw2 = bw * 2 + 1; + // allocate the forward and backward matrices f[][] and b[][] and the scaling array s[] + f = calloc(l_query+1, sizeof(void*)); + if (is_backward) b = calloc(l_query+1, sizeof(void*)); + for (i = 0; i <= l_query; ++i) { + f[i] = calloc(bw2 * 3 + 6, sizeof(double)); // FIXME: this is over-allocated for very short seqs + if (is_backward) b[i] = calloc(bw2 * 3 + 6, sizeof(double)); + } + s = calloc(l_query+2, sizeof(double)); // s[] is the scaling factor to avoid underflow + // initialize qual + _qual = calloc(l_query, sizeof(float)); + if (g_qual2prob[0] == 0) + for (i = 0; i < 256; ++i) + g_qual2prob[i] = pow(10, -i/10.); + for (i = 0; i < l_query; ++i) _qual[i] = g_qual2prob[iqual? iqual[i] : 30]; + qual = _qual - 1; + // initialize transition probability + sM = sI = 1. / (2 * l_query + 2); // the value here seems not to affect results; FIXME: need proof + m[0*3+0] = (1 - c->d - c->d) * (1 - sM); m[0*3+1] = m[0*3+2] = c->d * (1 - sM); + m[1*3+0] = (1 - c->e) * (1 - sI); m[1*3+1] = c->e * (1 - sI); m[1*3+2] = 0.; + m[2*3+0] = 1 - c->e; m[2*3+1] = 0.; m[2*3+2] = c->e; + bM = (1 - c->d) / l_ref; bI = c->d / l_ref; // (bM+bI)*l_ref==1 + /*** forward ***/ + // f[0] + set_u(k, bw, 0, 0); + f[0][k] = s[0] = 1.; + { // f[1] + double *fi = f[1], sum; + int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1, _beg, _end; + for (k = beg, sum = 0.; k <= end; ++k) { + int u; + double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] * EM; + set_u(u, bw, 1, k); + fi[u+0] = e * bM; fi[u+1] = EI * bI; + sum += fi[u] + fi[u+1]; + } + // rescale + s[1] = sum; + set_u(_beg, bw, 1, beg); set_u(_end, bw, 1, end); _end += 2; + for (k = _beg; k <= _end; ++k) fi[k] /= sum; + } + // f[2..l_query] + for (i = 2; i <= l_query; ++i) { + double *fi = f[i], *fi1 = f[i-1], sum, qli = qual[i]; + int beg = 1, end = l_ref, x, _beg, _end; + uint8_t qyi = query[i]; + x = i - bw; beg = beg > x? beg : x; // band start + x = i + bw; end = end < x? end : x; // band end + for (k = beg, sum = 0.; k <= end; ++k) { + int u, v11, v01, v10; + double e; + e = (ref[k] > 3 || qyi > 3)? 1. : ref[k] == qyi? 1. - qli : qli * EM; + set_u(u, bw, i, k); set_u(v11, bw, i-1, k-1); set_u(v10, bw, i-1, k); set_u(v01, bw, i, k-1); + fi[u+0] = e * (m[0] * fi1[v11+0] + m[3] * fi1[v11+1] + m[6] * fi1[v11+2]); + fi[u+1] = EI * (m[1] * fi1[v10+0] + m[4] * fi1[v10+1]); + fi[u+2] = m[2] * fi[v01+0] + m[8] * fi[v01+2]; + sum += fi[u] + fi[u+1] + fi[u+2]; +// fprintf(stderr, "F (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, fi[u], fi[u+1], fi[u+2]); // DEBUG + } + // rescale + s[i] = sum; + set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2; + for (k = _beg, sum = 1./sum; k <= _end; ++k) fi[k] *= sum; + } + { // f[l_query+1] + double sum; + for (k = 1, sum = 0.; k <= l_ref; ++k) { + int u; + set_u(u, bw, l_query, k); + if (u < 3 || u >= bw2*3+3) continue; + sum += f[l_query][u+0] * sM + f[l_query][u+1] * sI; + } + s[l_query+1] = sum; // the last scaling factor + } + { // compute likelihood + double p = 1., Pr1 = 0.; + for (i = 0; i <= l_query + 1; ++i) { + p *= s[i]; + if (p < 1e-100) Pr1 += -4.343 * log(p), p = 1.; + } + Pr1 += -4.343 * log(p * l_ref * l_query); + Pr = (int)(Pr1 + .499); + if (!is_backward) { // skip backward and MAP + for (i = 0; i <= l_query; ++i) free(f[i]); + free(f); free(s); free(_qual); + return Pr; + } + } + /*** backward ***/ + // b[l_query] (b[l_query+1][0]=1 and thus \tilde{b}[][]=1/s[l_query+1]; this is where s[l_query+1] comes from) + for (k = 1; k <= l_ref; ++k) { + int u; + double *bi = b[l_query]; + set_u(u, bw, l_query, k); + if (u < 3 || u >= bw2*3+3) continue; + bi[u+0] = sM / s[l_query] / s[l_query+1]; bi[u+1] = sI / s[l_query] / s[l_query+1]; + } + // b[l_query-1..1] + for (i = l_query - 1; i >= 1; --i) { + int beg = 1, end = l_ref, x, _beg, _end; + double *bi = b[i], *bi1 = b[i+1], y = (i > 1), qli1 = qual[i+1]; + uint8_t qyi1 = query[i+1]; + x = i - bw; beg = beg > x? beg : x; + x = i + bw; end = end < x? end : x; + for (k = end; k >= beg; --k) { + int u, v11, v01, v10; + double e; + set_u(u, bw, i, k); set_u(v11, bw, i+1, k+1); set_u(v10, bw, i+1, k); set_u(v01, bw, i, k+1); + e = (k >= l_ref? 0 : (ref[k+1] > 3 || qyi1 > 3)? 1. : ref[k+1] == qyi1? 1. - qli1 : qli1 * EM) * bi1[v11]; + bi[u+0] = e * m[0] + EI * m[1] * bi1[v10+1] + m[2] * bi[v01+2]; // bi1[v11] has been foled into e. + bi[u+1] = e * m[3] + EI * m[4] * bi1[v10+1]; + bi[u+2] = (e * m[6] + m[8] * bi[v01+2]) * y; +// fprintf(stderr, "B (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, bi[u], bi[u+1], bi[u+2]); // DEBUG + } + // rescale + set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2; + for (k = _beg, y = 1./s[i]; k <= _end; ++k) bi[k] *= y; + } + { // b[0] + int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1; + double sum = 0.; + for (k = end; k >= beg; --k) { + int u; + double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] * EM; + set_u(u, bw, 1, k); + if (u < 3 || u >= bw2*3+3) continue; + sum += e * b[1][u+0] * bM + EI * b[1][u+1] * bI; + } + set_u(k, bw, 0, 0); + pb = b[0][k] = sum / s[0]; // if everything works as is expected, pb == 1.0 + } + is_diff = fabs(pb - 1.) > 1e-7? 1 : 0; + /*** MAP ***/ + for (i = 1; i <= l_query; ++i) { + double sum = 0., *fi = f[i], *bi = b[i], max = 0.; + int beg = 1, end = l_ref, x, max_k = -1; + x = i - bw; beg = beg > x? beg : x; + x = i + bw; end = end < x? end : x; + for (k = beg; k <= end; ++k) { + int u; + double z; + set_u(u, bw, i, k); + z = fi[u+0] * bi[u+0]; if (z > max) max = z, max_k = (k-1)<<2 | 0; sum += z; + z = fi[u+1] * bi[u+1]; if (z > max) max = z, max_k = (k-1)<<2 | 1; sum += z; + } + max /= sum; sum *= s[i]; // if everything works as is expected, sum == 1.0 + if (state) state[i-1] = max_k; + if (q) k = (int)(-4.343 * log(1. - max) + .499), q[i-1] = k > 100? 99 : k; +#ifdef _MAIN + fprintf(stderr, "(%.10lg,%.10lg) (%d,%d:%c,%c:%d) %lg\n", pb, sum, i-1, max_k>>2, + "ACGT"[query[i]], "ACGT"[ref[(max_k>>2)+1]], max_k&3, max); // DEBUG +#endif + } + /*** free ***/ + for (i = 0; i <= l_query; ++i) { + free(f[i]); free(b[i]); + } + free(f); free(b); free(s); free(_qual); + return Pr; +} + +#ifdef _MAIN +#include +int main(int argc, char *argv[]) +{ + uint8_t conv[256], *iqual, *ref, *query; + int c, l_ref, l_query, i, q = 30, b = 10, P; + while ((c = getopt(argc, argv, "b:q:")) >= 0) { + switch (c) { + case 'b': b = atoi(optarg); break; + case 'q': q = atoi(optarg); break; + } + } + if (optind + 2 > argc) { + fprintf(stderr, "Usage: %s [-q %d] [-b %d] \n", argv[0], q, b); // example: acttc attc + return 1; + } + memset(conv, 4, 256); + conv['a'] = conv['A'] = 0; conv['c'] = conv['C'] = 1; + conv['g'] = conv['G'] = 2; conv['t'] = conv['T'] = 3; + ref = (uint8_t*)argv[optind]; query = (uint8_t*)argv[optind+1]; + l_ref = strlen((char*)ref); l_query = strlen((char*)query); + for (i = 0; i < l_ref; ++i) ref[i] = conv[ref[i]]; + for (i = 0; i < l_query; ++i) query[i] = conv[query[i]]; + iqual = malloc(l_query); + memset(iqual, q, l_query); + kpa_par_def.bw = b; + P = kpa_glocal(ref, l_ref, query, l_query, iqual, &kpa_par_alt, 0, 0); + fprintf(stderr, "%d\n", P); + free(iqual); + return 0; +} +#endif diff --git a/samtools-0.1.18/kprobaln.h b/samtools-0.1.18/kprobaln.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kprobaln.h @@ -0,0 +1,49 @@ +/* The MIT License + + Copyright (c) 2003-2006, 2008, 2009 by Heng Li + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef LH3_KPROBALN_H_ +#define LH3_KPROBALN_H_ + +#include + +typedef struct { + float d, e; + int bw; +} kpa_par_t; + +#ifdef __cplusplus +extern "C" { +#endif + + int kpa_glocal(const uint8_t *_ref, int l_ref, const uint8_t *_query, int l_query, const uint8_t *iqual, + const kpa_par_t *c, int *state, uint8_t *q); + +#ifdef __cplusplus +} +#endif + +extern kpa_par_t kpa_par_def, kpa_par_alt; + +#endif diff --git a/samtools-0.1.18/kseq.h b/samtools-0.1.18/kseq.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kseq.h @@ -0,0 +1,224 @@ +/* The MIT License + + Copyright (c) 2008, 2009, 2011 Attractive Chaos + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* Last Modified: 18AUG2011 */ + +#ifndef AC_KSEQ_H +#define AC_KSEQ_H + +#include +#include +#include + +#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r +#define KS_SEP_TAB 1 // isspace() && !' ' +#define KS_SEP_MAX 1 + +#define __KS_TYPE(type_t) \ + typedef struct __kstream_t { \ + unsigned char *buf; \ + int begin, end, is_eof; \ + type_t f; \ + } kstream_t; + +#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end) +#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0) + +#define __KS_BASIC(type_t, __bufsize) \ + static inline kstream_t *ks_init(type_t f) \ + { \ + kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t)); \ + ks->f = f; \ + ks->buf = malloc(__bufsize); \ + return ks; \ + } \ + static inline void ks_destroy(kstream_t *ks) \ + { \ + if (ks) { \ + free(ks->buf); \ + free(ks); \ + } \ + } + +#define __KS_GETC(__read, __bufsize) \ + static inline int ks_getc(kstream_t *ks) \ + { \ + if (ks->is_eof && ks->begin >= ks->end) return -1; \ + if (ks->begin >= ks->end) { \ + ks->begin = 0; \ + ks->end = __read(ks->f, ks->buf, __bufsize); \ + if (ks->end < __bufsize) ks->is_eof = 1; \ + if (ks->end == 0) return -1; \ + } \ + return (int)ks->buf[ks->begin++]; \ + } + +#ifndef KSTRING_T +#define KSTRING_T kstring_t +typedef struct __kstring_t { + size_t l, m; + char *s; +} kstring_t; +#endif + +#ifndef kroundup32 +#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) +#endif + +#define __KS_GETUNTIL(__read, __bufsize) \ + static int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append) \ + { \ + if (dret) *dret = 0; \ + str->l = append? str->l : 0; \ + if (ks->begin >= ks->end && ks->is_eof) return -1; \ + for (;;) { \ + int i; \ + if (ks->begin >= ks->end) { \ + if (!ks->is_eof) { \ + ks->begin = 0; \ + ks->end = __read(ks->f, ks->buf, __bufsize); \ + if (ks->end < __bufsize) ks->is_eof = 1; \ + if (ks->end == 0) break; \ + } else break; \ + } \ + if (delimiter > KS_SEP_MAX) { \ + for (i = ks->begin; i < ks->end; ++i) \ + if (ks->buf[i] == delimiter) break; \ + } else if (delimiter == KS_SEP_SPACE) { \ + for (i = ks->begin; i < ks->end; ++i) \ + if (isspace(ks->buf[i])) break; \ + } else if (delimiter == KS_SEP_TAB) { \ + for (i = ks->begin; i < ks->end; ++i) \ + if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break; \ + } else i = 0; /* never come to here! */ \ + if (str->m - str->l < i - ks->begin + 1) { \ + str->m = str->l + (i - ks->begin) + 1; \ + kroundup32(str->m); \ + str->s = (char*)realloc(str->s, str->m); \ + } \ + memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin); \ + str->l = str->l + (i - ks->begin); \ + ks->begin = i + 1; \ + if (i < ks->end) { \ + if (dret) *dret = ks->buf[i]; \ + break; \ + } \ + } \ + if (str->s == 0) { \ + str->m = 1; \ + str->s = (char*)calloc(1, 1); \ + } \ + str->s[str->l] = '\0'; \ + return str->l; \ + } \ + static inline int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \ + { return ks_getuntil2(ks, delimiter, str, dret, 0); } + +#define KSTREAM_INIT(type_t, __read, __bufsize) \ + __KS_TYPE(type_t) \ + __KS_BASIC(type_t, __bufsize) \ + __KS_GETC(__read, __bufsize) \ + __KS_GETUNTIL(__read, __bufsize) + +#define __KSEQ_BASIC(type_t) \ + static inline kseq_t *kseq_init(type_t fd) \ + { \ + kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t)); \ + s->f = ks_init(fd); \ + return s; \ + } \ + static inline void kseq_rewind(kseq_t *ks) \ + { \ + ks->last_char = 0; \ + ks->f->is_eof = ks->f->begin = ks->f->end = 0; \ + } \ + static inline void kseq_destroy(kseq_t *ks) \ + { \ + if (!ks) return; \ + free(ks->name.s); free(ks->comment.s); free(ks->seq.s); free(ks->qual.s); \ + ks_destroy(ks->f); \ + free(ks); \ + } + +/* Return value: + >=0 length of the sequence (normal) + -1 end-of-file + -2 truncated quality string + */ +#define __KSEQ_READ \ + static int kseq_read(kseq_t *seq) \ + { \ + int c; \ + kstream_t *ks = seq->f; \ + if (seq->last_char == 0) { /* then jump to the next header line */ \ + while ((c = ks_getc(ks)) != -1 && c != '>' && c != '@'); \ + if (c == -1) return -1; /* end of file */ \ + seq->last_char = c; \ + } /* else: the first header char has been read in the previous call */ \ + seq->comment.l = seq->seq.l = seq->qual.l = 0; /* reset all members */ \ + if (ks_getuntil(ks, 0, &seq->name, &c) < 0) return -1; /* normal exit: EOF */ \ + if (c != '\n') ks_getuntil(ks, '\n', &seq->comment, 0); /* read FASTA/Q comment */ \ + if (seq->seq.s == 0) { /* we can do this in the loop below, but that is slower */ \ + seq->seq.m = 256; \ + seq->seq.s = (char*)malloc(seq->seq.m); \ + } \ + while ((c = ks_getc(ks)) != -1 && c != '>' && c != '+' && c != '@') { \ + seq->seq.s[seq->seq.l++] = c; /* this is safe: we always have enough space for 1 char */ \ + ks_getuntil2(ks, '\n', &seq->seq, 0, 1); /* read the rest of the line */ \ + } \ + if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */ \ + if (seq->seq.l + 1 >= seq->seq.m) { /* seq->seq.s[seq->seq.l] below may be out of boundary */ \ + seq->seq.m = seq->seq.l + 2; \ + kroundup32(seq->seq.m); /* rounded to the next closest 2^k */ \ + seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \ + } \ + seq->seq.s[seq->seq.l] = 0; /* null terminated string */ \ + if (c != '+') return seq->seq.l; /* FASTA */ \ + if (seq->qual.m < seq->seq.m) { /* allocate memory for qual in case insufficient */ \ + seq->qual.m = seq->seq.m; \ + seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \ + } \ + while ((c = ks_getc(ks)) != -1 && c != '\n'); /* skip the rest of '+' line */ \ + if (c == -1) return -2; /* error: no quality string */ \ + while (ks_getuntil2(ks, '\n', &seq->qual, 0, 1) >= 0 && seq->qual.l < seq->seq.l); \ + seq->last_char = 0; /* we have not come to the next header line */ \ + if (seq->seq.l != seq->qual.l) return -2; /* error: qual string is of a different length */ \ + return seq->seq.l; \ + } + +#define __KSEQ_TYPE(type_t) \ + typedef struct { \ + kstring_t name, comment, seq, qual; \ + int last_char; \ + kstream_t *f; \ + } kseq_t; + +#define KSEQ_INIT(type_t, __read) \ + KSTREAM_INIT(type_t, __read, 16384) \ + __KSEQ_TYPE(type_t) \ + __KSEQ_BASIC(type_t) \ + __KSEQ_READ + +#endif diff --git a/samtools-0.1.18/ksort.h b/samtools-0.1.18/ksort.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/ksort.h @@ -0,0 +1,281 @@ +/* The MIT License + + Copyright (c) 2008 Genome Research Ltd (GRL). + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* Contact: Heng Li */ + +/* + 2008-11-16 (0.1.4): + + * Fixed a bug in introsort() that happens in rare cases. + + 2008-11-05 (0.1.3): + + * Fixed a bug in introsort() for complex comparisons. + + * Fixed a bug in mergesort(). The previous version is not stable. + + 2008-09-15 (0.1.2): + + * Accelerated introsort. On my Mac (not on another Linux machine), + my implementation is as fast as std::sort on random input. + + * Added combsort and in introsort, switch to combsort if the + recursion is too deep. + + 2008-09-13 (0.1.1): + + * Added k-small algorithm + + 2008-09-05 (0.1.0): + + * Initial version + +*/ + +#ifndef AC_KSORT_H +#define AC_KSORT_H + +#include +#include + +typedef struct { + void *left, *right; + int depth; +} ks_isort_stack_t; + +#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; } + +#define KSORT_INIT(name, type_t, __sort_lt) \ + void ks_mergesort_##name(size_t n, type_t array[], type_t temp[]) \ + { \ + type_t *a2[2], *a, *b; \ + int curr, shift; \ + \ + a2[0] = array; \ + a2[1] = temp? temp : (type_t*)malloc(sizeof(type_t) * n); \ + for (curr = 0, shift = 0; (1ul<> 1) - 1; i != (size_t)(-1); --i) \ + ks_heapadjust_##name(i, lsize, l); \ + } \ + void ks_heapsort_##name(size_t lsize, type_t l[]) \ + { \ + size_t i; \ + for (i = lsize - 1; i > 0; --i) { \ + type_t tmp; \ + tmp = *l; *l = l[i]; l[i] = tmp; ks_heapadjust_##name(0, i, l); \ + } \ + } \ + inline void __ks_insertsort_##name(type_t *s, type_t *t) \ + { \ + type_t *i, *j, swap_tmp; \ + for (i = s + 1; i < t; ++i) \ + for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) { \ + swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp; \ + } \ + } \ + void ks_combsort_##name(size_t n, type_t a[]) \ + { \ + const double shrink_factor = 1.2473309501039786540366528676643; \ + int do_swap; \ + size_t gap = n; \ + type_t tmp, *i, *j; \ + do { \ + if (gap > 2) { \ + gap = (size_t)(gap / shrink_factor); \ + if (gap == 9 || gap == 10) gap = 11; \ + } \ + do_swap = 0; \ + for (i = a; i < a + n - gap; ++i) { \ + j = i + gap; \ + if (__sort_lt(*j, *i)) { \ + tmp = *i; *i = *j; *j = tmp; \ + do_swap = 1; \ + } \ + } \ + } while (do_swap || gap > 2); \ + if (gap != 1) __ks_insertsort_##name(a, a + n); \ + } \ + void ks_introsort_##name(size_t n, type_t a[]) \ + { \ + int d; \ + ks_isort_stack_t *top, *stack; \ + type_t rp, swap_tmp; \ + type_t *s, *t, *i, *j, *k; \ + \ + if (n < 1) return; \ + else if (n == 2) { \ + if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \ + return; \ + } \ + for (d = 2; 1ul<>1) + 1; \ + if (__sort_lt(*k, *i)) { \ + if (__sort_lt(*k, *j)) k = j; \ + } else k = __sort_lt(*j, *i)? i : j; \ + rp = *k; \ + if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; } \ + for (;;) { \ + do ++i; while (__sort_lt(*i, rp)); \ + do --j; while (i <= j && __sort_lt(rp, *j)); \ + if (j <= i) break; \ + swap_tmp = *i; *i = *j; *j = swap_tmp; \ + } \ + swap_tmp = *i; *i = *t; *t = swap_tmp; \ + if (i-s > t-i) { \ + if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \ + s = t-i > 16? i+1 : t; \ + } else { \ + if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \ + t = i-s > 16? i-1 : s; \ + } \ + } else { \ + if (top == stack) { \ + free(stack); \ + __ks_insertsort_##name(a, a+n); \ + return; \ + } else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \ + } \ + } \ + } \ + /* This function is adapted from: http://ndevilla.free.fr/median/ */ \ + /* 0 <= kk < n */ \ + type_t ks_ksmall_##name(size_t n, type_t arr[], size_t kk) \ + { \ + type_t *low, *high, *k, *ll, *hh, *mid; \ + low = arr; high = arr + n - 1; k = arr + kk; \ + for (;;) { \ + if (high <= low) return *k; \ + if (high == low + 1) { \ + if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \ + return *k; \ + } \ + mid = low + (high - low) / 2; \ + if (__sort_lt(*high, *mid)) KSORT_SWAP(type_t, *mid, *high); \ + if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \ + if (__sort_lt(*low, *mid)) KSORT_SWAP(type_t, *mid, *low); \ + KSORT_SWAP(type_t, *mid, *(low+1)); \ + ll = low + 1; hh = high; \ + for (;;) { \ + do ++ll; while (__sort_lt(*ll, *low)); \ + do --hh; while (__sort_lt(*low, *hh)); \ + if (hh < ll) break; \ + KSORT_SWAP(type_t, *ll, *hh); \ + } \ + KSORT_SWAP(type_t, *low, *hh); \ + if (hh <= k) low = ll; \ + if (hh >= k) high = hh - 1; \ + } \ + } \ + void ks_shuffle_##name(size_t n, type_t a[]) \ + { \ + int i, j; \ + for (i = n; i > 1; --i) { \ + type_t tmp; \ + j = (int)(drand48() * i); \ + tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp; \ + } \ + } + +#define ks_mergesort(name, n, a, t) ks_mergesort_##name(n, a, t) +#define ks_introsort(name, n, a) ks_introsort_##name(n, a) +#define ks_combsort(name, n, a) ks_combsort_##name(n, a) +#define ks_heapsort(name, n, a) ks_heapsort_##name(n, a) +#define ks_heapmake(name, n, a) ks_heapmake_##name(n, a) +#define ks_heapadjust(name, i, n, a) ks_heapadjust_##name(i, n, a) +#define ks_ksmall(name, n, a, k) ks_ksmall_##name(n, a, k) +#define ks_shuffle(name, n, a) ks_shuffle_##name(n, a) + +#define ks_lt_generic(a, b) ((a) < (b)) +#define ks_lt_str(a, b) (strcmp((a), (b)) < 0) + +typedef const char *ksstr_t; + +#define KSORT_INIT_GENERIC(type_t) KSORT_INIT(type_t, type_t, ks_lt_generic) +#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str) + +#endif diff --git a/samtools-0.1.18/kstring.c b/samtools-0.1.18/kstring.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kstring.c @@ -0,0 +1,212 @@ +#include +#include +#include +#include +#include +#include "kstring.h" + +int ksprintf(kstring_t *s, const char *fmt, ...) +{ + va_list ap; + int l; + va_start(ap, fmt); + l = vsnprintf(s->s + s->l, s->m - s->l, fmt, ap); // This line does not work with glibc 2.0. See `man snprintf'. + va_end(ap); + if (l + 1 > s->m - s->l) { + s->m = s->l + l + 2; + kroundup32(s->m); + s->s = (char*)realloc(s->s, s->m); + va_start(ap, fmt); + l = vsnprintf(s->s + s->l, s->m - s->l, fmt, ap); + } + va_end(ap); + s->l += l; + return l; +} + +char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux) +{ + const char *p, *start; + if (sep) { // set up the table + if (str == 0 && (aux->tab[0]&1)) return 0; // no need to set up if we have finished + aux->finished = 0; + if (sep[1]) { + aux->sep = -1; + aux->tab[0] = aux->tab[1] = aux->tab[2] = aux->tab[3] = 0; + for (p = sep; *p; ++p) aux->tab[*p>>6] |= 1ull<<(*p&0x3f); + } else aux->sep = sep[0]; + } + if (aux->finished) return 0; + else if (str) aux->p = str - 1, aux->finished = 0; + if (aux->sep < 0) { + for (p = start = aux->p + 1; *p; ++p) + if (aux->tab[*p>>6]>>(*p&0x3f)&1) break; + } else { + for (p = start = aux->p + 1; *p; ++p) + if (*p == aux->sep) break; + } + aux->p = p; // end of token + if (*p == 0) aux->finished = 1; // no more tokens + return (char*)start; +} + +// s MUST BE a null terminated string; l = strlen(s) +int ksplit_core(char *s, int delimiter, int *_max, int **_offsets) +{ + int i, n, max, last_char, last_start, *offsets, l; + n = 0; max = *_max; offsets = *_offsets; + l = strlen(s); + +#define __ksplit_aux do { \ + if (_offsets) { \ + s[i] = 0; \ + if (n == max) { \ + max = max? max<<1 : 2; \ + offsets = (int*)realloc(offsets, sizeof(int) * max); \ + } \ + offsets[n++] = last_start; \ + } else ++n; \ + } while (0) + + for (i = 0, last_char = last_start = 0; i <= l; ++i) { + if (delimiter == 0) { + if (isspace(s[i]) || s[i] == 0) { + if (isgraph(last_char)) __ksplit_aux; // the end of a field + } else { + if (isspace(last_char) || last_char == 0) last_start = i; + } + } else { + if (s[i] == delimiter || s[i] == 0) { + if (last_char != 0 && last_char != delimiter) __ksplit_aux; // the end of a field + } else { + if (last_char == delimiter || last_char == 0) last_start = i; + } + } + last_char = s[i]; + } + *_max = max; *_offsets = offsets; + return n; +} + +/********************** + * Boyer-Moore search * + **********************/ + +typedef unsigned char ubyte_t; + +// reference: http://www-igm.univ-mlv.fr/~lecroq/string/node14.html +static int *ksBM_prep(const ubyte_t *pat, int m) +{ + int i, *suff, *prep, *bmGs, *bmBc; + prep = calloc(m + 256, sizeof(int)); + bmGs = prep; bmBc = prep + m; + { // preBmBc() + for (i = 0; i < 256; ++i) bmBc[i] = m; + for (i = 0; i < m - 1; ++i) bmBc[pat[i]] = m - i - 1; + } + suff = calloc(m, sizeof(int)); + { // suffixes() + int f = 0, g; + suff[m - 1] = m; + g = m - 1; + for (i = m - 2; i >= 0; --i) { + if (i > g && suff[i + m - 1 - f] < i - g) + suff[i] = suff[i + m - 1 - f]; + else { + if (i < g) g = i; + f = i; + while (g >= 0 && pat[g] == pat[g + m - 1 - f]) --g; + suff[i] = f - g; + } + } + } + { // preBmGs() + int j = 0; + for (i = 0; i < m; ++i) bmGs[i] = m; + for (i = m - 1; i >= 0; --i) + if (suff[i] == i + 1) + for (; j < m - 1 - i; ++j) + if (bmGs[j] == m) + bmGs[j] = m - 1 - i; + for (i = 0; i <= m - 2; ++i) + bmGs[m - 1 - suff[i]] = m - 1 - i; + } + free(suff); + return prep; +} + +void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep) +{ + int i, j, *prep = 0, *bmGs, *bmBc; + const ubyte_t *str, *pat; + str = (const ubyte_t*)_str; pat = (const ubyte_t*)_pat; + prep = (_prep == 0 || *_prep == 0)? ksBM_prep(pat, m) : *_prep; + if (_prep && *_prep == 0) *_prep = prep; + bmGs = prep; bmBc = prep + m; + j = 0; + while (j <= n - m) { + for (i = m - 1; i >= 0 && pat[i] == str[i+j]; --i); + if (i >= 0) { + int max = bmBc[str[i+j]] - m + 1 + i; + if (max < bmGs[i]) max = bmGs[i]; + j += max; + } else return (void*)(str + j); + } + if (_prep == 0) free(prep); + return 0; +} + +char *kstrstr(const char *str, const char *pat, int **_prep) +{ + return (char*)kmemmem(str, strlen(str), pat, strlen(pat), _prep); +} + +char *kstrnstr(const char *str, const char *pat, int n, int **_prep) +{ + return (char*)kmemmem(str, n, pat, strlen(pat), _prep); +} + +/*********************** + * The main() function * + ***********************/ + +#ifdef KSTRING_MAIN +#include +int main() +{ + kstring_t *s; + int *fields, n, i; + ks_tokaux_t aux; + char *p; + s = (kstring_t*)calloc(1, sizeof(kstring_t)); + // test ksprintf() + ksprintf(s, " abcdefg: %d ", 100); + printf("'%s'\n", s->s); + // test ksplit() + fields = ksplit(s, 0, &n); + for (i = 0; i < n; ++i) + printf("field[%d] = '%s'\n", i, s->s + fields[i]); + // test kstrtok() + s->l = 0; + for (p = kstrtok("ab:cde:fg/hij::k", ":/", &aux); p; p = kstrtok(0, 0, &aux)) { + kputsn(p, aux.p - p, s); + kputc('\n', s); + } + printf("%s", s->s); + // free + free(s->s); free(s); free(fields); + + { + static char *str = "abcdefgcdgcagtcakcdcd"; + static char *pat = "cd"; + char *ret, *s = str; + int *prep = 0; + while ((ret = kstrstr(s, pat, &prep)) != 0) { + printf("match: %s\n", ret); + s = ret + prep[0]; + } + free(prep); + } + return 0; +} +#endif diff --git a/samtools-0.1.18/kstring.h b/samtools-0.1.18/kstring.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/kstring.h @@ -0,0 +1,117 @@ +#ifndef KSTRING_H +#define KSTRING_H + +#include +#include +#include + +#ifndef kroundup32 +#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) +#endif + +#ifndef KSTRING_T +#define KSTRING_T kstring_t +typedef struct __kstring_t { + size_t l, m; + char *s; +} kstring_t; +#endif + +typedef struct { + uint64_t tab[4]; + int sep, finished; + const char *p; // end of the current token +} ks_tokaux_t; + +#ifdef __cplusplus +extern "C" { +#endif + + int ksprintf(kstring_t *s, const char *fmt, ...); + int ksplit_core(char *s, int delimiter, int *_max, int **_offsets); + char *kstrstr(const char *str, const char *pat, int **_prep); + char *kstrnstr(const char *str, const char *pat, int n, int **_prep); + void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep); + + /* kstrtok() is similar to strtok_r() except that str is not + * modified and both str and sep can be NULL. For efficiency, it is + * actually recommended to set both to NULL in the subsequent calls + * if sep is not changed. */ + char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux); + +#ifdef __cplusplus +} +#endif + +static inline int kputsn(const char *p, int l, kstring_t *s) +{ + if (s->l + l + 1 >= s->m) { + s->m = s->l + l + 2; + kroundup32(s->m); + s->s = (char*)realloc(s->s, s->m); + } + memcpy(s->s + s->l, p, l); + s->l += l; + s->s[s->l] = 0; + return l; +} + +static inline int kputs(const char *p, kstring_t *s) +{ + return kputsn(p, strlen(p), s); +} + +static inline int kputc(int c, kstring_t *s) +{ + if (s->l + 1 >= s->m) { + s->m = s->l + 2; + kroundup32(s->m); + s->s = (char*)realloc(s->s, s->m); + } + s->s[s->l++] = c; + s->s[s->l] = 0; + return c; +} + +static inline int kputw(int c, kstring_t *s) +{ + char buf[16]; + int l, x; + if (c == 0) return kputc('0', s); + for (l = 0, x = c < 0? -c : c; x > 0; x /= 10) buf[l++] = x%10 + '0'; + if (c < 0) buf[l++] = '-'; + if (s->l + l + 1 >= s->m) { + s->m = s->l + l + 2; + kroundup32(s->m); + s->s = (char*)realloc(s->s, s->m); + } + for (x = l - 1; x >= 0; --x) s->s[s->l++] = buf[x]; + s->s[s->l] = 0; + return 0; +} + +static inline int kputuw(unsigned c, kstring_t *s) +{ + char buf[16]; + int l, i; + unsigned x; + if (c == 0) return kputc('0', s); + for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0'; + if (s->l + l + 1 >= s->m) { + s->m = s->l + l + 2; + kroundup32(s->m); + s->s = (char*)realloc(s->s, s->m); + } + for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i]; + s->s[s->l] = 0; + return 0; +} + +static inline int *ksplit(kstring_t *s, int delimiter, int *n) +{ + int max = 0, *offsets = 0; + *n = ksplit_core(s->s, delimiter, &max, &offsets); + return offsets; +} + +#endif diff --git a/samtools-0.1.18/razf.c b/samtools-0.1.18/razf.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/razf.c @@ -0,0 +1,853 @@ +/* + * RAZF : Random Access compressed(Z) File + * Version: 1.0 + * Release Date: 2008-10-27 + * + * Copyright 2008, Jue Ruan , Heng Li + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _NO_RAZF + +#include +#include +#include +#include +#include +#include "razf.h" + + +#if ZLIB_VERNUM < 0x1221 +struct _gz_header_s { + int text; + uLong time; + int xflags; + int os; + Bytef *extra; + uInt extra_len; + uInt extra_max; + Bytef *name; + uInt name_max; + Bytef *comment; + uInt comm_max; + int hcrc; + int done; +}; +#warning "zlib < 1.2.2.1; RAZF writing is disabled." +#endif + +#define DEF_MEM_LEVEL 8 + +static inline uint32_t byte_swap_4(uint32_t v){ + v = ((v & 0x0000FFFFU) << 16) | (v >> 16); + return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8); +} + +static inline uint64_t byte_swap_8(uint64_t v){ + v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32); + v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16); + return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8); +} + +static inline int is_big_endian(){ + int x = 0x01; + char *c = (char*)&x; + return (c[0] != 0x01); +} + +#ifndef _RZ_READONLY +static void add_zindex(RAZF *rz, int64_t in, int64_t out){ + if(rz->index->size == rz->index->cap){ + rz->index->cap = rz->index->cap * 1.5 + 2; + rz->index->cell_offsets = realloc(rz->index->cell_offsets, sizeof(int) * rz->index->cap); + rz->index->bin_offsets = realloc(rz->index->bin_offsets, sizeof(int64_t) * (rz->index->cap/RZ_BIN_SIZE + 1)); + } + if(rz->index->size % RZ_BIN_SIZE == 0) rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE] = out; + rz->index->cell_offsets[rz->index->size] = out - rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE]; + rz->index->size ++; +} + +static void save_zindex(RAZF *rz, int fd){ + int32_t i, v32; + int is_be; + is_be = is_big_endian(); + if(is_be) write(fd, &rz->index->size, sizeof(int)); + else { + v32 = byte_swap_4((uint32_t)rz->index->size); + write(fd, &v32, sizeof(uint32_t)); + } + v32 = rz->index->size / RZ_BIN_SIZE + 1; + if(!is_be){ + for(i=0;iindex->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]); + for(i=0;iindex->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]); + } + write(fd, rz->index->bin_offsets, sizeof(int64_t) * v32); + write(fd, rz->index->cell_offsets, sizeof(int32_t) * rz->index->size); +} +#endif + +#ifdef _USE_KNETFILE +static void load_zindex(RAZF *rz, knetFile *fp){ +#else +static void load_zindex(RAZF *rz, int fd){ +#endif + int32_t i, v32; + int is_be; + if(!rz->load_index) return; + if(rz->index == NULL) rz->index = malloc(sizeof(ZBlockIndex)); + is_be = is_big_endian(); +#ifdef _USE_KNETFILE + knet_read(fp, &rz->index->size, sizeof(int)); +#else + read(fd, &rz->index->size, sizeof(int)); +#endif + if(!is_be) rz->index->size = byte_swap_4((uint32_t)rz->index->size); + rz->index->cap = rz->index->size; + v32 = rz->index->size / RZ_BIN_SIZE + 1; + rz->index->bin_offsets = malloc(sizeof(int64_t) * v32); +#ifdef _USE_KNETFILE + knet_read(fp, rz->index->bin_offsets, sizeof(int64_t) * v32); +#else + read(fd, rz->index->bin_offsets, sizeof(int64_t) * v32); +#endif + rz->index->cell_offsets = malloc(sizeof(int) * rz->index->size); +#ifdef _USE_KNETFILE + knet_read(fp, rz->index->cell_offsets, sizeof(int) * rz->index->size); +#else + read(fd, rz->index->cell_offsets, sizeof(int) * rz->index->size); +#endif + if(!is_be){ + for(i=0;iindex->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]); + for(i=0;iindex->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]); + } +} + +#ifdef _RZ_READONLY +static RAZF* razf_open_w(int fd) +{ + fprintf(stderr, "[razf_open_w] Writing is not available with zlib ver < 1.2.2.1\n"); + return 0; +} +#else +static RAZF* razf_open_w(int fd){ + RAZF *rz; +#ifdef _WIN32 + setmode(fd, O_BINARY); +#endif + rz = calloc(1, sizeof(RAZF)); + rz->mode = 'w'; +#ifdef _USE_KNETFILE + rz->x.fpw = fd; +#else + rz->filedes = fd; +#endif + rz->stream = calloc(sizeof(z_stream), 1); + rz->inbuf = malloc(RZ_BUFFER_SIZE); + rz->outbuf = malloc(RZ_BUFFER_SIZE); + rz->index = calloc(sizeof(ZBlockIndex), 1); + deflateInit2(rz->stream, RZ_COMPRESS_LEVEL, Z_DEFLATED, WINDOW_BITS + 16, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); + rz->stream->avail_out = RZ_BUFFER_SIZE; + rz->stream->next_out = rz->outbuf; + rz->header = calloc(sizeof(gz_header), 1); + rz->header->os = 0x03; //Unix + rz->header->text = 0; + rz->header->time = 0; + rz->header->extra = malloc(7); + strncpy((char*)rz->header->extra, "RAZF", 4); + rz->header->extra[4] = 1; // obsolete field + // block size = RZ_BLOCK_SIZE, Big-Endian + rz->header->extra[5] = RZ_BLOCK_SIZE >> 8; + rz->header->extra[6] = RZ_BLOCK_SIZE & 0xFF; + rz->header->extra_len = 7; + rz->header->name = rz->header->comment = 0; + rz->header->hcrc = 0; + deflateSetHeader(rz->stream, rz->header); + rz->block_pos = rz->block_off = 0; + return rz; +} + +static void _razf_write(RAZF* rz, const void *data, int size){ + int tout; + rz->stream->avail_in = size; + rz->stream->next_in = (void*)data; + while(1){ + tout = rz->stream->avail_out; + deflate(rz->stream, Z_NO_FLUSH); + rz->out += tout - rz->stream->avail_out; + if(rz->stream->avail_out) break; +#ifdef _USE_KNETFILE + write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#else + write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#endif + rz->stream->avail_out = RZ_BUFFER_SIZE; + rz->stream->next_out = rz->outbuf; + if(rz->stream->avail_in == 0) break; + }; + rz->in += size - rz->stream->avail_in; + rz->block_off += size - rz->stream->avail_in; +} + +static void razf_flush(RAZF *rz){ + uint32_t tout; + if(rz->buf_len){ + _razf_write(rz, rz->inbuf, rz->buf_len); + rz->buf_off = rz->buf_len = 0; + } + if(rz->stream->avail_out){ +#ifdef _USE_KNETFILE + write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#else + write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#endif + rz->stream->avail_out = RZ_BUFFER_SIZE; + rz->stream->next_out = rz->outbuf; + } + while(1){ + tout = rz->stream->avail_out; + deflate(rz->stream, Z_FULL_FLUSH); + rz->out += tout - rz->stream->avail_out; + if(rz->stream->avail_out == 0){ +#ifdef _USE_KNETFILE + write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#else + write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#endif + rz->stream->avail_out = RZ_BUFFER_SIZE; + rz->stream->next_out = rz->outbuf; + } else break; + } + rz->block_pos = rz->out; + rz->block_off = 0; +} + +static void razf_end_flush(RAZF *rz){ + uint32_t tout; + if(rz->buf_len){ + _razf_write(rz, rz->inbuf, rz->buf_len); + rz->buf_off = rz->buf_len = 0; + } + while(1){ + tout = rz->stream->avail_out; + deflate(rz->stream, Z_FINISH); + rz->out += tout - rz->stream->avail_out; + if(rz->stream->avail_out < RZ_BUFFER_SIZE){ +#ifdef _USE_KNETFILE + write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#else + write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out); +#endif + rz->stream->avail_out = RZ_BUFFER_SIZE; + rz->stream->next_out = rz->outbuf; + } else break; + } +} + +static void _razf_buffered_write(RAZF *rz, const void *data, int size){ + int i, n; + while(1){ + if(rz->buf_len == RZ_BUFFER_SIZE){ + _razf_write(rz, rz->inbuf, rz->buf_len); + rz->buf_len = 0; + } + if(size + rz->buf_len < RZ_BUFFER_SIZE){ + for(i=0;iinbuf + rz->buf_len)[i] = ((char*)data)[i]; + rz->buf_len += size; + return; + } else { + n = RZ_BUFFER_SIZE - rz->buf_len; + for(i=0;iinbuf + rz->buf_len)[i] = ((char*)data)[i]; + size -= n; + data += n; + rz->buf_len += n; + } + } +} + +int razf_write(RAZF* rz, const void *data, int size){ + int ori_size, n; + int64_t next_block; + ori_size = size; + next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE; + while(rz->in + rz->buf_len + size >= next_block){ + n = next_block - rz->in - rz->buf_len; + _razf_buffered_write(rz, data, n); + data += n; + size -= n; + razf_flush(rz); + add_zindex(rz, rz->in, rz->out); + next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE; + } + _razf_buffered_write(rz, data, size); + return ori_size; +} +#endif + +/* gzip flag byte */ +#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ +#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ +#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ +#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ +#define COMMENT 0x10 /* bit 4 set: file comment present */ +#define RESERVED 0xE0 /* bits 5..7: reserved */ + +static int _read_gz_header(unsigned char *data, int size, int *extra_off, int *extra_len){ + int method, flags, n, len; + if(size < 2) return 0; + if(data[0] != 0x1f || data[1] != 0x8b) return 0; + if(size < 4) return 0; + method = data[2]; + flags = data[3]; + if(method != Z_DEFLATED || (flags & RESERVED)) return 0; + n = 4 + 6; // Skip 6 bytes + *extra_off = n + 2; + *extra_len = 0; + if(flags & EXTRA_FIELD){ + if(size < n + 2) return 0; + len = ((int)data[n + 1] << 8) | data[n]; + n += 2; + *extra_off = n; + while(len){ + if(n >= size) return 0; + n ++; + len --; + } + *extra_len = n - (*extra_off); + } + if(flags & ORIG_NAME) while(n < size && data[n++]); + if(flags & COMMENT) while(n < size && data[n++]); + if(flags & HEAD_CRC){ + if(n + 2 > size) return 0; + n += 2; + } + return n; +} + +#ifdef _USE_KNETFILE +static RAZF* razf_open_r(knetFile *fp, int _load_index){ +#else +static RAZF* razf_open_r(int fd, int _load_index){ +#endif + RAZF *rz; + int ext_off, ext_len; + int n, is_be, ret; + int64_t end; + unsigned char c[] = "RAZF"; + rz = calloc(1, sizeof(RAZF)); + rz->mode = 'r'; +#ifdef _USE_KNETFILE + rz->x.fpr = fp; +#else +#ifdef _WIN32 + setmode(fd, O_BINARY); +#endif + rz->filedes = fd; +#endif + rz->stream = calloc(sizeof(z_stream), 1); + rz->inbuf = malloc(RZ_BUFFER_SIZE); + rz->outbuf = malloc(RZ_BUFFER_SIZE); + rz->end = rz->src_end = 0x7FFFFFFFFFFFFFFFLL; +#ifdef _USE_KNETFILE + n = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE); +#else + n = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE); +#endif + ret = _read_gz_header(rz->inbuf, n, &ext_off, &ext_len); + if(ret == 0){ + PLAIN_FILE: + rz->in = n; + rz->file_type = FILE_TYPE_PLAIN; + memcpy(rz->outbuf, rz->inbuf, n); + rz->buf_len = n; + free(rz->stream); + rz->stream = NULL; + return rz; + } + rz->header_size = ret; + ret = inflateInit2(rz->stream, -WINDOW_BITS); + if(ret != Z_OK){ inflateEnd(rz->stream); goto PLAIN_FILE;} + rz->stream->avail_in = n - rz->header_size; + rz->stream->next_in = rz->inbuf + rz->header_size; + rz->stream->avail_out = RZ_BUFFER_SIZE; + rz->stream->next_out = rz->outbuf; + rz->file_type = FILE_TYPE_GZ; + rz->in = rz->header_size; + rz->block_pos = rz->header_size; + rz->next_block_pos = rz->header_size; + rz->block_off = 0; + if(ext_len < 7 || memcmp(rz->inbuf + ext_off, c, 4) != 0) return rz; + if(((((unsigned char*)rz->inbuf)[ext_off + 5] << 8) | ((unsigned char*)rz->inbuf)[ext_off + 6]) != RZ_BLOCK_SIZE){ + fprintf(stderr, " -- WARNING: RZ_BLOCK_SIZE is not %d, treat source as gz file. in %s -- %s:%d --\n", RZ_BLOCK_SIZE, __FUNCTION__, __FILE__, __LINE__); + return rz; + } + rz->load_index = _load_index; + rz->file_type = FILE_TYPE_RZ; +#ifdef _USE_KNETFILE + if(knet_seek(fp, -16, SEEK_END) == -1){ +#else + if(lseek(fd, -16, SEEK_END) == -1){ +#endif + UNSEEKABLE: + rz->seekable = 0; + rz->index = NULL; + rz->src_end = rz->end = 0x7FFFFFFFFFFFFFFFLL; + } else { + is_be = is_big_endian(); + rz->seekable = 1; +#ifdef _USE_KNETFILE + knet_read(fp, &end, sizeof(int64_t)); +#else + read(fd, &end, sizeof(int64_t)); +#endif + if(!is_be) rz->src_end = (int64_t)byte_swap_8((uint64_t)end); + else rz->src_end = end; + +#ifdef _USE_KNETFILE + knet_read(fp, &end, sizeof(int64_t)); +#else + read(fd, &end, sizeof(int64_t)); +#endif + if(!is_be) rz->end = (int64_t)byte_swap_8((uint64_t)end); + else rz->end = end; + if(n > rz->end){ + rz->stream->avail_in -= n - rz->end; + n = rz->end; + } + if(rz->end > rz->src_end){ +#ifdef _USE_KNETFILE + knet_seek(fp, rz->in, SEEK_SET); +#else + lseek(fd, rz->in, SEEK_SET); +#endif + goto UNSEEKABLE; + } +#ifdef _USE_KNETFILE + knet_seek(fp, rz->end, SEEK_SET); + if(knet_tell(fp) != rz->end){ + knet_seek(fp, rz->in, SEEK_SET); +#else + if(lseek(fd, rz->end, SEEK_SET) != rz->end){ + lseek(fd, rz->in, SEEK_SET); +#endif + goto UNSEEKABLE; + } +#ifdef _USE_KNETFILE + load_zindex(rz, fp); + knet_seek(fp, n, SEEK_SET); +#else + load_zindex(rz, fd); + lseek(fd, n, SEEK_SET); +#endif + } + return rz; +} + +#ifdef _USE_KNETFILE +RAZF* razf_dopen(int fd, const char *mode){ + if (strstr(mode, "r")) fprintf(stderr,"[razf_dopen] implement me\n"); + else if(strstr(mode, "w")) return razf_open_w(fd); + return NULL; +} + +RAZF* razf_dopen2(int fd, const char *mode) +{ + fprintf(stderr,"[razf_dopen2] implement me\n"); + return NULL; +} +#else +RAZF* razf_dopen(int fd, const char *mode){ + if(strstr(mode, "r")) return razf_open_r(fd, 1); + else if(strstr(mode, "w")) return razf_open_w(fd); + else return NULL; +} + +RAZF* razf_dopen2(int fd, const char *mode) +{ + if(strstr(mode, "r")) return razf_open_r(fd, 0); + else if(strstr(mode, "w")) return razf_open_w(fd); + else return NULL; +} +#endif + +static inline RAZF* _razf_open(const char *filename, const char *mode, int _load_index){ + int fd; + RAZF *rz; + if(strstr(mode, "r")){ +#ifdef _USE_KNETFILE + knetFile *fd = knet_open(filename, "r"); + if (fd == 0) { + fprintf(stderr, "[_razf_open] fail to open %s\n", filename); + return NULL; + } +#else +#ifdef _WIN32 + fd = open(filename, O_RDONLY | O_BINARY); +#else + fd = open(filename, O_RDONLY); +#endif +#endif + if(fd < 0) return NULL; + rz = razf_open_r(fd, _load_index); + } else if(strstr(mode, "w")){ +#ifdef _WIN32 + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666); +#else + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666); +#endif + if(fd < 0) return NULL; + rz = razf_open_w(fd); + } else return NULL; + return rz; +} + +RAZF* razf_open(const char *filename, const char *mode){ + return _razf_open(filename, mode, 1); +} + +RAZF* razf_open2(const char *filename, const char *mode){ + return _razf_open(filename, mode, 0); +} + +int razf_get_data_size(RAZF *rz, int64_t *u_size, int64_t *c_size){ + int64_t n; + if(rz->mode != 'r' && rz->mode != 'R') return 0; + switch(rz->file_type){ + case FILE_TYPE_PLAIN: + if(rz->end == 0x7fffffffffffffffLL){ +#ifdef _USE_KNETFILE + if(knet_seek(rz->x.fpr, 0, SEEK_CUR) == -1) return 0; + n = knet_tell(rz->x.fpr); + knet_seek(rz->x.fpr, 0, SEEK_END); + rz->end = knet_tell(rz->x.fpr); + knet_seek(rz->x.fpr, n, SEEK_SET); +#else + if((n = lseek(rz->filedes, 0, SEEK_CUR)) == -1) return 0; + rz->end = lseek(rz->filedes, 0, SEEK_END); + lseek(rz->filedes, n, SEEK_SET); +#endif + } + *u_size = *c_size = rz->end; + return 1; + case FILE_TYPE_GZ: + return 0; + case FILE_TYPE_RZ: + if(rz->src_end == rz->end) return 0; + *u_size = rz->src_end; + *c_size = rz->end; + return 1; + default: + return 0; + } +} + +static int _razf_read(RAZF* rz, void *data, int size){ + int ret, tin; + if(rz->z_eof || rz->z_err) return 0; + if (rz->file_type == FILE_TYPE_PLAIN) { +#ifdef _USE_KNETFILE + ret = knet_read(rz->x.fpr, data, size); +#else + ret = read(rz->filedes, data, size); +#endif + if (ret == 0) rz->z_eof = 1; + return ret; + } + rz->stream->avail_out = size; + rz->stream->next_out = data; + while(rz->stream->avail_out){ + if(rz->stream->avail_in == 0){ + if(rz->in >= rz->end){ rz->z_eof = 1; break; } + if(rz->end - rz->in < RZ_BUFFER_SIZE){ +#ifdef _USE_KNETFILE + rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, rz->end -rz->in); +#else + rz->stream->avail_in = read(rz->filedes, rz->inbuf, rz->end -rz->in); +#endif + } else { +#ifdef _USE_KNETFILE + rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE); +#else + rz->stream->avail_in = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE); +#endif + } + if(rz->stream->avail_in == 0){ + rz->z_eof = 1; + break; + } + rz->stream->next_in = rz->inbuf; + } + tin = rz->stream->avail_in; + ret = inflate(rz->stream, Z_BLOCK); + rz->in += tin - rz->stream->avail_in; + if(ret == Z_NEED_DICT || ret == Z_MEM_ERROR || ret == Z_DATA_ERROR){ + fprintf(stderr, "[_razf_read] inflate error: %d %s (at %s:%d)\n", ret, rz->stream->msg ? rz->stream->msg : "", __FILE__, __LINE__); + rz->z_err = 1; + break; + } + if(ret == Z_STREAM_END){ + rz->z_eof = 1; + break; + } + if ((rz->stream->data_type&128) && !(rz->stream->data_type&64)){ + rz->buf_flush = 1; + rz->next_block_pos = rz->in; + break; + } + } + return size - rz->stream->avail_out; +} + +int razf_read(RAZF *rz, void *data, int size){ + int ori_size, i; + ori_size = size; + while(size > 0){ + if(rz->buf_len){ + if(size < rz->buf_len){ + for(i=0;ioutbuf + rz->buf_off)[i]; + rz->buf_off += size; + rz->buf_len -= size; + data += size; + rz->block_off += size; + size = 0; + break; + } else { + for(i=0;ibuf_len;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i]; + data += rz->buf_len; + size -= rz->buf_len; + rz->block_off += rz->buf_len; + rz->buf_off = 0; + rz->buf_len = 0; + if(rz->buf_flush){ + rz->block_pos = rz->next_block_pos; + rz->block_off = 0; + rz->buf_flush = 0; + } + } + } else if(rz->buf_flush){ + rz->block_pos = rz->next_block_pos; + rz->block_off = 0; + rz->buf_flush = 0; + } + if(rz->buf_flush) continue; + rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE); + if(rz->z_eof && rz->buf_len == 0) break; + } + rz->out += ori_size - size; + return ori_size - size; +} + +int razf_skip(RAZF* rz, int size){ + int ori_size; + ori_size = size; + while(size > 0){ + if(rz->buf_len){ + if(size < rz->buf_len){ + rz->buf_off += size; + rz->buf_len -= size; + rz->block_off += size; + size = 0; + break; + } else { + size -= rz->buf_len; + rz->buf_off = 0; + rz->buf_len = 0; + rz->block_off += rz->buf_len; + if(rz->buf_flush){ + rz->block_pos = rz->next_block_pos; + rz->block_off = 0; + rz->buf_flush = 0; + } + } + } else if(rz->buf_flush){ + rz->block_pos = rz->next_block_pos; + rz->block_off = 0; + rz->buf_flush = 0; + } + if(rz->buf_flush) continue; + rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE); + if(rz->z_eof || rz->z_err) break; + } + rz->out += ori_size - size; + return ori_size - size; +} + +static void _razf_reset_read(RAZF *rz, int64_t in, int64_t out){ +#ifdef _USE_KNETFILE + knet_seek(rz->x.fpr, in, SEEK_SET); +#else + lseek(rz->filedes, in, SEEK_SET); +#endif + rz->in = in; + rz->out = out; + rz->block_pos = in; + rz->next_block_pos = in; + rz->block_off = 0; + rz->buf_flush = 0; + rz->z_eof = rz->z_err = 0; + inflateReset(rz->stream); + rz->stream->avail_in = 0; + rz->buf_off = rz->buf_len = 0; +} + +int64_t razf_jump(RAZF *rz, int64_t block_start, int block_offset){ + int64_t pos; + rz->z_eof = 0; + if(rz->file_type == FILE_TYPE_PLAIN){ + rz->buf_off = rz->buf_len = 0; + pos = block_start + block_offset; +#ifdef _USE_KNETFILE + knet_seek(rz->x.fpr, pos, SEEK_SET); + pos = knet_tell(rz->x.fpr); +#else + pos = lseek(rz->filedes, pos, SEEK_SET); +#endif + rz->out = rz->in = pos; + return pos; + } + if(block_start == rz->block_pos && block_offset >= rz->block_off) { + block_offset -= rz->block_off; + goto SKIP; // Needn't reset inflate + } + if(block_start == 0) block_start = rz->header_size; // Automaticly revist wrong block_start + _razf_reset_read(rz, block_start, 0); + SKIP: + if(block_offset) razf_skip(rz, block_offset); + return rz->block_off; +} + +int64_t razf_seek(RAZF* rz, int64_t pos, int where){ + int64_t idx; + int64_t seek_pos, new_out; + rz->z_eof = 0; + if (where == SEEK_CUR) pos += rz->out; + else if (where == SEEK_END) pos += rz->src_end; + if(rz->file_type == FILE_TYPE_PLAIN){ +#ifdef _USE_KNETFILE + knet_seek(rz->x.fpr, pos, SEEK_SET); + seek_pos = knet_tell(rz->x.fpr); +#else + seek_pos = lseek(rz->filedes, pos, SEEK_SET); +#endif + rz->buf_off = rz->buf_len = 0; + rz->out = rz->in = seek_pos; + return seek_pos; + } else if(rz->file_type == FILE_TYPE_GZ){ + if(pos >= rz->out) goto SKIP; + return rz->out; + } + if(pos == rz->out) return pos; + if(pos > rz->src_end) return rz->out; + if(!rz->seekable || !rz->load_index){ + if(pos >= rz->out) goto SKIP; + } + idx = pos / RZ_BLOCK_SIZE - 1; + seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]); + new_out = (idx + 1) * RZ_BLOCK_SIZE; + if(pos > rz->out && new_out <= rz->out) goto SKIP; + _razf_reset_read(rz, seek_pos, new_out); + SKIP: + razf_skip(rz, (int)(pos - rz->out)); + return rz->out; +} + +uint64_t razf_tell2(RAZF *rz) +{ + /* + if (rz->load_index) { + int64_t idx, seek_pos; + idx = rz->out / RZ_BLOCK_SIZE - 1; + seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]); + if (seek_pos != rz->block_pos || rz->out%RZ_BLOCK_SIZE != rz->block_off) + fprintf(stderr, "[razf_tell2] inconsistent block offset: (%lld, %lld) != (%lld, %lld)\n", + (long long)seek_pos, (long long)rz->out%RZ_BLOCK_SIZE, (long long)rz->block_pos, (long long) rz->block_off); + } + */ + return (uint64_t)rz->block_pos<<16 | (rz->block_off&0xffff); +} + +int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where) +{ + if (where != SEEK_SET) return -1; + return razf_jump(rz, voffset>>16, voffset&0xffff); +} + +void razf_close(RAZF *rz){ + if(rz->mode == 'w'){ +#ifndef _RZ_READONLY + razf_end_flush(rz); + deflateEnd(rz->stream); +#ifdef _USE_KNETFILE + save_zindex(rz, rz->x.fpw); + if(is_big_endian()){ + write(rz->x.fpw, &rz->in, sizeof(int64_t)); + write(rz->x.fpw, &rz->out, sizeof(int64_t)); + } else { + uint64_t v64 = byte_swap_8((uint64_t)rz->in); + write(rz->x.fpw, &v64, sizeof(int64_t)); + v64 = byte_swap_8((uint64_t)rz->out); + write(rz->x.fpw, &v64, sizeof(int64_t)); + } +#else + save_zindex(rz, rz->filedes); + if(is_big_endian()){ + write(rz->filedes, &rz->in, sizeof(int64_t)); + write(rz->filedes, &rz->out, sizeof(int64_t)); + } else { + uint64_t v64 = byte_swap_8((uint64_t)rz->in); + write(rz->filedes, &v64, sizeof(int64_t)); + v64 = byte_swap_8((uint64_t)rz->out); + write(rz->filedes, &v64, sizeof(int64_t)); + } +#endif +#endif + } else if(rz->mode == 'r'){ + if(rz->stream) inflateEnd(rz->stream); + } + if(rz->inbuf) free(rz->inbuf); + if(rz->outbuf) free(rz->outbuf); + if(rz->header){ + free(rz->header->extra); + free(rz->header->name); + free(rz->header->comment); + free(rz->header); + } + if(rz->index){ + free(rz->index->bin_offsets); + free(rz->index->cell_offsets); + free(rz->index); + } + free(rz->stream); +#ifdef _USE_KNETFILE + if (rz->mode == 'r') + knet_close(rz->x.fpr); + if (rz->mode == 'w') + close(rz->x.fpw); +#else + close(rz->filedes); +#endif + free(rz); +} + +#endif diff --git a/samtools-0.1.18/razf.h b/samtools-0.1.18/razf.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/razf.h @@ -0,0 +1,134 @@ + /*- + * RAZF : Random Access compressed(Z) File + * Version: 1.0 + * Release Date: 2008-10-27 + * + * Copyright 2008, Jue Ruan , Heng Li + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +#ifndef __RAZF_RJ_H +#define __RAZF_RJ_H + +#include +#include +#include "zlib.h" + +#ifdef _USE_KNETFILE +#include "knetfile.h" +#endif + +#if ZLIB_VERNUM < 0x1221 +#define _RZ_READONLY +struct _gz_header_s; +typedef struct _gz_header_s _gz_header; +#define gz_header _gz_header +#endif + +#define WINDOW_BITS 15 + +#ifndef RZ_BLOCK_SIZE +#define RZ_BLOCK_SIZE (1<mode from HEAD to TYPE after call inflateReset */ + int buf_off, buf_len; + int z_err, z_eof; + int seekable; + /* Indice where the source is seekable */ + int load_index; + /* set has_index to 0 in mode 'w', then index will be discarded */ +} RAZF; + +#ifdef __cplusplus +extern "C" { +#endif + + RAZF* razf_dopen(int data_fd, const char *mode); + RAZF *razf_open(const char *fn, const char *mode); + int razf_write(RAZF* rz, const void *data, int size); + int razf_read(RAZF* rz, void *data, int size); + int64_t razf_seek(RAZF* rz, int64_t pos, int where); + void razf_close(RAZF* rz); + +#define razf_tell(rz) ((rz)->out) + + RAZF* razf_open2(const char *filename, const char *mode); + RAZF* razf_dopen2(int fd, const char *mode); + uint64_t razf_tell2(RAZF *rz); + int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/samtools-0.1.18/sam.c b/samtools-0.1.18/sam.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/sam.c @@ -0,0 +1,179 @@ +#include +#include +#include "faidx.h" +#include "sam.h" + +#define TYPE_BAM 1 +#define TYPE_READ 2 + +bam_header_t *bam_header_dup(const bam_header_t *h0) +{ + bam_header_t *h; + int i; + h = bam_header_init(); + *h = *h0; + h->hash = h->dict = h->rg2lib = 0; + h->text = (char*)calloc(h->l_text + 1, 1); + memcpy(h->text, h0->text, h->l_text); + h->target_len = (uint32_t*)calloc(h->n_targets, 4); + h->target_name = (char**)calloc(h->n_targets, sizeof(void*)); + for (i = 0; i < h->n_targets; ++i) { + h->target_len[i] = h0->target_len[i]; + h->target_name[i] = strdup(h0->target_name[i]); + } + return h; +} +static void append_header_text(bam_header_t *header, char* text, int len) +{ + int x = header->l_text + 1; + int y = header->l_text + len + 1; // 1 byte null + if (text == 0) return; + kroundup32(x); + kroundup32(y); + if (x < y) header->text = (char*)realloc(header->text, y); + strncpy(header->text + header->l_text, text, len); // we cannot use strcpy() here. + header->l_text += len; + header->text[header->l_text] = 0; +} + +samfile_t *samopen(const char *fn, const char *mode, const void *aux) +{ + samfile_t *fp; + fp = (samfile_t*)calloc(1, sizeof(samfile_t)); + if (strchr(mode, 'r')) { // read + fp->type |= TYPE_READ; + if (strchr(mode, 'b')) { // binary + fp->type |= TYPE_BAM; + fp->x.bam = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r"); + if (fp->x.bam == 0) goto open_err_ret; + fp->header = bam_header_read(fp->x.bam); + } else { // text + fp->x.tamr = sam_open(fn); + if (fp->x.tamr == 0) goto open_err_ret; + fp->header = sam_header_read(fp->x.tamr); + if (fp->header->n_targets == 0) { // no @SQ fields + if (aux) { // check if aux is present + bam_header_t *textheader = fp->header; + fp->header = sam_header_read2((const char*)aux); + if (fp->header == 0) goto open_err_ret; + append_header_text(fp->header, textheader->text, textheader->l_text); + bam_header_destroy(textheader); + } + if (fp->header->n_targets == 0 && bam_verbose >= 1) + fprintf(stderr, "[samopen] no @SQ lines in the header.\n"); + } else if (bam_verbose >= 2) fprintf(stderr, "[samopen] SAM header is present: %d sequences.\n", fp->header->n_targets); + } + } else if (strchr(mode, 'w')) { // write + fp->header = bam_header_dup((const bam_header_t*)aux); + if (strchr(mode, 'b')) { // binary + char bmode[3]; + int i, compress_level = -1; + for (i = 0; mode[i]; ++i) if (mode[i] >= '0' && mode[i] <= '9') break; + if (mode[i]) compress_level = mode[i] - '0'; + if (strchr(mode, 'u')) compress_level = 0; + bmode[0] = 'w'; bmode[1] = compress_level < 0? 0 : compress_level + '0'; bmode[2] = 0; + fp->type |= TYPE_BAM; + fp->x.bam = strcmp(fn, "-")? bam_open(fn, bmode) : bam_dopen(fileno(stdout), bmode); + if (fp->x.bam == 0) goto open_err_ret; + bam_header_write(fp->x.bam, fp->header); + } else { // text + // open file + fp->x.tamw = strcmp(fn, "-")? fopen(fn, "w") : stdout; + if (fp->x.tamr == 0) goto open_err_ret; + if (strchr(mode, 'X')) fp->type |= BAM_OFSTR<<2; + else if (strchr(mode, 'x')) fp->type |= BAM_OFHEX<<2; + else fp->type |= BAM_OFDEC<<2; + // write header + if (strchr(mode, 'h')) { + int i; + bam_header_t *alt; + // parse the header text + alt = bam_header_init(); + alt->l_text = fp->header->l_text; alt->text = fp->header->text; + sam_header_parse(alt); + alt->l_text = 0; alt->text = 0; + // check if there are @SQ lines in the header + fwrite(fp->header->text, 1, fp->header->l_text, fp->x.tamw); // FIXME: better to skip the trailing NULL + if (alt->n_targets) { // then write the header text without dumping ->target_{name,len} + if (alt->n_targets != fp->header->n_targets && bam_verbose >= 1) + fprintf(stderr, "[samopen] inconsistent number of target sequences. Output the text header.\n"); + } else { // then dump ->target_{name,len} + for (i = 0; i < fp->header->n_targets; ++i) + fprintf(fp->x.tamw, "@SQ\tSN:%s\tLN:%d\n", fp->header->target_name[i], fp->header->target_len[i]); + } + bam_header_destroy(alt); + } + } + } + return fp; + +open_err_ret: + free(fp); + return 0; +} + +void samclose(samfile_t *fp) +{ + if (fp == 0) return; + if (fp->header) bam_header_destroy(fp->header); + if (fp->type & TYPE_BAM) bam_close(fp->x.bam); + else if (fp->type & TYPE_READ) sam_close(fp->x.tamr); + else fclose(fp->x.tamw); + free(fp); +} + +int samread(samfile_t *fp, bam1_t *b) +{ + if (fp == 0 || !(fp->type & TYPE_READ)) return -1; // not open for reading + if (fp->type & TYPE_BAM) return bam_read1(fp->x.bam, b); + else return sam_read1(fp->x.tamr, fp->header, b); +} + +int samwrite(samfile_t *fp, const bam1_t *b) +{ + if (fp == 0 || (fp->type & TYPE_READ)) return -1; // not open for writing + if (fp->type & TYPE_BAM) return bam_write1(fp->x.bam, b); + else { + char *s = bam_format1_core(fp->header, b, fp->type>>2&3); + int l = strlen(s); + fputs(s, fp->x.tamw); fputc('\n', fp->x.tamw); + free(s); + return l + 1; + } +} + +int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *func_data) +{ + bam_plbuf_t *buf; + int ret; + bam1_t *b; + b = bam_init1(); + buf = bam_plbuf_init(func, func_data); + bam_plbuf_set_mask(buf, mask); + while ((ret = samread(fp, b)) >= 0) + bam_plbuf_push(b, buf); + bam_plbuf_push(0, buf); + bam_plbuf_destroy(buf); + bam_destroy1(b); + return 0; +} + +char *samfaipath(const char *fn_ref) +{ + char *fn_list = 0; + if (fn_ref == 0) return 0; + fn_list = calloc(strlen(fn_ref) + 5, 1); + strcat(strcpy(fn_list, fn_ref), ".fai"); + if (access(fn_list, R_OK) == -1) { // fn_list is unreadable + if (access(fn_ref, R_OK) == -1) { + fprintf(stderr, "[samfaipath] fail to read file %s.\n", fn_ref); + } else { + if (bam_verbose >= 3) fprintf(stderr, "[samfaipath] build FASTA index...\n"); + if (fai_build(fn_ref) == -1) { + fprintf(stderr, "[samfaipath] fail to build FASTA index.\n"); + free(fn_list); fn_list = 0; + } + } + } + return fn_list; +} diff --git a/samtools-0.1.18/sam.h b/samtools-0.1.18/sam.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/sam.h @@ -0,0 +1,98 @@ +#ifndef BAM_SAM_H +#define BAM_SAM_H + +#include "bam.h" + +/*! + @header + + This file provides higher level of I/O routines and unifies the APIs + for SAM and BAM formats. These APIs are more convenient and + recommended. + + @copyright Genome Research Ltd. + */ + +/*! @typedef + @abstract SAM/BAM file handler + @field type type of the handler; bit 1 for BAM, 2 for reading and bit 3-4 for flag format + @field bam BAM file handler; valid if (type&1) == 1 + @field tamr SAM file handler for reading; valid if type == 2 + @field tamw SAM file handler for writing; valid if type == 0 + @field header header struct + */ +typedef struct { + int type; + union { + tamFile tamr; + bamFile bam; + FILE *tamw; + } x; + bam_header_t *header; +} samfile_t; + +#ifdef __cplusplus +extern "C" { +#endif + + /*! + @abstract Open a SAM/BAM file + + @param fn SAM/BAM file name; "-" is recognized as stdin (for + reading) or stdout (for writing). + + @param mode open mode /[rw](b?)(u?)(h?)([xX]?)/: 'r' for reading, + 'w' for writing, 'b' for BAM I/O, 'u' for uncompressed BAM output, + 'h' for outputing header in SAM, 'x' for HEX flag and 'X' for + string flag. If 'b' present, it must immediately follow 'r' or + 'w'. Valid modes are "r", "w", "wh", "wx", "whx", "wX", "whX", + "rb", "wb" and "wbu" exclusively. + + @param aux auxiliary data; if mode[0]=='w', aux points to + bam_header_t; if strcmp(mode, "rb")!=0 and @SQ header lines in SAM + are absent, aux points the file name of the list of the reference; + aux is not used otherwise. If @SQ header lines are present in SAM, + aux is not used, either. + + @return SAM/BAM file handler + */ + samfile_t *samopen(const char *fn, const char *mode, const void *aux); + + /*! + @abstract Close a SAM/BAM handler + @param fp file handler to be closed + */ + void samclose(samfile_t *fp); + + /*! + @abstract Read one alignment + @param fp file handler + @param b alignment + @return bytes read + */ + int samread(samfile_t *fp, bam1_t *b); + + /*! + @abstract Write one alignment + @param fp file handler + @param b alignment + @return bytes written + */ + int samwrite(samfile_t *fp, const bam1_t *b); + + /*! + @abstract Get the pileup for a whole alignment file + @param fp file handler + @param mask mask transferred to bam_plbuf_set_mask() + @param func user defined function called in the pileup process + #param data user provided data for func() + */ + int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *data); + + char *samfaipath(const char *fn_ref); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/samtools-0.1.18/sam_header.c b/samtools-0.1.18/sam_header.c new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/sam_header.c @@ -0,0 +1,736 @@ +#include "sam_header.h" +#include +#include +#include +#include +#include + +#include "khash.h" +KHASH_MAP_INIT_STR(str, const char *) + +struct _HeaderList +{ + struct _HeaderList *last; // Hack: Used and maintained only by list_append_to_end. Maintained in the root node only. + struct _HeaderList *next; + void *data; +}; +typedef struct _HeaderList list_t; +typedef list_t HeaderDict; + +typedef struct +{ + char key[2]; + char *value; +} +HeaderTag; + +typedef struct +{ + char type[2]; + list_t *tags; +} +HeaderLine; + +const char *o_hd_tags[] = {"SO","GO",NULL}; +const char *r_hd_tags[] = {"VN",NULL}; + +const char *o_sq_tags[] = {"AS","M5","UR","SP",NULL}; +const char *r_sq_tags[] = {"SN","LN",NULL}; +const char *u_sq_tags[] = {"SN",NULL}; + +const char *o_rg_tags[] = {"CN","DS","DT","FO","KS","LB","PG","PI","PL","PU","SM",NULL}; +const char *r_rg_tags[] = {"ID",NULL}; +const char *u_rg_tags[] = {"ID",NULL}; + +const char *o_pg_tags[] = {"VN","CL",NULL}; +const char *r_pg_tags[] = {"ID",NULL}; + +const char *types[] = {"HD","SQ","RG","PG","CO",NULL}; +const char **optional_tags[] = {o_hd_tags,o_sq_tags,o_rg_tags,o_pg_tags,NULL,NULL}; +const char **required_tags[] = {r_hd_tags,r_sq_tags,r_rg_tags,r_pg_tags,NULL,NULL}; +const char **unique_tags[] = {NULL, u_sq_tags,u_rg_tags,NULL,NULL,NULL}; + + +static void debug(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); +} + +#if 0 +// Replaced by list_append_to_end +static list_t *list_prepend(list_t *root, void *data) +{ + list_t *l = malloc(sizeof(list_t)); + l->next = root; + l->data = data; + return l; +} +#endif + +// Relies on the root->last being correct. Do not use with the other list_* +// routines unless they are fixed to modify root->last as well. +static list_t *list_append_to_end(list_t *root, void *data) +{ + list_t *l = malloc(sizeof(list_t)); + l->last = l; + l->next = NULL; + l->data = data; + + if ( !root ) + return l; + + root->last->next = l; + root->last = l; + return root; +} + +static list_t *list_append(list_t *root, void *data) +{ + list_t *l = root; + while (l && l->next) + l = l->next; + if ( l ) + { + l->next = malloc(sizeof(list_t)); + l = l->next; + } + else + { + l = malloc(sizeof(list_t)); + root = l; + } + l->data = data; + l->next = NULL; + return root; +} + +static void list_free(list_t *root) +{ + list_t *l = root; + while (root) + { + l = root; + root = root->next; + free(l); + } +} + + + +// Look for a tag "XY" in a predefined const char *[] array. +static int tag_exists(const char *tag, const char **tags) +{ + int itag=0; + if ( !tags ) return -1; + while ( tags[itag] ) + { + if ( tags[itag][0]==tag[0] && tags[itag][1]==tag[1] ) return itag; + itag++; + } + return -1; +} + + + +// Mimics the behaviour of getline, except it returns pointer to the next chunk of the text +// or NULL if everything has been read. The lineptr should be freed by the caller. The +// newline character is stripped. +static const char *nextline(char **lineptr, size_t *n, const char *text) +{ + int len; + const char *to = text; + + if ( !*to ) return NULL; + + while ( *to && *to!='\n' && *to!='\r' ) to++; + len = to - text + 1; + + if ( *to ) + { + // Advance the pointer for the next call + if ( *to=='\n' ) to++; + else if ( *to=='\r' && *(to+1)=='\n' ) to+=2; + } + if ( !len ) + return to; + + if ( !*lineptr ) + { + *lineptr = malloc(len); + *n = len; + } + else if ( *nkey[0] = name[0]; + tag->key[1] = name[1]; + tag->value = malloc(len+1); + memcpy(tag->value,value_from,len+1); + tag->value[len] = 0; + return tag; +} + +static HeaderTag *header_line_has_tag(HeaderLine *hline, const char *key) +{ + list_t *tags = hline->tags; + while (tags) + { + HeaderTag *tag = tags->data; + if ( tag->key[0]==key[0] && tag->key[1]==key[1] ) return tag; + tags = tags->next; + } + return NULL; +} + + +// Return codes: +// 0 .. different types or unique tags differ or conflicting tags, cannot be merged +// 1 .. all tags identical -> no need to merge, drop one +// 2 .. the unique tags match and there are some conflicting tags (same tag, different value) -> error, cannot be merged nor duplicated +// 3 .. there are some missing complementary tags and no unique conflict -> can be merged into a single line +static int sam_header_compare_lines(HeaderLine *hline1, HeaderLine *hline2) +{ + HeaderTag *t1, *t2; + + if ( hline1->type[0]!=hline2->type[0] || hline1->type[1]!=hline2->type[1] ) + return 0; + + int itype = tag_exists(hline1->type,types); + if ( itype==-1 ) { + debug("[sam_header_compare_lines] Unknown type [%c%c]\n", hline1->type[0],hline1->type[1]); + return -1; // FIXME (lh3): error; I do not know how this will be handled in Petr's code + } + + if ( unique_tags[itype] ) + { + t1 = header_line_has_tag(hline1,unique_tags[itype][0]); + t2 = header_line_has_tag(hline2,unique_tags[itype][0]); + if ( !t1 || !t2 ) // this should never happen, the unique tags are required + return 2; + + if ( strcmp(t1->value,t2->value) ) + return 0; // the unique tags differ, cannot be merged + } + if ( !required_tags[itype] && !optional_tags[itype] ) + { + t1 = hline1->tags->data; + t2 = hline2->tags->data; + if ( !strcmp(t1->value,t2->value) ) return 1; // identical comments + return 0; + } + + int missing=0, itag=0; + while ( required_tags[itype] && required_tags[itype][itag] ) + { + t1 = header_line_has_tag(hline1,required_tags[itype][itag]); + t2 = header_line_has_tag(hline2,required_tags[itype][itag]); + if ( !t1 && !t2 ) + return 2; // this should never happen + else if ( !t1 || !t2 ) + missing = 1; // there is some tag missing in one of the hlines + else if ( strcmp(t1->value,t2->value) ) + { + if ( unique_tags[itype] ) + return 2; // the lines have a matching unique tag but have a conflicting tag + + return 0; // the lines contain conflicting tags, cannot be merged + } + itag++; + } + itag = 0; + while ( optional_tags[itype] && optional_tags[itype][itag] ) + { + t1 = header_line_has_tag(hline1,optional_tags[itype][itag]); + t2 = header_line_has_tag(hline2,optional_tags[itype][itag]); + if ( !t1 && !t2 ) + { + itag++; + continue; + } + if ( !t1 || !t2 ) + missing = 1; // there is some tag missing in one of the hlines + else if ( strcmp(t1->value,t2->value) ) + { + if ( unique_tags[itype] ) + return 2; // the lines have a matching unique tag but have a conflicting tag + + return 0; // the lines contain conflicting tags, cannot be merged + } + itag++; + } + if ( missing ) return 3; // there are some missing complementary tags with no conflicts, can be merged + return 1; +} + + +static HeaderLine *sam_header_line_clone(const HeaderLine *hline) +{ + list_t *tags; + HeaderLine *out = malloc(sizeof(HeaderLine)); + out->type[0] = hline->type[0]; + out->type[1] = hline->type[1]; + out->tags = NULL; + + tags = hline->tags; + while (tags) + { + HeaderTag *old = tags->data; + + HeaderTag *new = malloc(sizeof(HeaderTag)); + new->key[0] = old->key[0]; + new->key[1] = old->key[1]; + new->value = strdup(old->value); + out->tags = list_append(out->tags, new); + + tags = tags->next; + } + return out; +} + +static int sam_header_line_merge_with(HeaderLine *out_hline, const HeaderLine *tmpl_hline) +{ + list_t *tmpl_tags; + + if ( out_hline->type[0]!=tmpl_hline->type[0] || out_hline->type[1]!=tmpl_hline->type[1] ) + return 0; + + tmpl_tags = tmpl_hline->tags; + while (tmpl_tags) + { + HeaderTag *tmpl_tag = tmpl_tags->data; + HeaderTag *out_tag = header_line_has_tag(out_hline, tmpl_tag->key); + if ( !out_tag ) + { + HeaderTag *tag = malloc(sizeof(HeaderTag)); + tag->key[0] = tmpl_tag->key[0]; + tag->key[1] = tmpl_tag->key[1]; + tag->value = strdup(tmpl_tag->value); + out_hline->tags = list_append(out_hline->tags,tag); + } + tmpl_tags = tmpl_tags->next; + } + return 1; +} + + +static HeaderLine *sam_header_line_parse(const char *headerLine) +{ + HeaderLine *hline; + HeaderTag *tag; + const char *from, *to; + from = headerLine; + + if ( *from != '@' ) { + debug("[sam_header_line_parse] expected '@', got [%s]\n", headerLine); + return 0; + } + to = ++from; + + while (*to && *to!='\t') to++; + if ( to-from != 2 ) { + debug("[sam_header_line_parse] expected '@XY', got [%s]\nHint: The header tags must be tab-separated.\n", headerLine); + return 0; + } + + hline = malloc(sizeof(HeaderLine)); + hline->type[0] = from[0]; + hline->type[1] = from[1]; + hline->tags = NULL; + + int itype = tag_exists(hline->type, types); + + from = to; + while (*to && *to=='\t') to++; + if ( to-from != 1 ) { + debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from)); + return 0; + } + from = to; + while (*from) + { + while (*to && *to!='\t') to++; + + if ( !required_tags[itype] && !optional_tags[itype] ) + { + // CO is a special case, it can contain anything, including tabs + if ( *to ) { to++; continue; } + tag = new_tag(" ",from,to-1); + } + else + tag = new_tag(from,from+3,to-1); + + if ( header_line_has_tag(hline,tag->key) ) + debug("The tag '%c%c' present (at least) twice on line [%s]\n", tag->key[0],tag->key[1], headerLine); + hline->tags = list_append(hline->tags, tag); + + from = to; + while (*to && *to=='\t') to++; + if ( *to && to-from != 1 ) { + debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from)); + return 0; + } + + from = to; + } + return hline; +} + + +// Must be of an existing type, all tags must be recognised and all required tags must be present +static int sam_header_line_validate(HeaderLine *hline) +{ + list_t *tags; + HeaderTag *tag; + int itype, itag; + + // Is the type correct? + itype = tag_exists(hline->type, types); + if ( itype==-1 ) + { + debug("The type [%c%c] not recognised.\n", hline->type[0],hline->type[1]); + return 0; + } + + // Has all required tags? + itag = 0; + while ( required_tags[itype] && required_tags[itype][itag] ) + { + if ( !header_line_has_tag(hline,required_tags[itype][itag]) ) + { + debug("The tag [%c%c] required for [%c%c] not present.\n", required_tags[itype][itag][0],required_tags[itype][itag][1], + hline->type[0],hline->type[1]); + return 0; + } + itag++; + } + + // Are all tags recognised? + tags = hline->tags; + while ( tags ) + { + tag = tags->data; + if ( !tag_exists(tag->key,required_tags[itype]) && !tag_exists(tag->key,optional_tags[itype]) ) + { + debug("Unknown tag [%c%c] for [%c%c].\n", tag->key[0],tag->key[1], hline->type[0],hline->type[1]); + return 0; + } + tags = tags->next; + } + + return 1; +} + + +static void print_header_line(FILE *fp, HeaderLine *hline) +{ + list_t *tags = hline->tags; + HeaderTag *tag; + + fprintf(fp, "@%c%c", hline->type[0],hline->type[1]); + while (tags) + { + tag = tags->data; + + fprintf(fp, "\t"); + if ( tag->key[0]!=' ' || tag->key[1]!=' ' ) + fprintf(fp, "%c%c:", tag->key[0],tag->key[1]); + fprintf(fp, "%s", tag->value); + + tags = tags->next; + } + fprintf(fp,"\n"); +} + + +static void sam_header_line_free(HeaderLine *hline) +{ + list_t *tags = hline->tags; + while (tags) + { + HeaderTag *tag = tags->data; + free(tag->value); + free(tag); + tags = tags->next; + } + list_free(hline->tags); + free(hline); +} + +void sam_header_free(void *_header) +{ + HeaderDict *header = (HeaderDict*)_header; + list_t *hlines = header; + while (hlines) + { + sam_header_line_free(hlines->data); + hlines = hlines->next; + } + list_free(header); +} + +HeaderDict *sam_header_clone(const HeaderDict *dict) +{ + HeaderDict *out = NULL; + while (dict) + { + HeaderLine *hline = dict->data; + out = list_append(out, sam_header_line_clone(hline)); + dict = dict->next; + } + return out; +} + +// Returns a newly allocated string +char *sam_header_write(const void *_header) +{ + const HeaderDict *header = (const HeaderDict*)_header; + char *out = NULL; + int len=0, nout=0; + const list_t *hlines; + + // Calculate the length of the string to allocate + hlines = header; + while (hlines) + { + len += 4; // @XY and \n + + HeaderLine *hline = hlines->data; + list_t *tags = hline->tags; + while (tags) + { + HeaderTag *tag = tags->data; + len += strlen(tag->value) + 1; // \t + if ( tag->key[0]!=' ' || tag->key[1]!=' ' ) + len += strlen(tag->value) + 3; // XY: + tags = tags->next; + } + hlines = hlines->next; + } + + nout = 0; + out = malloc(len+1); + hlines = header; + while (hlines) + { + HeaderLine *hline = hlines->data; + + nout += sprintf(out+nout,"@%c%c",hline->type[0],hline->type[1]); + + list_t *tags = hline->tags; + while (tags) + { + HeaderTag *tag = tags->data; + nout += sprintf(out+nout,"\t"); + if ( tag->key[0]!=' ' || tag->key[1]!=' ' ) + nout += sprintf(out+nout,"%c%c:", tag->key[0],tag->key[1]); + nout += sprintf(out+nout,"%s", tag->value); + tags = tags->next; + } + hlines = hlines->next; + nout += sprintf(out+nout,"\n"); + } + out[len] = 0; + return out; +} + +void *sam_header_parse2(const char *headerText) +{ + list_t *hlines = NULL; + HeaderLine *hline; + const char *text; + char *buf=NULL; + size_t nbuf = 0; + int tovalidate = 0; + + if ( !headerText ) + return 0; + + text = headerText; + while ( (text=nextline(&buf, &nbuf, text)) ) + { + hline = sam_header_line_parse(buf); + if ( hline && (!tovalidate || sam_header_line_validate(hline)) ) + // With too many (~250,000) reference sequences the header parsing was too slow with list_append. + hlines = list_append_to_end(hlines, hline); + else + { + if (hline) sam_header_line_free(hline); + sam_header_free(hlines); + if ( buf ) free(buf); + return NULL; + } + } + if ( buf ) free(buf); + + return hlines; +} + +void *sam_header2tbl(const void *_dict, char type[2], char key_tag[2], char value_tag[2]) +{ + const HeaderDict *dict = (const HeaderDict*)_dict; + const list_t *l = dict; + khash_t(str) *tbl = kh_init(str); + khiter_t k; + int ret; + + if (_dict == 0) return tbl; // return an empty (not null) hash table + while (l) + { + HeaderLine *hline = l->data; + if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] ) + { + l = l->next; + continue; + } + + HeaderTag *key, *value; + key = header_line_has_tag(hline,key_tag); + value = header_line_has_tag(hline,value_tag); + if ( !key || !value ) + { + l = l->next; + continue; + } + + k = kh_get(str, tbl, key->value); + if ( k != kh_end(tbl) ) + debug("[sam_header_lookup_table] They key %s not unique.\n", key->value); + k = kh_put(str, tbl, key->value, &ret); + kh_value(tbl, k) = value->value; + + l = l->next; + } + return tbl; +} + +char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n) +{ + const HeaderDict *dict = (const HeaderDict*)_dict; + const list_t *l = dict; + int max, n; + char **ret; + + ret = 0; *_n = max = n = 0; + while (l) + { + HeaderLine *hline = l->data; + if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] ) + { + l = l->next; + continue; + } + + HeaderTag *key; + key = header_line_has_tag(hline,key_tag); + if ( !key ) + { + l = l->next; + continue; + } + + if (n == max) { + max = max? max<<1 : 4; + ret = realloc(ret, max * sizeof(void*)); + } + ret[n++] = key->value; + + l = l->next; + } + *_n = n; + return ret; +} + +const char *sam_tbl_get(void *h, const char *key) +{ + khash_t(str) *tbl = (khash_t(str)*)h; + khint_t k; + k = kh_get(str, tbl, key); + return k == kh_end(tbl)? 0 : kh_val(tbl, k); +} + +int sam_tbl_size(void *h) +{ + khash_t(str) *tbl = (khash_t(str)*)h; + return h? kh_size(tbl) : 0; +} + +void sam_tbl_destroy(void *h) +{ + khash_t(str) *tbl = (khash_t(str)*)h; + kh_destroy(str, tbl); +} + +void *sam_header_merge(int n, const void **_dicts) +{ + const HeaderDict **dicts = (const HeaderDict**)_dicts; + HeaderDict *out_dict; + int idict, status; + + if ( n<2 ) return NULL; + + out_dict = sam_header_clone(dicts[0]); + + for (idict=1; idictdata, out_hlines->data); + if ( status==0 ) + { + out_hlines = out_hlines->next; + continue; + } + + if ( status==2 ) + { + print_header_line(stderr,tmpl_hlines->data); + print_header_line(stderr,out_hlines->data); + debug("Conflicting lines, cannot merge the headers.\n"); + return 0; + } + if ( status==3 ) + sam_header_line_merge_with(out_hlines->data, tmpl_hlines->data); + + inserted = 1; + break; + } + if ( !inserted ) + out_dict = list_append(out_dict, sam_header_line_clone(tmpl_hlines->data)); + + tmpl_hlines = tmpl_hlines->next; + } + } + + return out_dict; +} + + diff --git a/samtools-0.1.18/sam_header.h b/samtools-0.1.18/sam_header.h new file mode 100644 --- /dev/null +++ b/samtools-0.1.18/sam_header.h @@ -0,0 +1,24 @@ +#ifndef __SAM_HEADER_H__ +#define __SAM_HEADER_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + void *sam_header_parse2(const char *headerText); + void *sam_header_merge(int n, const void **dicts); + void sam_header_free(void *header); + char *sam_header_write(const void *headerDict); // returns a newly allocated string + + char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n); + + void *sam_header2tbl(const void *dict, char type[2], char key_tag[2], char value_tag[2]); + const char *sam_tbl_get(void *h, const char *key); + int sam_tbl_size(void *h); + void sam_tbl_destroy(void *h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/samtools.cabal b/samtools.cabal --- a/samtools.cabal +++ b/samtools.cabal @@ -1,5 +1,5 @@ Name: samtools -Version: 0.1.3 +Version: 0.2 Synopsis: Binding to the C samtools library Description: Binding to the C samtools library, which reads and writes SAM format alignments, both binary and tab- @@ -17,13 +17,13 @@ Cabal-version: >=1.4 Extra-source-files: cbits/samtools.c, include/samtools.h, - samtools-0.1.13/bgzf.c, samtools-0.1.13/kstring.c, samtools-0.1.13/bam_aux.c, - samtools-0.1.13/bam.c, samtools-0.1.13/bam_import.c, samtools-0.1.13/sam.c, - samtools-0.1.13/bam_index.c, samtools-0.1.13/bam_pileup.c, samtools-0.1.13/bam_lpileup.c, - samtools-0.1.13/bam_md.c, samtools-0.1.13/glf.c, samtools-0.1.13/razf.c, - samtools-0.1.13/faidx.c, samtools-0.1.13/knetfile.c, samtools-0.1.13/bam_sort.c, - samtools-0.1.13/sam_header.c, samtools-0.1.13/bam_reheader.c, samtools-0.1.13/kprobaln.c, - samtools-0.1.13/bam_maqcns.c, samtools-0.1.13/errmod.c, samtools-0.1.13/kaln.c + samtools-0.1.18/bgzf.c, samtools-0.1.18/kstring.c, samtools-0.1.18/bam_aux.c, + samtools-0.1.18/bam.c, samtools-0.1.18/bam_import.c, samtools-0.1.18/sam.c, + samtools-0.1.18/bam_index.c, samtools-0.1.18/bam_pileup.c, samtools-0.1.18/bam_lpileup.c, + samtools-0.1.18/bam_md.c, samtools-0.1.18/razf.c, + samtools-0.1.18/faidx.c, samtools-0.1.18/knetfile.c, samtools-0.1.18/bam_sort.c, + samtools-0.1.18/sam_header.c, samtools-0.1.18/bam_reheader.c, samtools-0.1.18/kprobaln.c, + samtools-0.1.18/errmod.c, samtools-0.1.18/kaln.c Flag Tests Description: Build test program @@ -33,25 +33,25 @@ Exposed-modules: Bio.SamTools.Bam, Bio.SamTools.Cigar, Bio.SamTools.BamIndex, Bio.SamTools.FaIdx Other-modules: Bio.SamTools.LowLevel, Bio.SamTools.Internal, C2HS - Build-depends: base >= 4.2 && < 5, bytestring, haskell98, vector >= 0.7, seqloc + Build-depends: base >= 4.2 && < 5, bytestring, haskell98, vector >= 0.7, seqloc >= 0.3.1 Hs-Source-Dirs: src Build-tools: c2hs - Include-dirs: samtools-0.1.13, include + Include-dirs: samtools-0.1.18, include C-sources: cbits/samtools.c, - samtools-0.1.13/bgzf.c, samtools-0.1.13/kstring.c, samtools-0.1.13/bam_aux.c, - samtools-0.1.13/bam.c, samtools-0.1.13/bam_import.c, samtools-0.1.13/sam.c, - samtools-0.1.13/bam_index.c, samtools-0.1.13/bam_pileup.c, samtools-0.1.13/bam_lpileup.c, - samtools-0.1.13/bam_md.c, samtools-0.1.13/glf.c, samtools-0.1.13/razf.c, - samtools-0.1.13/faidx.c, samtools-0.1.13/knetfile.c, samtools-0.1.13/bam_sort.c, - samtools-0.1.13/sam_header.c, samtools-0.1.13/bam_reheader.c, samtools-0.1.13/kprobaln.c, - samtools-0.1.13/bam_maqcns.c, samtools-0.1.13/errmod.c, samtools-0.1.13/kaln.c + samtools-0.1.18/bgzf.c, samtools-0.1.18/kstring.c, samtools-0.1.18/bam_aux.c, + samtools-0.1.18/bam.c, samtools-0.1.18/bam_import.c, samtools-0.1.18/sam.c, + samtools-0.1.18/bam_index.c, samtools-0.1.18/bam_pileup.c, samtools-0.1.18/bam_lpileup.c, + samtools-0.1.18/bam_md.c, samtools-0.1.18/razf.c, + samtools-0.1.18/faidx.c, samtools-0.1.18/knetfile.c, samtools-0.1.18/bam_sort.c, + samtools-0.1.18/sam_header.c, samtools-0.1.18/bam_reheader.c, samtools-0.1.18/kprobaln.c, + samtools-0.1.18/errmod.c, samtools-0.1.18/kaln.c Install-includes: include/samtools.h, - samtools-0.1.13/bam.h, samtools-0.1.13/bam_endian.h, samtools-0.1.13/faidx.h, - samtools-0.1.13/glf.h, samtools-0.1.13/khash.h, - samtools-0.1.13/kseq.h, samtools-0.1.13/ksort.h, samtools-0.1.13/kstring.h, - samtools-0.1.13/razf.h, samtools-0.1.13/sam.h, samtools-0.1.13/sam_header.h, - samtools-0.1.13/bgzf.h, samtools-0.1.13/knetfile.h, samtools-0.1.13/kaln.h, - samtools-0.1.13/kprobaln.h, samtools-0.1.13/bam_maqcns.h, samtools-0.1.13/errmod.h + samtools-0.1.18/bam.h, samtools-0.1.18/bam_endian.h, samtools-0.1.18/faidx.h, + samtools-0.1.18/khash.h, + samtools-0.1.18/kseq.h, samtools-0.1.18/ksort.h, samtools-0.1.18/kstring.h, + samtools-0.1.18/razf.h, samtools-0.1.18/sam.h, samtools-0.1.18/sam_header.h, + samtools-0.1.18/bgzf.h, samtools-0.1.18/knetfile.h, samtools-0.1.18/kaln.h, + samtools-0.1.18/kprobaln.h, samtools-0.1.18/errmod.h CC-options: -g -D_FILE_OFFSET_BITS=64 -D_USE_KNETFILE Extra-libraries: z Ghc-options: -Wall @@ -63,25 +63,25 @@ Other-modules: Bio.SamTools.Bam, Bio.SamTools.Cigar, Bio.SamTools.BamIndex, Bio.SamTools.FaIdx, Bio.SamTools.LowLevel, Bio.SamTools.Internal, C2HS - Build-depends: base >= 4.2 && < 5, bytestring, haskell98, vector >= 0.7, seqloc, process, filepath + Build-depends: base >= 4.2 && < 5, bytestring, haskell98, vector >= 0.7, seqloc >= 0.3.1, process, filepath Hs-Source-Dirs: src, test Build-tools: c2hs - Include-dirs: samtools-0.1.13, include + Include-dirs: samtools-0.1.18, include C-sources: cbits/samtools.c, - samtools-0.1.13/bgzf.c, samtools-0.1.13/kstring.c, samtools-0.1.13/bam_aux.c, - samtools-0.1.13/bam.c, samtools-0.1.13/bam_import.c, samtools-0.1.13/sam.c, - samtools-0.1.13/bam_index.c, samtools-0.1.13/bam_pileup.c, samtools-0.1.13/bam_lpileup.c, - samtools-0.1.13/bam_md.c, samtools-0.1.13/glf.c, samtools-0.1.13/razf.c, - samtools-0.1.13/faidx.c, samtools-0.1.13/knetfile.c, samtools-0.1.13/bam_sort.c, - samtools-0.1.13/sam_header.c, samtools-0.1.13/bam_reheader.c, samtools-0.1.13/kprobaln.c, - samtools-0.1.13/bam_maqcns.c, samtools-0.1.13/errmod.c, samtools-0.1.13/kaln.c + samtools-0.1.18/bgzf.c, samtools-0.1.18/kstring.c, samtools-0.1.18/bam_aux.c, + samtools-0.1.18/bam.c, samtools-0.1.18/bam_import.c, samtools-0.1.18/sam.c, + samtools-0.1.18/bam_index.c, samtools-0.1.18/bam_pileup.c, samtools-0.1.18/bam_lpileup.c, + samtools-0.1.18/bam_md.c, samtools-0.1.18/razf.c, + samtools-0.1.18/faidx.c, samtools-0.1.18/knetfile.c, samtools-0.1.18/bam_sort.c, + samtools-0.1.18/sam_header.c, samtools-0.1.18/bam_reheader.c, samtools-0.1.18/kprobaln.c, + samtools-0.1.18/errmod.c, samtools-0.1.18/kaln.c Install-includes: include/samtools.h, - samtools-0.1.13/bam.h, samtools-0.1.13/bam_endian.h, samtools-0.1.13/faidx.h, - samtools-0.1.13/glf.h, samtools-0.1.13/khash.h, - samtools-0.1.13/kseq.h, samtools-0.1.13/ksort.h, samtools-0.1.13/kstring.h, - samtools-0.1.13/razf.h, samtools-0.1.13/sam.h, samtools-0.1.13/sam_header.h, - samtools-0.1.13/bgzf.h, samtools-0.1.13/knetfile.h, samtools-0.1.13/kaln.h, - samtools-0.1.13/kprobaln.h, samtools-0.1.13/bam_maqcns.h, samtools-0.1.13/errmod.h + samtools-0.1.18/bam.h, samtools-0.1.18/bam_endian.h, samtools-0.1.18/faidx.h, + samtools-0.1.18/khash.h, + samtools-0.1.18/kseq.h, samtools-0.1.18/ksort.h, samtools-0.1.18/kstring.h, + samtools-0.1.18/razf.h, samtools-0.1.18/sam.h, samtools-0.1.18/sam_header.h, + samtools-0.1.18/bgzf.h, samtools-0.1.18/knetfile.h, samtools-0.1.18/kaln.h, + samtools-0.1.18/kprobaln.h, samtools-0.1.18/errmod.h CC-options: -g -D_FILE_OFFSET_BITS=64 -D_USE_KNETFILE Extra-libraries: z Ghc-options: -Wall diff --git a/src/Bio/SamTools/Bam.hs b/src/Bio/SamTools/Bam.hs --- a/src/Bio/SamTools/Bam.hs +++ b/src/Bio/SamTools/Bam.hs @@ -261,12 +261,12 @@ refSpLoc :: Bam1 -> Maybe SpLoc.SpliceLoc refSpLoc b | isUnmap b = Nothing | otherwise = liftM (stranded strand) $! liftM2 (cigarToSpLoc) (position b) (Just . cigars $ b) - where strand = if isReverse b then RevCompl else Fwd + where strand = if isReverse b then Minus else Plus -- | 'Just' the reference sequence location (as per 'refSpLoc') on -- the target reference (as per 'targetName') refSeqLoc :: Bam1 -> Maybe SpliceSeqLoc -refSeqLoc b = liftM2 OnSeq (liftM SeqName $! targetName b) (refSpLoc b) +refSeqLoc b = liftM2 OnSeq (liftM toSeqLabel $! targetName b) (refSpLoc b) -- | Handle for reading SAM/BAM format alignments data InHandle = InHandle { inFilename :: !FilePath diff --git a/src/Bio/SamTools/Cigar.hs b/src/Bio/SamTools/Cigar.hs --- a/src/Bio/SamTools/Cigar.hs +++ b/src/Bio/SamTools/Cigar.hs @@ -75,7 +75,7 @@ entry start (Cigar SoftClip _len) = (start, Nothing) entry start (Cigar HardClip _len) = (start, Nothing) entry start (Cigar Pad _len) = (start, Nothing) - contig start len = Just $! Loc.fromBoundsStrand (fromIntegral start) (fromIntegral $ start + len - 1) Fwd + contig start len = Just $! Loc.fromBoundsStrand (fromIntegral start) (fromIntegral $ start + len - 1) Plus mergeAdj :: Loc.ContigLoc -> [Loc.ContigLoc] -> [Loc.ContigLoc] mergeAdj cprev [] = [cprev] @@ -97,4 +97,4 @@ , zip (poses read0 len) (repeat Nothing)) cigarStep (read0, ref0) (Cigar HardClip _len) = ((read0, ref0), []) cigarStep (read0, ref0) (Cigar Pad _len) = ((read0, ref0), []) - poses start len = [ Just $! Pos.Pos (fromIntegral $ start + i) Fwd | i <- [0..(len-1)] ] + poses start len = [ Just $! Pos.Pos (fromIntegral $ start + i) Plus | i <- [0..(len-1)] ] diff --git a/src/Bio/SamTools/FaIdx.hs b/src/Bio/SamTools/FaIdx.hs --- a/src/Bio/SamTools/FaIdx.hs +++ b/src/Bio/SamTools/FaIdx.hs @@ -65,8 +65,8 @@ return sout fetchContig :: InHandle -> OnSeq Loc.ContigLoc -> IO (Maybe BS.ByteString) -fetchContig inh (OnSeq (SeqName name) cl) - = liftM result $ fetch inh name intbounds +fetchContig inh (OnSeq name cl) + = liftM result $ fetch inh (unSeqLabel name) intbounds where intbounds = (fromIntegral *** fromIntegral) $ Loc.bounds cl result sequ | BS.null sequ = Nothing | otherwise = return $! stranded (Loc.strand cl) sequ