packages feed

HsHTSLib (empty) → 1.3.2.0

raw patch · 90 files changed

+45613/−0 lines, 90 filesdep +basedep +bytestringdep +bytestring-lexingsetup-changed

Dependencies added: base, bytestring, bytestring-lexing, conduit-combinators, containers, inline-c, mtl, template-haskell

Files

+ ChangeLog.md view
@@ -0,0 +1,5 @@+# Revision history for passwd-manager++## 0.1.0  -- YYYY-mm-dd++* First version. Released on an unsuspecting world.
+ HsHTSLib.cabal view
@@ -0,0 +1,88 @@+name:                HsHTSLib+version:             1.3.2.0+synopsis:            High level bindings to htslib.+description:         This package provides high level bindings to htslib, a library+                     for processing high throughput DNA sequencing data.+license:             MIT+license-file:        LICENSE+author:              Kai Zhang+maintainer:          kai@kzhang.org+copyright:           (c) 2016 Kai Zhang+category:            Bio+build-type:          Simple+cabal-version:       >=1.10+extra-source-files:+  ChangeLog.md+  htslib-1.3.2/*.h+  htslib-1.3.2/htslib/*.h+  htslib-1.3.2/cram/*.h++library+  ghc-options:         -Wall+  exposed-modules:+    Bio.HTS+    Bio.HTS.Types++  build-depends:+      base >=4.8 && <5.0+    , bytestring+    , bytestring-lexing+    , containers+    , conduit-combinators+    , inline-c+    , mtl+    , template-haskell++  hs-source-dirs:      src+  extra-libraries: pthread+  include-dirs:+      htslib-1.3.2++  c-sources:+      src/Bio/HTS.c++      htslib-1.3.2/bgzf.c+      htslib-1.3.2/bgzip.c+      htslib-1.3.2/faidx.c+      htslib-1.3.2/hfile.c+      htslib-1.3.2/hfile_net.c+      htslib-1.3.2/hts.c+      htslib-1.3.2/kfunc.c+      htslib-1.3.2/knetfile.c+      htslib-1.3.2/kstring.c+      htslib-1.3.2/md5.c+      htslib-1.3.2/regidx.c+      htslib-1.3.2/sam.c+      htslib-1.3.2/synced_bcf_reader.c+      htslib-1.3.2/tbx.c+      htslib-1.3.2/vcf.c+      htslib-1.3.2/vcf_sweep.c+      htslib-1.3.2/vcfutils.c++      htslib-1.3.2/cram/cram_codecs.c+      htslib-1.3.2/cram/cram_decode.c+      htslib-1.3.2/cram/cram_encode.c+      htslib-1.3.2/cram/cram_external.c+      htslib-1.3.2/cram/cram_index.c+      htslib-1.3.2/cram/cram_io.c+      htslib-1.3.2/cram/cram_samtools.c+      htslib-1.3.2/cram/cram_stats.c+      htslib-1.3.2/cram/files.c+      htslib-1.3.2/cram/mFILE.c+      htslib-1.3.2/cram/open_trace_file.c+      htslib-1.3.2/cram/pooled_alloc.c+      htslib-1.3.2/cram/rANS_static.c+      htslib-1.3.2/cram/sam_header.c+      htslib-1.3.2/cram/string_alloc.c+      htslib-1.3.2/cram/thread_pool.c+      htslib-1.3.2/cram/vlen.c+      htslib-1.3.2/cram/zfio.c++      -- htslib-1.3.2/tabix.c+      -- htslib-1.3.2/htsfile.c++  default-language:    Haskell2010++source-repository  head+  type: git+  location: https://github.com/kaizhang/HsHTSLib.git
+ LICENSE view
@@ -0,0 +1,97 @@+[Files in this distribution outwith the htslib/ subdirectory are distributed+according to the terms of the following MIT license.]++The MIT License (MIT)++Copyright (c) 2016 Kai Zhang++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.+++[htslib C library are distributed according to the terms of the license.]++[Files in this distribution outwith the cram/ subdirectory are distributed+according to the terms of the following MIT/Expat license.]++The MIT/Expat License++Copyright (C) 2012-2014 Genome Research Ltd.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.+++[Files within the cram/ subdirectory in this distribution are distributed+according to the terms of the following Modified 3-Clause BSD license.]++The Modified-BSD License++Copyright (C) 2012-2014 Genome Research Ltd.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice,+   this list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+   this list of conditions and the following disclaimer in the documentation+   and/or other materials provided with the distribution.++3. Neither the names Genome Research Ltd and Wellcome Trust Sanger Institute+   nor the names of its contributors may be used to endorse or promote products+   derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR ITS CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+++[The use of a range of years within a copyright notice in this distribution+should be interpreted as being equivalent to a list of years including the+first and last year specified and all consecutive years between them.++For example, a copyright notice that reads "Copyright (C) 2005, 2007-2009,+2011-2012" should be interpreted as being identical to a notice that reads+"Copyright (C) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright notice+that reads "Copyright (C) 2005-2012" should be interpreted as being identical+to a notice that reads "Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010,+2011, 2012".]
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ htslib-1.3.2/bgzf.c view
@@ -0,0 +1,1330 @@+/* The MIT License++   Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology+                 2011, 2012 Attractive Chaos <attractor@live.co.uk>+   Copyright (C) 2009, 2013-2016 Genome Research Ltd++   Permission is hereby granted, free of charge, to any person obtaining a copy+   of this software and associated documentation files (the "Software"), to deal+   in the Software without restriction, including without limitation the rights+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+   copies of the Software, and to permit persons to whom the Software is+   furnished to do so, subject to the following conditions:++   The above copyright notice and this permission notice shall be included in+   all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+   THE SOFTWARE.+*/++#include <config.h>++#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <errno.h>+#include <unistd.h>+#include <assert.h>+#include <pthread.h>+#include <sys/types.h>+#include <inttypes.h>++#include "htslib/hts.h"+#include "htslib/bgzf.h"+#include "htslib/hfile.h"++#define BGZF_CACHE+#define BGZF_MT++#define BLOCK_HEADER_LENGTH 18+#define BLOCK_FOOTER_LENGTH 8+++/* BGZF/GZIP header (speciallized from RFC 1952; little endian):+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---++ | 31|139|  8|  4|              0|  0|255|      6| 66| 67|      2|BLK_LEN|+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---++  BGZF extension:+                ^                              ^   ^   ^+                |                              |   |   |+               FLG.EXTRA                     XLEN  B   C++  BGZF format is compatible with GZIP. It limits the size of each compressed+  block to 2^16 bytes and adds and an extra "BC" field in the gzip header which+  records the size.++*/+static const uint8_t g_magic[19] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\0\0";++#ifdef BGZF_CACHE+typedef struct {+    int size;+    uint8_t *block;+    int64_t end_offset;+} cache_t;+#include "htslib/khash.h"+KHASH_MAP_INIT_INT64(cache, cache_t)+#endif++typedef struct+{+    uint64_t uaddr;  // offset w.r.t. uncompressed data+    uint64_t caddr;  // offset w.r.t. compressed data+}+bgzidx1_t;++struct __bgzidx_t+{+    int noffs, moffs;       // the size of the index, n:used, m:allocated+    bgzidx1_t *offs;        // offsets+    uint64_t ublock_addr;   // offset of the current block (uncompressed data)+};++void bgzf_index_destroy(BGZF *fp);+int bgzf_index_add_block(BGZF *fp);++static inline void packInt16(uint8_t *buffer, uint16_t value)+{+    buffer[0] = value;+    buffer[1] = value >> 8;+}++static inline int unpackInt16(const uint8_t *buffer)+{+    return buffer[0] | buffer[1] << 8;+}++static inline void packInt32(uint8_t *buffer, uint32_t value)+{+    buffer[0] = value;+    buffer[1] = value >> 8;+    buffer[2] = value >> 16;+    buffer[3] = value >> 24;+}++static const char *bgzf_zerr(int errnum, z_stream *zs)+{+    static char buffer[32];++    /* Return zs->msg if available.+       zlib doesn't set this very reliably.  Looking at the source suggests+       that it may get set to a useful message for deflateInit2, inflateInit2+       and inflate when it returns Z_DATA_ERROR. For inflate with other+       return codes, deflate, deflateEnd and inflateEnd it doesn't appear+       to be useful.  For the likely non-useful cases, the caller should+       pass NULL into zs. */++    if (zs && zs->msg) return zs->msg;++    // gzerror OF((gzFile file, int *errnum)+    switch (errnum) {+    case Z_ERRNO:+        return strerror(errno);+    case Z_STREAM_ERROR:+        return "invalid parameter/compression level, or inconsistent stream state";+    case Z_DATA_ERROR:+        return "invalid or incomplete IO";+    case Z_MEM_ERROR:+        return "out of memory";+    case Z_BUF_ERROR:+        return "progress temporarily not possible, or in() / out() returned an error";+    case Z_VERSION_ERROR:+        return "zlib version mismatch";+    case Z_OK: // 0: maybe gzgets error Z_NULL+    default:+        snprintf(buffer, sizeof(buffer), "[%d] unknown", errnum);+        return buffer;  // FIXME: Not thread-safe.+    }+}++static BGZF *bgzf_read_init(hFILE *hfpr)+{+    BGZF *fp;+    uint8_t magic[18];+    ssize_t n = hpeek(hfpr, magic, 18);+    if (n < 0) return NULL;++    fp = (BGZF*)calloc(1, sizeof(BGZF));+    if (fp == NULL) return NULL;++    fp->is_write = 0;+    fp->is_compressed = (n==2 && magic[0]==0x1f && magic[1]==0x8b);+    fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);+    fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);+    fp->is_compressed = (n==18 && magic[0]==0x1f && magic[1]==0x8b) ? 1 : 0;+    fp->is_gzip = ( !fp->is_compressed || ((magic[3]&4) && memcmp(&magic[12], "BC\2\0",4)==0) ) ? 0 : 1;+#ifdef BGZF_CACHE+    fp->cache = kh_init(cache);+#endif+    return fp;+}++// get the compress level from the mode string: compress_level==-1 for the default level, -2 plain uncompressed+static int mode2level(const char *__restrict mode)+{+    int i, compress_level = -1;+    for (i = 0; mode[i]; ++i)+        if (mode[i] >= '0' && mode[i] <= '9') break;+    if (mode[i]) compress_level = (int)mode[i] - '0';+    if (strchr(mode, 'u')) compress_level = -2;+    return compress_level;+}+static BGZF *bgzf_write_init(const char *mode)+{+    BGZF *fp;+    fp = (BGZF*)calloc(1, sizeof(BGZF));+    if (fp == NULL) goto mem_fail;+    fp->is_write = 1;+    int compress_level = mode2level(mode);+    if ( compress_level==-2 )+    {+        fp->is_compressed = 0;+        return fp;+    }+    fp->is_compressed = 1;++    fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);+    if (fp->uncompressed_block == NULL) goto mem_fail;+    fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);+    if (fp->compressed_block == NULL) goto mem_fail;++    fp->compress_level = compress_level < 0? Z_DEFAULT_COMPRESSION : compress_level; // Z_DEFAULT_COMPRESSION==-1+    if (fp->compress_level > 9) fp->compress_level = Z_DEFAULT_COMPRESSION;+    if ( strchr(mode,'g') )+    {+        // gzip output+        fp->is_gzip = 1;+        fp->gz_stream = (z_stream*)calloc(1,sizeof(z_stream));+        if (fp->gz_stream == NULL) goto mem_fail;+        fp->gz_stream->zalloc = NULL;+        fp->gz_stream->zfree  = NULL;+        fp->gz_stream->msg    = NULL;++        int ret = deflateInit2(fp->gz_stream, fp->compress_level, Z_DEFLATED, 15|16, 8, Z_DEFAULT_STRATEGY);+        if (ret!=Z_OK) {+            if (hts_verbose >= 1) {+                fprintf(stderr, "[E::%s] deflateInit2 failed: %s\n",+                        __func__, bgzf_zerr(ret, fp->gz_stream));+            }+            goto fail;+        }+    }+    return fp;++ mem_fail:+    if (hts_verbose >= 1) {+        fprintf(stderr, "[E::%s] %s\n", __func__, strerror(errno));+    }+ fail:+    if (fp != NULL) {+        free(fp->uncompressed_block);+        free(fp->compressed_block);+        free(fp->gz_stream);+        free(fp);+    }+    return NULL;+}++BGZF *bgzf_open(const char *path, const char *mode)+{+    BGZF *fp = 0;+    assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);+    if (strchr(mode, 'r')) {+        hFILE *fpr;+        if ((fpr = hopen(path, mode)) == 0) return 0;+        fp = bgzf_read_init(fpr);+        if (fp == 0) { hclose_abruptly(fpr); return NULL; }+        fp->fp = fpr;+    } else if (strchr(mode, 'w') || strchr(mode, 'a')) {+        hFILE *fpw;+        if ((fpw = hopen(path, mode)) == 0) return 0;+        fp = bgzf_write_init(mode);+        if (fp == NULL) return NULL;+        fp->fp = fpw;+    }+    else { errno = EINVAL; return 0; }++    fp->is_be = ed_is_big();+    return fp;+}++BGZF *bgzf_dopen(int fd, const char *mode)+{+    BGZF *fp = 0;+    assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);+    if (strchr(mode, 'r')) {+        hFILE *fpr;+        if ((fpr = hdopen(fd, mode)) == 0) return 0;+        fp = bgzf_read_init(fpr);+        if (fp == 0) { hclose_abruptly(fpr); return NULL; } // FIXME this closes fd+        fp->fp = fpr;+    } else if (strchr(mode, 'w') || strchr(mode, 'a')) {+        hFILE *fpw;+        if ((fpw = hdopen(fd, mode)) == 0) return 0;+        fp = bgzf_write_init(mode);+        if (fp == NULL) return NULL;+        fp->fp = fpw;+    }+    else { errno = EINVAL; return 0; }++    fp->is_be = ed_is_big();+    return fp;+}++BGZF *bgzf_hopen(hFILE *hfp, const char *mode)+{+    BGZF *fp = NULL;+    assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);+    if (strchr(mode, 'r')) {+        fp = bgzf_read_init(hfp);+        if (fp == NULL) return NULL;+    } else if (strchr(mode, 'w') || strchr(mode, 'a')) {+        fp = bgzf_write_init(mode);+        if (fp == NULL) return NULL;+    }+    else { errno = EINVAL; return 0; }++    fp->fp = hfp;+    fp->is_be = ed_is_big();+    return fp;+}++int bgzf_compress(void *_dst, size_t *dlen, const void *src, size_t slen, int level)+{+    uint32_t crc;+    z_stream zs;+    uint8_t *dst = (uint8_t*)_dst;++    // compress the body+    zs.zalloc = NULL; zs.zfree = NULL;+    zs.msg = NULL;+    zs.next_in  = (Bytef*)src;+    zs.avail_in = slen;+    zs.next_out = dst + BLOCK_HEADER_LENGTH;+    zs.avail_out = *dlen - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH;+    int ret = deflateInit2(&zs, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); // -15 to disable zlib header/footer+    if (ret!=Z_OK) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] deflateInit2 failed: %s\n",+                    __func__, bgzf_zerr(ret, &zs));+        }+        return -1;+    }+    if ((ret = deflate(&zs, Z_FINISH)) != Z_STREAM_END) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] deflate failed: %s\n",+                    __func__, bgzf_zerr(ret, ret == Z_DATA_ERROR ? &zs : NULL));+        }+        return -1;+    }+    if ((ret = deflateEnd(&zs)) != Z_OK) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] deflateEnd failed: %s\n",+                    __func__, bgzf_zerr(ret, NULL));+        }+        return -1;+    }+    *dlen = zs.total_out + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;+    // write the header+    memcpy(dst, g_magic, BLOCK_HEADER_LENGTH); // the last two bytes are a place holder for the length of the block+    packInt16(&dst[16], *dlen - 1); // write the compressed length; -1 to fit 2 bytes+    // write the footer+    crc = crc32(crc32(0L, NULL, 0L), (Bytef*)src, slen);+    packInt32((uint8_t*)&dst[*dlen - 8], crc);+    packInt32((uint8_t*)&dst[*dlen - 4], slen);+    return 0;+}++static int bgzf_gzip_compress(BGZF *fp, void *_dst, size_t *dlen, const void *src, size_t slen, int level)+{+    uint8_t *dst = (uint8_t*)_dst;+    z_stream *zs = fp->gz_stream;+    int flush = slen ? Z_NO_FLUSH : Z_FINISH;+    zs->next_in   = (Bytef*)src;+    zs->avail_in  = slen;+    zs->next_out  = dst;+    zs->avail_out = *dlen;+    int ret = deflate(zs, flush);+    if (ret == Z_STREAM_ERROR) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] deflate failed: %s\n",+                    __func__, bgzf_zerr(ret, NULL));+        }+        return -1;+    }+    *dlen = *dlen - zs->avail_out;+    return 0;+}++// Deflate the block in fp->uncompressed_block into fp->compressed_block. Also adds an extra field that stores the compressed block length.+static int deflate_block(BGZF *fp, int block_length)+{+    size_t comp_size = BGZF_MAX_BLOCK_SIZE;+    int ret;+    if ( !fp->is_gzip )+        ret = bgzf_compress(fp->compressed_block, &comp_size, fp->uncompressed_block, block_length, fp->compress_level);+    else+        ret = bgzf_gzip_compress(fp, fp->compressed_block, &comp_size, fp->uncompressed_block, block_length, fp->compress_level);++    if ( ret != 0 )+    {+        if (hts_verbose >= 3) {+            fprintf(stderr, "[E::%s] compression error %d\n", __func__, ret);+        }+        fp->errcode |= BGZF_ERR_ZLIB;+        return -1;+    }+    fp->block_offset = 0;+    return comp_size;+}++// Inflate the block in fp->compressed_block into fp->uncompressed_block+static int inflate_block(BGZF* fp, int block_length)+{+    z_stream zs;+    zs.zalloc = NULL;+    zs.zfree = NULL;+    zs.msg = NULL;+    zs.next_in = (Bytef*)fp->compressed_block + 18;+    zs.avail_in = block_length - 16;+    zs.next_out = (Bytef*)fp->uncompressed_block;+    zs.avail_out = BGZF_MAX_BLOCK_SIZE;++    int ret = inflateInit2(&zs, -15);+    if (ret != Z_OK) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] inflateInit2 failed: %s\n",+                    __func__, bgzf_zerr(ret, &zs));+        }+        fp->errcode |= BGZF_ERR_ZLIB;+        return -1;+    }+    if ((ret = inflate(&zs, Z_FINISH)) != Z_STREAM_END) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] inflate failed: %s\n",+                    __func__, bgzf_zerr(ret, ret == Z_DATA_ERROR ? &zs : NULL));+        }+        if ((ret = inflateEnd(&zs)) != Z_OK) {+            if (hts_verbose >= 2) {+                fprintf(stderr, "[E::%s] inflateEnd failed: %s\n",+                        __func__, bgzf_zerr(ret, NULL));+            }+        }+        fp->errcode |= BGZF_ERR_ZLIB;+        return -1;+    }+    if ((ret = inflateEnd(&zs)) != Z_OK) {+        if (hts_verbose >= 1) {+            fprintf(stderr, "[E::%s] inflateEnd failed: %s\n",+                    __func__, bgzf_zerr(ret, NULL));+        }+        fp->errcode |= BGZF_ERR_ZLIB;+        return -1;+    }+    return zs.total_out;+}++static int inflate_gzip_block(BGZF *fp, int cached)+{+    int ret = Z_OK;+    do+    {+        if ( !cached && fp->gz_stream->avail_out!=0 )+        {+            fp->gz_stream->avail_in = hread(fp->fp, fp->compressed_block, BGZF_BLOCK_SIZE);+            if ( fp->gz_stream->avail_in<=0 ) return fp->gz_stream->avail_in;+            if ( fp->gz_stream->avail_in==0 ) break;+            fp->gz_stream->next_in = fp->compressed_block;+        }+        else cached = 0;+        do+        {+            fp->gz_stream->next_out = (Bytef*)fp->uncompressed_block + fp->block_offset;+            fp->gz_stream->avail_out = BGZF_MAX_BLOCK_SIZE - fp->block_offset;+            fp->gz_stream->msg = NULL;+            ret = inflate(fp->gz_stream, Z_NO_FLUSH);+            if ( ret==Z_BUF_ERROR ) continue;   // non-critical error+            if ( ret<0 ) {+                if (hts_verbose >= 1) {+                    fprintf(stderr, "[E::%s] inflate failed: %s\n",+                            __func__,+                            bgzf_zerr(ret, ret == Z_DATA_ERROR ? fp->gz_stream : NULL));+                }+                fp->errcode |= BGZF_ERR_ZLIB;+                return -1;+            }+            unsigned int have = BGZF_MAX_BLOCK_SIZE - fp->gz_stream->avail_out;+            if ( have ) return have;+        }+        while ( fp->gz_stream->avail_out == 0 );+    }+    while (ret != Z_STREAM_END);+    return BGZF_MAX_BLOCK_SIZE - fp->gz_stream->avail_out;+}++// Returns: 0 on success (BGZF header); -1 on non-BGZF GZIP header; -2 on error+static int check_header(const uint8_t *header)+{+    if ( header[0] != 31 || header[1] != 139 || header[2] != 8 ) return -2;+    return ((header[3] & 4) != 0+            && unpackInt16((uint8_t*)&header[10]) == 6+            && header[12] == 'B' && header[13] == 'C'+            && unpackInt16((uint8_t*)&header[14]) == 2) ? 0 : -1;+}++#ifdef BGZF_CACHE+static void free_cache(BGZF *fp)+{+    khint_t k;+    khash_t(cache) *h = (khash_t(cache)*)fp->cache;+    if (fp->is_write) return;+    for (k = kh_begin(h); k < kh_end(h); ++k)+        if (kh_exist(h, k)) free(kh_val(h, k).block);+    kh_destroy(cache, h);+}++static int load_block_from_cache(BGZF *fp, int64_t block_address)+{+    khint_t k;+    cache_t *p;+    khash_t(cache) *h = (khash_t(cache)*)fp->cache;+    k = kh_get(cache, h, block_address);+    if (k == kh_end(h)) return 0;+    p = &kh_val(h, k);+    if (fp->block_length != 0) fp->block_offset = 0;+    fp->block_address = block_address;+    fp->block_length = p->size;+    memcpy(fp->uncompressed_block, p->block, BGZF_MAX_BLOCK_SIZE);+    if ( hseek(fp->fp, p->end_offset, SEEK_SET) < 0 )+    {+        // todo: move the error up+        fprintf(stderr,"Could not hseek to %"PRId64"\n", p->end_offset);+        exit(1);+    }+    return p->size;+}++static void cache_block(BGZF *fp, int size)+{+    int ret;+    khint_t k;+    cache_t *p;+    khash_t(cache) *h = (khash_t(cache)*)fp->cache;+    if (BGZF_MAX_BLOCK_SIZE >= fp->cache_size) return;+    if ((kh_size(h) + 1) * BGZF_MAX_BLOCK_SIZE > (uint32_t)fp->cache_size) {+        /* A better way would be to remove the oldest block in the+         * cache, but here we remove a random one for simplicity. This+         * should not have a big impact on performance. */+        for (k = kh_begin(h); k < kh_end(h); ++k)+            if (kh_exist(h, k)) break;+        if (k < kh_end(h)) {+            free(kh_val(h, k).block);+            kh_del(cache, h, k);+        }+    }+    k = kh_put(cache, h, fp->block_address, &ret);+    if (ret == 0) return; // if this happens, a bug!+    p = &kh_val(h, k);+    p->size = fp->block_length;+    p->end_offset = fp->block_address + size;+    p->block = (uint8_t*)malloc(BGZF_MAX_BLOCK_SIZE);+    memcpy(kh_val(h, k).block, fp->uncompressed_block, BGZF_MAX_BLOCK_SIZE);+}+#else+static void free_cache(BGZF *fp) {}+static int load_block_from_cache(BGZF *fp, int64_t block_address) {return 0;}+static void cache_block(BGZF *fp, int size) {}+#endif++int bgzf_read_block(BGZF *fp)+{+    uint8_t header[BLOCK_HEADER_LENGTH], *compressed_block;+    int count, size = 0, block_length, remaining;++    // Reading an uncompressed file+    if ( !fp->is_compressed )+    {+        count = hread(fp->fp, fp->uncompressed_block, BGZF_MAX_BLOCK_SIZE);+        if (count < 0)  // Error+        {+            fp->errcode |= BGZF_ERR_IO;+            return -1;+        }+        else if (count == 0)  // EOF+        {+            fp->block_length = 0;+            return 0;+        }+        if (fp->block_length != 0) fp->block_offset = 0;+        fp->block_address += count;+        fp->block_length = count;+        return 0;+    }++    // Reading compressed file+    int64_t block_address;+    block_address = htell(fp->fp);+    if ( fp->is_gzip && fp->gz_stream ) // is this is a initialized gzip stream?+    {+        count = inflate_gzip_block(fp, 0);+        if ( count<0 )+        {+            fp->errcode |= BGZF_ERR_ZLIB;+            return -1;+        }+        fp->block_length = count;+        fp->block_address = block_address;+        return 0;+    }+    if (fp->cache_size && load_block_from_cache(fp, block_address)) return 0;+    count = hread(fp->fp, header, sizeof(header));+    if (count == 0) { // no data read+        fp->block_length = 0;+        return 0;+    }+    int ret;+    if ( count != sizeof(header) || (ret=check_header(header))==-2 )+    {+        fp->errcode |= BGZF_ERR_HEADER;+        return -1;+    }+    if ( ret==-1 )+    {+        // GZIP, not BGZF+        uint8_t *cblock = (uint8_t*)fp->compressed_block;+        memcpy(cblock, header, sizeof(header));+        count = hread(fp->fp, cblock+sizeof(header), BGZF_BLOCK_SIZE - sizeof(header)) + sizeof(header);+        int nskip = 10;++        // Check optional fields to skip: FLG.FNAME,FLG.FCOMMENT,FLG.FHCRC,FLG.FEXTRA+        // Note: Some of these fields are untested, I did not have appropriate data available+        if ( header[3] & 0x4 ) // FLG.FEXTRA+        {+            nskip += unpackInt16(&cblock[nskip]) + 2;+        }+        if ( header[3] & 0x8 ) // FLG.FNAME+        {+            while ( nskip<count && cblock[nskip] ) nskip++;+            nskip++;+        }+        if ( header[3] & 0x10 ) // FLG.FCOMMENT+        {+            while ( nskip<count && cblock[nskip] ) nskip++;+            nskip++;+        }+        if ( header[3] & 0x2 ) nskip += 2;  //  FLG.FHCRC++        /* FIXME: Should handle this better.  There's no reason why+           someone shouldn't include a massively long comment in their+           gzip stream. */+        if ( nskip >= count )+        {+            fp->errcode |= BGZF_ERR_HEADER;+            return -1;+        }++        fp->is_gzip = 1;+        fp->gz_stream = (z_stream*) calloc(1,sizeof(z_stream));+        int ret = inflateInit2(fp->gz_stream, -15);+        if (ret != Z_OK)+        {+            if (hts_verbose >= 1) {+                fprintf(stderr, "[E::%s] inflateInit2 failed: %s",+                        __func__, bgzf_zerr(ret, fp->gz_stream));+            }+            fp->errcode |= BGZF_ERR_ZLIB;+            return -1;+        }+        fp->gz_stream->avail_in = count - nskip;+        fp->gz_stream->next_in  = cblock + nskip;+        count = inflate_gzip_block(fp, 1);+        if ( count<0 )+        {+            fp->errcode |= BGZF_ERR_ZLIB;+            return -1;+        }+        fp->block_length = count;+        fp->block_address = block_address;+        if ( fp->idx_build_otf ) return -1; // cannot build index for gzip+        return 0;+    }+    size = count;+    block_length = unpackInt16((uint8_t*)&header[16]) + 1; // +1 because when writing this number, we used "-1"+    compressed_block = (uint8_t*)fp->compressed_block;+    memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);+    remaining = block_length - BLOCK_HEADER_LENGTH;+    count = hread(fp->fp, &compressed_block[BLOCK_HEADER_LENGTH], remaining);+    if (count != remaining) {+        fp->errcode |= BGZF_ERR_IO;+        return -1;+    }+    size += count;+    if ((count = inflate_block(fp, block_length)) < 0) {+        if (hts_verbose >= 2) fprintf(stderr, "[E::%s] inflate_block error %d\n", __func__, count);+        fp->errcode |= BGZF_ERR_ZLIB;+        return -1;+    }+    if (fp->block_length != 0) fp->block_offset = 0; // Do not reset offset if this read follows a seek.+    fp->block_address = block_address;+    fp->block_length = count;+    if ( fp->idx_build_otf )+    {+        bgzf_index_add_block(fp);+        fp->idx->ublock_addr += count;+    }+    cache_block(fp, size);+    return 0;+}++ssize_t bgzf_read(BGZF *fp, void *data, size_t length)+{+    ssize_t bytes_read = 0;+    uint8_t *output = (uint8_t*)data;+    if (length <= 0) return 0;+    assert(fp->is_write == 0);+    while (bytes_read < length) {+        int copy_length, available = fp->block_length - fp->block_offset;+        uint8_t *buffer;+        if (available <= 0) {+            int ret = bgzf_read_block(fp);+            if (ret != 0) {+                if (hts_verbose >= 2) {+                    fprintf(stderr, "[E::%s] bgzf_read_block error %d after %zd of %zu bytes\n", __func__, ret, bytes_read, length);+                }+                fp->errcode |= BGZF_ERR_ZLIB;+                return -1;+            }+            available = fp->block_length - fp->block_offset;+            if (available <= 0) break;+        }+        copy_length = length - bytes_read < available? length - bytes_read : available;+        buffer = (uint8_t*)fp->uncompressed_block;+        memcpy(output, buffer + fp->block_offset, copy_length);+        fp->block_offset += copy_length;+        output += copy_length;+        bytes_read += copy_length;+    }+    if (fp->block_offset == fp->block_length) {+        fp->block_address = htell(fp->fp);+        fp->block_offset = fp->block_length = 0;+    }+    fp->uncompressed_address += bytes_read;+    return bytes_read;+}++ssize_t bgzf_raw_read(BGZF *fp, void *data, size_t length)+{+    return hread(fp->fp, data, length);+}++#ifdef BGZF_MT++typedef struct {+    struct bgzf_mtaux_t *mt;+    void *buf;+    int i, errcode, toproc, compress_level;+} worker_t;++typedef struct bgzf_mtaux_t {+    int n_threads, n_blks, curr, done;+    volatile int proc_cnt;+    void **blk;+    int *len;+    worker_t *w;+    pthread_t *tid;+    pthread_mutex_t lock;+    pthread_cond_t cv;+} mtaux_t;++static int worker_aux(worker_t *w)+{+    int i, stop = 0;+    // wait for condition: to process or all done+    pthread_mutex_lock(&w->mt->lock);+    while (!w->toproc && !w->mt->done)+        pthread_cond_wait(&w->mt->cv, &w->mt->lock);+    if (w->mt->done) stop = 1;+    w->toproc = 0;+    pthread_mutex_unlock(&w->mt->lock);+    if (stop) return 1; // to quit the thread+    w->errcode = 0;+    for (i = w->i; i < w->mt->curr; i += w->mt->n_threads) {+        size_t clen = BGZF_MAX_BLOCK_SIZE;+        int ret = bgzf_compress(w->buf, &clen, w->mt->blk[i], w->mt->len[i], w->compress_level);+        if (ret != 0) {+            if (hts_verbose >= 2) fprintf(stderr, "[E::%s] bgzf_compress error %d\n", __func__, ret);+            w->errcode |= BGZF_ERR_ZLIB; // Report error+            // We're not going to do any more, so set remaining lengths to 0+            for (; i < w->mt->curr; i += w->mt->n_threads) w->mt->len[i] = 0;+            break; // Give up+        } else {+            memcpy(w->mt->blk[i], w->buf, clen);+            w->mt->len[i] = clen;+        }+    }+    __sync_fetch_and_add(&w->mt->proc_cnt, 1);+    return 0;+}++static void *mt_worker(void *data)+{+    while (worker_aux((worker_t*)data) == 0);+    return 0;+}++int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks)+{+    int i;+    mtaux_t *mt;+    pthread_attr_t attr;+    if (!fp->is_write || fp->mt || n_threads <= 1) return -1;+    mt = (mtaux_t*)calloc(1, sizeof(mtaux_t));+    mt->n_threads = n_threads;+    mt->n_blks = n_threads * n_sub_blks;+    mt->len = (int*)calloc(mt->n_blks, sizeof(int));+    mt->blk = (void**)calloc(mt->n_blks, sizeof(void*));+    for (i = 0; i < mt->n_blks; ++i)+        mt->blk[i] = malloc(BGZF_MAX_BLOCK_SIZE);+    mt->tid = (pthread_t*)calloc(mt->n_threads, sizeof(pthread_t)); // tid[0] is not used, as the worker 0 is launched by the master+    mt->w = (worker_t*)calloc(mt->n_threads, sizeof(worker_t));+    for (i = 0; i < mt->n_threads; ++i) {+        mt->w[i].i = i;+        mt->w[i].mt = mt;+        mt->w[i].compress_level = fp->compress_level;+        mt->w[i].buf = malloc(BGZF_MAX_BLOCK_SIZE);+    }+    pthread_attr_init(&attr);+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);+    pthread_mutex_init(&mt->lock, 0);+    pthread_cond_init(&mt->cv, 0);+    for (i = 1; i < mt->n_threads; ++i) // worker 0 is effectively launched by the master thread+        pthread_create(&mt->tid[i], &attr, mt_worker, &mt->w[i]);+    fp->mt = mt;+    return 0;+}++static void mt_destroy(mtaux_t *mt)+{+    int i;+    // signal all workers to quit+    pthread_mutex_lock(&mt->lock);+    mt->done = 1; mt->proc_cnt = 0;+    pthread_cond_broadcast(&mt->cv);+    pthread_mutex_unlock(&mt->lock);+    for (i = 1; i < mt->n_threads; ++i) pthread_join(mt->tid[i], 0); // worker 0 is effectively launched by the master thread+    // free other data allocated on heap+    for (i = 0; i < mt->n_blks; ++i) free(mt->blk[i]);+    for (i = 0; i < mt->n_threads; ++i) free(mt->w[i].buf);+    free(mt->blk); free(mt->len); free(mt->w); free(mt->tid);+    pthread_cond_destroy(&mt->cv);+    pthread_mutex_destroy(&mt->lock);+    free(mt);+}++static void mt_queue(BGZF *fp)+{+    mtaux_t *mt = fp->mt;+    assert(mt->curr < mt->n_blks); // guaranteed by the caller+    memcpy(mt->blk[mt->curr], fp->uncompressed_block, fp->block_offset);+    mt->len[mt->curr] = fp->block_offset;+    fp->block_offset = 0;+    ++mt->curr;+}++static int mt_flush_queue(BGZF *fp)+{+    int i;+    mtaux_t *mt = fp->mt;+    // signal all the workers to compress+    pthread_mutex_lock(&mt->lock);+    for (i = 0; i < mt->n_threads; ++i) mt->w[i].toproc = 1;+    mt->proc_cnt = 0;+    pthread_cond_broadcast(&mt->cv);+    pthread_mutex_unlock(&mt->lock);+    // worker 0 is doing things here+    worker_aux(&mt->w[0]);+    // wait for all the threads to complete+    while (mt->proc_cnt < mt->n_threads);+    // dump data to disk+    for (i = 0; i < mt->n_threads; ++i) fp->errcode |= mt->w[i].errcode;+    if (fp->errcode == 0) {+        /* Only try to write if all the threads worked, as otherwise we+           could get a file with holes in it */+        for (i = 0; i < mt->curr; ++i) {+            if (hwrite(fp->fp, mt->blk[i], mt->len[i]) != mt->len[i]) {+                fp->errcode |= BGZF_ERR_IO;+                break;+            }+        }+    }+    mt->curr = 0;+    return (fp->errcode == 0)? 0 : -1;+}++static int lazy_flush(BGZF *fp)+{+    if (fp->mt) {+        if (fp->block_offset) mt_queue(fp);+        return (fp->mt->curr < fp->mt->n_blks)? 0 : mt_flush_queue(fp);+    }+    else return bgzf_flush(fp);+}++#else  // ~ #ifdef BGZF_MT++int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks)+{+    return 0;+}++static inline int lazy_flush(BGZF *fp)+{+    return bgzf_flush(fp);+}++#endif // ~ #ifdef BGZF_MT++int bgzf_flush(BGZF *fp)+{+    if (!fp->is_write) return 0;+#ifdef BGZF_MT+    if (fp->mt) {+        if (fp->block_offset) mt_queue(fp); // guaranteed that assertion does not fail+        return mt_flush_queue(fp);+    }+#endif+    while (fp->block_offset > 0) {+        int block_length;+        if ( fp->idx_build_otf )+        {+            bgzf_index_add_block(fp);+            fp->idx->ublock_addr += fp->block_offset;+        }+        block_length = deflate_block(fp, fp->block_offset);+        if (block_length < 0) {+            if (hts_verbose >= 3) fprintf(stderr, "[E::%s] deflate_block error %d\n", __func__, block_length);+            return -1;+        }+        if (hwrite(fp->fp, fp->compressed_block, block_length) != block_length) {+            if (hts_verbose >= 1) fprintf(stderr, "[E::%s] hwrite error (wrong size)\n", __func__);+            fp->errcode |= BGZF_ERR_IO; // possibly truncated file+            return -1;+        }+        fp->block_address += block_length;+    }+    return 0;+}++int bgzf_flush_try(BGZF *fp, ssize_t size)+{+    if (fp->block_offset + size > BGZF_BLOCK_SIZE) return lazy_flush(fp);+    return 0;+}++ssize_t bgzf_write(BGZF *fp, const void *data, size_t length)+{+    if ( !fp->is_compressed )+        return hwrite(fp->fp, data, length);++    const uint8_t *input = (const uint8_t*)data;+    ssize_t remaining = length;+    assert(fp->is_write);+    while (remaining > 0) {+        uint8_t* buffer = (uint8_t*)fp->uncompressed_block;+        int copy_length = BGZF_BLOCK_SIZE - fp->block_offset;+        if (copy_length > remaining) copy_length = remaining;+        memcpy(buffer + fp->block_offset, input, copy_length);+        fp->block_offset += copy_length;+        input += copy_length;+        remaining -= copy_length;+        if (fp->block_offset == BGZF_BLOCK_SIZE) {+            if (lazy_flush(fp) != 0) return -1;+        }+    }+    return length - remaining;+}++ssize_t bgzf_raw_write(BGZF *fp, const void *data, size_t length)+{+    return hwrite(fp->fp, data, length);+}++int bgzf_close(BGZF* fp)+{+    int ret, block_length;+    if (fp == 0) return -1;+    if (fp->is_write && fp->is_compressed) {+        if (bgzf_flush(fp) != 0) return -1;+        fp->compress_level = -1;+        block_length = deflate_block(fp, 0); // write an empty block+        if (block_length < 0) {+            if (hts_verbose >= 3) fprintf(stderr, "[E::%s] deflate_block error %d\n", __func__, block_length);+            return -1;+        }+        if (hwrite(fp->fp, fp->compressed_block, block_length) < 0+            || hflush(fp->fp) != 0) {+            if (hts_verbose >= 1) fprintf(stderr, "[E::%s] file write error\n", __func__);+            fp->errcode |= BGZF_ERR_IO;+            return -1;+        }+#ifdef BGZF_MT+        if (fp->mt) mt_destroy(fp->mt);+#endif+    }+    if ( fp->is_gzip )+    {+        if (!fp->is_write) ret = inflateEnd(fp->gz_stream);+        else ret = deflateEnd(fp->gz_stream);+        if (ret != Z_OK && hts_verbose >= 1)+            fprintf(stderr, "[E::%s] inflateEnd/deflateEnd failed: %s\n",+                    __func__, bgzf_zerr(ret, NULL));+        free(fp->gz_stream);+    }+    ret = hclose(fp->fp);+    if (ret != 0) return -1;+    bgzf_index_destroy(fp);+    free(fp->uncompressed_block);+    free(fp->compressed_block);+    free_cache(fp);+    free(fp);+    return 0;+}++void bgzf_set_cache_size(BGZF *fp, int cache_size)+{+    if (fp) fp->cache_size = cache_size;+}++int bgzf_check_EOF(BGZF *fp)+{+    uint8_t buf[28];+    off_t offset = htell(fp->fp);+    if (hseek(fp->fp, -28, SEEK_END) < 0) {+        if (errno == ESPIPE) { hclearerr(fp->fp); return 2; }+        else return -1;+    }+    if ( hread(fp->fp, buf, 28) != 28 ) return -1;+    if ( hseek(fp->fp, offset, SEEK_SET) < 0 ) return -1;+    return (memcmp("\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0", buf, 28) == 0)? 1 : 0;+}++int64_t bgzf_seek(BGZF* fp, int64_t pos, int where)+{+    int block_offset;+    int64_t block_address;++    if (fp->is_write || where != SEEK_SET) {+        fp->errcode |= BGZF_ERR_MISUSE;+        return -1;+    }+    block_offset = pos & 0xFFFF;+    block_address = pos >> 16;+    if (hseek(fp->fp, block_address, SEEK_SET) < 0) {+        fp->errcode |= BGZF_ERR_IO;+        return -1;+    }+    fp->block_length = 0;  // indicates current block has not been loaded+    fp->block_address = block_address;+    fp->block_offset = block_offset;+    return 0;+}++int bgzf_is_bgzf(const char *fn)+{+    uint8_t buf[16];+    int n;+    hFILE *fp;+    if ((fp = hopen(fn, "r")) == 0) return 0;+    n = hread(fp, buf, 16);+    if ( hclose(fp) < 0 ) return -1;+    if (n != 16) return 0;+    return memcmp(g_magic, buf, 16) == 0? 1 : 0;+}++int bgzf_getc(BGZF *fp)+{+    int c;+    if (fp->block_offset >= fp->block_length) {+        if (bgzf_read_block(fp) != 0) return -2; /* error */+        if (fp->block_length == 0) return -1; /* end-of-file */+    }+    c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];+    if (fp->block_offset == fp->block_length) {+        fp->block_address = htell(fp->fp);+        fp->block_offset = 0;+        fp->block_length = 0;+    }+    fp->uncompressed_address++;+    return c;+}++#ifndef kroundup32+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))+#endif++int bgzf_getline(BGZF *fp, int delim, kstring_t *str)+{+    int l, state = 0;+    unsigned char *buf = (unsigned char*)fp->uncompressed_block;+    str->l = 0;+    do {+        if (fp->block_offset >= fp->block_length) {+            if (bgzf_read_block(fp) != 0) { state = -2; break; }+            if (fp->block_length == 0) { state = -1; break; }+        }+        for (l = fp->block_offset; l < fp->block_length && buf[l] != delim; ++l);+        if (l < fp->block_length) state = 1;+        l -= fp->block_offset;+        if (str->l + l + 1 >= str->m) {+            str->m = str->l + l + 2;+            kroundup32(str->m);+            str->s = (char*)realloc(str->s, str->m);+        }+        memcpy(str->s + str->l, buf + fp->block_offset, l);+        str->l += l;+        fp->block_offset += l + 1;+        if (fp->block_offset >= fp->block_length) {+            fp->block_address = htell(fp->fp);+            fp->block_offset = 0;+            fp->block_length = 0;+        }+    } while (state == 0);+    if (str->l == 0 && state < 0) return state;+    fp->uncompressed_address += str->l;+    if ( delim=='\n' && str->l>0 && str->s[str->l-1]=='\r' ) str->l--;+    str->s[str->l] = 0;+    return str->l;+}++void bgzf_index_destroy(BGZF *fp)+{+    if ( !fp->idx ) return;+    free(fp->idx->offs);+    free(fp->idx);+    fp->idx = NULL;+    fp->idx_build_otf = 0;+}++int bgzf_index_build_init(BGZF *fp)+{+    bgzf_index_destroy(fp);+    fp->idx = (bgzidx_t*) calloc(1,sizeof(bgzidx_t));+    if ( !fp->idx ) return -1;+    fp->idx_build_otf = 1;  // build index on the fly+    return 0;+}++int bgzf_index_add_block(BGZF *fp)+{+    fp->idx->noffs++;+    if ( fp->idx->noffs > fp->idx->moffs )+    {+        fp->idx->moffs = fp->idx->noffs;+        kroundup32(fp->idx->moffs);+        fp->idx->offs = (bgzidx1_t*) realloc(fp->idx->offs, fp->idx->moffs*sizeof(bgzidx1_t));+        if ( !fp->idx->offs ) return -1;+    }+    fp->idx->offs[ fp->idx->noffs-1 ].uaddr = fp->idx->ublock_addr;+    fp->idx->offs[ fp->idx->noffs-1 ].caddr = fp->block_address;+    return 0;+}++static inline int fwrite_uint64(uint64_t x, FILE *f)+{+    if (ed_is_big()) x = ed_swap_8(x);+    if (fwrite(&x, sizeof x, 1, f) != 1) return -1;+    return 0;+}++int bgzf_index_dump(BGZF *fp, const char *bname, const char *suffix)+{+    if (bgzf_flush(fp) != 0) return -1;++    assert(fp->idx);+    char *tmp = NULL;+    if ( suffix )+    {+        int blen = strlen(bname);+        int slen = strlen(suffix);+        tmp = (char*) malloc(blen + slen + 1);+        if ( !tmp ) return -1;+        memcpy(tmp,bname,blen);+        memcpy(tmp+blen,suffix,slen+1);+    }++    FILE *idx = fopen(tmp?tmp:bname,"wb");+    if ( tmp ) free(tmp);+    if ( !idx ) {+        if (hts_verbose > 1)+        {+            fprintf(stderr, "[E::%s] Error opening %s%s : %s\n",+                    __func__, bname, suffix ? suffix : "", strerror(errno));+        }+        return -1;+    }++    // Note that the index contains one extra record when indexing files opened+    // for reading. The terminating record is not present when opened for writing.+    // This is not a bug.++    int i;+    if (fwrite_uint64(fp->idx->noffs - 1, idx) < 0) goto fail;+    for (i=1; i<fp->idx->noffs; i++)+    {+        if (fwrite_uint64(fp->idx->offs[i].caddr, idx) < 0) goto fail;+        if (fwrite_uint64(fp->idx->offs[i].uaddr, idx) < 0) goto fail;+    }++    if (fclose(idx) < 0)+    {+        if (hts_verbose > 1)+        {+            fprintf(stderr, "[E::%s] Error on closing %s%s : %s\n",+                    __func__, bname, suffix ? suffix : "", strerror(errno));+        }+        return -1;+    }+    return 0;++ fail:+    if (hts_verbose > 1)+    {+        fprintf(stderr, "[E::%s] Error writing to %s%s : %s\n",+                __func__, bname, suffix ? suffix : "", strerror(errno));+    }+    fclose(idx);+    return -1;+}++static inline int fread_uint64(uint64_t *xptr, FILE *f)+{+    if (fread(xptr, sizeof *xptr, 1, f) != 1) return -1;+    if (ed_is_big()) ed_swap_8p(xptr);+    return 0;+}++int bgzf_index_load(BGZF *fp, const char *bname, const char *suffix)+{+    char *tmp = NULL;+    if ( suffix )+    {+        int blen = strlen(bname);+        int slen = strlen(suffix);+        tmp = (char*) malloc(blen + slen + 1);+        if ( !tmp ) return -1;+        memcpy(tmp,bname,blen);+        memcpy(tmp+blen,suffix,slen+1);+    }++    FILE *idx = fopen(tmp?tmp:bname,"rb");+    if ( tmp ) free(tmp);+    if ( !idx ) {+        if (hts_verbose > 1) {+            fprintf(stderr, "[E::%s] Error opening %s%s : %s\n",+                    __func__, bname, suffix ? suffix : "", strerror(errno));+        }+        return -1;+    }++    fp->idx = (bgzidx_t*) calloc(1,sizeof(bgzidx_t));+    if (fp->idx == NULL) goto fail;+    uint64_t x;+    if (fread_uint64(&x, idx) < 0) goto fail;++    fp->idx->noffs = fp->idx->moffs = x + 1;+    fp->idx->offs  = (bgzidx1_t*) malloc(fp->idx->moffs*sizeof(bgzidx1_t));+    if (fp->idx->offs == NULL) goto fail;+    fp->idx->offs[0].caddr = fp->idx->offs[0].uaddr = 0;++    int i;+    for (i=1; i<fp->idx->noffs; i++)+    {+        if (fread_uint64(&fp->idx->offs[i].caddr, idx) < 0) goto fail;+        if (fread_uint64(&fp->idx->offs[i].uaddr, idx) < 0) goto fail;+    }++    if (fclose(idx) != 0) goto fail;+    return 0;++ fail:+    if (hts_verbose > 1)+    {+        fprintf(stderr, "[E::%s] Error reading %s%s : %s\n",+                __func__, bname, suffix ? suffix : "", strerror(errno));+    }+    fclose(idx);+    if (fp->idx) {+        free(fp->idx->offs);+        free(fp->idx);+        fp->idx = NULL;+    }+    return -1;+}++int bgzf_useek(BGZF *fp, long uoffset, int where)+{+    if ( !fp->is_compressed )+    {+        if (hseek(fp->fp, uoffset, SEEK_SET) < 0)+        {+            fp->errcode |= BGZF_ERR_IO;+            return -1;+        }+        fp->block_length = 0;  // indicates current block has not been loaded+        fp->block_address = uoffset;+        fp->block_offset = 0;+        if (bgzf_read_block(fp) < 0) {+            fp->errcode |= BGZF_ERR_IO;+            return -1;+        }+        fp->uncompressed_address = uoffset;+        return 0;+    }++    if ( !fp->idx )+    {+        fp->errcode |= BGZF_ERR_IO;+        return -1;+    }++    // binary search+    int ilo = 0, ihi = fp->idx->noffs - 1;+    while ( ilo<=ihi )+    {+        int i = (ilo+ihi)*0.5;+        if ( uoffset < fp->idx->offs[i].uaddr ) ihi = i - 1;+        else if ( uoffset >= fp->idx->offs[i].uaddr ) ilo = i + 1;+        else break;+    }+    int i = ilo-1;+    if (hseek(fp->fp, fp->idx->offs[i].caddr, SEEK_SET) < 0)+    {+        fp->errcode |= BGZF_ERR_IO;+        return -1;+    }+    fp->block_length = 0;  // indicates current block has not been loaded+    fp->block_address = fp->idx->offs[i].caddr;+    fp->block_offset = 0;+    if ( bgzf_read_block(fp) < 0 ) {+        fp->errcode |= BGZF_ERR_IO;+        return -1;+    }+    if ( uoffset - fp->idx->offs[i].uaddr > 0 )+    {+        fp->block_offset = uoffset - fp->idx->offs[i].uaddr;+        assert( fp->block_offset <= fp->block_length );     // todo: skipped, unindexed, blocks+    }+    fp->uncompressed_address = uoffset;+    return 0;+}++long bgzf_utell(BGZF *fp)+{+    return fp->uncompressed_address;    // currently maintained only when reading+}
+ htslib-1.3.2/bgzip.c view
@@ -0,0 +1,311 @@+/* bgzip.c -- Block compression/decompression utility.++   Copyright (C) 2008, 2009 Broad Institute / Massachusetts Institute of Technology+   Copyright (C) 2010, 2013-2016 Genome Research Ltd.++   Permission is hereby granted, free of charge, to any person obtaining a copy+   of this software and associated documentation files (the "Software"), to deal+   in the Software without restriction, including without limitation the rights+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+   copies of the Software, and to permit persons to whom the Software is+   furnished to do so, subject to the following conditions:++   The above copyright notices and this permission notice shall be included in+   all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+   THE SOFTWARE.+*/++#include <config.h>++#include <stdlib.h>+#include <string.h>+#include <stdio.h>+#include <fcntl.h>+#include <unistd.h>+#include <errno.h>+#include <stdarg.h>+#include <getopt.h>+#include <sys/stat.h>+#include "htslib/bgzf.h"+#include "htslib/hts.h"++static const int WINDOW_SIZE = 64 * 1024;++static void error(const char *format, ...)+{+    va_list ap;+    va_start(ap, format);+    vfprintf(stderr, format, ap);+    va_end(ap);+    exit(EXIT_FAILURE);+}++static int confirm_overwrite(const char *fn)+{+    int save_errno = errno;+    int ret = 0;++    if (isatty(STDIN_FILENO)) {+        char c;+        fprintf(stderr, "[bgzip] %s already exists; do you wish to overwrite (y or n)? ", fn);+        if (scanf("%c", &c) == 1 && (c == 'Y' || c == 'y')) ret = 1;+    }++    errno = save_errno;+    return ret;+}++static int bgzip_main_usage(void)+{+    fprintf(stderr, "\n");+    fprintf(stderr, "Version: %s\n", hts_version());+    fprintf(stderr, "Usage:   bgzip [OPTIONS] [FILE] ...\n");+    fprintf(stderr, "Options:\n");+    fprintf(stderr, "   -b, --offset INT        decompress at virtual file pointer (0-based uncompressed offset)\n");+    fprintf(stderr, "   -c, --stdout            write on standard output, keep original files unchanged\n");+    fprintf(stderr, "   -d, --decompress        decompress\n");+    fprintf(stderr, "   -f, --force             overwrite files without asking\n");+    fprintf(stderr, "   -h, --help              give this help\n");+    fprintf(stderr, "   -i, --index             compress and create BGZF index\n");+    fprintf(stderr, "   -I, --index-name FILE   name of BGZF index file [file.gz.gzi]\n");+    fprintf(stderr, "   -r, --reindex           (re)index compressed file\n");+    fprintf(stderr, "   -s, --size INT          decompress INT bytes (uncompressed size)\n");+    fprintf(stderr, "   -@, --threads INT       number of compression threads to use [1]\n");+    fprintf(stderr, "\n");+    return 1;+}++int main(int argc, char **argv)+{+    int c, compress, pstdout, is_forced, index = 0, reindex = 0;+    BGZF *fp;+    void *buffer;+    long start, end, size;+    char *index_fname = NULL;+    int threads = 1;++    static const struct option loptions[] =+    {+        {"help", no_argument, NULL, 'h'},+        {"offset", required_argument, NULL, 'b'},+        {"stdout", no_argument, NULL, 'c'},+        {"decompress", no_argument, NULL, 'd'},+        {"force", no_argument, NULL, 'f'},+        {"index", no_argument, NULL, 'i'},+        {"index-name", required_argument, NULL, 'I'},+        {"reindex", no_argument, NULL, 'r'},+        {"size", required_argument, NULL, 's'},+        {"threads", required_argument, NULL, '@'},+        {"version", no_argument, NULL, 1},+        {NULL, 0, NULL, 0}+    };++    compress = 1; pstdout = 0; start = 0; size = -1; end = -1; is_forced = 0;+    while((c  = getopt_long(argc, argv, "cdh?fb:@:s:iI:r",loptions,NULL)) >= 0){+        switch(c){+        case 'd': compress = 0; break;+        case 'c': pstdout = 1; break;+        case 'b': start = atol(optarg); compress = 0; pstdout = 1; break;+        case 's': size = atol(optarg); pstdout = 1; break;+        case 'f': is_forced = 1; break;+        case 'i': index = 1; break;+        case 'I': index_fname = optarg; break;+        case 'r': reindex = 1; compress = 0; break;+        case '@': threads = atoi(optarg); break;+        case 1:+            printf(+"bgzip (htslib) %s\n"+"Copyright (C) 2016 Genome Research Ltd.\n", hts_version());+            return EXIT_SUCCESS;+        case 'h':+        case '?': return bgzip_main_usage();+        }+    }+    if (size >= 0) end = start + size;+    if (end >= 0 && end < start) {+        fprintf(stderr, "[bgzip] Illegal region: [%ld, %ld]\n", start, end);+        return 1;+    }+    if (compress == 1) {+        struct stat sbuf;+        int f_src = fileno(stdin);++        if ( argc>optind )+        {+            if ( stat(argv[optind],&sbuf)<0 )+            {+                fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);+                return 1;+            }++            if ((f_src = open(argv[optind], O_RDONLY)) < 0) {+                fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);+                return 1;+            }++            if (pstdout)+                fp = bgzf_open("-", "w");+            else+            {+                char *name = malloc(strlen(argv[optind]) + 5);+                strcpy(name, argv[optind]);+                strcat(name, ".gz");+                fp = bgzf_open(name, is_forced? "w" : "wx");+                if (fp == NULL && errno == EEXIST && confirm_overwrite(name))+                    fp = bgzf_open(name, "w");+                if (fp == NULL) {+                    fprintf(stderr, "[bgzip] can't create %s: %s\n", name, strerror(errno));+                    free(name);+                    return 1;+                }+                free(name);+            }+        }+        else if (!pstdout && isatty(fileno((FILE *)stdout)) )+            return bgzip_main_usage();+        else if ( index && !index_fname )+        {+            fprintf(stderr, "[bgzip] Index file name expected when writing to stdout\n");+            return 1;+        }+        else+            fp = bgzf_open("-", "w");++        if (threads > 1)+            bgzf_mt(fp, threads, 256);++        if ( index ) bgzf_index_build_init(fp);+        buffer = malloc(WINDOW_SIZE);+        while ((c = read(f_src, buffer, WINDOW_SIZE)) > 0)+            if (bgzf_write(fp, buffer, c) < 0) error("Could not write %d bytes: Error %d\n", c, fp->errcode);+        if ( index )+        {+            if (index_fname) {+                if (bgzf_index_dump(fp, index_fname, NULL) < 0)+                    error("Could not write index to '%s'\n", index_fname);+            } else {+                if (bgzf_index_dump(fp, argv[optind], ".gz.gzi") < 0)+                    error("Could not write index to '%s.gz.gzi'", argv[optind]);+            }+        }+        if (bgzf_close(fp) < 0) error("Close failed: Error %d", fp->errcode);+        if (argc > optind && !pstdout) unlink(argv[optind]);+        free(buffer);+        close(f_src);+        return 0;+    }+    else if ( reindex )+    {+        if ( argc>optind )+        {+            fp = bgzf_open(argv[optind], "r");+            if ( !fp ) error("[bgzip] Could not open file: %s\n", argv[optind]);+        }+        else+        {+            if ( !index_fname ) error("[bgzip] Index file name expected when reading from stdin\n");+            fp = bgzf_open("-", "r");+            if ( !fp ) error("[bgzip] Could not read from stdin: %s\n", strerror(errno));+        }++        buffer = malloc(BGZF_BLOCK_SIZE);+        bgzf_index_build_init(fp);+        int ret;+        while ( (ret=bgzf_read(fp, buffer, BGZF_BLOCK_SIZE))>0 ) ;+        free(buffer);+        if ( ret<0 ) error("Is the file gzipped or bgzipped? The latter is required for indexing.\n");++        if ( index_fname ) {+            if (bgzf_index_dump(fp, index_fname, NULL) < 0)+                error("Could not write index to '%s'\n", index_fname);+        } else {+            if (bgzf_index_dump(fp, argv[optind], ".gzi") < 0)+                error("Could not write index to '%s.gzi'\n", argv[optind]);+        }++        if ( bgzf_close(fp)<0 ) error("Close failed: Error %d\n",fp->errcode);+        return 0;+    }+    else+    {+        struct stat sbuf;+        int f_dst;++        if ( argc>optind )+        {+            if ( stat(argv[optind],&sbuf)<0 )+            {+                fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);+                return 1;+            }+            char *name;+            int len = strlen(argv[optind]);+            if ( strcmp(argv[optind]+len-3,".gz") )+            {+                fprintf(stderr, "[bgzip] %s: unknown suffix -- ignored\n", argv[optind]);+                return 1;+            }+            fp = bgzf_open(argv[optind], "r");+            if (fp == NULL) {+                fprintf(stderr, "[bgzip] Could not open file: %s\n", argv[optind]);+                return 1;+            }++            if (pstdout) {+                f_dst = fileno(stdout);+            }+            else {+                const int wrflags = O_WRONLY | O_CREAT | O_TRUNC;+                name = strdup(argv[optind]);+                name[strlen(name) - 3] = '\0';+                f_dst = open(name, is_forced? wrflags : wrflags|O_EXCL, 0666);+                if (f_dst < 0 && errno == EEXIST && confirm_overwrite(name))+                    f_dst = open(name, wrflags, 0666);+                if (f_dst < 0) {+                    fprintf(stderr, "[bgzip] can't create %s: %s\n", name, strerror(errno));+                    free(name);+                    return 1;+                }+                free(name);+            }+        }+        else if (!pstdout && isatty(fileno((FILE *)stdin)) )+            return bgzip_main_usage();+        else+        {+            f_dst = fileno(stdout);+            fp = bgzf_open("-", "r");+            if (fp == NULL) {+                fprintf(stderr, "[bgzip] Could not read from stdin: %s\n", strerror(errno));+                return 1;+            }+        }+        buffer = malloc(WINDOW_SIZE);+        if ( start>0 )+        {+            if ( bgzf_index_load(fp, argv[optind], ".gzi") < 0 ) error("Could not load index: %s.gzi\n", argv[optind]);+            if ( bgzf_useek(fp, start, SEEK_SET) < 0 ) error("Could not seek to %d-th (uncompressd) byte\n", start);+        }+        while (1) {+            if (end < 0) c = bgzf_read(fp, buffer, WINDOW_SIZE);+            else c = bgzf_read(fp, buffer, (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start));+            if (c == 0) break;+            if (c < 0) error("Could not read %d bytes: Error %d\n", (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start), fp->errcode);+            start += c;+            if ( write(f_dst, buffer, c) != c ) error("Could not write %d bytes\n", c);+            if (end >= 0 && start >= end) break;+        }+        free(buffer);+        if (bgzf_close(fp) < 0) error("Close failed: Error %d\n",fp->errcode);+        if (!pstdout) unlink(argv[optind]);+        return 0;+    }+    return 0;+}
+ htslib-1.3.2/config.h view
+ htslib-1.3.2/cram/cram.h view
@@ -0,0 +1,61 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*! \file+ * CRAM interface.+ *+ * Consider using the higher level hts_*() API for programs that wish to+ * be file format agnostic (see htslib/hts.h).+ *+ * This API should be used for CRAM specific code. The specifics of the+ * public API are implemented in cram_io.h, cram_encode.h and cram_decode.h+ * although these should not be included directly (use this file instead).+ */++#ifndef _CRAM_H_+#define _CRAM_H_++#include "cram/cram_samtools.h"+#include "cram/sam_header.h"+#include "cram_structs.h"+#include "cram_io.h"+#include "cram_encode.h"+#include "cram_decode.h"+#include "cram_stats.h"+#include "cram_codecs.h"+#include "cram_index.h"++// Validate against the external cram.h,+//+// This contains duplicated portions from cram_io.h and cram_structs.h,+// so we want to ensure that the prototypes match.+#include "htslib/cram.h"++#endif
+ htslib-1.3.2/cram/cram_codecs.c view
@@ -0,0 +1,1950 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+ * FIXME: add checking of cram_external_type to return NULL on unsupported+ * {codec,type} tuples.+ */++#include <config.h>++#include <stdlib.h>+#include <string.h>+#include <assert.h>+#include <limits.h>++#include "cram/cram.h"++/*+ * ---------------------------------------------------------------------------+ * Block bit-level I/O functions.+ * All defined static here to promote easy inlining by the compiler.+ */++#if 0+/* Get a single bit, MSB first */+static signed int get_bit_MSB(cram_block *block) {+    unsigned int val;++    if (block->byte > block->alloc)+	return -1;++    val = block->data[block->byte] >> block->bit;+    if (--block->bit == -1) {+	block->bit = 7;+	block->byte++;+	//printf("(%02X)", block->data[block->byte]);+    }++    //printf("-B%d-", val&1);++    return val & 1;+}+#endif++/*+ * Count number of successive 0 and 1 bits+ */+static int get_one_bits_MSB(cram_block *block) {+    int n = 0, b;+    if (block->byte >= block->uncomp_size)+        return -1;+    do {+	b = block->data[block->byte] >> block->bit;+	if (--block->bit == -1) {+	    block->bit = 7;+	    block->byte++;+	    if (block->byte == block->uncomp_size && (b&1))+	        return -1;+	}+	n++;+    } while (b&1);++    return n-1;+}++static int get_zero_bits_MSB(cram_block *block) {+    int n = 0, b;+    if (block->byte >= block->uncomp_size)+        return -1;+    do {+	b = block->data[block->byte] >> block->bit;+	if (--block->bit == -1) {+	    block->bit = 7;+	    block->byte++;+	    if (block->byte == block->uncomp_size && !(b&1))+	        return -1;+	}+	n++;+    } while (!(b&1));++    return n-1;+}++#if 0+/* Stores a single bit */+static void store_bit_MSB(cram_block *block, unsigned int bit) {+    if (block->byte >= block->alloc) {+	block->alloc = block->alloc ? block->alloc*2 : 1024;+	block->data = realloc(block->data, block->alloc);+    }++    if (bit)+	block->data[block->byte] |= (1 << block->bit);++    if (--block->bit == -1) {+	block->bit = 7;+	block->byte++;+	block->data[block->byte] = 0;+    }+}+#endif++#if 0+/* Rounds to the next whole byte boundary first */+static void store_bytes_MSB(cram_block *block, char *bytes, int len) {+    if (block->bit != 7) {+	block->bit = 7;+	block->byte++;+    }++    while (block->byte + len >= block->alloc) {+	block->alloc = block->alloc ? block->alloc*2 : 1024;+	block->data = realloc(block->data, block->alloc);+    }++    memcpy(&block->data[block->byte], bytes, len);+    block->byte += len;+}+#endif++/* Local optimised copy for inlining */+static inline unsigned int get_bits_MSB(cram_block *block, int nbits) {+    unsigned int val = 0;+    int i;++#if 0+    // Fits within the current byte */+    if (nbits <= block->bit+1) {+	val = (block->data[block->byte]>>(block->bit-(nbits-1))) & ((1<<nbits)-1);+	if ((block->bit -= nbits) == -1) {+	    block->bit = 7;+	    block->byte++;+	}+	return val;+    }++    // partial first byte+    val = block->data[block->byte] & ((1<<(block->bit+1))-1);+    nbits -= block->bit+1;+    block->bit = 7;+    block->byte++;++    // whole middle bytes+    while (nbits >= 8) {+	val = (val << 8) | block->data[block->byte++];+	nbits -= 8;+    }++    val <<= nbits;+    val |= (block->data[block->byte]>>(block->bit-(nbits-1))) & ((1<<nbits)-1);+    block->bit -= nbits;+    return val;+#endif++#if 0+    /* Inefficient implementation! */+    //printf("{");+    for (i = 0; i < nbits; i++)+	//val = (val << 1) | get_bit_MSB(block);+	GET_BIT_MSB(block, val);+#endif++#if 1+    /* Combination of 1st two methods */+    if (nbits <= block->bit+1) {+	val = (block->data[block->byte]>>(block->bit-(nbits-1))) & ((1<<nbits)-1);+	if ((block->bit -= nbits) == -1) {+	    block->bit = 7;+	    block->byte++;+	}+	return val;+    }++    switch(nbits) {+//    case 15: GET_BIT_MSB(block, val);+//    case 14: GET_BIT_MSB(block, val);+//    case 13: GET_BIT_MSB(block, val);+//    case 12: GET_BIT_MSB(block, val);+//    case 11: GET_BIT_MSB(block, val);+//    case 10: GET_BIT_MSB(block, val);+//    case  9: GET_BIT_MSB(block, val);+    case  8: GET_BIT_MSB(block, val);+    case  7: GET_BIT_MSB(block, val);+    case  6: GET_BIT_MSB(block, val);+    case  5: GET_BIT_MSB(block, val);+    case  4: GET_BIT_MSB(block, val);+    case  3: GET_BIT_MSB(block, val);+    case  2: GET_BIT_MSB(block, val);+    case  1: GET_BIT_MSB(block, val);+	break;++    default:+	for (i = 0; i < nbits; i++)+	    //val = (val << 1) | get_bit_MSB(block);+	    GET_BIT_MSB(block, val);+    }+#endif++    //printf("=0x%x}", val);++    return val;+}++/*+ * Can store up to 24-bits worth of data encoded in an integer value+ * Possibly we'd want to have a less optimal store_bits function when dealing+ * with nbits > 24, but for now we assume the codes generated are never+ * that big. (Given this is only possible with 121392 or more+ * characters with exactly the correct frequency distribution we check+ * for it elsewhere.)+ */+static int store_bits_MSB(cram_block *block, unsigned int val, int nbits) {+    /* fprintf(stderr, " store_bits: %02x %d\n", val, nbits); */++    /*+     * Use slow mode until we tweak the huffman generator to never generate+     * codes longer than 24-bits.+     */+    unsigned int mask;++    if (block->byte+4 >= block->alloc) {+	if (block->byte) {+	    block->alloc *= 2;+	    block->data = realloc(block->data, block->alloc + 4);+	    if (!block->data)+		return -1;+	} else {+	    block->alloc = 1024;+	    block->data = realloc(block->data, block->alloc + 4);+	    if (!block->data)+		return -1;+	    block->data[0] = 0; // initialise first byte of buffer+	}+    }++    /* fits in current bit-field */+    if (nbits <= block->bit+1) {+	block->data[block->byte] |= (val << (block->bit+1-nbits));+	if ((block->bit-=nbits) == -1) {+	    block->bit = 7;+	    block->byte++;+	    block->data[block->byte] = 0;+	}+	return 0;+    }++    block->data[block->byte] |= (val >> (nbits -= block->bit+1));+    block->bit = 7;+    block->byte++;+    block->data[block->byte] = 0;+				 +    mask = 1<<(nbits-1);+    do {+	if (val & mask)+	    block->data[block->byte] |= (1 << block->bit);+	if (--block->bit == -1) {+	    block->bit = 7;+	    block->byte++;+	    block->data[block->byte] = 0;+	}+	mask >>= 1;+    } while(--nbits);++    return 0;+}++/*+ * Returns the next 'size' bytes from a block, or NULL if insufficient+ * data left.This is just a pointer into the block data and not an+ * allocated object, so do not free the result.+ */+static char *cram_extract_block(cram_block *b, int size) {+    char *cp = (char *)b->data + b->idx;+    b->idx += size;+    if (b->idx > b->uncomp_size)+	return NULL;++    return cp;+}++/*+ * ---------------------------------------------------------------------------+ * EXTERNAL+ */+int cram_external_decode_int(cram_slice *slice, cram_codec *c,+			     cram_block *in, char *out, int *out_size) {+    int l;+    char *cp;+    cram_block *b;++    /* Find the external block */+    b = cram_get_block_by_id(slice, c->external.content_id);+    if (!b)+        return *out_size?-1:0;++    cp = (char *)b->data + b->idx;+    // E_INT and E_LONG are guaranteed single item queries+    l = safe_itf8_get(cp, (char *)b->data + b->uncomp_size, (int32_t *)out);+    b->idx += l;+    *out_size = 1;++    return l > 0 ? 0 : -1;+}++int cram_external_decode_char(cram_slice *slice, cram_codec *c,+			      cram_block *in, char *out,+			      int *out_size) {+    char *cp;+    cram_block *b;++    /* Find the external block */+    b = cram_get_block_by_id(slice, c->external.content_id);+    if (!b)+        return *out_size?-1:0;++    cp = cram_extract_block(b, *out_size);+    if (!cp)+	return -1;++    if (out)+	memcpy(out, cp, *out_size);+    return 0;+}++static int cram_external_decode_block(cram_slice *slice, cram_codec *c,+				      cram_block *in, char *out_,+				      int *out_size) {+    char *cp;+    cram_block *b = NULL;+    cram_block *out = (cram_block *)out_;++    /* Find the external block */+    b = cram_get_block_by_id(slice, c->external.content_id);+    if (!b)+        return *out_size?-1:0;++    cp = cram_extract_block(b, *out_size);+    if (!cp)+	return -1;++    BLOCK_APPEND(out, cp, *out_size);+    return 0;+}++void cram_external_decode_free(cram_codec *c) {+    if (c)+	free(c);+}++cram_codec *cram_external_decode_init(char *data, int size,+				      enum cram_external_type option,+				      int version) {+    cram_codec *c;+    char *cp = data;++    if (!(c = malloc(sizeof(*c))))+	return NULL;++    c->codec  = E_EXTERNAL;+    if (option == E_INT || option == E_LONG)+	c->decode = cram_external_decode_int;+    else if (option == E_BYTE_ARRAY || option == E_BYTE)+	c->decode = cram_external_decode_char;+    else+	c->decode = cram_external_decode_block;+    c->free   = cram_external_decode_free;+    +    cp += itf8_get(cp, &c->external.content_id);++    if (cp - data != size) {+	fprintf(stderr, "Malformed external header stream\n");+	free(c);+	return NULL;+    }++    c->external.type = option;++    return c;+}++int cram_external_encode_int(cram_slice *slice, cram_codec *c,+			     char *in, int in_size) {+    uint32_t *i32 = (uint32_t *)in;++    itf8_put_blk(c->out, *i32);+    return 0;+}++int cram_external_encode_char(cram_slice *slice, cram_codec *c,+			      char *in, int in_size) {+    BLOCK_APPEND(c->out, in, in_size);+    return 0;+}++void cram_external_encode_free(cram_codec *c) {+    if (!c)+	return;+    free(c);+}++int cram_external_encode_store(cram_codec *c, cram_block *b, char *prefix,+			       int version) {+    char tmp[99], *tp = tmp;+    int len = 0;++    if (prefix) {+	size_t l = strlen(prefix);+	BLOCK_APPEND(b, prefix, l);+	len += l;+    }++    tp += itf8_put(tp, c->e_external.content_id);+    len += itf8_put_blk(b, c->codec);+    len += itf8_put_blk(b, tp-tmp);+    BLOCK_APPEND(b, tmp, tp-tmp);+    len += tp-tmp;++    return len;+}++cram_codec *cram_external_encode_init(cram_stats *st,+				      enum cram_external_type option,+				      void *dat,+				      int version) {+    cram_codec *c;++    c = malloc(sizeof(*c));+    if (!c)+	return NULL;+    c->codec = E_EXTERNAL;+    c->free = cram_external_encode_free;+    if (option == E_INT || option == E_LONG)+	c->encode = cram_external_encode_int;+    else if (option == E_BYTE_ARRAY || option == E_BYTE)+	c->encode = cram_external_encode_char;+    else+	abort();+    c->store = cram_external_encode_store;++    c->e_external.content_id = (size_t)dat;++    return c;+}++/*+ * ---------------------------------------------------------------------------+ * BETA+ */+int cram_beta_decode_int(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {+    int32_t *out_i = (int32_t *)out;+    int i, n;++    if (c->beta.nbits) {+        if (cram_not_enough_bits(in, c->beta.nbits))+            return -1;++	for (i = 0, n = *out_size; i < n; i++)+	    out_i[i] = get_bits_MSB(in, c->beta.nbits) - c->beta.offset;+    } else {+	for (i = 0, n = *out_size; i < n; i++)+	    out_i[i] = -c->beta.offset;+    }++    return 0;+}++int cram_beta_decode_char(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {+    int i, n;+++    if (c->beta.nbits) {+        if (cram_not_enough_bits(in, c->beta.nbits))+            return -1;++	if (out)+	    for (i = 0, n = *out_size; i < n; i++)+		out[i] = get_bits_MSB(in, c->beta.nbits) - c->beta.offset;+	else+	    for (i = 0, n = *out_size; i < n; i++)+		get_bits_MSB(in, c->beta.nbits);+    } else {+	if (out)+	    for (i = 0, n = *out_size; i < n; i++)+		out[i] = -c->beta.offset;+    }++    return 0;+}++void cram_beta_decode_free(cram_codec *c) {+    if (c)+	free(c);+}++cram_codec *cram_beta_decode_init(char *data, int size,+				  enum cram_external_type option,+				  int version) {+    cram_codec *c;+    char *cp = data;++    if (!(c = malloc(sizeof(*c))))+	return NULL;++    c->codec  = E_BETA;+    if (option == E_INT || option == E_LONG)+	c->decode = cram_beta_decode_int;+    else if (option == E_BYTE_ARRAY || option == E_BYTE)+	c->decode = cram_beta_decode_char;+    else+	abort();+    c->free   = cram_beta_decode_free;+    +    cp += itf8_get(cp, &c->beta.offset);+    cp += itf8_get(cp, &c->beta.nbits);++    if (cp - data != size+        || c->beta.nbits < 0 || c->beta.nbits > 8 * sizeof(int)) {+	fprintf(stderr, "Malformed beta header stream\n");+	free(c);+	return NULL;+    }++    return c;+}++int cram_beta_encode_store(cram_codec *c, cram_block *b,+			   char *prefix, int version) {+    int len = 0;++    if (prefix) {+	size_t l = strlen(prefix);+	BLOCK_APPEND(b, prefix, l);+	len += l;+    }++    len += itf8_put_blk(b, c->codec);+    len += itf8_put_blk(b, itf8_size(c->e_beta.offset)+			+ itf8_size(c->e_beta.nbits)); // codec length+    len += itf8_put_blk(b, c->e_beta.offset);+    len += itf8_put_blk(b, c->e_beta.nbits);++    return len;+}++int cram_beta_encode_int(cram_slice *slice, cram_codec *c,+			 char *in, int in_size) {+    int *syms = (int *)in;+    int i, r = 0;++    for (i = 0; i < in_size; i++)+	r |= store_bits_MSB(c->out, syms[i] + c->e_beta.offset,+			    c->e_beta.nbits);++    return r;+}++int cram_beta_encode_char(cram_slice *slice, cram_codec *c,+			  char *in, int in_size) {+    unsigned char *syms = (unsigned char *)in;+    int i, r = 0;++    for (i = 0; i < in_size; i++)+	r |= store_bits_MSB(c->out, syms[i] + c->e_beta.offset,+			    c->e_beta.nbits);++    return r;+}++void cram_beta_encode_free(cram_codec *c) {+    if (c) free(c);+}++cram_codec *cram_beta_encode_init(cram_stats *st,+				  enum cram_external_type option,+				  void *dat,+				  int version) {+    cram_codec *c;+    int min_val, max_val, len = 0;++    c = malloc(sizeof(*c));+    if (!c)+	return NULL;+    c->codec  = E_BETA;+    c->free   = cram_beta_encode_free;+    if (option == E_INT)+	c->encode = cram_beta_encode_int;+    else+	c->encode = cram_beta_encode_char;+    c->store  = cram_beta_encode_store;++    if (dat) {+	min_val = ((int *)dat)[0];+	max_val = ((int *)dat)[1];+    } else {+	min_val = INT_MAX;+	max_val = INT_MIN;+	int i;+	for (i = 0; i < MAX_STAT_VAL; i++) {+	    if (!st->freqs[i])+		continue;+	    if (min_val > i)+		min_val = i;+	    max_val = i;+	}+	if (st->h) {+	    khint_t k;++	    for (k = kh_begin(st->h); k != kh_end(st->h); k++) {+		if (!kh_exist(st->h, k))+		    continue;+	    +		i = kh_key(st->h, k);+		if (min_val > i)+		    min_val = i;+		if (max_val < i)+		    max_val = i;+	    }+	}+    }++    assert(max_val >= min_val);+    c->e_beta.offset = -min_val;+    max_val -= min_val;+    while (max_val) {+	len++;+	max_val >>= 1;+    }+    c->e_beta.nbits = len;++    return c;+}++/*+ * ---------------------------------------------------------------------------+ * SUBEXP+ */+int cram_subexp_decode(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {+    int32_t *out_i = (int32_t *)out;+    int n, count;+    int k = c->subexp.k;++    for (count = 0, n = *out_size; count < n; count++) {+	int i = 0, tail;+	int val;++	/* Get number of 1s */+	//while (get_bit_MSB(in) == 1) i++;+	i = get_one_bits_MSB(in);+        if (i < 0 || cram_not_enough_bits(in, i > 0 ? i + k - 1 : k))+            return -1;+	/*+	 * Val is+	 * i > 0:  2^(k+i-1) + k+i-1 bits+	 * i = 0:  k bits+	 */+	if (i) {+	    tail = i + k-1;+	    val = 0;+	    while (tail) {+		//val = val<<1; val |= get_bit_MSB(in);+		GET_BIT_MSB(in, val);+		tail--;+	    }+	    val += 1 << (i + k-1);+	} else {+	    tail = k;+	    val = 0;+	    while (tail) {+		//val = val<<1; val |= get_bit_MSB(in);+		GET_BIT_MSB(in, val);+		tail--;+	    }+	}++	out_i[count] = val - c->subexp.offset;+    }++    return 0;+}++void cram_subexp_decode_free(cram_codec *c) {+    if (c)+	free(c);+}++cram_codec *cram_subexp_decode_init(char *data, int size,+				    enum cram_external_type option,+				    int version) {+    cram_codec *c;+    char *cp = data;++    if (!(c = malloc(sizeof(*c))))+	return NULL;++    c->codec  = E_SUBEXP;+    c->decode = cram_subexp_decode;+    c->free   = cram_subexp_decode_free;+    c->subexp.k = -1;++    cp += safe_itf8_get(cp, data + size, &c->subexp.offset);+    cp += safe_itf8_get(cp, data + size, &c->subexp.k);++    if (cp - data != size || c->subexp.k < 0) {+	fprintf(stderr, "Malformed subexp header stream\n");+	free(c);+	return NULL;+    }++    return c;+}++/*+ * ---------------------------------------------------------------------------+ * GAMMA+ */+int cram_gamma_decode(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {+    int32_t *out_i = (int32_t *)out;+    int i, n;++    for (i = 0, n = *out_size; i < n; i++) {+	int nz = 0;+	int val;+	//while (get_bit_MSB(in) == 0) nz++;+	nz = get_zero_bits_MSB(in);+        if (cram_not_enough_bits(in, nz))+            return -1;+	val = 1;+	while (nz > 0) {+	    //val <<= 1; val |= get_bit_MSB(in);+	    GET_BIT_MSB(in, val);+	    nz--;+	}++	out_i[i] = val - c->gamma.offset;+    }++    return 0;+}++void cram_gamma_decode_free(cram_codec *c) {+    if (c)+	free(c);+}++cram_codec *cram_gamma_decode_init(char *data, int size,+				   enum cram_external_type option,+				   int version) {+    cram_codec *c;+    char *cp = data;++    if (!(c = malloc(sizeof(*c))))+	return NULL;++    c->codec  = E_GAMMA;+    c->decode = cram_gamma_decode;+    c->free   = cram_gamma_decode_free;+    +    cp += itf8_get(cp, &c->gamma.offset);++    if (cp - data != size) {+	fprintf(stderr, "Malformed gamma header stream\n");+	free(c);+	return NULL;+    }++    return c;+}++/*+ * ---------------------------------------------------------------------------+ * HUFFMAN+ */++static int code_sort(const void *vp1, const void *vp2) {+    const cram_huffman_code *c1 = (const cram_huffman_code *)vp1;+    const cram_huffman_code *c2 = (const cram_huffman_code *)vp2;++    if (c1->len != c2->len)+	return c1->len - c2->len;+    else+	return c1->symbol - c2->symbol;+}++void cram_huffman_decode_free(cram_codec *c) {+    if (!c)+	return;++    if (c->huffman.codes)+	free(c->huffman.codes);+    free(c);+}++int cram_huffman_decode_null(cram_slice *slice, cram_codec *c,+			     cram_block *in, char *out, int *out_size) {+    return -1;+}++int cram_huffman_decode_char0(cram_slice *slice, cram_codec *c,+			      cram_block *in, char *out, int *out_size) {+    int i, n;++    if (!out)+	return 0;++    /* Special case of 0 length codes */+    for (i = 0, n = *out_size; i < n; i++) {+	out[i] = c->huffman.codes[0].symbol;+    }+    return 0;+}++int cram_huffman_decode_char(cram_slice *slice, cram_codec *c,+			     cram_block *in, char *out, int *out_size) {+    int i, n, ncodes = c->huffman.ncodes;+    const cram_huffman_code * const codes = c->huffman.codes;++    for (i = 0, n = *out_size; i < n; i++) {+	int idx = 0;+	int val = 0, len = 0, last_len = 0;++	for (;;) {+	    int dlen = codes[idx].len - last_len;+	    if (cram_not_enough_bits(in, dlen))+		return -1;++	    //val <<= dlen;+	    //val  |= get_bits_MSB(in, dlen);+	    //last_len = (len += dlen);++	    last_len = (len += dlen);+	    for (; dlen; dlen--) GET_BIT_MSB(in, val);++	    idx = val - codes[idx].p;+	    if (idx >= ncodes || idx < 0)+		return -1;++	    if (codes[idx].code == val && codes[idx].len == len) {+		out[i] = codes[idx].symbol;+		break;+	    }+	}+    }++    return 0;+}++int cram_huffman_decode_int0(cram_slice *slice, cram_codec *c,+			     cram_block *in, char *out, int *out_size) {+    int32_t *out_i = (int32_t *)out;+    int i, n;+    const cram_huffman_code * const codes = c->huffman.codes;++    /* Special case of 0 length codes */+    for (i = 0, n = *out_size; i < n; i++) {+	out_i[i] = codes[0].symbol;+    }+    return 0;+}++int cram_huffman_decode_int(cram_slice *slice, cram_codec *c,+			    cram_block *in, char *out, int *out_size) {+    int32_t *out_i = (int32_t *)out;+    int i, n, ncodes = c->huffman.ncodes;+    const cram_huffman_code * const codes = c->huffman.codes;++    for (i = 0, n = *out_size; i < n; i++) {+	int idx = 0;+	int val = 0, len = 0, last_len = 0;++	// Now one bit at a time for remaining checks+	for (;;) {+	    int dlen = codes[idx].len - last_len;+	    if (cram_not_enough_bits(in, dlen))+		return -1;+	    +	    //val <<= dlen;+	    //val  |= get_bits_MSB(in, dlen);+	    //last_len = (len += dlen);++	    last_len = (len += dlen);+	    for (; dlen; dlen--) GET_BIT_MSB(in, val);++	    idx = val - codes[idx].p;+	    if (idx >= ncodes || idx < 0)+		return -1;++	    if (codes[idx].code == val && codes[idx].len == len) {+		out_i[i] = codes[idx].symbol;+		break;+	    }+	}+    }++    return 0;+}++/*+ * Initialises a huffman decoder from an encoding data stream.+ */+cram_codec *cram_huffman_decode_init(char *data, int size,+				     enum cram_external_type option,+				     int version) {+    int32_t ncodes = 0, i, j;+    char *cp = data, *data_end = &data[size];+    cram_codec *h;+    cram_huffman_code *codes;+    int32_t val, last_len, max_len = 0;+    int l;++    cp += safe_itf8_get(cp, data_end, &ncodes);+    h = calloc(1, sizeof(*h));+    if (!h)+	return NULL;++    h->codec  = E_HUFFMAN;+    h->free   = cram_huffman_decode_free;++    h->huffman.ncodes = ncodes;+    codes = h->huffman.codes = malloc(ncodes * sizeof(*codes));+    if (!codes) {+	free(h);+	return NULL;+    }++    /* Read symbols and bit-lengths */+    for (i = 0, l = 1; i < ncodes && l > 0; i++, cp += l) {+	l = safe_itf8_get(cp, data_end, &codes[i].symbol);+    }++    if (l < 1) {+	fprintf(stderr, "Malformed huffman header stream\n");+	free(h);+	return NULL;+    }+    cp += safe_itf8_get(cp, data_end, &i);+    if (i != ncodes) {+	fprintf(stderr, "Malformed huffman header stream\n");+	free(h);+	return NULL;+    }++    if (ncodes == 0) {+	/* NULL huffman stream.  Ensure it returns an error if+           anything tries to use it. */+        h->decode = cram_huffman_decode_null;+        return h;+    }++    for (i = 0, l = 1; i < ncodes; i++, cp += l) {+        l = safe_itf8_get(cp, data_end, &codes[i].len);+        if (l < 1)+            break;+	if (max_len < codes[i].len)+	    max_len = codes[i].len;+    }+    if (l < 1 || cp - data != size || max_len >= ncodes) {+	fprintf(stderr, "Malformed huffman header stream\n");+	free(h);+	return NULL;+    }++    /* Sort by bit length and then by symbol value */+    qsort(codes, ncodes, sizeof(*codes), code_sort);++    /* Assign canonical codes */+    val = -1, last_len = 0;+    for (i = 0; i < ncodes; i++) {+	val++;+	if (codes[i].len > last_len) {+	    while (codes[i].len > last_len) {+		val <<= 1;+		last_len++;+	    }+	}+	codes[i].code = val;+    }++    /*+     * Compute the next starting point, offset by the i'th value.+     * For example if codes 10, 11, 12, 13 are 30, 31, 32, 33 then+     * codes[10..13].p = 30 - 10.+     */+    last_len = 0;+    for (i = j = 0; i < ncodes; i++) {+	if (codes[i].len > last_len) {+	    j = codes[i].code - i;+	    last_len = codes[i].len;+	}+	codes[i].p = j;+    }++//    puts("==HUFF LEN==");+//    for (i = 0; i <= last_len+1; i++) {+//	printf("len %d=%d prefix %d\n", i, h->huffman.lengths[i], h->huffman.prefix[i]); +//    }+//    puts("===HUFFMAN CODES===");+//    for (i = 0; i < ncodes; i++) {+//	int j;+//	printf("%d: %d %d %d ", i, codes[i].symbol, codes[i].len, codes[i].code);+//	j = codes[i].len;+//	while (j) {+//	    putchar(codes[i].code & (1 << --j) ? '1' : '0');+//	}+//	printf(" %d\n", codes[i].code);+//    }++    if (option == E_BYTE || option == E_BYTE_ARRAY) {+	if (h->huffman.codes[0].len == 0)+	    h->decode = cram_huffman_decode_char0;+	else+	    h->decode = cram_huffman_decode_char;+    } else if (option == E_BYTE_ARRAY_BLOCK) {+	abort();+    } else {+	if (h->huffman.codes[0].len == 0)+	    h->decode = cram_huffman_decode_int0;+	else+	    h->decode = cram_huffman_decode_int;+    }++    return (cram_codec *)h;+}++int cram_huffman_encode_char0(cram_slice *slice, cram_codec *c,+			      char *in, int in_size) {+    return 0;+}++int cram_huffman_encode_char(cram_slice *slice, cram_codec *c,+			     char *in, int in_size) {+    int i, code, len, r = 0;+    unsigned char *syms = (unsigned char *)in;++    while (in_size--) {+	int sym = *syms++;+	if (sym >= -1 && sym < MAX_HUFF) {+	    i = c->e_huffman.val2code[sym+1];+	    assert(c->e_huffman.codes[i].symbol == sym);+	    code = c->e_huffman.codes[i].code;+	    len  = c->e_huffman.codes[i].len;+	} else {+	    /* Slow - use a lookup table for when sym < MAX_HUFF? */+	    for (i = 0; i < c->e_huffman.nvals; i++) {+		if (c->e_huffman.codes[i].symbol == sym)+		    break;+	    }+	    if (i == c->e_huffman.nvals)+		return -1;+    +	    code = c->e_huffman.codes[i].code;+	    len  = c->e_huffman.codes[i].len;+	}++	r |= store_bits_MSB(c->out, code, len);+    }++    return r;+}++int cram_huffman_encode_int0(cram_slice *slice, cram_codec *c,+			     char *in, int in_size) {+    return 0;+}++int cram_huffman_encode_int(cram_slice *slice, cram_codec *c,+			    char *in, int in_size) {+    int i, code, len, r = 0;+    int *syms = (int *)in;++    while (in_size--) {+	int sym = *syms++;++	if (sym >= -1 && sym < MAX_HUFF) {+	    i = c->e_huffman.val2code[sym+1];+	    assert(c->e_huffman.codes[i].symbol == sym);+	    code = c->e_huffman.codes[i].code;+	    len  = c->e_huffman.codes[i].len;+	} else {+	    /* Slow - use a lookup table for when sym < MAX_HUFFMAN_SYM? */+	    for (i = 0; i < c->e_huffman.nvals; i++) {+		if (c->e_huffman.codes[i].symbol == sym)+		    break;+	    }+	    if (i == c->e_huffman.nvals)+		return -1;+    +	    code = c->e_huffman.codes[i].code;+	    len  = c->e_huffman.codes[i].len;+	}++	r |= store_bits_MSB(c->out, code, len);+    }++    return r;+}++void cram_huffman_encode_free(cram_codec *c) {+    if (!c)+	return;++    if (c->e_huffman.codes)+	free(c->e_huffman.codes);+    free(c);+}++/*+ * Encodes a huffman tree.+ * Returns number of bytes written.+ */+int cram_huffman_encode_store(cram_codec *c, cram_block *b, char *prefix,+			      int version) {+    int i, len = 0;+    cram_huffman_code *codes = c->e_huffman.codes;+    /*+     * Up to code length 127 means 2.5e+26 bytes of data required (worst+     * case huffman tree needs symbols with freqs matching the Fibonacci+     * series). So guaranteed 1 byte per code.+     *+     * Symbols themselves could be 5 bytes (eg -1 is 5 bytes in itf8).+     *+     * Therefore 6*ncodes + 5 + 5 + 1 + 5 is max memory+     */+    char *tmp = malloc(6*c->e_huffman.nvals+16);+    char *tp = tmp;++    if (!tmp)+	return -1;++    if (prefix) {+	size_t l = strlen(prefix);+	BLOCK_APPEND(b, prefix, l);+	len += l;+    }++    tp += itf8_put(tp, c->e_huffman.nvals);+    for (i = 0; i < c->e_huffman.nvals; i++) {+	tp += itf8_put(tp, codes[i].symbol);+    }++    tp += itf8_put(tp, c->e_huffman.nvals);+    for (i = 0; i < c->e_huffman.nvals; i++) {+	tp += itf8_put(tp, codes[i].len);+    }++    len += itf8_put_blk(b, c->codec);+    len += itf8_put_blk(b, tp-tmp);+    BLOCK_APPEND(b, tmp, tp-tmp);+    len += tp-tmp;++    free(tmp);++    return len;+}++cram_codec *cram_huffman_encode_init(cram_stats *st,+				     enum cram_external_type option,+				     void *dat,+				     int version) {+    int *vals = NULL, *freqs = NULL, vals_alloc = 0, *lens, code, len;+    int nvals, i, ntot = 0, max_val = 0, min_val = INT_MAX, k;+    cram_codec *c;+    cram_huffman_code *codes;++    c = malloc(sizeof(*c));+    if (!c)+	return NULL;+    c->codec = E_HUFFMAN;++    /* Count number of unique symbols */+    for (nvals = i = 0; i < MAX_STAT_VAL; i++) {+	if (!st->freqs[i])+	    continue;+	if (nvals >= vals_alloc) {+	    vals_alloc = vals_alloc ? vals_alloc*2 : 1024;+	    vals  = realloc(vals,  vals_alloc * sizeof(int));+	    freqs = realloc(freqs, vals_alloc * sizeof(int));+	    if (!vals || !freqs) {+		if (vals)  free(vals);+		if (freqs) free(freqs);+		free(c);+		return NULL;+	    }+	}+	vals[nvals] = i;+	freqs[nvals] = st->freqs[i];+	assert(st->freqs[i] > 0);+	ntot += freqs[nvals];+	if (max_val < i) max_val = i;+	if (min_val > i) min_val = i;+	nvals++;+    }+    if (st->h) {+	khint_t k;++	for (k = kh_begin(st->h); k != kh_end(st->h); k++) {+	    if (!kh_exist(st->h, k))+		continue;+	    if (nvals >= vals_alloc) {+		vals_alloc = vals_alloc ? vals_alloc*2 : 1024;+		vals  = realloc(vals,  vals_alloc * sizeof(int));+		freqs = realloc(freqs, vals_alloc * sizeof(int));+		if (!vals || !freqs)+		    return NULL;+	    }+	    vals[nvals]= kh_key(st->h, k);+	    freqs[nvals] = kh_val(st->h, k);+	    assert(freqs[nvals] > 0);+	    ntot += freqs[nvals];+	    if (max_val < i) max_val = i;+	    if (min_val > i) min_val = i;+	    nvals++;+	}+    }++    assert(nvals > 0);++    freqs = realloc(freqs, 2*nvals*sizeof(*freqs));+    lens = calloc(2*nvals, sizeof(*lens));+    if (!lens || !freqs)+	return NULL;++    /* Inefficient, use pointers to form chain so we can insert and maintain+     * a sorted list? This is currently O(nvals^2) complexity.+     */+    for (;;) {+	int low1 = INT_MAX, low2 = INT_MAX;+	int ind1 = 0, ind2 = 0;+	for (i = 0; i < nvals; i++) {+	    if (freqs[i] < 0)+		continue;+	    if (low1 > freqs[i]) +		low2 = low1, ind2 = ind1, low1 = freqs[i], ind1 = i;+	    else if (low2 > freqs[i])+		low2 = freqs[i], ind2 = i;+	}+	if (low2 == INT_MAX)+	    break;++	freqs[nvals] = low1 + low2;+	lens[ind1] = nvals;+	lens[ind2] = nvals;+	freqs[ind1] *= -1;+	freqs[ind2] *= -1;+	nvals++;+    }+    nvals = nvals/2+1;++    /* Assign lengths */+    for (i = 0; i < nvals; i++) {+	int code_len = 0;+	for (k = lens[i]; k; k = lens[k])+	    code_len++;+	lens[i] = code_len;+	freqs[i] *= -1;+	//fprintf(stderr, "%d / %d => %d\n", vals[i], freqs[i], lens[i]);+    }+++    /* Sort, need in a struct */+    if (!(codes = malloc(nvals * sizeof(*codes))))+	return NULL;+    for (i = 0; i < nvals; i++) {+	codes[i].symbol = vals[i];+	codes[i].len = lens[i];+    }+    qsort(codes, nvals, sizeof(*codes), code_sort);++    /*+     * Generate canonical codes from lengths.+     * Sort by length.+     * Start with 0.+     * Every new code of same length is +1.+     * Every new code of new length is +1 then <<1 per extra length.+     *+     * /\+     * a/\+     * /\/\+     * bcd/\+     *    ef+     * +     * a 1  0+     * b 3  4 (0+1)<<2+     * c 3  5+     * d 3  6+     * e 4  14  (6+1)<<1+     * f 5  15     +     */+    code = 0; len = codes[0].len;+    for (i = 0; i < nvals; i++) {+	while (len != codes[i].len) {+	    code<<=1;+	    len++;+	}+	codes[i].code = code++;++	if (codes[i].symbol >= -1 && codes[i].symbol < MAX_HUFF)+	    c->e_huffman.val2code[codes[i].symbol+1] = i;++	//fprintf(stderr, "sym %d, code %d, len %d\n",+	//	codes[i].symbol, codes[i].code, codes[i].len);+    }++    free(lens);+    free(vals);+    free(freqs);++    c->e_huffman.codes = codes;+    c->e_huffman.nvals = nvals;++    c->free = cram_huffman_encode_free;+    if (option == E_BYTE || option == E_BYTE_ARRAY) {+	if (c->e_huffman.codes[0].len == 0)+	    c->encode = cram_huffman_encode_char0;+	else+	    c->encode = cram_huffman_encode_char;+    } else {+	if (c->e_huffman.codes[0].len == 0)+	    c->encode = cram_huffman_encode_int0;+	else+	    c->encode = cram_huffman_encode_int;+    }+    c->store = cram_huffman_encode_store;++    return c;+}++/*+ * ---------------------------------------------------------------------------+ * BYTE_ARRAY_LEN+ */+int cram_byte_array_len_decode(cram_slice *slice, cram_codec *c,+			       cram_block *in, char *out,+			       int *out_size) {+    /* Fetch length */+    int32_t len = 0, one = 1;+    int r;++    r = c->byte_array_len.len_codec->decode(slice, c->byte_array_len.len_codec,+                                            in, (char *)&len, &one);+    //printf("ByteArray Len=%d\n", len);++    if (!r && c->byte_array_len.val_codec && len >= 0) {+	r = c->byte_array_len.val_codec->decode(slice,+						c->byte_array_len.val_codec,+						in, out, &len);+    } else {+	return -1;+    }++    *out_size = len;++    return r;+}++void cram_byte_array_len_decode_free(cram_codec *c) {+    if (!c) return;++    if (c->byte_array_len.len_codec)+	c->byte_array_len.len_codec->free(c->byte_array_len.len_codec);++    if (c->byte_array_len.val_codec)+	c->byte_array_len.val_codec->free(c->byte_array_len.val_codec);++    free(c);+}++cram_codec *cram_byte_array_len_decode_init(char *data, int size,+					    enum cram_external_type option,+					    int version) {+    cram_codec *c;+    char *cp   = data;+    char *endp = data + size;+    int32_t encoding = 0;+    int32_t sub_size = -1;++    if (!(c = malloc(sizeof(*c))))+	return NULL;++    c->codec  = E_BYTE_ARRAY_LEN;+    c->decode = cram_byte_array_len_decode;+    c->free   = cram_byte_array_len_decode_free;+    +    cp += safe_itf8_get(cp, endp, &encoding);+    cp += safe_itf8_get(cp, endp, &sub_size);+    if (sub_size < 0 || endp - cp < sub_size)+        goto malformed;+    c->byte_array_len.len_codec = cram_decoder_init(encoding, cp, sub_size,+						    E_INT, version);+    if (c->byte_array_len.len_codec == NULL)+        goto no_codec;+    cp += sub_size;++    sub_size = -1;+    cp += safe_itf8_get(cp, endp, &encoding);+    cp += safe_itf8_get(cp, endp, &sub_size);+    if (sub_size < 0 || endp - cp < sub_size)+        goto malformed;+    c->byte_array_len.val_codec = cram_decoder_init(encoding, cp, sub_size,+						    option, version);+    if (c->byte_array_len.val_codec == NULL)+        goto no_codec;+    cp += sub_size;++    if (cp - data != size)+        goto malformed;++    return c;++ malformed:+    fprintf(stderr, "Malformed byte_array_len header stream\n");+ no_codec:+    free(c);+    return NULL;+}++int cram_byte_array_len_encode(cram_slice *slice, cram_codec *c,+			       char *in, int in_size) {+    int32_t i32 = in_size;+    int r = 0;++    r |= c->e_byte_array_len.len_codec->encode(slice,+					       c->e_byte_array_len.len_codec,+					       (char *)&i32, 1);+    r |= c->e_byte_array_len.val_codec->encode(slice,+					       c->e_byte_array_len.val_codec,+					       in, in_size);+    return r;+}++void cram_byte_array_len_encode_free(cram_codec *c) {+    if (!c)+	return;++    if (c->e_byte_array_len.len_codec)+	c->e_byte_array_len.len_codec->free(c->e_byte_array_len.len_codec);++    if (c->e_byte_array_len.val_codec)+	c->e_byte_array_len.val_codec->free(c->e_byte_array_len.val_codec);++    free(c);+}++int cram_byte_array_len_encode_store(cram_codec *c, cram_block *b,+				     char *prefix, int version) {+    int len = 0, len2, len3;+    cram_codec *tc;+    cram_block *b_len, *b_val;++    if (prefix) {+	size_t l = strlen(prefix);+	BLOCK_APPEND(b, prefix, l);+	len += l;+    }++    tc = c->e_byte_array_len.len_codec; +    b_len = cram_new_block(0, 0);+    len2 = tc->store(tc, b_len, NULL, version);++    tc = c->e_byte_array_len.val_codec;+    b_val = cram_new_block(0, 0);+    len3 = tc->store(tc, b_val, NULL, version);++    len += itf8_put_blk(b, c->codec);+    len += itf8_put_blk(b, len2+len3);+    BLOCK_APPEND(b, BLOCK_DATA(b_len), BLOCK_SIZE(b_len));+    BLOCK_APPEND(b, BLOCK_DATA(b_val), BLOCK_SIZE(b_val));++    cram_free_block(b_len);+    cram_free_block(b_val);++    return len + len2 + len3;+}++cram_codec *cram_byte_array_len_encode_init(cram_stats *st,+					    enum cram_external_type option,+					    void *dat,+					    int version) {+    cram_codec *c;+    cram_byte_array_len_encoder *e = (cram_byte_array_len_encoder *)dat;++    c = malloc(sizeof(*c));+    if (!c)+	return NULL;+    c->codec = E_BYTE_ARRAY_LEN;+    c->free = cram_byte_array_len_encode_free;+    c->encode = cram_byte_array_len_encode;+    c->store = cram_byte_array_len_encode_store;++    c->e_byte_array_len.len_codec = cram_encoder_init(e->len_encoding,+						      NULL, E_INT, +						      e->len_dat,+						      version);+    c->e_byte_array_len.val_codec = cram_encoder_init(e->val_encoding,+						      NULL, E_BYTE_ARRAY, +						      e->val_dat,+						      version);++    return c;+}++/*+ * ---------------------------------------------------------------------------+ * BYTE_ARRAY_STOP+ */+static int cram_byte_array_stop_decode_char(cram_slice *slice, cram_codec *c,+					    cram_block *in, char *out,+					    int *out_size) {+    cram_block *b = NULL;+    char *cp, ch;++    b = cram_get_block_by_id(slice, c->byte_array_stop.content_id);+    if (!b)+        return *out_size?-1:0;++    if (b->idx >= b->uncomp_size)+	return -1;++    cp = (char *)b->data + b->idx;+    if (out) {+	while ((ch = *cp) != (char)c->byte_array_stop.stop) {+	    if (cp - (char *)b->data >= b->uncomp_size)+		return -1;+	    *out++ = ch;+	    cp++;+	}+    } else {+	// Consume input, but produce no output+	while ((ch = *cp) != (char)c->byte_array_stop.stop) {+	    if (cp - (char *)b->data >= b->uncomp_size)+		return -1;+	    cp++;+	}+    }++    *out_size = cp - (char *)(b->data + b->idx);+    b->idx = cp - (char *)b->data + 1;++    return 0;+}++int cram_byte_array_stop_decode_block(cram_slice *slice, cram_codec *c,+				      cram_block *in, char *out_,+				      int *out_size) {+    cram_block *b;+    cram_block *out = (cram_block *)out_;+    char *cp, *out_cp, *cp_end;+    char stop;++    b = cram_get_block_by_id(slice, c->byte_array_stop.content_id);+    if (!b)+        return *out_size?-1:0;++    if (b->idx >= b->uncomp_size)+	return -1;+    cp = (char *)b->data + b->idx;+    cp_end = (char *)b->data + b->uncomp_size;+    out_cp = (char *)BLOCK_END(out);++    stop = c->byte_array_stop.stop;+    if (cp_end - cp < out->alloc - out->byte) {+	while (cp != cp_end && *cp != stop)+	    *out_cp++ = *cp++;+	BLOCK_SIZE(out) = out_cp - (char *)BLOCK_DATA(out);+    } else {+	char *cp_start;+	for (cp_start = cp; cp != cp_end && *cp != stop; cp++)+	    ;+	BLOCK_APPEND(out, cp_start, cp - cp_start);+	BLOCK_GROW(out, cp - cp_start);+    }++    *out_size = cp - (char *)(b->data + b->idx);+    b->idx = cp - (char *)b->data + 1;++    return 0;+}++void cram_byte_array_stop_decode_free(cram_codec *c) {+    if (!c) return;++    free(c);+}++cram_codec *cram_byte_array_stop_decode_init(char *data, int size,+					     enum cram_external_type option,+					     int version) {+    cram_codec *c;+    unsigned char *cp = (unsigned char *)data;++    if (!(c = malloc(sizeof(*c))))+	return NULL;++    c->codec  = E_BYTE_ARRAY_STOP;+    switch (option) {+    case E_BYTE_ARRAY_BLOCK:+        c->decode = cram_byte_array_stop_decode_block;+	break;+    case E_BYTE_ARRAY:+        c->decode = cram_byte_array_stop_decode_char;+	break;+    default:+        fprintf(stderr, "byte_array_stop codec only supports BYTE_ARRAYs.\n");+        free(c);+        return NULL;+    }+    c->free   = cram_byte_array_stop_decode_free;+    +    c->byte_array_stop.stop = *cp++;+    if (CRAM_MAJOR_VERS(version) == 1) {+	c->byte_array_stop.content_id = cp[0] + (cp[1]<<8) + (cp[2]<<16)+	    + (cp[3]<<24);+	cp += 4;+    } else {+	cp += itf8_get(cp, &c->byte_array_stop.content_id);+    }++    if ((char *)cp - data != size) {+	fprintf(stderr, "Malformed byte_array_stop header stream\n");+	free(c);+	return NULL;+    }++    return c;+}++int cram_byte_array_stop_encode(cram_slice *slice, cram_codec *c,+				char *in, int in_size) {+    BLOCK_APPEND(c->out, in, in_size);+    BLOCK_APPEND_CHAR(c->out, c->e_byte_array_stop.stop);+    return 0;+}++void cram_byte_array_stop_encode_free(cram_codec *c) {+    if (!c)+	return;+    free(c);+}++int cram_byte_array_stop_encode_store(cram_codec *c, cram_block *b,+				      char *prefix, int version) {+    int len = 0;+    char buf[20], *cp = buf;++    if (prefix) {+	size_t l = strlen(prefix);+	BLOCK_APPEND(b, prefix, l);+	len += l;+    }++    cp += itf8_put(cp, c->codec);++    if (CRAM_MAJOR_VERS(version) == 1) {+	cp += itf8_put(cp, 5);+	*cp++ = c->e_byte_array_stop.stop;+	*cp++ = (c->e_byte_array_stop.content_id >>  0) & 0xff;+	*cp++ = (c->e_byte_array_stop.content_id >>  8) & 0xff;+	*cp++ = (c->e_byte_array_stop.content_id >> 16) & 0xff;+	*cp++ = (c->e_byte_array_stop.content_id >> 24) & 0xff;+    } else {+	cp += itf8_put(cp, 1 + itf8_size(c->e_byte_array_stop.content_id));+	*cp++ = c->e_byte_array_stop.stop;+	cp += itf8_put(cp, c->e_byte_array_stop.content_id);+    }++    BLOCK_APPEND(b, buf, cp-buf);+    len += cp-buf;++    return len;+}++cram_codec *cram_byte_array_stop_encode_init(cram_stats *st,+					     enum cram_external_type option,+					     void *dat,+					     int version) {+    cram_codec *c;++    c = malloc(sizeof(*c));+    if (!c)+	return NULL;+    c->codec = E_BYTE_ARRAY_STOP;+    c->free = cram_byte_array_stop_encode_free;+    c->encode = cram_byte_array_stop_encode;+    c->store = cram_byte_array_stop_encode_store;++    c->e_byte_array_stop.stop = ((int *)dat)[0];+    c->e_byte_array_stop.content_id = ((int *)dat)[1];++    return c;+}++/*+ * ---------------------------------------------------------------------------+ */++const char *cram_encoding2str(enum cram_encoding t) {+    switch (t) {+    case E_NULL:            return "NULL";+    case E_EXTERNAL:        return "EXTERNAL";+    case E_GOLOMB:          return "GOLOMB";+    case E_HUFFMAN:         return "HUFFMAN";+    case E_BYTE_ARRAY_LEN:  return "BYTE_ARRAY_LEN";+    case E_BYTE_ARRAY_STOP: return "BYTE_ARRAY_STOP";+    case E_BETA:            return "BETA";+    case E_SUBEXP:          return "SUBEXP";+    case E_GOLOMB_RICE:     return "GOLOMB_RICE";+    case E_GAMMA:           return "GAMMA";+    case E_NUM_CODECS:+    default:                return "?";+    }+}++static cram_codec *(*decode_init[])(char *data,+				    int size,+				    enum cram_external_type option,+				    int version) = {+    NULL,+    cram_external_decode_init,+    NULL,+    cram_huffman_decode_init,+    cram_byte_array_len_decode_init,+    cram_byte_array_stop_decode_init,+    cram_beta_decode_init,+    cram_subexp_decode_init,+    NULL,+    cram_gamma_decode_init,+};++cram_codec *cram_decoder_init(enum cram_encoding codec,+			      char *data, int size,+			      enum cram_external_type option,+			      int version) {+    if (codec >= E_NULL && codec < E_NUM_CODECS && decode_init[codec]) {+	return decode_init[codec](data, size, option, version);+    } else {+	fprintf(stderr, "Unimplemented codec of type %s\n", cram_encoding2str(codec));+	return NULL;+    }+}++static cram_codec *(*encode_init[])(cram_stats *stx,+				    enum cram_external_type option,+				    void *opt,+				    int version) = {+    NULL,+    cram_external_encode_init,+    NULL,+    cram_huffman_encode_init,+    cram_byte_array_len_encode_init,+    cram_byte_array_stop_encode_init,+    cram_beta_encode_init,+    NULL, //cram_subexp_encode_init,+    NULL,+    NULL, //cram_gamma_encode_init,+};++cram_codec *cram_encoder_init(enum cram_encoding codec,+			      cram_stats *st,+			      enum cram_external_type option,+			      void *dat,+			      int version) {+    if (st && !st->nvals)+	return NULL;++    if (encode_init[codec]) {+	cram_codec *r;+	if ((r = encode_init[codec](st, option, dat, version)))+	    r->out = NULL;+	return r;+    } else {+	fprintf(stderr, "Unimplemented codec of type %s\n", cram_encoding2str(codec));+	abort();+    }+}++/*+ * Returns the content_id used by this codec, also in id2 if byte_array_len.+ * Returns -1 for the CORE block and -2 for unneeded.+ * id2 is only filled out for BYTE_ARRAY_LEN which uses 2 codecs.+ */+int cram_codec_to_id(cram_codec *c, int *id2) {+    int bnum1, bnum2 = -2;++    switch (c->codec) {+    case E_HUFFMAN:+	bnum1 = c->huffman.ncodes == 1 ? -2 : -1;+	break;+    case E_GOLOMB:+    case E_BETA:+    case E_SUBEXP:+    case E_GOLOMB_RICE:+    case E_GAMMA:+	bnum1 = -1;+	break;+    case E_EXTERNAL:+	bnum1 = c->external.content_id;+	break;+    case E_BYTE_ARRAY_LEN:+	bnum1 = cram_codec_to_id(c->byte_array_len.len_codec, NULL);+	bnum2 = cram_codec_to_id(c->byte_array_len.val_codec, NULL);+	break;+    case E_BYTE_ARRAY_STOP:+	bnum1 = c->byte_array_stop.content_id;+	break;+    case E_NULL:+	bnum1 = -2;+	break;+    default:+	fprintf(stderr, "Unknown codec type %d\n", c->codec);+	bnum1 = -1;+    }++    if (id2)+	*id2 = bnum2;+    return bnum1;+}+++/*+ * cram_codec structures are specialised for decoding or encoding.+ * Unfortunately this makes turning a decoder into an encoder (such as+ * when transcoding files) problematic.+ *+ * This function converts a cram decoder codec into an encoder version+ * in-place (ie it modifiers the codec itself).+ *+ * Returns 0 on success;+ *        -1 on failure.+ */+int cram_codec_decoder2encoder(cram_fd *fd, cram_codec *c) {+    int j;++    switch (c->codec) {+    case E_EXTERNAL:+	// shares struct with decode+	c->free = cram_external_encode_free;+	c->store = cram_external_encode_store;+	if (c->decode == cram_external_decode_int)+	    c->encode = cram_external_encode_int;+	else if (c->decode == cram_external_decode_char)+	    c->encode = cram_external_encode_char;+	else+	    return -1;+	break;++    case E_HUFFMAN: {+	// New structure, so switch.+	// FIXME: we huffman and e_huffman structs amended, we could+	// unify this.+	cram_codec *t = malloc(sizeof(*t));+	t->codec = E_HUFFMAN;+	t->free = cram_huffman_encode_free;+	t->store = cram_huffman_encode_store;+	t->e_huffman.codes = c->huffman.codes;+	t->e_huffman.nvals = c->huffman.ncodes;+	for (j = 0; j < t->e_huffman.nvals; j++) {+	    int32_t sym = t->e_huffman.codes[j].symbol;+	    if (sym >= -1 && sym < MAX_HUFF)+		t->e_huffman.val2code[sym+1] = j;+	}++	if (c->decode == cram_huffman_decode_char0)+	    t->encode = cram_huffman_encode_char0;+	else if (c->decode == cram_huffman_decode_char)+	    t->encode = cram_huffman_encode_char;+	else if (c->decode == cram_huffman_decode_int0)+	    t->encode = cram_huffman_encode_int0;+	else if (c->decode == cram_huffman_decode_int)+	    t->encode = cram_huffman_encode_int;+	else {+	    free(t);+	    return -1;+	}+	*c = *t;+	free(t);+	break;+    }++    case E_BETA:+	// shares struct with decode+	c->free = cram_beta_encode_free;+	c->store = cram_beta_encode_store;+	if (c->decode == cram_beta_decode_int)+	    c->encode = cram_beta_encode_int;+	else if (c->decode == cram_beta_decode_char)+	    c->encode = cram_beta_encode_char;+	else+	    return -1;+	break;++    case E_BYTE_ARRAY_LEN: {+	cram_codec *t = malloc(sizeof(*t));+	t->codec = E_BYTE_ARRAY_LEN;+	t->free   = cram_byte_array_len_encode_free;+	t->store  = cram_byte_array_len_encode_store;+	t->encode = cram_byte_array_len_encode;+	t->e_byte_array_len.len_codec = c->byte_array_len.len_codec;+	t->e_byte_array_len.val_codec = c->byte_array_len.val_codec;+	if (cram_codec_decoder2encoder(fd, t->e_byte_array_len.len_codec) == -1 ||+	    cram_codec_decoder2encoder(fd, t->e_byte_array_len.val_codec) == -1) {+	    t->free(t);+	    return -1;+	}++	// {len,val}_{encoding,dat} are undefined, but unused.+	// Leaving them unset here means we can test that assertion.+	*c = *t;+	free(t);+	break;+    }++    case E_BYTE_ARRAY_STOP:+	// shares struct with decode+	c->free   = cram_byte_array_stop_encode_free;+	c->store  = cram_byte_array_stop_encode_store;+	c->encode = cram_byte_array_stop_encode;+	break;++    default:+	return -1;+    }++    return 0;+}
+ htslib-1.3.2/cram/cram_codecs.h view
@@ -0,0 +1,194 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _CRAM_ENCODINGS_H_+#define _CRAM_ENCODINGS_H_++#include <inttypes.h>++#ifdef __cplusplus+extern "C" {+#endif++struct cram_codec;++/*+ * Slow but simple huffman decoder to start with.+ * Read a bit at a time, keeping track of {length, value}+ * eg. 1 1 0 1 => {1,1},  {2,3}, {3,6}, {4,13}+ *+ * Keep track of this through the huffman code table.+ * For fast scanning we have an index of where the first code of length X+ * appears.+ */+typedef struct {+    int32_t symbol;+    int32_t p; // next code start value, minus index to codes[]+    int32_t code;+    int32_t len;+} cram_huffman_code;++typedef struct {+    int ncodes;+    cram_huffman_code *codes;+} cram_huffman_decoder;++#define MAX_HUFF 128+typedef struct {+    cram_huffman_code *codes;+    int nvals;+    int val2code[MAX_HUFF+1]; // value to code lookup for small values+} cram_huffman_encoder;++typedef struct {+    int32_t offset;+    int32_t nbits;+} cram_beta_decoder;++typedef struct {+    int32_t offset;+} cram_gamma_decoder;++typedef struct {+    int32_t offset;+    int32_t k;+} cram_subexp_decoder;++typedef struct {+    int32_t content_id;+    enum cram_external_type type;+} cram_external_decoder;++typedef struct {+    struct cram_codec *len_codec;+    struct cram_codec *val_codec;+} cram_byte_array_len_decoder;++typedef struct {+    unsigned char stop;+    int32_t content_id;+} cram_byte_array_stop_decoder;++typedef struct {+    enum cram_encoding len_encoding;+    enum cram_encoding val_encoding;+    void *len_dat;+    void *val_dat;+    struct cram_codec *len_codec;+    struct cram_codec *val_codec;+} cram_byte_array_len_encoder;++/*+ * A generic codec structure.+ */+typedef struct cram_codec {+    enum cram_encoding codec;+    cram_block *out;+    void (*free)(struct cram_codec *codec);+    int (*decode)(cram_slice *slice, struct cram_codec *codec,+		  cram_block *in, char *out, int *out_size);+    int (*encode)(cram_slice *slice, struct cram_codec *codec,+		  char *in, int in_size);+    int (*store)(struct cram_codec *codec, cram_block *b, char *prefix,+		 int version);+    union {+	cram_huffman_decoder         huffman;+	cram_external_decoder        external;+	cram_beta_decoder            beta;+	cram_gamma_decoder           gamma;+	cram_subexp_decoder          subexp;+	cram_byte_array_len_decoder  byte_array_len;+	cram_byte_array_stop_decoder byte_array_stop;++	cram_huffman_encoder         e_huffman;+	cram_external_decoder        e_external;+	cram_byte_array_stop_decoder e_byte_array_stop;+	cram_byte_array_len_encoder  e_byte_array_len;+	cram_beta_decoder            e_beta;+    };+} cram_codec;++const char *cram_encoding2str(enum cram_encoding t);++cram_codec *cram_decoder_init(enum cram_encoding codec, char *data, int size,+			      enum cram_external_type option,+			      int version);+cram_codec *cram_encoder_init(enum cram_encoding codec, cram_stats *st,+			      enum cram_external_type option, void *dat,+			      int version);++//int cram_decode(void *codes, char *in, int in_size, char *out, int *out_size);+//void cram_decoder_free(void *codes);++//#define GET_BIT_MSB(b,v) (void)(v<<=1, v|=(b->data[b->byte] >> b->bit)&1, (--b->bit == -1) && (b->bit = 7, b->byte++))++#define GET_BIT_MSB(b,v) (void)(v<<=1, v|=(b->data[b->byte] >> b->bit)&1, b->byte += (--b->bit<0), b->bit&=7)++/*+ * Check that enough bits are left in a block to satisy a bit-based decoder.+ * Return  0 if there are enough+ *         1 if not.+ */++static inline int cram_not_enough_bits(cram_block *blk, int nbits) {+    if (nbits < 0 ||+	(blk->byte >= blk->uncomp_size && nbits > 0) ||+	(blk->uncomp_size - blk->byte <= INT32_MAX / 8 + 1 &&+	 (blk->uncomp_size - blk->byte) * 8 + blk->bit - 7 < nbits)) {+        return 1;+    }+    return 0;+}++/*+ * Returns the content_id used by this codec, also in id2 if byte_array_len.+ * Returns -1 for the CORE block and -2 for unneeded.+ * id2 is only filled out for BYTE_ARRAY_LEN which uses 2 codecs.+ */+int cram_codec_to_id(cram_codec *c, int *id2);++/*+ * cram_codec structures are specialised for decoding or encoding.+ * Unfortunately this makes turning a decoder into an encoder (such as+ * when transcoding files) problematic.+ *+ * This function converts a cram decoder codec into an encoder version+ * in-place (ie it modifiers the codec itself).+ *+ * Returns 0 on success;+ *        -1 on failure.+ */+int cram_codec_decoder2encoder(cram_fd *fd, cram_codec *c);++#ifdef __cplusplus+}+#endif++#endif /* _CRAM_ENCODINGS_H_ */
+ htslib-1.3.2/cram/cram_decode.c view
@@ -0,0 +1,3159 @@+/*+Copyright (c) 2012-2014 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+ * - In-memory decoding of CRAM data structures.+ * - Iterator for reading CRAM record by record.+ */++#include <config.h>++#include <stdio.h>+#include <errno.h>+#include <assert.h>+#include <stdlib.h>+#include <string.h>+#include <zlib.h>+#include <sys/types.h>+#include <sys/stat.h>+#include <math.h>+#include <ctype.h>++#include "cram/cram.h"+#include "cram/os.h"+#include "htslib/hts.h"++//Whether CIGAR has just M or uses = and X to indicate match and mismatch+//#define USE_X++/* ----------------------------------------------------------------------+ * CRAM compression headers+ */++/*+ * Decodes the Tag Dictionary record in the preservation map+ * Updates the cram compression header.+ * + * Returns number of bytes decoded on success+ *        -1 on failure+ */+int cram_decode_TD(char *cp, const char *endp, cram_block_compression_hdr *h) {+    char *op = cp;+    unsigned char *dat;+    cram_block *b;+    int32_t blk_size = 0;+    int nTL, i, sz;++    if (!(b = cram_new_block(0, 0)))+	return -1;++    /* Decode */+    cp += safe_itf8_get(cp, endp, &blk_size);+    if (!blk_size) {+	h->nTL = 0;+	h->TL = NULL;+	cram_free_block(b);+	return cp - op;+    }++    if (blk_size < 0 || endp - cp < blk_size) {+        cram_free_block(b);+	return -1;+    }++    BLOCK_APPEND(b, cp, blk_size);+    cp += blk_size;+    sz = cp - op;+    // Force nul termination if missing+    if (BLOCK_DATA(b)[BLOCK_SIZE(b)-1])+	BLOCK_APPEND_CHAR(b, '\0');++    /* Set up TL lookup table */+    dat = BLOCK_DATA(b);++    // Count+    for (nTL = i = 0; i < BLOCK_SIZE(b); i++) {+	nTL++;+	while (dat[i])+	    i++;+    }++    // Copy+    h->nTL = nTL;+    if (!(h->TL = calloc(h->nTL, sizeof(unsigned char *)))) {+        cram_free_block(b);+        return -1;+    }+    for (nTL = i = 0; i < BLOCK_SIZE(b); i++) {+	h->TL[nTL++] = &dat[i];+	while (dat[i])+	    i++;+    }+    h->TD_blk = b;+    +    return sz;+}++/*+ * Decodes a CRAM block compression header.+ * Returns header ptr on success+ *         NULL on failure+ */+cram_block_compression_hdr *cram_decode_compression_header(cram_fd *fd,+							   cram_block *b) {+    char *cp, *endp, *cp_copy;+    cram_block_compression_hdr *hdr = calloc(1, sizeof(*hdr));+    int i;+    int32_t map_size = 0, map_count = 0;++    if (!hdr)+	return NULL;++    if (b->method != RAW) {+	if (cram_uncompress_block(b)) {+	    free(hdr);+	    return NULL;+	}+    }++    cp = (char *)b->data;+    endp = cp + b->uncomp_size;++    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	cp += safe_itf8_get(cp, endp, &hdr->ref_seq_id);+	cp += safe_itf8_get(cp, endp, &hdr->ref_seq_start);+	cp += safe_itf8_get(cp, endp, &hdr->ref_seq_span);+	cp += safe_itf8_get(cp, endp, &hdr->num_records);+	cp += safe_itf8_get(cp, endp, &hdr->num_landmarks);+	if (!(hdr->landmark = malloc(hdr->num_landmarks * sizeof(int32_t)))) {+	    free(hdr);+	    return NULL;+	}+	for (i = 0; i < hdr->num_landmarks; i++) {+	    cp += safe_itf8_get(cp, endp, &hdr->landmark[i]);+	}+    }++    hdr->preservation_map = kh_init(map);++    memset(hdr->rec_encoding_map, 0,+	   CRAM_MAP_HASH * sizeof(hdr->rec_encoding_map[0]));+    memset(hdr->tag_encoding_map, 0,+	   CRAM_MAP_HASH * sizeof(hdr->tag_encoding_map[0]));++    if (!hdr->preservation_map) {+	cram_free_compression_header(hdr);+	return NULL;+    }++    /* Initialise defaults for preservation map */+    hdr->mapped_qs_included = 0;+    hdr->unmapped_qs_included = 0;+    hdr->unmapped_placed = 0;+    hdr->qs_included = 0;+    hdr->read_names_included = 0;+    hdr->AP_delta = 1;+    memcpy(hdr->substitution_matrix, "CGTNAGTNACTNACGNACGT", 20);++    /* Preservation map */+    cp += safe_itf8_get(cp, endp, &map_size); cp_copy = cp;+    cp += safe_itf8_get(cp, endp, &map_count);+    for (i = 0; i < map_count; i++) {+	pmap_t hd;+	khint_t k;+	int r;++	if (endp - cp < 2) {+	    cram_free_compression_header(hdr);+	    return NULL;+	}+	cp += 2;+	switch(CRAM_KEY(cp[-2],cp[-1])) {+	case CRAM_KEY('M','I'):+	    hd.i = *cp++;+	    k = kh_put(map, hdr->preservation_map, "MI", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+                return NULL;+            }++	    kh_val(hdr->preservation_map, k) = hd;+	    hdr->mapped_qs_included = hd.i;+	    break;++	case CRAM_KEY('U','I'):+	    hd.i = *cp++;+	    k = kh_put(map, hdr->preservation_map, "UI", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+                return NULL;+            }++	    kh_val(hdr->preservation_map, k) = hd;+	    hdr->unmapped_qs_included = hd.i;+	    break;++	case CRAM_KEY('P','I'):+	    hd.i = *cp++;+	    k = kh_put(map, hdr->preservation_map, "PI", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+                return NULL;+            }++	    kh_val(hdr->preservation_map, k) = hd;+	    hdr->unmapped_placed = hd.i;+	    break;++	case CRAM_KEY('R','N'):+	    hd.i = *cp++;+	    k = kh_put(map, hdr->preservation_map, "RN", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+                return NULL;+            }++	    kh_val(hdr->preservation_map, k) = hd;+	    hdr->read_names_included = hd.i;+	    break;++	case CRAM_KEY('A','P'):+	    hd.i = *cp++;+	    k = kh_put(map, hdr->preservation_map, "AP", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+                return NULL;+            }++	    kh_val(hdr->preservation_map, k) = hd;+	    hdr->AP_delta = hd.i;+	    break;++	case CRAM_KEY('R','R'):+	    hd.i = *cp++;+	    k = kh_put(map, hdr->preservation_map, "RR", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+                return NULL;+            }++	    kh_val(hdr->preservation_map, k) = hd;+	    fd->no_ref = !hd.i;+	    break;++	case CRAM_KEY('S','M'):+	    if (endp - cp < 5) {+	        cram_free_compression_header(hdr);+		return NULL;+	    }+	    hdr->substitution_matrix[0][(cp[0]>>6)&3] = 'C';+	    hdr->substitution_matrix[0][(cp[0]>>4)&3] = 'G';+	    hdr->substitution_matrix[0][(cp[0]>>2)&3] = 'T';+	    hdr->substitution_matrix[0][(cp[0]>>0)&3] = 'N';++	    hdr->substitution_matrix[1][(cp[1]>>6)&3] = 'A';+	    hdr->substitution_matrix[1][(cp[1]>>4)&3] = 'G';+	    hdr->substitution_matrix[1][(cp[1]>>2)&3] = 'T';+	    hdr->substitution_matrix[1][(cp[1]>>0)&3] = 'N';++	    hdr->substitution_matrix[2][(cp[2]>>6)&3] = 'A';+	    hdr->substitution_matrix[2][(cp[2]>>4)&3] = 'C';+	    hdr->substitution_matrix[2][(cp[2]>>2)&3] = 'T';+	    hdr->substitution_matrix[2][(cp[2]>>0)&3] = 'N';++	    hdr->substitution_matrix[3][(cp[3]>>6)&3] = 'A';+	    hdr->substitution_matrix[3][(cp[3]>>4)&3] = 'C';+	    hdr->substitution_matrix[3][(cp[3]>>2)&3] = 'G';+	    hdr->substitution_matrix[3][(cp[3]>>0)&3] = 'N';++	    hdr->substitution_matrix[4][(cp[4]>>6)&3] = 'A';+	    hdr->substitution_matrix[4][(cp[4]>>4)&3] = 'C';+	    hdr->substitution_matrix[4][(cp[4]>>2)&3] = 'G';+	    hdr->substitution_matrix[4][(cp[4]>>0)&3] = 'T';++	    hd.p = cp;+	    cp += 5;++	    k = kh_put(map, hdr->preservation_map, "SM", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	    kh_val(hdr->preservation_map, k) = hd;+	    break;++	case CRAM_KEY('T','D'): {+	    int sz = cram_decode_TD(cp, endp, hdr); // tag dictionary+	    if (sz < 0) {+		cram_free_compression_header(hdr);+		return NULL;+	    }++	    hd.p = cp;+	    cp += sz;++	    k = kh_put(map, hdr->preservation_map, "TD", &r);+	    if (-1 == r) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	    kh_val(hdr->preservation_map, k) = hd;+	    break;+	}++	default:+	    fprintf(stderr, "Unrecognised preservation map key %c%c\n",+		    cp[-2], cp[-1]);+	    // guess byte;+	    cp++;+	    break;+	}+    }+    if (cp - cp_copy != map_size) {+	cram_free_compression_header(hdr);+	return NULL;+    }++    /* Record encoding map */+    cp += safe_itf8_get(cp, endp, &map_size); cp_copy = cp;+    cp += safe_itf8_get(cp, endp, &map_count);+    for (i = 0; i < map_count; i++) {+	char *key = cp;+	int32_t encoding = E_NULL;+	int32_t size = 0;+	cram_map *m = malloc(sizeof(*m)); // FIXME: use pooled_alloc++	if (!m || endp - cp < 4) {+	    free(m);+	    cram_free_compression_header(hdr);+	    return NULL;+	}++	cp += 2;+	cp += safe_itf8_get(cp, endp, &encoding);+	cp += safe_itf8_get(cp, endp, &size);++	// Fill out cram_map purely for cram_dump to dump out.+	m->key = (key[0]<<8)|key[1];+	m->encoding = encoding;+	m->size     = size;+	m->offset   = cp - (char *)b->data;+	m->codec = NULL;++	if (m->encoding == E_NULL)+	    continue;++	if (size < 0 || endp - cp < size) {+	    free(m);+	    cram_free_compression_header(hdr);+	    return NULL;+	}++	//printf("%s codes for %.2s\n", cram_encoding2str(encoding), key);++	/*+	 * For CRAM1.0 CF and BF are Byte and not Int.+	 * Practically speaking it makes no difference unless we have a+	 * 1.0 format file that stores these in EXTERNAL as only then+	 * does Byte vs Int matter.+	 *+	 * Neither this C code nor Java reference implementations did this,+	 * so we gloss over it and treat them as int.+	 */++	if (key[0] == 'B' && key[1] == 'F') {+	    if (!(hdr->codecs[DS_BF] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'C' && key[1] == 'F') {+	    if (!(hdr->codecs[DS_CF] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'R' && key[1] == 'I') {+	    if (!(hdr->codecs[DS_RI] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'R' && key[1] == 'L') {+	    if (!(hdr->codecs[DS_RL] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'A' && key[1] == 'P') {+	    if (!(hdr->codecs[DS_AP] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'R' && key[1] == 'G') {+	    if (!(hdr->codecs[DS_RG] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'M' && key[1] == 'F') {+	    if (!(hdr->codecs[DS_MF] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'N' && key[1] == 'S') {+	    if (!(hdr->codecs[DS_NS] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'N' && key[1] == 'P') {+	    if (!(hdr->codecs[DS_NP] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'T' && key[1] == 'S') {+	    if (!(hdr->codecs[DS_TS] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'N' && key[1] == 'F') {+	    if (!(hdr->codecs[DS_NF] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'T' && key[1] == 'C') {+	    if (!(hdr->codecs[DS_TC] = cram_decoder_init(encoding, cp, size,+							 E_BYTE,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'T' && key[1] == 'N') {+	    if (!(hdr->codecs[DS_TN] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'F' && key[1] == 'N') {+	    if (!(hdr->codecs[DS_FN] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'F' && key[1] == 'C') {+	    if (!(hdr->codecs[DS_FC] = cram_decoder_init(encoding, cp, size,+							 E_BYTE,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'F' && key[1] == 'P') {+	    if (!(hdr->codecs[DS_FP] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'B' && key[1] == 'S') {+	    if (!(hdr->codecs[DS_BS] = cram_decoder_init(encoding, cp, size,+							 E_BYTE,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'I' && key[1] == 'N') {+	    if (!(hdr->codecs[DS_IN] = cram_decoder_init(encoding, cp, size,+							 E_BYTE_ARRAY,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'S' && key[1] == 'C') {+	    if (!(hdr->codecs[DS_SC] = cram_decoder_init(encoding, cp, size,+							 E_BYTE_ARRAY,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'D' && key[1] == 'L') {+	    if (!(hdr->codecs[DS_DL] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'B' && key[1] == 'A') {+	    if (!(hdr->codecs[DS_BA] = cram_decoder_init(encoding, cp, size,+							 E_BYTE,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'B' && key[1] == 'B') {+	    if (!(hdr->codecs[DS_BB] = cram_decoder_init(encoding, cp, size,+							 E_BYTE_ARRAY,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'R' && key[1] == 'S') {+	    if (!(hdr->codecs[DS_RS] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'P' && key[1] == 'D') {+	    if (!(hdr->codecs[DS_PD] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'H' && key[1] == 'C') {+	    if (!(hdr->codecs[DS_HC] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'M' && key[1] == 'Q') {+	    if (!(hdr->codecs[DS_MQ] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'R' && key[1] == 'N') {+	    if (!(hdr->codecs[DS_RN] = cram_decoder_init(encoding, cp, size,+							 E_BYTE_ARRAY_BLOCK,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'Q' && key[1] == 'S') {+	    if (!(hdr->codecs[DS_QS] = cram_decoder_init(encoding, cp, size,+							 E_BYTE,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'Q' && key[1] == 'Q') {+	    if (!(hdr->codecs[DS_QQ] = cram_decoder_init(encoding, cp, size,+							 E_BYTE_ARRAY,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'T' && key[1] == 'L') {+	    if (!(hdr->codecs[DS_TL] = cram_decoder_init(encoding, cp, size,+							 E_INT,+							 fd->version))) {+		cram_free_compression_header(hdr);+		return NULL;+	    }+	} else if (key[0] == 'T' && key[1] == 'M') {+	} else if (key[0] == 'T' && key[1] == 'V') {+	} else+	    fprintf(stderr, "Unrecognised key: %.2s\n", key);++	cp += size;++	m->next = hdr->rec_encoding_map[CRAM_MAP(key[0], key[1])];+	hdr->rec_encoding_map[CRAM_MAP(key[0], key[1])] = m;+    }+    if (cp - cp_copy != map_size) {+	cram_free_compression_header(hdr);+	return NULL;+    }++    /* Tag encoding map */+    cp += safe_itf8_get(cp, endp, &map_size); cp_copy = cp;+    cp += safe_itf8_get(cp, endp, &map_count);+    for (i = 0; i < map_count; i++) {+	int32_t encoding = E_NULL;+	int32_t size = 0;+	cram_map *m = malloc(sizeof(*m)); // FIXME: use pooled_alloc+	char *key;++	if (!m || endp - cp < 6) {+	    free(m);+	    cram_free_compression_header(hdr);+	    return NULL;+	}++	key = cp + 1;+	m->key = (key[0]<<16)|(key[1]<<8)|key[2];++	cp += 4; // Strictly ITF8, but this suffices+	cp += safe_itf8_get(cp, endp, &encoding);+	cp += safe_itf8_get(cp, endp, &size);++	m->encoding = encoding;+	m->size     = size;+	m->offset   = cp - (char *)b->data;+	if (size < 0 || endp - cp < size ||+	    !(m->codec = cram_decoder_init(encoding, cp, size,+					   E_BYTE_ARRAY_BLOCK, fd->version))) {+	    cram_free_compression_header(hdr);+	    free(m);+	    return NULL;+	}+	+	cp += size;++	m->next = hdr->tag_encoding_map[CRAM_MAP(key[0],key[1])];+	hdr->tag_encoding_map[CRAM_MAP(key[0],key[1])] = m;+    }+    if (cp - cp_copy != map_size) {+	cram_free_compression_header(hdr);+	return NULL;+    }++    return hdr;+}++/*+ * Note we also need to scan through the record encoding map to+ * see which data series share the same block, either external or+ * CORE. For example if we need the BF data series but MQ and CF+ * are also encoded in the same block then we need to add those in+ * as a dependency in order to correctly decode BF.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_dependent_data_series(cram_fd *fd,+			       cram_block_compression_hdr *hdr,+			       cram_slice *s) {+    int *block_used;+    int core_used = 0;+    int i;+    static int i_to_id[] = {+	DS_BF, DS_AP, DS_FP, DS_RL, DS_DL, DS_NF, DS_BA, DS_QS,+	DS_FC, DS_FN, DS_BS, DS_IN, DS_RG, DS_MQ, DS_TL, DS_RN,+	DS_NS, DS_NP, DS_TS, DS_MF, DS_CF, DS_RI, DS_RS, DS_PD,+	DS_HC, DS_SC, DS_BB, DS_QQ,+    };+    uint32_t orig_ds;++    /*+     * Set the data_series bit field based on fd->required_fields+     * contents.+     */+    if (fd->required_fields && fd->required_fields != INT_MAX) {+	hdr->data_series = 0;++	if (fd->required_fields & SAM_QNAME)+	    hdr->data_series |= CRAM_RN;++	if (fd->required_fields & SAM_FLAG)+	    hdr->data_series |= CRAM_BF;++	if (fd->required_fields & SAM_RNAME)+	    hdr->data_series |= CRAM_RI | CRAM_BF;++	if (fd->required_fields & SAM_POS)+	    hdr->data_series |= CRAM_AP | CRAM_BF;++	if (fd->required_fields & SAM_MAPQ)+	    hdr->data_series |= CRAM_MQ;++	if (fd->required_fields & SAM_CIGAR)+	    hdr->data_series |= CRAM_CIGAR;++	if (fd->required_fields & SAM_RNEXT)+	    hdr->data_series |= CRAM_CF | CRAM_NF | CRAM_RI | CRAM_NS |CRAM_BF;++	if (fd->required_fields & SAM_PNEXT)+	    hdr->data_series |= CRAM_CF | CRAM_NF | CRAM_AP | CRAM_NP | CRAM_BF;++	if (fd->required_fields & SAM_TLEN)+	    hdr->data_series |= CRAM_CF | CRAM_NF | CRAM_AP | CRAM_TS |+		CRAM_BF | CRAM_MF | CRAM_RI | CRAM_CIGAR;++	if (fd->required_fields & SAM_SEQ)+	    hdr->data_series |= CRAM_SEQ;++	if (!(fd->required_fields & SAM_AUX))+	    // No easy way to get MD/NM without other tags at present+	    fd->decode_md = 0;++	if (fd->required_fields & SAM_QUAL)+	    hdr->data_series |= CRAM_QUAL;++	if (fd->required_fields & SAM_AUX)+	    hdr->data_series |= CRAM_RG | CRAM_TL | CRAM_aux;++	if (fd->required_fields & SAM_RGAUX)+	    hdr->data_series |= CRAM_RG | CRAM_BF;++	// Always uncompress CORE block+	if (cram_uncompress_block(s->block[0]))+	    return -1;+    } else {+	hdr->data_series = CRAM_ALL;++	for (i = 0; i < s->hdr->num_blocks; i++) {+	    if (cram_uncompress_block(s->block[i]))+		return -1;+	}++	return 0;+    }++    block_used = calloc(s->hdr->num_blocks+1, sizeof(int));+    if (!block_used)+	return -1;++    do {+	/*+	 * Also set data_series based on code prerequisites. Eg if we need+	 * CRAM_QS then we also need to know CRAM_RL so we know how long it+	 * is, or if we need FC/FP then we also need FN (number of features).+	 *+	 * It's not reciprocal though. We may be needing to decode FN+	 * but have no need to decode FC, FP and cigar ops.+	 */+	if (hdr->data_series & CRAM_RS)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_PD)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_HC)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_QS)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_IN)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_SC)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_BS)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_DL)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_BA)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_BB)    hdr->data_series |= CRAM_FC|CRAM_FP;+	if (hdr->data_series & CRAM_QQ)    hdr->data_series |= CRAM_FC|CRAM_FP;++	// cram_decode_seq() needs seq[] array+	if (hdr->data_series & (CRAM_SEQ|CRAM_CIGAR)) hdr->data_series |= CRAM_RL;++	if (hdr->data_series & CRAM_FP)    hdr->data_series |= CRAM_FC;+	if (hdr->data_series & CRAM_FC)    hdr->data_series |= CRAM_FN;+	if (hdr->data_series & CRAM_aux)   hdr->data_series |= CRAM_TL;+	if (hdr->data_series & CRAM_MF)    hdr->data_series |= CRAM_CF;+	if (hdr->data_series & CRAM_MQ)    hdr->data_series |= CRAM_BF;+	if (hdr->data_series & CRAM_BS)    hdr->data_series |= CRAM_RI;+	if (hdr->data_series & (CRAM_MF |CRAM_NS |CRAM_NP |CRAM_TS |CRAM_NF))+	    hdr->data_series |= CRAM_CF;+	if (!hdr->read_names_included && hdr->data_series & CRAM_RN)+	    hdr->data_series |= CRAM_CF | CRAM_NF;+	if (hdr->data_series & (CRAM_BA | CRAM_QS | CRAM_BB | CRAM_QQ))+	    hdr->data_series |= CRAM_BF | CRAM_CF | CRAM_RL;++	orig_ds = hdr->data_series;++	// Find which blocks are in use.+	for (i = 0; i < sizeof(i_to_id)/sizeof(*i_to_id); i++) {+	    int bnum1, bnum2, j;+	    cram_codec *c = hdr->codecs[i_to_id[i]];++	    if (!(hdr->data_series & (1<<i)))+		continue;++	    if (!c)+		continue;++	    bnum1 = cram_codec_to_id(c, &bnum2);++	    for (;;) {+		switch (bnum1) {+		case -2:+		    break;++		case -1:+		    core_used = 1;+		    break;++		default:+		    for (j = 0; j < s->hdr->num_blocks; j++) {+			if (s->block[j]->content_type == EXTERNAL &&+			    s->block[j]->content_id == bnum1) {+			    block_used[j] = 1;+			    if (cram_uncompress_block(s->block[j])) {+				free(block_used);+				return -1;+			    }+			}+		    }+		    break;+		}++		if (bnum2 == -2 || bnum1 == bnum2)+		    break;++		bnum1 = bnum2; // 2nd pass+	    }+	}++	// Tags too+	if ((fd->required_fields & SAM_AUX) ||+	    (hdr->data_series & CRAM_aux)) {+	    for (i = 0; i < CRAM_MAP_HASH; i++) {+		int bnum1, bnum2, j;+		cram_map *m = hdr->tag_encoding_map[i];++		while (m) {+		    cram_codec *c = m->codec;+		    if (!c)+			continue;++		    bnum1 = cram_codec_to_id(c, &bnum2);++		    for (;;) {+			switch (bnum1) {+			case -2:+			    break;++			case -1:+			    core_used = 1;+			    break;++			default:+			    for (j = 0; j < s->hdr->num_blocks; j++) {+				if (s->block[j]->content_type == EXTERNAL &&+				    s->block[j]->content_id == bnum1) {+				    block_used[j] = 1;+				    if (cram_uncompress_block(s->block[j])) {+					free(block_used);+					return -1;+				    }+				}+			    }+			    break;+			}++			if (bnum2 == -2 || bnum1 == bnum2)+			    break;++			bnum1 = bnum2; // 2nd pass+		    }++		    m = m->next;+		}+	    }+	}++	// We now know which blocks are in used, so repeat and find+	// which other data series need to be added.+	for (i = 0; i < sizeof(i_to_id)/sizeof(*i_to_id); i++) {+	    int bnum1, bnum2, j;+	    cram_codec *c = hdr->codecs[i_to_id[i]];++	    if (!c)+		continue;++	    bnum1 = cram_codec_to_id(c, &bnum2);++	    for (;;) {+		switch (bnum1) {+		case -2:+		    break;++		case -1:+		    if (core_used) {+			//printf(" + data series %08x:\n", 1<<i);+			hdr->data_series |= 1<<i;+		    }+		    break;++		default:+		    for (j = 0; j < s->hdr->num_blocks; j++) {+			if (s->block[j]->content_type == EXTERNAL &&+			    s->block[j]->content_id == bnum1) {+			    if (block_used[j]) {+				//printf(" + data series %08x:\n", 1<<i);+				hdr->data_series |= 1<<i;+			    }+			}+		    }+		    break;+		}++		if (bnum2 == -2 || bnum1 == bnum2)+		    break;++		bnum1 = bnum2; // 2nd pass+	    }+	}++	// Tags too+	for (i = 0; i < CRAM_MAP_HASH; i++) {+	    int bnum1, bnum2, j;+	    cram_map *m = hdr->tag_encoding_map[i];++	    while (m) {+		cram_codec *c = m->codec;+		if (!c)+		    continue;++		bnum1 = cram_codec_to_id(c, &bnum2);+		+		for (;;) {+		    switch (bnum1) {+		    case -2:+			break;++		    case -1:+			//printf(" + data series %08x:\n", CRAM_aux);+			hdr->data_series |= CRAM_aux;+			break;++		    default:+			for (j = 0; j < s->hdr->num_blocks; j++) {+			    if (s->block[j]->content_type &&+				s->block[j]->content_id == bnum1) {+				if (block_used[j]) {+				    //printf(" + data series %08x:\n",+				    //       CRAM_aux);+				    hdr->data_series |= CRAM_aux;+				}+			    }+			}+			break;+		    }++		    if (bnum2 == -2 || bnum1 == bnum2)+			break;++		    bnum1 = bnum2; // 2nd pass+		}++		m = m->next;+	    }+	}+    } while (orig_ds != hdr->data_series);++    free(block_used);+    return 0;+}++/*+ * Checks whether an external block is used solely by a single data series.+ * Returns the codec type if so (EXTERNAL, BYTE_ARRAY_LEN, BYTE_ARRAY_STOP)+ *         or 0 if not (E_NULL).+ */+static int cram_ds_unique(cram_block_compression_hdr *hdr, cram_codec *c,+			  int id) {+    int i, n_id = 0;+    enum cram_encoding e_type = 0;++    for (i = 0; i < DS_END; i++) {+	cram_codec *c;+	int bnum1, bnum2, old_n_id;++	if (!(c = hdr->codecs[i]))+	    continue;++	bnum1 = cram_codec_to_id(c, &bnum2);++	old_n_id = n_id;+	if (bnum1 == id) {+	    n_id++;+	    e_type = c->codec;+	}+	if (bnum2 == id) {+	    n_id++;+	    e_type = c->codec;+	}++	if (n_id == old_n_id+2)+	    n_id--; // len/val in same place counts once only.+    }++    return n_id == 1 ? e_type : 0;+}++/*+ * Attempts to estimate the size of some blocks so we can preallocate them+ * before decoding.  Although decoding will automatically grow the blocks,+ * it is typically more efficient to preallocate.+ */+void cram_decode_estimate_sizes(cram_block_compression_hdr *hdr, cram_slice *s,+				int *qual_size, int *name_size,+				int *q_id) {+    int bnum1, bnum2;+    cram_codec *cd;++    *qual_size = 0;+    *name_size = 0;++    /* Qual */+    cd = hdr->codecs[DS_QS];+    bnum1 = cram_codec_to_id(cd, &bnum2);+    if (bnum1 < 0 && bnum2 >= 0) bnum1 = bnum2;+    if (cram_ds_unique(hdr, cd, bnum1)) {+	cram_block *b = cram_get_block_by_id(s, bnum1);+	if (b) *qual_size = b->uncomp_size;+	if (q_id && cd->codec == E_EXTERNAL)+	    *q_id = bnum1;+    }++    /* Name */+    cd = hdr->codecs[DS_RN];+    bnum1 = cram_codec_to_id(cd, &bnum2);+    if (bnum1 < 0 && bnum2 >= 0) bnum1 = bnum2;+    if (cram_ds_unique(hdr, cd, bnum1)) {+	cram_block *b = cram_get_block_by_id(s, bnum1);+	if (b) *name_size = b->uncomp_size;+    }+}+++/* ----------------------------------------------------------------------+ * CRAM slices+ */++/*+ * Decodes a CRAM (un)mapped slice header block.+ * Returns slice header ptr on success+ *         NULL on failure+ */+cram_block_slice_hdr *cram_decode_slice_header(cram_fd *fd, cram_block *b) {+    cram_block_slice_hdr *hdr;+    char *cp;+    char *cp_end;+    int i;++    if (b->method != RAW) {+        /* Spec. says slice header should be RAW, but we can future-proof+	   by trying to decode it if it isn't. */+        if (cram_uncompress_block(b) < 0)+            return NULL;+    }+    cp =  (char *)b->data;+    cp_end = cp + b->uncomp_size;++    if (b->content_type != MAPPED_SLICE &&+	b->content_type != UNMAPPED_SLICE)+	return NULL;++    if (!(hdr  = calloc(1, sizeof(*hdr))))+	return NULL;++    hdr->content_type = b->content_type;++    if (b->content_type == MAPPED_SLICE) {+        cp += safe_itf8_get(cp,  cp_end, &hdr->ref_seq_id);+        cp += safe_itf8_get(cp,  cp_end, &hdr->ref_seq_start);+        cp += safe_itf8_get(cp,  cp_end, &hdr->ref_seq_span);+    }+    cp += safe_itf8_get(cp,  cp_end, &hdr->num_records);+    hdr->record_counter = 0;+    if (CRAM_MAJOR_VERS(fd->version) == 2) {+	int32_t i32 = 0;+	cp += safe_itf8_get(cp, cp_end, &i32);+	hdr->record_counter = i32;+    } else if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	cp += ltf8_get(cp, &hdr->record_counter);+    }++    cp += safe_itf8_get(cp, cp_end, &hdr->num_blocks);++    cp += safe_itf8_get(cp, cp_end, &hdr->num_content_ids);+    if (hdr->num_content_ids < 1 ||+	hdr->num_content_ids >= SIZE_MAX / sizeof(int32_t)) {+        /* Slice must have at least one data block,+	   and malloc'd size shouldn't wrap. */+        free(hdr);+        return NULL;+    }+    hdr->block_content_ids = malloc(hdr->num_content_ids * sizeof(int32_t));+    if (!hdr->block_content_ids) {+	free(hdr);+	return NULL;+    }++    for (i = 0; i < hdr->num_content_ids; i++) {+	int l = safe_itf8_get(cp, cp_end,+			      &hdr->block_content_ids[i]);+	if (l <= 0) {+	    free(hdr->block_content_ids);+	    free(hdr);+	    return NULL;+	}+	cp += l;+    }++    if (b->content_type == MAPPED_SLICE) {+        cp += safe_itf8_get(cp,  cp_end, &hdr->ref_base_id);+    }++    if (CRAM_MAJOR_VERS(fd->version) != 1) {+        if (cp_end - cp < 16) {+            free(hdr->block_content_ids);+            free(hdr);+            return NULL;+        }+	memcpy(hdr->md5, cp, 16);+    } else {+	memset(hdr->md5, 0, 16);+    }++    return hdr;+}+++#if 0+/* Returns the number of bits set in val; it the highest bit used */+static int nbits(int v) {+    static const int MultiplyDeBruijnBitPosition[32] = {+	1, 10, 2, 11, 14, 22, 3, 30, 12, 15, 17, 19, 23, 26, 4, 31,+	9, 13, 21, 29, 16, 18, 25, 8, 20, 28, 24, 7, 27, 6, 5, 32+    };++    v |= v >> 1; // first up to set all bits 1 after the first 1 */+    v |= v >> 2;+    v |= v >> 4;+    v |= v >> 8;+    v |= v >> 16;++    // DeBruijn magic to find top bit+    return MultiplyDeBruijnBitPosition[(uint32_t)(v * 0x07C4ACDDU) >> 27];+}+#endif++#if 0+static int sort_freqs(const void *vp1, const void *vp2) {+    const int i1 = *(const int *)vp1;+    const int i2 = *(const int *)vp2;+    return i1-i2;+}+#endif++/* ----------------------------------------------------------------------+ * Primary CRAM sequence decoder+ */++/*+ * Internal part of cram_decode_slice().+ * Generates the sequence, quality and cigar components.+ */+static int cram_decode_seq(cram_fd *fd, cram_container *c, cram_slice *s,+			   cram_block *blk, cram_record *cr, SAM_hdr *bfd,+			   int cf, char *seq, char *qual,+			   int has_MD, int has_NM) {+    int prev_pos = 0, f, r = 0, out_sz = 1;+    int seq_pos = 1;+    int cig_len = 0, ref_pos = cr->apos;+    int32_t fn, i32;+    enum cigar_op cig_op = BAM_CMATCH;+    uint32_t *cigar = s->cigar;+    uint32_t ncigar = s->ncigar;+    uint32_t cigar_alloc = s->cigar_alloc;+    uint32_t nm = 0;+    int32_t md_dist = 0;+    int orig_aux = 0;+    int decode_md = fd->decode_md && s->ref && !has_MD;+    int decode_nm = fd->decode_md && s->ref && !has_NM;+    uint32_t ds = c->comp_hdr->data_series;++    if ((ds & CRAM_QS) && !(cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {+	memset(qual, 255, cr->len);+    }++    if (cr->cram_flags & CRAM_FLAG_NO_SEQ)+	decode_md = decode_nm = 0;++    if (decode_md) {+	orig_aux = BLOCK_SIZE(s->aux_blk);+	BLOCK_APPEND(s->aux_blk, "MDZ", 3);+    }+    +    if (ds & CRAM_FN) {+	if (!c->comp_hdr->codecs[DS_FN]) return -1;+	r |= c->comp_hdr->codecs[DS_FN]->decode(s,c->comp_hdr->codecs[DS_FN],+						blk, (char *)&fn, &out_sz);+        if (r) return r;+    } else {+	fn = 0;+    }++    ref_pos--; // count from 0+    cr->cigar = ncigar;++    if (!(ds & (CRAM_FC | CRAM_FP)))+	goto skip_cigar;++    for (f = 0; f < fn; f++) {+	int32_t pos = 0;+	char op;++	if (ncigar+2 >= cigar_alloc) {+	    cigar_alloc = cigar_alloc ? cigar_alloc*2 : 1024;+	    s->cigar = cigar;+	    if (!(cigar = realloc(cigar, cigar_alloc * sizeof(*cigar))))+		return -1;+	}++	if (ds & CRAM_FC) {+	    if (!c->comp_hdr->codecs[DS_FC]) return -1;+	    r |= c->comp_hdr->codecs[DS_FC]->decode(s,+						    c->comp_hdr->codecs[DS_FC],+						    blk,+						    &op,  &out_sz);+	    if (r) return r;+	}++	if (!(ds & CRAM_FP))+	    continue;++	if (!c->comp_hdr->codecs[DS_FP]) return -1;+	r |= c->comp_hdr->codecs[DS_FP]->decode(s,+						c->comp_hdr->codecs[DS_FP],+						blk,+						(char *)&pos, &out_sz);+	if (r) return r;+	pos += prev_pos;++	if (pos <= 0) {+	    fprintf(stderr, "Error: feature position %d before start of read.\n",+		    pos);+	    return -1;+	}++	if (pos > seq_pos) {+	    if (pos > cr->len+1)+		return -1;++	    if (s->ref && cr->ref_id >= 0) {+		if (ref_pos + pos - seq_pos > bfd->ref[cr->ref_id].len) {+		    static int whinged = 0;+		    int rlen;+		    if (!whinged)+			fprintf(stderr, "Ref pos outside of ref "+				"sequence boundary\n");+		    whinged = 1;+		    rlen = bfd->ref[cr->ref_id].len - ref_pos;+		    if (rlen > 0) {+			memcpy(&seq[seq_pos-1],+			       &s->ref[ref_pos - s->ref_start +1], rlen);+			if ((pos - seq_pos) - rlen > 0)+			    memset(&seq[seq_pos-1+rlen], 'N',+				   (pos - seq_pos) - rlen);+		    } else {+		        memset(&seq[seq_pos-1], 'N', cr->len - seq_pos + 1);+		    }+		} else {+		    memcpy(&seq[seq_pos-1], &s->ref[ref_pos - s->ref_start +1],+			   pos - seq_pos);+		}+	    }+#ifdef USE_X+	    if (cig_len && cig_op != BAM_CBASE_MATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    cig_op = BAM_CBASE_MATCH;+#else+	    if (cig_len && cig_op != BAM_CMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    cig_op = BAM_CMATCH;+#endif+	    cig_len += pos - seq_pos;+	    ref_pos += pos - seq_pos;+	    if (md_dist >= 0)+		md_dist += pos - seq_pos;+	    seq_pos = pos;+	}++	prev_pos = pos;++	if (!(ds & CRAM_FC))+	    goto skip_cigar;++	if (!(ds & CRAM_FC))+	    continue;++	switch(op) {+	case 'S': { // soft clip: IN+	    int32_t out_sz2 = 1;+	    int have_sc = 0;++	    if (cig_len) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    switch (CRAM_MAJOR_VERS(fd->version)) {+	    case 1:+	        if (ds & CRAM_IN) {+		    r |= c->comp_hdr->codecs[DS_IN]+			? c->comp_hdr->codecs[DS_IN]+			             ->decode(s, c->comp_hdr->codecs[DS_IN],+					      blk,+					      cr->len ? &seq[pos-1] : NULL,+					      &out_sz2)+			: (seq[pos-1] = 'N', out_sz2 = 1, 0);+		    have_sc = 1;+		}+		break;+	    case 2:+	    default:+	        if (ds & CRAM_SC) {+		    r |= c->comp_hdr->codecs[DS_SC]+			? c->comp_hdr->codecs[DS_SC]+			             ->decode(s, c->comp_hdr->codecs[DS_SC],+					      blk,+					      cr->len ? &seq[pos-1] : NULL,+					      &out_sz2)+			: (seq[pos-1] = 'N', out_sz2 = 1, 0);+		    have_sc = 1;+		}+		break;++//		default:+//		    r |= c->comp_hdr->codecs[DS_BB]+//			? c->comp_hdr->codecs[DS_BB]+//			             ->decode(s, c->comp_hdr->codecs[DS_BB],+//					      blk, &seq[pos-1], &out_sz2)+//			: (seq[pos-1] = 'N', out_sz2 = 1, 0);+	    }+	    if (have_sc) {+		if (r) return r;+		cigar[ncigar++] = (out_sz2<<4) + BAM_CSOFT_CLIP;+		cig_op = BAM_CSOFT_CLIP;+		seq_pos += out_sz2;+	    }+	    break;+	}++	case 'X': { // Substitution; BS+	    unsigned char base;+#ifdef USE_X+	    if (cig_len && cig_op != BAM_CBASE_MISMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_BS) {+		if (!c->comp_hdr->codecs[DS_BS]) return -1;+		r |= c->comp_hdr->codecs[DS_BS]+		                ->decode(s, c->comp_hdr->codecs[DS_BS], blk,+					 (char *)&base, &out_sz);+		if (pos-1 < cr->len)+		    seq[pos-1] = 'N'; // FIXME look up BS=base value+	    }+	    cig_op = BAM_CBASE_MISMATCH;+#else+	    int ref_base;+	    if (cig_len && cig_op != BAM_CMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_BS) {+		if (!c->comp_hdr->codecs[DS_BS]) return -1;+		r |= c->comp_hdr->codecs[DS_BS]+		                ->decode(s, c->comp_hdr->codecs[DS_BS], blk,+					 (char *)&base, &out_sz);+		if (r) return -1;+		if (ref_pos >= bfd->ref[cr->ref_id].len || !s->ref) {+		    if (pos-1 < cr->len)+			seq[pos-1] = c->comp_hdr->+			    substitution_matrix[fd->L1['N']][base];+		    if (decode_md || decode_nm) {+			if (md_dist >= 0 && decode_md)+			    BLOCK_APPEND_UINT(s->aux_blk, md_dist);+			md_dist = -1;+			nm--;+		    }+		} else {+		    unsigned char ref_call = ref_pos <= s->ref_end +			? (uc)s->ref[ref_pos - s->ref_start +1]+			: 'N';+		    ref_base = fd->L1[ref_call];+		    if (pos-1 < cr->len)+			seq[pos-1] = c->comp_hdr->+			    substitution_matrix[ref_base][base];+		    if (decode_md) {+			BLOCK_APPEND_UINT(s->aux_blk, md_dist);+			BLOCK_APPEND_CHAR(s->aux_blk, ref_call);+			md_dist = 0;+		    }+		}+	    }+	    cig_op = BAM_CMATCH;+#endif+	    nm++;+	    cig_len++;+	    seq_pos++;+	    ref_pos++;+	    break;+	}++	case 'D': { // Deletion; DL+	    if (cig_len && cig_op != BAM_CDEL) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_DL) {+		if (!c->comp_hdr->codecs[DS_DL]) return -1;+		r |= c->comp_hdr->codecs[DS_DL]+		                ->decode(s, c->comp_hdr->codecs[DS_DL], blk,+					 (char *)&i32, &out_sz);+		if (r) return r;+		if (decode_md || decode_nm) {+		    if (md_dist >= 0 && decode_md)+			BLOCK_APPEND_UINT(s->aux_blk, md_dist);+		    if (ref_pos + i32 <= bfd->ref[cr->ref_id].len) {+			if (decode_md) {+			    BLOCK_APPEND_CHAR(s->aux_blk, '^');+			    BLOCK_APPEND(s->aux_blk,+					 &s->ref[ref_pos - s->ref_start +1],+					 i32);+			    md_dist = 0;+			}+			nm += i32;+		    } else {+			uint32_t dlen;+			if (bfd->ref[cr->ref_id].len >= ref_pos) {+			    if (decode_md) {+				BLOCK_APPEND_CHAR(s->aux_blk, '^');+				BLOCK_APPEND(s->aux_blk,+					     &s->ref[ref_pos - s->ref_start+1],+					     bfd->ref[cr->ref_id].len-ref_pos);+				BLOCK_APPEND_UINT(s->aux_blk, 0);+			    }+			    dlen = i32 - (bfd->ref[cr->ref_id].len - ref_pos);+			    nm += i32 - dlen;+			} else {+			    dlen = i32;+			}++			md_dist = -1;+		    }+		}+		cig_op = BAM_CDEL;+		cig_len += i32;+		ref_pos += i32;+		//printf("  %d: DL = %d (ret %d)\n", f, i32, r);+	    }+	    break;+	}++	case 'I': { // Insertion (several bases); IN+	    int32_t out_sz2 = 1;++	    if (cig_len && cig_op != BAM_CINS) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }++	    if (ds & CRAM_IN) {+		if (!c->comp_hdr->codecs[DS_IN]) return -1;+		r |= c->comp_hdr->codecs[DS_IN]+		                ->decode(s, c->comp_hdr->codecs[DS_IN], blk,+					 cr->len ? &seq[pos-1] : NULL,+					 &out_sz2);+		if (r) return r;+		cig_op = BAM_CINS;+		cig_len += out_sz2;+		seq_pos += out_sz2;+		nm      += out_sz2;+		//printf("  %d: IN(I) = %.*s (ret %d, out_sz %d)\n", f, out_sz2, dat, r, out_sz2);+	    }+	    break;+	}++	case 'i': { // Insertion (single base); BA+	    if (cig_len && cig_op != BAM_CINS) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_BA) {+		if (!c->comp_hdr->codecs[DS_BA]) return -1;+		r |= c->comp_hdr->codecs[DS_BA]+		                ->decode(s, c->comp_hdr->codecs[DS_BA], blk,+					 cr->len ? &seq[pos-1] : NULL,+					 &out_sz);+		if (r) return r;+	    }+	    cig_op = BAM_CINS;+	    cig_len++;+	    seq_pos++;+	    nm++;+	    break;+	}++	case 'b': { // Several bases+	    int32_t len = 1;++	    if (cig_len && cig_op != BAM_CMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }++	    if (ds & CRAM_BB) {+		if (!c->comp_hdr->codecs[DS_BB]) return -1;+		r |= c->comp_hdr->codecs[DS_BB]+		    ->decode(s, c->comp_hdr->codecs[DS_BB], blk,+			     cr->len ? &seq[pos-1] : NULL,+			     &len);+		if (r) return r;++		if (decode_md || decode_nm) {+		    int x;+		    if (md_dist >= 0 && decode_md)+			BLOCK_APPEND_UINT(s->aux_blk, md_dist);++		    for (x = 0; x < len; x++) {+			if (x && decode_md)+			    BLOCK_APPEND_UINT(s->aux_blk, 0);+			if (ref_pos+x >= bfd->ref[cr->ref_id].len || !s->ref) {+			    md_dist = -1;+			    break;+			} else {+			    if (decode_md) {+				char r = s->ref[ref_pos+x-s->ref_start +1];+				BLOCK_APPEND_CHAR(s->aux_blk, r);+			    }+			}+		    }++		    nm += x;+		}+	    }++	    cig_op = BAM_CMATCH;++	    cig_len+=len;+	    seq_pos+=len;+	    ref_pos+=len;+	    //prev_pos+=len;+	    break;+	}++	case 'q': { // Several quality values+	    int32_t len = 1;++	    if (cig_len && cig_op != BAM_CMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }++	    if (ds & CRAM_QQ) {+		if (!c->comp_hdr->codecs[DS_QQ]) return -1;+		r |= c->comp_hdr->codecs[DS_QQ]+		    ->decode(s, c->comp_hdr->codecs[DS_QQ], blk,+			     (char *)&qual[pos-1], &len);+		if (r) return r;+	    }++	    cig_op = BAM_CMATCH;++	    cig_len+=len;+	    seq_pos+=len;+	    ref_pos+=len;+	    //prev_pos+=len;+	    break;+	}++	case 'B': { // Read base; BA, QS+#ifdef USE_X+	    if (cig_len && cig_op != BAM_CBASE_MISMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+#else+	    if (cig_len && cig_op != BAM_CMATCH) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+#endif+	    if (ds & CRAM_BA) {+		if (!c->comp_hdr->codecs[DS_BA]) return -1;+		r |= c->comp_hdr->codecs[DS_BA]+		                ->decode(s, c->comp_hdr->codecs[DS_BA], blk,+					 cr->len ? &seq[pos-1] : NULL,+					 &out_sz);++		if (decode_md || decode_nm) {+		    if (md_dist >= 0 && decode_md)+			BLOCK_APPEND_UINT(s->aux_blk, md_dist);+		    if (ref_pos >= bfd->ref[cr->ref_id].len || !s->ref) {+			md_dist = -1;+		    } else {+			if (decode_md)+			    BLOCK_APPEND_CHAR(s->aux_blk,+					      s->ref[ref_pos-s->ref_start +1]);+			nm++;+			md_dist = 0;+		    }+		}+	    }+	    if (ds & CRAM_QS) {+		if (!c->comp_hdr->codecs[DS_QS]) return -1;+		r |= c->comp_hdr->codecs[DS_QS]+		                ->decode(s, c->comp_hdr->codecs[DS_QS], blk,+					 (char *)&qual[pos-1], &out_sz);+	    }+#ifdef USE_X+	    cig_op = BAM_CBASE_MISMATCH;+#else+	    cig_op = BAM_CMATCH;+#endif+	    cig_len++;+	    seq_pos++;+	    ref_pos++;+	    //printf("  %d: BA/QS(B) = %c/%d (ret %d)\n", f, i32, qc, r);+	    break;+	}++	case 'Q': { // Quality score; QS+	    if (ds & CRAM_QS) {+		if (!c->comp_hdr->codecs[DS_QS]) return -1;+		r |= c->comp_hdr->codecs[DS_QS]+		                ->decode(s, c->comp_hdr->codecs[DS_QS], blk,+					 (char *)&qual[pos-1], &out_sz);+		//printf("  %d: QS = %d (ret %d)\n", f, qc, r);+	    }+	    break;+	}++	case 'H': { // hard clip; HC+	    if (cig_len && cig_op != BAM_CHARD_CLIP) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_HC) {+		if (!c->comp_hdr->codecs[DS_HC]) return -1;+		r |= c->comp_hdr->codecs[DS_HC]+		                ->decode(s, c->comp_hdr->codecs[DS_HC], blk,+					 (char *)&i32, &out_sz);+		if (r) return r;+		cig_op = BAM_CHARD_CLIP;+		cig_len += i32;+	    }+	    break;+	}++	case 'P': { // padding; PD+	    if (cig_len && cig_op != BAM_CPAD) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_PD) {+		if (!c->comp_hdr->codecs[DS_PD]) return -1;+		r |= c->comp_hdr->codecs[DS_PD]+		                ->decode(s, c->comp_hdr->codecs[DS_PD], blk,+					 (char *)&i32, &out_sz);+		if (r) return r;+		cig_op = BAM_CPAD;+		cig_len += i32;+		nm      += i32;+	    }+	    break;+	}++	case 'N': { // Ref skip; RS+	    if (cig_len && cig_op != BAM_CREF_SKIP) {+		cigar[ncigar++] = (cig_len<<4) + cig_op;+		cig_len = 0;+	    }+	    if (ds & CRAM_RS) {+		if (!c->comp_hdr->codecs[DS_RS]) return -1;+		r |= c->comp_hdr->codecs[DS_RS]+		                ->decode(s, c->comp_hdr->codecs[DS_RS], blk,+					 (char *)&i32, &out_sz);+		if (r) return r;+		cig_op = BAM_CREF_SKIP;+		cig_len += i32;+		ref_pos += i32;+		nm      += i32;+	    }+	    break;+	}++	default:+            fprintf(stderr, "Error: Unknown feature code '%c'\n", op);+	    return -1;+	}+    }++    if (!(ds & CRAM_FC))+	goto skip_cigar;++    /* An implicit match op for any unaccounted for bases */+    if ((ds & CRAM_FN) && cr->len >= seq_pos) {+	if (s->ref) {+	    if (ref_pos + cr->len - seq_pos + 1 > bfd->ref[cr->ref_id].len) {+		static int whinged = 0;+		int rlen;+		if (!whinged)+		    fprintf(stderr, "Ref pos outside of ref sequence boundary\n");+		whinged = 1;+		rlen = bfd->ref[cr->ref_id].len - ref_pos;+		if (rlen > 0) {+		    if (seq_pos-1 + rlen < cr->len)+			memcpy(&seq[seq_pos-1],+			       &s->ref[ref_pos - s->ref_start +1], rlen);+		    if ((cr->len - seq_pos + 1) - rlen > 0)+		        memset(&seq[seq_pos-1+rlen], 'N',+                               (cr->len - seq_pos + 1) - rlen);+		} else {+		    if (cr->len - seq_pos + 1 > 0)+			memset(&seq[seq_pos-1], 'N', cr->len - seq_pos + 1);+		}+	    } else {+		if (cr->len - seq_pos + 1 > 0)+		    memcpy(&seq[seq_pos-1], &s->ref[ref_pos - s->ref_start +1],+			   cr->len - seq_pos + 1);+		ref_pos += cr->len - seq_pos + 1;+		if (md_dist >= 0)+		    md_dist += cr->len - seq_pos + 1;+	    }+	}++	if (ncigar+1 >= cigar_alloc) {+	    cigar_alloc = cigar_alloc ? cigar_alloc*2 : 1024;+	    s->cigar = cigar;+	    if (!(cigar = realloc(cigar, cigar_alloc * sizeof(*cigar))))+		return -1;+	}+#ifdef USE_X+	if (cig_len && cig_op != BAM_CBASE_MATCH) {+	    cigar[ncigar++] = (cig_len<<4) + cig_op;+	    cig_len = 0;+	}+	cig_op = BAM_CBASE_MATCH;+#else+	if (cig_len && cig_op != BAM_CMATCH) {+	    cigar[ncigar++] = (cig_len<<4) + cig_op;+	    cig_len = 0;+	}+	cig_op = BAM_CMATCH;+#endif+	cig_len += cr->len - seq_pos+1;+    }++ skip_cigar:++    if ((ds & CRAM_FN) && decode_md) {+	if (md_dist >= 0)+	    BLOCK_APPEND_UINT(s->aux_blk, md_dist);+    }++    if (cig_len) {+	if (ncigar >= cigar_alloc) {+	    cigar_alloc = cigar_alloc ? cigar_alloc*2 : 1024;+	    s->cigar = cigar;+	    if (!(cigar = realloc(cigar, cigar_alloc * sizeof(*cigar))))+		return -1;+	}++	cigar[ncigar++] = (cig_len<<4) + cig_op;+    }++    cr->ncigar = ncigar - cr->cigar;+    cr->aend = ref_pos;++    //printf("2: %.*s %d .. %d\n", cr->name_len, DSTRING_STR(name_ds) + cr->name, cr->apos, ref_pos);++    if (ds & CRAM_MQ) {+	if (!c->comp_hdr->codecs[DS_MQ]) return -1;+	r |= c->comp_hdr->codecs[DS_MQ]+	                ->decode(s, c->comp_hdr->codecs[DS_MQ], blk,+				 (char *)&cr->mqual, &out_sz);+    } else {+	cr->mqual = 40;+    }++    if ((ds & CRAM_QS) && (cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {+	int32_t out_sz2 = cr->len;++	if (ds & CRAM_QS) {+	    if (!c->comp_hdr->codecs[DS_QS]) return -1;+	    r |= c->comp_hdr->codecs[DS_QS]+		            ->decode(s, c->comp_hdr->codecs[DS_QS], blk,+				     qual, &out_sz2);+	}+    }++    s->cigar = cigar;+    s->cigar_alloc = cigar_alloc;+    s->ncigar = ncigar;++    if (cr->cram_flags & CRAM_FLAG_NO_SEQ)+	cr->len = 0;++    if (decode_md) {+	BLOCK_APPEND_CHAR(s->aux_blk, '\0'); // null terminate MD:Z:+	cr->aux_size += BLOCK_SIZE(s->aux_blk) - orig_aux;+    }++    if (decode_nm) {+	char buf[7];+	buf[0] = 'N'; buf[1] = 'M'; buf[2] = 'I';+	buf[3] = (nm>> 0) & 0xff;+	buf[4] = (nm>> 8) & 0xff;+	buf[5] = (nm>>16) & 0xff;+	buf[6] = (nm>>24) & 0xff;+	BLOCK_APPEND(s->aux_blk, buf, 7);+	cr->aux_size += 7;+    }++    return r;+}++/*+ * Quick and simple hash lookup for cram_map arrays+ */+static cram_map *map_find(cram_map **map, unsigned char *key, int id) {+    cram_map *m;++    m = map[CRAM_MAP(key[0],key[1])];+    while (m && m->key != id)+	m= m->next;++    return m;+}++//#define map_find(M,K,I) M[CRAM_MAP(K[0],K[1])];while (m && m->key != I);m= m->next+++static int cram_decode_aux_1_0(cram_container *c, cram_slice *s,+			       cram_block *blk, cram_record *cr) {+    int i, r = 0, out_sz = 1;+    unsigned char ntags;+	    +    if (!c->comp_hdr->codecs[DS_TC]) return -1;+    r |= c->comp_hdr->codecs[DS_TC]->decode(s, c->comp_hdr->codecs[DS_TC], blk,+					    (char *)&ntags, &out_sz);+    cr->ntags = ntags;++    //printf("TC=%d\n", cr->ntags);+    cr->aux_size = 0;+    cr->aux = BLOCK_SIZE(s->aux_blk);++    for (i = 0; i < cr->ntags; i++) {+	int32_t id, out_sz = 1;+	unsigned char tag_data[3];+	cram_map *m;++	//printf("Tag %d/%d\n", i+1, cr->ntags);+	if (!c->comp_hdr->codecs[DS_TN]) return -1;+	r |= c->comp_hdr->codecs[DS_TN]->decode(s, c->comp_hdr->codecs[DS_TN],+						blk, (char *)&id, &out_sz);+	if (out_sz == 3) {+	    tag_data[0] = ((char *)&id)[0];+	    tag_data[1] = ((char *)&id)[1];+	    tag_data[2] = ((char *)&id)[2];+	} else {+	    tag_data[0] = (id>>16) & 0xff;+	    tag_data[1] = (id>>8)  & 0xff;+	    tag_data[2] = id       & 0xff;+	} ++	m = map_find(c->comp_hdr->tag_encoding_map, tag_data, id);+	if (!m)+	    return -1;+	BLOCK_APPEND(s->aux_blk, (char *)tag_data, 3);++	if (!m->codec) return -1;+	r |= m->codec->decode(s, m->codec, blk, (char *)s->aux_blk, &out_sz);++	cr->aux_size += out_sz + 3;+    }+    +    return r;+}++static int cram_decode_aux(cram_container *c, cram_slice *s,+			   cram_block *blk, cram_record *cr,+			   int *has_MD, int *has_NM) {+    int i, r = 0, out_sz = 1;+    int32_t TL = 0;+    unsigned char *TN;+    uint32_t ds = c->comp_hdr->data_series;+	    +    if (!(ds & (CRAM_TL|CRAM_aux))) {+	cr->aux = 0;+	cr->aux_size = 0;+	return 0;+    }++    if (!c->comp_hdr->codecs[DS_TL]) return -1;+    r |= c->comp_hdr->codecs[DS_TL]->decode(s, c->comp_hdr->codecs[DS_TL], blk,+					    (char *)&TL, &out_sz);+    if (r || TL < 0 || TL >= c->comp_hdr->nTL)+	return -1;++    TN = c->comp_hdr->TL[TL];+    cr->ntags = strlen((char *)TN)/3; // optimise to remove strlen++    //printf("TC=%d\n", cr->ntags);+    cr->aux_size = 0;+    cr->aux = BLOCK_SIZE(s->aux_blk);++    if (!(ds & CRAM_aux))+	return 0;++    for (i = 0; i < cr->ntags; i++) {+	int32_t id, out_sz = 1;+	unsigned char tag_data[3];+	cram_map *m;++	if (TN[0] == 'M' && TN[1] == 'D' && has_MD)+	    *has_MD = 1;+	if (TN[0] == 'N' && TN[1] == 'M' && has_NM)+	    *has_NM = 1;++	//printf("Tag %d/%d\n", i+1, cr->ntags);+	tag_data[0] = *TN++;+	tag_data[1] = *TN++;+	tag_data[2] = *TN++;+	id = (tag_data[0]<<16) | (tag_data[1]<<8) | tag_data[2];++	m = map_find(c->comp_hdr->tag_encoding_map, tag_data, id);+	if (!m)+	    return -1;+	BLOCK_APPEND(s->aux_blk, (char *)tag_data, 3);++	if (!m->codec) return -1;+	r |= m->codec->decode(s, m->codec, blk, (char *)s->aux_blk, &out_sz);+	if (r) break;+	cr->aux_size += out_sz + 3;+    }+    +    return r;+}++/* Resolve mate pair cross-references between recs within this slice */+static int cram_decode_slice_xref(cram_slice *s, int required_fields) {+    int rec;++    if (!(required_fields & (SAM_RNEXT | SAM_PNEXT | SAM_TLEN))) {+	for (rec = 0; rec < s->hdr->num_records; rec++) {+	    cram_record *cr = &s->crecs[rec];++	    cr->tlen = 0;+	    cr->mate_pos = 0;+	    cr->mate_ref_id = -1;+	}++	return 0;+    }++    for (rec = 0; rec < s->hdr->num_records; rec++) {+	cram_record *cr = &s->crecs[rec];++	if (cr->mate_line >= 0) {+	    if (cr->mate_line < s->hdr->num_records) {+		/*+		 * On the first read, loop through computing lengths.+		 * It's not perfect as we have one slice per reference so we+		 * cannot detect when TLEN should be zero due to seqs that+		 * map to multiple references.+		 *+		 * We also cannot set tlen correct when it spans a slice for+		 * other reasons. This may make tlen too small. Should we+		 * fix this by forcing TLEN to be stored verbatim in such cases?+		 *+		 * Or do we just admit defeat and output 0 for tlen? It's the+		 * safe option...+		 */+		if (cr->tlen == INT_MIN) {+		    int id1 = rec, id2 = rec;+		    int aleft = cr->apos, aright = cr->aend;+		    int tlen;+		    int ref = cr->ref_id;++		    // number of segments starting at the same point.+		    int left_cnt = 0;++		    do {+			if (aleft > s->crecs[id2].apos)+			    aleft = s->crecs[id2].apos, left_cnt = 1;+			else if (aleft == s->crecs[id2].apos)+			    left_cnt++;+			if (aright < s->crecs[id2].aend)+			    aright = s->crecs[id2].aend;+			if (s->crecs[id2].mate_line == -1) {+			    s->crecs[id2].mate_line = rec;+			    break;+			}+			if (s->crecs[id2].mate_line <= id2 ||+                            s->crecs[id2].mate_line >= s->hdr->num_records)+			    return -1;+			id2 = s->crecs[id2].mate_line;++			if (s->crecs[id2].ref_id != ref)+			    ref = -1;+		    } while (id2 != id1);++		    if (ref != -1) {+			tlen = aright - aleft + 1;+			id1 = id2 = rec;++			/*+			 * When we have two seqs with identical start and+			 * end coordinates, set +/- tlen based on 1st/last+			 * bit flags instead, as a tie breaker.+			 */+			if (s->crecs[id2].apos == aleft) {+			    if (left_cnt == 1 || +				(s->crecs[id2].flags & BAM_FREAD1))+				s->crecs[id2].tlen = tlen;+			    else+				s->crecs[id2].tlen = -tlen;+			} else {+			    s->crecs[id2].tlen = -tlen;+			}++			id2 = s->crecs[id2].mate_line;+			while (id2 != id1) {+			    if (s->crecs[id2].apos == aleft) {+				if (left_cnt == 1 || +				    (s->crecs[id2].flags & BAM_FREAD1))+				    s->crecs[id2].tlen = tlen;+				else+				    s->crecs[id2].tlen = -tlen;+			    } else {+				s->crecs[id2].tlen = -tlen;+			    }+			    id2 = s->crecs[id2].mate_line;+			}+		    } else {+			id1 = id2 = rec;++			s->crecs[id2].tlen = 0;+			id2 = s->crecs[id2].mate_line;+			while (id2 != id1) {+			    s->crecs[id2].tlen = 0;+			    id2 = s->crecs[id2].mate_line;+			}+		    }+		}++		cr->mate_pos = s->crecs[cr->mate_line].apos;+		cr->mate_ref_id = s->crecs[cr->mate_line].ref_id;++		// paired+		cr->flags |= BAM_FPAIRED;++		// set mate unmapped if needed+		if (s->crecs[cr->mate_line].flags & BAM_FUNMAP) {+		    cr->flags |= BAM_FMUNMAP;+		    cr->tlen = 0;+		}+		if (cr->flags & BAM_FUNMAP) {+		    cr->tlen = 0;+		}++		// set mate reversed if needed+		if (s->crecs[cr->mate_line].flags & BAM_FREVERSE)+		    cr->flags |= BAM_FMREVERSE;+	    } else {+		fprintf(stderr, "Mate line out of bounds: %d vs [0, %d]\n",+			cr->mate_line, s->hdr->num_records-1);+	    }++	    /* FIXME: construct read names here too if needed */+	} else {+	    if (cr->mate_flags & CRAM_M_REVERSE) {+		cr->flags |= BAM_FPAIRED | BAM_FMREVERSE;+	    }+	    if (cr->mate_flags & CRAM_M_UNMAP) {+		cr->flags |= BAM_FMUNMAP;+		//cr->mate_ref_id = -1;+	    }+	    if (!(cr->flags & BAM_FPAIRED))+		cr->mate_ref_id = -1;+	}++	if (cr->tlen == INT_MIN)+	    cr->tlen = 0; // Just incase+    }+    return 0;+}++static char *md5_print(unsigned char *md5, char *out) {+    int i;+    for (i = 0; i < 16; i++) {+	out[i*2+0] = "0123456789abcdef"[md5[i]>>4];+	out[i*2+1] = "0123456789abcdef"[md5[i]&15];+    }+    out[32] = 0;++    return out;+}++/*+ * Decode an entire slice from container blocks. Fills out s->crecs[] array.+ * Returns 0 on success+ *        -1 on failure+ */+int cram_decode_slice(cram_fd *fd, cram_container *c, cram_slice *s,+		      SAM_hdr *bfd) {+    cram_block *blk = s->block[0];+    int32_t bf, ref_id;+    unsigned char cf;+    int out_sz, r = 0;+    int rec;+    char *seq = NULL, *qual = NULL;+    int unknown_rg = -1;+    int embed_ref;+    char **refs = NULL;+    uint32_t ds;++    if (cram_dependent_data_series(fd, c->comp_hdr, s) != 0)+	return -1;++    ds = c->comp_hdr->data_series;++    blk->bit = 7; // MSB first++    // Study the blocks and estimate approx sizes to preallocate.+    // This looks to speed up decoding by around 8-9%.+    // We can always shrink back down at the end if we overestimated.+    // However it's likely that this also saves memory as own growth+    // factor (*=1.5) is never applied.+    {+	int qsize, nsize, q_id;+	cram_decode_estimate_sizes(c->comp_hdr, s, &qsize, &nsize, &q_id);+	//fprintf(stderr, "qsize=%d nsize=%d\n", qsize, nsize);+	+	if (qsize && (ds & CRAM_RL)) BLOCK_RESIZE_EXACT(s->seqs_blk, qsize+1);+	if (qsize && (ds & CRAM_RL)) BLOCK_RESIZE_EXACT(s->qual_blk, qsize+1);+	if (nsize && (ds & CRAM_NS)) BLOCK_RESIZE_EXACT(s->name_blk, nsize+1);++	// To do - consider using q_id here to usurp the quality block and+	// avoid a memcpy during decode.+	// Specifically when quality is an external block uniquely used by+	// DS_QS only, then we can set s->qual_blk directly to this+	// block and save the codec->decode() calls. (Approx 3% cpu saving)+    }++    /* Look for unknown RG, added as last by Java CRAM? */+    if (bfd->nrg > 0 &&+	bfd->rg[bfd->nrg-1].name != NULL &&+	!strcmp(bfd->rg[bfd->nrg-1].name, "UNKNOWN"))+	unknown_rg = bfd->nrg-1;++    if (blk->content_type != CORE)+	return -1;++    if (s->crecs)+	free(s->crecs);+    if (!(s->crecs = malloc(s->hdr->num_records * sizeof(*s->crecs))))+	return -1;++    ref_id = s->hdr->ref_seq_id;+    embed_ref = s->hdr->ref_base_id >= 0 ? 1 : 0;++    if (ref_id >= 0) {+	if (embed_ref) {+	    cram_block *b;+	    if (s->hdr->ref_base_id < 0) {+		fprintf(stderr, "No reference specified and "+			"no embedded reference is available.\n");+		return -1;+	    }+	    b = cram_get_block_by_id(s, s->hdr->ref_base_id);+	    if (!b)+		return -1;+	    if (cram_uncompress_block(b) != 0)+		return -1;+	    s->ref = (char *)BLOCK_DATA(b);+	    s->ref_start = s->hdr->ref_seq_start;+	    s->ref_end   = s->hdr->ref_seq_start + s->hdr->ref_seq_span-1;+	} else if (!fd->no_ref) {+	    //// Avoid Java cramtools bug by loading entire reference seq +	    //s->ref = cram_get_ref(fd, s->hdr->ref_seq_id, 1, 0);+	    //s->ref_start = 1;++	    if (fd->required_fields & SAM_SEQ)+		s->ref =+		cram_get_ref(fd, s->hdr->ref_seq_id,+			     s->hdr->ref_seq_start,+			     s->hdr->ref_seq_start + s->hdr->ref_seq_span -1);+	    s->ref_start = s->hdr->ref_seq_start;+	    s->ref_end   = s->hdr->ref_seq_start + s->hdr->ref_seq_span-1;++	    /* Sanity check */+	    if (s->ref_start < 0) {+		fprintf(stderr, "Slice starts before base 1.\n");+		s->ref_start = 0;+	    }+	    pthread_mutex_lock(&fd->ref_lock);+	    pthread_mutex_lock(&fd->refs->lock);+	    if ((fd->required_fields & SAM_SEQ) &&+		s->ref_end > fd->refs->ref_id[ref_id]->length) {+		s->ref_end = fd->refs->ref_id[ref_id]->length;+	    }+	    pthread_mutex_unlock(&fd->refs->lock);+	    pthread_mutex_unlock(&fd->ref_lock);+	}+    }++    if ((fd->required_fields & SAM_SEQ) &&+	s->ref == NULL && s->hdr->ref_seq_id >= 0 && !fd->no_ref) {+	fprintf(stderr, "Unable to fetch reference #%d %d..%d\n",+		s->hdr->ref_seq_id, s->hdr->ref_seq_start,+		s->hdr->ref_seq_start + s->hdr->ref_seq_span-1);+	return -1;+    }++    if (CRAM_MAJOR_VERS(fd->version) != 1+	&& (fd->required_fields & SAM_SEQ)+	&& s->hdr->ref_seq_id >= 0+	&& !fd->ignore_md5+	&& memcmp(s->hdr->md5, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 16)) {+	hts_md5_context *md5;+	unsigned char digest[16];++	if (s->ref && s->hdr->ref_seq_id >= 0) {+	    int start, len;++	    if (s->hdr->ref_seq_start >= s->ref_start) {+		start = s->hdr->ref_seq_start - s->ref_start;+	    } else {+		fprintf(stderr, "Slice starts before base 1.\n");+		start = 0;+	    }++	    if (s->hdr->ref_seq_span <= s->ref_end - s->ref_start + 1) {+		len = s->hdr->ref_seq_span;+	    } else {+		fprintf(stderr, "Slice ends beyond reference end.\n");+		len = s->ref_end - s->ref_start + 1;+	    }++	    if (!(md5 = hts_md5_init()))+		return -1;+	    if (start + len > s->ref_end - s->ref_start + 1)+		len = s->ref_end - s->ref_start + 1 - start;+	    if (len >= 0)+		hts_md5_update(md5, s->ref + start, len);+	    hts_md5_final(digest, md5);+	    hts_md5_destroy(md5);+	} else if (!s->ref && s->hdr->ref_base_id >= 0) {+	    cram_block *b = cram_get_block_by_id(s, s->hdr->ref_base_id);+	    if (b) {+		if (!(md5 = hts_md5_init()))+		    return -1;+		hts_md5_update(md5, b->data, b->uncomp_size);+		hts_md5_final(digest, md5);+		hts_md5_destroy(md5);+	    }+	}++	if ((!s->ref && s->hdr->ref_base_id < 0)+	    || memcmp(digest, s->hdr->md5, 16) != 0) {+	    char M[33];+	    fprintf(stderr, "ERROR: md5sum reference mismatch for ref "+		    "%d pos %d..%d\n", ref_id, s->ref_start, s->ref_end);+	    fprintf(stderr, "CRAM: %s\n", md5_print(s->hdr->md5, M));+	    fprintf(stderr, "Ref : %s\n", md5_print(digest, M));+	    return -1;+	}+    }++    if (ref_id == -2) {+	pthread_mutex_lock(&fd->ref_lock);+	pthread_mutex_lock(&fd->refs->lock);+	refs = calloc(fd->refs->nref, sizeof(char *));+	pthread_mutex_unlock(&fd->refs->lock);+	pthread_mutex_unlock(&fd->ref_lock);+	if (!refs)+	    return -1;+    }++    int last_ref_id = -9; // Arbitrary -ve marker for not-yet-set+    for (rec = 0; rec < s->hdr->num_records; rec++) {+	cram_record *cr = &s->crecs[rec];+	int has_MD, has_NM;++	//fprintf(stderr, "Decode seq %d, %d/%d\n", rec, blk->byte, blk->bit);++	cr->s = s;++	out_sz = 1; /* decode 1 item */+	if (ds & CRAM_BF) {+	    if (!c->comp_hdr->codecs[DS_BF]) return -1;+	    r |= c->comp_hdr->codecs[DS_BF]+		            ->decode(s, c->comp_hdr->codecs[DS_BF], blk,+				     (char *)&bf, &out_sz);+	    if (r || bf < 0 ||+		bf >= sizeof(fd->bam_flag_swap)/sizeof(*fd->bam_flag_swap))+		return -1;+	    bf = fd->bam_flag_swap[bf];+	    cr->flags = bf;+	} else {+	    cr->flags = bf = 0x4; // unmapped+	}++	if (ds & CRAM_CF) {+	    if (CRAM_MAJOR_VERS(fd->version) == 1) {+		/* CF is byte in 1.0, int32 in 2.0 */+		if (!c->comp_hdr->codecs[DS_CF]) return -1;+		r |= c->comp_hdr->codecs[DS_CF]+		                ->decode(s, c->comp_hdr->codecs[DS_CF], blk,+				 	 (char *)&cf, &out_sz);+		if (r) return -1;+		cr->cram_flags = cf;+	    } else {+		if (!c->comp_hdr->codecs[DS_CF]) return -1;+		r |= c->comp_hdr->codecs[DS_CF]+		                ->decode(s, c->comp_hdr->codecs[DS_CF], blk,+					 (char *)&cr->cram_flags, &out_sz);+		if (r) return -1;+		cf = cr->cram_flags;+	    }+	} else {+	    cf = cr->cram_flags = 0;+	}++	if (CRAM_MAJOR_VERS(fd->version) != 1 && ref_id == -2) {+	    if (ds & CRAM_RI) {+		if (!c->comp_hdr->codecs[DS_RI]) return -1;+		r |= c->comp_hdr->codecs[DS_RI]+		                ->decode(s, c->comp_hdr->codecs[DS_RI], blk,+					 (char *)&cr->ref_id, &out_sz);+		if (r) return -1;+		if ((fd->required_fields & (SAM_SEQ|SAM_TLEN))+		    && cr->ref_id >= 0+		    && cr->ref_id != last_ref_id) {+		    if (!fd->no_ref) {+			if (!refs[cr->ref_id])+			    refs[cr->ref_id] = cram_get_ref(fd, cr->ref_id,+							    1, 0);+			s->ref = refs[cr->ref_id];++			if (!fd->unsorted && last_ref_id >= 0 && refs[last_ref_id]) {+			    cram_ref_decr(fd->refs, last_ref_id);+			    refs[last_ref_id] = NULL;+			}+		    }+		    s->ref_start = 1;+		    pthread_mutex_lock(&fd->ref_lock);+		    pthread_mutex_lock(&fd->refs->lock);+		    s->ref_end = fd->refs->ref_id[cr->ref_id]->length;+		    pthread_mutex_unlock(&fd->refs->lock);+		    pthread_mutex_unlock(&fd->ref_lock);++		    last_ref_id = cr->ref_id;+		}+	    } else {+		cr->ref_id = -1;+	    }+	} else {+	    cr->ref_id = ref_id; // Forced constant in CRAM 1.0+	}+	if (cr->ref_id >= bfd->nref) {+	    fprintf(stderr, "Requested unknown reference ID %d\n", cr->ref_id);+            return -1;+	}++	if (ds & CRAM_RL) {+	    if (!c->comp_hdr->codecs[DS_RL]) return -1;+	    r |= c->comp_hdr->codecs[DS_RL]+		            ->decode(s, c->comp_hdr->codecs[DS_RL], blk,+				     (char *)&cr->len, &out_sz);+	    if (r) return r;+	    if (cr->len < 0) {+	        fprintf(stderr, "Read has negative length\n");+		return -1;+	    }+	}++	if (ds & CRAM_AP) {+	    if (!c->comp_hdr->codecs[DS_AP]) return -1;+	    r |= c->comp_hdr->codecs[DS_AP]+		            ->decode(s, c->comp_hdr->codecs[DS_AP], blk,+				     (char *)&cr->apos, &out_sz);+	    if (r) return r;+	    if (c->comp_hdr->AP_delta)+		cr->apos += s->last_apos;+	    s->last_apos=  cr->apos;+	} else {+	    cr->apos = c->ref_seq_start;+	}+		    +	if (ds & CRAM_RG) {+	    if (!c->comp_hdr->codecs[DS_RG]) return -1;+	    r |= c->comp_hdr->codecs[DS_RG]+		           ->decode(s, c->comp_hdr->codecs[DS_RG], blk,+				    (char *)&cr->rg, &out_sz);+	    if (r) return r;+	    if (cr->rg == unknown_rg)+		cr->rg = -1;+	} else {+	    cr->rg = -1;+	}++	cr->name_len = 0;++	if (c->comp_hdr->read_names_included) {+	    int32_t out_sz2 = 1;++	    // Read directly into name cram_block+	    cr->name = BLOCK_SIZE(s->name_blk);+	    if (ds & CRAM_RN) {+		if (!c->comp_hdr->codecs[DS_RN]) return -1;+		r |= c->comp_hdr->codecs[DS_RN]+		                ->decode(s, c->comp_hdr->codecs[DS_RN], blk,+					 (char *)s->name_blk, &out_sz2);+		if (r) return r;+		cr->name_len = out_sz2;+	    }+	}++	cr->mate_pos = 0;+	cr->mate_line = -1;+	cr->mate_ref_id = -1;+	if ((ds & CRAM_CF) && (cf & CRAM_FLAG_DETACHED)) {+	    if (ds & CRAM_MF) {+		if (CRAM_MAJOR_VERS(fd->version) == 1) {+		    /* MF is byte in 1.0, int32 in 2.0 */+		    unsigned char mf;+		    if (!c->comp_hdr->codecs[DS_MF]) return -1;+		    r |= c->comp_hdr->codecs[DS_MF]+			            ->decode(s, c->comp_hdr->codecs[DS_MF],+					     blk, (char *)&mf, &out_sz);+		    if (r) return r;+		    cr->mate_flags = mf;+		} else {+		    if (!c->comp_hdr->codecs[DS_MF]) return -1;+		    r |= c->comp_hdr->codecs[DS_MF]+			            ->decode(s, c->comp_hdr->codecs[DS_MF],+					     blk,+					     (char *)&cr->mate_flags,+					     &out_sz);+		    if (r) return r;+		}+	    } else {+		cr->mate_flags = 0;+	    }++	    if (!c->comp_hdr->read_names_included) {+		int32_t out_sz2 = 1;+	    +		// Read directly into name cram_block+		cr->name = BLOCK_SIZE(s->name_blk);+		if (ds & CRAM_RN) {+		    if (!c->comp_hdr->codecs[DS_RN]) return -1;+		    r |= c->comp_hdr->codecs[DS_RN]+			            ->decode(s, c->comp_hdr->codecs[DS_RN],+					     blk, (char *)s->name_blk,+					     &out_sz2);+		    if (r) return r;+		    cr->name_len = out_sz2;+		}+	    }+		    +	    if (ds & CRAM_NS) {+		if (!c->comp_hdr->codecs[DS_NS]) return -1;+		r |= c->comp_hdr->codecs[DS_NS]+		                ->decode(s, c->comp_hdr->codecs[DS_NS], blk,+					 (char *)&cr->mate_ref_id, &out_sz);+		if (r) return r;+	    }++// Skip as mate_ref of "*" is legit. It doesn't mean unmapped, just unknown.+//	    if (cr->mate_ref_id == -1 && cr->flags & 0x01) {+//		/* Paired, but unmapped */+//		cr->flags |= BAM_FMUNMAP;+//	    }++	    if (ds & CRAM_NP) {+		if (!c->comp_hdr->codecs[DS_NP]) return -1;+		r |= c->comp_hdr->codecs[DS_NP]+		                ->decode(s, c->comp_hdr->codecs[DS_NP], blk,+					 (char *)&cr->mate_pos, &out_sz);+		if (r) return r;+	    }++	    if (ds & CRAM_TS) {+		if (!c->comp_hdr->codecs[DS_TS]) return -1;+		r |= c->comp_hdr->codecs[DS_TS]+		                ->decode(s, c->comp_hdr->codecs[DS_TS], blk,+					 (char *)&cr->tlen, &out_sz);+		if (r) return r;+	    } else {+		cr->tlen = INT_MIN;+	    }+	} else if ((ds & CRAM_CF) && (cf & CRAM_FLAG_MATE_DOWNSTREAM)) {+	    if (ds & CRAM_NF) {+		if (!c->comp_hdr->codecs[DS_NF]) return -1;+		r |= c->comp_hdr->codecs[DS_NF]+		                ->decode(s, c->comp_hdr->codecs[DS_NF], blk,+					 (char *)&cr->mate_line, &out_sz);+		if (r) return r;+		cr->mate_line += rec + 1;++		//cr->name_len = sprintf(name, "%d", name_id++);+		//cr->name = DSTRING_LEN(name_ds);+		//dstring_nappend(name_ds, name, cr->name_len);++		cr->mate_ref_id = -1;+		cr->tlen = INT_MIN;+		cr->mate_pos = 0;+	    } else  {+		cr->mate_flags = 0;+		cr->tlen = INT_MIN;+	    }+	} else {+	    cr->mate_flags = 0;+	    cr->tlen = INT_MIN;+	}+	/*+	else if (!name[0]) {+	    //name[0] = '?'; name[1] = 0;+	    //cr->name_len = 1;+	    //cr->name=  DSTRING_LEN(s->name_ds);+	    //dstring_nappend(s->name_ds, "?", 1);++	    cr->mate_ref_id = -1;+	    cr->tlen = 0;+	    cr->mate_pos = 0;+	}+	*/++	/* Auxiliary tags */+	has_MD = has_NM = 0;+	if (CRAM_MAJOR_VERS(fd->version) == 1)+	    r |= cram_decode_aux_1_0(c, s, blk, cr);+	else+	    r |= cram_decode_aux(c, s, blk, cr, &has_MD, &has_NM);+	if (r) return r;++	/* Fake up dynamic string growth and appending */+	if (ds & CRAM_RL) {+	    cr->seq = BLOCK_SIZE(s->seqs_blk);+	    BLOCK_GROW(s->seqs_blk, cr->len);+	    seq = (char *)BLOCK_END(s->seqs_blk);+	    BLOCK_SIZE(s->seqs_blk) += cr->len;++	    if (!seq)+		return -1;+	+	    cr->qual = BLOCK_SIZE(s->qual_blk);+	    BLOCK_GROW(s->qual_blk, cr->len);+	    qual = (char *)BLOCK_END(s->qual_blk);+	    BLOCK_SIZE(s->qual_blk) += cr->len;++	    if (!s->ref)+		memset(seq, '=', cr->len);+	}++	if (!(bf & BAM_FUNMAP)) {+            if ((ds & CRAM_AP) && cr->apos <= 0) {+                fprintf(stderr,+			"Read has alignment position %d but no unmapped flag\n",+			cr->apos);+		return -1;+	    }+	    /* Decode sequence and generate CIGAR */+	    if (ds & (CRAM_SEQ | CRAM_MQ)) {+		r |= cram_decode_seq(fd, c, s, blk, cr, bfd, cf, seq, qual,+				     has_MD, has_NM);+		if (r) return r;+	    } else {+		cr->cigar = 0;+		cr->ncigar = 0;+		cr->aend = cr->apos;+		cr->mqual = 0;+	    }+	} else {+	    int out_sz2 = cr->len;++	    //puts("Unmapped");+	    cr->cigar = 0;+	    cr->ncigar = 0;+	    cr->aend = cr->apos;+	    cr->mqual = 0;++	    if (ds & CRAM_BA && cr->len) {+		if (!c->comp_hdr->codecs[DS_BA]) return -1;+		r |= c->comp_hdr->codecs[DS_BA]+		                ->decode(s, c->comp_hdr->codecs[DS_BA], blk,+					 (char *)seq, &out_sz2);+		if (r) return r;+	    }++	    if ((ds & CRAM_CF) && (cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {+		out_sz2 = cr->len;+		if (ds & CRAM_QS && cr->len >= 0) {+		    if (!c->comp_hdr->codecs[DS_QS]) return -1;+		    r |= c->comp_hdr->codecs[DS_QS]+			            ->decode(s, c->comp_hdr->codecs[DS_QS],+					     blk, qual, &out_sz2);+		    if (r) return r;+		}+	    } else {+		if (ds & CRAM_RL)+		    memset(qual, 255, cr->len);+	    }+	}+    }++    pthread_mutex_lock(&fd->ref_lock);+    if (refs) {+	int i;+	for (i = 0; i < fd->refs->nref; i++) {+	    if (refs[i])+		cram_ref_decr(fd->refs, i);+	}+	free(refs);+    } else if (ref_id >= 0 && s->ref != fd->ref_free && !embed_ref) {+	cram_ref_decr(fd->refs, ref_id);+    }+    pthread_mutex_unlock(&fd->ref_lock);++    /* Resolve mate pair cross-references between recs within this slice */+    r |= cram_decode_slice_xref(s, fd->required_fields);++    // Free the original blocks as we no longer need these.+    {+	int i;+	for (i = 0; i < s->hdr->num_blocks; i++) {+	    cram_block *b = s->block[i];+	    cram_free_block(b);+	    s->block[i] = NULL;+	}+    }++    // Also see initial BLOCK_RESIZE_EXACT at top of function.+    // As we grow blocks we overallocate by up to 50%. So shrink+    // back to their final sizes here.+    //+//    fprintf(stderr, "%d %d // %d %d // %d %d // %d %d\n",+//    	    (int)s->seqs_blk->byte, (int)s->seqs_blk->alloc, +//    	    (int)s->qual_blk->byte, (int)s->qual_blk->alloc, +//    	    (int)s->name_blk->byte, (int)s->name_blk->alloc, +//    	    (int)s->aux_blk->byte,  (int)s->aux_blk->alloc);+    BLOCK_RESIZE_EXACT(s->seqs_blk, BLOCK_SIZE(s->seqs_blk)+1);+    BLOCK_RESIZE_EXACT(s->qual_blk, BLOCK_SIZE(s->qual_blk)+1);+    BLOCK_RESIZE_EXACT(s->name_blk, BLOCK_SIZE(s->name_blk)+1);+    BLOCK_RESIZE_EXACT(s->aux_blk,  BLOCK_SIZE(s->aux_blk)+1);++    return r;+}++typedef struct {+    cram_fd *fd;+    cram_container *c;+    cram_slice *s;+    SAM_hdr *h;+    int exit_code;+} cram_decode_job;++void *cram_decode_slice_thread(void *arg) {+    cram_decode_job *j = (cram_decode_job *)arg;++    j->exit_code = cram_decode_slice(j->fd, j->c, j->s, j->h);++    return j;+}++/*+ * Spawn a multi-threaded version of cram_decode_slice().+ */+int cram_decode_slice_mt(cram_fd *fd, cram_container *c, cram_slice *s,+			 SAM_hdr *bfd) {+    cram_decode_job *j;+    int nonblock;++    if (!fd->pool)+	return cram_decode_slice(fd, c, s, bfd);++    if (!(j = malloc(sizeof(*j))))+	return -1;++    j->fd = fd;+    j->c  = c;+    j->s  = s;+    j->h  = bfd;+    +    nonblock = t_pool_results_queue_sz(fd->rqueue) ? 1 : 0;++    if (-1 == t_pool_dispatch2(fd->pool, fd->rqueue, cram_decode_slice_thread,+			       j, nonblock)) {+	/* Would block */+	fd->job_pending = j;+    } else {+	fd->job_pending = NULL;+    }++    // flush too+    return 0;+}+++/* ----------------------------------------------------------------------+ * CRAM sequence iterators.+ */++/*+ * Converts a cram in-memory record into a bam in-memory record. We+ * pass a pointer to a bam_seq_t pointer along with the a pointer to+ * the allocated size. These can initially be pointers to NULL and zero.+ *+ * This function will reallocate the bam buffer as required and update+ * (*bam)->alloc accordingly, allowing it to be used within a loop+ * efficiently without needing to allocate new bam objects over and+ * over again.+ *+ * Returns the used size of the bam record on success+ *         -1 on failure.+ */+static int cram_to_bam(SAM_hdr *bfd, cram_fd *fd, cram_slice *s,+		       cram_record *cr, int rec, bam_seq_t **bam) {+    int bam_idx, rg_len;+    char name_a[1024], *name;+    int name_len;+    char *aux, *aux_orig;+    char *seq, *qual;++    /* Assign names if not explicitly set */+    if (fd->required_fields & SAM_QNAME) {+	if (cr->name_len) {+	    name = (char *)BLOCK_DATA(s->name_blk) + cr->name;+	    name_len = cr->name_len;+	} else {+	    name = name_a;+	    name_len = strlen(fd->prefix);+	    memcpy(name, fd->prefix, name_len);+	    name += name_len;+	    *name++ = ':';+	    if (cr->mate_line >= 0 && cr->mate_line < rec)+		name = (char *)append_uint64((unsigned char *)name,+					     s->hdr->record_counter ++					     cr->mate_line + 1);+	    else+		name = (char *)append_uint64((unsigned char *)name,+					     s->hdr->record_counter ++					     rec + 1);+	    name_len = name - name_a;+	    name = name_a;+	}+    } else {+	name = "?";+	name_len = 1;+    }++    /* Generate BAM record */+    if (cr->rg < -1 || cr->rg >= bfd->nrg)+	return -1;+    rg_len = (cr->rg != -1) ? bfd->rg[cr->rg].name_len + 4 : 0;++    if (fd->required_fields & (SAM_SEQ | SAM_QUAL)) {+	if (!BLOCK_DATA(s->seqs_blk))+	    return -1;+	seq = (char *)BLOCK_DATA(s->seqs_blk) + cr->seq;+    } else {+	seq = "*";+	cr->len = 0;+    }+++    if (fd->required_fields & SAM_QUAL) {+	if (!BLOCK_DATA(s->qual_blk))+	    return -1;+	qual = (char *)BLOCK_DATA(s->qual_blk) + cr->qual;+    } else {+	qual = NULL;+    }++    bam_idx = bam_construct_seq(bam, cr->aux_size + rg_len,+				name, name_len,+				cr->flags,+				cr->ref_id,+				cr->apos,+				cr->aend,+				cr->mqual,+				cr->ncigar, &s->cigar[cr->cigar],+				cr->mate_ref_id,+				cr->mate_pos,+				cr->tlen,+				cr->len,+				seq,+				qual);+    if (bam_idx == -1)+	return -1;++    aux = aux_orig = (char *)bam_aux(*bam);++    /* Auxiliary strings */+    if (cr->aux_size != 0) {+	memcpy(aux, BLOCK_DATA(s->aux_blk) + cr->aux, cr->aux_size);+	aux += cr->aux_size;+    }++    /* RG:Z: */+    if (cr->rg != -1) {+	int len = bfd->rg[cr->rg].name_len;+	*aux++ = 'R'; *aux++ = 'G'; *aux++ = 'Z';+	memcpy(aux, bfd->rg[cr->rg].name, len);+	aux += len;+	*aux++ = 0;+    }+    +    return bam_idx + (aux - aux_orig);+}++/*+ * Here be dragons! The multi-threading code in this is crufty beyond belief.+ */+static cram_slice *cram_next_slice(cram_fd *fd, cram_container **cp) {+    cram_container *c;+    cram_slice *s = NULL;++    if (!(c = fd->ctr)) {+	// Load first container.+	do {+	    if (!(c = fd->ctr = cram_read_container(fd)))+		return NULL;+	} while (c->length == 0);++	/*+	 * The first container may be a result of a sub-range query.+	 * In which case it may still not be the optimal starting point+	 * due to skipped containers/slices in the index. +	 */+	if (fd->range.refid != -2) {+	    while (c->ref_seq_id != -2 &&+		   (c->ref_seq_id < fd->range.refid ||+		    c->ref_seq_start + c->ref_seq_span-1 < fd->range.start)) {+		if (0 != cram_seek(fd, c->length, SEEK_CUR))+		    return NULL;+		cram_free_container(fd->ctr);+		do {+		    if (!(c = fd->ctr = cram_read_container(fd)))+			return NULL;+		} while (c->length == 0);+	    }++	    if (c->ref_seq_id != -2 && c->ref_seq_id != fd->range.refid)+		return NULL;+	}++	if (!(c->comp_hdr_block = cram_read_block(fd)))+	    return NULL;+	if (c->comp_hdr_block->content_type != COMPRESSION_HEADER)+	    return NULL;++	c->comp_hdr = cram_decode_compression_header(fd, c->comp_hdr_block);+	if (!c->comp_hdr)+	    return NULL;+	if (!c->comp_hdr->AP_delta &&+	    sam_hdr_sort_order(fd->header) != ORDER_COORD) {+	    pthread_mutex_lock(&fd->ref_lock);+	    fd->unsorted = 1;+	    pthread_mutex_unlock(&fd->ref_lock);+	}+    }++    if ((s = c->slice)) {+	c->slice = NULL;+	cram_free_slice(s);+	s = NULL;+    }++    if (c->curr_slice == c->max_slice) {+	cram_free_container(c);+	c = NULL;+    }++    /* Sorry this is so contorted! */+    for (;;) {+	if (fd->job_pending) {+	    cram_decode_job *j = (cram_decode_job *)fd->job_pending;+	    c = j->c;+	    s = j->s;+	    free(fd->job_pending);+	    fd->job_pending = NULL;+	} else if (!fd->ooc) {+	empty_container:+	    if (!c || c->curr_slice == c->max_slice) {+		// new container+		do {+		    if (!(c = fd->ctr = cram_read_container(fd))) {+			if (fd->pool) {+			    fd->ooc = 1;+			    break;+			}++			return NULL;+		    }+		} while (c->length == 0);+		if (fd->ooc)+		    break;++		/* Skip containers not yet spanning our range */+		if (fd->range.refid != -2 && c->ref_seq_id != -2) {+		    fd->required_fields |= SAM_POS;++		    if (c->ref_seq_id != fd->range.refid) {+			cram_free_container(c);+			fd->ctr = NULL;+			fd->ooc = 1;+			fd->eof = 1;+			break;+		    }++		    if (c->ref_seq_start > fd->range.end) {+			cram_free_container(c);+			fd->ctr = NULL;+			fd->ooc = 1;+			fd->eof = 1;+			break;+		    }++		    if (c->ref_seq_start + c->ref_seq_span-1 <+			fd->range.start) {+			c->curr_rec = c->max_rec;+			c->curr_slice = c->max_slice;+			cram_seek(fd, c->length, SEEK_CUR);+			cram_free_container(c);+			c = NULL;+			continue;+		    }+		}++		if (!(c->comp_hdr_block = cram_read_block(fd)))+		    return NULL;+		if (c->comp_hdr_block->content_type != COMPRESSION_HEADER)+		    return NULL;++		c->comp_hdr =+		    cram_decode_compression_header(fd, c->comp_hdr_block);+		if (!c->comp_hdr)+		    return NULL;++		if (!c->comp_hdr->AP_delta &&+		    sam_hdr_sort_order(fd->header) != ORDER_COORD) {+		    pthread_mutex_lock(&fd->ref_lock);+		    fd->unsorted = 1;+		    pthread_mutex_unlock(&fd->ref_lock);+		}+	    }++	    if (c->num_records == 0) {+		cram_free_container(c); c = NULL;+		goto empty_container;+	    }+++	    if (!(s = c->slice = cram_read_slice(fd)))+		return NULL;+	    c->curr_slice++;+	    c->curr_rec = 0;+	    c->max_rec = s->hdr->num_records;++	    s->last_apos = s->hdr->ref_seq_start;+	    +	    /* Skip slices not yet spanning our range */+	    if (fd->range.refid != -2 && s->hdr->ref_seq_id != -2) {+		if (s->hdr->ref_seq_id != fd->range.refid) {+		    fd->eof = 1;+		    cram_free_slice(s);+		    c->slice = NULL;+		    return NULL;+		}++		if (s->hdr->ref_seq_start > fd->range.end) {+		    fd->eof = 1;+		    cram_free_slice(s);+		    c->slice = NULL;+		    return NULL;+		}++		if (s->hdr->ref_seq_start + s->hdr->ref_seq_span-1 <+		    fd->range.start) {+		    cram_free_slice(s);+		    c->slice = NULL;+		    cram_free_container(c);+		    c = NULL;+		    continue;+		}+	    }+	}++	/* Test decoding of 1st seq */+	if (!c || !s)+	    break;++	if (cram_decode_slice_mt(fd, c, s, fd->header) != 0) {+	    //	if (cram_decode_slice(fd, c, s, fd->header) != 0) {+	    fprintf(stderr, "Failure to decode slice\n");+	    cram_free_slice(s);+	    c->slice = NULL;+	    return NULL;+	}++	if (!fd->pool || fd->job_pending)+	    break;++	// Push it a bit far, to qsize in queue rather than pending arrival,+	// as cram tends to be a bit bursty in decode timings.+	if (t_pool_results_queue_len(fd->rqueue) > fd->pool->qsize)+	    break;+    }++    if (fd->pool) {+	t_pool_result *res;+	cram_decode_job *j;+	+//	fprintf(stderr, "Thread pool len = %d, %d\n",+//		t_pool_results_queue_len(fd->rqueue),+//		t_pool_results_queue_sz(fd->rqueue));++	if (fd->ooc && t_pool_results_queue_empty(fd->rqueue))+	    return NULL;++	res = t_pool_next_result_wait(fd->rqueue);++	if (!res || !res->data) {+	    fprintf(stderr, "t_pool_next_result failure\n");+	    return NULL;+	}++	j = (cram_decode_job *)res->data;+	c = j->c;+	s = j->s;++	fd->ctr = c;++	t_pool_delete_result(res, 1);+    }++    *cp = c;+    return s;+}++/*+ * Read the next cram record and return it.+ * Note that to decode cram_record the caller will need to look up some data+ * in the current slice, pointed to by fd->ctr->slice. This is valid until+ * the next call to cram_get_seq (which may invalidate it).+ *+ * Returns record pointer on success (do not free)+ *        NULL on failure+ */+cram_record *cram_get_seq(cram_fd *fd) {+    cram_container *c;+    cram_slice *s;++    for (;;) {+	c = fd->ctr;+	if (c && c->slice && c->curr_rec < c->max_rec) {+	    s = c->slice;+	} else {+	    if (!(s = cram_next_slice(fd, &c)))+		return NULL;+	    continue; /* In case slice contains no records */+	}++	if (fd->range.refid != -2) {+	    if (fd->range.refid == -1 && s->crecs[c->curr_rec].ref_id != -1) {+		// Special case when looking for unmapped blocks at end.+		// If these are mixed in with mapped data (c->ref_id == -2)+		// then we need skip until we find the unmapped data, if at all+		c->curr_rec++;+		continue;+	    }+	    if (s->crecs[c->curr_rec].ref_id < fd->range.refid &&+		s->crecs[c->curr_rec].ref_id != -1) {+		// Looking for a mapped read, but not there yet.  Special case+		// as -1 (unmapped) shouldn't be considered < refid.+		c->curr_rec++;+		continue;+	    }++	    if (s->crecs[c->curr_rec].ref_id != fd->range.refid) {+		fd->eof = 1;+		cram_free_slice(s);+		c->slice = NULL;+		return NULL;+	    }++	    if (fd->range.refid != -1 && s->crecs[c->curr_rec].apos > fd->range.end) {+		fd->eof = 1;+		cram_free_slice(s);+		c->slice = NULL;+		return NULL;+	    }++	    if (fd->range.refid != -1 && s->crecs[c->curr_rec].aend < fd->range.start) {+		c->curr_rec++;+		continue;+	    }+	}++	break;+    }++    fd->ctr = c;+    c->slice = s;+    return &s->crecs[c->curr_rec++];+}++/*+ * Read the next cram record and convert it to a bam_seq_t struct.+ *+ * Returns 0 on success+ *        -1 on EOF or failure (check fd->err)+ */+int cram_get_bam_seq(cram_fd *fd, bam_seq_t **bam) {+    cram_record *cr;+    cram_container *c;+    cram_slice *s;++    if (!(cr = cram_get_seq(fd)))+	return -1;++    c = fd->ctr;+    s = c->slice;++    return cram_to_bam(fd->header, fd, s, cr, c->curr_rec-1, bam);+}
+ htslib-1.3.2/cram/cram_decode.h view
@@ -0,0 +1,112 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*! \file+ * Include cram.h instead.+ *+ * This is an internal part of the CRAM system and is automatically included+ * when you #include cram.h.+ *+ * Implements the decoding portion of CRAM I/O. Also see+ * cram_codecs.[ch] for the actual encoding functions themselves.+ */++#ifndef _CRAM_READ_H_+#define _CRAM_READ_H_++#ifdef __cplusplus+extern "C" {+#endif++/* ----------------------------------------------------------------------+ * CRAM sequence iterators.+ */++/*! Read the next cram record and return it as a cram_record.+ *+ * Note that to decode cram_record the caller will need to look up some data+ * in the current slice, pointed to by fd->ctr->slice. This is valid until+ * the next call to cram_get_seq (which may invalidate it).+ *+ * @return+ * Returns record pointer on success (do not free);+ *        NULL on failure+ */+cram_record *cram_get_seq(cram_fd *fd);++/*! Read the next cram record and convert it to a bam_seq_t struct.+ *+ * @return+ * Returns 0 on success;+ *        -1 on EOF or failure (check fd->err)+ */+int cram_get_bam_seq(cram_fd *fd, bam_seq_t **bam);+++/* ----------------------------------------------------------------------+ * Internal functions+ */++/*! INTERNAL:+ * Decodes a CRAM block compression header.+ *+ * @return+ * Returns header ptr on success;+ *         NULL on failure+ */+cram_block_compression_hdr *cram_decode_compression_header(cram_fd *fd,+							   cram_block *b);++/*! INTERNAL:+ * Decodes a CRAM (un)mapped slice header block.+ *+ * @return+ * Returns slice header ptr on success;+ *         NULL on failure+ */+cram_block_slice_hdr *cram_decode_slice_header(cram_fd *fd, cram_block *b);+++/*! INTERNAL:+ * Decode an entire slice from container blocks. Fills out s->crecs[] array.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_decode_slice(cram_fd *fd, cram_container *c, cram_slice *s,+		      SAM_hdr *hdr);+++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/cram/cram_encode.c view
@@ -0,0 +1,3105 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdio.h>+#include <errno.h>+#include <assert.h>+#include <stdlib.h>+#include <string.h>+#include <zlib.h>+#include <sys/types.h>+#include <sys/stat.h>+#include <math.h>+#include <ctype.h>++#include "cram/cram.h"+#include "cram/os.h"+#include "htslib/hts.h"++#define Z_CRAM_STRAT Z_FILTERED+//#define Z_CRAM_STRAT Z_RLE+//#define Z_CRAM_STRAT Z_HUFFMAN_ONLY+//#define Z_CRAM_STRAT Z_DEFAULT_STRATEGY++static int process_one_read(cram_fd *fd, cram_container *c,+			    cram_slice *s, cram_record *cr,+			    bam_seq_t *b, int rnum);++/*+ * Returns index of val into key.+ * Basically strchr(key, val)-key;+ */+static int sub_idx(char *key, char val) {+    int i;++    for (i = 0; *key && *key++ != val; i++);+    return i;+}++/*+ * Encodes a compression header block into a generic cram_block structure.+ *+ * Returns cram_block ptr on success+ *         NULL on failure+ */+cram_block *cram_encode_compression_header(cram_fd *fd, cram_container *c,+					   cram_block_compression_hdr *h) {+    cram_block *cb  = cram_new_block(COMPRESSION_HEADER, 0);+    cram_block *map = cram_new_block(COMPRESSION_HEADER, 0);+    int i, mc;++    if (!cb || !map)+	return NULL;++    /*+     * This is a concatenation of several blocks of data:+     * header + landmarks, preservation map, read encoding map, and the tag+     * encoding map.+     * All 4 are variable sized and we need to know how large these are+     * before creating the compression header itself as this starts with+     * the total size (stored as a variable length string).+     */++    // Duplicated from container itself, and removed in 1.1+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	itf8_put_blk(cb, h->ref_seq_id);+	itf8_put_blk(cb, h->ref_seq_start);+	itf8_put_blk(cb, h->ref_seq_span);+	itf8_put_blk(cb, h->num_records);+	itf8_put_blk(cb, h->num_landmarks);+	for (i = 0; i < h->num_landmarks; i++) {+	    itf8_put_blk(cb, h->landmark[i]);+	}+    }++    if (h->preservation_map)+	kh_destroy(map, h->preservation_map);++    /* Create in-memory preservation map */+    /* FIXME: should create this when we create the container */+    {+	khint_t k;+	int r;++	if (!(h->preservation_map = kh_init(map)))+	    return NULL;++	k = kh_put(map, h->preservation_map, "RN", &r);+	if (-1 == r) return NULL;+	kh_val(h->preservation_map, k).i = 1;++	if (CRAM_MAJOR_VERS(fd->version) == 1) {+	    k = kh_put(map, h->preservation_map, "PI", &r);+	    if (-1 == r) return NULL;+	    kh_val(h->preservation_map, k).i = 0;++	    k = kh_put(map, h->preservation_map, "UI", &r);+	    if (-1 == r) return NULL;+	    kh_val(h->preservation_map, k).i = 1;++	    k = kh_put(map, h->preservation_map, "MI", &r);+	    if (-1 == r) return NULL;+	    kh_val(h->preservation_map, k).i = 1;++	} else {+	    // Technically SM was in 1.0, but wasn't in Java impl.+	    k = kh_put(map, h->preservation_map, "SM", &r);+	    if (-1 == r) return NULL;+	    kh_val(h->preservation_map, k).i = 0;++	    k = kh_put(map, h->preservation_map, "TD", &r);+	    if (-1 == r) return NULL;+	    kh_val(h->preservation_map, k).i = 0;++	    k = kh_put(map, h->preservation_map, "AP", &r);+	    if (-1 == r) return NULL;+	    kh_val(h->preservation_map, k).i = h->AP_delta;++	    if (fd->no_ref || fd->embed_ref) {+		// Reference Required == No+		k = kh_put(map, h->preservation_map, "RR", &r);+		if (-1 == r) return NULL;+		kh_val(h->preservation_map, k).i = 0;+	    }+	}+    }++    /* Encode preservation map; could collapse this and above into one */+    mc = 0;+    BLOCK_SIZE(map) = 0;+    if (h->preservation_map) {+	khint_t k;++	for (k = kh_begin(h->preservation_map);+	     k != kh_end(h->preservation_map);+	     k++) {+	    const char *key;+	    khash_t(map) *pmap = h->preservation_map;+++	    if (!kh_exist(pmap, k))+		continue;++	    key = kh_key(pmap, k);+	    BLOCK_APPEND(map, key, 2);++	    switch(CRAM_KEY(key[0], key[1])) {+	    case CRAM_KEY('M','I'):+		BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);+		break;++	    case CRAM_KEY('U','I'):+		BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);+		break;++	    case CRAM_KEY('P','I'):+		BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);+		break;++	    case CRAM_KEY('A','P'):+		BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);+		break;++	    case CRAM_KEY('R','N'):+		BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);+		break;++	    case CRAM_KEY('R','R'):+		BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);+		break;++	    case CRAM_KEY('S','M'): {+		char smat[5], *mp = smat;+		*mp++ =+		    (sub_idx("CGTN", h->substitution_matrix[0][0]) << 6) |+		    (sub_idx("CGTN", h->substitution_matrix[0][1]) << 4) |+		    (sub_idx("CGTN", h->substitution_matrix[0][2]) << 2) |+		    (sub_idx("CGTN", h->substitution_matrix[0][3]) << 0);+		*mp++ =+		    (sub_idx("AGTN", h->substitution_matrix[1][0]) << 6) |+		    (sub_idx("AGTN", h->substitution_matrix[1][1]) << 4) |+		    (sub_idx("AGTN", h->substitution_matrix[1][2]) << 2) |+		    (sub_idx("AGTN", h->substitution_matrix[1][3]) << 0);+		*mp++ =+		    (sub_idx("ACTN", h->substitution_matrix[2][0]) << 6) |+		    (sub_idx("ACTN", h->substitution_matrix[2][1]) << 4) |+		    (sub_idx("ACTN", h->substitution_matrix[2][2]) << 2) |+		    (sub_idx("ACTN", h->substitution_matrix[2][3]) << 0);+		*mp++ =+		    (sub_idx("ACGN", h->substitution_matrix[3][0]) << 6) |+		    (sub_idx("ACGN", h->substitution_matrix[3][1]) << 4) |+		    (sub_idx("ACGN", h->substitution_matrix[3][2]) << 2) |+		    (sub_idx("ACGN", h->substitution_matrix[3][3]) << 0);+		*mp++ =+		    (sub_idx("ACGT", h->substitution_matrix[4][0]) << 6) |+		    (sub_idx("ACGT", h->substitution_matrix[4][1]) << 4) |+		    (sub_idx("ACGT", h->substitution_matrix[4][2]) << 2) |+		    (sub_idx("ACGT", h->substitution_matrix[4][3]) << 0);+		BLOCK_APPEND(map, smat, 5);+		break;+	    }++	    case CRAM_KEY('T','D'): {+		itf8_put_blk(map, BLOCK_SIZE(h->TD_blk));+		BLOCK_APPEND(map,+			     BLOCK_DATA(h->TD_blk),+			     BLOCK_SIZE(h->TD_blk));+		break;+	    }++	    default:+		fprintf(stderr, "Unknown preservation key '%.2s'\n", key);+		break;+	    }++	    mc++;+        }+    }+    itf8_put_blk(cb, BLOCK_SIZE(map) + itf8_size(mc));+    itf8_put_blk(cb, mc);    +    BLOCK_APPEND(cb, BLOCK_DATA(map), BLOCK_SIZE(map));+    +    /* rec encoding map */+    mc = 0;+    BLOCK_SIZE(map) = 0;+    if (h->codecs[DS_BF]) {+	if (-1 == h->codecs[DS_BF]->store(h->codecs[DS_BF], map, "BF",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_CF]) {+	if (-1 == h->codecs[DS_CF]->store(h->codecs[DS_CF], map, "CF",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_RL]) {+	if (-1 == h->codecs[DS_RL]->store(h->codecs[DS_RL], map, "RL",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_AP]) {+	if (-1 == h->codecs[DS_AP]->store(h->codecs[DS_AP], map, "AP",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_RG]) {+	if (-1 == h->codecs[DS_RG]->store(h->codecs[DS_RG], map, "RG",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_MF]) {+	if (-1 == h->codecs[DS_MF]->store(h->codecs[DS_MF], map, "MF",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_NS]) {+	if (-1 == h->codecs[DS_NS]->store(h->codecs[DS_NS], map, "NS",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_NP]) {+	if (-1 == h->codecs[DS_NP]->store(h->codecs[DS_NP], map, "NP",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_TS]) {+	if (-1 == h->codecs[DS_TS]->store(h->codecs[DS_TS], map, "TS",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_NF]) {+	if (-1 == h->codecs[DS_NF]->store(h->codecs[DS_NF], map, "NF",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_TC]) {+	if (-1 == h->codecs[DS_TC]->store(h->codecs[DS_TC], map, "TC",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_TN]) {+	if (-1 == h->codecs[DS_TN]->store(h->codecs[DS_TN], map, "TN",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_TL]) {+	if (-1 == h->codecs[DS_TL]->store(h->codecs[DS_TL], map, "TL",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_FN]) {+	if (-1 == h->codecs[DS_FN]->store(h->codecs[DS_FN], map, "FN",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_FC]) {+	if (-1 == h->codecs[DS_FC]->store(h->codecs[DS_FC], map, "FC",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_FP]) {+	if (-1 == h->codecs[DS_FP]->store(h->codecs[DS_FP], map, "FP",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_BS]) {+	if (-1 == h->codecs[DS_BS]->store(h->codecs[DS_BS], map, "BS",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_IN]) {+	if (-1 == h->codecs[DS_IN]->store(h->codecs[DS_IN], map, "IN",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_DL]) {+	if (-1 == h->codecs[DS_DL]->store(h->codecs[DS_DL], map, "DL",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_BA]) {+	if (-1 == h->codecs[DS_BA]->store(h->codecs[DS_BA], map, "BA",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_BB]) {+	if (-1 == h->codecs[DS_BB]->store(h->codecs[DS_BB], map, "BB",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_MQ]) {+	if (-1 == h->codecs[DS_MQ]->store(h->codecs[DS_MQ], map, "MQ",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_RN]) {+	if (-1 == h->codecs[DS_RN]->store(h->codecs[DS_RN], map, "RN",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_QS]) {+	if (-1 == h->codecs[DS_QS]->store(h->codecs[DS_QS], map, "QS",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_QQ]) {+	if (-1 == h->codecs[DS_QQ]->store(h->codecs[DS_QQ], map, "QQ",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_RI]) {+	if (-1 == h->codecs[DS_RI]->store(h->codecs[DS_RI], map, "RI",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (CRAM_MAJOR_VERS(fd->version) != 1) {+	if (h->codecs[DS_SC]) {+	    if (-1 == h->codecs[DS_SC]->store(h->codecs[DS_SC], map, "SC",+					      fd->version))+		return NULL;+	    mc++;+	}+	if (h->codecs[DS_RS]) {+	    if (-1 == h->codecs[DS_RS]->store(h->codecs[DS_RS], map, "RS",+					      fd->version))+		return NULL;+	    mc++;+	}+	if (h->codecs[DS_PD]) {+	    if (-1 == h->codecs[DS_PD]->store(h->codecs[DS_PD], map, "PD",+					      fd->version))+		return NULL;+	    mc++;+	}+	if (h->codecs[DS_HC]) {+	    if (-1 == h->codecs[DS_HC]->store(h->codecs[DS_HC], map, "HC",+					      fd->version))+		return NULL;+	    mc++;+	}+    }+    if (h->codecs[DS_TM]) {+	if (-1 == h->codecs[DS_TM]->store(h->codecs[DS_TM], map, "TM",+					  fd->version))+	    return NULL;+	mc++;+    }+    if (h->codecs[DS_TV]) {+	if (-1 == h->codecs[DS_TV]->store(h->codecs[DS_TV], map, "TV",+					  fd->version))+	    return NULL;+	mc++;+    }+    itf8_put_blk(cb, BLOCK_SIZE(map) + itf8_size(mc));+    itf8_put_blk(cb, mc);    +    BLOCK_APPEND(cb, BLOCK_DATA(map), BLOCK_SIZE(map));++    /* tag encoding map */+#if 0+    mp = map; mc = 0;+    if (h->tag_encoding_map) {+        HashItem *hi;+        HashIter *iter = HashTableIterCreate();+	if (!iter)+	    return NULL;++        while ((hi = HashTableIterNext(h->tag_encoding_map, iter))) {+            cram_map *m = hi->data.p;+	    int sz;++	    mp += itf8_put(mp, (hi->key[0]<<16)|(hi->key[1]<<8)|hi->key[2]);+	    if (-1 == (sz = m->codec->store(m->codec, mp, NULL, fd->version)))+		return NULL;+	    mp += sz;+	    mc++;+        }++        HashTableIterDestroy(iter);+    }+#else+    mc = 0;+    BLOCK_SIZE(map) = 0;+    if (c->tags_used) {+	khint_t k;++#define TAG_ID(a) ((#a[0]<<8)+#a[1])++	for (k = kh_begin(c->tags_used); k != kh_end(c->tags_used); k++) {+	    int key;+	    if (!kh_exist(c->tags_used, k))+		continue;++	    mc++;+	    itf8_put_blk(map, kh_key(c->tags_used, k));++	    // use block content id 4+	    switch((key = kh_key(c->tags_used, k)) & 0xff) {+	    case 'Z': case 'H':+		// string as byte_array_stop+		if (CRAM_MAJOR_VERS(fd->version) == 1) {+		    BLOCK_APPEND(map,+				 "\005" // BYTE_ARRAY_STOP+				 "\005" // len+				 "\t"   // stop-byte is also SAM separator+				 DS_aux_S "\000\000\000",+				 7);+		} else {+		    if (key>>8 == TAG_ID(OQ))+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_OQ_S,+				     4);+		    else if (key>>8 == TAG_ID(BQ))+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_BQ_S,+				     4);+		    else if (key>>8 == TAG_ID(BD))+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_BD_S,+				     4);+		    else if (key>>8 == TAG_ID(BI))+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_BI_S,+				     4);+		    else if ((key>>8 == TAG_ID(Q2)) ||+			     (key>>8 == TAG_ID(U2)) ||+			     (key>>8 == TAG_ID(QT)) ||+			     (key>>8 == TAG_ID(CQ)))+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_oq_S,+				     4);+		    else if ((key>>8 == TAG_ID(R2)) ||+			     (key>>8 == TAG_ID(E2)) ||+			     (key>>8 == TAG_ID(CS)) ||+			     (key>>8 == TAG_ID(BC)) ||+			     (key>>8 == TAG_ID(RT)))+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_os_S,+				     4);+		    else+			BLOCK_APPEND(map,+				     "\005" // BYTE_ARRAY_STOP+				     "\002" // len+				     "\t"   // stop-byte is also SAM separator+				     DS_aux_oz_S,+				     4);+		}+		break;++	    case 'A': case 'c': case 'C':+		// byte array len, 1 byte+		BLOCK_APPEND(map,+			     "\004" // BYTE_ARRAY_LEN+			     "\011" // length+			     "\003" // HUFFMAN (len)+			     "\004" // huffman-len+			     "\001" // 1 symbol+			     "\001" // symbol=1 byte value+			     "\001" // 1 length+			     "\000" // length=0+			     "\001" // EXTERNAL (val)+			     "\001" // external-len+			     DS_aux_S,// content-id+			     11);+		break;++	    case 's': case 'S':+		// byte array len, 2 byte+		BLOCK_APPEND(map,+			     "\004" // BYTE_ARRAY_LEN+			     "\011" // length+			     "\003" // HUFFMAN (len)+			     "\004" // huffman-len+			     "\001" // 1 symbol+			     "\002" // symbol=2 byte value+			     "\001" // 1 length+			     "\000" // length=0+			     "\001" // EXTERNAL (val)+			     "\001" // external-len+			     DS_aux_S,// content-id+			     11);+		break;++	    case 'i': case 'I': case 'f':+		// byte array len, 4 byte+		BLOCK_APPEND(map,+			     "\004" // BYTE_ARRAY_LEN+			     "\011" // length+			     "\003" // HUFFMAN (len)+			     "\004" // huffman-len+			     "\001" // 1 symbol+			     "\004" // symbol=4 byte value+			     "\001" // 1 length+			     "\000" // length=0+			     "\001" // EXTERNAL (val)+			     "\001" // external-len+			     DS_aux_S,// content-id+			     11);+		break;++	    case 'B':+		// Byte array of variable size, but we generate our tag+		// byte stream at the wrong stage (during reading and not+		// after slice header construction). So we use+		// BYTE_ARRAY_LEN with the length codec being external+		// too.+		if ((key>>8 == TAG_ID(FZ)) || (key>>8 == TAG_ID(ZM)))+		    BLOCK_APPEND(map,+				 "\004" // BYTE_ARRAY_LEN+				 "\006" // length+				 "\001" // EXTERNAL (len)+				 "\001" // external-len+				 DS_aux_FZ_S // content-id+				 "\001" // EXTERNAL (val)+				 "\001" // external-len+				 DS_aux_FZ_S,// content-id+				 8);+		else+		    BLOCK_APPEND(map,+				 "\004" // BYTE_ARRAY_LEN+				 "\006" // length+				 "\001" // EXTERNAL (len)+				 "\001" // external-len+				 DS_aux_S // content-id+				 "\001" // EXTERNAL (val)+				 "\001" // external-len+				 DS_aux_S,// content-id+				 8);+		break;++	    default:+		fprintf(stderr, "Unsupported SAM aux type '%c'\n",+			kh_key(c->tags_used, k) & 0xff);+	    }+	    //mp += m->codec->store(m->codec, mp, NULL, fd->version);+	}+    }+#endif+    itf8_put_blk(cb, BLOCK_SIZE(map) + itf8_size(mc));+    itf8_put_blk(cb, mc);    +    BLOCK_APPEND(cb, BLOCK_DATA(map), BLOCK_SIZE(map));++    if (fd->verbose)+	fprintf(stderr, "Wrote compression block header in %d bytes\n",+		(int)BLOCK_SIZE(cb));++    BLOCK_UPLEN(cb);++    cram_free_block(map);++    return cb;+}+++/*+ * Encodes a slice compression header. + *+ * Returns cram_block on success+ *         NULL on failure+ */+cram_block *cram_encode_slice_header(cram_fd *fd, cram_slice *s) {+    char *buf;+    char *cp;+    cram_block *b = cram_new_block(MAPPED_SLICE, 0);+    int j;++    if (!b)+	return NULL;++    if (NULL == (cp = buf = malloc(16+5*(8+s->hdr->num_blocks)))) {+	cram_free_block(b);+	return NULL;+    }++    cp += itf8_put(cp, s->hdr->ref_seq_id);+    cp += itf8_put(cp, s->hdr->ref_seq_start);+    cp += itf8_put(cp, s->hdr->ref_seq_span);+    cp += itf8_put(cp, s->hdr->num_records);+    if (CRAM_MAJOR_VERS(fd->version) == 2)+	cp += itf8_put(cp, s->hdr->record_counter);+    else if (CRAM_MAJOR_VERS(fd->version) >= 3)+	cp += ltf8_put(cp, s->hdr->record_counter);+    cp += itf8_put(cp, s->hdr->num_blocks);+    cp += itf8_put(cp, s->hdr->num_content_ids);+    for (j = 0; j < s->hdr->num_content_ids; j++) {+	cp += itf8_put(cp, s->hdr->block_content_ids[j]);+    }+    if (s->hdr->content_type == MAPPED_SLICE)+	cp += itf8_put(cp, s->hdr->ref_base_id);++    if (CRAM_MAJOR_VERS(fd->version) != 1) {+	memcpy(cp, s->hdr->md5, 16); cp += 16;+    }+    +    assert(cp-buf <= 16+5*(8+s->hdr->num_blocks));++    b->data = (unsigned char *)buf;+    b->comp_size = b->uncomp_size = cp-buf;++    return b;+}+++/*+ * Encodes a single read.+ *+ * Returns 0 on success+ *        -1 on failure+ */+static int cram_encode_slice_read(cram_fd *fd,+				  cram_container *c,+				  cram_block_compression_hdr *h,+				  cram_slice *s,+				  cram_record *cr,+				  int *last_pos) {+    int r = 0;+    int32_t i32;+    unsigned char uc;++    //fprintf(stderr, "Encode seq %d, %d/%d FN=%d, %s\n", rec, core->byte, core->bit, cr->nfeature, s->name_ds->str + cr->name);++    //printf("BF=0x%x\n", cr->flags);+    //	    bf = cram_flag_swap[cr->flags];+    i32 = fd->cram_flag_swap[cr->flags & 0xfff];+    r |= h->codecs[DS_BF]->encode(s, h->codecs[DS_BF], (char *)&i32, 1);++    i32 = cr->cram_flags & CRAM_FLAG_MASK;+    r |= h->codecs[DS_CF]->encode(s, h->codecs[DS_CF], (char *)&i32, 1);++    if (CRAM_MAJOR_VERS(fd->version) != 1 && s->hdr->ref_seq_id == -2)+	r |= h->codecs[DS_RI]->encode(s, h->codecs[DS_RI], (char *)&cr->ref_id, 1);++    r |= h->codecs[DS_RL]->encode(s, h->codecs[DS_RL], (char *)&cr->len, 1);++    if (c->pos_sorted) {+	i32 = cr->apos - *last_pos;+	r |= h->codecs[DS_AP]->encode(s, h->codecs[DS_AP], (char *)&i32, 1);+	*last_pos = cr->apos;+    } else {+	i32 = cr->apos;+	r |= h->codecs[DS_AP]->encode(s, h->codecs[DS_AP], (char *)&i32, 1);+    }++    r |= h->codecs[DS_RG]->encode(s, h->codecs[DS_RG], (char *)&cr->rg, 1);++    if (c->comp_hdr->read_names_included) {+	// RN codec: Already stored in block[3].+    }++    if (cr->cram_flags & CRAM_FLAG_DETACHED) {+	i32 = cr->mate_flags;+	r |= h->codecs[DS_MF]->encode(s, h->codecs[DS_MF], (char *)&i32, 1);++	if (!c->comp_hdr->read_names_included) {+	    // RN codec: Already stored in block[3].+	}++	r |= h->codecs[DS_NS]->encode(s, h->codecs[DS_NS],+				      (char *)&cr->mate_ref_id, 1);++	r |= h->codecs[DS_NP]->encode(s, h->codecs[DS_NP],+				      (char *)&cr->mate_pos, 1);++	r |= h->codecs[DS_TS]->encode(s, h->codecs[DS_TS],+				      (char *)&cr->tlen, 1);+    } else if (cr->cram_flags & CRAM_FLAG_MATE_DOWNSTREAM) {+	r |= h->codecs[DS_NF]->encode(s, h->codecs[DS_NF],+				      (char *)&cr->mate_line, 1);+    }++    /* Aux tags */+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	int j;+	uc = cr->ntags;+	r |= h->codecs[DS_TC]->encode(s, h->codecs[DS_TC], (char *)&uc, 1);++	for (j = 0; j < cr->ntags; j++) {+	    uint32_t i32 = s->TN[cr->TN_idx + j]; // id+	    r |= h->codecs[DS_TN]->encode(s, h->codecs[DS_TN], (char *)&i32, 1);+	}+    } else {+	r |= h->codecs[DS_TL]->encode(s, h->codecs[DS_TL], (char *)&cr->TL, 1);+    }++    // qual+    // QS codec : Already stored in block[2].++    // features (diffs)+    if (!(cr->flags & BAM_FUNMAP)) {+	int prev_pos = 0, j;++	r |= h->codecs[DS_FN]->encode(s, h->codecs[DS_FN],+				      (char *)&cr->nfeature, 1);+	for (j = 0; j < cr->nfeature; j++) {+	    cram_feature *f = &s->features[cr->feature + j];++	    uc = f->X.code;+	    r |= h->codecs[DS_FC]->encode(s, h->codecs[DS_FC], (char *)&uc, 1);+	    i32 = f->X.pos - prev_pos;+	    r |= h->codecs[DS_FP]->encode(s, h->codecs[DS_FP], (char *)&i32, 1);+	    prev_pos = f->X.pos;++	    switch(f->X.code) {+		//char *seq;++	    case 'X':+		//fprintf(stderr, "    FC=%c FP=%d base=%d\n", f->X.code, i32, f->X.base);+		+		uc = f->X.base;+		r |= h->codecs[DS_BS]->encode(s, h->codecs[DS_BS],+					      (char *)&uc, 1);+		break;+	    case 'S':+		// Already done+//		r |= h->codecs[DS_SC]->encode(s, h->codecs[DS_SC],+//					      BLOCK_DATA(s->soft_blk) + f->S.seq_idx,+//					      f->S.len);++//		if (IS_CRAM_3_VERS(fd)) {+//		    r |= h->codecs[DS_BB]->encode(s, h->codecs[DS_BB], +//						  BLOCK_DATA(s->seqs_blk) + f->S.seq_idx,+//						  f->S.len);+//		}+		break;+	    case 'I':+		//seq = DSTRING_STR(s->seqs_ds) + f->S.seq_idx;+		//r |= h->codecs[DS_IN]->encode(s, h->codecs[DS_IN],+		//			     seq, f->S.len);+//		if (IS_CRAM_3_VERS(fd)) {+//		    r |= h->codecs[DS_BB]->encode(s, h->codecs[DS_BB], +//						  BLOCK_DATA(s->seqs_blk) + f->I.seq_idx,+//						  f->I.len);+//		}+		break;+	    case 'i':+		uc = f->i.base;+		r |= h->codecs[DS_BA]->encode(s, h->codecs[DS_BA],+					      (char *)&uc, 1);+		//seq = DSTRING_STR(s->seqs_ds) + f->S.seq_idx;+		//r |= h->codecs[DS_IN]->encode(s, h->codecs[DS_IN], +		//			     seq, 1);+		break;+	    case 'D':+		i32 = f->D.len;+		r |= h->codecs[DS_DL]->encode(s, h->codecs[DS_DL],+					      (char *)&i32, 1);+		break;++	    case 'B':+		//		    // Used when we try to store a non ACGTN base or an N+		//		    // that aligns against a non ACGTN reference++		uc  = f->B.base;+		r |= h->codecs[DS_BA]->encode(s, h->codecs[DS_BA],+					      (char *)&uc, 1);++		//                  Already added+		//		    uc  = f->B.qual;+		//		    r |= h->codecs[DS_QS]->encode(s, h->codecs[DS_QS], +		//					     (char *)&uc, 1);+		break;++	    case 'b':+		// string of bases+		r |= h->codecs[DS_BB]->encode(s, h->codecs[DS_BB], +					      (char *)BLOCK_DATA(s->seqs_blk)+					              + f->b.seq_idx,+					      f->b.len);+		break;++	    case 'Q':+		//                  Already added+		//		    uc  = f->B.qual;+		//		    r |= h->codecs[DS_QS]->encode(s, h->codecs[DS_QS], +		//					     (char *)&uc, 1);+		break;++	    case 'N':+		i32 = f->N.len;+		r |= h->codecs[DS_RS]->encode(s, h->codecs[DS_RS],+					      (char *)&i32, 1);+		break;+		    +	    case 'P':+		i32 = f->P.len;+		r |= h->codecs[DS_PD]->encode(s, h->codecs[DS_PD],+					      (char *)&i32, 1);+		break;+		    +	    case 'H':+		i32 = f->H.len;+		r |= h->codecs[DS_HC]->encode(s, h->codecs[DS_HC],+					      (char *)&i32, 1);+		break;+		    ++	    default:+		fprintf(stderr, "unhandled feature code %c\n",+			f->X.code);+		return -1;+	    }+	}++	r |= h->codecs[DS_MQ]->encode(s, h->codecs[DS_MQ],+				      (char *)&cr->mqual, 1);+    } else {+	char *seq = (char *)BLOCK_DATA(s->seqs_blk) + cr->seq;+	if (cr->len)+	    r |= h->codecs[DS_BA]->encode(s, h->codecs[DS_BA], seq, cr->len);+    }++    return r ? -1 : 0;+}+++/*+ * Applies various compression methods to specific blocks, depending on+ * known observations of how data series compress.+ *+ * Returns 0 on success+ *        -1 on failure+ */+static int cram_compress_slice(cram_fd *fd, cram_slice *s) {+    int level = fd->level, i;+    int method = 1<<GZIP | 1<<GZIP_RLE, methodF = method;++    /* Compress the CORE Block too, with minimal zlib level */+    if (level > 5 && s->block[0]->uncomp_size > 500)+	cram_compress_block(fd, s->block[0], NULL, GZIP, 1);+ +    if (fd->use_bz2)+	method |= 1<<BZIP2;++    if (fd->use_rans)+	method |= (1<<RANS0) | (1<<RANS1);++    if (fd->use_lzma)+	method |= (1<<LZMA);++    /* Faster method for data series we only need entropy encoding on */+    methodF = method & ~(1<<GZIP | 1<<BZIP2 | 1<<LZMA);+    if (level >= 6)+	methodF = method;+    ++    /* Specific compression methods for certain block types */+    if (cram_compress_block(fd, s->block[DS_IN], fd->m[DS_IN], //IN (seq)+			    method, level))+	return -1;++    if (fd->level == 0) {+	/* Do nothing */+    } else if (fd->level == 1) {+	if (cram_compress_block(fd, s->block[DS_QS], fd->m[DS_QS],+				methodF, 1))+	    return -1;+	for (i = DS_aux; i <= DS_aux_oz; i++) {+	    if (s->block[i])+		if (cram_compress_block(fd, s->block[i], fd->m[i],+					method, 1))+		    return -1;+	}+    } else if (fd->level < 3) {+	if (cram_compress_block(fd, s->block[DS_QS], fd->m[DS_QS],+				method, 1))+	    return -1;+	if (cram_compress_block(fd, s->block[DS_BA], fd->m[DS_BA],+				method, 1))+	    return -1;+	if (s->block[DS_BB])+	    if (cram_compress_block(fd, s->block[DS_BB], fd->m[DS_BB],+				    method, 1))+	    return -1;+	for (i = DS_aux; i <= DS_aux_oz; i++) {+	    if (s->block[i])+		if (cram_compress_block(fd, s->block[i], fd->m[i],+					method, level))+		    return -1;+	}+    } else {+	if (cram_compress_block(fd, s->block[DS_QS], fd->m[DS_QS],+				method, level))+	    return -1;+	if (cram_compress_block(fd, s->block[DS_BA], fd->m[DS_BA],+				method, level))+	    return -1;+	if (s->block[DS_BB])+	    if (cram_compress_block(fd, s->block[DS_BB], fd->m[DS_BB],+				    method, level))+	    return -1;+	for (i = DS_aux; i <= DS_aux_oz; i++) {+	    if (s->block[i])+		if (cram_compress_block(fd, s->block[i], fd->m[i],+					method, level))+		    return -1;+	}+    }++    // NAME: best is generally xz, bzip2, zlib then rans1+    // It benefits well from a little bit extra compression level.+    if (cram_compress_block(fd, s->block[DS_RN], fd->m[DS_RN],+			    method & ~(1<<RANS0 | 1<<GZIP_RLE),+			    MIN(9,level)))+	return -1;++    // NS shows strong local correlation as rearrangements are localised+    if (s->block[DS_NS] != s->block[0])+	if (cram_compress_block(fd, s->block[DS_NS], fd->m[DS_NS],+				method, level))+	    return -1;+++    /*+     * Minimal compression of any block still uncompressed, bar CORE+     */+    {+	int i;+	for (i = 1; i < DS_END; i++) {+	    if (!s->block[i] || s->block[i] == s->block[0])+		continue;++	    // fast methods only+	    if (s->block[i]->method == RAW) {+		cram_compress_block(fd, s->block[i], fd->m[i],+				    methodF, level);+	    }+	}+    }++    return 0;+}++/*+ * Encodes a single slice from a container+ *+ * Returns 0 on success+ *        -1 on failure+ */+static int cram_encode_slice(cram_fd *fd, cram_container *c,+			     cram_block_compression_hdr *h, cram_slice *s) {+    int rec, r = 0, last_pos;+    int embed_ref;+    enum cram_DS_ID id;++    embed_ref = fd->embed_ref && s->hdr->ref_seq_id != -1 ? 1 : 0;++    /*+     * Slice external blocks:+     * ID 0 => base calls (insertions, soft-clip)+     * ID 1 => qualities+     * ID 2 => names+     * ID 3 => TS (insert size), NP (next frag)+     * ID 4 => tag values+     * ID 6 => tag IDs (TN), if CRAM_V1.0+     * ID 7 => TD tag dictionary, if !CRAM_V1.0+     */++    /* Create cram slice header */+    s->hdr->ref_base_id = embed_ref ? DS_ref : -1;+    s->hdr->record_counter = c->num_records + c->record_counter;+    c->num_records += s->hdr->num_records;++    s->block = calloc(DS_END, sizeof(s->block[0]));+    s->hdr->block_content_ids = malloc(DS_END * sizeof(int32_t));+    if (!s->block || !s->hdr->block_content_ids)+	return -1;++    // Create first fixed blocks, always external.+    // CORE+    if (!(s->block[0] = cram_new_block(CORE, 0)))+	return -1;++    // TN block for CRAM v1+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	if (h->codecs[DS_TN]->codec == E_EXTERNAL) {+	    if (!(s->block[DS_TN] = cram_new_block(EXTERNAL,DS_TN))) return -1;+	    h->codecs[DS_TN]->external.content_id = DS_TN;+	} else {+	    s->block[DS_TN] = s->block[0];+	}+	s->block[DS_TN] = s->block[DS_TN];+    }++    // Embedded reference+    if (embed_ref) {+	if (!(s->block[DS_ref] = cram_new_block(EXTERNAL, DS_ref)))+	    return -1;+	s->ref_id = DS_ref; // needed?+	BLOCK_APPEND(s->block[DS_ref],+		     c->ref + c->first_base - c->ref_start,+		     c->last_base - c->first_base + 1);+    }++    /*+     * All the data-series blocks if appropriate. +     */+    for (id = DS_BF; id < DS_TN; id++) {+	if (h->codecs[id] && (h->codecs[id]->codec == E_EXTERNAL ||+			      h->codecs[id]->codec == E_BYTE_ARRAY_STOP ||+			      h->codecs[id]->codec == E_BYTE_ARRAY_LEN)) {+	    switch (h->codecs[id]->codec) {+	    case E_EXTERNAL:+		if (!(s->block[id] = cram_new_block(EXTERNAL, id)))+		    return -1;+		h->codecs[id]->external.content_id = id;+		break;++	    case E_BYTE_ARRAY_STOP:+		if (!(s->block[id] = cram_new_block(EXTERNAL, id)))+		    return -1;+		h->codecs[id]->byte_array_stop.content_id = id;+		break;++	    case E_BYTE_ARRAY_LEN: {+		cram_codec *cc;++		cc = h->codecs[id]->e_byte_array_len.len_codec;+		if (cc->codec == E_EXTERNAL) {+		    int eid = cc->external.content_id;+		    if (!(s->block[eid] = cram_new_block(EXTERNAL, eid)))+			return -1;+		    cc->external.content_id = eid;+		    cc->out = s->block[eid];+		}++		cc = h->codecs[id]->e_byte_array_len.val_codec;+		if (cc->codec == E_EXTERNAL) {+		    int eid = cc->external.content_id;+		    if (!s->block[eid])+			if (!(s->block[eid] = cram_new_block(EXTERNAL, eid)))+			    return -1;+		    cc->external.content_id = eid;+		    cc->out = s->block[eid];+		}+		break;+	    }+	    default:+		break;+	    }+	} else {+	    if (!(id == DS_BB && !h->codecs[DS_BB]))+		s->block[id] = s->block[0];+	}+	if (h->codecs[id])+	    h->codecs[id]->out = s->block[id];+    }++    /* Encode reads */+    last_pos = s->hdr->ref_seq_start;+    for (rec = 0; rec < s->hdr->num_records; rec++) {+	cram_record *cr = &s->crecs[rec];+	if (cram_encode_slice_read(fd, c, h, s, cr, &last_pos) == -1)+	    return -1;+    }++    s->block[0]->uncomp_size = s->block[0]->byte + (s->block[0]->bit < 7);+    s->block[0]->comp_size = s->block[0]->uncomp_size;++    // Make sure the fixed blocks point to the correct sources+    s->block[DS_IN] = s->base_blk; s->base_blk = NULL;+    s->block[DS_QS] = s->qual_blk; s->qual_blk = NULL;+    s->block[DS_RN] = s->name_blk; s->name_blk = NULL;+    s->block[DS_SC] = s->soft_blk; s->soft_blk = NULL;+    s->block[DS_aux]= s->aux_blk;  s->aux_blk  = NULL;+    s->block[DS_aux_OQ]= s->aux_OQ_blk;  s->aux_OQ_blk  = NULL;+    s->block[DS_aux_BQ]= s->aux_BQ_blk;  s->aux_BQ_blk  = NULL;+    s->block[DS_aux_BD]= s->aux_BD_blk;  s->aux_BD_blk  = NULL;+    s->block[DS_aux_BI]= s->aux_BI_blk;  s->aux_BI_blk  = NULL;+    s->block[DS_aux_FZ]= s->aux_FZ_blk;  s->aux_FZ_blk  = NULL;+    s->block[DS_aux_oq]= s->aux_oq_blk;  s->aux_oq_blk  = NULL;+    s->block[DS_aux_os]= s->aux_os_blk;  s->aux_os_blk  = NULL;+    s->block[DS_aux_oz]= s->aux_oz_blk;  s->aux_oz_blk  = NULL;++    // Ensure block sizes are up to date.+    for (id = 1; id < DS_END; id++) {+	if (!s->block[id] || s->block[id] == s->block[0])+	    continue;++	if (s->block[id]->uncomp_size == 0)+	    BLOCK_UPLEN(s->block[id]);+    }++    // Compress it all+    if (cram_compress_slice(fd, s) == -1)+	return -1;++    // Collapse empty blocks and create hdr_block+    {+	int i, j;+	for (i = j = 1; i < DS_END; i++) {+	    if (!s->block[i] || s->block[i] == s->block[0])+		continue;+	    if (s->block[i]->uncomp_size == 0) {+		cram_free_block(s->block[i]);+		s->block[i] = NULL;+		continue;+	    }+	    s->block[j] = s->block[i];+	    s->hdr->block_content_ids[j-1] = s->block[i]->content_id;+	    j++;+	}+	s->hdr->num_content_ids = j-1;+	s->hdr->num_blocks = j;++	if (!(s->hdr_block = cram_encode_slice_header(fd, s)))+	    return -1;+    }++    return r ? -1 : 0;+}++/*+ * Encodes all slices in a container into blocks.+ * Returns 0 on success+ *        -1 on failure+ */+int cram_encode_container(cram_fd *fd, cram_container *c) {+    int i, j, slice_offset;+    cram_block_compression_hdr *h = c->comp_hdr;+    cram_block *c_hdr;+    int multi_ref = 0;+    int r1, r2, sn, nref;+    spare_bams *spares;++    /* Cache references up-front if we have unsorted access patterns */+    pthread_mutex_lock(&fd->ref_lock);+    nref = fd->refs->nref;+    pthread_mutex_unlock(&fd->ref_lock);++    if (!fd->no_ref && c->refs_used) {+	for (i = 0; i < nref; i++) {+	    if (c->refs_used[i])+		cram_get_ref(fd, i, 1, 0);+	}+    }++    /* To create M5 strings */+    /* Fetch reference sequence */+    if (!fd->no_ref) {+	bam_seq_t *b = c->bams[0];+	char *ref;++	ref = cram_get_ref(fd, bam_ref(b), 1, 0);+	if (!ref && bam_ref(b) >= 0) {+	    fprintf(stderr, "Failed to load reference #%d\n", bam_ref(b));+	    return -1;+	}+	if ((c->ref_id = bam_ref(b)) >= 0) {+	    c->ref_seq_id = c->ref_id;+	    c->ref       = fd->refs->ref_id[c->ref_seq_id]->seq;+	    c->ref_start = 1;+	    c->ref_end   = fd->refs->ref_id[c->ref_seq_id]->length;+	} else {+	    c->ref_seq_id = c->ref_id; // FIXME remove one var!+	}+    } else {+	c->ref_id = bam_ref(c->bams[0]);+	cram_ref_incr(fd->refs, c->ref_id);+	c->ref_seq_id = c->ref_id;+    }++    /* Turn bams into cram_records and gather basic stats */+    for (r1 = sn = 0; r1 < c->curr_c_rec; sn++) {+	cram_slice *s = c->slices[sn];+	int first_base = INT_MAX, last_base = INT_MIN;++	assert(sn < c->curr_slice);++	/* FIXME: we could create our slice objects here too instead of+	 * in cram_put_bam_seq. It's more natural here and also this is+	 * bit is threaded so it's less work in the main thread.+	 */++	for (r2 = 0; r1 < c->curr_c_rec && r2 < c->max_rec; r1++, r2++) {+	    cram_record *cr = &s->crecs[r2];+	    bam_seq_t *b = c->bams[r1];++	    /* If multi-ref we need to cope with changing reference per seq */+	    if (c->multi_seq && !fd->no_ref) {+		if (bam_ref(b) != c->ref_seq_id && bam_ref(b) >= 0) {+		    if (c->ref_seq_id >= 0)+			cram_ref_decr(fd->refs, c->ref_seq_id);++		    if (!cram_get_ref(fd, bam_ref(b), 1, 0)) {+			fprintf(stderr, "Failed to load reference #%d\n",+				bam_ref(b));+			return -1;+		    }++		    c->ref_seq_id = bam_ref(b); // overwritten later by -2+		    assert(fd->refs->ref_id[c->ref_seq_id]->seq);+		    c->ref       = fd->refs->ref_id[c->ref_seq_id]->seq;+		    c->ref_start = 1;+		    c->ref_end   = fd->refs->ref_id[c->ref_seq_id]->length;+		}+	    }++	    if (process_one_read(fd, c, s, cr, b, r2) != 0)+		return -1;++	    if (first_base > cr->apos)+		first_base = cr->apos;++	    if (last_base < cr->aend)+		last_base = cr->aend;+	}++	if (c->multi_seq) {+	    s->hdr->ref_seq_id    = -2;+	    s->hdr->ref_seq_start = 0;+	    s->hdr->ref_seq_span  = 0;+	} else {+	    s->hdr->ref_seq_id    = c->ref_id;+	    s->hdr->ref_seq_start = first_base;+	    s->hdr->ref_seq_span  = last_base - first_base + 1;+	}+	s->hdr->num_records = r2;+    }++    if (c->multi_seq && !fd->no_ref) {+	if (c->ref_seq_id >= 0)+	    cram_ref_decr(fd->refs, c->ref_seq_id);+    }++    /* Link our bams[] array onto the spare bam list for reuse */+    spares = malloc(sizeof(*spares));+    pthread_mutex_lock(&fd->bam_list_lock);+    spares->bams = c->bams;+    spares->next = fd->bl;+    fd->bl = spares;+    pthread_mutex_unlock(&fd->bam_list_lock);+    c->bams = NULL;++    /* Detect if a multi-seq container */+    cram_stats_encoding(fd, c->stats[DS_RI]);+    multi_ref = c->stats[DS_RI]->nvals > 1;++    if (multi_ref) {+	if (fd->verbose)+	    fprintf(stderr, "Multi-ref container\n");+	c->ref_seq_id = -2;+	c->ref_seq_start = 0;+	c->ref_seq_span = 0;+    }+++    /* Compute MD5s */+    for (i = 0; i < c->curr_slice; i++) {+	cram_slice *s = c->slices[i];+	+	if (CRAM_MAJOR_VERS(fd->version) != 1) {+	    if (s->hdr->ref_seq_id >= 0 && c->multi_seq == 0 && !fd->no_ref) {+		hts_md5_context *md5 = hts_md5_init();+		if (!md5)+		    return -1;+		hts_md5_update(md5,+			   c->ref + s->hdr->ref_seq_start - c->ref_start,+			   s->hdr->ref_seq_span);+		hts_md5_final(s->hdr->md5, md5);+		hts_md5_destroy(md5);+	    } else {+		memset(s->hdr->md5, 0, 16);+	    }+	}+    }++    c->num_records = 0;+    c->num_blocks = 1; // cram_block_compression_hdr+    c->length = 0;++    //fprintf(stderr, "=== BF ===\n");+    h->codecs[DS_BF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_BF]),+					 c->stats[DS_BF], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== CF ===\n");+    h->codecs[DS_CF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_CF]),+					 c->stats[DS_CF], E_INT, NULL,+					 fd->version);+//    fprintf(stderr, "=== RN ===\n");+//    h->codecs[DS_RN] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RN]),+//				    c->stats[DS_RN], E_BYTE_ARRAY, NULL,+//				    fd->version);++    //fprintf(stderr, "=== AP ===\n");+    if (c->pos_sorted) {+	h->codecs[DS_AP] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_AP]),+					     c->stats[DS_AP], E_INT, NULL,+					     fd->version);+    } else {+	int p[2] = {0, c->max_apos};+	h->codecs[DS_AP] = cram_encoder_init(E_BETA, NULL, E_INT, p,+					     fd->version);+    }++    //fprintf(stderr, "=== RG ===\n");+    h->codecs[DS_RG] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RG]),+					 c->stats[DS_RG], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== MQ ===\n");+    h->codecs[DS_MQ] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_MQ]),+					 c->stats[DS_MQ], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== NS ===\n");+    h->codecs[DS_NS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_NS]),+					 c->stats[DS_NS], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== MF ===\n");+    h->codecs[DS_MF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_MF]),+					 c->stats[DS_MF], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== TS ===\n");+    h->codecs[DS_TS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TS]),+					 c->stats[DS_TS], E_INT, NULL,+					 fd->version);+    //fprintf(stderr, "=== NP ===\n");+    h->codecs[DS_NP] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_NP]),+					 c->stats[DS_NP], E_INT, NULL,+					 fd->version);+    //fprintf(stderr, "=== NF ===\n");+    h->codecs[DS_NF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_NF]),+					 c->stats[DS_NF], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== RL ===\n");+    h->codecs[DS_RL] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RL]),+					 c->stats[DS_RL], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== FN ===\n");+    h->codecs[DS_FN] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_FN]),+					 c->stats[DS_FN], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== FC ===\n");+    h->codecs[DS_FC] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_FC]),+					 c->stats[DS_FC], E_BYTE, NULL,+					 fd->version);++    //fprintf(stderr, "=== FP ===\n");+    h->codecs[DS_FP] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_FP]),+					 c->stats[DS_FP], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== DL ===\n");+    h->codecs[DS_DL] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_DL]),+					 c->stats[DS_DL], E_INT, NULL,+					 fd->version);++    //fprintf(stderr, "=== BA ===\n");+    h->codecs[DS_BA] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_BA]),+					 c->stats[DS_BA], E_BYTE, NULL,+					 fd->version);++    if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	cram_byte_array_len_encoder e;++	e.len_encoding = E_EXTERNAL;+	e.len_dat = (void *)DS_BB_len;+	//e.len_dat = (void *)DS_BB;++	e.val_encoding = E_EXTERNAL;+	e.val_dat = (void *)DS_BB;++	h->codecs[DS_BB] = cram_encoder_init(E_BYTE_ARRAY_LEN, NULL,+					     E_BYTE_ARRAY, (void *)&e,+					     fd->version);+    } else {+	h->codecs[DS_BB] = NULL;+    }++    //fprintf(stderr, "=== BS ===\n");+    h->codecs[DS_BS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_BS]),+					 c->stats[DS_BS], E_BYTE, NULL,+					 fd->version);++    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	h->codecs[DS_TL] = NULL;+	h->codecs[DS_RI] = NULL;+	h->codecs[DS_RS] = NULL;+	h->codecs[DS_PD] = NULL;+	h->codecs[DS_HC] = NULL;+	h->codecs[DS_SC] = NULL;++	//fprintf(stderr, "=== TC ===\n");+	h->codecs[DS_TC] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TC]),+					     c->stats[DS_TC], E_BYTE, NULL,+					     fd->version);++    //fprintf(stderr, "=== TN ===\n");+	h->codecs[DS_TN] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TN]),+					     c->stats[DS_TN], E_INT, NULL,+					     fd->version);+    } else {+	h->codecs[DS_TC] = NULL;+	h->codecs[DS_TN] = NULL;++	//fprintf(stderr, "=== TL ===\n");+	h->codecs[DS_TL] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TL]),+					     c->stats[DS_TL], E_INT, NULL,+					     fd->version);+++	//fprintf(stderr, "=== RI ===\n");+	h->codecs[DS_RI] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RI]),+					     c->stats[DS_RI], E_INT, NULL,+					     fd->version);++	//fprintf(stderr, "=== RS ===\n");+	h->codecs[DS_RS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RS]),+					     c->stats[DS_RS], E_INT, NULL,+					     fd->version);++	//fprintf(stderr, "=== PD ===\n");+	h->codecs[DS_PD] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_PD]),+					     c->stats[DS_PD], E_INT, NULL,+					     fd->version);++	//fprintf(stderr, "=== HC ===\n");+	h->codecs[DS_HC] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_HC]),+					     c->stats[DS_HC], E_INT, NULL,+					     fd->version);++	//fprintf(stderr, "=== SC ===\n");+	if (1) {+	    int i2[2] = {0, DS_SC};++	    h->codecs[DS_SC] = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,+						 E_BYTE_ARRAY, (void *)i2,+						 fd->version);+	} else {+	    // Appears to be no practical benefit to using this method,+	    // but it may work better if we start mixing SC, IN and BB+	    // elements into the same external block.+	    cram_byte_array_len_encoder e;++	    e.len_encoding = E_EXTERNAL;+	    e.len_dat = (void *)DS_SC_len;++	    e.val_encoding = E_EXTERNAL;+	    e.val_dat = (void *)DS_SC;++	    h->codecs[DS_SC] = cram_encoder_init(E_BYTE_ARRAY_LEN, NULL,+						 E_BYTE_ARRAY, (void *)&e,+						 fd->version);+	}+    }+    +    //fprintf(stderr, "=== IN ===\n");+    {+	int i2[2] = {0, DS_IN};+	h->codecs[DS_IN] = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,+					     E_BYTE_ARRAY, (void *)i2,+					     fd->version);+    }++    h->codecs[DS_QS] = cram_encoder_init(E_EXTERNAL, NULL, E_BYTE,+					 (void *)DS_QS,+					 fd->version);+    {+	int i2[2] = {0, DS_RN};+	h->codecs[DS_RN] = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,+					     E_BYTE_ARRAY, (void *)i2,+					     fd->version);+    }+++    /* Encode slices */+    for (i = 0; i < c->curr_slice; i++) {+	if (fd->verbose)+	    fprintf(stderr, "Encode slice %d\n", i);+	if (cram_encode_slice(fd, c, h, c->slices[i]) != 0)+	    return -1;+    }++    /* Create compression header */+    {+	h->ref_seq_id    = c->ref_seq_id;+	h->ref_seq_start = c->ref_seq_start;+	h->ref_seq_span  = c->ref_seq_span;+	h->num_records   = c->num_records;+	+	h->mapped_qs_included = 0;   // fixme+	h->unmapped_qs_included = 0; // fixme+	h->AP_delta = c->pos_sorted;+	// h->...  fixme+	memcpy(h->substitution_matrix, CRAM_SUBST_MATRIX, 20);++	if (!(c_hdr = cram_encode_compression_header(fd, c, h)))+	    return -1;+    }++    /* Compute landmarks */+    /* Fill out slice landmarks */+    c->num_landmarks = c->curr_slice;+    c->landmark = malloc(c->num_landmarks * sizeof(*c->landmark));+    if (!c->landmark)+	return -1;++    /*+     * Slice offset starts after the first block, so we need to simulate+     * writing it to work out the correct offset+     */+    {+	slice_offset = c_hdr->method == RAW+	    ? c_hdr->uncomp_size+	    : c_hdr->comp_size;+	slice_offset += 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) ++	    itf8_size(c_hdr->content_id) ++	    itf8_size(c_hdr->comp_size) ++	    itf8_size(c_hdr->uncomp_size);+    }++    c->ref_seq_id    = c->slices[0]->hdr->ref_seq_id;+    c->ref_seq_start = c->slices[0]->hdr->ref_seq_start;+    c->ref_seq_span  = c->slices[0]->hdr->ref_seq_span;+    for (i = 0; i < c->curr_slice; i++) {+	cram_slice *s = c->slices[i];+	+        c->num_blocks += s->hdr->num_blocks + 1; // slice header+	c->landmark[i] = slice_offset;++	if (s->hdr->ref_seq_start + s->hdr->ref_seq_span >+	    c->ref_seq_start + c->ref_seq_span) {+	    c->ref_seq_span = s->hdr->ref_seq_start + s->hdr->ref_seq_span+		- c->ref_seq_start;+	}+	+	slice_offset += s->hdr_block->method == RAW+	    ? s->hdr_block->uncomp_size+	    : s->hdr_block->comp_size;++	slice_offset += 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) ++	    itf8_size(s->hdr_block->content_id) ++	    itf8_size(s->hdr_block->comp_size) ++	    itf8_size(s->hdr_block->uncomp_size);++	for (j = 0; j < s->hdr->num_blocks; j++) {+	    slice_offset += 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) ++		itf8_size(s->block[j]->content_id) ++		itf8_size(s->block[j]->comp_size) ++		itf8_size(s->block[j]->uncomp_size);++	    slice_offset += s->block[j]->method == RAW+		? s->block[j]->uncomp_size+		: s->block[j]->comp_size;+	}+    }+    c->length += slice_offset; // just past the final slice++    c->comp_hdr_block = c_hdr;++    if (c->ref_seq_id >= 0) {+	cram_ref_decr(fd->refs, c->ref_seq_id);+    }++    /* Cache references up-front if we have unsorted access patterns */+    if (!fd->no_ref && c->refs_used) {+	for (i = 0; i < fd->refs->nref; i++) {+	    if (c->refs_used[i])+		cram_ref_decr(fd->refs, i);+	}+    }++    return 0;+}+++/*+ * Adds a feature code to a read within a slice. For purposes of minimising+ * memory allocations and fragmentation we have one array of features for all+ * reads within the slice. We return the index into this array for this new+ * feature.+ *+ * Returns feature index on success+ *         -1 on failure.+ */+static int cram_add_feature(cram_container *c, cram_slice *s,+			    cram_record *r, cram_feature *f) {+    if (s->nfeatures >= s->afeatures) {+	s->afeatures = s->afeatures ? s->afeatures*2 : 1024;+	s->features = realloc(s->features, s->afeatures*sizeof(*s->features));+	if (!s->features)+	    return -1;+    }++    if (!r->nfeature++) {+	r->feature = s->nfeatures;+	cram_stats_add(c->stats[DS_FP], f->X.pos);+    } else {+	cram_stats_add(c->stats[DS_FP],+		       f->X.pos - s->features[r->feature + r->nfeature-2].X.pos);+    }+    cram_stats_add(c->stats[DS_FC], f->X.code);++    s->features[s->nfeatures++] = *f;++    return 0;+}++static int cram_add_substitution(cram_fd *fd, cram_container *c,+				 cram_slice *s, cram_record *r,+				 int pos, char base, char qual, char ref) {+    cram_feature f;++    // seq=ACGTN vs ref=ACGT or seq=ACGT vs ref=ACGTN+    if (fd->L2[(uc)base]<4 || (fd->L2[(uc)base]<5 && fd->L2[(uc)ref]<4)) {+	f.X.pos = pos+1;+	f.X.code = 'X';+	f.X.base = fd->cram_sub_matrix[ref&0x1f][base&0x1f];+	cram_stats_add(c->stats[DS_BS], f.X.base);+    } else {+	f.B.pos = pos+1;+	f.B.code = 'B';+	f.B.base = base;+	f.B.qual = qual;+	cram_stats_add(c->stats[DS_BA], f.B.base);+	cram_stats_add(c->stats[DS_QS], f.B.qual);+	BLOCK_APPEND_CHAR(s->qual_blk, qual);+    }+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_bases(cram_fd *fd, cram_container *c,+			  cram_slice *s, cram_record *r,+			  int pos, int len, char *base) {+    cram_feature f;++    f.b.pos = pos+1;+    f.b.code = 'b';+    f.b.seq_idx = base - (char *)BLOCK_DATA(s->seqs_blk);+    f.b.len = len;++    return cram_add_feature(c, s, r, &f);+}++static int cram_add_base(cram_fd *fd, cram_container *c,+			 cram_slice *s, cram_record *r,+			 int pos, char base, char qual) {+    cram_feature f;+    f.B.pos = pos+1;+    f.B.code = 'B';+    f.B.base = base;+    f.B.qual = qual;+    cram_stats_add(c->stats[DS_BA], base);+    cram_stats_add(c->stats[DS_QS], qual);+    BLOCK_APPEND_CHAR(s->qual_blk, qual);+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_quality(cram_fd *fd, cram_container *c,+			    cram_slice *s, cram_record *r,+			    int pos, char qual) {+    cram_feature f;+    f.Q.pos = pos+1;+    f.Q.code = 'Q';+    f.Q.qual = qual;+    cram_stats_add(c->stats[DS_QS], qual);+    BLOCK_APPEND_CHAR(s->qual_blk, qual);+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_deletion(cram_container *c, cram_slice *s, cram_record *r,+			     int pos, int len, char *base) {+    cram_feature f;+    f.D.pos = pos+1;+    f.D.code = 'D';+    f.D.len = len;+    cram_stats_add(c->stats[DS_DL], len);+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_softclip(cram_container *c, cram_slice *s, cram_record *r,+			     int pos, int len, char *base, int version) {+    cram_feature f;+    f.S.pos = pos+1;+    f.S.code = 'S';+    f.S.len = len;+    switch (CRAM_MAJOR_VERS(version)) {+    case 1:+	f.S.seq_idx = BLOCK_SIZE(s->base_blk);+	BLOCK_APPEND(s->base_blk, base, len);+	BLOCK_APPEND_CHAR(s->base_blk, '\0');+	break;++    case 2:+    default:+	f.S.seq_idx = BLOCK_SIZE(s->soft_blk);+	if (base) {+	    BLOCK_APPEND(s->soft_blk, base, len);+	} else {+	    int i;+	    for (i = 0; i < len; i++)+		BLOCK_APPEND_CHAR(s->soft_blk, 'N');+	}+	BLOCK_APPEND_CHAR(s->soft_blk, '\0');+	break;++//    default:+//	// v3.0 onwards uses BB data-series+//	f.S.seq_idx = BLOCK_SIZE(s->soft_blk);+    }+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_hardclip(cram_container *c, cram_slice *s, cram_record *r,+			     int pos, int len, char *base) {+    cram_feature f;+    f.S.pos = pos+1;+    f.S.code = 'H';+    f.S.len = len;+    cram_stats_add(c->stats[DS_HC], len);+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_skip(cram_container *c, cram_slice *s, cram_record *r,+			     int pos, int len, char *base) {+    cram_feature f;+    f.S.pos = pos+1;+    f.S.code = 'N';+    f.S.len = len;+    cram_stats_add(c->stats[DS_RS], len);+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_pad(cram_container *c, cram_slice *s, cram_record *r,+			     int pos, int len, char *base) {+    cram_feature f;+    f.S.pos = pos+1;+    f.S.code = 'P';+    f.S.len = len;+    cram_stats_add(c->stats[DS_PD], len);+    return cram_add_feature(c, s, r, &f);+}++static int cram_add_insertion(cram_container *c, cram_slice *s, cram_record *r,+			      int pos, int len, char *base) {+    cram_feature f;+    f.I.pos = pos+1;+    if (len == 1) {+	char b = base ? *base : 'N';+	f.i.code = 'i';+	f.i.base = b;+	cram_stats_add(c->stats[DS_BA], b);+    } else {+	f.I.code = 'I';+	f.I.len = len;+	f.S.seq_idx = BLOCK_SIZE(s->base_blk);+	if (base) {+	    BLOCK_APPEND(s->base_blk, base, len);+	} else {+	    int i;+	    for (i = 0; i < len; i++)+		BLOCK_APPEND_CHAR(s->base_blk, 'N');+	}+	BLOCK_APPEND_CHAR(s->base_blk, '\0');+    }+    return cram_add_feature(c, s, r, &f);+}++/*+ * Encodes auxiliary data.+ * Returns the read-group parsed out of the BAM aux fields on success+ *         NULL on failure or no rg present (FIXME)+ */+static char *cram_encode_aux_1_0(cram_fd *fd, bam_seq_t *b, cram_container *c,+				 cram_slice *s, cram_record *cr) {+    char *aux, *tmp, *rg = NULL;+    int aux_size = bam_blk_size(b) -+	((char *)bam_aux(b) - (char *)&bam_ref(b));+	+    /* Worst case is 1 nul char on every ??:Z: string, so +33% */+    BLOCK_GROW(s->aux_blk, aux_size*1.34+1);+    tmp = (char *)BLOCK_END(s->aux_blk);++    aux = (char *)bam_aux(b);+    cr->TN_idx = s->nTN;++    while (aux[0] != 0) {+	int32_t i32;+	int r;++	if (aux[0] == 'R' && aux[1] == 'G' && aux[2] == 'Z') {+	    rg = &aux[3];+	    while (*aux++);+	    continue;+	}+	if (aux[0] == 'M' && aux[1] == 'D' && aux[2] == 'Z') {+	    while (*aux++);+	    continue;+	}+	if (aux[0] == 'N' && aux[1] == 'M') {+	    switch(aux[2]) {+	    case 'A': case 'C': case 'c': aux+=4; break;+	    case 'I': case 'i': case 'f': aux+=7; break;+	    default:+		fprintf(stderr, "Unhandled type code for NM tag\n");+		return NULL;+	    }+	    continue;+	}++	cr->ntags++;++	i32 = (aux[0]<<16) | (aux[1]<<8) | aux[2];+	kh_put(s_i2i, c->tags_used, i32, &r);+	if (-1 == r)+	    return NULL;++	if (s->nTN >= s->aTN) {+	    s->aTN = s->aTN ? s->aTN*2 : 1024;+	    if (!(s->TN = realloc(s->TN, s->aTN * sizeof(*s->TN))))+		return NULL;+	}+	s->TN[s->nTN++] = i32;+	cram_stats_add(c->stats[DS_TN], i32);++	switch(aux[2]) {+	case 'A': case 'C': case 'c':+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++;+	    break;++	case 'S': case 's':+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++; *tmp++=*aux++;+	    break;++	case 'I': case 'i': case 'f':+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    break;++	case 'd':+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    break;++	case 'Z': case 'H':+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t'; // stop byte+	    break;++	case 'B': {+	    int type = aux[3], blen;+	    uint32_t count = (uint32_t)((((unsigned char *)aux)[4]<< 0) ++					(((unsigned char *)aux)[5]<< 8) ++					(((unsigned char *)aux)[6]<<16) ++					(((unsigned char *)aux)[7]<<24));+	    // skip TN field+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;++	    // We use BYTE_ARRAY_LEN with external length, so store that first+	    switch (type) {+	    case 'c': case 'C':+		blen = count;+		break;+	    case 's': case 'S':+		blen = 2*count;+		break;+	    case 'i': case 'I': case 'f':+		blen = 4*count;+		break;+	    default:+		fprintf(stderr, "Unknown sub-type '%c' for aux type 'B'\n",+			type);+		return NULL;+		    +	    }++	    tmp += itf8_put(tmp, blen+5);++	    *tmp++=*aux++; // sub-type & length+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;++	    // The tag data itself+	    memcpy(tmp, aux, blen); tmp += blen; aux += blen;++	    //cram_stats_add(c->aux_B_stats, blen);+	    break;+	}+	default:+	    fprintf(stderr, "Unknown aux type '%c'\n", aux[2]);+	    return NULL;+	}+    }+    cram_stats_add(c->stats[DS_TC], cr->ntags);++    cr->aux = BLOCK_SIZE(s->aux_blk);+    cr->aux_size = (uc *)tmp - (BLOCK_DATA(s->aux_blk) + cr->aux);+    BLOCK_SIZE(s->aux_blk) = (uc *)tmp - BLOCK_DATA(s->aux_blk);+    assert(s->aux_blk->byte <= s->aux_blk->alloc);++    return rg;+}++/*+ * Encodes auxiliary data. Largely duplicated from above, but done so to+ * keep it simple and avoid a myriad of version ifs.+ *+ * Returns the read-group parsed out of the BAM aux fields on success+ *         NULL on failure or no rg present (FIXME)+ */+static char *cram_encode_aux(cram_fd *fd, bam_seq_t *b, cram_container *c,+			     cram_slice *s, cram_record *cr) {+    char *aux, *orig, *tmp, *rg = NULL;+    int aux_size = bam_get_l_aux(b);+    cram_block *td_b = c->comp_hdr->TD_blk;+    int TD_blk_size = BLOCK_SIZE(td_b), new;+    char *key;+    khint_t k;+++    /* Worst case is 1 nul char on every ??:Z: string, so +33% */+    BLOCK_GROW(s->aux_blk, aux_size*1.34+1);+    tmp = (char *)BLOCK_END(s->aux_blk);+++    orig = aux = (char *)bam_aux(b);++    // Copy aux keys to td_b and aux values to s->aux_blk+    while (aux - orig < aux_size && aux[0] != 0) {+	uint32_t i32;+	int r;++	// RG:Z+	if (aux[0] == 'R' && aux[1] == 'G' && aux[2] == 'Z') {+	    rg = &aux[3];+	    while (*aux++);+	    continue;+	}++	// MD:Z+	if (aux[0] == 'M' && aux[1] == 'D' && aux[2] == 'Z') {+	    if (cr->len && !fd->no_ref && !(cr->flags & BAM_FUNMAP)) {+		while (*aux++);+		continue;+	    }+	}++	// NM:i+	if (aux[0] == 'N' && aux[1] == 'M') {+	    if (cr->len && !fd->no_ref && !(cr->flags & BAM_FUNMAP)) {+		switch(aux[2]) {+		case 'A': case 'C': case 'c': aux+=4; break;+		case 'S': case 's':           aux+=5; break;+		case 'I': case 'i': case 'f': aux+=7; break;+		default:+		    fprintf(stderr, "Unhandled type code for NM tag\n");+		    return NULL;+		}+		continue;+	    }+	}++	BLOCK_APPEND(td_b, aux, 3);++	i32 = (aux[0]<<16) | (aux[1]<<8) | aux[2];+	kh_put(s_i2i, c->tags_used, i32, &r);+	if (-1 == r)+	    return NULL;++	// BQ:Z+	if (aux[0] == 'B' && aux[1] == 'Q' && aux[2] == 'Z') {+	    char *tmp;+	    if (!s->aux_BQ_blk)+		if (!(s->aux_BQ_blk = cram_new_block(EXTERNAL, DS_aux_BQ)))+		    return NULL;+	    BLOCK_GROW(s->aux_BQ_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_BQ_blk);+	    aux += 3;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t';+	    BLOCK_SIZE(s->aux_BQ_blk) = (uc *)tmp - BLOCK_DATA(s->aux_BQ_blk);+	    continue;+	}++	// BD:Z+	if (aux[0] == 'B' && aux[1]=='D' && aux[2] == 'Z') {+	    char *tmp;+	    if (!s->aux_BD_blk)+		if (!(s->aux_BD_blk = cram_new_block(EXTERNAL, DS_aux_BD)))+		    return NULL;+	    BLOCK_GROW(s->aux_BD_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_BD_blk);+	    aux += 3;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t';+	    BLOCK_SIZE(s->aux_BD_blk) = (uc *)tmp - BLOCK_DATA(s->aux_BD_blk);+	    continue;+	}++	// BI:Z+	if (aux[0] == 'B' && aux[1]=='I' && aux[2] == 'Z') {+	    char *tmp;+	    if (!s->aux_BI_blk)+		if (!(s->aux_BI_blk = cram_new_block(EXTERNAL, DS_aux_BI)))+		    return NULL;+	    BLOCK_GROW(s->aux_BI_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_BI_blk);+	    aux += 3;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t';+	    BLOCK_SIZE(s->aux_BI_blk) = (uc *)tmp - BLOCK_DATA(s->aux_BI_blk);+	    continue;+	}++	// OQ:Z:+	if (aux[0] == 'O' && aux[1] == 'Q' && aux[2] == 'Z') {+	    char *tmp;+	    if (!s->aux_OQ_blk)+		if (!(s->aux_OQ_blk = cram_new_block(EXTERNAL, DS_aux_OQ)))+		    return NULL;+	    BLOCK_GROW(s->aux_OQ_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_OQ_blk);+	    aux += 3;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t';+	    BLOCK_SIZE(s->aux_OQ_blk) = (uc *)tmp - BLOCK_DATA(s->aux_OQ_blk);+	    continue;+	}++	// FZ:B or ZM:B+	if ((aux[0] == 'F' && aux[1] == 'Z' && aux[2] == 'B') ||+	    (aux[0] == 'Z' && aux[1] == 'M' && aux[2] == 'B')) {+	    int type = aux[3], blen;+	    uint32_t count = (uint32_t)((((unsigned char *)aux)[4]<< 0) ++					(((unsigned char *)aux)[5]<< 8) ++					(((unsigned char *)aux)[6]<<16) ++					(((unsigned char *)aux)[7]<<24));+	    char *tmp;+	    if (!s->aux_FZ_blk)+		if (!(s->aux_FZ_blk = cram_new_block(EXTERNAL, DS_aux_FZ)))+		    return NULL;+	    BLOCK_GROW(s->aux_FZ_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_FZ_blk);++	    // skip TN field+	    aux+=3;++	    // We use BYTE_ARRAY_LEN with external length, so store that first+	    switch (type) {+	    case 'c': case 'C':+		blen = count;+		break;+	    case 's': case 'S':+		blen = 2*count;+		break;+	    case 'i': case 'I': case 'f':+		blen = 4*count;+		break;+	    default:+		fprintf(stderr, "Unknown sub-type '%c' for aux type 'B'\n",+			type);+		return NULL;+		    +	    }++	    blen += 5; // sub-type & length+	    tmp += itf8_put(tmp, blen);++	    // The tag data itself+	    memcpy(tmp, aux, blen); tmp += blen; aux += blen;++	    BLOCK_SIZE(s->aux_FZ_blk) = (uc *)tmp - BLOCK_DATA(s->aux_FZ_blk);+	    continue;+	}++	// Other quality data - {Q2,E2,U2,CQ}:Z and similar+	if (((aux[0] == 'Q' && aux[1] == '2') ||+	     (aux[0] == 'U' && aux[1] == '2') ||+	     (aux[0] == 'Q' && aux[1] == 'T') ||+	     (aux[0] == 'C' && aux[1] == 'Q')) && aux[2] == 'Z') {+	    char *tmp;+	    if (!s->aux_oq_blk)+		if (!(s->aux_oq_blk = cram_new_block(EXTERNAL, DS_aux_oq)))+		    return NULL;+	    BLOCK_GROW(s->aux_oq_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_oq_blk);+	    aux += 3;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t';+	    BLOCK_SIZE(s->aux_oq_blk) = (uc *)tmp - BLOCK_DATA(s->aux_oq_blk);+	    continue;+	}++	// Other sequence data - {R2,E2,CS,BC,RT}:Z and similar+	if (((aux[0] == 'R' && aux[1] == '2') ||+	     (aux[0] == 'E' && aux[1] == '2') ||+	     (aux[0] == 'C' && aux[1] == 'S') ||+	     (aux[0] == 'B' && aux[1] == 'C') ||+	     (aux[0] == 'R' && aux[1] == 'T')) && aux[2] == 'Z') {+	    char *tmp;+	    if (!s->aux_os_blk)+		if (!(s->aux_os_blk = cram_new_block(EXTERNAL, DS_aux_os)))+		    return NULL;+	    BLOCK_GROW(s->aux_os_blk, aux_size*1.34+1);+	    tmp = (char *)BLOCK_END(s->aux_os_blk);+	    aux += 3;+	    while ((*tmp++=*aux++));+	    *tmp++ = '\t';+	    BLOCK_SIZE(s->aux_os_blk) = (uc *)tmp - BLOCK_DATA(s->aux_os_blk);+	    continue;+	}+++	switch(aux[2]) {+	case 'A': case 'C': case 'c':+	    aux+=3;+	    *tmp++=*aux++;+	    break;++	case 'S': case 's':+	    aux+=3;+	    *tmp++=*aux++; *tmp++=*aux++;+	    break;++	case 'I': case 'i': case 'f':+	    aux+=3;+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    break;++	case 'd':+	    aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;+	    break;++	case 'Z': case 'H':+	    {+		char *tmp;+		if (!s->aux_oz_blk)+		    if (!(s->aux_oz_blk = cram_new_block(EXTERNAL, DS_aux_oz)))+			return NULL;+		BLOCK_GROW(s->aux_oz_blk, aux_size*1.34+1);+		tmp = (char *)BLOCK_END(s->aux_oz_blk);+		aux += 3;+		while ((*tmp++=*aux++));+		*tmp++ = '\t';+		BLOCK_SIZE(s->aux_oz_blk) = (uc *)tmp -+		    BLOCK_DATA(s->aux_oz_blk);+	    }+	    break;++	case 'B': {+	    int type = aux[3], blen;+	    uint32_t count = (uint32_t)((((unsigned char *)aux)[4]<< 0) ++					(((unsigned char *)aux)[5]<< 8) ++					(((unsigned char *)aux)[6]<<16) ++					(((unsigned char *)aux)[7]<<24));+	    // skip TN field+	    aux+=3;++	    // We use BYTE_ARRAY_LEN with external length, so store that first+	    switch (type) {+	    case 'c': case 'C':+		blen = count;+		break;+	    case 's': case 'S':+		blen = 2*count;+		break;+	    case 'i': case 'I': case 'f':+		blen = 4*count;+		break;+	    default:+		fprintf(stderr, "Unknown sub-type '%c' for aux type 'B'\n",+			type);+		return NULL;+		    +	    }++	    blen += 5; // sub-type & length+	    tmp += itf8_put(tmp, blen);++	    // The tag data itself+	    memcpy(tmp, aux, blen); tmp += blen; aux += blen;++	    //cram_stats_add(c->aux_B_stats, blen);+	    break;+	}+	default:+	    fprintf(stderr, "Unknown aux type '%c'\n", aux[2]);+	    return NULL;+	}+    }++    // FIXME: sort BLOCK_DATA(td_b) by char[3] triples+    +    // And and increment TD hash entry+    BLOCK_APPEND_CHAR(td_b, 0);++    // Duplicate key as BLOCK_DATA() can be realloced to a new pointer.+    key = string_ndup(c->comp_hdr->TD_keys, +		      (char *)BLOCK_DATA(td_b) + TD_blk_size,+		      BLOCK_SIZE(td_b) - TD_blk_size);+    k = kh_put(m_s2i, c->comp_hdr->TD_hash, key, &new);+    if (new < 0) {+	return NULL;+    } else if (new == 0) {+	BLOCK_SIZE(td_b) = TD_blk_size;+    } else {+	kh_val(c->comp_hdr->TD_hash, k) = c->comp_hdr->nTL;+	c->comp_hdr->nTL++;+    }++    cr->TL = kh_val(c->comp_hdr->TD_hash, k);+    cram_stats_add(c->stats[DS_TL], cr->TL);++    cr->aux = BLOCK_SIZE(s->aux_blk);+    cr->aux_size = (uc *)tmp - (BLOCK_DATA(s->aux_blk) + cr->aux);+    BLOCK_SIZE(s->aux_blk) = (uc *)tmp - BLOCK_DATA(s->aux_blk);+    assert(s->aux_blk->byte <= s->aux_blk->alloc);++    return rg;+}+++/*+ * Handles creation of a new container or new slice, flushing any+ * existing containers when appropriate. + *+ * Really this is next slice, which may or may not lead to a new container.+ *+ * Returns cram_container pointer on success+ *         NULL on failure.+ */+static cram_container *cram_next_container(cram_fd *fd, bam_seq_t *b) {+    cram_container *c = fd->ctr;+    cram_slice *s;+    int i;++    /* First occurence */+    if (c->curr_ref == -2)+	c->curr_ref = bam_ref(b);++    if (c->slice) {+	s = c->slice;+	if (c->multi_seq) {+	    s->hdr->ref_seq_id    = -2;+	    s->hdr->ref_seq_start = 0;+	    s->hdr->ref_seq_span  = 0;+	} else {+	    s->hdr->ref_seq_id    = c->curr_ref;+	    s->hdr->ref_seq_start = c->first_base;+	    s->hdr->ref_seq_span  = c->last_base - c->first_base + 1;+	}+	s->hdr->num_records   = c->curr_rec;++	if (c->curr_slice == 0) {+	    if (c->ref_seq_id != s->hdr->ref_seq_id)+		c->ref_seq_id  = s->hdr->ref_seq_id;+	    c->ref_seq_start = c->first_base;+	}++	c->curr_slice++;+    }++    /* Flush container */+    if (c->curr_slice == c->max_slice ||+	(bam_ref(b) != c->curr_ref && !c->multi_seq)) {+	c->ref_seq_span = fd->last_base - c->ref_seq_start + 1;+	if (fd->verbose)+	    fprintf(stderr, "Flush container %d/%d..%d\n",+		    c->ref_seq_id, c->ref_seq_start,+		    c->ref_seq_start + c->ref_seq_span -1);++	/* Encode slices */+	if (fd->pool) {+	    if (-1 == cram_flush_container_mt(fd, c))+		return NULL;+	} else {+	    if (-1 == cram_flush_container(fd, c))+		return NULL;++	    // Move to sep func, as we need cram_flush_container for+	    // the closing phase to flush the partial container.+	    for (i = 0; i < c->max_slice; i++) {+		cram_free_slice(c->slices[i]);+		c->slices[i] = NULL;+	    }++	    c->slice = NULL;+	    c->curr_slice = 0;++	    /* Easy approach for purposes of freeing stats */+	    cram_free_container(c);+	}++	c = fd->ctr = cram_new_container(fd->seqs_per_slice,+					 fd->slices_per_container);+	if (!c)+	    return NULL;+	c->record_counter = fd->record_counter;+	c->curr_ref = bam_ref(b);+    }++    c->last_pos = c->first_base = c->last_base = bam_pos(b)+1;++    /* New slice */+    c->slice = c->slices[c->curr_slice] =+	cram_new_slice(MAPPED_SLICE, c->max_rec);+    if (!c->slice)+	return NULL;++    if (c->multi_seq) {+	c->slice->hdr->ref_seq_id = -2;+	c->slice->hdr->ref_seq_start = 0;+	c->slice->last_apos = 1;+    } else {+	c->slice->hdr->ref_seq_id = bam_ref(b);+	// wrong for unsorted data, will fix during encoding.+	c->slice->hdr->ref_seq_start = bam_pos(b)+1;+	c->slice->last_apos = bam_pos(b)+1;+    }++    c->curr_rec = 0;++    return c;+}++/*+ * Converts a single bam record into a cram record.+ * Possibly used within a thread.+ *+ * Returns 0 on success;+ *        -1 on failure+ */+static int process_one_read(cram_fd *fd, cram_container *c,+			    cram_slice *s, cram_record *cr,+			    bam_seq_t *b, int rnum) {+    int i, fake_qual = -1;+    char *cp, *rg;+    char *ref, *seq, *qual;++    // FIXME: multi-ref containers++    ref = c->ref;+    cr->flags       = bam_flag(b);+    cr->len         = bam_seq_len(b);++    //fprintf(stderr, "%s => %d\n", rg ? rg : "\"\"", cr->rg);++    // Fields to resolve later+    //cr->mate_line;    // index to another cram_record+    //cr->mate_flags;   // MF+    //cr->ntags;        // TC+    cr->ntags      = 0; //cram_stats_add(c->stats[DS_TC], cr->ntags);+    if (CRAM_MAJOR_VERS(fd->version) == 1)+	rg = cram_encode_aux_1_0(fd, b, c, s, cr);+    else+	rg = cram_encode_aux(fd, b, c, s, cr);++    //cr->aux_size = b->blk_size - ((char *)bam_aux(b) - (char *)&bam_ref(b));+    //cr->aux = DSTRING_LEN(s->aux_ds);+    //dstring_nappend(s->aux_ds, bam_aux(b), cr->aux_size);++    /* Read group, identified earlier */+    if (rg) {+	SAM_RG *brg = sam_hdr_find_rg(fd->header, rg);+	cr->rg = brg ? brg->id : -1;+    } else if (CRAM_MAJOR_VERS(fd->version) == 1) {+	SAM_RG *brg = sam_hdr_find_rg(fd->header, "UNKNOWN");+	assert(brg);+    } else {+	cr->rg = -1;+    }+    cram_stats_add(c->stats[DS_RG], cr->rg);++    +    cr->ref_id      = bam_ref(b);  cram_stats_add(c->stats[DS_RI], cr->ref_id);+    cram_stats_add(c->stats[DS_BF], fd->cram_flag_swap[cr->flags & 0xfff]);++    // Non reference based encoding means storing the bases verbatim as features, which in+    // turn means every base also has a quality already stored.+    if (!fd->no_ref || CRAM_MAJOR_VERS(fd->version) >= 3)+	cr->cram_flags = CRAM_FLAG_PRESERVE_QUAL_SCORES;+    else+	cr->cram_flags = 0;++    if (cr->len <= 0 && CRAM_MAJOR_VERS(fd->version) >= 3)+	cr->cram_flags |= CRAM_FLAG_NO_SEQ;+    //cram_stats_add(c->stats[DS_CF], cr->cram_flags & CRAM_FLAG_MASK);++    c->num_bases   += cr->len;+    cr->apos        = bam_pos(b)+1;+    if (c->pos_sorted) {+	if (cr->apos < s->last_apos) {+	    c->pos_sorted = 0;+	} else {+	    cram_stats_add(c->stats[DS_AP], cr->apos - s->last_apos);+	    s->last_apos = cr->apos;+	}+    } else {+	//cram_stats_add(c->stats[DS_AP], cr->apos);+    }+    c->max_apos += (cr->apos > c->max_apos) * (cr->apos - c->max_apos);++    cr->name        = BLOCK_SIZE(s->name_blk);+    cr->name_len    = bam_name_len(b);+    cram_stats_add(c->stats[DS_RN], cr->name_len);++    BLOCK_APPEND(s->name_blk, bam_name(b), bam_name_len(b));+++    /*+     * This seqs_ds is largely pointless and it could reuse the same memory+     * over and over.+     * s->base_blk is what we need for encoding.+     */+    cr->seq         = BLOCK_SIZE(s->seqs_blk);+    cr->qual        = BLOCK_SIZE(s->qual_blk);+    BLOCK_GROW(s->seqs_blk, cr->len+1);+    BLOCK_GROW(s->qual_blk, cr->len);+    seq = cp = (char *)BLOCK_END(s->seqs_blk);++    *seq = 0;+#ifdef ALLOW_UAC+    {+	// Convert seq 2 bases at a time for speed.+	static const uint16_t code2base[256] = {+	    15677, 16701, 17213, 19773, 18237, 21053, 21309, 22077,+	    21565, 22333, 22845, 18493, 19261, 17469, 16957, 20029,+	    15681, 16705, 17217, 19777, 18241, 21057, 21313, 22081,+	    21569, 22337, 22849, 18497, 19265, 17473, 16961, 20033,+	    15683, 16707, 17219, 19779, 18243, 21059, 21315, 22083,+	    21571, 22339, 22851, 18499, 19267, 17475, 16963, 20035,+	    15693, 16717, 17229, 19789, 18253, 21069, 21325, 22093,+	    21581, 22349, 22861, 18509, 19277, 17485, 16973, 20045,+	    15687, 16711, 17223, 19783, 18247, 21063, 21319, 22087,+	    21575, 22343, 22855, 18503, 19271, 17479, 16967, 20039,+	    15698, 16722, 17234, 19794, 18258, 21074, 21330, 22098,+	    21586, 22354, 22866, 18514, 19282, 17490, 16978, 20050,+	    15699, 16723, 17235, 19795, 18259, 21075, 21331, 22099,+	    21587, 22355, 22867, 18515, 19283, 17491, 16979, 20051,+	    15702, 16726, 17238, 19798, 18262, 21078, 21334, 22102,+	    21590, 22358, 22870, 18518, 19286, 17494, 16982, 20054,+	    15700, 16724, 17236, 19796, 18260, 21076, 21332, 22100,+	    21588, 22356, 22868, 18516, 19284, 17492, 16980, 20052,+	    15703, 16727, 17239, 19799, 18263, 21079, 21335, 22103,+	    21591, 22359, 22871, 18519, 19287, 17495, 16983, 20055,+	    15705, 16729, 17241, 19801, 18265, 21081, 21337, 22105,+	    21593, 22361, 22873, 18521, 19289, 17497, 16985, 20057,+	    15688, 16712, 17224, 19784, 18248, 21064, 21320, 22088,+	    21576, 22344, 22856, 18504, 19272, 17480, 16968, 20040,+	    15691, 16715, 17227, 19787, 18251, 21067, 21323, 22091,+	    21579, 22347, 22859, 18507, 19275, 17483, 16971, 20043,+	    15684, 16708, 17220, 19780, 18244, 21060, 21316, 22084,+	    21572, 22340, 22852, 18500, 19268, 17476, 16964, 20036,+	    15682, 16706, 17218, 19778, 18242, 21058, 21314, 22082,+	    21570, 22338, 22850, 18498, 19266, 17474, 16962, 20034,+	    15694, 16718, 17230, 19790, 18254, 21070, 21326, 22094,+	    21582, 22350, 22862, 18510, 19278, 17486, 16974, 20046+	};++	int l2 = cr->len / 2;+	unsigned char *from = (unsigned char *)bam_seq(b);+	uint16_t *cpi = (uint16_t *)cp;+	cp[0] = 0;+	for (i = 0; i < l2; i++)+	    cpi[i] = le_int2(code2base[from[i]]);+	if ((i *= 2) < cr->len)+	    cp[i] = seq_nt16_str[bam_seqi(bam_seq(b), i)];+    }+#else+    for (i = 0; i < cr->len; i++)+	cp[i] = seq_nt16_str[bam_seqi(bam_seq(b), i)];+#endif+    BLOCK_SIZE(s->seqs_blk) += cr->len;++    qual = cp = (char *)bam_qual(b);++    /* Copy and parse */+    if (!(cr->flags & BAM_FUNMAP)) {+	uint32_t *cig_to, *cig_from;+	int apos = cr->apos-1, spos = 0;++	cr->cigar       = s->ncigar;+	cr->ncigar      = bam_cigar_len(b);+	while (cr->cigar + cr->ncigar >= s->cigar_alloc) {+	    s->cigar_alloc = s->cigar_alloc ? s->cigar_alloc*2 : 1024;+	    s->cigar = realloc(s->cigar, s->cigar_alloc * sizeof(*s->cigar));+	    if (!s->cigar)+		return -1;+	}++	cig_to = (uint32_t *)s->cigar;+	cig_from = (uint32_t *)bam_cigar(b);++	cr->feature = 0;+	cr->nfeature = 0;+	for (i = 0; i < cr->ncigar; i++) {+	    enum cigar_op cig_op = cig_from[i] & BAM_CIGAR_MASK;+	    uint32_t cig_len = cig_from[i] >> BAM_CIGAR_SHIFT;+	    cig_to[i] = cig_from[i];++	    /* Can also generate events from here for CRAM diffs */++	    switch (cig_op) {+		int l;++		// Don't trust = and X ops to be correct.+	    case BAM_CMATCH:+	    case BAM_CBASE_MATCH:+	    case BAM_CBASE_MISMATCH:+		//fprintf(stderr, "\nBAM_CMATCH\nR: %.*s\nS: %.*s\n",+		//	cig_len, &ref[apos], cig_len, &seq[spos]);+		l = 0;+		if (!fd->no_ref && cr->len) {+		    int end = cig_len+apos < c->ref_end+			? cig_len : c->ref_end - apos;+		    char *sp = &seq[spos];+		    char *rp = &ref[apos];+		    char *qp = &qual[spos];+		    if (end > cr->len) {+			fprintf(stderr, "CIGAR and query sequence are of "+				"different length\n");+			return -1;+		    }+		    for (l = 0; l < end; l++) {+			if (rp[l] != sp[l]) {+			    if (!sp[l])+				break;+			    if (0 && CRAM_MAJOR_VERS(fd->version) >= 3) {+				// Disabled for the time being as it doesn't+				// seem to gain us much.+				int ol=l;+				while (l<end && rp[l] != sp[l])+				    l++;+				if (l-ol > 1) {+				    if (cram_add_bases(fd, c, s, cr, spos+ol,+						       l-ol, &seq[spos+ol]))+					return -1;+				    l--;+				} else {+				    l = ol;+				    if (cram_add_substitution(fd, c, s, cr,+							      spos+l, sp[l],+							      qp[l], rp[l]))+					return -1;+				}+			    } else {+				if (cram_add_substitution(fd, c, s, cr, spos+l,+							  sp[l], qp[l], rp[l]))+				    return -1;+			    }+			}+		    }+		    spos += l;+		    apos += l;+		}++		if (l < cig_len && cr->len) {+		    if (fd->no_ref) {+			if (CRAM_MAJOR_VERS(fd->version) == 3) {+			    if (cram_add_bases(fd, c, s, cr, spos,+					       cig_len-l, &seq[spos]))+				return -1;+			    spos += cig_len-l;+			} else {+			    for (; l < cig_len && seq[spos]; l++, spos++) {+				if (cram_add_base(fd, c, s, cr, spos,+						  seq[spos], qual[spos]))+				    return -1;+			    }+			}+		    } else {+			/* off end of sequence or non-ref based output */+			for (; l < cig_len && seq[spos]; l++, spos++) {+			    if (cram_add_base(fd, c, s, cr, spos,+					      seq[spos], qual[spos]))+				return -1;+			}+		    }+		    apos += cig_len;+		} else if (!cr->len) {+		    /* Seq "*" */+		    apos += cig_len;+		    spos += cig_len;+		}+		break;+		+	    case BAM_CDEL:+		if (cram_add_deletion(c, s, cr, spos, cig_len, &seq[spos]))+		    return -1;+		apos += cig_len;+		break;++	    case BAM_CREF_SKIP:+		if (cram_add_skip(c, s, cr, spos, cig_len, &seq[spos]))+		    return -1;+		apos += cig_len;+		break;++	    case BAM_CINS:+		if (cram_add_insertion(c, s, cr, spos, cig_len,+				       cr->len ? &seq[spos] : NULL))+		    return -1;+		if (fd->no_ref && cr->len) {+		    for (l = 0; l < cig_len; l++, spos++) {+			cram_add_quality(fd, c, s, cr, spos, qual[spos]);+		    }+		} else {+		    spos += cig_len;+		}+		break;++	    case BAM_CSOFT_CLIP:+		if (cram_add_softclip(c, s, cr, spos, cig_len,+				      cr->len ? &seq[spos] : NULL,+				      fd->version))+		    return -1;+		if (fd->no_ref &&+		    !(cr->cram_flags & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {+		    if (cr->len) {+			for (l = 0; l < cig_len; l++, spos++) {+			    cram_add_quality(fd, c, s, cr, spos, qual[spos]);+			}+		    } else {+			for (l = 0; l < cig_len; l++, spos++) {+			    cram_add_quality(fd, c, s, cr, spos, -1);+			}+		    }+		} else {+		    spos += cig_len;+		}+		break;++	    case BAM_CHARD_CLIP:+		if (cram_add_hardclip(c, s, cr, spos, cig_len, &seq[spos]))+		    return -1;+		break;+	+	    case BAM_CPAD:+		if (cram_add_pad(c, s, cr, spos, cig_len, &seq[spos]))+		    return -1;+		break;+	    }+	}+	if (cr->len && spos != cr->len) {+	    fprintf(stderr, "CIGAR and query sequence are of different "+		    "length\n");+	    return -1;+	}+	fake_qual = spos;+	cr->aend = fd->no_ref ? apos : MIN(apos, c->ref_end);+	cram_stats_add(c->stats[DS_FN], cr->nfeature);+    } else {+	// Unmapped+	cr->cram_flags |= CRAM_FLAG_PRESERVE_QUAL_SCORES;+	cr->cigar  = 0;+	cr->ncigar = 0;+	cr->nfeature = 0;+	cr->aend = cr->apos;+	for (i = 0; i < cr->len; i++)+	    cram_stats_add(c->stats[DS_BA], seq[i]);+	fake_qual = 0;+    }++    /*+     * Append to the qual block now. We do this here as+     * cram_add_substitution() can generate BA/QS events which need to +     * be in the qual block before we append the rest of the data.+     */+    if (cr->cram_flags & CRAM_FLAG_PRESERVE_QUAL_SCORES) {+	/* Special case of seq "*" */+	if (cr->len == 0) {+	    cr->len = fake_qual;+	    BLOCK_GROW(s->qual_blk, cr->len);+	    cp = (char *)BLOCK_END(s->qual_blk);+	    memset(cp, 255, cr->len);+	} else {+	    BLOCK_GROW(s->qual_blk, cr->len);+	    cp = (char *)BLOCK_END(s->qual_blk);+	    char *from = (char *)&bam_qual(b)[0];+	    char *to = &cp[0];+	    memcpy(to, from, cr->len);+	    //for (i = 0; i < cr->len; i++) cp[i] = from[i];+	}+	BLOCK_SIZE(s->qual_blk) += cr->len;+    } else {+	if (cr->len == 0)+	    cr->len = fake_qual >= 0 ? fake_qual : cr->aend - cr->apos + 1;+    }++    cram_stats_add(c->stats[DS_RL], cr->len);++    /* Now we know apos and aend both, update mate-pair information */+    {+	int new;+	khint_t k;+	int sec = (cr->flags & BAM_FSECONDARY) ? 1 : 0;++	//fprintf(stderr, "Checking %"PRId64"/%.*s\t", rnum,+	//	cr->name_len, DSTRING_STR(s->name_ds)+cr->name);+	if (cr->flags & BAM_FPAIRED) {+	    char *key = string_ndup(s->pair_keys,+				    (char *)BLOCK_DATA(s->name_blk)+cr->name,+				    cr->name_len);+	    if (!key)+		return -1;++	    k = kh_put(m_s2i, s->pair[sec], key, &new);+	    if (-1 == new)+		return -1;+	    else if (new > 0)+		kh_val(s->pair[sec], k) = rnum;+	} else {+	    new = 1;+	}++	if (new == 0) {+	    cram_record *p = &s->crecs[kh_val(s->pair[sec], k)];+	    int aleft, aright, sign;++	    aleft = MIN(cr->apos, p->apos);+	    aright = MAX(cr->aend, p->aend);+	    if (cr->apos < p->apos) {+		sign = 1;+	    } else if (cr->apos > p->apos) {+		sign = -1;+	    } else if (cr->flags & BAM_FREAD1) {+		sign = 1;+	    } else {+		sign = -1;+	    }+	    +	    //fprintf(stderr, "paired %"PRId64"\n", kh_val(s->pair[sec], k));++	    // This vs p: tlen, matepos, flags+	    if (bam_ins_size(b) != sign*(aright-aleft+1))+		goto detached;++	    if (MAX(bam_mate_pos(b)+1, 0) != p->apos)+		goto detached;++	    if (((bam_flag(b) & BAM_FMUNMAP) != 0) !=+		((p->flags & BAM_FUNMAP) != 0))+		goto detached;++	    if (((bam_flag(b) & BAM_FMREVERSE) != 0) !=+		((p->flags & BAM_FREVERSE) != 0))+		goto detached;+++	    // p vs this: tlen, matepos, flags+	    if (p->ref_id != cr->ref_id)+		goto detached;++	    if (p->tlen != -sign*(aright-aleft+1))+		goto detached;++	    if (p->mate_pos != cr->apos)+		goto detached;++	    if (((p->flags & BAM_FMUNMAP) != 0) !=+		((p->mate_flags & CRAM_M_UNMAP) != 0))+		goto detached;++	    if (((p->flags & BAM_FMREVERSE) != 0) !=+		((p->mate_flags & CRAM_M_REVERSE) != 0))+		goto detached;++	    // Supplementary reads are just too ill defined+	    if ((cr->flags & BAM_FSUPPLEMENTARY) ||+		(p->flags & BAM_FSUPPLEMENTARY))+		goto detached;++	    /*+	     * The fields below are unused when encoding this read as it is+	     * no longer detached.  In theory they may get referred to when+	     * processing a 3rd or 4th read in this template?, so we set them+	     * here just to be sure.+	     *+	     * They do not need cram_stats_add() calls those as they are+	     * not emitted.+	     */+	    cr->mate_pos = p->apos;+	    cr->tlen = sign*(aright-aleft+1);+	    cr->mate_flags =+	    	((p->flags & BAM_FMUNMAP)   == BAM_FMUNMAP)   * CRAM_M_UNMAP ++	    	((p->flags & BAM_FMREVERSE) == BAM_FMREVERSE) * CRAM_M_REVERSE;++	    // Decrement statistics aggregated earlier+	    if (p->cram_flags & CRAM_FLAG_STATS_ADDED) {+		cram_stats_del(c->stats[DS_NP], p->mate_pos);+		cram_stats_del(c->stats[DS_MF], p->mate_flags);+		cram_stats_del(c->stats[DS_TS], p->tlen);+		cram_stats_del(c->stats[DS_NS], p->mate_ref_id);+	    }++	    /* Similarly we could correct the p-> values too, but these will no+	     * longer have any code that refers back to them as the new 'p'+	     * for this template is our current 'cr'.+	     */+	    //p->mate_pos = cr->apos;+	    //p->mate_flags =+	    //	((cr->flags & BAM_FMUNMAP)   == BAM_FMUNMAP)  * CRAM_M_UNMAP ++	    //	((cr->flags & BAM_FMREVERSE) == BAM_FMREVERSE)* CRAM_M_REVERSE;+	    //p->tlen = p->apos - cr->aend;++	    // Clear detached from cr flags+	    cr->cram_flags &= ~CRAM_FLAG_DETACHED;+	    cram_stats_add(c->stats[DS_CF], cr->cram_flags & CRAM_FLAG_MASK);++	    // Clear detached from p flags and set downstream+	    if (p->cram_flags & CRAM_FLAG_STATS_ADDED) {+		cram_stats_del(c->stats[DS_CF], p->cram_flags & CRAM_FLAG_MASK);+		p->cram_flags &= ~CRAM_FLAG_STATS_ADDED;+	    }++	    p->cram_flags  &= ~CRAM_FLAG_DETACHED;+	    p->cram_flags  |=  CRAM_FLAG_MATE_DOWNSTREAM;+	    cram_stats_add(c->stats[DS_CF], p->cram_flags & CRAM_FLAG_MASK);++	    p->mate_line = rnum - (kh_val(s->pair[sec], k) + 1);+	    cram_stats_add(c->stats[DS_NF], p->mate_line);++	    kh_val(s->pair[sec], k) = rnum;+	} else {+	detached:+	    //fprintf(stderr, "unpaired\n");++	    /* Derive mate flags from this flag */+	    cr->mate_flags = 0;+	    if (bam_flag(b) & BAM_FMUNMAP)+		cr->mate_flags |= CRAM_M_UNMAP;+	    if (bam_flag(b) & BAM_FMREVERSE)+		cr->mate_flags |= CRAM_M_REVERSE;++	    cram_stats_add(c->stats[DS_MF], cr->mate_flags);++	    cr->mate_pos    = MAX(bam_mate_pos(b)+1, 0);+	    cram_stats_add(c->stats[DS_NP], cr->mate_pos);++	    cr->tlen        = bam_ins_size(b);+	    cram_stats_add(c->stats[DS_TS], cr->tlen);++	    cr->cram_flags |= CRAM_FLAG_DETACHED;+	    cram_stats_add(c->stats[DS_CF], cr->cram_flags & CRAM_FLAG_MASK);+	    cram_stats_add(c->stats[DS_NS], bam_mate_ref(b));++	    cr->cram_flags |= CRAM_FLAG_STATS_ADDED;+	}+    }++    cr->mqual       = bam_map_qual(b);+    cram_stats_add(c->stats[DS_MQ], cr->mqual);++    cr->mate_ref_id = bam_mate_ref(b);++    if (!(bam_flag(b) & BAM_FUNMAP)) {+	if (c->first_base > cr->apos)+	    c->first_base = cr->apos;++	if (c->last_base < cr->aend)+	    c->last_base = cr->aend;+    }++    return 0;+}++/*+ * Write iterator: put BAM format sequences into a CRAM file.+ * We buffer up a containers worth of data at a time.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_put_bam_seq(cram_fd *fd, bam_seq_t *b) {+    cram_container *c;++    if (!fd->ctr) {+	fd->ctr = cram_new_container(fd->seqs_per_slice,+				     fd->slices_per_container);+	if (!fd->ctr)+	    return -1;+	fd->ctr->record_counter = fd->record_counter;+    }+    c = fd->ctr;++    if (!c->slice || c->curr_rec == c->max_rec ||+	(bam_ref(b) != c->curr_ref && c->curr_ref >= -1)) {+	int slice_rec, curr_rec, multi_seq = fd->multi_seq == 1;+	int curr_ref = c->slice ? c->curr_ref : bam_ref(b);+++	/*+	 * Start packing slices when we routinely have under 1/4tr full.+	 *+	 * This option isn't available if we choose to embed references+	 * since we can only have one per slice.+	 */+	if (fd->multi_seq == -1 && c->curr_rec < c->max_rec/4+10 &&+	    fd->last_slice && fd->last_slice < c->max_rec/4+10 &&+	    !fd->embed_ref) {+	    if (fd->verbose && !c->multi_seq)+		fprintf(stderr, "Multi-ref enabled for this container\n");+	    multi_seq = 1;+	}++	slice_rec = c->slice_rec;+	curr_rec  = c->curr_rec;++	if (CRAM_MAJOR_VERS(fd->version) == 1 ||+	    c->curr_rec == c->max_rec || fd->multi_seq != 1 || !c->slice) {+	    if (NULL == (c = cram_next_container(fd, b))) {+		if (fd->ctr) {+		    // prevent cram_close attempting to flush+		    cram_free_container(fd->ctr);+		    fd->ctr = NULL;+		}+		return -1;+	    }+	}++	/*+	 * Due to our processing order, some things we've already done we+	 * cannot easily undo. So when we first notice we should be packing+	 * multiple sequences per container we emit the small partial+	 * container as-is and then start a fresh one in a different mode.+	 */+	if (multi_seq) {+	    fd->multi_seq = 1;+	    c->multi_seq = 1;+	    c->pos_sorted = 0; // required atm for multi_seq slices++	    if (!c->refs_used) {+		pthread_mutex_lock(&fd->ref_lock);+		c->refs_used = calloc(fd->refs->nref, sizeof(int));+		pthread_mutex_unlock(&fd->ref_lock);+		if (!c->refs_used)+		    return -1;+	    }+	}++	fd->last_slice = curr_rec - slice_rec;+	c->slice_rec = c->curr_rec;++	// Have we seen this reference before?+	if (bam_ref(b) >= 0 && bam_ref(b) != curr_ref && !fd->embed_ref &&+	    !fd->unsorted && multi_seq) {+	    +	    if (!c->refs_used) {+		pthread_mutex_lock(&fd->ref_lock);+		c->refs_used = calloc(fd->refs->nref, sizeof(int));+		pthread_mutex_unlock(&fd->ref_lock);+		if (!c->refs_used)+		    return -1;+	    } else if (c->refs_used && c->refs_used[bam_ref(b)]) {+		pthread_mutex_lock(&fd->ref_lock);+		fd->unsorted = 1;+		pthread_mutex_unlock(&fd->ref_lock);+		fd->multi_seq = 1;+	    }+	}++	c->curr_ref = bam_ref(b);+	if (c->refs_used && c->curr_ref >= 0) c->refs_used[c->curr_ref]++;+    }++    if (!c->bams) {+	/* First time through, allocate a set of bam pointers */+	pthread_mutex_lock(&fd->bam_list_lock);+	if (fd->bl) {+	    spare_bams *spare = fd->bl;+	    c->bams = spare->bams;+	    fd->bl = spare->next;+	    free(spare);+	} else {+	    c->bams = calloc(c->max_c_rec, sizeof(bam_seq_t *));+	    if (!c->bams)+		return -1;+	}+	pthread_mutex_unlock(&fd->bam_list_lock);+    }++    /* Copy or alloc+copy the bam record, for later encoding */+    if (c->bams[c->curr_c_rec])+	bam_copy1(c->bams[c->curr_c_rec], b);+    else+	c->bams[c->curr_c_rec] = bam_dup(b);++    c->curr_rec++;+    c->curr_c_rec++;+    fd->record_counter++;++    return 0;+}
+ htslib-1.3.2/cram/cram_encode.h view
@@ -0,0 +1,105 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*! \file+ * Include cram.h instead.+ *+ * This is an internal part of the CRAM system and is automatically included+ * when you #include cram.h.+ *+ * Implements the encoding portion of CRAM I/O. Also see+ * cram_codecs.[ch] for the actual encoding functions themselves.+ */++#ifndef _CRAM_WRITE_H_+#define _CRAM_WRITE_H_++#ifdef __cplusplus+extern "C" {+#endif++/* ----------------------------------------------------------------------+ * CRAM sequence iterators.+ */++/*! Write iterator: put BAM format sequences into a CRAM file.+ *+ * We buffer up a containers worth of data at a time.+ *+ * FIXME: break this into smaller pieces.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_put_bam_seq(cram_fd *fd, bam_seq_t *b);+++/* ----------------------------------------------------------------------+ * Internal functions+ */++/*! INTERNAL:+ * Encodes a compression header block into a generic cram_block structure.+ *+ * @return+ * Returns cram_block ptr on success;+ *         NULL on failure+ */+cram_block *cram_encode_compression_header(cram_fd *fd, cram_container *c,+					   cram_block_compression_hdr *h);++/*! INTERNAL:+ * Encodes a slice compression header. + *+ * @return+ * Returns cram_block on success;+ *         NULL on failure+ */+cram_block *cram_encode_slice_header(cram_fd *fd, cram_slice *s);++/*! INTERNAL:+ * Encodes all slices in a container into blocks.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ *+ * FIXME: separate into encode_container and write_container. Ideally+ * we should be able to do read_container / write_container or+ * decode_container / encode_container.+ */+int cram_encode_container(cram_fd *fd, cram_container *c);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/cram/cram_external.c view
@@ -0,0 +1,377 @@+/*+Copyright (c) 2015 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice,+this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice,+this list of conditions and the following disclaimer in the documentation+and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*! \file+ * External CRAM interface.+ *+ * Internally we're happy to use macros and to grub around in the cram+ * structures.  This isn't very sustainable for an externally usable+ * ABI though, so we have anonymous structs and accessor functions too+ * to permit software such as samtools reheader to manipulate cram+ * containers and blocks in a robust manner.+ */++#include <config.h>++#include "htslib/hfile.h"+#include "cram/cram.h"++/*+ *-----------------------------------------------------------------------------+ * cram_fd+ */+SAM_hdr *cram_fd_get_header(cram_fd *fd) { return fd->header; }+void cram_fd_set_header(cram_fd *fd, SAM_hdr *hdr) { fd->header = hdr; }++int cram_fd_get_version(cram_fd *fd) { return fd->version; }+void cram_fd_set_version(cram_fd *fd, int vers) { fd->version = vers; }++int cram_major_vers(cram_fd *fd) { return CRAM_MAJOR_VERS(fd->version); }+int cram_minor_vers(cram_fd *fd) { return CRAM_MINOR_VERS(fd->version); }++hFILE *cram_fd_get_fp(cram_fd *fd) { return fd->fp; }+void cram_fd_set_fp(cram_fd *fd, hFILE *fp) { fd->fp = fp; }+++/*+ *-----------------------------------------------------------------------------+ * cram_container+ */+int32_t cram_container_get_length(cram_container *c) {+    return c->length;+}++void cram_container_set_length(cram_container *c, int32_t length) {+    c->length = length;+}+++int32_t cram_container_get_num_blocks(cram_container *c) {+    return c->num_blocks;+}++void cram_container_set_num_blocks(cram_container *c, int32_t num_blocks) {+    c->num_blocks = num_blocks;+}+++/* Returns the landmarks[] array and the number of elements+ * in num_landmarks.+ */+int32_t *cram_container_get_landmarks(cram_container *c, int32_t *num_landmarks) {+    *num_landmarks = c->num_landmarks;+    return c->landmark;+}++/* Sets the landmarks[] array (pointer copy, not a memory dup) and+ * num_landmarks value.+ */+void cram_container_set_landmarks(cram_container *c, int32_t num_landmarks,+				  int32_t *landmarks) {+    c->num_landmarks = num_landmarks;+    c->landmark = landmarks;+}+++/* Returns true if the container is empty (EOF marker) */+int cram_container_is_empty(cram_fd *fd) {+    return fd->empty_container;+}+++/*+ *-----------------------------------------------------------------------------+ * cram_block_compression_hdr+ */++/*+ * Utility function to edit an RG id.+ * This is only possible if there is one single RG value used and it+ * is in the container compression header using HUFFMAN or BETA+ * codec.  In this case it is essentially hard coded and needs no+ * editing of external (or worse, CORE) blocks.+ *+ * Returns 0 on success+ *        -1 on failure+ */+// Or arbitrary set compression header constant?++static int cram_block_compression_hdr_set_DS(cram_block_compression_hdr *ch,+					     int ds, int new_rg) {+    if (!ch || !ch->codecs[ds])+	return -1;++    switch (ch->codecs[ds]->codec) {+    case E_HUFFMAN:+	if (ch->codecs[ds]->huffman.ncodes != 1)+	    return -1;+	ch->codecs[ds]->huffman.codes[0].symbol = new_rg;+	return 0;++    case E_BETA:+	if (ch->codecs[ds]->beta.nbits != 0)+	    return -1;+	ch->codecs[ds]->beta.offset = -new_rg;+	return 0;++    default:+	return -1;+    }++    return 0;+}++int cram_block_compression_hdr_set_rg(cram_block_compression_hdr *ch, int new_rg) {+    return cram_block_compression_hdr_set_DS(ch, DS_RG, new_rg);+}++/*+ * Converts a cram_block_compression_hdr struct used for decoding to+ * one used for encoding.  Maybe this should be a transparent+ * operation applied on-demand.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_block_compression_hdr_decoder2encoder(cram_fd *fd,+					       cram_block_compression_hdr *ch) {+    int i;++    if (!ch)+	return -1;++    for (i = 0; i < DS_END; i++) {+	cram_codec *co = ch->codecs[i];+	if (!co)+	    continue;++	if (-1 == cram_codec_decoder2encoder(fd, co))+	    return -1;+    }++    return 0;+}++/*+ *-----------------------------------------------------------------------------+ * cram_slice+ */+int32_t cram_slice_hdr_get_num_blocks(cram_block_slice_hdr *hdr) {+    return hdr->num_blocks;+}+++/*+ *-----------------------------------------------------------------------------+ * cram_block+ */+int32_t cram_block_get_content_id(cram_block *b)  { return b->content_id; }+int32_t cram_block_get_comp_size(cram_block *b)   { return b->comp_size; }+int32_t cram_block_get_uncomp_size(cram_block *b) { return b->uncomp_size; }+int32_t cram_block_get_crc32(cram_block *b)       { return b->crc32; }+void *  cram_block_get_data(cram_block *b)        { return BLOCK_DATA(b); }+int32_t cram_block_get_size(cram_block *b)        { return BLOCK_SIZE(b); }+enum cram_content_type cram_block_get_content_type(cram_block *b) {+    return b->content_type;+}++void cram_block_set_content_id(cram_block *b, int32_t id) { b->content_id = id; }+void cram_block_set_comp_size(cram_block *b, int32_t size) { b->comp_size = size; }+void cram_block_set_uncomp_size(cram_block *b, int32_t size) { b->uncomp_size = size; }+void cram_block_set_crc32(cram_block *b, int32_t crc) { b->crc32 = crc; }+void cram_block_set_data(cram_block *b, void *data) { BLOCK_DATA(b) = data; }+void cram_block_set_size(cram_block *b, int32_t size) { BLOCK_SIZE(b) = size; }++int cram_block_append(cram_block *b, void *data, int size) {+    BLOCK_APPEND(b, data, size);+    return BLOCK_DATA(b) ? 0 : -1; // It'll do for now...+}+void cram_block_update_size(cram_block *b) { BLOCK_UPLEN(b); }++// Offset is known as "size" internally, but it can be confusing.+size_t cram_block_get_offset(cram_block *b) { return BLOCK_SIZE(b); }+void cram_block_set_offset(cram_block *b, size_t offset) { BLOCK_SIZE(b) = offset; }+++/*+ * Copies the blocks representing the next num_slice slices from a+ * container from 'in' to 'out'.  It is expected that the file pointer+ * is just after the read of the cram_container and cram compression+ * header.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_copy_slice(cram_fd *in, cram_fd *out, int32_t num_slice) {+    int32_t i, j;++    for (i = 0; i < num_slice; i++) {+	cram_block *blk;+	cram_block_slice_hdr *hdr;++	if (!(blk = cram_read_block(in)))+	    return -1;+	if (!(hdr = cram_decode_slice_header(in, blk))) {+	    cram_free_block(blk);+	    return -1;+	}+	if (cram_write_block(out, blk) != 0) {+	    cram_free_block(blk);+	    return -1;+	}+	cram_free_block(blk);++	int num_blocks = cram_slice_hdr_get_num_blocks(hdr);+	for (j = 0; j < num_blocks; j++) {+	    blk = cram_read_block(in);+	    if (!blk || cram_write_block(out, blk) != 0) {+		if (blk) cram_free_block(blk);+		return -1;+	    }+	    cram_free_block(blk);+	}+	cram_free_slice_header(hdr);+    }++    return 0;+}++/*+ * Renumbers RG numbers in a cram compression header.+ *+ * CRAM stores RG as the Nth number in the header, rather than a+ * string holding the ID: tag.  This is smaller in space, but means+ * "samtools cat" to join files together that contain single but+ * different RG lines needs a way of renumbering them.+ *+ * The file descriptor is expected to be immediately after the+ * cram_container structure (ie before the cram compression header).+ * Due to the nature of the CRAM format, this needs to read and write+ * the blocks itself.  Note that there may be multiple slices within+ * the container, meaning multiple compression headers to manipulate.+ * Changing RG may change the size of the compression header and+ * therefore the length field in the container.  Hence we rewrite all+ * blocks just incase and also emit the adjusted container.+ *+ * The current implementation can only cope with renumbering a single+ * RG (and only then if it is using HUFFMAN or BETA codecs).  In+ * theory it *may* be possible to renumber multiple RGs if they use+ * HUFFMAN to the CORE block or use an external block unshared by any+ * other data series.  So we have an API that can be upgraded to+ * support this, but do not implement it for now.  An example+ * implementation of RG as an EXTERNAL block would be to find that+ * block and rewrite it, returning the number of blocks consumed.+ *+ * Returns 0 on success;+ *        -1 if unable to edit;+ *        -2 on other errors (eg I/O).+ */+int cram_transcode_rg(cram_fd *in, cram_fd *out,+		      cram_container *c,+		      int nrg, int *in_rg, int *out_rg) {+    int new_rg = *out_rg, old_size, new_size;+    cram_block *o_blk, *n_blk;+    cram_block_compression_hdr *ch;++    if (nrg != 1) {+	fprintf(stderr, "[%s] ERROR: not implemented for nrg != 1\n",+		__func__);+	return -2;+    }++    // Produce a new block holding the updated compression header,+    // with RG transcoded to a new value. (Single only supported.)+    o_blk = cram_read_block(in);+    old_size = cram_block_size(o_blk);+    ch = cram_decode_compression_header(in, o_blk);+    if (cram_block_compression_hdr_set_rg(ch, new_rg) != 0)+	return -1;+    cram_block_compression_hdr_decoder2encoder(in, ch);+    n_blk = cram_encode_compression_header(in, c, ch);+    cram_free_compression_header(ch);++    /*+     * Warning: this has internal knowledge of the cram compression+     * header format.+     *+     * The decoder doesn't set c->tags_used, so the encoder puts a two+     * byte blank segment.  This means n_blk is too short.  We skip+     * through the decoded old block (o_blk) and copy from there.+     */+    char *cp = cram_block_get_data(o_blk);+    char *op = cp;+    char *endp = cp + cram_block_get_uncomp_size(o_blk);+    //fprintf(stderr, "sz = %d\n", (int)(endp-cp));+    int32_t i32;++    cp += safe_itf8_get(cp, endp, &i32);+    cp += i32;+    cp += safe_itf8_get(cp, endp, &i32);+    cp += i32;+    op = cp;+    cp += safe_itf8_get(cp, endp, &i32);+    i32 += (cp-op);++    //fprintf(stderr, "remaining %d bytes\n", i32);+    cram_block_set_size(n_blk, cram_block_get_size(n_blk)-2);+    cram_block_append(n_blk, op, i32);+    cram_block_update_size(n_blk);++    new_size = cram_block_size(n_blk);++    //fprintf(stderr, "size %d -> %d\n", old_size, new_size);++    // Now we've constructedthe updated compression header,+    // amend the container too (it may have changed size).+    int32_t *landmarks, num_landmarks;+    landmarks = cram_container_get_landmarks(c, &num_landmarks);++    if (old_size != new_size) {+	int diff = new_size - old_size, j;++	for (j = 0; j < num_landmarks; j++)+	    landmarks[j] += diff;+	//cram_container_set_landmarks(c, num_landmarks, landmarks);+	cram_container_set_length(c, cram_container_get_length(c) + diff);+    }++    // Finally write it all out; container, compression header,+    // and then all the remaining slice blocks.+    if (cram_write_container(out, c) != 0)+	return -2;++    cram_write_block(out, n_blk);+    cram_free_block(o_blk);+    cram_free_block(n_blk);++    // Container num_blocks can be invalid, due to a bug.+    // Instead we iterate in slice context instead.+    return cram_copy_slice(in, out, num_landmarks);+}
+ htslib-1.3.2/cram/cram_index.c view
@@ -0,0 +1,582 @@+/*+Copyright (c) 2013-2014 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+ * The index is a gzipped tab-delimited text file with one line per slice.+ * The columns are:+ * 1: reference number (0 to N-1, as per BAM ref_id)+ * 2: reference position of 1st read in slice (1..?)+ * 3: number of reads in slice+ * 4: offset of container start (relative to end of SAM header, so 1st+ *    container is offset 0).+ * 5: slice number within container (ie which landmark).+ *+ * In memory, we hold this in a nested containment list. Each list element is+ * a cram_index struct. Each element in turn can contain its own list of+ * cram_index structs.+ *+ * Any start..end range which is entirely contained within another (and+ * earlier as it is sorted) range will be held within it. This ensures that+ * the outer list will never have containments and we can safely do a+ * binary search to find the first range which overlaps any given coordinate.+ */++#include <config.h>++#include <stdio.h>+#include <errno.h>+#include <assert.h>+#include <stdlib.h>+#include <string.h>+#include <zlib.h>+#include <sys/types.h>+#include <sys/stat.h>+#include <math.h>+#include <ctype.h>++#include "htslib/hfile.h"+#include "hts_internal.h"+#include "cram/cram.h"+#include "cram/os.h"+#include "cram/zfio.h"++#if 0+static void dump_index_(cram_index *e, int level) {+    int i, n;+    n = printf("%*s%d / %d .. %d, ", level*4, "", e->refid, e->start, e->end);+    printf("%*soffset %"PRId64"\n", MAX(0,50-n), "", e->offset);+    for (i = 0; i < e->nslice; i++) {+	dump_index_(&e->e[i], level+1);+    }+}++static void dump_index(cram_fd *fd) {+    int i;+    for (i = 0; i < fd->index_sz; i++) {+	dump_index_(&fd->index[i], 0);+    }+}+#endif++static int kget_int32(kstring_t *k, size_t *pos, int32_t *val_p) {+    int sign = 1;+    int32_t val = 0;+    size_t p = *pos;++    while (p < k->l && (k->s[p] == ' ' || k->s[p] == '\t'))+	   p++;++    if (p < k->l && k->s[p] == '-')+	sign = -1, p++;++    if (p >= k->l || !(k->s[p] >= '0' && k->s[p] <= '9'))+	return -1;++    while (p < k->l && k->s[p] >= '0' && k->s[p] <= '9')+	val = val*10 + k->s[p++]-'0';+    +    *pos = p;+    *val_p = sign*val;++    return 0;+}++static int kget_int64(kstring_t *k, size_t *pos, int64_t *val_p) {+    int sign = 1;+    int64_t val = 0;+    size_t p = *pos;++    while (p < k->l && (k->s[p] == ' ' || k->s[p] == '\t'))+	   p++;++    if (p < k->l && k->s[p] == '-')+	sign = -1, p++;++    if (p >= k->l || !(k->s[p] >= '0' && k->s[p] <= '9'))+	return -1;++    while (p < k->l && k->s[p] >= '0' && k->s[p] <= '9')+	val = val*10 + k->s[p++]-'0';+    +    *pos = p;+    *val_p = sign*val;++    return 0;+}++/*+ * Loads a CRAM .crai index into memory.+ *+ * Returns 0 for success+ *        -1 for failure+ */+int cram_index_load(cram_fd *fd, const char *fn, const char *fn_idx) {+    char *fn2 = NULL;+    char buf[65536];+    ssize_t len;+    kstring_t kstr = {0};+    FILE *fp;+    cram_index *idx;+    cram_index **idx_stack = NULL, *ep, e;+    int idx_stack_alloc = 0, idx_stack_ptr = 0;+    size_t pos = 0;++    /* Check if already loaded */+    if (fd->index)+	return 0;++    fd->index = calloc((fd->index_sz = 1), sizeof(*fd->index));+    if (!fd->index)+	return -1;++    idx = &fd->index[0];+    idx->refid = -1;+    idx->start = INT_MIN;+    idx->end   = INT_MAX;++    idx_stack = calloc(++idx_stack_alloc, sizeof(*idx_stack));+    idx_stack[idx_stack_ptr] = idx;++    if (!fn_idx) {+	fn2 = hts_idx_getfn(fn, ".crai");+	if (!fn2) {+	    free(idx_stack);+	    return -1;+	}+	fn_idx = fn2;+    }++    if (!(fp = fopen(fn_idx, "r"))) {+	perror(fn_idx);+	free(idx_stack);+	free(fn2);+	return -1;+    }++    // Load the file into memory+    while ((len = fread(buf, 1, 65536, fp)) > 0)+	kputsn(buf, len, &kstr);+    if (len < 0 || kstr.l < 2) {+	if (kstr.s)+	    free(kstr.s);+	free(idx_stack);+	free(fn2);+	return -1;+    }++    if (fclose(fp)) {+	if (kstr.s)+	    free(kstr.s);+	free(idx_stack);+	free(fn2);+	return -1;+    }+	++    // Uncompress if required+    if (kstr.s[0] == 31 && (uc)kstr.s[1] == 139) {+	size_t l;+	char *s = zlib_mem_inflate(kstr.s, kstr.l, &l);+	free(kstr.s);+	if (!s) {+	    free(idx_stack);+	    free(fn2);+	    return -1;+	}+	kstr.s = s;+	kstr.l = l;+	kstr.m = l; // conservative estimate of the size allocated+	kputsn("", 0, &kstr); // ensure kstr.s is NUL-terminated+    }+++    // Parse it line at a time+    do {+	/* 1.1 layout */+	if (kget_int32(&kstr, &pos, &e.refid) == -1) {+	    free(kstr.s); free(idx_stack); free(fn2); return -1;+	}+	if (kget_int32(&kstr, &pos, &e.start) == -1) {+	    free(kstr.s); free(idx_stack); free(fn2); return -1;+	}+	if (kget_int32(&kstr, &pos, &e.end) == -1) {+	    free(kstr.s); free(idx_stack); free(fn2); return -1;+	}+	if (kget_int64(&kstr, &pos, &e.offset) == -1) {+	    free(kstr.s); free(idx_stack); free(fn2); return -1;+	}+	if (kget_int32(&kstr, &pos, &e.slice) == -1) {+	    free(kstr.s); free(idx_stack); free(fn2); return -1;+	}+	if (kget_int32(&kstr, &pos, &e.len) == -1) {+	    free(kstr.s); free(idx_stack); free(fn2); return -1;+	}++	e.end += e.start-1;+	//printf("%d/%d..%d\n", e.refid, e.start, e.end);++	if (e.refid < -1) {+	    free(kstr.s);+	    free(idx_stack);+	    free(fn2);+	    fprintf(stderr, "Malformed index file, refid %d\n", e.refid);+	    return -1;+	}++	if (e.refid != idx->refid) {+	    if (fd->index_sz < e.refid+2) {+		size_t index_end = fd->index_sz * sizeof(*fd->index);+		fd->index_sz = e.refid+2;+		fd->index = realloc(fd->index,+				    fd->index_sz * sizeof(*fd->index));+		memset(((char *)fd->index) + index_end, 0,+		       fd->index_sz * sizeof(*fd->index) - index_end);+	    }+	    idx = &fd->index[e.refid+1];+	    idx->refid = e.refid;+	    idx->start = INT_MIN;+	    idx->end   = INT_MAX;+	    idx->nslice = idx->nalloc = 0;+	    idx->e = NULL;+	    idx_stack[(idx_stack_ptr = 0)] = idx;+	}++	while (!(e.start >= idx->start && e.end <= idx->end) || idx->end == 0) {+	    idx = idx_stack[--idx_stack_ptr];+	}++	// Now contains, so append+	if (idx->nslice+1 >= idx->nalloc) {+	    idx->nalloc = idx->nalloc ? idx->nalloc*2 : 16;+	    idx->e = realloc(idx->e, idx->nalloc * sizeof(*idx->e));+	}++	e.nalloc = e.nslice = 0; e.e = NULL;+	*(ep = &idx->e[idx->nslice++]) = e;+	idx = ep;++	if (++idx_stack_ptr >= idx_stack_alloc) {+	    idx_stack_alloc *= 2;+	    idx_stack = realloc(idx_stack, idx_stack_alloc*sizeof(*idx_stack));+	}+	idx_stack[idx_stack_ptr] = idx;++	while (pos < kstr.l && kstr.s[pos] != '\n')+	    pos++;+	pos++;+    } while (pos < kstr.l);++    free(idx_stack);+    free(kstr.s);+    free(fn2);++    // dump_index(fd);++    return 0;+}++static void cram_index_free_recurse(cram_index *e) {+    if (e->e) {+	int i;+	for (i = 0; i < e->nslice; i++) {+	    cram_index_free_recurse(&e->e[i]);+	}+	free(e->e);+    }+}++void cram_index_free(cram_fd *fd) {+    int i;++    if (!fd->index)+	return;+    +    for (i = 0; i < fd->index_sz; i++) {+	cram_index_free_recurse(&fd->index[i]);+    }+    free(fd->index);++    fd->index = NULL;+}++/*+ * Searches the index for the first slice overlapping a reference ID+ * and position, or one immediately preceding it if none is found in+ * the index to overlap this position. (Our index may have missing+ * entries, but we require at least one per reference.)+ *+ * If the index finds multiple slices overlapping this position we+ * return the first one only. Subsequent calls should specifying+ * "from" as the last slice we checked to find the next one. Otherwise+ * set "from" to be NULL to find the first one.+ *+ * Returns the cram_index pointer on sucess+ *         NULL on failure+ */+cram_index *cram_index_query(cram_fd *fd, int refid, int pos, +			     cram_index *from) {+    int i, j, k;+    cram_index *e;++    if (refid+1 < 0 || refid+1 >= fd->index_sz)+	return NULL;++    if (!from)+	from = &fd->index[refid+1];++    // Ref with nothing aligned against it.+    if (!from->e)+	return NULL;++    // This sequence is covered by the index, so binary search to find+    // the optimal starting block.+    i = 0, j = fd->index[refid+1].nslice-1;+    for (k = j/2; k != i; k = (j-i)/2 + i) {+	if (from->e[k].refid > refid) {+	    j = k;+	    continue;+	}++	if (from->e[k].refid < refid) {+	    i = k;+	    continue;+	}++	if (from->e[k].start >= pos) {+	    j = k;+	    continue;+	}++	if (from->e[k].start < pos) {+	    i = k;+	    continue;+	}+    }+    // i==j or i==j-1. Check if j is better.+    if (j >= 0 && from->e[j].start < pos && from->e[j].refid == refid)+	i = j;++    /* The above found *a* bin overlapping, but not necessarily the first */+    while (i > 0 && from->e[i-1].end >= pos)+	i--;++    /* We may be one bin before the optimum, so check */+    while (i+1 < from->nslice &&+	   (from->e[i].refid < refid ||+	    from->e[i].end < pos))+	i++;++    e = &from->e[i];++    return e;+}+++/*+ * Skips to a container overlapping the start coordinate listed in+ * cram_range.+ *+ * In theory we call cram_index_query multiple times, once per slice+ * overlapping the range. However slices may be absent from the index+ * which makes this problematic. Instead we find the left-most slice+ * and then read from then on, skipping decoding of slices and/or+ * whole containers when they don't overlap the specified cram_range.+ *+ * Returns 0 on success+ *        -1 on general failure+ *        -2 on no-data (empty chromosome)+ */+int cram_seek_to_refpos(cram_fd *fd, cram_range *r) {+    cram_index *e;++    // Ideally use an index, so see if we have one.+    if ((e = cram_index_query(fd, r->refid, r->start, NULL))) {+	if (0 != cram_seek(fd, e->offset, SEEK_SET))+	    if (0 != cram_seek(fd, e->offset - fd->first_container, SEEK_CUR))+		return -1;+    } else {+	// Absent from index, but this most likely means it simply has no data.+	return -2;+    }++    if (fd->ctr) {+	cram_free_container(fd->ctr);+	fd->ctr = NULL;+	fd->ooc = 0;+    }++    return 0;+}+++/*+ * A specialised form of cram_index_build (below) that deals with slices+ * having multiple references in this (ref_id -2). In this scenario we+ * decode the slice to look at the RI data series instead.+ *+ * Returns 0 on success+ *        -1 on failure+ */+static int cram_index_build_multiref(cram_fd *fd,+				     cram_container *c,+				     cram_slice *s,+				     zfp *fp,+				     off_t cpos,+				     int32_t landmark,+				     int sz) {+    int i, ref = -2, ref_start = 0, ref_end;+    char buf[1024];++    if (0 != cram_decode_slice(fd, c, s, fd->header))+	return -1;++    ref_end = INT_MIN;+    for (i = 0; i < s->hdr->num_records; i++) {+	if (s->crecs[i].ref_id == ref) {+	    if (ref_end < s->crecs[i].aend)+		ref_end = s->crecs[i].aend;+	    continue;+	}++	if (ref != -2) {+	    sprintf(buf, "%d\t%d\t%d\t%"PRId64"\t%d\t%d\n",+		    ref, ref_start, ref_end - ref_start + 1,+		    (int64_t)cpos, landmark, sz);+	    zfputs(buf, fp);+	}++	ref = s->crecs[i].ref_id;+	ref_start = s->crecs[i].apos;+	ref_end = INT_MIN;+    }++    if (ref != -2) {+	sprintf(buf, "%d\t%d\t%d\t%"PRId64"\t%d\t%d\n",+		ref, ref_start, ref_end - ref_start + 1,+		(int64_t)cpos, landmark, sz);+	zfputs(buf, fp);+    }++    return 0;+}++/*+ * Builds an index file.+ *+ * fd is a newly opened cram file that we wish to index.+ * fn_base is the filename of the associated CRAM file.+ * fn_idx is the filename of the index file to be written;+ * if NULL, we add ".crai" to fn_base to get the index filename.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_index_build(cram_fd *fd, const char *fn_base, const char *fn_idx) {+    cram_container *c;+    off_t cpos, spos, hpos;+    zfp *fp;+    kstring_t fn_idx_str = {0};++    if (! fn_idx) {+        kputs(fn_base, &fn_idx_str);+        kputs(".crai", &fn_idx_str);+        fn_idx = fn_idx_str.s;+    }++    if (!(fp = zfopen(fn_idx, "wz"))) {+        perror(fn_idx);+        free(fn_idx_str.s);+        return -1;+    }++    free(fn_idx_str.s);++    cpos = htell(fd->fp);+    while ((c = cram_read_container(fd))) {+        int j;++        if (fd->err) {+            perror("Cram container read");+            return -1;+        }++        hpos = htell(fd->fp);++        if (!(c->comp_hdr_block = cram_read_block(fd)))+            return -1;+        assert(c->comp_hdr_block->content_type == COMPRESSION_HEADER);++        c->comp_hdr = cram_decode_compression_header(fd, c->comp_hdr_block);+        if (!c->comp_hdr)+            return -1;++        // 2.0 format+        for (j = 0; j < c->num_landmarks; j++) {+            char buf[1024];+            cram_slice *s;+            int sz;++            spos = htell(fd->fp);+            assert(spos - cpos - c->offset == c->landmark[j]);++            if (!(s = cram_read_slice(fd))) {+		zfclose(fp);+		return -1;+	    }++            sz = (int)(htell(fd->fp) - spos);++	    if (s->hdr->ref_seq_id == -2) {+		cram_index_build_multiref(fd, c, s, fp,+					  cpos, c->landmark[j], sz);+	    } else {+		sprintf(buf, "%d\t%d\t%d\t%"PRId64"\t%d\t%d\n",+			s->hdr->ref_seq_id, s->hdr->ref_seq_start,+			s->hdr->ref_seq_span, (int64_t)cpos,+			c->landmark[j], sz);+		zfputs(buf, fp);+	    }++            cram_free_slice(s);+        }++        cpos = htell(fd->fp);+        assert(cpos == hpos + c->length);++        cram_free_container(c);+    }+    if (fd->err) {+	zfclose(fp);+	return -1;+    }+	++    return (zfclose(fp) >= 0)? 0 : -1;+}
+ htslib-1.3.2/cram/cram_index.h view
@@ -0,0 +1,99 @@+/*+Copyright (c) 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _CRAM_INDEX_H_+#define _CRAM_INDEX_H_++#ifdef __cplusplus+extern "C" {+#endif++/*+ * Loads a CRAM .crai index into memory.+ * Returns 0 for success+ *        -1 for failure+ */+int cram_index_load(cram_fd *fd, const char *fn, const char *fn_idx);++void cram_index_free(cram_fd *fd);++/*+ * Searches the index for the first slice overlapping a reference ID+ * and position.+ *+ * Returns the cram_index pointer on sucess+ *         NULL on failure+ */+cram_index *cram_index_query(cram_fd *fd, int refid, int pos, cram_index *frm);++/*+ * Skips to a container overlapping the start coordinate listed in+ * cram_range.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_seek_to_refpos(cram_fd *fd, cram_range *r);++void cram_index_free(cram_fd *fd);++/*+ * Skips to a container overlapping the start coordinate listed in+ * cram_range.+ *+ * In theory we call cram_index_query multiple times, once per slice+ * overlapping the range. However slices may be absent from the index+ * which makes this problematic. Instead we find the left-most slice+ * and then read from then on, skipping decoding of slices and/or+ * whole containers when they don't overlap the specified cram_range.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_seek_to_refpos(cram_fd *fd, cram_range *r);++/*+ * Builds an index file.+ *+ * fd is a newly opened cram file that we wish to index.+ * fn_base is the filename of the associated CRAM file.+ * fn_idx is the filename of the index file to be written;+ * if NULL, we add ".crai" to fn_base to get the index filename.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_index_build(cram_fd *fd, const char *fn_base, const char *fn_idx);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/cram/cram_io.c view
@@ -0,0 +1,4608 @@+/*+Copyright (c) 2012-2014 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+ * CRAM I/O primitives.+ *+ * - ITF8 encoding and decoding.+ * - Block based I/O+ * - Zlib inflating and deflating (memory)+ * - CRAM basic data structure reading and writing+ * - File opening / closing+ * - Reference sequence handling+ */++/*+ * TODO: BLOCK_GROW, BLOCK_RESIZE, BLOCK_APPEND and itf8_put_blk all need+ * a way to return errors for when malloc fails.+ */++#include <config.h>++#include <stdio.h>+#include <errno.h>+#include <assert.h>+#include <stdlib.h>+#include <string.h>+#include <unistd.h>+#include <zlib.h>+#ifdef HAVE_LIBBZ2+#include <bzlib.h>+#endif+#ifdef HAVE_LIBLZMA+#include <lzma.h>+#endif+#include <sys/types.h>+#include <sys/stat.h>+#include <math.h>+#include <ctype.h>+#include <time.h>++#include "cram/cram.h"+#include "cram/os.h"+#include "htslib/hts.h"+#include "cram/open_trace_file.h"+#include "cram/rANS_static.h"++//#define REF_DEBUG++#ifdef REF_DEBUG+#include <sys/syscall.h>+#define gettid() (int)syscall(SYS_gettid)++#define RP(...) fprintf (stderr, __VA_ARGS__)+#else+#define RP(...) +#endif++#include "htslib/hfile.h"+#include "htslib/bgzf.h"+#include "htslib/faidx.h"++#ifndef PATH_MAX+#define PATH_MAX FILENAME_MAX+#endif++#define TRIAL_SPAN 50+#define NTRIALS 3+++/* ----------------------------------------------------------------------+ * ITF8 encoding and decoding.+ *+* Also see the itf8_get and itf8_put macros in cram_io.h+ */++/*+ * LEGACY: consider using itf8_decode_crc.+ *+ * Reads an integer in ITF-8 encoding from 'cp' and stores it in+ * *val.+ *+ * Returns the number of bytes read on success+ *        -1 on failure+ */+int itf8_decode(cram_fd *fd, int32_t *val_p) {+    static int nbytes[16] = {+	0,0,0,0, 0,0,0,0,                               // 0000xxxx - 0111xxxx+	1,1,1,1,                                        // 1000xxxx - 1011xxxx+	2,2,                                            // 1100xxxx - 1101xxxx+	3,                                              // 1110xxxx+	4,                                              // 1111xxxx+    };++    static int nbits[16] = {+	0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, // 0000xxxx - 0111xxxx+	0x3f, 0x3f, 0x3f, 0x3f,                         // 1000xxxx - 1011xxxx+	0x1f, 0x1f,                                     // 1100xxxx - 1101xxxx+	0x0f,                                           // 1110xxxx+	0x0f,                                           // 1111xxxx+    };++    int32_t val = hgetc(fd->fp);+    if (val == -1)+	return -1;++    int i = nbytes[val>>4];+    val &= nbits[val>>4];++    switch(i) {+    case 0:+	*val_p = val;+	return 1;++    case 1:+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val;+	return 2;++    case 2:+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val;+	return 3;++    case 3:+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val;+	return 4;++    case 4: // really 3.5 more, why make it different?+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<4) | (((unsigned char)hgetc(fd->fp)) & 0x0f);+	*val_p = val;+    }++    return 5;+}++int itf8_decode_crc(cram_fd *fd, int32_t *val_p, uint32_t *crc) {+    static int nbytes[16] = {+	0,0,0,0, 0,0,0,0,                               // 0000xxxx - 0111xxxx+	1,1,1,1,                                        // 1000xxxx - 1011xxxx+	2,2,                                            // 1100xxxx - 1101xxxx+	3,                                              // 1110xxxx+	4,                                              // 1111xxxx+    };++    static int nbits[16] = {+	0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, // 0000xxxx - 0111xxxx+	0x3f, 0x3f, 0x3f, 0x3f,                         // 1000xxxx - 1011xxxx+	0x1f, 0x1f,                                     // 1100xxxx - 1101xxxx+	0x0f,                                           // 1110xxxx+	0x0f,                                           // 1111xxxx+    };+    unsigned char c[5];++    int32_t val = hgetc(fd->fp);+    if (val == -1)+	return -1;++    c[0]=val;++    int i = nbytes[val>>4];+    val &= nbits[val>>4];++    switch(i) {+    case 0:+	*val_p = val;+	*crc = crc32(*crc, c, 1);+	return 1;++    case 1:+	val = (val<<8) | (c[1]=hgetc(fd->fp));+	*val_p = val;+	*crc = crc32(*crc, c, 2);+	return 2;++    case 2:+	val = (val<<8) | (c[1]=hgetc(fd->fp));+	val = (val<<8) | (c[2]=hgetc(fd->fp));+	*val_p = val;+	*crc = crc32(*crc, c, 3);+	return 3;++    case 3:+	val = (val<<8) | (c[1]=hgetc(fd->fp));+	val = (val<<8) | (c[2]=hgetc(fd->fp));+	val = (val<<8) | (c[3]=hgetc(fd->fp));+	*val_p = val;+	*crc = crc32(*crc, c, 4);+	return 4;++    case 4: // really 3.5 more, why make it different?+	val = (val<<8) |   (c[1]=hgetc(fd->fp));+	val = (val<<8) |   (c[2]=hgetc(fd->fp));+	val = (val<<8) |   (c[3]=hgetc(fd->fp));+	val = (val<<4) | (((c[4]=hgetc(fd->fp))) & 0x0f);+	*val_p = val;+	*crc = crc32(*crc, c, 5);+    }++    return 5;+}++/*+ * Encodes and writes a single integer in ITF-8 format.+ * Returns 0 on success+ *        -1 on failure+ */+int itf8_encode(cram_fd *fd, int32_t val) {+    char buf[5];+    int len = itf8_put(buf, val);+    return hwrite(fd->fp, buf, len) == len ? 0 : -1;+}+ +const int itf8_bytes[16] = {+    1, 1, 1, 1, 1, 1, 1, 1,+    2, 2, 2, 2, 3, 3, 4, 5+};++#ifndef ITF8_MACROS+/*+ * As above, but decoding from memory+ */+int itf8_get(char *cp, int32_t *val_p) {+    unsigned char *up = (unsigned char *)cp;+    +    if (up[0] < 0x80) {+	*val_p =   up[0];+	return 1;+    } else if (up[0] < 0xc0) {+	*val_p = ((up[0] <<8) |  up[1])                           & 0x3fff;+	return 2;+    } else if (up[0] < 0xe0) {+	*val_p = ((up[0]<<16) | (up[1]<< 8) |  up[2])             & 0x1fffff;+	return 3;+    } else if (up[0] < 0xf0) {+	*val_p = ((up[0]<<24) | (up[1]<<16) | (up[2]<<8) | up[3]) & 0x0fffffff;+	return 4;+    } else {+	*val_p = ((up[0] & 0x0f)<<28) | (up[1]<<20) | (up[2]<<12) | (up[3]<<4) | (up[4] & 0x0f);+	return 5;+    }+}++/*+ * Stores a value to memory in ITF-8 format.+ *+ * Returns the number of bytes required to store the number.+ * This is a maximum of 5 bytes.+ */+int itf8_put(char *cp, int32_t val) {+    if        (!(val & ~0x00000007f)) { // 1 byte+	*cp = val;+	return 1;+    } else if (!(val & ~0x00003fff)) { // 2 byte+	*cp++ = (val >> 8 ) | 0x80;+	*cp   = val & 0xff;+	return 2;+    } else if (!(val & ~0x01fffff)) { // 3 byte+	*cp++ = (val >> 16) | 0xc0;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 3;+    } else if (!(val & ~0x0fffffff)) { // 4 byte+	*cp++ = (val >> 24) | 0xe0;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 4;+    } else {                           // 5 byte+	*cp++ = 0xf0 | ((val>>28) & 0xff);+	*cp++ = (val >> 20) & 0xff;+	*cp++ = (val >> 12) & 0xff;+	*cp++ = (val >> 4 ) & 0xff;+	*cp = val & 0x0f;+	return 5;+    }+}+#endif++/* 64-bit itf8 variant */+int ltf8_put(char *cp, int64_t val) {+    if        (!(val & ~((1LL<<7)-1))) {+	*cp = val;+	return 1;+    } else if (!(val & ~((1LL<<(6+8))-1))) {+	*cp++ = (val >> 8 ) | 0x80;+	*cp   = val & 0xff;+	return 2;+    } else if (!(val & ~((1LL<<(5+2*8))-1))) {+	*cp++ = (val >> 16) | 0xc0;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 3;+    } else if (!(val & ~((1LL<<(4+3*8))-1))) {+	*cp++ = (val >> 24) | 0xe0;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 4;+    } else if (!(val & ~((1LL<<(3+4*8))-1))) {+	*cp++ = (val >> 32) | 0xf0;+	*cp++ = (val >> 24) & 0xff;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 5;+    } else if (!(val & ~((1LL<<(2+5*8))-1))) {+	*cp++ = (val >> 40) | 0xf8;+	*cp++ = (val >> 32) & 0xff;+	*cp++ = (val >> 24) & 0xff;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 6;+    } else if (!(val & ~((1LL<<(1+6*8))-1))) {+	*cp++ = (val >> 48) | 0xfc;+	*cp++ = (val >> 40) & 0xff;+	*cp++ = (val >> 32) & 0xff;+	*cp++ = (val >> 24) & 0xff;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 7;+    } else if (!(val & ~((1LL<<(7*8))-1))) {+	*cp++ = (val >> 56) | 0xfe;+	*cp++ = (val >> 48) & 0xff;+	*cp++ = (val >> 40) & 0xff;+	*cp++ = (val >> 32) & 0xff;+	*cp++ = (val >> 24) & 0xff;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 8;+    } else {+	*cp++ = 0xff;+	*cp++ = (val >> 56) & 0xff;+	*cp++ = (val >> 48) & 0xff;+	*cp++ = (val >> 40) & 0xff;+	*cp++ = (val >> 32) & 0xff;+	*cp++ = (val >> 24) & 0xff;+	*cp++ = (val >> 16) & 0xff;+	*cp++ = (val >> 8 ) & 0xff;+	*cp   = val & 0xff;+	return 9;+    }+}++int ltf8_get(char *cp, int64_t *val_p) {+    unsigned char *up = (unsigned char *)cp;+    +    if (up[0] < 0x80) {+	*val_p =   up[0];+	return 1;+    } else if (up[0] < 0xc0) {+	*val_p = (((uint64_t)up[0]<< 8) |+		   (uint64_t)up[1]) & (((1LL<<(6+8)))-1);+	return 2;+    } else if (up[0] < 0xe0) {+	*val_p = (((uint64_t)up[0]<<16) |+		  ((uint64_t)up[1]<< 8) |+		   (uint64_t)up[2]) & ((1LL<<(5+2*8))-1);+	return 3;+    } else if (up[0] < 0xf0) {+	*val_p = (((uint64_t)up[0]<<24) |+		  ((uint64_t)up[1]<<16) |+		  ((uint64_t)up[2]<< 8) |+		   (uint64_t)up[3]) & ((1LL<<(4+3*8))-1);+	return 4;+    } else if (up[0] < 0xf8) {+	*val_p = (((uint64_t)up[0]<<32) |+		  ((uint64_t)up[1]<<24) |+		  ((uint64_t)up[2]<<16) |+		  ((uint64_t)up[3]<< 8) |+		   (uint64_t)up[4]) & ((1LL<<(3+4*8))-1);+	return 5;+    } else if (up[0] < 0xfc) {+	*val_p = (((uint64_t)up[0]<<40) |+		  ((uint64_t)up[1]<<32) |+		  ((uint64_t)up[2]<<24) |+		  ((uint64_t)up[3]<<16) |+		  ((uint64_t)up[4]<< 8) |+		   (uint64_t)up[5]) & ((1LL<<(2+5*8))-1);+	return 6;+    } else if (up[0] < 0xfe) {+	*val_p = (((uint64_t)up[0]<<48) |+		  ((uint64_t)up[1]<<40) |+		  ((uint64_t)up[2]<<32) |+		  ((uint64_t)up[3]<<24) |+		  ((uint64_t)up[4]<<16) |+		  ((uint64_t)up[5]<< 8) |+		   (uint64_t)up[6]) & ((1LL<<(1+6*8))-1);+	return 7;+    } else if (up[0] < 0xff) {+	*val_p = (((uint64_t)up[1]<<48) |+		  ((uint64_t)up[2]<<40) |+		  ((uint64_t)up[3]<<32) |+		  ((uint64_t)up[4]<<24) |+		  ((uint64_t)up[5]<<16) |+		  ((uint64_t)up[6]<< 8) |+		   (uint64_t)up[7]) & ((1LL<<(7*8))-1);+	return 8;+    } else {+	*val_p = (((uint64_t)up[1]<<56) |+		  ((uint64_t)up[2]<<48) |+		  ((uint64_t)up[3]<<40) |+		  ((uint64_t)up[4]<<32) |+		  ((uint64_t)up[5]<<24) |+		  ((uint64_t)up[6]<<16) |+		  ((uint64_t)up[7]<< 8) |+		   (uint64_t)up[8]);+	return 9;+    }+}++/*+ * LEGACY: consider using ltf8_decode_crc.+ */+int ltf8_decode(cram_fd *fd, int64_t *val_p) {+    int c = hgetc(fd->fp);+    int64_t val = (unsigned char)c;+    if (c == -1)+	return -1;++    if (val < 0x80) {+	*val_p =   val;+	return 1;++    } else if (val < 0xc0) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & (((1LL<<(6+8)))-1);+	return 2;++    } else if (val < 0xe0) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & ((1LL<<(5+2*8))-1);+	return 3;++    } else if (val < 0xf0) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & ((1LL<<(4+3*8))-1);+	return 4;++    } else if (val < 0xf8) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & ((1LL<<(3+4*8))-1);+	return 5;++    } else if (val < 0xfc) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & ((1LL<<(2+5*8))-1);+	return 6;++    } else if (val < 0xfe) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & ((1LL<<(1+6*8))-1);+	return 7;++    } else if (val < 0xff) {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val & ((1LL<<(7*8))-1);+	return 8;++    } else {+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	val = (val<<8) | (unsigned char)hgetc(fd->fp);+	*val_p = val;+    }++    return 9;+}++int ltf8_decode_crc(cram_fd *fd, int64_t *val_p, uint32_t *crc) {+    unsigned char c[9];+    int64_t val = (unsigned char)hgetc(fd->fp);+    if (val == -1)+	return -1;++    c[0] = val;++    if (val < 0x80) {+	*val_p =   val;+	*crc = crc32(*crc, c, 1);+	return 1;++    } else if (val < 0xc0) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	*val_p = val & (((1LL<<(6+8)))-1);+	*crc = crc32(*crc, c, 2);+	return 2;++    } else if (val < 0xe0) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	*val_p = val & ((1LL<<(5+2*8))-1);+	*crc = crc32(*crc, c, 3);+	return 3;++    } else if (val < 0xf0) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	val = (val<<8) | (c[3]=hgetc(fd->fp));;+	*val_p = val & ((1LL<<(4+3*8))-1);+	*crc = crc32(*crc, c, 4);+	return 4;++    } else if (val < 0xf8) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	val = (val<<8) | (c[3]=hgetc(fd->fp));;+	val = (val<<8) | (c[4]=hgetc(fd->fp));;+	*val_p = val & ((1LL<<(3+4*8))-1);+	*crc = crc32(*crc, c, 5);+	return 5;++    } else if (val < 0xfc) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	val = (val<<8) | (c[3]=hgetc(fd->fp));;+	val = (val<<8) | (c[4]=hgetc(fd->fp));;+	val = (val<<8) | (c[5]=hgetc(fd->fp));;+	*val_p = val & ((1LL<<(2+5*8))-1);+	*crc = crc32(*crc, c, 6);+	return 6;++    } else if (val < 0xfe) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	val = (val<<8) | (c[3]=hgetc(fd->fp));;+	val = (val<<8) | (c[4]=hgetc(fd->fp));;+	val = (val<<8) | (c[5]=hgetc(fd->fp));;+	val = (val<<8) | (c[6]=hgetc(fd->fp));;+	*val_p = val & ((1LL<<(1+6*8))-1);+	*crc = crc32(*crc, c, 7);+	return 7;++    } else if (val < 0xff) {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	val = (val<<8) | (c[3]=hgetc(fd->fp));;+	val = (val<<8) | (c[4]=hgetc(fd->fp));;+	val = (val<<8) | (c[5]=hgetc(fd->fp));;+	val = (val<<8) | (c[6]=hgetc(fd->fp));;+	val = (val<<8) | (c[7]=hgetc(fd->fp));;+	*val_p = val & ((1LL<<(7*8))-1);+	*crc = crc32(*crc, c, 8);+	return 8;++    } else {+	val = (val<<8) | (c[1]=hgetc(fd->fp));;+	val = (val<<8) | (c[2]=hgetc(fd->fp));;+	val = (val<<8) | (c[3]=hgetc(fd->fp));;+	val = (val<<8) | (c[4]=hgetc(fd->fp));;+	val = (val<<8) | (c[5]=hgetc(fd->fp));;+	val = (val<<8) | (c[6]=hgetc(fd->fp));;+	val = (val<<8) | (c[7]=hgetc(fd->fp));;+	val = (val<<8) | (c[8]=hgetc(fd->fp));;+	*crc = crc32(*crc, c, 9);+	*val_p = val;+    }++    return 9;+}++/*+ * Pushes a value in ITF8 format onto the end of a block.+ * This shouldn't be used for high-volume data as it is not the fastest+ * method.+ *+ * Returns the number of bytes written+ */+int itf8_put_blk(cram_block *blk, int val) {+    char buf[5];+    int sz;++    sz = itf8_put(buf, val);+    BLOCK_APPEND(blk, buf, sz);+    return sz;+}++/*+ * Decodes a 32-bit little endian value from fd and stores in val.+ *+ * Returns the number of bytes read on success+ *         -1 on failure+ */+int int32_decode(cram_fd *fd, int32_t *val) {+    int32_t i;+    if (4 != hread(fd->fp, &i, 4))+	return -1;++    *val = le_int4(i);+    return 4;+}++/*+ * Encodes a 32-bit little endian value 'val' and writes to fd.+ *+ * Returns the number of bytes written on success+ *         -1 on failure+ */+int int32_encode(cram_fd *fd, int32_t val) {+    val = le_int4(val);+    if (4 != hwrite(fd->fp, &val, 4))+	return -1;++    return 4;+}++/* As int32_decoded/encode, but from/to blocks instead of cram_fd */+int int32_get_blk(cram_block *b, int32_t *val) {+    if (b->uncomp_size - BLOCK_SIZE(b) < 4)+	return -1;++    *val =+	 b->data[b->byte  ]        |+	(b->data[b->byte+1] <<  8) |+	(b->data[b->byte+2] << 16) |+	(b->data[b->byte+3] << 24);+    BLOCK_SIZE(b) += 4;+    return 4;+}++/* As int32_decoded/encode, but from/to blocks instead of cram_fd */+int int32_put_blk(cram_block *b, int32_t val) {+    unsigned char cp[4];+    cp[0] = ( val      & 0xff);+    cp[1] = ((val>>8)  & 0xff);+    cp[2] = ((val>>16) & 0xff);+    cp[3] = ((val>>24) & 0xff);++    BLOCK_APPEND(b, cp, 4);+    return b->data ? 0 : -1;+}++/* ----------------------------------------------------------------------+ * zlib compression code - from Gap5's tg_iface_g.c+ * They're static here as they're only used within the cram_compress_block+ * and cram_uncompress_block functions, which are the external interface.+ */+char *zlib_mem_inflate(char *cdata, size_t csize, size_t *size) {+    z_stream s;+    unsigned char *data = NULL; /* Uncompressed output */+    int data_alloc = 0;+    int err;++    /* Starting point at uncompressed size, and scale after that */+    data = malloc(data_alloc = csize*1.2+100);+    if (!data)+	return NULL;++    /* Initialise zlib stream */+    s.zalloc = Z_NULL; /* use default allocation functions */+    s.zfree  = Z_NULL;+    s.opaque = Z_NULL;+    s.next_in  = (unsigned char *)cdata;+    s.avail_in = csize;+    s.total_in = 0;+    s.next_out  = data;+    s.avail_out = data_alloc;+    s.total_out = 0;++    //err = inflateInit(&s);+    err = inflateInit2(&s, 15 + 32);+    if (err != Z_OK) {+	fprintf(stderr, "zlib inflateInit error: %s\n", s.msg);+	free(data);+	return NULL;+    }++    /* Decode to 'data' array */+    for (;s.avail_in;) {+	unsigned char *data_tmp;+	int alloc_inc;++	s.next_out = &data[s.total_out];+	err = inflate(&s, Z_NO_FLUSH);+	if (err == Z_STREAM_END)+	    break;++	if (err != Z_OK) {+	    fprintf(stderr, "zlib inflate error: %s\n", s.msg);+	    if (data)+		free(data);+	    return NULL;+	}++	/* More to come, so realloc based on growth so far */+	alloc_inc = (double)s.avail_in/s.total_in * s.total_out + 100;+	data = realloc((data_tmp = data), data_alloc += alloc_inc);+	if (!data) {+	    free(data_tmp);+	    return NULL;+	}+	s.avail_out += alloc_inc;+    }+    inflateEnd(&s);++    *size = s.total_out;+    return (char *)data;+}++static char *zlib_mem_deflate(char *data, size_t size, size_t *cdata_size,+			      int level, int strat) {+    z_stream s;+    unsigned char *cdata = NULL; /* Compressed output */+    int cdata_alloc = 0;+    int cdata_pos = 0;+    int err;++    cdata = malloc(cdata_alloc = size*1.05+100);+    if (!cdata)+	return NULL;+    cdata_pos = 0;++    /* Initialise zlib stream */+    s.zalloc = Z_NULL; /* use default allocation functions */+    s.zfree  = Z_NULL;+    s.opaque = Z_NULL;+    s.next_in  = (unsigned char *)data;+    s.avail_in = size;+    s.total_in = 0;+    s.next_out  = cdata;+    s.avail_out = cdata_alloc;+    s.total_out = 0;+    s.data_type = Z_BINARY;++    err = deflateInit2(&s, level, Z_DEFLATED, 15|16, 9, strat);+    if (err != Z_OK) {+	fprintf(stderr, "zlib deflateInit2 error: %s\n", s.msg);+	return NULL;+    }++    /* Encode to 'cdata' array */+    for (;s.avail_in;) {+	s.next_out = &cdata[cdata_pos];+	s.avail_out = cdata_alloc - cdata_pos;+	if (cdata_alloc - cdata_pos <= 0) {+	    fprintf(stderr, "Deflate produced larger output than expected. Abort\n"); +	    return NULL;+	}+	err = deflate(&s, Z_NO_FLUSH);+	cdata_pos = cdata_alloc - s.avail_out;+	if (err != Z_OK) {+	    fprintf(stderr, "zlib deflate error: %s\n", s.msg);+	    break;+	}+    }+    if (deflate(&s, Z_FINISH) != Z_STREAM_END) {+	fprintf(stderr, "zlib deflate error: %s\n", s.msg);+    }+    *cdata_size = s.total_out;++    if (deflateEnd(&s) != Z_OK) {+	fprintf(stderr, "zlib deflate error: %s\n", s.msg);+    }+    return (char *)cdata;+}++#ifdef HAVE_LIBLZMA+/* ------------------------------------------------------------------------ */+/*+ * Data compression routines using liblzma (xz)+ *+ * On a test set this shrunk the main db from 136157104 bytes to 114796168, but+ * caused tg_index to grow from 2m43.707s to 15m3.961s. Exporting as bfastq+ * went from 18.3s to 36.3s. So decompression suffers too, but not as bad+ * as compression times.+ *+ * For now we disable this functionality. If it's to be reenabled make sure you+ * improve the mem_inflate implementation as it's just a test hack at the+ * moment.+ */++static char *lzma_mem_deflate(char *data, size_t size, size_t *cdata_size,+			      int level) {+    char *out;+    size_t out_size = lzma_stream_buffer_bound(size);+    *cdata_size = 0;++    out = malloc(out_size);++    /* Single call compression */+    if (LZMA_OK != lzma_easy_buffer_encode(level, LZMA_CHECK_CRC32, NULL,+					   (uint8_t *)data, size,+					   (uint8_t *)out, cdata_size,+					   out_size))+    	return NULL;++    return out;+}++static char *lzma_mem_inflate(char *cdata, size_t csize, size_t *size) {+    lzma_stream strm = LZMA_STREAM_INIT;+    size_t out_size = 0, out_pos = 0;+    char *out = NULL;+    int r;++    /* Initiate the decoder */+    if (LZMA_OK != lzma_stream_decoder(&strm, 50000000, 0))+	return NULL;++    /* Decode loop */+    strm.avail_in = csize;+    strm.next_in = (uint8_t *)cdata;++    for (;strm.avail_in;) {+	if (strm.avail_in > out_size - out_pos) {+	    out_size += strm.avail_in * 4 + 32768;+	    out = realloc(out, out_size);+	}+	strm.avail_out = out_size - out_pos;+	strm.next_out = (uint8_t *)&out[out_pos];++	r = lzma_code(&strm, LZMA_RUN);+	if (LZMA_OK != r && LZMA_STREAM_END != r) {+	    fprintf(stderr, "r=%d\n", r);+	    fprintf(stderr, "mem=%"PRId64"d\n", (int64_t)lzma_memusage(&strm));+	    return NULL;+	}++	out_pos = strm.total_out;++	if (r == LZMA_STREAM_END)+	    break;+    }++    /* finish up any unflushed data; necessary? */+    r = lzma_code(&strm, LZMA_FINISH);+    if (r != LZMA_OK && r != LZMA_STREAM_END) {+	fprintf(stderr, "r=%d\n", r);+	return NULL;+    }++    out = realloc(out, strm.total_out);+    *size = strm.total_out;++    lzma_end(&strm);++    return out;+}+#endif++/* ----------------------------------------------------------------------+ * CRAM blocks - the dynamically growable data block. We have code to+ * create, update, (un)compress and read/write.+ *+ * These are derived from the deflate_interlaced.c blocks, but with the+ * CRAM extension of content types and IDs.+ */++/*+ * Allocates a new cram_block structure with a specified content_type and+ * id.+ *+ * Returns block pointer on success+ *         NULL on failure+ */+cram_block *cram_new_block(enum cram_content_type content_type,+			   int content_id) {+    cram_block *b = malloc(sizeof(*b));+    if (!b)+	return NULL;+    b->method = b->orig_method = RAW;+    b->content_type = content_type;+    b->content_id = content_id;+    b->comp_size = 0;+    b->uncomp_size = 0;+    b->data = NULL;+    b->alloc = 0;+    b->byte = 0;+    b->bit = 7; // MSB++    return b;+}++/*+ * Reads a block from a cram file.+ * Returns cram_block pointer on success.+ *         NULL on failure+ */+cram_block *cram_read_block(cram_fd *fd) {+    cram_block *b = malloc(sizeof(*b));+    unsigned char c;+    uint32_t crc = 0;+    if (!b)+	return NULL;++    //fprintf(stderr, "Block at %d\n", (int)ftell(fd->fp));++    if (-1 == (b->method      = hgetc(fd->fp))) { free(b); return NULL; }+    c = b->method; crc = crc32(crc, &c, 1);+    if (-1 == (b->content_type= hgetc(fd->fp))) { free(b); return NULL; }+    c = b->content_type; crc = crc32(crc, &c, 1);+    if (-1 == itf8_decode_crc(fd, &b->content_id, &crc))  { free(b); return NULL; }+    if (-1 == itf8_decode_crc(fd, &b->comp_size, &crc))   { free(b); return NULL; }+    if (-1 == itf8_decode_crc(fd, &b->uncomp_size, &crc)) { free(b); return NULL; }++    //    fprintf(stderr, "  method %d, ctype %d, cid %d, csize %d, ucsize %d\n",+    //	    b->method, b->content_type, b->content_id, b->comp_size, b->uncomp_size);++    if (b->method == RAW) {+	b->alloc = b->uncomp_size;+	if (!(b->data = malloc(b->uncomp_size))){ free(b); return NULL; }+	if (b->uncomp_size != hread(fd->fp, b->data, b->uncomp_size)) {+	    free(b->data);+	    free(b);+	    return NULL;+	}+    } else {+	b->alloc = b->comp_size;+	if (!(b->data = malloc(b->comp_size)))  { free(b); return NULL; }+	if (b->comp_size != hread(fd->fp, b->data, b->comp_size)) {+	    free(b->data);+	    free(b);+	    return NULL;+	}+    }++    if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	if (-1 == int32_decode(fd, (int32_t *)&b->crc32)) {+	    free(b);+	    return NULL;+	}++	crc = crc32(crc, b->data ? b->data : (uc *)"", b->alloc);+	if (crc != b->crc32) {+	    fprintf(stderr, "Block CRC32 failure\n");+	    free(b->data);+	    free(b);+	    return NULL;+	}+    }++    b->orig_method = b->method;+    b->idx = 0;+    b->byte = 0;+    b->bit = 7; // MSB++    return b;+}+++/*+ * Computes the size of a cram block, including the block+ * header itself.+ */+uint32_t cram_block_size(cram_block *b) {+    unsigned char dat[100], *cp = dat;;+    uint32_t sz;++    *cp++ = b->method;+    *cp++ = b->content_type;+    cp += itf8_put(cp, b->content_id);+    cp += itf8_put(cp, b->comp_size);+    cp += itf8_put(cp, b->uncomp_size);++    sz = cp-dat + 4;+    sz += b->method == RAW ? b->uncomp_size : b->comp_size;++    return sz;+}++/*+ * Writes a CRAM block.+ * Returns 0 on success+ *        -1 on failure+ */+int cram_write_block(cram_fd *fd, cram_block *b) {+    assert(b->method != RAW || (b->comp_size == b->uncomp_size));++    if (hputc(b->method,       fd->fp)  == EOF) return -1;+    if (hputc(b->content_type, fd->fp)  == EOF) return -1;+    if (itf8_encode(fd, b->content_id)  ==  -1) return -1;+    if (itf8_encode(fd, b->comp_size)   ==  -1) return -1;+    if (itf8_encode(fd, b->uncomp_size) ==  -1) return -1;++    if (b->method == RAW) {+	if (b->uncomp_size != hwrite(fd->fp, b->data, b->uncomp_size))+	    return -1;+    } else {+	if (b->comp_size != hwrite(fd->fp, b->data, b->comp_size))+	    return -1;+    }++    if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	unsigned char dat[100], *cp = dat;;+	uint32_t crc;++	*cp++ = b->method;+	*cp++ = b->content_type;+	cp += itf8_put(cp, b->content_id);+	cp += itf8_put(cp, b->comp_size);+	cp += itf8_put(cp, b->uncomp_size);+	crc = crc32(0L, dat, cp-dat);++	if (b->method == RAW) {+	    b->crc32 = crc32(crc, b->data ? b->data : (uc*)"", b->uncomp_size);+	} else {+	    b->crc32 = crc32(crc, b->data ? b->data : (uc*)"", b->comp_size);+	}++	if (-1 == int32_encode(fd, b->crc32))+	    return -1;+    }++    return 0;+}++/*+ * Frees a CRAM block, deallocating internal data too.+ */+void cram_free_block(cram_block *b) {+    if (!b)+	return;+    if (b->data)+	free(b->data);+    free(b);+}++/*+ * Uncompresses a CRAM block, if compressed.+ */+int cram_uncompress_block(cram_block *b) {+    char *uncomp;+    size_t uncomp_size = 0;++    if (b->uncomp_size == 0) {+	// blank block+	b->method = RAW;+	return 0;+    }++    switch (b->method) {+    case RAW:+	return 0;++    case GZIP:+	uncomp = zlib_mem_inflate((char *)b->data, b->comp_size, &uncomp_size);+	if (!uncomp)+	    return -1;+	if ((int)uncomp_size != b->uncomp_size) {+	    free(uncomp);+	    return -1;+	}+	free(b->data);+	b->data = (unsigned char *)uncomp;+	b->alloc = uncomp_size;+	b->method = RAW;+	break;++#ifdef HAVE_LIBBZ2+    case BZIP2: {+	unsigned int usize = b->uncomp_size;+	if (!(uncomp = malloc(usize)))+	    return -1;+	if (BZ_OK != BZ2_bzBuffToBuffDecompress(uncomp, &usize,+						(char *)b->data, b->comp_size,+						0, 0)) {+	    free(uncomp);+	    return -1;+	}+	free(b->data);+	b->data = (unsigned char *)uncomp;+	b->alloc = usize;+	b->method = RAW;+	b->uncomp_size = usize; // Just incase it differs+	break;+    }+#else+    case BZIP2:+	fprintf(stderr, "Bzip2 compression is not compiled into this "+		"version.\nPlease rebuild and try again.\n");+	return -1;+#endif++#ifdef HAVE_LIBLZMA+    case LZMA:+	uncomp = lzma_mem_inflate((char *)b->data, b->comp_size, &uncomp_size);+	if (!uncomp)+	    return -1;+	if ((int)uncomp_size != b->uncomp_size)+	    return -1;+	free(b->data);+	b->data = (unsigned char *)uncomp;+	b->alloc = uncomp_size;+	b->method = RAW;+	break;+#else+    case LZMA:+	fprintf(stderr, "Lzma compression is not compiled into this "+		"version.\nPlease rebuild and try again.\n");+	return -1;+	break;+#endif++    case RANS: {+	unsigned int usize = b->uncomp_size, usize2;+	uncomp = (char *)rans_uncompress(b->data, b->comp_size, &usize2);+	if (!uncomp || usize != usize2)+ 	    return -1;+	free(b->data);+	b->data = (unsigned char *)uncomp;+	b->alloc = usize2;+	b->method = RAW;+	b->uncomp_size = usize2; // Just incase it differs+	//fprintf(stderr, "Expanded %d to %d\n", b->comp_size, b->uncomp_size);+	break;+    }++    default:+	return -1;+    }++    return 0;+}++static char *cram_compress_by_method(char *in, size_t in_size,+				     size_t *out_size,+				     enum cram_block_method method,+				     int level, int strat) {+    switch (method) {+    case GZIP:+	return zlib_mem_deflate(in, in_size, out_size, level, strat);++    case BZIP2: {+#ifdef HAVE_LIBBZ2+	unsigned int comp_size = in_size*1.01 + 600;+	char *comp = malloc(comp_size);+	if (!comp)+	    return NULL;++	if (BZ_OK != BZ2_bzBuffToBuffCompress(comp, &comp_size,+					      in, in_size,+					      level, 0, 30)) {+	    free(comp);+	    return NULL;+	}+	*out_size = comp_size;+	return comp;+#else+	return NULL;+#endif+    }++    case LZMA:+#ifdef HAVE_LIBLZMA+	return lzma_mem_deflate(in, in_size, out_size, level);+#else+	return NULL;+#endif++    case RANS0: {+	unsigned int out_size_i;+	unsigned char *cp;+	cp = rans_compress((unsigned char *)in, in_size, &out_size_i, 0);+	*out_size = out_size_i;+	return (char *)cp;+    }++    case RANS1: {+	unsigned int out_size_i;+	unsigned char *cp;+	+	cp = rans_compress((unsigned char *)in, in_size, &out_size_i, 1);+	*out_size = out_size_i;+	return (char *)cp;+    }++    case RAW:+	break;++    default:+	return NULL;+    }++    return NULL;+}+++/*+ * Compresses a block using one of two different zlib strategies. If we only+ * want one choice set strat2 to be -1.+ *+ * The logic here is that sometimes Z_RLE does a better job than Z_FILTERED+ * or Z_DEFAULT_STRATEGY on quality data. If so, we'd rather use it as it is+ * significantly faster.+ *+ * Method and level -1 implies defaults, as specified in cram_fd.+ */+int cram_compress_block(cram_fd *fd, cram_block *b, cram_metrics *metrics,+			int method, int level) {++    char *comp = NULL;+    size_t comp_size = 0;+    int strat;++    if (b->method != RAW) {+        // Maybe already compressed if s->block[0] was compressed and+        // we have e.g. s->block[DS_BA] set to s->block[0] due to only+        // one base type present and hence using E_HUFFMAN on block 0.+        // A second explicit attempt to compress the same block then+        // occurs.+        return 0;+    }++    if (method == -1) {+	method = 1<<GZIP;+	if (fd->use_bz2)+	    method |= 1<<BZIP2;+	if (fd->use_lzma)+	    method |= 1<<LZMA;+    }++    if (level == -1)+	level = fd->level;++    //fprintf(stderr, "IN: block %d, sz %d\n", b->content_id, b->uncomp_size);++    if (method == RAW || level == 0 || b->uncomp_size == 0) {+	b->method = RAW;+	b->comp_size = b->uncomp_size;+	//fprintf(stderr, "Skip block id %d\n", b->content_id);+	return 0;+    }++    if (metrics) {+	pthread_mutex_lock(&fd->metrics_lock);+	if (metrics->trial > 0 || --metrics->next_trial <= 0) {+	    size_t sz_best = INT_MAX;+	    size_t sz_gz_rle = 0;+	    size_t sz_gz_def = 0;+	    size_t sz_rans0 = 0;+	    size_t sz_rans1 = 0;+	    size_t sz_bzip2 = 0;+	    size_t sz_lzma = 0;+	    int method_best = 0;+	    char *c_best = NULL, *c = NULL;++	    if (metrics->revised_method)+		method = metrics->revised_method;+	    else+		metrics->revised_method = method;++	    if (metrics->next_trial == 0) {+		metrics->next_trial = TRIAL_SPAN;+		metrics->trial = NTRIALS;+		metrics->sz_gz_rle /= 2;+		metrics->sz_gz_def /= 2;+		metrics->sz_rans0  /= 2;+		metrics->sz_rans1  /= 2;+		metrics->sz_bzip2  /= 2;+		metrics->sz_lzma   /= 2;+	    }++	    pthread_mutex_unlock(&fd->metrics_lock);+	    +	    if (method & (1<<GZIP_RLE)) {+		c = cram_compress_by_method((char *)b->data, b->uncomp_size,+					    &sz_gz_rle, GZIP, 1, Z_RLE);+		if (c && sz_best > sz_gz_rle) {+		    sz_best = sz_gz_rle;+		    method_best = GZIP_RLE;+		    if (c_best)+			free(c_best);+		    c_best = c;+		} else if (c) {+		    free(c);+		} else {+		    sz_gz_rle = b->uncomp_size*2+1000;+		}++		//fprintf(stderr, "Block %d; %d->%d\n", b->content_id, b->uncomp_size, sz_gz_rle);+	    }++	    if (method & (1<<GZIP)) {+		c = cram_compress_by_method((char *)b->data, b->uncomp_size,+					    &sz_gz_def, GZIP, level,+					    Z_FILTERED);+		if (c && sz_best > sz_gz_def) {+		    sz_best = sz_gz_def;+		    method_best = GZIP;+		    if (c_best)+			free(c_best);+		    c_best = c;+		} else if (c) {+		    free(c);+		} else {+		    sz_gz_def = b->uncomp_size*2+1000;+		}++		//fprintf(stderr, "Block %d; %d->%d\n", b->content_id, b->uncomp_size, sz_gz_def);+	    }++	    if (method & (1<<RANS0)) {+		c = cram_compress_by_method((char *)b->data, b->uncomp_size,+					    &sz_rans0, RANS0, 0, 0);+		if (c && sz_best > sz_rans0) {+		    sz_best = sz_rans0;+		    method_best = RANS0;+		    if (c_best)+			free(c_best);+		    c_best = c;+		} else if (c) {+		    free(c);+		} else {+		    sz_rans0 = b->uncomp_size*2+1000;+		}+	    }++	    if (method & (1<<RANS1)) {+		c = cram_compress_by_method((char *)b->data, b->uncomp_size,+					    &sz_rans1, RANS1, 0, 0);+		if (c && sz_best > sz_rans1) {+		    sz_best = sz_rans1;+		    method_best = RANS1;+		    if (c_best)+			free(c_best);+		    c_best = c;+		} else if (c) {+		    free(c);+		} else {+		    sz_rans1 = b->uncomp_size*2+1000;+		}+	    }++	    if (method & (1<<BZIP2)) {+		c = cram_compress_by_method((char *)b->data, b->uncomp_size,+					    &sz_bzip2, BZIP2, level, 0);+		if (c && sz_best > sz_bzip2) {+		    sz_best = sz_bzip2;+		    method_best = BZIP2;+		    if (c_best)+			free(c_best);+		    c_best = c;+		} else if (c) {+		    free(c);+		} else {+		    sz_bzip2 = b->uncomp_size*2+1000;+		}+	    }++	    if (method & (1<<LZMA)) {+		c = cram_compress_by_method((char *)b->data, b->uncomp_size,+					    &sz_lzma, LZMA, level, 0);+		if (c && sz_best > sz_lzma) {+		    sz_best = sz_lzma;+		    method_best = LZMA;+		    if (c_best)+			free(c_best);+		    c_best = c;+		} else if (c) {+		    free(c);+		} else {+		    sz_lzma = b->uncomp_size*2+1000;+		}+	    }++	    //fprintf(stderr, "sz_best = %d\n", sz_best);++	    free(b->data);+	    b->data = (unsigned char *)c_best;+	    //printf("method_best = %s\n", cram_block_method2str(method_best));+	    b->method = method_best == GZIP_RLE ? GZIP : method_best;+	    b->comp_size = sz_best;++	    pthread_mutex_lock(&fd->metrics_lock);+	    metrics->sz_gz_rle += sz_gz_rle;+	    metrics->sz_gz_def += sz_gz_def;+	    metrics->sz_rans0  += sz_rans0;+	    metrics->sz_rans1  += sz_rans1;+	    metrics->sz_bzip2  += sz_bzip2;+	    metrics->sz_lzma   += sz_lzma;+	    if (--metrics->trial == 0) {+		int best_method = RAW;+		int best_sz = INT_MAX;++		// Scale methods by cost+		if (fd->level <= 3) {+		    metrics->sz_rans1  *= 1.02;+		    metrics->sz_gz_def *= 1.04;+		    metrics->sz_bzip2  *= 1.08;+		    metrics->sz_lzma   *= 1.10;+		} else if (fd->level <= 6) {+		    metrics->sz_rans1  *= 1.01;+		    metrics->sz_gz_def *= 1.02;+		    metrics->sz_bzip2  *= 1.03;+		    metrics->sz_lzma   *= 1.05;+		}++		if (method & (1<<GZIP_RLE) && best_sz > metrics->sz_gz_rle)+		    best_sz = metrics->sz_gz_rle, best_method = GZIP_RLE;++		if (method & (1<<GZIP) && best_sz > metrics->sz_gz_def)+		    best_sz = metrics->sz_gz_def, best_method = GZIP;++		if (method & (1<<RANS0) && best_sz > metrics->sz_rans0)+		    best_sz = metrics->sz_rans0, best_method = RANS0;++		if (method & (1<<RANS1) && best_sz > metrics->sz_rans1)+		    best_sz = metrics->sz_rans1, best_method = RANS1;++		if (method & (1<<BZIP2) && best_sz > metrics->sz_bzip2)+		    best_sz = metrics->sz_bzip2, best_method = BZIP2;++		if (method & (1<<LZMA) && best_sz > metrics->sz_lzma)+		    best_sz = metrics->sz_lzma, best_method = LZMA;++		if (best_method == GZIP_RLE) {+		    metrics->method = GZIP;+		    metrics->strat  = Z_RLE;+		} else {+		    metrics->method = best_method;+		    metrics->strat  = Z_FILTERED;+		}++		// If we see at least MAXFAIL trials in a row for a specific+		// compression method with more than MAXDELTA aggregate+		// size then we drop this from the list of methods used+		// for this block type.+#define MAXDELTA 0.20+#define MAXFAILS 4+		if (best_method == GZIP_RLE) {+		    metrics->gz_rle_cnt = 0;+		    metrics->gz_rle_extra = 0;+		} else if (best_sz < metrics->sz_gz_rle) {+		    double r = (double)metrics->sz_gz_rle / best_sz - 1;+		    if (++metrics->gz_rle_cnt >= MAXFAILS && +			(metrics->gz_rle_extra += r) >= MAXDELTA)+			method &= ~(1<<GZIP_RLE);+		}++		if (best_method == GZIP) {+		    metrics->gz_def_cnt = 0;+		    metrics->gz_def_extra = 0;+		} else if (best_sz < metrics->sz_gz_def) {+		    double r = (double)metrics->sz_gz_def / best_sz - 1;+		    if (++metrics->gz_def_cnt >= MAXFAILS &&+			(metrics->gz_def_extra += r) >= MAXDELTA)+			method &= ~(1<<GZIP);+		}++		if (best_method == RANS0) {+		    metrics->rans0_cnt = 0;+		    metrics->rans0_extra = 0;+		} else if (best_sz < metrics->sz_rans0) {+		    double r = (double)metrics->sz_rans0 / best_sz - 1;+		    if (++metrics->rans0_cnt >= MAXFAILS &&+			(metrics->rans0_extra += r) >= MAXDELTA)+			method &= ~(1<<RANS0);+		}++		if (best_method == RANS1) {+		    metrics->rans1_cnt = 0;+		    metrics->rans1_extra = 0;+		} else if (best_sz < metrics->sz_rans1) {+		    double r = (double)metrics->sz_rans1 / best_sz - 1;+		    if (++metrics->rans1_cnt >= MAXFAILS &&+			(metrics->rans1_extra += r) >= MAXDELTA)+			method &= ~(1<<RANS1);+		}++		if (best_method == BZIP2) {+		    metrics->bzip2_cnt = 0;+		    metrics->bzip2_extra = 0;+		} else if (best_sz < metrics->sz_bzip2) {+		    double r = (double)metrics->sz_bzip2 / best_sz - 1;+		    if (++metrics->bzip2_cnt >= MAXFAILS &&+			(metrics->bzip2_extra += r) >= MAXDELTA)+			method &= ~(1<<BZIP2);+		}++		if (best_method == LZMA) {+		    metrics->lzma_cnt = 0;+		    metrics->lzma_extra = 0;+		} else if (best_sz < metrics->sz_lzma) {+		    double r = (double)metrics->sz_lzma / best_sz - 1;+		    if (++metrics->lzma_cnt >= MAXFAILS &&+			(metrics->lzma_extra += r) >= MAXDELTA)+			method &= ~(1<<LZMA);+		}++		//if (method != metrics->revised_method)+		//    fprintf(stderr, "%d: method from %x to %x\n",+		//	    b->content_id, metrics->revised_method, method);+		metrics->revised_method = method;+	    }+	    pthread_mutex_unlock(&fd->metrics_lock);+	} else {+	    strat = metrics->strat;+	    method = metrics->method;++	    pthread_mutex_unlock(&fd->metrics_lock);+	    comp = cram_compress_by_method((char *)b->data, b->uncomp_size,+					   &comp_size, method,+					   level, strat);+	    if (!comp)+		return -1;+	    free(b->data);+	    b->data = (unsigned char *)comp;+	    b->comp_size = comp_size;+	    b->method = method;+	}++    } else {+	// no cached metrics, so just do zlib?+	comp = cram_compress_by_method((char *)b->data, b->uncomp_size,+				       &comp_size, GZIP, level, Z_FILTERED);+	if (!comp) {+	    fprintf(stderr, "Compression failed!\n");+	    return -1;+	}+	free(b->data);+	b->data = (unsigned char *)comp;+	b->comp_size = comp_size;+	b->method = GZIP;+    }++    if (fd->verbose)+	fprintf(stderr, "Compressed block ID %d from %d to %d by method %s\n",+		b->content_id, b->uncomp_size, b->comp_size,+		cram_block_method2str(b->method));++    if (b->method == RANS1)+	b->method = RANS0; // Spec just has RANS (not 0/1) with auto-sensing++    return 0;+}++cram_metrics *cram_new_metrics(void) {+    cram_metrics *m = calloc(1, sizeof(*m));+    if (!m)+	return NULL;+    m->trial = NTRIALS-1;+    m->next_trial = TRIAL_SPAN;+    m->method = RAW;+    m->strat = 0;+    m->revised_method = 0;++    return m;+}++char *cram_block_method2str(enum cram_block_method m) {+    switch(m) {+    case RAW:	   return "RAW";+    case GZIP:	   return "GZIP";+    case BZIP2:	   return "BZIP2";+    case LZMA:     return "LZMA";+    case RANS0:    return "RANS0";+    case RANS1:    return "RANS1";+    case GZIP_RLE: return "GZIP_RLE";+    case ERROR:    break;+    }+    return "?";+}++char *cram_content_type2str(enum cram_content_type t) {+    switch (t) {+    case FILE_HEADER:         return "FILE_HEADER";+    case COMPRESSION_HEADER:  return "COMPRESSION_HEADER";+    case MAPPED_SLICE:        return "MAPPED_SLICE";+    case UNMAPPED_SLICE:      return "UNMAPPED_SLICE";+    case EXTERNAL:            return "EXTERNAL";+    case CORE:                return "CORE";+    case CT_ERROR:            break;+    }+    return "?";+}++/*+ * Extra error checking on fclose to really ensure data is written.+ * Care needs to be taken to handle pipes vs real files.+ *+ * Returns 0 on success+ *        -1 on failure.+ */+int paranoid_fclose(FILE *fp) {+    if (-1 == fflush(fp) && errno != EBADF) {+	fclose(fp);+	return -1;+    }++    errno = 0;+    if (-1 == fsync(fileno(fp))) {+	if (errno != EINVAL) { // eg pipe+	    fclose(fp);+	    return -1;+	}+    }+    return fclose(fp);+}++/* ----------------------------------------------------------------------+ * Reference sequence handling+ *+ * These revolve around the refs_t structure, which may potentially be+ * shared between multiple cram_fd.+ *+ * We start with refs_create() to allocate an empty refs_t and then+ * populate it with @SQ line data using refs_from_header(). This is done on+ * cram_open().  Also at start up we can call cram_load_reference() which+ * is used with "scramble -r foo.fa". This replaces the fd->refs with the+ * new one specified. In either case refs2id() is then called which+ * maps ref_entry names to @SQ ids (refs_t->ref_id[]).+ *+ * Later, possibly within a thread, we will want to know the actual ref+ * seq itself, obtained by calling cram_get_ref().  This may use the+ * UR: or M5: fields or the filename specified in the original+ * cram_load_reference() call.+ *+ * Given the potential for multi-threaded reference usage, we have+ * reference counting (sorry for the confusing double use of "ref") to+ * track the number of callers interested in any specific reference.+ */++/*+ * Frees/unmaps a reference sequence and associated file handles.+ */+static void ref_entry_free_seq(ref_entry *e) {+    if (e->mf)+	mfclose(e->mf);+    if (e->seq && !e->mf)+	free(e->seq);++    e->seq = NULL;+    e->mf = NULL;+}++void refs_free(refs_t *r) {+    RP("refs_free()\n");++    if (--r->count > 0)+	return;++    if (!r)+	return;++    if (r->pool)+	string_pool_destroy(r->pool);++    if (r->h_meta) {+	khint_t k;++	for (k = kh_begin(r->h_meta); k != kh_end(r->h_meta); k++) {+	    ref_entry *e;++	    if (!kh_exist(r->h_meta, k))+		continue;+	    if (!(e = kh_val(r->h_meta, k)))+		continue;+	    ref_entry_free_seq(e);+	    free(e);+	}++	kh_destroy(refs, r->h_meta);+    }+    +    if (r->ref_id)+	free(r->ref_id);++    if (r->fp)+	bgzf_close(r->fp);++    pthread_mutex_destroy(&r->lock);++    free(r);+}++static refs_t *refs_create(void) {+    refs_t *r = calloc(1, sizeof(*r));++    RP("refs_create()\n");++    if (!r)+	return NULL;++    if (!(r->pool = string_pool_create(8192)))+	goto err;++    r->ref_id = NULL; // see refs2id() to populate.+    r->count = 1;+    r->last = NULL;+    r->last_id = -1;++    if (!(r->h_meta = kh_init(refs)))+	goto err;++    pthread_mutex_init(&r->lock, NULL);++    return r;++ err:+    refs_free(r);+    return NULL;+}++/*+ * Opens a reference fasta file as a BGZF stream, allowing for+ * compressed files.  It automatically builds a .fai file if+ * required and if compressed a .gzi bgzf index too.+ *+ * Returns a BGZF handle on success;+ *         NULL on failure.+ */+static BGZF *bgzf_open_ref(char *fn, char *mode, int is_md5) {+    BGZF *fp;++    if (!is_md5) {+	char fai_file[PATH_MAX];++	snprintf(fai_file, PATH_MAX, "%s.fai", fn);+	if (access(fai_file, R_OK) != 0)+	    if (fai_build(fn) != 0)+		return NULL;+    }++    if (!(fp = bgzf_open(fn, mode))) {+	perror(fn);+	return NULL;+    }++    if (fp->is_compressed == 1 && bgzf_index_load(fp, fn, ".gzi") < 0) {+	fprintf(stderr, "Unable to load .gzi index '%s.gzi'\n", fn);+	bgzf_close(fp);+	return NULL;+    }++    return fp;+}++/*+ * Loads a FAI file for a reference.fasta.+ * "is_err" indicates whether failure to load is worthy of emitting an+ * error message. In some cases (eg with embedded references) we+ * speculatively load, just incase, and silently ignore errors.+ *+ * Returns the refs_t struct on success (maybe newly allocated);+ *         NULL on failure+ */+static refs_t *refs_load_fai(refs_t *r_orig, char *fn, int is_err) {+    struct stat sb;+    FILE *fp = NULL;+    char fai_fn[PATH_MAX];+    char line[8192];+    refs_t *r = r_orig;+    size_t fn_l = strlen(fn);+    int id = 0, id_alloc = 0;++    RP("refs_load_fai %s\n", fn);++    if (!r)+	if (!(r = refs_create()))+	    goto err;++    /* Open reference, for later use */+    if (stat(fn, &sb) != 0) {+	if (is_err)+	    perror(fn);+	goto err;+    }++    if (r->fp)+	if (bgzf_close(r->fp) != 0)+	    goto err;+    r->fp = NULL;++    if (!(r->fn = string_dup(r->pool, fn)))+	goto err;+	+    if (fn_l > 4 && strcmp(&fn[fn_l-4], ".fai") == 0)+	r->fn[fn_l-4] = 0;++    if (!(r->fp = bgzf_open_ref(r->fn, "r", 0)))+	goto err;++    /* Parse .fai file and load meta-data */+    sprintf(fai_fn, "%.*s.fai", PATH_MAX-5, r->fn);++    if (stat(fai_fn, &sb) != 0) {+	if (is_err)+	    perror(fai_fn);+	goto err;+    }+    if (!(fp = fopen(fai_fn, "r"))) {+	if (is_err)+	    perror(fai_fn);+	goto err;+    }+    while (fgets(line, 8192, fp) != NULL) {+	ref_entry *e = malloc(sizeof(*e));+	char *cp;+	int n;+	khint_t k;++	if (!e)+	    return NULL;++	// id+	for (cp = line; *cp && !isspace(*cp); cp++)+	    ;+	*cp++ = 0;+	e->name = string_dup(r->pool, line);+	+	// length+	while (*cp && isspace(*cp))+	    cp++;+	e->length = strtoll(cp, &cp, 10);++	// offset+	while (*cp && isspace(*cp))+	    cp++;+	e->offset = strtoll(cp, &cp, 10);++	// bases per line+	while (*cp && isspace(*cp))+	    cp++;+	e->bases_per_line = strtol(cp, &cp, 10);++	// line length+	while (*cp && isspace(*cp))+	    cp++;+	e->line_length = strtol(cp, &cp, 10);++	// filename+	e->fn = r->fn;++	e->count = 0;+	e->seq = NULL;+	e->mf = NULL;+	e->is_md5 = 0;++	k = kh_put(refs, r->h_meta, e->name, &n);+	if (-1 == n)  {+	    free(e);+	    return NULL;+	}++	if (n) {+	    kh_val(r->h_meta, k) = e;+	} else {+	    ref_entry *re = kh_val(r->h_meta, k);+	    if (re && (re->count != 0 || re->length != 0)) {+		/* Keep old */+		free(e);+	    } else {+		/* Replace old */+		if (re)+		    free(re);+		kh_val(r->h_meta, k) = e;+	    }+	}++	if (id >= id_alloc) {+	    int x;++	    id_alloc = id_alloc ?id_alloc*2 : 16;+	    r->ref_id = realloc(r->ref_id, id_alloc * sizeof(*r->ref_id));++	    for (x = id; x < id_alloc; x++)+		r->ref_id[x] = NULL;+	}+	r->ref_id[id] = e;+	r->nref = ++id;+    }++    return r;++ err:+    if (fp)+	fclose(fp);++    if (!r_orig)+	refs_free(r);+    +    return NULL;+}++/*+ * Verifies that the CRAM @SQ lines and .fai files match.+ */+static void sanitise_SQ_lines(cram_fd *fd) {+    int i;++    if (!fd->header)+	return;++    if (!fd->refs || !fd->refs->h_meta)+	return;++    for (i = 0; i < fd->header->nref; i++) {+	char *name = fd->header->ref[i].name;+	khint_t k = kh_get(refs, fd->refs->h_meta, name);+	ref_entry *r;++	// We may have @SQ lines which have no known .fai, but do not+	// in themselves pose a problem because they are unused in the file.+	if (k == kh_end(fd->refs->h_meta))+	    continue;++	if (!(r = (ref_entry *)kh_val(fd->refs->h_meta, k)))+	    continue;++	if (r->length && r->length != fd->header->ref[i].len) {+	    assert(strcmp(r->name, fd->header->ref[i].name) == 0);++	    // Should we also check MD5sums here to ensure the correct+	    // reference was given?+	    fprintf(stderr, "WARNING: Header @SQ length mismatch for "+		    "ref %s, %d vs %d\n",+		    r->name, fd->header->ref[i].len, (int)r->length);++	    // Fixing the parsed @SQ header will make MD:Z: strings work+	    // and also stop it producing N for the sequence.+	    fd->header->ref[i].len = r->length;+	}+    }+}++/*+ * Indexes references by the order they appear in a BAM file. This may not+ * necessarily be the same order they appear in the fasta reference file.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int refs2id(refs_t *r, SAM_hdr *h) {+    int i;++    if (r->ref_id)+	free(r->ref_id);+    if (r->last)+	r->last = NULL;++    r->ref_id = calloc(h->nref, sizeof(*r->ref_id));+    if (!r->ref_id)+	return -1;++    r->nref = h->nref;+    for (i = 0; i < h->nref; i++) {+	khint_t k = kh_get(refs, r->h_meta, h->ref[i].name);+	if (k != kh_end(r->h_meta)) {+	    r->ref_id[i] = kh_val(r->h_meta, k);+	} else {+	    fprintf(stderr, "Unable to find ref name '%s'\n",+		    h->ref[i].name);+	}+    }++    return 0;+}++/*+ * Generates refs_t entries based on @SQ lines in the header.+ * Returns 0 on success+ *         -1 on failure+ */+static int refs_from_header(refs_t *r, cram_fd *fd, SAM_hdr *h) {+    int i, j;++    if (!r)+	return -1;++    if (!h || h->nref == 0)+	return 0;++    //fprintf(stderr, "refs_from_header for %p mode %c\n", fd, fd->mode);++    /* Existing refs are fine, as long as they're compatible with the hdr. */+    if (!(r->ref_id = realloc(r->ref_id, (r->nref + h->nref) * sizeof(*r->ref_id))))+	return -1;++    /* Copy info from h->ref[i] over to r */+    for (i = 0, j = r->nref; i < h->nref; i++) {+	SAM_hdr_type *ty;+	SAM_hdr_tag *tag;+	khint_t k;+	int n;++	k = kh_get(refs, r->h_meta, h->ref[i].name);+	if (k != kh_end(r->h_meta))+	    // Ref already known about+	    continue;++	if (!(r->ref_id[j] = calloc(1, sizeof(ref_entry))))+	    return -1;++	if (!h->ref[i].name)+	    return -1;++	r->ref_id[j]->name = string_dup(r->pool, h->ref[i].name);+	r->ref_id[j]->length = 0; // marker for not yet loaded++	/* Initialise likely filename if known */+	if ((ty = sam_hdr_find(h, "SQ", "SN", h->ref[i].name))) {+	    if ((tag = sam_hdr_find_key(h, ty, "M5", NULL))) {+		r->ref_id[j]->fn = string_dup(r->pool, tag->str+3);+		//fprintf(stderr, "Tagging @SQ %s / %s\n", r->ref_id[h]->name, r->ref_id[h]->fn);+	    }+	}++	k = kh_put(refs, r->h_meta, r->ref_id[j]->name, &n);+	if (n <= 0) // already exists or error+	    return -1;+	kh_val(r->h_meta, k) = r->ref_id[j];++	j++;+    }+    r->nref = j;++    return 0;+}++/*+ * Attaches a header to a cram_fd.+ *+ * This should be used when creating a new cram_fd for writing where+ * we have an SAM_hdr already constructed (eg from a file we've read+ * in).+ */+int cram_set_header(cram_fd *fd, SAM_hdr *hdr) {+    if (fd->header)+	sam_hdr_free(fd->header);+    fd->header = hdr;+    return refs_from_header(fd->refs, fd, hdr);+}++/*+ * Converts a directory and a filename into an expanded path, replacing %s+ * in directory with the filename and %[0-9]+s with portions of the filename+ * Any remaining parts of filename are added to the end with /%s.+ */+void expand_cache_path(char *path, char *dir, char *fn) {+    char *cp;++    while ((cp = strchr(dir, '%'))) {+	strncpy(path, dir, cp-dir);+	path += cp-dir;++	if (*++cp == 's') {+	    strcpy(path, fn);+	    path += strlen(fn);+	    fn += strlen(fn);+	    cp++;+	} else if (*cp >= '0' && *cp <= '9') {+	    char *endp;+	    long l;++	    l = strtol(cp, &endp, 10);+	    l = MIN(l, strlen(fn));+	    if (*endp == 's') {+		strncpy(path, fn, l);+		path += l;+		fn += l;+		*path = 0;+		cp = endp+1;+	    } else {+		*path++ = '%';+		*path++ = *cp++;+	    }+	} else {+	    *path++ = '%';+	    *path++ = *cp++;+	}+	dir = cp;+    }+    strcpy(path, dir);+    path += strlen(dir);+    if (*fn && path[-1] != '/')+	*path++ = '/';+    strcpy(path, fn);+}++/*+ * Make the directory containing path and any prefix directories.+ */+void mkdir_prefix(char *path, int mode) {+    char *cp = strrchr(path, '/');+    if (!cp)+	return;++    *cp = 0;+    if (is_directory(path)) {+	*cp = '/';+	return;+    }++    if (mkdir(path, mode) == 0) {+	chmod(path, mode);+	*cp = '/';+	return;+    }++    mkdir_prefix(path, mode);+    mkdir(path, mode);+    chmod(path, mode);+    *cp = '/';+}++/*+ * Return the cache directory to use, based on the first of these+ * environment variables to be set to a non-empty value.+ */+static const char *get_cache_basedir(const char **extra) {+    char *base;++    *extra = "";++    base = getenv("XDG_CACHE_HOME");+    if (base && *base) return base;++    base = getenv("HOME");+    if (base && *base) { *extra = "/.cache"; return base; }++    base = getenv("TMPDIR");+    if (base && *base) return base;++    base = getenv("TEMP");+    if (base && *base) return base;++    return "/tmp";+}++/*+ * Return an integer representation of pthread_self().+ */+static unsigned get_int_threadid() {+    pthread_t pt = pthread_self();+    unsigned char *s = (unsigned char *) &pt;+    size_t i;+    unsigned h = 0;+    for (i = 0; i < sizeof(pthread_t); i++)+	h = (h << 5) - h + s[i];+    return h;+}++/*+ * Queries the M5 string from the header and attempts to populate the+ * reference from this using the REF_PATH environment.+ *+ * Returns 0 on sucess+ *        -1 on failure+ */+static int cram_populate_ref(cram_fd *fd, int id, ref_entry *r) {+    char *ref_path = getenv("REF_PATH");+    SAM_hdr_type *ty;+    SAM_hdr_tag *tag;+    char path[PATH_MAX], path_tmp[PATH_MAX];+    char cache[PATH_MAX], cache_root[PATH_MAX];+    char *local_cache = getenv("REF_CACHE");+    mFILE *mf;+    int local_path = 0;++    if (fd->verbose)+	fprintf(stderr, "cram_populate_ref on fd %p, id %d\n", fd, id);++    cache_root[0] = '\0';++    if (!ref_path || *ref_path == '\0') {+	/*+	 * If we have no ref path, we use the EBI server.+	 * However to avoid spamming it we require a local ref cache too.+	 */+	ref_path = "http://www.ebi.ac.uk:80/ena/cram/md5/%s";+	if (!local_cache || *local_cache == '\0') {+	    const char *extra;+	    const char *base = get_cache_basedir(&extra);+	    snprintf(cache_root, PATH_MAX, "%s%s/hts-ref", base, extra);+	    snprintf(cache,PATH_MAX, "%s%s/hts-ref/%%2s/%%2s/%%s", base, extra);+	    local_cache = cache;+	    if (fd->verbose)+		fprintf(stderr, "Populating local cache: %s\n", local_cache);+	}+    }++    if (!r->name)+	return -1;++    if (!(ty = sam_hdr_find(fd->header, "SQ", "SN", r->name)))+	return -1;++    if (!(tag = sam_hdr_find_key(fd->header, ty, "M5", NULL)))+	goto no_M5;++    if (fd->verbose)+	fprintf(stderr, "Querying ref %s\n", tag->str+3);++    /* Use cache if available */+    if (local_cache && *local_cache) {+	expand_cache_path(path, local_cache, tag->str+3);+	local_path = 1;+    }++#ifndef HAVE_MMAP+    char *path2;+    /* Search local files in REF_PATH; we can open them and return as above */+    if (!local_path && (path2 = find_path(tag->str+3, ref_path))) {+	strncpy(path, path2, PATH_MAX);+	free(path2);+	if (is_file(path)) // incase it's too long+	    local_path = 1;+    }+#endif++    /* Found via REF_CACHE or local REF_PATH file */+    if (local_path) {+	struct stat sb;+	BGZF *fp;++	if (0 == stat(path, &sb) && (fp = bgzf_open(path, "r"))) {+	    r->length = sb.st_size;+	    r->offset = r->line_length = r->bases_per_line = 0;++	    r->fn = string_dup(fd->refs->pool, path);++	    if (fd->refs->fp)+		if (bgzf_close(fd->refs->fp) != 0)+		    return -1;+	    fd->refs->fp = fp;+	    fd->refs->fn = r->fn;+	    r->is_md5 = 1;++	    // Fall back to cram_get_ref() where it'll do the actual+	    // reading of the file.+	    return 0;+	}+    }+++    /* Otherwise search full REF_PATH; slower as loads entire file */+    if ((mf = open_path_mfile(tag->str+3, ref_path, NULL))) {+	size_t sz;+	r->seq = mfsteal(mf, &sz);+	if (r->seq) {+	    r->mf = NULL;+	} else {+	    // keep mf around as we couldn't detach+	    r->seq = mf->data;+	    r->mf = mf;+	}+	r->length = sz;+	r->is_md5 = 1;+    } else {+	refs_t *refs;+	char *fn;++    no_M5:+	/* Failed to find in search path or M5 cache, see if @SQ UR: tag? */+	if (!(tag = sam_hdr_find_key(fd->header, ty, "UR", NULL)))+	    return -1;++	fn = (strncmp(tag->str+3, "file:", 5) == 0)+	    ? tag->str+8+	    : tag->str+3;++	if (fd->refs->fp) {+	    if (bgzf_close(fd->refs->fp) != 0)+		return -1;+	    fd->refs->fp = NULL;+	}+	if (!(refs = refs_load_fai(fd->refs, fn, 0)))+	    return -1;+	sanitise_SQ_lines(fd);++	fd->refs = refs;+	if (fd->refs->fp) {+	    if (bgzf_close(fd->refs->fp) != 0)+		return -1;+	    fd->refs->fp = NULL;+	}++	if (!fd->refs->fn)+	    return -1;++	if (-1 == refs2id(fd->refs, fd->header))+	    return -1;+	if (!fd->refs->ref_id || !fd->refs->ref_id[id])+	    return -1;++	// Local copy already, so fall back to cram_get_ref().+	return 0;+    }++    /* Populate the local disk cache if required */+    if (local_cache && *local_cache) {+	int pid = (int) getpid();+	unsigned thrid = get_int_threadid();+	FILE *fp;++	if (*cache_root && !is_directory(cache_root) && hts_verbose >= 1)+	    fprintf(stderr,+"Creating reference cache directory %s\n"+"This may become large; see the samtools(1) manual page REF_CACHE discussion\n",+		    cache_root);++	expand_cache_path(path, local_cache, tag->str+3);+	if (fd->verbose)+	    fprintf(stderr, "Writing cache file '%s'\n", path);+	mkdir_prefix(path, 01777);++	do {+	    // Attempt to further uniquify the temporary filename+	    unsigned t = ((unsigned) time(NULL)) ^ ((unsigned) clock());+	    thrid++; // Ensure filename changes even if time/clock haven't++	    sprintf(path_tmp, "%s.tmp_%d_%u_%u", path, pid, thrid, t);+	    fp = fopen(path_tmp, "wx");+	} while (fp == NULL && errno == EEXIST);+	if (!fp) {+	    perror(path_tmp);++	    // Not fatal - we have the data already so keep going.+	    return 0;+	}++	// Check md5sum+	hts_md5_context *md5;+	char unsigned md5_buf1[16];+	char md5_buf2[33];++	if (!(md5 = hts_md5_init())) {+	    unlink(path_tmp);+	    fclose(fp);+	    return -1;+	}+	hts_md5_update(md5, r->seq, r->length);+	hts_md5_final(md5_buf1, md5);+	hts_md5_destroy(md5);+	hts_md5_hex(md5_buf2, md5_buf1);++	if (strncmp(tag->str+3, md5_buf2, 32) != 0) {+	    fprintf(stderr, "[E::%s] mismatching md5sum for downloaded reference.\n", __func__);+	    unlink(path_tmp);+	    fclose(fp);+	    return -1;+	}++	if (r->length != fwrite(r->seq, 1, r->length, fp)) {+	    perror(path);+	}+	if (-1 == paranoid_fclose(fp)) {+	    unlink(path_tmp);+	} else {+	    if (0 == chmod(path_tmp, 0444))+		rename(path_tmp, path);+	    else+		unlink(path_tmp);+	}+    }++    return 0;+}++static void cram_ref_incr_locked(refs_t *r, int id) {+    RP("%d INC REF %d, %d %p\n", gettid(), id, (int)(id>=0?r->ref_id[id]->count+1:-999), id>=0?r->ref_id[id]->seq:(char *)1);++    if (id < 0 || !r->ref_id[id]->seq)+	return;++    if (r->last_id == id)+	r->last_id = -1;++    ++r->ref_id[id]->count;+}++void cram_ref_incr(refs_t *r, int id) {+    pthread_mutex_lock(&r->lock);+    cram_ref_incr_locked(r, id);+    pthread_mutex_unlock(&r->lock);+}++static void cram_ref_decr_locked(refs_t *r, int id) {+    RP("%d DEC REF %d, %d %p\n", gettid(), id, (int)(id>=0?r->ref_id[id]->count-1:-999), id>=0?r->ref_id[id]->seq:(char *)1);++    if (id < 0 || !r->ref_id[id]->seq) {+	assert(r->ref_id[id]->count >= 0);+	return;+    }++    if (--r->ref_id[id]->count <= 0) {+	assert(r->ref_id[id]->count == 0);+	if (r->last_id >= 0) {+	    if (r->ref_id[r->last_id]->count <= 0 &&+		r->ref_id[r->last_id]->seq) {+		RP("%d FREE REF %d (%p)\n", gettid(),+		   r->last_id, r->ref_id[r->last_id]->seq);+		ref_entry_free_seq(r->ref_id[r->last_id]);+		r->ref_id[r->last_id]->length = 0;+	    }+	}+	r->last_id = id;+    }+}++void cram_ref_decr(refs_t *r, int id) {+    pthread_mutex_lock(&r->lock);+    cram_ref_decr_locked(r, id);+    pthread_mutex_unlock(&r->lock);+}++/*+ * Used by cram_ref_load and cram_ref_get. The file handle will have+ * already been opened, so we can catch it. The ref_entry *e informs us+ * of whether this is a multi-line fasta file or a raw MD5 style file.+ * Either way we create a single contiguous sequence.+ *+ * Returns all or part of a reference sequence on success (malloced);+ *         NULL on failure.+ */+static char *load_ref_portion(BGZF *fp, ref_entry *e, int start, int end) {+    off_t offset, len;+    char *seq;++    if (end < start)+	end = start;++    /*+     * Compute locations in file. This is trivial for the MD5 files, but+     * is still necessary for the fasta variants.+     */+    offset = e->line_length+	? e->offset + (start-1)/e->bases_per_line * e->line_length ++	  (start-1) % e->bases_per_line+	: start-1;++    len = (e->line_length+	   ? e->offset + (end-1)/e->bases_per_line * e->line_length + +	     (end-1) % e->bases_per_line+	   : end-1) - offset + 1;++    if (bgzf_useek(fp, offset, SEEK_SET) < 0) {+	perror("bgzf_useek() on reference file");+	return NULL;+    }++    if (len == 0 || !(seq = malloc(len))) {+	return NULL;+    }++    if (len != bgzf_read(fp, seq, len)) {+	perror("bgzf_read() on reference file");+	free(seq);+	return NULL;+    }++    /* Strip white-space if required. */+    if (len != end-start+1) {+	int i, j;+	char *cp = seq;+	char *cp_to;++	for (i = j = 0; i < len; i++) {+	    if (cp[i] >= '!' && cp[i] <= '~')+		cp[j++] = toupper(cp[i]);+	}+	cp_to = cp+j;++	if (cp_to - seq != end-start+1) {+	    fprintf(stderr, "Malformed reference file?\n");+	    free(seq);+	    return NULL;+	}+    } else {+	int i;+	for (i = 0; i < len; i++) {+	    seq[i] = toupper(seq[i]);+	}+    }++    return seq;+}++/*+ * Load the entire reference 'id'.+ * This also increments the reference count by 1.+ *+ * Returns ref_entry on success;+ *         NULL on failure+ */+ref_entry *cram_ref_load(refs_t *r, int id, int is_md5) {+    ref_entry *e = r->ref_id[id];+    int start = 1, end = e->length;+    char *seq;++    if (e->seq) {+	return e;+    }++    assert(e->count == 0);++    if (r->last) {+#ifdef REF_DEBUG+	int idx = 0;+	for (idx = 0; idx < r->nref; idx++)+	    if (r->last == r->ref_id[idx])+		break;+	RP("%d cram_ref_load DECR %d\n", gettid(), idx);+#endif+	assert(r->last->count > 0);+	if (--r->last->count <= 0) {+	    RP("%d FREE REF %d (%p)\n", gettid(), id, r->ref_id[id]->seq);+	    if (r->last->seq)+		ref_entry_free_seq(r->last);+	}+    }++    /* Open file if it's not already the current open reference */+    if (strcmp(r->fn, e->fn) || r->fp == NULL) {+	if (r->fp)+	    if (bgzf_close(r->fp) != 0)+		return NULL;+	r->fn = e->fn;+	if (!(r->fp = bgzf_open_ref(r->fn, "r", is_md5)))+	    return NULL;+    }++    RP("%d Loading ref %d (%d..%d)\n", gettid(), id, start, end);++    if (!(seq = load_ref_portion(r->fp, e, start, end))) {+	return NULL;+    }++    RP("%d Loaded ref %d (%d..%d) = %p\n", gettid(), id, start, end, seq);++    RP("%d INC REF %d, %d\n", gettid(), id, (int)(e->count+1));+    e->seq = seq;+    e->mf = NULL;+    e->count++;++    /*+     * Also keep track of last used ref so incr/decr loops on the same+     * sequence don't cause load/free loops.+     */+    RP("%d cram_ref_load INCR %d => %d\n", gettid(), id, e->count+1);+    r->last = e;+    e->count++; ++    return e;+}++/*+ * Returns a portion of a reference sequence from start to end inclusive.+ * The returned pointer is owned by either the cram_file fd or by the+ * internal refs_t structure and should not be freed  by the caller.+ *+ * The difference is whether or not this refs_t is in use by just the one+ * cram_fd or by multiples, or whether we have multiple threads accessing+ * references. In either case fd->shared will be true and we start using+ * reference counting to track the number of users of a specific reference+ * sequence.+ *+ * Otherwise the ref seq returned is allocated as part of cram_fd itself+ * and will be freed up on the next call to cram_get_ref or cram_close.+ *+ * To return the entire reference sequence, specify start as 1 and end+ * as 0.+ *+ * To cease using a reference, call cram_ref_decr().+ *+ * Returns reference on success,+ *         NULL on failure+ */+char *cram_get_ref(cram_fd *fd, int id, int start, int end) {+    ref_entry *r;+    char *seq;+    int ostart = start;++    if (id == -1)+	return NULL;++    /* FIXME: axiomatic query of r->seq being true?+     * Or shortcut for unsorted data where we load once and never free?+     */++    //fd->shared_ref = 1; // hard code for now to simplify things++    pthread_mutex_lock(&fd->ref_lock);++    RP("%d cram_get_ref on fd %p, id %d, range %d..%d\n", gettid(), fd, id, start, end);++    /*+     * Unsorted data implies we want to fetch an entire reference at a time.+     * We just deal with this at the moment by claiming we're sharing+     * references instead, which has the same requirement.+     */+    if (fd->unsorted)+	fd->shared_ref = 1;+++    /* Sanity checking: does this ID exist? */+    if (id >= fd->refs->nref) {+	fprintf(stderr, "No reference found for id %d\n", id);+	pthread_mutex_unlock(&fd->ref_lock);+	return NULL;+    }++    if (!fd->refs || !fd->refs->ref_id[id]) {+	fprintf(stderr, "No reference found for id %d\n", id);+	pthread_mutex_unlock(&fd->ref_lock);+	return NULL;+    }++    if (!(r = fd->refs->ref_id[id])) {+	fprintf(stderr, "No reference found for id %d\n", id);+	pthread_mutex_unlock(&fd->ref_lock);+	return NULL;+    }+++    /*+     * It has an entry, but may not have been populated yet.+     * Any manually loaded .fai files have their lengths known.+     * A ref entry computed from @SQ lines (M5 or UR field) will have+     * r->length == 0 unless it's been loaded once and verified that we have+     * an on-disk filename for it.+     *+     * 19 Sep 2013: Moved the lock here as the cram_populate_ref code calls+     * open_path_mfile and libcurl, which isn't multi-thread safe unless I+     * rewrite my code to have one curl handle per thread.+     */+    pthread_mutex_lock(&fd->refs->lock);+    if (r->length == 0) {+	if (cram_populate_ref(fd, id, r) == -1) {+	    fprintf(stderr, "Failed to populate reference for id %d\n", id);+	    pthread_mutex_unlock(&fd->refs->lock);+	    pthread_mutex_unlock(&fd->ref_lock);+	    return NULL;+	}+	r = fd->refs->ref_id[id];+	if (fd->unsorted)+	    cram_ref_incr_locked(fd->refs, id);+    }+++    /*+     * We now know that we the filename containing the reference, so check+     * for limits. If it's over half the reference we'll load all of it in+     * memory as this will speed up subsequent calls.+     */+    if (end < 1)+	end = r->length;+    if (end >= r->length)+	end  = r->length; +    assert(start >= 1);++    if (end - start >= 0.5*r->length || fd->shared_ref) {+	start = 1;+	end = r->length;+    }+    +    /*+     * Maybe we have it cached already? If so use it.+     *+     * Alternatively if we don't have the sequence but we're sharing+     * references and/or are asking for the entire length of it, then+     * load the full reference into the refs structure and return+     * a pointer to that one instead.+     */+    if (fd->shared_ref || r->seq || (start == 1 && end == r->length)) {+	char *cp;++	if (id >= 0) {+	    if (r->seq) {+		cram_ref_incr_locked(fd->refs, id);+	    } else {+		ref_entry *e;+		if (!(e = cram_ref_load(fd->refs, id, r->is_md5))) {+		    pthread_mutex_unlock(&fd->refs->lock);+		    pthread_mutex_unlock(&fd->ref_lock);+		    return NULL;+		}++		/* unsorted data implies cache ref indefinitely, to avoid+		 * continually loading and unloading.+		 */+		if (fd->unsorted)+		    cram_ref_incr_locked(fd->refs, id);+	    }	    ++	    fd->ref = NULL; /* We never access it directly */+	    fd->ref_start = 1;+	    fd->ref_end   = r->length;+	    fd->ref_id    = id;++	    cp = fd->refs->ref_id[id]->seq + ostart-1;+	} else {+	    fd->ref = NULL;+	    cp = NULL;+	}++	RP("%d cram_get_ref returning for id %d, count %d\n", gettid(), id, (int)r->count);++	pthread_mutex_unlock(&fd->refs->lock);+	pthread_mutex_unlock(&fd->ref_lock);+	return cp;+    }++    /*+     * Otherwise we're not sharing, we don't have a copy of it already and+     * we're only asking for a small portion of it.+     *+     * In this case load up just that segment ourselves, freeing any old+     * small segments in the process.+     */++    /* Unmapped ref ID */+    if (id < 0) {+	if (fd->ref_free) {+	    free(fd->ref_free);+	    fd->ref_free = NULL;+	}+	fd->ref = NULL;+	fd->ref_id = id;+	pthread_mutex_unlock(&fd->refs->lock);+	pthread_mutex_unlock(&fd->ref_lock);+	return NULL;+    }++    /* Open file if it's not already the current open reference */+    if (strcmp(fd->refs->fn, r->fn) || fd->refs->fp == NULL) {+	if (fd->refs->fp)+	    if (bgzf_close(fd->refs->fp) != 0)+		return NULL;+	fd->refs->fn = r->fn;+	if (!(fd->refs->fp = bgzf_open_ref(fd->refs->fn, "r", r->is_md5))) {+	    pthread_mutex_unlock(&fd->refs->lock);+	    pthread_mutex_unlock(&fd->ref_lock);+	    return NULL;+	}+    }++    if (!(fd->ref = load_ref_portion(fd->refs->fp, r, start, end))) {+	pthread_mutex_unlock(&fd->refs->lock);+	pthread_mutex_unlock(&fd->ref_lock);+	return NULL;+    }++    if (fd->ref_free)+	free(fd->ref_free);++    fd->ref_id    = id;+    fd->ref_start = start;+    fd->ref_end   = end;+    fd->ref_free = fd->ref;+    seq = fd->ref;++    pthread_mutex_unlock(&fd->refs->lock);+    pthread_mutex_unlock(&fd->ref_lock);++    return seq + ostart - start;+}++/*+ * If fd has been opened for reading, it may be permitted to specify 'fn'+ * as NULL and let the code auto-detect the reference by parsing the+ * SAM header @SQ lines.+ */+int cram_load_reference(cram_fd *fd, char *fn) {+    int ret = 0;++    if (fn) {+	fd->refs = refs_load_fai(fd->refs, fn,+				 !(fd->embed_ref && fd->mode == 'r'));+	fn = fd->refs ? fd->refs->fn : NULL;+	if (!fn)+	    ret = -1;+	sanitise_SQ_lines(fd);+    }+    fd->ref_fn = fn;++    if ((!fd->refs || (fd->refs->nref == 0 && !fn)) && fd->header) {+	if (fd->refs)+	    refs_free(fd->refs);+	if (!(fd->refs = refs_create()))+	    return -1;+	if (-1 == refs_from_header(fd->refs, fd, fd->header))+	    return -1;+    }++    if (fd->header)+	if (-1 == refs2id(fd->refs, fd->header))+	    return -1;++    return ret;+}++/* ----------------------------------------------------------------------+ * Containers+ */++/*+ * Creates a new container, specifying the maximum number of slices+ * and records permitted.+ *+ * Returns cram_container ptr on success+ *         NULL on failure+ */+cram_container *cram_new_container(int nrec, int nslice) {+    cram_container *c = calloc(1, sizeof(*c));+    enum cram_DS_ID id;++    if (!c)+	return NULL;++    c->curr_ref = -2;++    c->max_c_rec = nrec * nslice;+    c->curr_c_rec = 0;++    c->max_rec = nrec;+    c->record_counter = 0;+    c->num_bases = 0;++    c->max_slice = nslice;+    c->curr_slice = 0;++    c->pos_sorted = 1;+    c->max_apos   = 0;+    c->multi_seq  = 0;++    c->bams = NULL;++    if (!(c->slices = (cram_slice **)calloc(nslice, sizeof(cram_slice *))))+	goto err;+    c->slice = NULL;++    if (!(c->comp_hdr = cram_new_compression_header()))+	goto err;+    c->comp_hdr_block = NULL;++    for (id = DS_RN; id < DS_TN; id++)+	if (!(c->stats[id] = cram_stats_create())) goto err;+    +    //c->aux_B_stats = cram_stats_create();++    if (!(c->tags_used = kh_init(s_i2i)))+	goto err;+    c->refs_used = 0;++    return c;++ err:+    if (c) {+	if (c->slices)+	    free(c->slices);+	free(c);+    }+    return NULL;+}++void cram_free_container(cram_container *c) {+    enum cram_DS_ID id;+    int i;++    if (!c)+	return;++    if (c->refs_used)+	free(c->refs_used);++    if (c->landmark)+	free(c->landmark);++    if (c->comp_hdr)+	cram_free_compression_header(c->comp_hdr);++    if (c->comp_hdr_block)+	cram_free_block(c->comp_hdr_block);++    if (c->slices) {+	for (i = 0; i < c->max_slice; i++)+	    if (c->slices[i])+		cram_free_slice(c->slices[i]);+	free(c->slices);+    }++    for (id = DS_RN; id < DS_TN; id++)+	if (c->stats[id]) cram_stats_free(c->stats[id]);++    //if (c->aux_B_stats) cram_stats_free(c->aux_B_stats);+    +    if (c->tags_used) kh_destroy(s_i2i, c->tags_used);++    free(c);+}++/*+ * Reads a container header.+ *+ * Returns cram_container on success+ *         NULL on failure or no container left (fd->err == 0).+ */+cram_container *cram_read_container(cram_fd *fd) {+    cram_container c2, *c;+    int i, s;+    size_t rd = 0;+    uint32_t crc = 0;+    +    fd->err = 0;+    fd->eof = 0;++    memset(&c2, 0, sizeof(c2));+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	if ((s = itf8_decode_crc(fd, &c2.length, &crc)) == -1) {+	    fd->eof = fd->empty_container ? 1 : 2;+	    return NULL;+	} else {+	    rd+=s;+	}+    } else {+	uint32_t len;+	if ((s = int32_decode(fd, &c2.length)) == -1) {+	    if (CRAM_MAJOR_VERS(fd->version) == 2 &&+		CRAM_MINOR_VERS(fd->version) == 0)+		fd->eof = 1; // EOF blocks arrived in v2.1+	    else+		fd->eof = fd->empty_container ? 1 : 2;+	    return NULL;+	} else {+	    rd+=s;+	}+	len = le_int4(c2.length);+	crc = crc32(0L, (unsigned char *)&len, 4);+    }+    if ((s = itf8_decode_crc(fd, &c2.ref_seq_id, &crc))   == -1) return NULL; else rd+=s;+    if ((s = itf8_decode_crc(fd, &c2.ref_seq_start, &crc))== -1) return NULL; else rd+=s;+    if ((s = itf8_decode_crc(fd, &c2.ref_seq_span, &crc)) == -1) return NULL; else rd+=s;+    if ((s = itf8_decode_crc(fd, &c2.num_records, &crc))  == -1) return NULL; else rd+=s;++    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	c2.record_counter = 0;+	c2.num_bases = 0;+    } else {+	if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	    if ((s = ltf8_decode_crc(fd, &c2.record_counter, &crc)) == -1)+		return NULL;+	    else+		rd += s;+	} else {+	    int32_t i32;+	    if ((s = itf8_decode_crc(fd, &i32, &crc)) == -1)+		return NULL;+	    else+		rd += s;+	    c2.record_counter = i32;+	}++	if ((s = ltf8_decode_crc(fd, &c2.num_bases, &crc))== -1)+	    return NULL;+	else+	    rd += s;+    }+    if ((s = itf8_decode_crc(fd, &c2.num_blocks, &crc))   == -1) return NULL; else rd+=s;+    if ((s = itf8_decode_crc(fd, &c2.num_landmarks, &crc))== -1) return NULL; else rd+=s;++    if (!(c = calloc(1, sizeof(*c))))+	return NULL;++    *c = c2;++    if (!(c->landmark = malloc(c->num_landmarks * sizeof(int32_t))) &&+	c->num_landmarks) {+	fd->err = errno;+	cram_free_container(c);+	return NULL;+    }  +    for (i = 0; i < c->num_landmarks; i++) {+	if ((s = itf8_decode_crc(fd, &c->landmark[i], &crc)) == -1) {+	    cram_free_container(c);+	    return NULL;+	} else {+	    rd += s;+	}+    }++    if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	if (-1 == int32_decode(fd, (int32_t *)&c->crc32))+	    return NULL;+	else+	    rd+=4;++	if (crc != c->crc32) {+	    fprintf(stderr, "Container header CRC32 failure\n");+	    cram_free_container(c);+	    return NULL;+	}+    }++    c->offset = rd;+    c->slices = NULL;+    c->curr_slice = 0;+    c->max_slice = c->num_landmarks;+    c->slice_rec = 0;+    c->curr_rec = 0;+    c->max_rec = 0;++    if (c->ref_seq_id == -2) {+	c->multi_seq = 1;+	fd->multi_seq = 1;+    }++    fd->empty_container =+	(c->num_records == 0 &&+	 c->ref_seq_id == -1 &&+	 c->ref_seq_start == 0x454f46 /* EOF */) ? 1 : 0;++    return c;+}+++/* MAXIMUM storage size needed for the container. */+int cram_container_size(cram_container *c) {+    return 55 + 5*c->num_landmarks;+}+++/*+ * Stores the container structure in dat and returns *size as the+ * number of bytes written to dat[].  The input size of dat is also+ * held in *size and should be initialised to cram_container_size(c).+ *+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_store_container(cram_fd *fd, cram_container *c, char *dat, int *size)+{+    char *cp = dat;+    int i;++    // Check the input buffer is large enough according to our stated+    // requirements. (NOTE: it may actually take less.)+    if (cram_container_size(c) > *size)+        return -1;++    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	cp += itf8_put(cp, c->length);+    } else {+	*(int32_t *)cp = le_int4(c->length);+	cp += 4;+    }+    if (c->multi_seq) {+	cp += itf8_put(cp, -2);+	cp += itf8_put(cp, 0);+	cp += itf8_put(cp, 0);+    } else {+	cp += itf8_put(cp, c->ref_seq_id);+	cp += itf8_put(cp, c->ref_seq_start);+	cp += itf8_put(cp, c->ref_seq_span);+    }+    cp += itf8_put(cp, c->num_records);+    if (CRAM_MAJOR_VERS(fd->version) == 2) {+	cp += itf8_put(cp, c->record_counter);+	cp += ltf8_put(cp, c->num_bases);+    } else if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	cp += ltf8_put(cp, c->record_counter);+	cp += ltf8_put(cp, c->num_bases);+    }++    cp += itf8_put(cp, c->num_blocks);+    cp += itf8_put(cp, c->num_landmarks);+    for (i = 0; i < c->num_landmarks; i++)+	cp += itf8_put(cp, c->landmark[i]);++    if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	c->crc32 = crc32(0L, (uc *)dat, cp-dat);+	cp[0] =  c->crc32        & 0xff;+	cp[1] = (c->crc32 >>  8) & 0xff;+	cp[2] = (c->crc32 >> 16) & 0xff;+	cp[3] = (c->crc32 >> 24) & 0xff;+	cp += 4;+    }++    *size = cp-dat; // actual used size++    return 0;+}+++/*+ * Writes a container structure.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_write_container(cram_fd *fd, cram_container *c) {+    char buf_a[1024], *buf = buf_a, *cp;+    int i;++    if (55 + c->num_landmarks * 5 >= 1024)+	buf = malloc(55 + c->num_landmarks * 5);+    cp = buf;++    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	cp += itf8_put(cp, c->length);+    } else {+	*(int32_t *)cp = le_int4(c->length);+	cp += 4;+    }+    if (c->multi_seq) {+	cp += itf8_put(cp, -2);+	cp += itf8_put(cp, 0);+	cp += itf8_put(cp, 0);+    } else {+	cp += itf8_put(cp, c->ref_seq_id);+	cp += itf8_put(cp, c->ref_seq_start);+	cp += itf8_put(cp, c->ref_seq_span);+    }+    cp += itf8_put(cp, c->num_records);+    if (CRAM_MAJOR_VERS(fd->version) == 2) {+	cp += itf8_put(cp, c->record_counter);+	cp += ltf8_put(cp, c->num_bases);+    } else if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	cp += ltf8_put(cp, c->record_counter);+	cp += ltf8_put(cp, c->num_bases);+    }++    cp += itf8_put(cp, c->num_blocks);+    cp += itf8_put(cp, c->num_landmarks);+    for (i = 0; i < c->num_landmarks; i++)+	cp += itf8_put(cp, c->landmark[i]);++    if (CRAM_MAJOR_VERS(fd->version) >= 3) {+	c->crc32 = crc32(0L, (uc *)buf, cp-buf);+	cp[0] =  c->crc32        & 0xff;+	cp[1] = (c->crc32 >>  8) & 0xff;+	cp[2] = (c->crc32 >> 16) & 0xff;+	cp[3] = (c->crc32 >> 24) & 0xff;+	cp += 4;+    }++    if (cp-buf != hwrite(fd->fp, buf, cp-buf)) {+	if (buf != buf_a)+	    free(buf);+	return -1;+    }++    if (buf != buf_a)+	free(buf);++    return 0;+}++// common component shared by cram_flush_container{,_mt}+static int cram_flush_container2(cram_fd *fd, cram_container *c) {+    int i, j;++    if (c->curr_slice > 0 && !c->slices)+	return -1;++    //fprintf(stderr, "Writing container %d, sum %u\n", c->record_counter, sum);++    /* Write the container struct itself */+    if (0 != cram_write_container(fd, c))+	return -1;++    /* And the compression header */+    if (0 != cram_write_block(fd, c->comp_hdr_block))+	return -1;++    /* Followed by the slice blocks */+    for (i = 0; i < c->curr_slice; i++) {+	cram_slice *s = c->slices[i];++	if (0 != cram_write_block(fd, s->hdr_block))+	    return -1;++	for (j = 0; j < s->hdr->num_blocks; j++) {+	    if (0 != cram_write_block(fd, s->block[j]))+		return -1;+	}+    }++    return hflush(fd->fp) == 0 ? 0 : -1;+}++/*+ * Flushes a completely or partially full container to disk, writing+ * container structure, header and blocks. This also calls the encoder+ * functions.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_flush_container(cram_fd *fd, cram_container *c) {+    /* Encode the container blocks and generate compression header */+    if (0 != cram_encode_container(fd, c))+	return -1;++    return cram_flush_container2(fd, c);+}++typedef struct {+    cram_fd *fd;+    cram_container *c;+} cram_job;++void *cram_flush_thread(void *arg) {+    cram_job *j = (cram_job *)arg;++    /* Encode the container blocks and generate compression header */+    if (0 != cram_encode_container(j->fd, j->c)) {+	fprintf(stderr, "cram_encode_container failed\n");+	return NULL;+    }++    return arg;+}++static int cram_flush_result(cram_fd *fd) {+    int i, ret = 0;+    t_pool_result *r;++    while ((r = t_pool_next_result(fd->rqueue))) {+	cram_job *j = (cram_job *)r->data;+	cram_container *c;++	if (!j) {+	    t_pool_delete_result(r, 0);+	    return -1;+	}++	fd = j->fd;+	c = j->c;++	if (0 != cram_flush_container2(fd, c))+	    return -1;++	/* Free the container */+	for (i = 0; i < c->max_slice; i++) {+	    cram_free_slice(c->slices[i]);+	    c->slices[i] = NULL;+	}++	c->slice = NULL;+	c->curr_slice = 0;++	cram_free_container(c);++	ret |= hflush(fd->fp) == 0 ? 0 : -1;++	t_pool_delete_result(r, 1);+    }++    return ret;+}++int cram_flush_container_mt(cram_fd *fd, cram_container *c) {+    cram_job *j;++    if (!fd->pool)+	return cram_flush_container(fd, c);++    if (!(j = malloc(sizeof(*j))))+	return -1;+    j->fd = fd;+    j->c = c;+    +    t_pool_dispatch(fd->pool, fd->rqueue, cram_flush_thread, j);++    return cram_flush_result(fd);+}++/* ----------------------------------------------------------------------+ * Compression headers; the first part of the container+ */++/*+ * Creates a new blank container compression header+ *+ * Returns header ptr on success+ *         NULL on failure+ */+cram_block_compression_hdr *cram_new_compression_header(void) {+    cram_block_compression_hdr *hdr = calloc(1, sizeof(*hdr));+    if (!hdr)+	return NULL;++    if (!(hdr->TD_blk = cram_new_block(CORE, 0))) {+	free(hdr);+	return NULL;+    }++    if (!(hdr->TD_hash = kh_init(m_s2i))) {+	cram_free_block(hdr->TD_blk);+	free(hdr);+	return NULL;+    }++    if (!(hdr->TD_keys = string_pool_create(8192))) {+	kh_destroy(m_s2i, hdr->TD_hash);+	cram_free_block(hdr->TD_blk);+	free(hdr);+	return NULL;+    }++    return hdr;+}++void cram_free_compression_header(cram_block_compression_hdr *hdr) {+    int i;++    if (hdr->landmark)+	free(hdr->landmark);++    if (hdr->preservation_map)+	kh_destroy(map, hdr->preservation_map);++    for (i = 0; i < CRAM_MAP_HASH; i++) {+	cram_map *m, *m2;+	for (m = hdr->rec_encoding_map[i]; m; m = m2) {+	    m2 = m->next;+	    if (m->codec)+		m->codec->free(m->codec);+	    free(m);+	}+    }++    for (i = 0; i < CRAM_MAP_HASH; i++) {+	cram_map *m, *m2;+	for (m = hdr->tag_encoding_map[i]; m; m = m2) {+	    m2 = m->next;+	    if (m->codec)+		m->codec->free(m->codec);+	    free(m);+	}+    }++    for (i = 0; i < DS_END; i++) {+	if (hdr->codecs[i])+	    hdr->codecs[i]->free(hdr->codecs[i]);+    }++    if (hdr->TL)+	free(hdr->TL);+    if (hdr->TD_blk)+	cram_free_block(hdr->TD_blk);+    if (hdr->TD_hash)+	kh_destroy(m_s2i, hdr->TD_hash);+    if (hdr->TD_keys)+	string_pool_destroy(hdr->TD_keys);++    free(hdr);+}+++/* ----------------------------------------------------------------------+ * Slices and slice headers+ */++void cram_free_slice_header(cram_block_slice_hdr *hdr) {+    if (!hdr)+	return;++    if (hdr->block_content_ids)+	free(hdr->block_content_ids);++    free(hdr);++    return;+}++void cram_free_slice(cram_slice *s) {+    if (!s)+	return;++    if (s->hdr_block)+	cram_free_block(s->hdr_block);++    if (s->block) {+	int i;++	if (s->hdr) {+	    for (i = 0; i < s->hdr->num_blocks; i++) {+		cram_free_block(s->block[i]);+	    }+	}+	free(s->block);+    }++    if (s->block_by_id)+	free(s->block_by_id);++    if (s->hdr)+	cram_free_slice_header(s->hdr);++    if (s->seqs_blk)+	cram_free_block(s->seqs_blk);++    if (s->qual_blk)+	cram_free_block(s->qual_blk);++    if (s->name_blk)+	cram_free_block(s->name_blk);++    if (s->aux_blk)+	cram_free_block(s->aux_blk);++    if (s->aux_OQ_blk)+	cram_free_block(s->aux_OQ_blk);++    if (s->aux_BQ_blk)+	cram_free_block(s->aux_BQ_blk);++    if (s->aux_FZ_blk)+	cram_free_block(s->aux_FZ_blk);++    if (s->aux_oq_blk)+	cram_free_block(s->aux_oq_blk);++    if (s->aux_os_blk)+	cram_free_block(s->aux_os_blk);++    if (s->aux_oz_blk)+	cram_free_block(s->aux_oz_blk);++    if (s->base_blk)+	cram_free_block(s->base_blk);++    if (s->soft_blk)+	cram_free_block(s->soft_blk);++    if (s->cigar)+	free(s->cigar);++    if (s->crecs)+	free(s->crecs);++    if (s->features)+	free(s->features);++    if (s->TN)+	free(s->TN);++    if (s->pair_keys)+	string_pool_destroy(s->pair_keys);++    if (s->pair[0])+	kh_destroy(m_s2i, s->pair[0]);+    if (s->pair[1])+	kh_destroy(m_s2i, s->pair[1]);++    free(s);+}++/*+ * Creates a new empty slice in memory, for subsequent writing to+ * disk.+ *+ * Returns cram_slice ptr on success+ *         NULL on failure+ */+cram_slice *cram_new_slice(enum cram_content_type type, int nrecs) {+    cram_slice *s = calloc(1, sizeof(*s));+    if (!s)+	return NULL;++    if (!(s->hdr = (cram_block_slice_hdr *)calloc(1, sizeof(*s->hdr))))+	goto err;+    s->hdr->content_type = type;++    s->hdr_block = NULL;+    s->block = NULL;+    s->block_by_id = NULL;+    s->last_apos = 0;+    if (!(s->crecs = malloc(nrecs * sizeof(cram_record))))  goto err;+    s->cigar = NULL;+    s->cigar_alloc = 0;+    s->ncigar = 0;++    if (!(s->seqs_blk = cram_new_block(EXTERNAL, 0)))       goto err;+    if (!(s->qual_blk = cram_new_block(EXTERNAL, DS_QS)))   goto err;+    if (!(s->name_blk = cram_new_block(EXTERNAL, DS_RN)))   goto err;+    if (!(s->aux_blk  = cram_new_block(EXTERNAL, DS_aux)))  goto err;+    if (!(s->base_blk = cram_new_block(EXTERNAL, DS_IN)))   goto err;+    if (!(s->soft_blk = cram_new_block(EXTERNAL, DS_SC)))   goto err;++    s->features = NULL;+    s->nfeatures = s->afeatures = 0;++#ifndef TN_external+    s->TN = NULL;+    s->nTN = s->aTN = 0;+#endif++    // Volatile keys as we do realloc in dstring+    if (!(s->pair_keys = string_pool_create(8192))) goto err;+    if (!(s->pair[0] = kh_init(m_s2i)))             goto err;+    if (!(s->pair[1] = kh_init(m_s2i)))             goto err;+    +#ifdef BA_external+    s->BA_len = 0;+#endif++    return s;++ err:+    if (s)+	cram_free_slice(s);++    return NULL;+}++/*+ * Loads an entire slice.+ * FIXME: In 1.0 the native unit of slices within CRAM is broken+ * as slices contain references to objects in other slices.+ * To work around this while keeping the slice oriented outer loop+ * we read all slices and stitch them together into a fake large+ * slice instead.+ *+ * Returns cram_slice ptr on success+ *         NULL on failure+ */+cram_slice *cram_read_slice(cram_fd *fd) {+    cram_block *b = cram_read_block(fd);+    cram_slice *s = calloc(1, sizeof(*s));+    int i, n, max_id, min_id;++    if (!b || !s)+	goto err;++    s->hdr_block = b;+    switch (b->content_type) {+    case MAPPED_SLICE:+    case UNMAPPED_SLICE:+	if (!(s->hdr = cram_decode_slice_header(fd, b)))+	    goto err;+	break;++    default:+	fprintf(stderr, "Unexpected block of type %s\n",+		cram_content_type2str(b->content_type));+	goto err;+    }++    if (s->hdr->num_blocks < 1) {+        fprintf(stderr, "Slice does not include any data blocks.\n");+	goto err;+    }++    s->block = calloc(n = s->hdr->num_blocks, sizeof(*s->block));+    if (!s->block)+	goto err;++    for (max_id = i = 0, min_id = INT_MAX; i < n; i++) {+	if (!(s->block[i] = cram_read_block(fd)))+	    goto err;++	if (s->block[i]->content_type == EXTERNAL) {+	    if (max_id < s->block[i]->content_id)+		max_id = s->block[i]->content_id;+	    if (min_id > s->block[i]->content_id)+		min_id = s->block[i]->content_id;+	}+    }+    if (min_id >= 0 && max_id < 1024) {+	if (!(s->block_by_id = calloc(1024, sizeof(s->block[0]))))+	    goto err;++	for (i = 0; i < n; i++) {+	    if (s->block[i]->content_type != EXTERNAL)+		continue;+	    s->block_by_id[s->block[i]->content_id] = s->block[i];+	}+    }++    /* Initialise encoding/decoding tables */+    s->cigar = NULL;+    s->cigar_alloc = 0;+    s->ncigar = 0;++    if (!(s->seqs_blk = cram_new_block(EXTERNAL, 0)))      goto err;+    if (!(s->qual_blk = cram_new_block(EXTERNAL, DS_QS)))  goto err;+    if (!(s->name_blk = cram_new_block(EXTERNAL, DS_RN)))  goto err;+    if (!(s->aux_blk  = cram_new_block(EXTERNAL, DS_aux))) goto err;+    if (!(s->base_blk = cram_new_block(EXTERNAL, DS_IN)))  goto err;+    if (!(s->soft_blk = cram_new_block(EXTERNAL, DS_SC)))  goto err;++    s->crecs = NULL;++    s->last_apos = s->hdr->ref_seq_start;+    +    return s;++ err:+    if (b)+	cram_free_block(b);+    if (s) {+	s->hdr_block = NULL;+	cram_free_slice(s);+    }+    return NULL;+}+++/* ----------------------------------------------------------------------+ * CRAM file definition (header)+ */++/*+ * Reads a CRAM file definition structure.+ * Returns file_def ptr on success+ *         NULL on failure+ */+cram_file_def *cram_read_file_def(cram_fd *fd) {+    cram_file_def *def = malloc(sizeof(*def));+    if (!def)+	return NULL;++    if (26 != hread(fd->fp, &def->magic[0], 26)) {+	free(def);+	return NULL;+    }++    if (memcmp(def->magic, "CRAM", 4) != 0) {+	free(def);+	return NULL;+    }++    if (def->major_version > 3) {+	fprintf(stderr, "CRAM version number mismatch\n"+		"Expected 1.x, 2.x or 3.x, got %d.%d\n",+		def->major_version, def->minor_version);+	free(def);+	return NULL;+    }++    fd->first_container += 26;+    fd->last_slice = 0;++    return def;+}++/*+ * Writes a cram_file_def structure to cram_fd.+ * Returns 0 on success+ *        -1 on failure+ */+int cram_write_file_def(cram_fd *fd, cram_file_def *def) {+    return (hwrite(fd->fp, &def->magic[0], 26) == 26) ? 0 : -1;+}++void cram_free_file_def(cram_file_def *def) {+    if (def) free(def);+}++/* ----------------------------------------------------------------------+ * SAM header I/O+ */+++/*+ * Reads the SAM header from the first CRAM data block.+ * Also performs minimal parsing to extract read-group+ * and sample information.++ * Returns SAM hdr ptr on success+ *         NULL on failure+ */+SAM_hdr *cram_read_SAM_hdr(cram_fd *fd) {+    int32_t header_len;+    char *header;+    SAM_hdr *hdr;++    /* 1.1 onwards stores the header in the first block of a container */+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	/* Length */+	if (-1 == int32_decode(fd, &header_len))+	    return NULL;++	/* Alloc and read */+	if (header_len < 0 || NULL == (header = malloc((size_t) header_len+1)))+	    return NULL;++	if (header_len != hread(fd->fp, header, header_len))+	    return NULL;+	header[header_len] = '\0';++	fd->first_container += 4 + header_len;+    } else {+	cram_container *c = cram_read_container(fd);+	cram_block *b;+	int i, len;++	if (!c)+	    return NULL;++	if (c->num_blocks < 1) {+	    cram_free_container(c);+	    return NULL;+	}++	if (!(b = cram_read_block(fd))) {+	    cram_free_container(c);+	    return NULL;+	}+	if (cram_uncompress_block(b) != 0) {+	    cram_free_container(c);+	    return NULL;+	}++	len = b->comp_size + 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) ++	    itf8_size(b->content_id) + +	    itf8_size(b->uncomp_size) + +	    itf8_size(b->comp_size);++	/* Extract header from 1st block */+	if (-1 == int32_get_blk(b, &header_len) ||+            header_len < 0 || /* Spec. says signed...  why? */+	    b->uncomp_size - 4 < header_len) {+	    cram_free_container(c);+	    cram_free_block(b);+	    return NULL;+	}+	if (NULL == (header = malloc((size_t) header_len+1))) {+	    cram_free_container(c);+	    cram_free_block(b);+	    return NULL;+	}+	memcpy(header, BLOCK_END(b), header_len);+	header[header_len]='\0';+	cram_free_block(b);++	/* Consume any remaining blocks */+	for (i = 1; i < c->num_blocks; i++) {+	    if (!(b = cram_read_block(fd))) {+		cram_free_container(c);+		return NULL;+	    }+	    len += b->comp_size + 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) ++		itf8_size(b->content_id) + +		itf8_size(b->uncomp_size) + +		itf8_size(b->comp_size);+	    cram_free_block(b);+	}++	if (c->length > 0 && len > 0 && c->length > len) {+	    // Consume padding+	    char *pads = malloc(c->length - len);+	    if (!pads) {+		cram_free_container(c);+		return NULL;+	    }++	    if (c->length - len != hread(fd->fp, pads, c->length - len)) {+		cram_free_container(c);+		return NULL;+	    }+	    free(pads);+	}++	cram_free_container(c);+    }++    /* Parse */+    hdr = sam_hdr_parse_(header, header_len);+    free(header);++    return hdr;+}++/*+ * Converts 'in' to a full pathname to store in out.+ * Out must be at least PATH_MAX bytes long.+ */+static void full_path(char *out, char *in) {+    if (*in == '/') {+	strncpy(out, in, PATH_MAX);+	out[PATH_MAX-1] = 0;+    } else {+	int len;++	// unable to get dir or out+in is too long+	if (!getcwd(out, PATH_MAX) ||+	    (len = strlen(out))+1+strlen(in) >= PATH_MAX) {+	    strncpy(out, in, PATH_MAX);+	    out[PATH_MAX-1] = 0;+	    return;+	}++	sprintf(out+len, "/%.*s", PATH_MAX - len, in);++	// FIXME: cope with `pwd`/../../../foo.fa ?+    }+}++/*+ * Writes a CRAM SAM header.+ * Returns 0 on success+ *        -1 on failure+ */+int cram_write_SAM_hdr(cram_fd *fd, SAM_hdr *hdr) {+    int header_len;+    int blank_block = (CRAM_MAJOR_VERS(fd->version) >= 3);++    /* Write CRAM MAGIC if not yet written. */+    if (fd->file_def->major_version == 0) {+	fd->file_def->major_version = CRAM_MAJOR_VERS(fd->version);+	fd->file_def->minor_version = CRAM_MINOR_VERS(fd->version);+	if (0 != cram_write_file_def(fd, fd->file_def))+	    return -1;+    }++    /* 1.0 requires an UNKNOWN read-group */+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	if (!sam_hdr_find_rg(hdr, "UNKNOWN"))+	    if (sam_hdr_add(hdr, "RG",+			    "ID", "UNKNOWN", "SM", "UNKNOWN", NULL))+		return -1;+    }++    /* Fix M5 strings */+    if (fd->refs && !fd->no_ref) {+	int i;+	for (i = 0; i < hdr->nref; i++) {+	    SAM_hdr_type *ty;+	    char *ref;++	    if (!(ty = sam_hdr_find(hdr, "SQ", "SN", hdr->ref[i].name)))+		return -1;++	    if (!sam_hdr_find_key(hdr, ty, "M5", NULL)) {+		char unsigned buf[16];+		char buf2[33];+		int rlen;+		hts_md5_context *md5;++		if (!fd->refs ||+		    !fd->refs->ref_id ||+		    !fd->refs->ref_id[i]) {+		    return -1;+		}+		rlen = fd->refs->ref_id[i]->length;+		if (!(md5 = hts_md5_init()))+		    return -1;+		ref = cram_get_ref(fd, i, 1, rlen);+		if (NULL == ref) return -1;+		rlen = fd->refs->ref_id[i]->length; /* In case it just loaded */+		hts_md5_update(md5, ref, rlen);+		hts_md5_final(buf, md5);+		hts_md5_destroy(md5);+		cram_ref_decr(fd->refs, i);++		hts_md5_hex(buf2, buf);+		if (sam_hdr_update(hdr, ty, "M5", buf2, NULL))+		    return -1;+	    }++	    if (fd->ref_fn) {+		char ref_fn[PATH_MAX];+		full_path(ref_fn, fd->ref_fn);+		if (sam_hdr_update(hdr, ty, "UR", ref_fn, NULL))+		    return -1;+	    }+	}+    }++    if (sam_hdr_rebuild(hdr))+	return -1;++    /* Length */+    header_len = sam_hdr_length(hdr);+    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	if (-1 == int32_encode(fd, header_len))+	    return -1;++	/* Text data */+	if (header_len != hwrite(fd->fp, sam_hdr_str(hdr), header_len))+	    return -1;+    } else {+	/* Create block(s) inside a container */+	cram_block *b = cram_new_block(FILE_HEADER, 0);+	cram_container *c = cram_new_container(0, 0);+	int padded_length;+	char *pads;+	int is_cram_3 = (CRAM_MAJOR_VERS(fd->version) >= 3);++	if (!b || !c) {+	    if (b) cram_free_block(b);+	    if (c) cram_free_container(c);+	    return -1;+	}++	int32_put_blk(b, header_len);+	BLOCK_APPEND(b, sam_hdr_str(hdr), header_len);+	BLOCK_UPLEN(b);++	// Compress header block if V3.0 and above+	if (CRAM_MAJOR_VERS(fd->version) >= 3)+	    cram_compress_block(fd, b, NULL, -1, -1);++	if (blank_block) {+	    c->length = b->comp_size + 2 + 4*is_cram_3 ++		itf8_size(b->content_id)   + +		itf8_size(b->uncomp_size)  ++		itf8_size(b->comp_size);++	    c->num_blocks = 2;+	    c->num_landmarks = 2;+	    if (!(c->landmark = malloc(2*sizeof(*c->landmark)))) {+		cram_free_block(b);+		cram_free_container(c);+		return -1;+	    }+	    c->landmark[0] = 0;+	    c->landmark[1] = c->length;++	    // Plus extra storage for uncompressed secondary blank block+	    padded_length = MIN(c->length*.5, 10000);+	    c->length += padded_length + 2 + 4*is_cram_3 ++		itf8_size(b->content_id) + +		itf8_size(padded_length)*2;+	} else {+	    // Pad the block instead.+	    c->num_blocks = 1;+	    c->num_landmarks = 1;+	    if (!(c->landmark = malloc(sizeof(*c->landmark))))+		return -1;+	    c->landmark[0] = 0;++	    padded_length = MAX(c->length*1.5, 10000) - c->length;++	    c->length = b->comp_size + padded_length ++		2 + 4*is_cram_3 ++		itf8_size(b->content_id)   + +		itf8_size(b->uncomp_size)  ++		itf8_size(b->comp_size);++	    if (NULL == (pads = calloc(1, padded_length))) {+		cram_free_block(b);+		cram_free_container(c);+		return -1;+	    }+	    BLOCK_APPEND(b, pads, padded_length);+	    BLOCK_UPLEN(b);+	    free(pads);+	}++	if (-1 == cram_write_container(fd, c)) {+	    cram_free_block(b);+	    cram_free_container(c);+	    return -1;+	}++	if (-1 == cram_write_block(fd, b)) {+	    cram_free_block(b);+	    cram_free_container(c);+	    return -1;+	}++	if (blank_block) {+	    BLOCK_RESIZE(b, padded_length);+	    memset(BLOCK_DATA(b), 0, padded_length);+	    BLOCK_SIZE(b) = padded_length;+	    BLOCK_UPLEN(b);+	    b->method = RAW;+	    if (-1 == cram_write_block(fd, b)) {+		cram_free_block(b);+		cram_free_container(c);+		return -1;+	    }+	}++	cram_free_block(b);+	cram_free_container(c);+    }++    if (-1 == refs_from_header(fd->refs, fd, fd->header))+	return -1;+    if (-1 == refs2id(fd->refs, fd->header))+	return -1;++    if (0 != hflush(fd->fp))+	return -1;++    RP("=== Finishing saving header ===\n");++    return 0;+}++/* ----------------------------------------------------------------------+ * The top-level cram opening, closing and option handling+ */++/*+ * Initialises the lookup tables. These could be global statics, but they're+ * clumsy to setup in a multi-threaded environment unless we generate+ * verbatim code and include that.+ */+static void cram_init_tables(cram_fd *fd) {+    int i;++    memset(fd->L1, 4, 256);+    fd->L1['A'] = 0; fd->L1['a'] = 0;+    fd->L1['C'] = 1; fd->L1['c'] = 1;+    fd->L1['G'] = 2; fd->L1['g'] = 2;+    fd->L1['T'] = 3; fd->L1['t'] = 3;++    memset(fd->L2, 5, 256);+    fd->L2['A'] = 0; fd->L2['a'] = 0;+    fd->L2['C'] = 1; fd->L2['c'] = 1;+    fd->L2['G'] = 2; fd->L2['g'] = 2;+    fd->L2['T'] = 3; fd->L2['t'] = 3;+    fd->L2['N'] = 4; fd->L2['n'] = 4;++    if (CRAM_MAJOR_VERS(fd->version) == 1) {+	for (i = 0; i < 0x200; i++) {+	    int f = 0;++	    if (i & CRAM_FPAIRED)      f |= BAM_FPAIRED;+	    if (i & CRAM_FPROPER_PAIR) f |= BAM_FPROPER_PAIR;+	    if (i & CRAM_FUNMAP)       f |= BAM_FUNMAP;+	    if (i & CRAM_FREVERSE)     f |= BAM_FREVERSE;+	    if (i & CRAM_FREAD1)       f |= BAM_FREAD1;+	    if (i & CRAM_FREAD2)       f |= BAM_FREAD2;+	    if (i & CRAM_FSECONDARY)   f |= BAM_FSECONDARY;+	    if (i & CRAM_FQCFAIL)      f |= BAM_FQCFAIL;+	    if (i & CRAM_FDUP)         f |= BAM_FDUP;++	    fd->bam_flag_swap[i]  = f;+	}+    +	for (i = 0; i < 0x1000; i++) {+	    int g = 0;++	    if (i & BAM_FPAIRED)	   g |= CRAM_FPAIRED;+	    if (i & BAM_FPROPER_PAIR)  g |= CRAM_FPROPER_PAIR;+	    if (i & BAM_FUNMAP)        g |= CRAM_FUNMAP;+	    if (i & BAM_FREVERSE)      g |= CRAM_FREVERSE;+	    if (i & BAM_FREAD1)        g |= CRAM_FREAD1;+	    if (i & BAM_FREAD2)        g |= CRAM_FREAD2;+	    if (i & BAM_FSECONDARY)    g |= CRAM_FSECONDARY;+	    if (i & BAM_FQCFAIL)       g |= CRAM_FQCFAIL;+	    if (i & BAM_FDUP)          g |= CRAM_FDUP;++	    fd->cram_flag_swap[i] = g;+	}+    } else {+	/* NOP */+	for (i = 0; i < 0x1000; i++)+	    fd->bam_flag_swap[i] = i;+	for (i = 0; i < 0x1000; i++)+	    fd->cram_flag_swap[i] = i;+    }++    memset(fd->cram_sub_matrix, 4, 32*32);+    for (i = 0; i < 32; i++) {+	fd->cram_sub_matrix[i]['A'&0x1f]=0;+	fd->cram_sub_matrix[i]['C'&0x1f]=1;+	fd->cram_sub_matrix[i]['G'&0x1f]=2;+	fd->cram_sub_matrix[i]['T'&0x1f]=3;+	fd->cram_sub_matrix[i]['N'&0x1f]=4;+    }+    for (i = 0; i < 20; i+=4) {+	int j;+	for (j = 0; j < 20; j++) {+	    fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;+	    fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;+	    fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;+	    fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;+	}+	fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+0]&0x1f]=0;+	fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+1]&0x1f]=1;+	fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+2]&0x1f]=2;+	fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+3]&0x1f]=3;+    }+}++// Default version numbers for CRAM+static int major_version = 3;+static int minor_version = 0;++/*+ * Opens a CRAM file for read (mode "rb") or write ("wb").+ * The filename may be "-" to indicate stdin or stdout.+ *+ * Returns file handle on success+ *         NULL on failure.+ */+cram_fd *cram_open(const char *filename, const char *mode) {+    hFILE *fp;+    cram_fd *fd;+    char fmode[3]= { mode[0], '\0', '\0' };++    if (strlen(mode) > 1 && (mode[1] == 'b' || mode[1] == 'c')) {+	fmode[1] = 'b';+    }++    fp = hopen(filename, fmode);+    if (!fp)+	return NULL;++    fd = cram_dopen(fp, filename, mode);+    if (!fd)+	hclose_abruptly(fp);++    return fd;+}++/* Opens an existing stream for reading or writing.+ *+ * Returns file handle on success;+ *         NULL on failure.+ */+cram_fd *cram_dopen(hFILE *fp, const char *filename, const char *mode) {+    int i;+    char *cp;+    cram_fd *fd = calloc(1, sizeof(*fd));+    if (!fd)+	return NULL;++    fd->level = 5;+    for (i = 0; mode[i]; i++) {+	if (mode[i] >= '0' && mode[i] <= '9') {+	    fd->level = mode[i] - '0';+	    break;+	}+    }++    fd->fp = fp;+    fd->mode = *mode;+    fd->first_container = 0;++    if (fd->mode == 'r') {+	/* Reader */++	if (!(fd->file_def = cram_read_file_def(fd)))+	    goto err;++	fd->version = fd->file_def->major_version * 256 ++	    fd->file_def->minor_version;++	if (!(fd->header = cram_read_SAM_hdr(fd)))+	    goto err;++    } else {+	/* Writer */+	cram_file_def *def = calloc(1, sizeof(*def));+	if (!def)+	    return NULL;++	fd->file_def = def;++	def->magic[0] = 'C';+	def->magic[1] = 'R';+	def->magic[2] = 'A';+	def->magic[3] = 'M';+	def->major_version = 0; // Indicator to write file def later.+	def->minor_version = 0;+	memset(def->file_id, 0, 20);+	strncpy(def->file_id, filename, 20);++	fd->version = major_version * 256 + minor_version;++	/* SAM header written later along with this file_def */+    }++    cram_init_tables(fd);++    fd->prefix = strdup((cp = strrchr(filename, '/')) ? cp+1 : filename);+    if (!fd->prefix)+	goto err;+    fd->first_base = fd->last_base = -1;+    fd->record_counter = 0;++    fd->ctr = NULL;+    fd->refs  = refs_create();+    if (!fd->refs)+	goto err;+    fd->ref_id = -2;+    fd->ref = NULL;++    fd->decode_md = 0;+    fd->verbose = 0;+    fd->seqs_per_slice = SEQS_PER_SLICE;+    fd->slices_per_container = SLICE_PER_CNT;+    fd->embed_ref = 0;+    fd->no_ref = 0;+    fd->ignore_md5 = 0;+    fd->use_bz2 = 0;+    fd->use_rans = (CRAM_MAJOR_VERS(fd->version) >= 3);+    fd->use_lzma = 0;+    fd->multi_seq = -1;+    fd->unsorted   = 0;+    fd->shared_ref = 0;++    fd->index       = NULL;+    fd->own_pool    = 0;+    fd->pool        = NULL;+    fd->rqueue      = NULL;+    fd->job_pending = NULL;+    fd->ooc         = 0;+    fd->required_fields = INT_MAX;++    for (i = 0; i < DS_END; i++)+	fd->m[i] = cram_new_metrics();++    fd->range.refid = -2; // no ref.+    fd->eof = 1;          // See samtools issue #150+    fd->ref_fn = NULL;++    fd->bl = NULL;++    /* Initialise dummy refs from the @SQ headers */+    if (-1 == refs_from_header(fd->refs, fd, fd->header))+	goto err;++    return fd;++ err:+    if (fd)+	free(fd);++    return NULL;+}++/*+ * Seek within a CRAM file.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_seek(cram_fd *fd, off_t offset, int whence) {+    char buf[65536];++    fd->ooc = 0;++    if (hseek(fd->fp, offset, whence) >= 0)+	return 0;++    if (!(whence == SEEK_CUR && offset >= 0))+	return -1;++    /* Couldn't fseek, but we're in SEEK_CUR mode so read instead */+    while (offset > 0) {+	int len = MIN(65536, offset);+	if (len != hread(fd->fp, buf, len))+	    return -1;+	offset -= len;+    }++    return 0;+}++/*+ * Flushes a CRAM file.+ * Useful for when writing to stdout without wishing to close the stream.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_flush(cram_fd *fd) {+    if (!fd)+	return -1;++    if (fd->mode == 'w' && fd->ctr) {+	if(fd->ctr->slice)+	    fd->ctr->curr_slice++;+	if (-1 == cram_flush_container_mt(fd, fd->ctr))+	    return -1;+    }++    return 0;+}++/*+ * Closes a CRAM file.+ * Returns 0 on success+ *        -1 on failure+ */+int cram_close(cram_fd *fd) {+    spare_bams *bl, *next;+    int i;+	+    if (!fd)+	return -1;++    if (fd->mode == 'w' && fd->ctr) {+	if(fd->ctr->slice)+	    fd->ctr->curr_slice++;+	if (-1 == cram_flush_container_mt(fd, fd->ctr))+	    return -1;+    }++    if (fd->pool && fd->eof >= 0) {+	t_pool_flush(fd->pool);++	if (0 != cram_flush_result(fd))+	    return -1;++	pthread_mutex_destroy(&fd->metrics_lock);+	pthread_mutex_destroy(&fd->ref_lock);+	pthread_mutex_destroy(&fd->bam_list_lock);++	fd->ctr = NULL; // prevent double freeing++	//fprintf(stderr, "CRAM: destroy queue %p\n", fd->rqueue);++	t_results_queue_destroy(fd->rqueue);+    }++    if (fd->mode == 'w') {+	/* Write EOF block */+	if (CRAM_MAJOR_VERS(fd->version) == 3) {+	    if (38 != hwrite(fd->fp,+			     "\x0f\x00\x00\x00\xff\xff\xff\xff" // Cont HDR+			     "\x0f\xe0\x45\x4f\x46\x00\x00\x00" // Cont HDR+			     "\x00\x01\x00"                     // Cont HDR+			     "\x05\xbd\xd9\x4f"                 // CRC32+			     "\x00\x01\x00\x06\x06"             // Comp.HDR blk+			     "\x01\x00\x01\x00\x01\x00"         // Comp.HDR blk+			     "\xee\x63\x01\x4b",                // CRC32+			     38))+		return -1;+	} else {+	    if (30 != hwrite(fd->fp,+			     "\x0b\x00\x00\x00\xff\xff\xff\xff"+			     "\x0f\xe0\x45\x4f\x46\x00\x00\x00"+			     "\x00\x01\x00\x00\x01\x00\x06\x06"+			     "\x01\x00\x01\x00\x01\x00", 30))+		return -1;+	}+    }++    for (bl = fd->bl; bl; bl = next) {+	int i, max_rec = fd->seqs_per_slice * fd->slices_per_container;++	next = bl->next;+	for (i = 0; i < max_rec; i++) {+	    if (bl->bams[i])+		bam_free(bl->bams[i]);+	}+	free(bl->bams);+	free(bl);+    }++    if (hclose(fd->fp) != 0)+	return -1;++    if (fd->file_def)+	cram_free_file_def(fd->file_def);++    if (fd->header)+	sam_hdr_free(fd->header);++    free(fd->prefix);++    if (fd->ctr)+	cram_free_container(fd->ctr);++    if (fd->refs)+	refs_free(fd->refs);+    if (fd->ref_free)+        free(fd->ref_free);++    for (i = 0; i < DS_END; i++)+	if (fd->m[i])+	    free(fd->m[i]);++    if (fd->index)+	cram_index_free(fd);++    if (fd->own_pool && fd->pool)+	t_pool_destroy(fd->pool, 0);++    free(fd);+    return 0;+}++/*+ * Returns 1 if we hit an EOF while reading.+ */+int cram_eof(cram_fd *fd) {+    return fd->eof;+}+++/* + * Sets options on the cram_fd. See CRAM_OPT_* definitions in cram_structs.h.+ * Use this immediately after opening.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_set_option(cram_fd *fd, enum hts_fmt_option opt, ...) {+    int r;+    va_list args;++    va_start(args, opt);+    r = cram_set_voption(fd, opt, args);+    va_end(args);++    return r;+}++/*+ * Sets options on the cram_fd. See CRAM_OPT_* definitions in cram_structs.h.+ * Use this immediately after opening.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_set_voption(cram_fd *fd, enum hts_fmt_option opt, va_list args) {+    refs_t *refs;++    if (!fd) {+	errno = EBADF;+	return -1;+    }++    switch (opt) {+    case CRAM_OPT_DECODE_MD:+	fd->decode_md = va_arg(args, int);+	break;++    case CRAM_OPT_PREFIX:+	if (fd->prefix)+	    free(fd->prefix);+	if (!(fd->prefix = strdup(va_arg(args, char *))))+	    return -1;+	break;++    case CRAM_OPT_VERBOSITY:+	fd->verbose = va_arg(args, int);+	break;++    case CRAM_OPT_SEQS_PER_SLICE:+	fd->seqs_per_slice = va_arg(args, int);+	break;++    case CRAM_OPT_SLICES_PER_CONTAINER:+	fd->slices_per_container = va_arg(args, int);+	break;++    case CRAM_OPT_EMBED_REF:+	fd->embed_ref = va_arg(args, int);+	break;++    case CRAM_OPT_NO_REF:+	fd->no_ref = va_arg(args, int);+	break;++    case CRAM_OPT_IGNORE_MD5:+	fd->ignore_md5 = va_arg(args, int);+	break;++    case CRAM_OPT_USE_BZIP2:+	fd->use_bz2 = va_arg(args, int);+	break;++    case CRAM_OPT_USE_RANS:+	fd->use_rans = va_arg(args, int);+	break;++    case CRAM_OPT_USE_LZMA:+	fd->use_lzma = va_arg(args, int);+	break;++    case CRAM_OPT_SHARED_REF:+	fd->shared_ref = 1;+	refs = va_arg(args, refs_t *);+	if (refs != fd->refs) {+	    if (fd->refs)+		refs_free(fd->refs);+	    fd->refs = refs;+	    fd->refs->count++;+	}+	break;++    case CRAM_OPT_RANGE:+	fd->range = *va_arg(args, cram_range *);+	return cram_seek_to_refpos(fd, &fd->range);++    case CRAM_OPT_REFERENCE:+	return cram_load_reference(fd, va_arg(args, char *));++    case CRAM_OPT_VERSION: {+	int major, minor;+	char *s = va_arg(args, char *);+	if (2 != sscanf(s, "%d.%d", &major, &minor)) {+	    fprintf(stderr, "Malformed version string %s\n", s);+	    return -1;+	}+	if (!((major == 1 &&  minor == 0) ||+	      (major == 2 && (minor == 0 || minor == 1)) ||+	      (major == 3 &&  minor == 0))) {+	    fprintf(stderr, "Unknown version string; "+		    "use 1.0, 2.0, 2.1 or 3.0\n");+	    errno = EINVAL;+	    return -1;+	}+	fd->version = major*256 + minor;++	if (CRAM_MAJOR_VERS(fd->version) >= 3)+	    fd->use_rans = 1;+	break;+    }++    case CRAM_OPT_MULTI_SEQ_PER_SLICE:+	fd->multi_seq = va_arg(args, int);+	break;++    case CRAM_OPT_NTHREADS: {+	int nthreads =  va_arg(args, int);+        if (nthreads > 1) {+            if (!(fd->pool = t_pool_init(nthreads*2, nthreads)))+                return -1;++	    fd->rqueue = t_results_queue_init();+	    pthread_mutex_init(&fd->metrics_lock, NULL);+	    pthread_mutex_init(&fd->ref_lock, NULL);+	    pthread_mutex_init(&fd->bam_list_lock, NULL);+	    fd->shared_ref = 1;+	    fd->own_pool = 1;+        }+	break;+    }++    case CRAM_OPT_THREAD_POOL:+	fd->pool = va_arg(args, t_pool *);+	if (fd->pool) {+	    fd->rqueue = t_results_queue_init();+	    pthread_mutex_init(&fd->metrics_lock, NULL);+	    pthread_mutex_init(&fd->ref_lock, NULL);+	    pthread_mutex_init(&fd->bam_list_lock, NULL);+	}+	fd->shared_ref = 1; // Needed to avoid clobbering ref between threads+	fd->own_pool = 0;++	//fd->qsize = 1;+	//fd->decoded = calloc(fd->qsize, sizeof(cram_container *));+	//t_pool_dispatch(fd->pool, cram_decoder_thread, fd);+	break;++    case CRAM_OPT_REQUIRED_FIELDS:+	fd->required_fields = va_arg(args, int);+	break;++    case HTS_OPT_COMPRESSION_LEVEL:+	fd->level = va_arg(args, int);+	break;++    default:+	fprintf(stderr, "Unknown CRAM option code %d\n", opt);+	errno = EINVAL;+	return -1;+    }++    return 0;+}
+ htslib-1.3.2/cram/cram_io.h view
@@ -0,0 +1,669 @@+/*+Copyright (c) 2012-2014 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*! \file+ * Include cram.h instead.+ *+ * This is an internal part of the CRAM system and is automatically included+ * when you #include cram.h.+ *+ * Implements the low level CRAM I/O primitives.+ * This includes basic data types such as byte, int, ITF-8,+ * maps, bitwise I/O, etc.+ */++#ifndef _CRAM_IO_H_+#define _CRAM_IO_H_++#define ITF8_MACROS++#include <stdint.h>+#include <cram/misc.h>++#ifdef __cplusplus+extern "C" {+#endif++/**@{ ----------------------------------------------------------------------+ * ITF8 encoding and decoding.+ *+ * Also see the itf8_get and itf8_put macros.+ */++/*! INTERNAL: Converts two characters into an integer for use in switch{} */+#define CRAM_KEY(a,b) (((a)<<8)|((b)))++/*! Reads an integer in ITF-8 encoding from 'fd' and stores it in+ * *val.+ *+ * @return+ * Returns the number of bytes read on success;+ *        -1 on failure+ */+int itf8_decode(cram_fd *fd, int32_t *val);++#ifndef ITF8_MACROS+/*! Reads an integer in ITF-8 encoding from 'cp' and stores it in+ * *val.+ *+ * @return+ * Returns the number of bytes read on success;+ *        -1 on failure+ */+int itf8_get(char *cp, int32_t *val_p);++/*! Stores a value to memory in ITF-8 format.+ *+ * @return+ * Returns the number of bytes required to store the number.+ * This is a maximum of 5 bytes.+ */+int itf8_put(char *cp, int32_t val);++#else++/*+ * Macro implementations of the above+ */+#define itf8_get(c,v) (((uc)(c)[0]<0x80)?(*(v)=(uc)(c)[0],1):(((uc)(c)[0]<0xc0)?(*(v)=(((uc)(c)[0]<<8)|(uc)(c)[1])&0x3fff,2):(((uc)(c)[0]<0xe0)?(*(v)=(((uc)(c)[0]<<16)|((uc)(c)[1]<<8)|(uc)(c)[2])&0x1fffff,3):(((uc)(c)[0]<0xf0)?(*(v)=(((uc)(c)[0]<<24)|((uc)(c)[1]<<16)|((uc)(c)[2]<<8)|(uc)(c)[3])&0x0fffffff,4):(*(v)=(((uc)(c)[0]&0x0f)<<28)|((uc)(c)[1]<<20)|((uc)(c)[2]<<12)|((uc)(c)[3]<<4)|((uc)(c)[4]&0x0f),5)))))++#define itf8_put(c,v) ((!((v)&~0x7f))?((c)[0]=(v),1):(!((v)&~0x3fff))?((c)[0]=((v)>>8)|0x80,(c)[1]=(v)&0xff,2):(!((v)&~0x1fffff))?((c)[0]=((v)>>16)|0xc0,(c)[1]=((v)>>8)&0xff,(c)[2]=(v)&0xff,3):(!((v)&~0xfffffff))?((c)[0]=((v)>>24)|0xe0,(c)[1]=((v)>>16)&0xff,(c)[2]=((v)>>8)&0xff,(c)[3]=(v)&0xff,4):((c)[0]=0xf0|(((v)>>28)&0xff),(c)[1]=((v)>>20)&0xff,(c)[2]=((v)>>12)&0xff,(c)[3]=((v)>>4)&0xff,(c)[4]=(v)&0xf,5))++#define itf8_size(v) ((!((v)&~0x7f))?1:(!((v)&~0x3fff))?2:(!((v)&~0x1fffff))?3:(!((v)&~0xfffffff))?4:5)++#endif++int ltf8_get(char *cp, int64_t *val_p);+int ltf8_put(char *cp, int64_t val);++  /* Version of itf8_get that checks it hasn't run out of input */++extern const int itf8_bytes[16];++static inline int safe_itf8_get(const char *cp, const char *endp,+                                int32_t *val_p) {+    const unsigned char *up = (unsigned char *)cp;++    if (endp - cp < 5 &&+        (cp >= endp || endp - cp < itf8_bytes[up[0]>>4])) {+        *val_p = 0;+        return 0;+    }++    if (up[0] < 0x80) {+        *val_p =   up[0];+        return 1;+    } else if (up[0] < 0xc0) {+        *val_p = ((up[0] <<8) |  up[1])                           & 0x3fff;+        return 2;+    } else if (up[0] < 0xe0) {+        *val_p = ((up[0]<<16) | (up[1]<< 8) |  up[2])             & 0x1fffff;+        return 3;+    } else if (up[0] < 0xf0) {+        *val_p = ((up[0]<<24) | (up[1]<<16) | (up[2]<<8) | up[3]) & 0x0fffffff;+        return 4;+    } else {+        *val_p = ((up[0] & 0x0f)<<28) | (up[1]<<20) | (up[2]<<12) | (up[3]<<4) | (up[4] & 0x0f);+        return 5;+    }+}++/*! Pushes a value in ITF8 format onto the end of a block.+ *+ * This shouldn't be used for high-volume data as it is not the fastest+ * method.+ *+ * @return+ * Returns the number of bytes written+ */+int itf8_put_blk(cram_block *blk, int val);++/*! Pulls a literal 32-bit value from a block.+ *+ * @returns the number of bytes decoded;+ *         -1 on failure.+ */+int int32_get_blk(cram_block *b, int32_t *val);++/*! Pushes a literal 32-bit value onto the end of a block.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure.+ */+int int32_put_blk(cram_block *blk, int32_t val);+++/**@}*/+/**@{ ----------------------------------------------------------------------+ * CRAM blocks - the dynamically growable data block. We have code to+ * create, update, (un)compress and read/write.+ *+ * These are derived from the deflate_interlaced.c blocks, but with the+ * CRAM extension of content types and IDs.+ */++/*! Allocates a new cram_block structure with a specified content_type and+ * id.+ *+ * @return+ * Returns block pointer on success;+ *         NULL on failure+ */+cram_block *cram_new_block(enum cram_content_type content_type,+			   int content_id);++/*! Reads a block from a cram file.+ *+ * @return+ * Returns cram_block pointer on success;+ *         NULL on failure+ */+cram_block *cram_read_block(cram_fd *fd);++/*! Writes a CRAM block.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_write_block(cram_fd *fd, cram_block *b);++/*! Frees a CRAM block, deallocating internal data too.+ */+void cram_free_block(cram_block *b);++/*! Uncompress a memory block using Zlib.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+char *zlib_mem_inflate(char *cdata, size_t csize, size_t *size);++/*! Uncompresses a CRAM block, if compressed.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_uncompress_block(cram_block *b);++/*! Compresses a block.+ *+ * Compresses a block using one of two different zlib strategies. If we only+ * want one choice set strat2 to be -1.+ *+ * The logic here is that sometimes Z_RLE does a better job than Z_FILTERED+ * or Z_DEFAULT_STRATEGY on quality data. If so, we'd rather use it as it is+ * significantly faster.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_compress_block(cram_fd *fd, cram_block *b, cram_metrics *metrics,+			int method, int level);++cram_metrics *cram_new_metrics(void);+char *cram_block_method2str(enum cram_block_method m);+char *cram_content_type2str(enum cram_content_type t);++/*+ * Find an external block by its content_id+ */++static inline cram_block *cram_get_block_by_id(cram_slice *slice, int id) {+    if (slice->block_by_id && id >= 0 && id < 1024) {+        return slice->block_by_id[id];+    } else {+        int i;+        for (i = 0; i < slice->hdr->num_blocks; i++) {+	    cram_block *b = slice->block[i];+	    if (b && b->content_type == EXTERNAL && b->content_id == id)+	        return b;+	}+    }+    return NULL;+}++/* --- Accessor macros for manipulating blocks on a byte by byte basis --- */++/* Block size and data pointer. */+#define BLOCK_SIZE(b) ((b)->byte)+#define BLOCK_DATA(b) ((b)->data)++/* Returns the address one past the end of the block */+#define BLOCK_END(b) (&(b)->data[(b)->byte])++/* Request block to be at least 'l' bytes long */+#define BLOCK_RESIZE(b,l)					\+    do {							\+	while((b)->alloc <= (l)) {				\+	    (b)->alloc = (b)->alloc ? (b)->alloc*1.5 : 1024;	\+	    (b)->data = realloc((b)->data, (b)->alloc);		\+	}							\+     } while(0)++/* Make block exactly 'l' bytes long */+#define BLOCK_RESIZE_EXACT(b,l)					\+    do {							\+        (b)->alloc = (l);                                       \+        (b)->data = realloc((b)->data, (b)->alloc);		\+     } while(0)++/* Ensure the block can hold at least another 'l' bytes */+#define BLOCK_GROW(b,l) BLOCK_RESIZE((b), BLOCK_SIZE((b)) + (l))++/* Append string 's' of length 'l' */+#define BLOCK_APPEND(b,s,l)		  \+    do {				  \+        BLOCK_GROW((b),(l));		  \+        memcpy(BLOCK_END((b)), (s), (l)); \+	BLOCK_SIZE((b)) += (l);		  \+    } while (0)++/* Append as single character 'c' */+#define BLOCK_APPEND_CHAR(b,c)		  \+    do {				  \+        BLOCK_GROW((b),1);		  \+	(b)->data[(b)->byte++] = (c);	  \+    } while (0)++/* Append a single unsigned integer */+#define BLOCK_APPEND_UINT(b,i)		             \+    do {					     \+        unsigned char *cp;			     \+        BLOCK_GROW((b),11);			     \+	cp = &(b)->data[(b)->byte];		     \+        (b)->byte += append_uint32(cp, (i)) - cp;	\+    } while (0)++static inline unsigned char *append_uint32(unsigned char *cp, uint32_t i) {+    uint32_t j;++    if (i == 0) {+	*cp++ = '0';+	return cp;+    }++    if (i < 100)        goto b1;+    if (i < 10000)      goto b3;+    if (i < 1000000)    goto b5;+    if (i < 100000000)  goto b7;++    if ((j = i / 1000000000)) {*cp++ = j + '0'; i -= j*1000000000; goto x8;}+    if ((j = i / 100000000))  {*cp++ = j + '0'; i -= j*100000000;  goto x7;}+ b7:if ((j = i / 10000000))   {*cp++ = j + '0'; i -= j*10000000;   goto x6;}+    if ((j = i / 1000000))    {*cp++ = j + '0', i -= j*1000000;    goto x5;}+ b5:if ((j = i / 100000))     {*cp++ = j + '0', i -= j*100000;     goto x4;}+    if ((j = i / 10000))      {*cp++ = j + '0', i -= j*10000;      goto x3;}+ b3:if ((j = i / 1000))       {*cp++ = j + '0', i -= j*1000;       goto x2;}+    if ((j = i / 100))        {*cp++ = j + '0', i -= j*100;        goto x1;}+ b1:if ((j = i / 10))         {*cp++ = j + '0', i -= j*10;         goto x0;}+    if (i)                     *cp++ = i + '0';+    return cp;++ x8: *cp++ = i / 100000000 + '0', i %= 100000000;+ x7: *cp++ = i / 10000000  + '0', i %= 10000000;+ x6: *cp++ = i / 1000000   + '0', i %= 1000000;+ x5: *cp++ = i / 100000    + '0', i %= 100000;+ x4: *cp++ = i / 10000     + '0', i %= 10000;+ x3: *cp++ = i / 1000      + '0', i %= 1000;+ x2: *cp++ = i / 100       + '0', i %= 100;+ x1: *cp++ = i / 10        + '0', i %= 10;+ x0: *cp++ = i             + '0';++    return cp;+}++static inline unsigned char *append_sub32(unsigned char *cp, uint32_t i) {+    *cp++ = i / 100000000 + '0', i %= 100000000;+    *cp++ = i / 10000000  + '0', i %= 10000000;+    *cp++ = i / 1000000   + '0', i %= 1000000;+    *cp++ = i / 100000    + '0', i %= 100000;+    *cp++ = i / 10000     + '0', i %= 10000;+    *cp++ = i / 1000      + '0', i %= 1000;+    *cp++ = i / 100       + '0', i %= 100;+    *cp++ = i / 10        + '0', i %= 10;+    *cp++ = i             + '0';++    return cp;+}++static inline unsigned char *append_uint64(unsigned char *cp, uint64_t i) {+    uint64_t j;++    if (i <= 0xffffffff)+	return append_uint32(cp, i);++    if ((j = i/1000000000) > 1000000000) {+	cp = append_uint32(cp, j/1000000000);+	j %= 1000000000;+	cp = append_sub32(cp, j);+    } else {+	cp = append_uint32(cp, i / 1000000000);+    }+    cp = append_sub32(cp, i % 1000000000);++    return cp;+}++#define BLOCK_UPLEN(b) \+    (b)->comp_size = (b)->uncomp_size = BLOCK_SIZE((b))++/**@}*/+/**@{ ----------------------------------------------------------------------+ * Reference sequence handling+ */++/*! Loads a reference set from fn and stores in the cram_fd.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_load_reference(cram_fd *fd, char *fn);++/*! Generates a lookup table in refs based on the SQ headers in SAM_hdr.+ *+ * Indexes references by the order they appear in a BAM file. This may not+ * necessarily be the same order they appear in the fasta reference file.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int refs2id(refs_t *r, SAM_hdr *bfd);++void refs_free(refs_t *r);++/*! Returns a portion of a reference sequence from start to end inclusive.+ *+ * The returned pointer is owned by the cram_file fd and should not be freed+ * by the caller. It is valid only until the next cram_get_ref is called+ * with the same fd parameter (so is thread-safe if given multiple files).+ *+ * To return the entire reference sequence, specify start as 1 and end+ * as 0.+ *+ * @return+ * Returns reference on success;+ *         NULL on failure+ */+char *cram_get_ref(cram_fd *fd, int id, int start, int end);+void cram_ref_incr(refs_t *r, int id);+void cram_ref_decr(refs_t *r, int id);+/**@}*/+/**@{ ----------------------------------------------------------------------+ * Containers+ */++/*! Creates a new container, specifying the maximum number of slices+ * and records permitted.+ *+ * @return+ * Returns cram_container ptr on success;+ *         NULL on failure+ */+cram_container *cram_new_container(int nrec, int nslice);+void cram_free_container(cram_container *c);++/*! Reads a container header.+ *+ * @return+ * Returns cram_container on success;+ *         NULL on failure or no container left (fd->err == 0).+ */+cram_container *cram_read_container(cram_fd *fd);++/*! Writes a container structure.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_write_container(cram_fd *fd, cram_container *h);++/*! Flushes a container to disk.+ *+ * Flushes a completely or partially full container to disk, writing+ * container structure, header and blocks. This also calls the encoder+ * functions.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_flush_container(cram_fd *fd, cram_container *c);+int cram_flush_container_mt(cram_fd *fd, cram_container *c);+++/**@}*/+/**@{ ----------------------------------------------------------------------+ * Compression headers; the first part of the container+ */++/*! Creates a new blank container compression header+ *+ * @return+ * Returns header ptr on success;+ *         NULL on failure+ */+cram_block_compression_hdr *cram_new_compression_header(void);++/*! Frees a cram_block_compression_hdr */+void cram_free_compression_header(cram_block_compression_hdr *hdr);+++/**@}*/+/**@{ ----------------------------------------------------------------------+ * Slices and slice headers+ */++/*! Frees a slice header */+void cram_free_slice_header(cram_block_slice_hdr *hdr);++/*! Frees a slice */+void cram_free_slice(cram_slice *s);++/*! Creates a new empty slice in memory, for subsequent writing to+ * disk.+ *+ * @return+ * Returns cram_slice ptr on success;+ *         NULL on failure+ */+cram_slice *cram_new_slice(enum cram_content_type type, int nrecs);++/*! Loads an entire slice.+ *+ * FIXME: In 1.0 the native unit of slices within CRAM is broken+ * as slices contain references to objects in other slices.+ * To work around this while keeping the slice oriented outer loop+ * we read all slices and stitch them together into a fake large+ * slice instead.+ *+ * @return+ * Returns cram_slice ptr on success;+ *         NULL on failure+ */+cram_slice *cram_read_slice(cram_fd *fd);++++/**@}*/+/**@{ ----------------------------------------------------------------------+ * CRAM file definition (header)+ */++/*! Reads a CRAM file definition structure.+ *+ * @return+ * Returns file_def ptr on success;+ *         NULL on failure+ */+cram_file_def *cram_read_file_def(cram_fd *fd);++/*! Writes a cram_file_def structure to cram_fd.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_write_file_def(cram_fd *fd, cram_file_def *def);++/*! Frees a cram_file_def structure. */+void cram_free_file_def(cram_file_def *def);+++/**@}*/+/**@{ ----------------------------------------------------------------------+ * SAM header I/O+ */++/*! Reads the SAM header from the first CRAM data block.+ *+ * Also performs minimal parsing to extract read-group+ * and sample information.+ *+ * @return+ * Returns SAM hdr ptr on success;+ *         NULL on failure+ */+SAM_hdr *cram_read_SAM_hdr(cram_fd *fd);++/*! Writes a CRAM SAM header.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_write_SAM_hdr(cram_fd *fd, SAM_hdr *hdr);+++/**@}*/+/**@{ ----------------------------------------------------------------------+ * The top-level cram opening, closing and option handling+ */++/*! Opens a CRAM file for read (mode "rb") or write ("wb").+ *+ * The filename may be "-" to indicate stdin or stdout.+ *+ * @return+ * Returns file handle on success;+ *         NULL on failure.+ */+cram_fd *cram_open(const char *filename, const char *mode);++/*! Opens an existing stream for reading or writing.+ *+ * @return+ * Returns file handle on success;+ *         NULL on failure.+ */+cram_fd *cram_dopen(struct hFILE *fp, const char *filename, const char *mode);++/*! Closes a CRAM file.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_close(cram_fd *fd);++/*+ * Seek within a CRAM file.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_seek(cram_fd *fd, off_t offset, int whence);++/*+ * Flushes a CRAM file.+ * Useful for when writing to stdout without wishing to close the stream.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_flush(cram_fd *fd);++/*! Checks for end of file on a cram_fd stream.+ *+ * @return+ * Returns 0 if not at end of file+ *         1 if we hit an expected EOF (end of range or EOF block)+ *         2 for other EOF (end of stream without EOF block)+ */+int cram_eof(cram_fd *fd);++/*! Sets options on the cram_fd.+ *+ * See CRAM_OPT_* definitions in cram_structs.h.+ * Use this immediately after opening.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_set_option(cram_fd *fd, enum hts_fmt_option opt, ...);++/*! Sets options on the cram_fd.+ *+ * See CRAM_OPT_* definitions in cram_structs.h.+ * Use this immediately after opening.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_set_voption(cram_fd *fd, enum hts_fmt_option opt, va_list args);++/*!+ * Attaches a header to a cram_fd.+ *+ * This should be used when creating a new cram_fd for writing where+ * we have an SAM_hdr already constructed (eg from a file we've read+ * in).+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_set_header(cram_fd *fd, SAM_hdr *hdr);+++#ifdef __cplusplus+}+#endif++#endif /* _CRAM_IO_H_ */
+ htslib-1.3.2/cram/cram_samtools.c view
@@ -0,0 +1,149 @@+/*+Copyright (c) 2010-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <assert.h>+#include <string.h>+#include <stdlib.h>++#include "cram/cram.h"+#include "htslib/sam.h"++/*---------------------------------------------------------------------------+ * Samtools compatibility portion+ */+int bam_construct_seq(bam_seq_t **bp, size_t extra_len,+		      const char *qname, size_t qname_len,+		      int flag,+		      int rname,      // Ref ID+		      int pos,+		      int end,        // aligned start/end coords+		      int mapq,+		      uint32_t ncigar, const uint32_t *cigar,+		      int mrnm,       // Mate Ref ID+		      int mpos,+		      int isize,+		      int len,+		      const char *seq,+		      const char *qual) {+    static const char L[256] = {+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15, 0,15,15,+	15, 1,14, 2,13,15,15, 4,11,15,15,12,15, 3,15,15,+	15,15, 5, 6, 8,15, 7, 9,15,10,15,15,15,15,15,15,+	15, 1,14, 2,13,15,15, 4,11,15,15,12,15, 3,15,15,+	15,15, 5, 6, 8,15, 7, 9,15,10,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,+	15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15+    };+    bam1_t *b = (bam1_t *)*bp;+    uint8_t *cp;+    int i, bam_len;++    //b->l_aux = extra_len; // we fill this out later++    bam_len = qname_len + 1 + ncigar*4 + (len+1)/2 + len + extra_len;+    if (b->m_data < bam_len) {+	b->m_data = bam_len;+	kroundup32(b->m_data);+	b->data = (uint8_t*)realloc(b->data, b->m_data);+	if (!b->data)+	    return -1;+    }+    b->l_data = bam_len;++    b->core.tid     = rname;+    b->core.pos     = pos-1;+    b->core.bin     = bam_reg2bin(pos-1, end);+    b->core.qual    = mapq;+    b->core.l_qname = qname_len+1;+    b->core.flag    = flag;+    b->core.n_cigar = ncigar;+    b->core.l_qseq  = len;+    b->core.mtid    = mrnm;+    b->core.mpos    = mpos-1;+    b->core.isize   = isize;++    cp = b->data;++    strncpy((char *)cp, qname, qname_len);+    cp[qname_len] = 0;+    cp += qname_len+1;+    memcpy(cp, cigar, ncigar*4);+    cp += ncigar*4;++    for (i = 0; i+1 < len; i+=2) {+	*cp++ = (L[(uc)seq[i]]<<4) + L[(uc)seq[i+1]];+    }+    if (i < len)+	*cp++ = L[(uc)seq[i]]<<4;++    if (qual)+	memcpy(cp, qual, len);+    else+	memset(cp, '\xff', len);++    return 0;+}++bam_hdr_t *cram_header_to_bam(SAM_hdr *h) {+    int i;+    bam_hdr_t *header = bam_hdr_init();++    header->l_text = ks_len(&h->text);+    header->text = malloc(header->l_text+1);+    memcpy(header->text, ks_str(&h->text), header->l_text);+    header->text[header->l_text] = 0;++    header->n_targets = h->nref;+    header->target_name = (char **)calloc(header->n_targets,+					  sizeof(char *));+    header->target_len = (uint32_t *)calloc(header->n_targets, 4);++    for (i = 0; i < h->nref; i++) {+	header->target_name[i] = strdup(h->ref[i].name);+	header->target_len[i] = h->ref[i].len;+    }++    return header;+}++SAM_hdr *bam_header_to_cram(bam_hdr_t *h) {+    return sam_hdr_parse_(h->text, h->l_text);+}
+ htslib-1.3.2/cram/cram_samtools.h view
@@ -0,0 +1,105 @@+/*+Copyright (c) 2010-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _CRAM_SAMTOOLS_H_+#define _CRAM_SAMTOOLS_H_++/* Samtools compatible API */+#define bam_blk_size(b)  ((b)->l_data)+#define bam_set_blk_size(b,v) ((b)->data_len = (v))++#define bam_ref(b)       (b)->core.tid+#define bam_pos(b)       (b)->core.pos+#define bam_mate_pos(b)  (b)->core.mpos+#define bam_mate_ref(b)  (b)->core.mtid+#define bam_ins_size(b)  (b)->core.isize+#define bam_seq_len(b)   (b)->core.l_qseq+#define bam_cigar_len(b) (b)->core.n_cigar+#define bam_flag(b)      (b)->core.flag+#define bam_bin(b)       (b)->core.bin+#define bam_map_qual(b)  (b)->core.qual+#define bam_name_len(b)  (b)->core.l_qname+#define bam_name(b)      bam_get_qname((b))+#define bam_qual(b)      bam_get_qual((b))+#define bam_seq(b)       bam_get_seq((b))+#define bam_cigar(b)     bam_get_cigar((b))+#define bam_aux(b)       bam_get_aux((b))++#define bam_dup(b)       bam_copy1(bam_init1(), (b))++#define bam_free(b)      bam_destroy1((b))++#define bam_reg2bin(beg,end) hts_reg2bin((beg),(end),14,5)++#include "htslib/sam.h"++enum cigar_op {+    BAM_CMATCH_=BAM_CMATCH,+    BAM_CINS_=BAM_CINS,+    BAM_CDEL_=BAM_CDEL,+    BAM_CREF_SKIP_=BAM_CREF_SKIP,+    BAM_CSOFT_CLIP_=BAM_CSOFT_CLIP,+    BAM_CHARD_CLIP_=BAM_CHARD_CLIP,+    BAM_CPAD_=BAM_CPAD,+    BAM_CBASE_MATCH=BAM_CEQUAL,+    BAM_CBASE_MISMATCH=BAM_CDIFF+};++typedef bam1_t bam_seq_t;++#include "cram/sam_header.h"++#ifdef __cplusplus+extern "C" {+#endif++bam_hdr_t *cram_header_to_bam(SAM_hdr *h);+SAM_hdr *bam_header_to_cram(bam_hdr_t *h);++int bam_construct_seq(bam_seq_t **bp, size_t extra_len,+		      const char *qname, size_t qname_len,+		      int flag,+		      int rname,      // Ref ID+		      int pos,+		      int end,        // aligned start/end coords+		      int mapq,+		      uint32_t ncigar, const uint32_t *cigar,+		      int mrnm,       // Mate Ref ID+		      int mpos,+		      int isize,+		      int len,+		      const char *seq,+		      const char *qual);++#ifdef __cplusplus+}+#endif++#endif /* _CRAM_SAMTOOLS_H_ */
+ htslib-1.3.2/cram/cram_stats.c view
@@ -0,0 +1,448 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdio.h>+#include <errno.h>+#include <assert.h>+#include <stdlib.h>+#include <string.h>+#include <zlib.h>+#include <sys/types.h>+#include <sys/stat.h>+#include <math.h>+#include <ctype.h>++#include "cram/cram.h"+#include "cram/os.h"++cram_stats *cram_stats_create(void) {+    return calloc(1, sizeof(cram_stats));+}++void cram_stats_add(cram_stats *st, int32_t val) {+    st->nsamp++;++    //assert(val >= 0);++    if (val < MAX_STAT_VAL && val >= 0) {+	st->freqs[val]++;+    } else {+	khint_t k;+	int r;++	if (!st->h) {+	    st->h = kh_init(m_i2i);+	}++	k = kh_put(m_i2i, st->h, val, &r);+	if (r == 0)+	    kh_val(st->h, k)++;+	else if (r != -1)+	    kh_val(st->h, k) = 1;+	else+	    ; // FIXME: handle error+    }+}++void cram_stats_del(cram_stats *st, int32_t val) {+    st->nsamp--;++    //assert(val >= 0);++    if (val < MAX_STAT_VAL && val >= 0) {+	st->freqs[val]--;+	assert(st->freqs[val] >= 0);+    } else if (st->h) {+	khint_t k = kh_get(m_i2i, st->h, val);++	if (k != kh_end(st->h)) {+	    if (--kh_val(st->h, k) == 0)+		kh_del(m_i2i, st->h, k);+	} else {+	    fprintf(stderr, "Failed to remove val %d from cram_stats\n", val);+	    st->nsamp++;+	}+    } else {+	fprintf(stderr, "Failed to remove val %d from cram_stats\n", val);+	st->nsamp++;+    }+}++void cram_stats_dump(cram_stats *st) {+    int i;+    fprintf(stderr, "cram_stats:\n");+    for (i = 0; i < MAX_STAT_VAL; i++) {+	if (!st->freqs[i])+	    continue;+	fprintf(stderr, "\t%d\t%d\n", i, st->freqs[i]);+    }+    if (st->h) {+	khint_t k;+	for (k = kh_begin(st->h); k != kh_end(st->h); k++) {+	    if (!kh_exist(st->h, k))+		continue;++	    fprintf(stderr, "\t%d\t%d\n", kh_key(st->h, k), kh_val(st->h, k));+	}+    }+}++#if 1+/* Returns the number of bits set in val; it the highest bit used */+static int nbits(int v) {+    static const int MultiplyDeBruijnBitPosition[32] = {+	1, 10, 2, 11, 14, 22, 3, 30, 12, 15, 17, 19, 23, 26, 4, 31,+	9, 13, 21, 29, 16, 18, 25, 8, 20, 28, 24, 7, 27, 6, 5, 32+    };++    v |= v >> 1; // first up to set all bits 1 after the first 1 */+    v |= v >> 2;+    v |= v >> 4;+    v |= v >> 8;+    v |= v >> 16;++    // DeBruijn magic to find top bit+    return MultiplyDeBruijnBitPosition[(uint32_t)(v * 0x07C4ACDDU) >> 27];+}+#endif++/*+ * Computes entropy from integer frequencies for various encoding methods and+ * picks the best encoding.+ *+ * FIXME: we could reuse some of the code here for the actual encoding+ * parameters too. Eg the best 'k' for SUBEXP or the code lengths for huffman.+ *+ * Returns the best codec to use.+ */+enum cram_encoding cram_stats_encoding(cram_fd *fd, cram_stats *st) {+    enum cram_encoding best_encoding = E_NULL;+    int best_size = INT_MAX, bits;+    int nvals, i, ntot = 0, max_val = 0, min_val = INT_MAX, k;+    int *vals = NULL, *freqs = NULL, vals_alloc = 0, *codes;++    //cram_stats_dump(st);++    /* Count number of unique symbols */+    for (nvals = i = 0; i < MAX_STAT_VAL; i++) {+	if (!st->freqs[i])+	    continue;+	if (nvals >= vals_alloc) {+	    vals_alloc = vals_alloc ? vals_alloc*2 : 1024;+	    vals  = realloc(vals,  vals_alloc * sizeof(int));+	    freqs = realloc(freqs, vals_alloc * sizeof(int));+	    if (!vals || !freqs) {+		if (vals)  free(vals);+		if (freqs) free(freqs);+		return E_HUFFMAN; // Cannot do much else atm+	    }+	}+	vals[nvals] = i;+	freqs[nvals] = st->freqs[i];+	ntot += freqs[nvals];+	if (max_val < i) max_val = i;+	if (min_val > i) min_val = i;+	nvals++;+    }+    if (st->h) {+	khint_t k;+	int i;++	for (k = kh_begin(st->h); k != kh_end(st->h); k++) {+	    if (!kh_exist(st->h, k))+		continue;++	    if (nvals >= vals_alloc) {+		vals_alloc = vals_alloc ? vals_alloc*2 : 1024;+		vals  = realloc(vals,  vals_alloc * sizeof(int));+		freqs = realloc(freqs, vals_alloc * sizeof(int));+		if (!vals || !freqs)+		    return E_HUFFMAN; // Cannot do much else atm+	    }+	    i = kh_key(st->h, k);+	    vals[nvals]=i;+	    freqs[nvals] = kh_val(st->h, k);+	    ntot += freqs[nvals];+	    if (max_val < i) max_val = i;+	    if (min_val > i) min_val = i;+	    nvals++;+	}+    }++    st->nvals = nvals;+    assert(ntot == st->nsamp);++    if (nvals <= 1) {+	free(vals);+	free(freqs);+	return E_HUFFMAN;+    }++    if (fd->verbose > 1)+	fprintf(stderr, "Range = %d..%d, nvals=%d, ntot=%d\n",+		min_val, max_val, nvals, ntot);++    /* Theoretical entropy */+//    if (fd->verbose > 1) {+//	double dbits = 0;+//	for (i = 0; i < nvals; i++) {+//	    dbits += freqs[i] * log((double)freqs[i]/ntot);+//	}+//	dbits /= -log(2);+//	if (fd->verbose > 1)+//	    fprintf(stderr, "Entropy = %f\n", dbits);+//    }++    if (nvals > 1 && ntot > 256) {+#if 0+	/*+	 * CRUDE huffman estimator. Round to closest and round up from 0+	 * to 1 bit.+	 *+	 * With and without ITF8 incase we have a few discrete values but with+	 * large magnitude.+	 *+	 * Note rans0/arith0 and Z_HUFFMAN_ONLY vs internal huffman can be+	 * compared in this way, but order-1 (eg rans1) or maybe LZ77 modes+	 * may detect the correlation of high bytes to low bytes in multi-+	 * byte values. So this predictor breaks down.+	 */+	double dbits = 0;  // entropy + ~huffman+	double dbitsH = 0;+	double dbitsE = 0; // external entropy + ~huffman+	double dbitsEH = 0;+	int F[256] = {0}, n = 0;+	double e = 0; // accumulated error bits+	for (i = 0; i < nvals; i++) {+	    double x; int X;+	    unsigned int v = vals[i];++	    //Better encoding would cope with sign.+	    //v = ABS(vals[i])*2+(vals[i]<0);++	    if (!(v & ~0x7f)) {+		F[v]             += freqs[i], n+=freqs[i];+	    } else if (!(v & ~0x3fff)) {+		F[(v>>8) |0x80] += freqs[i];+		F[ v     &0xff] += freqs[i], n+=2*freqs[i];+	    } else if (!(v & ~0x1fffff)) {+		F[(v>>16)|0xc0] += freqs[i];+		F[(v>>8 )&0xff] += freqs[i];+		F[ v     &0xff] += freqs[i], n+=3*freqs[i];+	    } else if (!(v & ~0x0fffffff)) {+		F[(v>>24)|0xe0] += freqs[i];+		F[(v>>16)&0xff] += freqs[i];+		F[(v>>8 )&0xff] += freqs[i];+		F[ v     &0xff] += freqs[i], n+=4*freqs[i];+	    } else {+		F[(v>>28)|0xf0] += freqs[i];+		F[(v>>20)&0xff] += freqs[i];+		F[(v>>12)&0xff] += freqs[i];+		F[(v>>4 )&0xff] += freqs[i];+		F[ v     &0x0f] += freqs[i], n+=5*freqs[i];+	    }++	    x = -log((double)freqs[i]/ntot)/.69314718055994530941;+	    X = x+0.5;+	    if ((int)(x+((double)e/freqs[i])+.5)>X) {+		X++;+	    } else if ((int)(x+((double)e/freqs[i])+.5)<X) {+		X--;+	    }+	    e-=freqs[i]*(X-x);+	    X += (X==0);++	    //fprintf(stderr, "Val %d = %d x %d (ent %f, %d) e %f\n", i, v, freqs[i], x, X, e);++	    dbits  += freqs[i] * x;+	    dbitsH += freqs[i] * X;+	}++	for (i = 0; i < 256; i++) {+	    if (F[i]) {+		double x = -log((double)F[i]/n)/.69314718055994530941;+		int X = x+0.5;+		X += (X==0);+		dbitsE  += F[i] * x;+		dbitsEH += F[i] * X;++		//fprintf(stderr, "Val %d = %d x %d (e %f, %d)\n", i, i, F[i], x, X);+	    }+	}++	//fprintf(stderr, "CORE Entropy = %f, %f\n", dbits/8, dbitsH/8);+	//fprintf(stderr, "Ext. Entropy = %f, %f\n", dbitsE/8, dbitsEH/8);++	if (dbitsE < 1000 || dbitsE / dbits > 1.1) {+	    //fprintf(stderr, "=> %d < 200 ? E_HUFFMAN : E_BETA\n", nvals);+	    free(vals); free(freqs);+	    return nvals < 200 ? E_HUFFMAN : E_BETA;+	}+#endif+	free(vals); free(freqs);+	return E_EXTERNAL;+    }++    /*+     * Avoid complex stats for now, just do heuristic of HUFFMAN for small+     * alphabets and BETA for anything large.+     */+    free(vals); free(freqs);+    return nvals < 200 ? E_HUFFMAN : E_BETA;+    //return E_HUFFMAN;+    //return E_EXTERNAL;+++    /* We only support huffman now anyway... */+    //free(vals); free(freqs); return E_HUFFMAN;++    /* Beta */+    bits = nbits(max_val - min_val) * ntot;+    if (fd->verbose > 1)+	fprintf(stderr, "BETA    = %d\n", bits);+    if (best_size > bits)+	best_size = bits, best_encoding = E_BETA;++#if 0+    /* Unary */+    if (min_val >= 0) {+	for (bits = i = 0; i < nvals; i++)+	    bits += freqs[i]*(vals[i]+1);+	if (fd->verbose > 1)+	    fprintf(stderr, "UNARY   = %d\n", bits);+	if (best_size > bits)+	    best_size = bits, best_encoding = E_NULL; //E_UNARY;+    }++    /* Gamma */+    for (bits = i = 0; i < nvals; i++)+	bits += ((nbits(vals[i]-min_val+1)-1) + nbits(vals[i]-min_val+1)) * freqs[i];+    if (fd->verbose > 1)+	fprintf(stderr, "GAMMA   = %d\n", bits);+    if (best_size > bits)+	best_size = bits, best_encoding = E_GAMMA;++    /* Subexponential */+    for (k = 0; k < 10; k++) {+	for (bits = i = 0; i < nvals; i++) {+	    if (vals[i]-min_val < (1<<k))+		bits += (1 + k)*freqs[i];+	    else+		bits += (nbits(vals[i]-min_val)*2-k)*freqs[i];+	}++	if (fd->verbose > 1)+	    fprintf(stderr, "SUBEXP%d = %d\n", k, bits);+	if (best_size > bits)+	    best_size = bits, best_encoding = E_SUBEXP;+    }+#endif++    /* byte array len */++    /* byte array stop */++    /* External? Guesswork! */++    /* Huffman */+//    qsort(freqs, nvals, sizeof(freqs[0]), sort_freqs);+//    for (i = 0; i < nvals; i++) {+//	fprintf(stderr, "%d = %d\n", i, freqs[i]);+//	vals[i] = 0;+//    }++    /* Grow freqs to 2*freqs, to store sums */+    /* Vals holds link data */+    freqs = realloc(freqs, 2*nvals*sizeof(*freqs));+    codes = calloc(2*nvals, sizeof(*codes));+    if (!freqs || !codes)+	return E_HUFFMAN; // Cannot do much else atm++    /* Inefficient, use pointers to form chain so we can insert and maintain+     * a sorted list? This is currently O(nvals^2) complexity.+     */+    for (;;) {+	int low1 = INT_MAX, low2 = INT_MAX;+	int ind1 = 0, ind2 = 0;+	for (i = 0; i < nvals; i++) {+	    if (freqs[i] < 0)+		continue;+	    if (low1 > freqs[i]) +		low2 = low1, ind2 = ind1, low1 = freqs[i], ind1 = i;+	    else if (low2 > freqs[i])+		low2 = freqs[i], ind2 = i;+	}+	if (low2 == INT_MAX)+	    break;++	//fprintf(stderr, "Merge ind %d (%d), %d (%d) = %d+%d, => %d=%d\n",+	//	ind1, vals[ind1], ind2, vals[ind2], low1, low2,+	//	nvals, low1+low2);++	freqs[nvals] = low1 + low2;+	codes[ind1] = nvals;+	codes[ind2] = nvals;+	freqs[ind1] *= -1;+	freqs[ind2] *= -1;+	nvals++;+    }+    nvals = nvals/2+1;++    for (i = 0; i < nvals; i++) {+	int code_len = 0;+	for (k = codes[i]; k; k = codes[k])+	    code_len++;+	codes[i] = code_len;+	freqs[i] *= -1;+	//fprintf(stderr, "%d / %d => %d\n", vals[i], freqs[i], codes[i]);+    }++    for (bits = i = 0; i < nvals; i++) {+	bits += freqs[i] * codes[i];+    }+    if (fd->verbose > 1)+	fprintf(stderr, "HUFFMAN = %d\n", bits);+    if (best_size >= bits)+	best_size = bits, best_encoding = E_HUFFMAN;+    free(codes);++    free(vals);+    free(freqs);++    return best_encoding;+}++void cram_stats_free(cram_stats *st) {+    if (st->h)+	kh_destroy(m_i2i, st->h);+    free(st);+}
+ htslib-1.3.2/cram/cram_stats.h view
@@ -0,0 +1,59 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _CRAM_STATS_H_+#define _CRAM_STATS_H_++#ifdef __cplusplus+extern "C" {+#endif++cram_stats *cram_stats_create(void);+void cram_stats_add(cram_stats *st, int32_t val);+void cram_stats_del(cram_stats *st, int32_t val);+void cram_stats_dump(cram_stats *st);+void cram_stats_free(cram_stats *st);++/*+ * Computes entropy from integer frequencies for various encoding methods and+ * picks the best encoding.+ *+ * FIXME: we could reuse some of the code here for the actual encoding+ * parameters too. Eg the best 'k' for SUBEXP or the code lengths for huffman.+ *+ * Returns the best codec to use.+ */+enum cram_encoding cram_stats_encoding(cram_fd *fd, cram_stats *st);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/cram/cram_structs.h view
@@ -0,0 +1,821 @@+/*+Copyright (c) 2012-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _CRAM_STRUCTS_H_+#define _CRAM_STRUCTS_H_++/*+ * Defines in-memory structs for the basic file-format objects in the+ * CRAM format.+ *+ * The basic file format is:+ *     File-def SAM-hdr Container Container ...+ *+ * Container:+ *     Service-block data-block data-block ...+ *+ * Multiple blocks in a container are grouped together as slices,+ * also sometimes referred to as landmarks in the spec.+ */+++#include <stdint.h>++#include "cram/thread_pool.h"+#include "cram/string_alloc.h"+#include "cram/mFILE.h"+#include "htslib/khash.h"++#ifdef __cplusplus+extern "C" {+#endif++// Generic hash-map integer -> integer+KHASH_MAP_INIT_INT(m_i2i, int)++// Generic hash-set integer -> (existance)+KHASH_SET_INIT_INT(s_i2i)++// For brevity+typedef unsigned char uc;++/*+ * A union for the preservation map. Required for khash.+ */+typedef union {+    int i;+    char *p;+} pmap_t;++// Generates static functions here which isn't ideal, but we have no way+// currently to declare the kh_map_t structure here without also declaring a+// duplicate in the .c files due to the nature of the KHASH macros.+KHASH_MAP_INIT_STR(map, pmap_t)++struct hFILE;++#define SEQS_PER_SLICE 10000+#define SLICE_PER_CNT  1++#define CRAM_SUBST_MATRIX "CGTNAGTNACTNACGNACGT"++#define MAX_STAT_VAL 1024+//#define MAX_STAT_VAL 16+typedef struct cram_stats {+    int freqs[MAX_STAT_VAL];+    khash_t(m_i2i) *h;+    int nsamp; // total number of values added+    int nvals; // total number of unique values added+} cram_stats;++/* NB: matches java impl, not the spec */+enum cram_encoding {+    E_NULL               = 0,+    E_EXTERNAL           = 1,+    E_GOLOMB             = 2,+    E_HUFFMAN            = 3,+    E_BYTE_ARRAY_LEN     = 4,+    E_BYTE_ARRAY_STOP    = 5,+    E_BETA               = 6,+    E_SUBEXP             = 7,+    E_GOLOMB_RICE        = 8,+    E_GAMMA              = 9,+    E_NUM_CODECS         = 10, /* Number of codecs, not a real one. */+};++enum cram_external_type {+    E_INT                = 1,+    E_LONG               = 2,+    E_BYTE               = 3,+    E_BYTE_ARRAY         = 4,+    E_BYTE_ARRAY_BLOCK   = 5,+};++/* External IDs used by this implementation (only assumed during writing) */+enum cram_DS_ID {+    DS_CORE   = 0,+    DS_aux    = 1, // aux_blk+    DS_aux_OQ = 2,+    DS_aux_BQ = 3,+    DS_aux_BD = 4,+    DS_aux_BI = 5,+    DS_aux_FZ = 6, // also ZM:B+    DS_aux_oq = 7, // other qualities+    DS_aux_os = 8, // other sequences+    DS_aux_oz = 9, // other strings+    DS_ref,+    DS_RN, // name_blk+    DS_QS, // qual_blk+    DS_IN, // base_blk+    DS_SC, // soft_blk++    DS_BF, // start loop+    DS_CF,+    DS_AP,+    DS_RG,+    DS_MQ,+    DS_NS,+    DS_MF,+    DS_TS,+    DS_NP,+    DS_NF,+    DS_RL,+    DS_FN,+    DS_FC,+    DS_FP,+    DS_DL,+    DS_BA,+    DS_BS,+    DS_TL,+    DS_RI,+    DS_RS,+    DS_PD,+    DS_HC,+    DS_BB,+    DS_QQ,++    DS_TN, // end loop++    DS_RN_len,+    DS_SC_len,+    DS_BB_len,+    DS_QQ_len,++    DS_TC, // CRAM v1.0 tags+    DS_TM, // test+    DS_TV, // test+    +    DS_END,+};++/* "File Definition Structure" */+typedef struct cram_file_def {+    char    magic[4];+    uint8_t major_version;+    uint8_t minor_version;+    char    file_id[20];      // Filename or SHA1 checksum+} cram_file_def;++#define CRAM_MAJOR_VERS(v) ((v) >> 8)+#define CRAM_MINOR_VERS(v) ((v) & 0xff)++struct cram_slice;++enum cram_block_method {+    ERROR    = -1,+    RAW      = 0,+    GZIP     = 1,+    BZIP2    = 2,+    LZMA     = 3,+    RANS     = 4,  // Generic; either order+    RANS0    = 4,+    RANS1    = 10, // Not externalised; stored as RANS (generic)+    GZIP_RLE = 11, // NB: not externalised in CRAM+};++enum cram_content_type {+    CT_ERROR           = -1,+    FILE_HEADER        = 0,+    COMPRESSION_HEADER = 1,+    MAPPED_SLICE       = 2,+    UNMAPPED_SLICE     = 3, // CRAM V1.0 only+    EXTERNAL           = 4,+    CORE               = 5,+};++/* Compression metrics */+typedef struct {+    // number of trials and time to next trial+    int trial;+    int next_trial;++    // aggregate sizes during trials+    int sz_gz_rle;+    int sz_gz_def;+    int sz_rans0;+    int sz_rans1;+    int sz_bzip2;+    int sz_lzma;++    // resultant method from trials+    int method;+    int strat;++    // Revisions of method, to allow culling of continually failing ones.+    int gz_rle_cnt;+    int gz_def_cnt;+    int rans0_cnt;+    int rans1_cnt;+    int bzip2_cnt;+    int lzma_cnt;+    int revised_method;++    double gz_rle_extra;+    double gz_def_extra;+    double rans0_extra;+    double rans1_extra;+    double bzip2_extra;+    double lzma_extra;+} cram_metrics;++/* Block */+typedef struct cram_block {+    enum cram_block_method  method, orig_method;+    enum cram_content_type  content_type;+    int32_t  content_id;+    int32_t  comp_size;+    int32_t  uncomp_size;+    uint32_t crc32;+    int32_t  idx; /* offset into data */+    unsigned char    *data;++    // For bit I/O+    size_t alloc;+    size_t byte;+    int bit;+} cram_block;++struct cram_codec; /* defined in cram_codecs.h */+struct cram_map;++#define CRAM_MAP_HASH 32+#define CRAM_MAP(a,b) (((a)*3+(b))&(CRAM_MAP_HASH-1))++/* Compression header block */+typedef struct cram_block_compression_hdr {+    int32_t ref_seq_id;+    int32_t ref_seq_start;+    int32_t ref_seq_span;+    int32_t num_records;+    int32_t num_landmarks;+    int32_t *landmark;++    /* Flags from preservation map */+    int mapped_qs_included;+    int unmapped_qs_included;+    int unmapped_placed;+    int qs_included;+    int read_names_included;+    int AP_delta;+    // indexed by ref-base and subst. code+    char substitution_matrix[5][4];++    // TD Dictionary as a concatenated block+    cram_block *TD_blk;          // Tag Dictionary+    int nTL;		         // number of TL entries in TD+    unsigned char **TL;          // array of size nTL, pointer into TD_blk.+    khash_t(m_s2i) *TD_hash;     // Keyed on TD strings, map to TL[] indices+    string_alloc_t *TD_keys;     // Pooled keys for TD hash.+    +    khash_t(map) *preservation_map;+    struct cram_map *rec_encoding_map[CRAM_MAP_HASH];+    struct cram_map *tag_encoding_map[CRAM_MAP_HASH];++    struct cram_codec *codecs[DS_END];++    char *uncomp; // A single block of uncompressed data+    size_t uncomp_size, uncomp_alloc;++    unsigned int data_series; // See cram_fields enum below+} cram_block_compression_hdr;++typedef struct cram_map {+    int key;    /* 0xe0 + 3 bytes */+    enum cram_encoding encoding;+    int offset; /* Offset into a single block of memory */+    int size;   /* Size */+    struct cram_codec *codec;+    struct cram_map *next; // for noddy internal hash+} cram_map;++/* Mapped or unmapped slice header block */+typedef struct cram_block_slice_hdr {+    enum cram_content_type content_type;+    int32_t ref_seq_id;     /* if content_type == MAPPED_SLICE */+    int32_t ref_seq_start;  /* if content_type == MAPPED_SLICE */+    int32_t ref_seq_span;   /* if content_type == MAPPED_SLICE */+    int32_t num_records;+    int64_t record_counter;+    int32_t num_blocks;+    int32_t num_content_ids;+    int32_t *block_content_ids;+    int32_t ref_base_id;    /* if content_type == MAPPED_SLICE */+    unsigned char md5[16];+} cram_block_slice_hdr;++struct ref_entry;++/*+ * Container.+ *+ * Conceptually a container is split into slices, and slices into blocks.+ * However on disk it's just a list of blocks and we need to query the+ * block types to identify the start/end points of the slices.+ *+ * OR... are landmarks the start/end points of slices?+ */+typedef struct cram_container {+    int32_t  length;+    int32_t  ref_seq_id;+    int32_t  ref_seq_start;+    int32_t  ref_seq_span;+    int64_t  record_counter;+    int64_t  num_bases;+    int32_t  num_records;+    int32_t  num_blocks;+    int32_t  num_landmarks;+    int32_t *landmark;++    /* Size of container header above */+    size_t   offset;+    +    /* Compression header is always the first block? */+    cram_block_compression_hdr *comp_hdr;+    cram_block *comp_hdr_block;++    /* For construction purposes */+    int max_slice, curr_slice;   // maximum number of slices+    int max_rec, curr_rec;       // current and max recs per slice+    int max_c_rec, curr_c_rec;   // current and max recs per container+    int slice_rec;               // rec no. for start of this slice+    int curr_ref;                // current ref ID. -2 for no previous+    int last_pos;                // last record position+    struct cram_slice **slices, *slice;+    int pos_sorted;              // boolean, 1=>position sorted data+    int max_apos;                // maximum position, used if pos_sorted==0+    int last_slice;              // number of reads in last slice (0 for 1st)+    int multi_seq;               // true if packing multi seqs per cont/slice+    int unsorted;		 // true is AP_delta is 0.++    /* Copied from fd before encoding, to allow multi-threading */+    int ref_start, first_base, last_base, ref_id, ref_end;+    char *ref;+    //struct ref_entry *ref;++    /* For multi-threading */+    bam_seq_t **bams;++    /* Statistics for encoding */+    cram_stats *stats[DS_END];++    khash_t(s_i2i) *tags_used; // set of tag types in use, for tag encoding map+    int *refs_used;       // array of frequency of ref seq IDs++    uint32_t crc32;       // CRC32+} cram_container;++/*+ * A single cram record+ */+typedef struct cram_record {+    struct cram_slice *s; // Filled out by cram_decode only++    int32_t ref_id;       // fixed for all recs in slice?+    int32_t flags;        // BF+    int32_t cram_flags;   // CF+    int32_t len;          // RL+    int32_t apos;         // AP+    int32_t rg;           // RG+    int32_t name;         // RN; idx to s->names_blk+    int32_t name_len;+    int32_t mate_line;    // index to another cram_record+    int32_t mate_ref_id;+    int32_t mate_pos;     // NP+    int32_t tlen;         // TS++    // Auxiliary data+    int32_t ntags;        // TC+    int32_t aux;          // idx to s->aux_blk+    int32_t aux_size;     // total size of packed ntags in aux_blk+#ifndef TN_external+    int32_t TN_idx;       // TN; idx to s->TN;+#else+    int32_t tn;           // idx to s->tn_blk+#endif+    int     TL;++    int32_t seq;          // idx to s->seqs_blk+    int32_t qual;         // idx to s->qual_blk+    int32_t cigar;        // idx to s->cigar+    int32_t ncigar;+    int32_t aend;         // alignment end+    int32_t mqual;        // MQ++    int32_t feature;      // idx to s->feature+    int32_t nfeature;     // number of features+    int32_t mate_flags;   // MF+} cram_record;++// Accessor macros as an analogue of the bam ones+#define cram_qname(c)    (&(c)->s->name_blk->data[(c)->name])+#define cram_seq(c)      (&(c)->s->seqs_blk->data[(c)->seq])+#define cram_qual(c)     (&(c)->s->qual_blk->data[(c)->qual])+#define cram_aux(c)      (&(c)->s->aux_blk->data[(c)->aux])+#define cram_seqi(c,i)   (cram_seq((c))[(i)])+#define cram_name_len(c) ((c)->name_len)+#define cram_strand(c)   (((c)->flags & BAM_FREVERSE) != 0)+#define cram_mstrand(c)  (((c)->flags & BAM_FMREVERSE) != 0)+#define cram_cigar(c)    (&((cr)->s->cigar)[(c)->cigar])++/*+ * A feature is a base difference, used for the sequence reference encoding.+ * (We generate these internally when writing CRAM.)+ */+typedef struct cram_feature {+    union {+	struct {+	    int pos;+	    int code;+	    int base;    // substitution code+	} X;+	struct {+	    int pos;+	    int code;+	    int base;    // actual base & qual+	    int qual;+	} B;+	struct {+	    int pos;+	    int code;+	    int seq_idx; // index to s->seqs_blk+	    int len;+	} b;+	struct {+	    int pos;+	    int code;+	    int qual;+	} Q;+	struct {+	    int pos;+	    int code;+	    int len;+	    int seq_idx; // soft-clip multiple bases+	} S;+	struct {+	    int pos;+	    int code;+	    int len;+	    int seq_idx; // insertion multiple bases+	} I;+	struct {+	    int pos;+	    int code;+	    int base; // insertion single base+	} i;+	struct {+	    int pos;+	    int code;+	    int len;+	} D;+	struct {+	    int pos;+	    int code;+	    int len;+	} N;+	struct {+	    int pos;+	    int code;+	    int len;+	} P;+	struct {+	    int pos;+	    int code;+	    int len;+	} H;+    };+} cram_feature;++/*+ * A slice is really just a set of blocks, but it+ * is the logical unit for decoding a number of+ * sequences.+ */+typedef struct cram_slice {+    cram_block_slice_hdr *hdr;+    cram_block *hdr_block;+    cram_block **block;+    cram_block **block_by_id;++    /* State used during encoding/decoding */+    int last_apos, max_apos;++    /* Array of decoded cram records */+    cram_record *crecs;++    /* An dynamically growing buffers for data pointed+     * to by crecs[] array.+     */+    uint32_t  *cigar;+    uint32_t   cigar_alloc;+    uint32_t   ncigar;++    cram_feature *features;+    int           nfeatures;+    int           afeatures; // allocated size of features++#ifndef TN_external+    // TN field (Tag Name)+    uint32_t      *TN;+    int           nTN, aTN;  // used and allocated size for TN[]+#else+    cram_block *tn_blk;+    int tn_id;+#endif++    // For variable sized elements which are always external blocks.+    cram_block *name_blk;+    cram_block *seqs_blk;+    cram_block *qual_blk;+    cram_block *base_blk;+    cram_block *soft_blk;+    cram_block *aux_blk;+    cram_block *aux_OQ_blk;+    cram_block *aux_BQ_blk;+    cram_block *aux_BD_blk;+    cram_block *aux_BI_blk;+    cram_block *aux_FZ_blk;+    cram_block *aux_oq_blk;+    cram_block *aux_os_blk;+    cram_block *aux_oz_blk;++    string_alloc_t *pair_keys; // Pooled keys for pair hash.+    khash_t(m_s2i) *pair[2];   // for identifying read-pairs in this slice.++    char *ref;               // slice of current reference+    int ref_start;           // start position of current reference;+    int ref_end;             // end position of current reference;+    int ref_id;+} cram_slice;++/*-----------------------------------------------------------------------------+ * Consider moving reference handling to cram_refs.[ch]+ */+// from fa.fai / samtools faidx files+typedef struct ref_entry {+    char *name;+    char *fn;+    int64_t length;+    int64_t offset;+    int bases_per_line;+    int line_length;+    int64_t count;	   // for shared references so we know to dealloc seq+    char *seq;+    mFILE *mf;+    int is_md5;            // Reference comes from a raw seq found by MD5+} ref_entry;++KHASH_MAP_INIT_STR(refs, ref_entry*)++// References structure.+typedef struct {+    string_alloc_t *pool;  // String pool for holding filenames and SN vals++    khash_t(refs) *h_meta; // ref_entry*, index by name+    ref_entry **ref_id;    // ref_entry*, index by ID+    int nref;              // number of ref_entry++    char *fn;              // current file opened+    BGZF *fp;              // and the hFILE* to go with it.++    int count;             // how many cram_fd sharing this refs struct++    pthread_mutex_t lock;  // Mutex for multi-threaded updating+    ref_entry *last;       // Last queried sequence+    int last_id;           // Used in cram_ref_decr_locked to delay free+} refs_t;++/*-----------------------------------------------------------------------------+ * CRAM index+ *+ * Detect format by number of entries per line.+ * 5 => 1.0 (refid, start, nseq, C offset, slice)+ * 6 => 1.1 (refid, start, span, C offset, S offset, S size)+ *+ * Indices are stored in a nested containment list, which is trivial to set+ * up as the indices are on sorted data so we're appending to the nclist+ * in sorted order. Basically if a slice entirely fits within a previous+ * slice then we append to that slices list. This is done recursively.+ *+ * Lists are sorted on two dimensions: ref id + slice coords.+ */+typedef struct cram_index {+    int nslice, nalloc;   // total number of slices+    struct cram_index *e; // array of size nslice++    int     refid;  // 1.0                 1.1+    int     start;  // 1.0                 1.1+    int     end;    //                     1.1+    int     nseq;   // 1.0 - undocumented+    int     slice;  // 1.0 landmark index, 1.1 landmark value+    int     len;    //                     1.1 - size of slice in bytes+    int64_t offset; // 1.0                 1.1+} cram_index;++typedef struct {+    int refid;+    int start;+    int end;+} cram_range;++/*-----------------------------------------------------------------------------+ */+/* CRAM File handle */++typedef struct spare_bams {+    bam_seq_t **bams;+    struct spare_bams *next;+} spare_bams;++typedef struct cram_fd {+    struct hFILE  *fp;+    int            mode;     // 'r' or 'w'+    int            version;+    cram_file_def *file_def;+    SAM_hdr       *header;++    char          *prefix;+    int64_t        record_counter;+    int            err;++    // Most recent compression header decoded+    //cram_block_compression_hdr *comp_hdr;+    //cram_block_slice_hdr       *slice_hdr;++    // Current container being processed.+    cram_container *ctr;++    // positions for encoding or decoding+    int first_base, last_base;++    // cached reference portion+    refs_t *refs;              // ref meta-data structure+    char *ref, *ref_free;      // current portion held in memory+    int   ref_id;+    int   ref_start;+    int   ref_end;+    char *ref_fn;   // reference fasta filename++    // compression level and metrics+    int level;+    cram_metrics *m[DS_END];++    // options+    int decode_md; // Whether to export MD and NM tags+    int verbose;+    int seqs_per_slice;+    int slices_per_container;+    int embed_ref;+    int no_ref;+    int ignore_md5;+    int use_bz2;+    int use_rans;+    int use_lzma;+    int shared_ref;+    unsigned int required_fields;+    cram_range range;++    // lookup tables, stored here so we can be trivially multi-threaded+    unsigned int bam_flag_swap[0x1000]; // cram -> bam flags+    unsigned int cram_flag_swap[0x1000];// bam -> cram flags+    unsigned char L1[256];              // ACGT{*} ->0123{4}+    unsigned char L2[256];              // ACGTN{*}->01234{5}+    char cram_sub_matrix[32][32];	// base substituion codes++    int         index_sz;+    cram_index *index;                  // array, sizeof index_sz+    off_t first_container;+    int eof;+    int last_slice;                     // number of recs encoded in last slice+    int multi_seq;+    int unsorted;+    int empty_container; 		// Marker for EOF block+    +    // thread pool+    int own_pool;+    t_pool *pool;+    t_results_queue *rqueue;+    pthread_mutex_t metrics_lock;+    pthread_mutex_t ref_lock;+    spare_bams *bl;+    pthread_mutex_t bam_list_lock;+    void *job_pending;+    int ooc;                            // out of containers.+} cram_fd;++// Translation of required fields to cram data series+enum cram_fields {+    CRAM_BF = 0x00000001,+    CRAM_AP = 0x00000002,+    CRAM_FP = 0x00000004,+    CRAM_RL = 0x00000008,+    CRAM_DL = 0x00000010,+    CRAM_NF = 0x00000020,+    CRAM_BA = 0x00000040,+    CRAM_QS = 0x00000080,+    CRAM_FC = 0x00000100,+    CRAM_FN = 0x00000200,+    CRAM_BS = 0x00000400,+    CRAM_IN = 0x00000800,+    CRAM_RG = 0x00001000,+    CRAM_MQ = 0x00002000,+    CRAM_TL = 0x00004000,+    CRAM_RN = 0x00008000,+    CRAM_NS = 0x00010000,+    CRAM_NP = 0x00020000,+    CRAM_TS = 0x00040000,+    CRAM_MF = 0x00080000,+    CRAM_CF = 0x00100000,+    CRAM_RI = 0x00200000,+    CRAM_RS = 0x00400000,+    CRAM_PD = 0x00800000,+    CRAM_HC = 0x01000000,+    CRAM_SC = 0x02000000,+    CRAM_BB = 0x04000000,+    CRAM_BB_len = 0x08000000,+    CRAM_QQ = 0x10000000,+    CRAM_QQ_len = 0x20000000,+    CRAM_aux= 0x40000000,+    CRAM_ALL= 0x7fffffff,+};++// A CIGAR opcode, but not necessarily the implications of it. Eg FC/FP may+// encode a base difference, but we don't need to know what it is for CIGAR.+// If we have a soft-clip or insertion, we do need SC/IN though to know how+// long that array is.+#define CRAM_CIGAR (CRAM_FN | CRAM_FP | CRAM_FC | CRAM_DL | CRAM_IN | \+		    CRAM_SC | CRAM_HC | CRAM_PD | CRAM_RS | CRAM_RL | CRAM_BF)++#define CRAM_SEQ (CRAM_CIGAR | CRAM_BA | CRAM_BS | \+		  CRAM_RL    | CRAM_AP | CRAM_BB)++#define CRAM_QUAL (CRAM_CIGAR | CRAM_RL | CRAM_AP | CRAM_QS | CRAM_QQ)++/* BF bitfields */+/* Corrected in 1.1. Use bam_flag_swap[bf] and BAM_* macros for 1.0 & 1.1 */+#define CRAM_FPAIRED      256+#define CRAM_FPROPER_PAIR 128+#define CRAM_FUNMAP        64+#define CRAM_FREVERSE      32+#define CRAM_FREAD1        16+#define CRAM_FREAD2         8+#define CRAM_FSECONDARY     4+#define CRAM_FQCFAIL        2+#define CRAM_FDUP           1++#define DS_aux_S "\001"+#define DS_aux_OQ_S "\002"+#define DS_aux_BQ_S "\003"+#define DS_aux_BD_S "\004"+#define DS_aux_BI_S "\005"+#define DS_aux_FZ_S "\006"+#define DS_aux_oq_S "\007"+#define DS_aux_os_S "\010"+#define DS_aux_oz_S "\011"++#define CRAM_M_REVERSE  1+#define CRAM_M_UNMAP    2+++/* CF bitfields */+#define CRAM_FLAG_PRESERVE_QUAL_SCORES (1<<0)+#define CRAM_FLAG_DETACHED             (1<<1)+#define CRAM_FLAG_MATE_DOWNSTREAM      (1<<2)+#define CRAM_FLAG_NO_SEQ               (1<<3)+#define CRAM_FLAG_MASK                 ((1<<4)-1)++/* Internal only */+#define CRAM_FLAG_STATS_ADDED          (1<<30)+++#ifdef __cplusplus+}+#endif++#endif /* _CRAM_STRUCTS_H_ */
+ htslib-1.3.2/cram/files.c view
@@ -0,0 +1,74 @@+/*+Copyright (c) 1994, 1996-1997, 2000, 2003 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1 Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2 Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++#include <config.h>++#include "cram/misc.h"++#include <sys/types.h>+#include <sys/stat.h>+/* Alliant's Concentrix <sys/stat.h> is hugely deficient */+/* Define things we require in this program              */+/* Methinks S_IFMT and S_IFDIR aren't defined in POSIX   */+#ifndef S_ISDIR+#define S_ISDIR(m)      (((m)&S_IFMT) == S_IFDIR)+#endif /*!S_ISDIR*/+#ifndef S_ISREG+#define S_ISREG(m)      (((m)&S_IFMT) == S_IFREG)+#endif /*!S_ISREG*/++int is_directory(char * fn)+{+    struct stat buf;+    if ( stat(fn,&buf) ) return 0;+    return S_ISDIR(buf.st_mode);+}++int is_file(char * fn)+{+    struct stat buf;+    if ( stat(fn,&buf) ) return 0;+    return S_ISREG(buf.st_mode);+}++int file_exists(char * fn)+{+    struct stat buf;+    return ( stat(fn,&buf) == 0);+}++int file_size(char * fn)+{+    struct stat buf;+    if ( stat(fn,&buf) != 0) return 0;+    return buf.st_size;+}+
+ htslib-1.3.2/cram/mFILE.c view
@@ -0,0 +1,694 @@+/*+Copyright (c) 2005-2006, 2008-2009, 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdio.h>+#include <stdlib.h>+#include <errno.h>+#include <string.h>+#include <sys/types.h>+#include <sys/stat.h>+#include <fcntl.h>+#include <unistd.h>+#include <stdarg.h>++#include "cram/os.h"+#include "cram/mFILE.h"+#include "cram/vlen.h"++#ifdef HAVE_MMAP+#include <sys/mman.h>+#endif++/*+ * This file contains memory-based versions of the most commonly used+ * (by io_lib) stdio functions.+ *+ * Actual file IO takes place either on opening or closing an mFILE.+ *+ * Coupled to this are a bunch of rather scary macros which can be obtained+ * by including stdio_hack.h. It is recommended though that you use mFILE.h+ * instead and replace fopen with mfopen (etc). This is more or less+ * mandatory if you wish to use both FILE and mFILE structs in a single file.+ */++static mFILE *m_channel[3];  /* stdin, stdout and stderr fakes */++/*+ * Reads the entirety of fp into memory. If 'fn' exists it is the filename+ * associated with fp. This will be used for more optimal reading (via a+ * stat to identify the size and a single read). Otherwise we use successive+ * reads until EOF.+ *+ * Returns a malloced buffer on success of length *size+ *         NULL on failure+ */+static char *mfload(FILE *fp, const char *fn, size_t *size, int binary) {+    struct stat sb;+    char *data = NULL;+    size_t allocated = 0, used = 0;+    int bufsize = 8192;++#ifdef _WIN32+    if (binary)+	_setmode(_fileno(fp), _O_BINARY);+    else +	_setmode(_fileno(fp), _O_TEXT);+#endif++    if (fn && -1 != stat(fn, &sb)) {+	data = malloc(allocated = sb.st_size);+	bufsize = sb.st_size;+    } else {+	fn = NULL;+    }++    do {+	size_t len;+	if (used + bufsize > allocated) {+	    allocated += bufsize;+	    data = realloc(data, allocated);+	}+	len = fread(data + used, 1, allocated - used, fp);+	if (len > 0)+	    used += len;+    } while (!feof(fp) && (fn == NULL || used < sb.st_size));++    *size = used;++    return data;+}+++#ifdef HAVE_MMAP+/*+ * mmaps in the file, but only for reading currently.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int mfmmap(mFILE *mf, FILE *fp, const char *fn) {+    struct stat sb;++    if (stat(fn, &sb) != 0)+	return -1;++    mf->size = sb.st_size;+    mf->data = mmap(NULL, mf->size, PROT_READ, MAP_SHARED,+		    fileno(fp), 0);++    if (!mf->data)+	return -1;++    mf->alloced = 0;+    return 0;+}+#endif+++/*+ * Creates and returns m_channel[0].+ * We initialise this on the first attempted read, which then slurps in+ * all of stdin until EOF is met.+ */+mFILE *mstdin(void) {+    if (m_channel[0])+	return m_channel[0];++    m_channel[0] = mfcreate(NULL, 0);+    if (NULL == m_channel[0]) return NULL;+    m_channel[0]->fp = stdin;+    return m_channel[0];+}++static void init_mstdin(void) {+    static int done_stdin = 0;+    if (done_stdin)+	return;++    m_channel[0]->data = mfload(stdin, NULL, &m_channel[0]->size, 1);+    m_channel[0]->mode = MF_READ;+    done_stdin = 1;+}++/*+ * Creates and returns m_channel[1]. This is the fake for stdout. It starts as+ * an empty buffer which is physically written out only when mfflush or+ * mfclose are called.+ */+mFILE *mstdout(void) {+    if (m_channel[1])+	return m_channel[1];++    m_channel[1] = mfcreate(NULL, 0);+    if (NULL == m_channel[1]) return NULL;+    m_channel[1]->fp = stdout;+    m_channel[1]->mode = MF_WRITE;+    return m_channel[1];+}++/*+ * Stderr as an mFILE.+ * The code handles stderr by returning m_channel[2], but also checking+ * for stderr in fprintf (the common usage of it) to auto-flush.+ */+mFILE *mstderr(void) {+    if (m_channel[2])+	return m_channel[2];++    m_channel[2] = mfcreate(NULL, 0);+    if (NULL == m_channel[2]) return NULL;+    m_channel[2]->fp = stderr;+    m_channel[2]->mode = MF_WRITE;+    return m_channel[2];+}+++/*+ * For creating existing mFILE pointers directly from memory buffers.+ */+mFILE *mfcreate(char *data, int size) {+    mFILE *mf = (mFILE *)malloc(sizeof(*mf));+    if (NULL == mf) return NULL;+    mf->fp = NULL;+    mf->data = data;+    mf->alloced = size;+    mf->size = size;+    mf->eof = 0;+    mf->offset = 0;+    mf->flush_pos = 0;+    mf->mode = MF_READ | MF_WRITE;+    return mf;+}++/*+ * Recreate an existing mFILE to house new data/size.+ * It also rewinds the file.+ */+void mfrecreate(mFILE *mf, char *data, int size) {+    if (mf->data)+	free(mf->data);+    mf->data = data;+    mf->size = size;+    mf->alloced = size;+    mf->eof = 0;+    mf->offset = 0;+    mf->flush_pos = 0;+}+++/*+ * Creates a new mFILE to contain the contents of the FILE pointer.+ * This mFILE is purely for in-memory operations and has no links to the+ * original FILE* it came from. It also doesn't close the FILE pointer.+ * Consider using mfreopen() is you need different behaviour.+ *+ * Returns mFILE * on success+ *         NULL on failure.+ */ +mFILE *mfcreate_from(const char *path, const char *mode_str, FILE *fp) {+   mFILE *mf; ++    /* Open using mfreopen() */+    if (NULL == (mf = mfreopen(path, mode_str, fp)))+	return NULL;+    +    /* Disassociate from the input stream */+    mf->fp = NULL;++    return mf;+}++/*+ * Converts a FILE * to an mFILE *.+ * Use this for wrapper functions to turn external prototypes requring+ * FILE * as an argument into internal code using mFILE *.+ */+mFILE *mfreopen(const char *path, const char *mode_str, FILE *fp) {+    mFILE *mf;+    int r = 0, w = 0, a = 0, b = 0, x = 0, mode = 0;++    /* Parse mode:+     * r = read file contents (if truncated => don't read)+     * w = write on close+     * a = position at end of buffer+     * x = position at same location as the original fp, don't seek on flush+     * + = for update (read and write)+     * m = mmap (read only)+     */+    if (strchr(mode_str, 'r'))+	r = 1, mode |= MF_READ;+    if (strchr(mode_str, 'w'))+	w = 1, mode |= MF_WRITE | MF_TRUNC;+    if (strchr(mode_str, 'a'))+	w = a = 1, mode |= MF_WRITE | MF_APPEND;+    if (strchr(mode_str, 'b'))+	b = 1, mode |= MF_BINARY;+    if (strchr(mode_str, 'x'))+	x = 1;+    if (strchr(mode_str, '+')) {+        w = 1, mode |= MF_READ | MF_WRITE;+	if (a)+	    r = 1;+    }+#ifdef HAVE_MMAP+    if (strchr(mode_str, 'm'))+	if (!w) mode |= MF_MMAP;+#endif++    if (r) {+	mf = mfcreate(NULL, 0);+	if (NULL == mf) return NULL;+	if (!(mode & MF_TRUNC)) {+#ifdef HAVE_MMAP+	    if (mode & MF_MMAP) {+		if (mfmmap(mf, fp, path) == -1) {+		    mf->data = NULL;+		    mode &= ~MF_MMAP;+		}+	    }+#endif+	    if (!mf->data) {+		mf->data = mfload(fp, path, &mf->size, b);+		mf->alloced = mf->size;+		if (!a)+		    fseek(fp, 0, SEEK_SET);+	    }+	}+    } else if (w) {+	/* Write - initialise the data structures */+	mf = mfcreate(NULL, 0);+	if (NULL == mf) return NULL;+    } else {+        fprintf(stderr, "Must specify either r, w or a for mode\n");+        return NULL;+    }+    mf->fp = fp;+    mf->mode = mode;++    if (x) {+	mf->mode |= MF_MODEX;+    }+    +    if (a) {+	mf->flush_pos = mf->size;+	fseek(fp, 0, SEEK_END);+    }++    return mf;+}++/*+ * Opens a file. If we have read access (r or a+) then it loads the entire+ * file into memory. If We have write access then the pathname is stored.+ * We do not actually write until an mfclose, which then checks this pathname.+ */+mFILE *mfopen(const char *path, const char *mode) {+    FILE *fp;++    if (NULL == (fp = fopen(path, mode)))+	return NULL;+    return mfreopen(path, mode, fp);+}++/*+ * Closes an mFILE. If the filename is known (implying write access) then this+ * also writes the data to disk.+ *+ * Stdout is handled by calling mfflush which writes to stdout if appropriate.+ */+int mfclose(mFILE *mf) {+    if (!mf)+	return -1;++    mfflush(mf);++#ifdef HAVE_MMAP+    if ((mf->mode & MF_MMAP) && mf->data) {+	/* Mmaped */+	munmap(mf->data, mf->size);+	mf->data = NULL;+    }+#endif++    if (mf->fp)+	fclose(mf->fp);++    mfdestroy(mf);++    return 0;+}++/*+ * Closes the file pointer contained within the mFILE without destroying+ * the in-memory data.+ *+ * Attempting to do this on an mmaped buffer is an error.+ */+int mfdetach(mFILE *mf) {+    if (!mf)+	return -1;++    mfflush(mf);+    if (mf->mode & MF_MMAP)+	return -1;++    if (mf->fp) {+	fclose(mf->fp);+	mf->fp = NULL;+    }++    return 0;+}++/*+ * Destroys an mFILE structure but does not flush or close it+ */+int mfdestroy(mFILE *mf) {+    if (!mf)+	return -1;++    if (mf->data)+	free(mf->data);+    free(mf);++    return 0;+}++/*+ * Steals that data out of an mFILE.  The mFILE itself will be closed.+ * It is up to the caller to free the stolen buffer.  If size_out is+ * not NULL, mf->size will be stored in it.+ * This is more-or-less the opposite of mfcreate().+ *+ * Note, we cannot steal the allocated buffer from an mmaped mFILE.+ */++void *mfsteal(mFILE *mf, size_t *size_out) {+    void *data;++    if (!mf) return NULL;++    data = mf->data;+    +    if (NULL != size_out) *size_out = mf->size;++    if (mfdetach(mf) != 0)+	return NULL;++    mf->data = NULL;+    mfdestroy(mf);++    return data;+}++/*+ * Seek/tell functions. Nothing more than updating and reporting an+ * in-memory index. NB we can seek on stdin or stdout even provided we+ * haven't been flushing.+ */+int mfseek(mFILE *mf, long offset, int whence) {+    switch (whence) {+    case SEEK_SET:+	mf->offset = offset;+	break;+    case SEEK_CUR:+	mf->offset += offset;+	break;+    case SEEK_END:+	mf->offset = mf->size + offset;+	break;+    default:+	errno = EINVAL;+	return -1;+    }++    mf->eof = 0;+    return 0;+}++long mftell(mFILE *mf) {+    return mf->offset;+}++void mrewind(mFILE *mf) {+    mf->offset = 0;+    mf->eof = 0;+}++/*+ * mftruncate is not directly a translation of ftruncate as the latter+ * takes a file descriptor instead of a FILE *. It performs the analogous+ * role though.+ *+ * If offset is -1 then the file is truncated to be the current file+ * offset.+ */+void mftruncate(mFILE *mf, long offset) {+    mf->size = offset != -1 ? offset : mf->offset;+    if (mf->offset > mf->size)+	mf->offset = mf->size;+}++int mfeof(mFILE *mf) {+    return mf->eof;+}++/*+ * mFILE read/write functions. Basically these turn fread/fwrite syntax+ * into memcpy statements, with appropriate memory handling for writing.+ */+size_t mfread(void *ptr, size_t size, size_t nmemb, mFILE *mf) {+    size_t len;+    char *cptr = (char *)ptr;+    +    if (mf == m_channel[0]) init_mstdin();++    if (mf->size <= mf->offset)+	return 0;++    len = size * nmemb <= mf->size - mf->offset+	? size * nmemb+	: mf->size - mf->offset;+    if (!size)+	return 0;++    memcpy(cptr, &mf->data[mf->offset], len);+    mf->offset += len;+    +    if (len != size * nmemb) {+	mf->eof = 1;+    }++    return len / size;+}++size_t mfwrite(void *ptr, size_t size, size_t nmemb, mFILE *mf) {+    if (!(mf->mode & MF_WRITE))+	return 0;++    /* Append mode => forced all writes to end of file */+    if (mf->mode & MF_APPEND)+	mf->offset = mf->size;++    /* Make sure we have enough room */+    while (size * nmemb + mf->offset > mf->alloced) {+	size_t new_alloced = mf->alloced ? mf->alloced * 2 : 1024;+	void * new_data = realloc(mf->data, new_alloced);+	if (NULL == new_data) return 0;+	mf->alloced = new_alloced;+	mf->data    = new_data;+    }++    /* Record where we need to reflush from */+    if (mf->offset < mf->flush_pos)+	mf->flush_pos = mf->offset;++    /* Copy the data over */+    memcpy(&mf->data[mf->offset], ptr, size * nmemb);+    mf->offset += size * nmemb;+    if (mf->size < mf->offset)+	mf->size = mf->offset;++    return nmemb;+}++int mfgetc(mFILE *mf) {+    if (mf == m_channel[0]) init_mstdin();+    if (mf->offset < mf->size) {+	return (unsigned char)mf->data[mf->offset++];+    }++    mf->eof = 1;+    return -1;+}++int mungetc(int c, mFILE *mf) {+    if (mf->offset > 0) {+	mf->data[--mf->offset] = c;+	return c;+    }+    +    mf->eof = 1;+    return -1;+}++char *mfgets(char *s, int size, mFILE *mf) {+    int i;++    if (mf == m_channel[0]) init_mstdin();+    *s = 0;+    for (i = 0; i < size-1;) {+	if (mf->offset < mf->size) {+	    s[i] = mf->data[mf->offset++];+	    if (s[i++] == '\n')+		break;+	} else {+	    mf->eof = 1;+	    break;+	}+    }++    s[i] = 0;+    return i ? s : NULL;+}++/*+ * Flushes an mFILE. If this is a real open of a file in write mode then+ * mFILE->fp will be set. We then write out any new data in mFILE since the+ * last flush. We cannot tell what may have been modified as we don't keep+ * track of that, so we typically rewrite out the entire file contents between+ * the last flush_pos and the end of file.+ *+ * For stderr/stdout we also reset the offsets so we cannot modify things+ * we've already output.+ */+int mfflush(mFILE *mf) {+    if (!mf->fp)+	return 0;++    /* FIXME: only do this when opened in write mode */+    if (mf == m_channel[1] || mf == m_channel[2]) {+	if (mf->flush_pos < mf->size) {+	    size_t bytes = mf->size - mf->flush_pos;+	    if (fwrite(mf->data + mf->flush_pos, 1, bytes, mf->fp) < bytes)+		return -1;+	    if (0 != fflush(mf->fp))+		return -1;+	}++	/* Stdout & stderr are non-seekable streams so throw away the data */+	mf->offset = mf->size = mf->flush_pos = 0;+    }++    /* only flush when opened in write mode */+    if (mf->mode & MF_WRITE) {+	if (mf->flush_pos < mf->size) {+	    size_t bytes = mf->size - mf->flush_pos;+	    if (!(mf->mode & MF_MODEX)) {+		fseek(mf->fp, mf->flush_pos, SEEK_SET);+	    }+	    if (fwrite(mf->data + mf->flush_pos, 1, bytes, mf->fp) < bytes)+		return -1;+	    if (0 != fflush(mf->fp))+		return -1;+	}+	if (ftell(mf->fp) != -1 &&+	    ftruncate(fileno(mf->fp), ftell(mf->fp)) == -1)+		return -1;+	mf->flush_pos = mf->size;+    }++    return 0;+}++/*+ * A wrapper around vsprintf() to write to an mFILE. This also uses vflen() to+ * estimate how many additional bytes of storage will be required for the+ * vsprintf to work.+ */+int mfprintf(mFILE *mf, char *fmt, ...) {+    int ret;+    size_t est_length;+    va_list args;++    va_start(args, fmt);+    est_length = vflen(fmt, args);+    va_end(args);+    while (est_length + mf->offset > mf->alloced) {+	size_t new_alloced = mf->alloced ? mf->alloced * 2 : 1024;+	void * new_data    = realloc(mf->data, new_alloced);+	if (NULL == new_data) return -1;+	mf->alloced = new_alloced;+	mf->data    = new_data;+    }++    va_start(args, fmt);+    ret = vsprintf(&mf->data[mf->offset], fmt, args);+    va_end(args);++    if (ret > 0) {+	mf->offset += ret;+	if (mf->size < mf->offset)+	    mf->size = mf->offset;+    }++    if (mf->fp == stderr) {+	/* Auto-flush for stderr */+	if (0 != mfflush(mf)) return -1;+    }++    return ret;+}++/*+ * Converts an mFILE from binary to ascii mode by replacing all+ * cr-nl with nl.+ *+ * Primarily used on windows when we've uncompressed a binary file which+ * happens to be a text file (eg Experiment File). Previously we would have+ * seeked back to the start and used _setmode(fileno(fp), _O_TEXT).+ *+ * Side effect: resets offset and flush_pos back to the start.+ */+void mfascii(mFILE *mf) {+    size_t p1, p2;++    for (p1 = p2 = 1; p1 < mf->size; p1++, p2++) {+	if (mf->data[p1] == '\n' && mf->data[p1-1] == '\r') {+	    p2--; /* delete the \r */+	}+	mf->data[p2] = mf->data[p1];+    }+    mf->size = p2;++    mf->offset = mf->flush_pos = 0;+}
+ htslib-1.3.2/cram/mFILE.h view
@@ -0,0 +1,89 @@+/*+Copyright (c) 2005-2006, 2008-2009 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _MFILE_H_+#define _MFILE_H_++#include <stdio.h>++#ifdef __cplusplus+extern "C" {+#endif++typedef struct {+    FILE *fp;+    char *data;+    size_t alloced;+    int eof;+    int mode; /* open mode in MF_?? define bit pattern */+    size_t size;+    size_t offset;+    size_t flush_pos;+} mFILE;++#define MF_READ    1+#define MF_WRITE   2+#define MF_APPEND  4+#define MF_BINARY  8+#define MF_TRUNC  16+#define MF_MODEX  32+#define MF_MMAP   64++mFILE *mfreopen(const char *path, const char *mode, FILE *fp);+mFILE *mfopen(const char *path, const char *mode);+int mfdetach(mFILE *mf);+int mfclose(mFILE *mf);+int mfdestroy(mFILE *mf);+int mfseek(mFILE *mf, long offset, int whence);+long mftell(mFILE *mf);+void mrewind(mFILE *mf);+void mftruncate(mFILE *mf, long offset);+int mfeof(mFILE *mf);+size_t mfread(void *ptr, size_t size, size_t nmemb, mFILE *mf);+size_t mfwrite(void *ptr, size_t size, size_t nmemb, mFILE *mf);+int mfgetc(mFILE *mf);+int mungetc(int c, mFILE *mf);+mFILE *mfcreate(char *data, int size);+mFILE *mfcreate_from(const char *path, const char *mode_str, FILE *fp);+void mfrecreate(mFILE *mf, char *data, int size);+void *mfsteal(mFILE *mf, size_t *size_out);+char *mfgets(char *s, int size, mFILE *mf);+int mfflush(mFILE *mf);+int mfprintf(mFILE *mf, char *fmt, ...);+mFILE *mstdin(void);+mFILE *mstdout(void);+mFILE *mstderr(void);+void mfascii(mFILE *mf);++#ifdef __cplusplus+}+#endif++#endif /* _MFILE_H_ */
+ htslib-1.3.2/cram/misc.h view
@@ -0,0 +1,110 @@+/*+Copyright (c) 1994-1997, 2001-2002 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1 Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2 Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++/*+Copyright (c) 2003-2013 Genome Research Ltd.++Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _misc_h+#define _misc_h++#include "cram/os.h"++#include <stdio.h>+#include <stdarg.h>  /* varargs needed for v*printf() prototypes */+#include <sys/types.h>++#ifdef __cplusplus+extern "C" {+#endif++/*+ * This informs gcc that crash() doesn't return, so it doesn't need to+ * concern itself that code paths going via crash could mean some variables+ * being undefined and then issuing uninitialised variable warnings.+ * This particularly affected convert.+ */+#ifdef __GNUC__+#    define __NORETURN__ __attribute__ ((__noreturn__))+#else+#    define __NORETURN__+#endif++/*+ * Used for printf style argument checking. We can request a function such+ * as vTcl_SetResult does argument checking, avoiding bugs with using+ * %d and passing in a 64-bit record.+ */+#ifdef __GNUC__+#    define __PRINTF_FORMAT__(a,b) __attribute__ ((format (printf, a, b)))+#else+#    define __PRINTF_FORMAT__(a,b)+#endif++extern int is_directory(char * fn);+extern int is_file(char * fn);+extern int file_size(char * fn);++#define MIN(A,B) ( ( (A) < (B) ) ? (A) : (B) )+#define MAX(A,B) ( ( (A) > (B) ) ? (A) : (B) )++#ifdef __cplusplus+}+#endif++#endif /*_misc_h*/
+ htslib-1.3.2/cram/open_trace_file.c view
@@ -0,0 +1,414 @@+/*+Author: James Bonfield++Copyright (c) 2000-2001 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+Copyright (c) 2008, 2009, 2013, 2014 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdlib.h>+#include <stdio.h>+#include <string.h>+#include <unistd.h>+#include <ctype.h>+#include <limits.h>+#include <sys/types.h>+#include <sys/stat.h>+#include "cram/os.h"+#ifndef PATH_MAX+#  define PATH_MAX 1024+#endif++#include "cram/open_trace_file.h"+#include "cram/misc.h"+#include "htslib/hfile.h"++/*+ * Tokenises the search path splitting on colons (unix) or semicolons+ * (windows).+ * We also  explicitly add a "./" to the end of the search path+ *+ * Returns: A new search path with items separated by nul chars. Two nul+ *          chars in a row represent the end of the tokenised path.+ * Returns NULL for a failure.+ *+ * The returned data has been malloced. It is up to the caller to free this+ * memory.+ */+char *tokenise_search_path(char *searchpath) {+    char *newsearch;+    unsigned int i, j;+    size_t len;+#ifdef _WIN32+    char path_sep = ';';+#else+    char path_sep = ':';+#endif++    if (!searchpath)+	searchpath="";++    newsearch = (char *)malloc((len = strlen(searchpath))+5);+    if (!newsearch)+	return NULL;++    for (i = 0, j = 0; i < len; i++) {+	/* "::" => ":". Used for escaping colons in http://foo */+	if (i < len-1 && searchpath[i] == ':' && searchpath[i+1] == ':') {+	    newsearch[j++] = ':';+	    i++;+	    continue;+	}++	/* Handle http:// and ftp:// too without :: */+	if (path_sep == ':') {+	    if ((i == 0 || (i > 0 && searchpath[i-1] == ':')) &&+		(!strncmp(&searchpath[i], "http:",     5) ||+		 !strncmp(&searchpath[i], "ftp:",      4) ||+		 !strncmp(&searchpath[i], "|http:",    6) ||+		 !strncmp(&searchpath[i], "|ftp:",     5) ||+		 !strncmp(&searchpath[i], "URL=http:", 9) ||+		 !strncmp(&searchpath[i], "URL=ftp:",  8))) {+		do {+		    newsearch[j++] = searchpath[i];+		} while (i<len && searchpath[i++] != ':');+		if (searchpath[i] == ':')+		    i++;+		if (searchpath[i]=='/')+		    newsearch[j++] = searchpath[i++];+		if (searchpath[i]=='/')+		    newsearch[j++] = searchpath[i++];+		// Look for host:port+		do {+		    newsearch[j++] = searchpath[i++];+		} while (i<len && searchpath[i] != ':' && searchpath[i] != '/');+		newsearch[j++] = searchpath[i++];+		if (searchpath[i] == ':')+		    i++;+	    }+	}++	if (searchpath[i] == path_sep) {+	    /* Skip blank path components */+	    if (j && newsearch[j-1] != 0)+		newsearch[j++] = 0;+	} else {+	    newsearch[j++] = searchpath[i];+	}+    }++    if (j)+	newsearch[j++] = 0;+    newsearch[j++] = '.';+    newsearch[j++] = '/';+    newsearch[j++] = 0;+    newsearch[j++] = 0;+    +    return newsearch;+}++mFILE *find_file_url(char *file, char *url) {+    char buf[8192], *cp;+    mFILE *mf = NULL;+    int maxlen = 8190 - strlen(file), len;+    hFILE *hf;++    /* Expand %s for the trace name */+    for (cp = buf; *url && cp - buf < maxlen; url++) {+	if (*url == '%' && *(url+1) == 's') {+	    url++;+	    cp += strlen(strcpy(cp, file));+	} else {+	    *cp++ = *url;+	}+    }+    *cp++ = 0;++    if (!(hf = hopen(buf, "r")))+	return NULL;++    if (NULL == (mf = mfcreate(NULL, 0)))+	return NULL;+    while ((len = hread(hf, buf, 8192)) > 0) {+	if (mfwrite(buf, len, 1, mf) <= 0) {+	    hclose_abruptly(hf);+	    mfdestroy(mf);+	    return NULL;+	}+    }+    if (hclose(hf) < 0 || len < 0) {+	mfdestroy(mf);+	return NULL;+    }++    mrewind(mf);+    return mf;+}++/*+ * Takes a dirname possibly including % rules and appends the filename+ * to it.+ *+ * Returns expanded pathname or NULL for malloc failure.+ */+static char *expand_path(char *file, char *dirname) {+    size_t len = strlen(dirname);+    size_t lenf = strlen(file);+    char *cp, *path;++    path = malloc(len+lenf+2); // worst expansion DIR/FILE+    if (!path)+	return NULL;++    if (dirname[len-1] == '/')+	len--;++    /* Special case for "./" or absolute filenames */+    if (*file == '/' || (len==1 && *dirname == '.')) {+	sprintf(path, "%s", file);+    } else {+	/* Handle %[0-9]*s expansions, if required */+	char *path_end = path;+	*path = 0;+	while ((cp = strchr(dirname, '%'))) {+	    char *endp;+	    long l = strtol(cp+1, &endp, 10);+	    if (*endp != 's') {+		strncpy(path_end, dirname, (endp+1)-dirname);+		path_end += (endp+1)-dirname;+		dirname = endp+1;+		continue;+	    }+	    +	    strncpy(path_end, dirname, cp-dirname);+	    path_end += cp-dirname;+	    if (l) {+		strncpy(path_end, file, l);+		path_end += MIN(strlen(file), l);+		file     += MIN(strlen(file), l);+	    } else {+		strcpy(path_end, file);+		path_end += strlen(file);+		file     += strlen(file);+	    }+	    len -= (endp+1) - dirname;+	    dirname = endp+1;+	}+	strncpy(path_end, dirname, len);+	path_end += MIN(strlen(dirname), len);+	*path_end = 0;+	if (*file) {+	    *path_end++ = '/';+	    strcpy(path_end, file);+	}+    }++    //fprintf(stderr, "*PATH=\"%s\"\n", path);+    return path;+}++/*+ * Searches for file in the directory 'dirname'. If it finds it, it opens+ * it. This also searches for compressed versions of the file in dirname+ * too.+ *+ * Returns mFILE pointer if found+ *         NULL if not+ */+static mFILE *find_file_dir(char *file, char *dirname) {+    char *path;+    mFILE *mf = NULL;++    path = expand_path(file, dirname);++    if (is_file(path))+	mf = mfopen(path, "rbm");++    free(path);+    return mf;+}++/*+ * ------------------------------------------------------------------------+ * Public functions below.+ */++/*+ * Opens a trace file named 'file'. This is initially looked for as a+ * pathname relative to a file named "relative_to". This may (for+ * example) be the name of an experiment file referencing the trace+ * file. In this case by passing relative_to as the experiment file+ * filename the trace file will be picked up in the same directory as+ * the experiment file. Relative_to may be supplied as NULL.+ *+ * 'file' is looked for at relative_to, then the current directory, and then+ * all of the locations listed in 'path' (which is a colon separated list).+ * If 'path' is NULL it uses the RAWDATA environment variable instead.+ *+ * Returns a mFILE pointer when found.+ *           NULL otherwise.+ */+mFILE *open_path_mfile(char *file, char *path, char *relative_to) {+    char *newsearch;+    char *ele;+    mFILE *fp;++    /* Use path first */+    if (!path)+	path = getenv("RAWDATA");+    if (NULL == (newsearch = tokenise_search_path(path)))+	return NULL;+    +    /*+     * Step through the search path testing out each component.+     * We now look through each path element treating some prefixes as+     * special, otherwise we treat the element as a directory.+     */+    for (ele = newsearch; *ele; ele += strlen(ele)+1) {+	char *ele2;++	/*+	 * '|' prefixing a path component indicates that we do not+	 * wish to perform the compression extension searching in that+	 * location.+	 *+	 * NB: this has been removed from the htslib implementation.+	 */+	if (*ele == '|') {+	    ele2 = ele+1;+	} else {+	    ele2 = ele;+	}++	if (0 == strncmp(ele2, "URL=", 4)) {+	    if ((fp = find_file_url(file, ele2+4))) {+		free(newsearch);+		return fp;+	    }+	} else if (!strncmp(ele2, "http:", 5) ||+		   !strncmp(ele2, "ftp:", 4)) {+	    if ((fp = find_file_url(file, ele2))) {+		free(newsearch);+		return fp;+	    }+	} else if ((fp = find_file_dir(file, ele2))) {+	    free(newsearch);+	    return fp;+	} +    }++    free(newsearch);++    /* Look in the same location as the incoming 'relative_to' filename */+    if (relative_to) {+	char *cp;+	char relative_path[PATH_MAX+1];+	strcpy(relative_path, relative_to);+	if ((cp = strrchr(relative_path, '/')))+	    *cp = 0;+	if ((fp = find_file_dir(file, relative_path)))+	    return fp;+    }++    return NULL;+}+++/*+ * As per open_path_mfile, but searching only for local filenames.+ * This is useful as we may avoid doing a full mfopen and loading+ * the entire file into memory.+ *+ * Returns the expanded pathname if found.+ *         NULL if not+ */+char *find_path(char *file, char *path) {+    char *newsearch;+    char *ele;+    char *outpath = NULL;++    /* Use path first */+    if (!path)+	path = getenv("RAWDATA");+    if (NULL == (newsearch = tokenise_search_path(path)))+	return NULL;+    +    for (ele = newsearch; *ele; ele += strlen(ele)+1) {+	char *ele2 = (*ele == '|') ? ele+1 : ele;++	if (!strncmp(ele2, "URL=", 4) ||+	    !strncmp(ele2, "http:", 5) ||+	    !strncmp(ele2, "ftp:", 4)) {+	    continue;+	} else {+	    outpath = expand_path(file, ele2);+	    if (is_file(outpath)) {+		free(newsearch);+		return outpath;+	    } else {+		free(outpath);+	    }+	} +    }++    free(newsearch);++    return NULL;+}
+ htslib-1.3.2/cram/open_trace_file.h view
@@ -0,0 +1,125 @@+/*+Author: James Bonfield++Copyright (c) 2000-2001 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   . Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   . Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   . Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+Copyright (c) 2008, 2009, 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _OPEN_TRACE_FILE_H_+#define _OPEN_TRACE_FILE_H_++#include "cram/mFILE.h"++#ifdef __cplusplus+extern "C" {+#endif++/*+ * Tokenises the search path splitting on colons (unix) or semicolons+ * (windows).+ * We also  explicitly add a "./" to the end of the search path+ *+ * Returns: A new search path with items separated by nul chars. Two nul+ *          chars in a row represent the end of the tokenised path.+ * Returns NULL for a failure.+ *+ * The returned data has been malloced. It is up to the caller to free this+ * memory.+ */+char *tokenise_search_path(char *searchpath);++/*+ * Opens a trace file named 'file'. This is initially looked for as a+ * pathname relative to a file named "relative_to". This may (for+ * example) be the name of an experiment file referencing the trace+ * file. In this case by passing relative_to as the experiment file+ * filename the trace file will be picked up in the same directory as+ * the experiment file. Relative_to may be supplied as NULL.+ *+ * 'file' is looked for at relative_to, then the current directory, and then+ * all of the locations listed in 'path' (which is a colon separated list).+ * If 'path' is NULL it uses the RAWDATA environment variable instead.+ *+ * Returns a mFILE pointer when found.+ *           NULL otherwise.+ */+mFILE *open_path_mfile(char *file, char *path, char *relative_to);++/*+ * Returns a mFILE containing the entire contents of the url;+ *         NULL on failure.+ */+mFILE *find_file_url(char *file, char *url);+++/*+ * As per open_path_mfile, but searching only for local filenames.+ * This is useful as we may avoid doing a full mfopen and loading+ * the entire file into memory.+ *+ * Returns the expanded pathname if found.+ *         NULL if not+ */+char *find_path(char *file, char *path);++#ifdef __cplusplus+}+#endif++#endif /* _OPEN_TRACE_FILE_H_ */
+ htslib-1.3.2/cram/os.h view
@@ -0,0 +1,308 @@+/*+Copyright (c) 1993, 1995-2002 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1 Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2 Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+Copyright (c) 2004, 2006, 2009-2011, 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+ * File: os.h+ *+ * Author: + *         MRC Laboratory of Molecular Biology+ *	   Hills Road+ *	   Cambridge CB2 2QH+ *	   United Kingdom+ *+ * Description: operating system specific type definitions+ *+ */++#ifndef _OS_H_+#define _OS_H_++#include <limits.h>+#include <inttypes.h>++#ifdef __cplusplus+extern "C" {+#endif++/*-----------------------------------------------------------------------------+ * Detection of endianness. The main part of this is done in autoconf, but+ * for the case of MacOS FAT binaries we fall back on auto-sensing based on+ * processor type too.+ */++/* Set by autoconf */+#define SP_LITTLE_ENDIAN++/* Mac FAT binaries or unknown. Auto detect based on CPU type */+#if !defined(SP_BIG_ENDIAN) && !defined(SP_LITTLE_ENDIAN)++/*+ * x86 equivalents+ */+#if defined(__i386__) || defined(__i386) || defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(__i686__) || defined(__i686)+#  if defined(SP_BIG_ENDIAN)+#    undef SP_BIG_ENDIAN+#  endif+#  define SP_LITTLE_ENDIAN+#endif++/*+ * DEC Alpha+ */+#if defined(__alpha__) || defined(__alpha)+#  if defined(SP_LITTLE_ENDIAN)+#    undef SP_LITTLE_ENDIAN+#  endif+#  define SP_BIG_ENDIAN+#endif++/*+ * SUN Sparc+ */+#if defined(__sparc__) || defined(__sparc)+#  if defined(SP_LITTLE_ENDIAN)+#    undef SP_LITTLE_ENDIAN+#  endif+#  define SP_BIG_ENDIAN+#endif++/*+ * PowerPC+ */+#if defined(__ppc__) || defined(__ppc)+#  if defined(SP_LITTLE_ENDIAN)+#    undef SP_LITTLE_ENDIAN+#  endif+#  define SP_BIG_ENDIAN+#endif++/* Some catch-alls */+#if defined(__LITTLE_ENDIAN__) || defined(__LITTLEENDIAN__)+#    define SP_LITTLE_ENDIAN+#endif++#if defined(__BIG_ENDIAN__) || defined(__BIGENDIAN__)+#    define SP_BIG_ENDIAN+#endif++#if defined(SP_BIG_ENDIAN) && defined(SP_LITTLE_ENDIAN)+#    error Both BIG and LITTLE endian defined. Fix os.h and/or Makefile+#endif++#if !defined(SP_BIG_ENDIAN) && !defined(SP_LITTLE_ENDIAN)+#    error Neither BIG nor LITTLE endian defined. Fix os.h and/or Makefile+#endif++#endif++/*-----------------------------------------------------------------------------+ * Allow for unaligned memory access. This is used in BAM code as the packed+ * structure has 4-byte cigar ints after the variable length name.+ *+ * Consider using AX_CHECK_ALIGNED_ACCESS_REQUIRED in autoconf.+ */+#if defined(__i386__) || defined(__i386) || defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(__i686__) || defined(__i686)+#  define ALLOW_UAC+#endif++/*-----------------------------------------------------------------------------+ * Byte swapping macros+ */++/*+ * Our new swap runs at the same speed on Ultrix, but substantially faster+ * (300% for swap_int4, ~50% for swap_int2) on an Alpha (due to the lack of+ * decent 'char' support).+ *+ * They also have the ability to swap in situ (src == dst). Newer code now+ * relies on this so don't change back!+ */+#define iswap_int8(x) \+    (((x & 0x00000000000000ffLL) << 56) + \+     ((x & 0x000000000000ff00LL) << 40) + \+     ((x & 0x0000000000ff0000LL) << 24) + \+     ((x & 0x00000000ff000000LL) <<  8) + \+     ((x & 0x000000ff00000000LL) >>  8) + \+     ((x & 0x0000ff0000000000LL) >> 24) + \+     ((x & 0x00ff000000000000LL) >> 40) + \+     ((x & 0xff00000000000000LL) >> 56))++#define iswap_int4(x) \+    (((x & 0x000000ff) << 24) + \+     ((x & 0x0000ff00) <<  8) + \+     ((x & 0x00ff0000) >>  8) + \+     ((x & 0xff000000) >> 24))++#define iswap_int2(x) \+    (((x & 0x00ff) << 8) + \+     ((x & 0xff00) >> 8))++/*+ * Linux systems may use byteswap.h to get assembly versions of byte-swap+ * on intel systems. This can be as trivial as the bswap opcode, which works+ * out at over 2-times faster than iswap_int4 above.+ */+#if 0+#if defined(__linux__)+#    include <byteswap.h>+#    undef iswap_int8+#    undef iswap_int4+#    undef iswap_int2+#    define iswap_int8 bswap_64+#    define iswap_int4 bswap_32+#    define iswap_int2 bswap_16+#endif+#endif+++/*+ * Macros to specify that data read in is of a particular endianness.+ * The macros here swap to the appropriate order for the particular machine+ * running the macro and return the new answer. These may also be used when+ * writing to a file to specify that we wish to write in (eg) big endian+ * format.+ *+ * This leads to efficient code as most of the time these macros are+ * trivial.+ */+#ifdef SP_BIG_ENDIAN+#define le_int4(x) iswap_int4((x))+#define le_int2(x) iswap_int2((x))+#endif++#ifdef SP_LITTLE_ENDIAN+#define le_int4(x) (x)+#define le_int2(x) (x)+#endif++/*-----------------------------------------------------------------------------+ * <inttypes.h> definitions, incase they're not present+ */++#ifndef PRId64+#define __PRI64__ "l"+#define PRId64 __PRI64__ "d"+#define PRId32 "d"+#define PRId16 "d"+#define PRId8  "d"+#define PRIu64 __PRI64__ "u"+#define PRIu32 "u"+#define PRIu16 "u"+#define PRIu8  "u"+#endif++/*-----------------------------------------------------------------------------+ * Operating system specifics.+ * These ought to be done by autoconf, but are legacy code.+ */+/*+ * SunOS 4.x+ * Even though we use the ANSI gcc, we make use the the standard SunOS 4.x+ * libraries and include files, which are non-ansi+ */+#if defined(__sun__) && !defined(__svr4__)+#define SEEK_SET 0+#define SEEK_CUR 1+#define SEEK_END 2+#endif++/*+ * Microsoft Visual C+++ * Windows+ */+#if defined(_MSC_VER)+#define popen _popen+#define pclose _pclose+#define ftruncate(fd,len) _chsize(fd,len)+#endif+++/*+ * Microsoft Windows running MinGW+ */+#if defined(__MINGW32__)+/* #define mkdir(filename,mode) mkdir((filename)) */+#define sysconf(x) 512+#define ftruncate(fd,len) _chsize(fd,len)+#endif++/* Generic WIN32 API issues */+#ifdef _WIN32+#  ifndef HAVE_FSEEKO+#    if __MSVCRT_VERSION__ >= 0x800+       /* if you have MSVCR80 installed then you can use these definitions: */+#      define off_t __int64+#      define fseeko _fseeki64+#      define ftello _ftelli64+#    else+       /* otherwise we're stuck with 32-bit file support */+#      define off_t long+#      define fseeko fseek+#      define ftello ftell+#    endif+#  endif /* !HAVE_FSEEKO */+#endif /* _WIN32 */++#ifdef __cplusplus+}+#endif++#endif /*_OS_H_*/
+ htslib-1.3.2/cram/pooled_alloc.c view
@@ -0,0 +1,188 @@+/*+Copyright (c) 2009 Genome Research Ltd.+Author: Rob Davies <rmd@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdlib.h>+#include <stdio.h>+#include <stdint.h>++#include "cram/pooled_alloc.h"+#include "cram/misc.h"++//#define TEST_MAIN++#define PSIZE 1024*1024++// credit to http://graphics.stanford.edu/~seander/bithacks.html+static int next_power_2(unsigned int v) {+    v--;+    v |= v >> 1;+    v |= v >> 2;+    v |= v >> 4;+    v |= v >> 8;+    v |= v >> 16;+    v++;++    return v;+}++/*+ * Creates a pool.+ * Pool allocations are approx minimum of 1024*dsize or PSIZE.+ * (Assumes we're not trying to use pools for >= 2Gb or more)+ */+pool_alloc_t *pool_create(size_t dsize) {+    pool_alloc_t *p;++    if (NULL == (p = (pool_alloc_t *)malloc(sizeof(*p))))+	return NULL;++    /* Minimum size is a pointer, for free list */+    dsize = (dsize + sizeof(void *) - 1) & ~(sizeof(void *)-1);+    if (dsize < sizeof(void *))+	dsize = sizeof(void *);+    p->dsize = dsize;+    p->psize = MIN(PSIZE, next_power_2(p->dsize*1024));++    p->npools = 0;+    p->pools = NULL;+    p->free  = NULL;++    return p;+}++static pool_t *new_pool(pool_alloc_t *p) {+    size_t n = p->psize / p->dsize;+    pool_t *pool;+    +    pool = realloc(p->pools, (p->npools + 1) * sizeof(*p->pools));+    if (NULL == pool) return NULL;+    p->pools = pool;+    pool = &p->pools[p->npools];++    pool->pool = malloc(n * p->dsize);+    if (NULL == pool->pool) return NULL;++    pool->used = 0;++    p->npools++;++    return pool;+}++void pool_destroy(pool_alloc_t *p) {+    size_t i;++    for (i = 0; i < p->npools; i++) {+        free(p->pools[i].pool);+    }+    free(p->pools);+    free(p);+}++void *pool_alloc(pool_alloc_t *p) {+    pool_t *pool;+    void *ret;++    /* Look on free list */+    if (NULL != p->free) {+        ret = p->free;+	p->free = *((void **)p->free);+	return ret;+    }++    /* Look for space in the last pool */+    if (p->npools) {+        pool = &p->pools[p->npools - 1];+        if (pool->used + p->dsize < p->psize) {+	    ret = ((char *) pool->pool) + pool->used;+	    pool->used += p->dsize;+	    return ret;+	}+    }++    /* Need a new pool */+    pool = new_pool(p);+    if (NULL == pool) return NULL;++    pool->used = p->dsize;+    return pool->pool;+}++void pool_free(pool_alloc_t *p, void *ptr) {+    *(void **)ptr = p->free;+    p->free = ptr;+}++#ifdef TEST_MAIN+typedef struct {+    int x, y, z;+} xyz;++#define NP 10000+int main(void) {+    int i;+    xyz *item;+    xyz **items;+    pool_alloc_t *p = pool_create(sizeof(xyz));++    items = (xyz **)malloc(NP * sizeof(*items));++    for (i = 0; i < NP; i++) {+	item = pool_alloc(p);+	item->x = i;+	item->y = i+1;+	item->z = i+2;+	items[i] = item;+    }++    for (i = 0; i < NP; i++) {+	item = items[i];+	if (i % 3)+	    pool_free(p, item);+    }++    for (i = 0; i < NP; i++) {+	item = pool_alloc(p);+	item->x = 1000000+i;+	item->y = 1000000+i+1;+	item->z = 1000000+i+2;+    }++    for (i = 0; i < NP; i++) {+	item = items[i];+	printf("%d\t%d\t%d\t%d\n", i, item->x, item->y, item->z);+	pool_free(p, item);+    }++    return 0;+}+#endif
+ htslib-1.3.2/cram/pooled_alloc.h view
@@ -0,0 +1,64 @@+/*+Copyright (c) 2009 Genome Research Ltd.+Author: Rob Davies <rmd@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _POOLED_ALLOC_H_+#define _POOLED_ALLOC_H_++#ifdef __cplusplus+extern "C" {+#endif++/*+ * Implements a pooled block allocator where all items are the same size,+ * but we need many of them.+ */+typedef struct {+    void   *pool;+    size_t  used;+} pool_t;++typedef struct {+    size_t dsize;+    size_t psize;+    size_t npools;+    pool_t *pools;+    void *free;+} pool_alloc_t;++pool_alloc_t *pool_create(size_t dsize);+void pool_destroy(pool_alloc_t *p);+void *pool_alloc(pool_alloc_t *p);+void pool_free(pool_alloc_t *p, void *ptr);++#ifdef __cplusplus+}+#endif++#endif /*_POOLED_ALLOC_H_*/
+ htslib-1.3.2/cram/rANS_byte.h view
@@ -0,0 +1,336 @@+/* rans_byte.h originally from https://github.com/rygorous/ryg_rans+ *+ * This is a public-domain implementation of several rANS variants. rANS is an+ * entropy coder from the ANS family, as described in Jarek Duda's paper+ * "Asymmetric numeral systems" (http://arxiv.org/abs/1311.2540).+ */++/*-------------------------------------------------------------------------- */++// Simple byte-aligned rANS encoder/decoder - public domain - Fabian 'ryg' Giesen 2014+//+// Not intended to be "industrial strength"; just meant to illustrate the general+// idea.++#ifndef RANS_BYTE_HEADER+#define RANS_BYTE_HEADER++#include <stdint.h>++#ifdef assert+#define RansAssert assert+#else+#define RansAssert(x)+#endif++// READ ME FIRST:+//+// This is designed like a typical arithmetic coder API, but there's three+// twists you absolutely should be aware of before you start hacking:+//+// 1. You need to encode data in *reverse* - last symbol first. rANS works+//    like a stack: last in, first out.+// 2. Likewise, the encoder outputs bytes *in reverse* - that is, you give+//    it a pointer to the *end* of your buffer (exclusive), and it will+//    slowly move towards the beginning as more bytes are emitted.+// 3. Unlike basically any other entropy coder implementation you might+//    have used, you can interleave data from multiple independent rANS+//    encoders into the same bytestream without any extra signaling;+//    you can also just write some bytes by yourself in the middle if+//    you want to. This is in addition to the usual arithmetic encoder+//    property of being able to switch models on the fly. Writing raw+//    bytes can be useful when you have some data that you know is+//    incompressible, and is cheaper than going through the rANS encode+//    function. Using multiple rANS coders on the same byte stream wastes+//    a few bytes compared to using just one, but execution of two+//    independent encoders can happen in parallel on superscalar and+//    Out-of-Order CPUs, so this can be *much* faster in tight decoding+//    loops.+//+//    This is why all the rANS functions take the write pointer as an+//    argument instead of just storing it in some context struct.++// --------------------------------------------------------------------------++// L ('l' in the paper) is the lower bound of our normalization interval.+// Between this and our byte-aligned emission, we use 31 (not 32!) bits.+// This is done intentionally because exact reciprocals for 31-bit uints+// fit in 32-bit uints: this permits some optimizations during encoding.+#define RANS_BYTE_L (1u << 23)  // lower bound of our normalization interval++// State for a rANS encoder. Yep, that's all there is to it.+typedef uint32_t RansState;++// Initialize a rANS encoder.+static inline void RansEncInit(RansState* r)+{+    *r = RANS_BYTE_L;+}++// Renormalize the encoder. Internal function.+static inline RansState RansEncRenorm(RansState x, uint8_t** pptr, uint32_t freq, uint32_t scale_bits)+{+    uint32_t x_max = ((RANS_BYTE_L >> scale_bits) << 8) * freq; // this turns into a shift.+    if (x >= x_max) {+        uint8_t* ptr = *pptr;+        do {+            *--ptr = (uint8_t) (x & 0xff);+            x >>= 8;+        } while (x >= x_max);+        *pptr = ptr;+    }+    return x;+}++// Encodes a single symbol with range start "start" and frequency "freq".+// All frequencies are assumed to sum to "1 << scale_bits", and the+// resulting bytes get written to ptr (which is updated).+//+// NOTE: With rANS, you need to encode symbols in *reverse order*, i.e. from+// beginning to end! Likewise, the output bytestream is written *backwards*:+// ptr starts pointing at the end of the output buffer and keeps decrementing.+static inline void RansEncPut(RansState* r, uint8_t** pptr, uint32_t start, uint32_t freq, uint32_t scale_bits)+{+    // renormalize+    RansState x = RansEncRenorm(*r, pptr, freq, scale_bits);++    // x = C(s,x)+    *r = ((x / freq) << scale_bits) + (x % freq) + start;+}++// Flushes the rANS encoder.+static inline void RansEncFlush(RansState* r, uint8_t** pptr)+{+    uint32_t x = *r;+    uint8_t* ptr = *pptr;++    ptr -= 4;+    ptr[0] = (uint8_t) (x >> 0);+    ptr[1] = (uint8_t) (x >> 8);+    ptr[2] = (uint8_t) (x >> 16);+    ptr[3] = (uint8_t) (x >> 24);++    *pptr = ptr;+}++// Initializes a rANS decoder.+// Unlike the encoder, the decoder works forwards as you'd expect.+static inline void RansDecInit(RansState* r, uint8_t** pptr)+{+    uint32_t x;+    uint8_t* ptr = *pptr;++    x  = ptr[0] << 0;+    x |= ptr[1] << 8;+    x |= ptr[2] << 16;+    x |= ptr[3] << 24;+    ptr += 4;++    *pptr = ptr;+    *r = x;+}++// Returns the current cumulative frequency (map it to a symbol yourself!)+static inline uint32_t RansDecGet(RansState* r, uint32_t scale_bits)+{+    return *r & ((1u << scale_bits) - 1);+}++// Advances in the bit stream by "popping" a single symbol with range start+// "start" and frequency "freq". All frequencies are assumed to sum to "1 << scale_bits",+// and the resulting bytes get written to ptr (which is updated).+static inline void RansDecAdvance(RansState* r, uint8_t** pptr, uint32_t start, uint32_t freq, uint32_t scale_bits)+{+    uint32_t mask = (1u << scale_bits) - 1;++    // s, x = D(x)+    uint32_t x = *r;+    x = freq * (x >> scale_bits) + (x & mask) - start;++    // renormalize+    if (x < RANS_BYTE_L) {+        uint8_t* ptr = *pptr;+        do x = (x << 8) | *ptr++; while (x < RANS_BYTE_L);+        *pptr = ptr;+    }++    *r = x;+}++// --------------------------------------------------------------------------++// That's all you need for a full encoder; below here are some utility+// functions with extra convenience or optimizations.++// Encoder symbol description+// This (admittedly odd) selection of parameters was chosen to make+// RansEncPutSymbol as cheap as possible.+typedef struct {+    uint32_t x_max;     // (Exclusive) upper bound of pre-normalization interval+    uint32_t rcp_freq;  // Fixed-point reciprocal frequency+    uint32_t bias;      // Bias+    uint16_t cmpl_freq; // Complement of frequency: (1 << scale_bits) - freq+    uint16_t rcp_shift; // Reciprocal shift+} RansEncSymbol;++// Decoder symbols are straightforward.+typedef struct {+    uint16_t start;     // Start of range.+    uint16_t freq;      // Symbol frequency.+} RansDecSymbol;++// Initializes an encoder symbol to start "start" and frequency "freq"+static inline void RansEncSymbolInit(RansEncSymbol* s, uint32_t start, uint32_t freq, uint32_t scale_bits)+{+    RansAssert(scale_bits <= 16);+    RansAssert(start <= (1u << scale_bits));+    RansAssert(freq <= (1u << scale_bits) - start);++    // Say M := 1 << scale_bits.+    //+    // The original encoder does:+    //   x_new = (x/freq)*M + start + (x%freq)+    //+    // The fast encoder does (schematically):+    //   q     = mul_hi(x, rcp_freq) >> rcp_shift   (division)+    //   r     = x - q*freq                         (remainder)+    //   x_new = q*M + bias + r                     (new x)+    // plugging in r into x_new yields:+    //   x_new = bias + x + q*(M - freq)+    //        =: bias + x + q*cmpl_freq             (*)+    //+    // and we can just precompute cmpl_freq. Now we just need to+    // set up our parameters such that the original encoder and+    // the fast encoder agree.++    s->x_max = ((RANS_BYTE_L >> scale_bits) << 8) * freq;+    s->cmpl_freq = (uint16_t) ((1 << scale_bits) - freq);+    if (freq < 2) {+        // freq=0 symbols are never valid to encode, so it doesn't matter what+        // we set our values to.+        //+        // freq=1 is tricky, since the reciprocal of 1 is 1; unfortunately,+        // our fixed-point reciprocal approximation can only multiply by values+        // smaller than 1.+        //+        // So we use the "next best thing": rcp_freq=0xffffffff, rcp_shift=0.+        // This gives:+        //   q = mul_hi(x, rcp_freq) >> rcp_shift+        //     = mul_hi(x, (1<<32) - 1)) >> 0+        //     = floor(x - x/(2^32))+        //     = x - 1 if 1 <= x < 2^32+        // and we know that x>0 (x=0 is never in a valid normalization interval).+        //+        // So we now need to choose the other parameters such that+        //   x_new = x*M + start+        // plug it in:+        //     x*M + start                   (desired result)+        //   = bias + x + q*cmpl_freq        (*)+        //   = bias + x + (x - 1)*(M - 1)    (plug in q=x-1, cmpl_freq)+        //   = bias + 1 + (x - 1)*M+        //   = x*M + (bias + 1 - M)+        //+        // so we have start = bias + 1 - M, or equivalently+        //   bias = start + M - 1.+        s->rcp_freq = ~0u;+        s->rcp_shift = 0;+        s->bias = start + (1 << scale_bits) - 1;+    } else {+        // Alverson, "Integer Division using reciprocals"+        // shift=ceil(log2(freq))+        uint32_t shift = 0;+        while (freq > (1u << shift))+            shift++;++        s->rcp_freq = (uint32_t) (((1ull << (shift + 31)) + freq-1) / freq);+        s->rcp_shift = shift - 1;++        // With these values, 'q' is the correct quotient, so we+        // have bias=start.+        s->bias = start;+    }++    s->rcp_shift += 32; // Avoid the extra >>32 in RansEncPutSymbol+}++// Initialize a decoder symbol to start "start" and frequency "freq"+static inline void RansDecSymbolInit(RansDecSymbol* s, uint32_t start, uint32_t freq)+{+    RansAssert(start <= (1 << 16));+    RansAssert(freq <= (1 << 16) - start);+    s->start = (uint16_t) start;+    s->freq = (uint16_t) freq;+}++// Encodes a given symbol. This is faster than straight RansEnc since we can do+// multiplications instead of a divide.+//+// See RansEncSymbolInit for a description of how this works.+static inline void RansEncPutSymbol(RansState* r, uint8_t** pptr, RansEncSymbol const* sym)+{+    RansAssert(sym->x_max != 0); // can't encode symbol with freq=0++    // renormalize+    uint32_t x = *r;+    uint32_t x_max = sym->x_max;++    if (x >= x_max) {+	uint8_t* ptr = *pptr;+	do {+	    *--ptr = (uint8_t) (x & 0xff);+	    x >>= 8;+	} while (x >= x_max);+	*pptr = ptr;+    }++    // x = C(s,x)+    // NOTE: written this way so we get a 32-bit "multiply high" when+    // available. If you're on a 64-bit platform with cheap multiplies+    // (e.g. x64), just bake the +32 into rcp_shift.+    //uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> 32) >> sym->rcp_shift;++    // The extra >>32 has already been added to RansEncSymbolInit+    uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> sym->rcp_shift);+    *r = x + sym->bias + q * sym->cmpl_freq;+}++// Equivalent to RansDecAdvance that takes a symbol.+static inline void RansDecAdvanceSymbol(RansState* r, uint8_t** pptr, RansDecSymbol const* sym, uint32_t scale_bits)+{+    RansDecAdvance(r, pptr, sym->start, sym->freq, scale_bits);+}++// Advances in the bit stream by "popping" a single symbol with range start+// "start" and frequency "freq". All frequencies are assumed to sum to "1 << scale_bits".+// No renormalization or output happens.+static inline void RansDecAdvanceStep(RansState* r, uint32_t start, uint32_t freq, uint32_t scale_bits)+{+    uint32_t mask = (1u << scale_bits) - 1;++    // s, x = D(x)+    uint32_t x = *r;+    *r = freq * (x >> scale_bits) + (x & mask) - start;+}++// Equivalent to RansDecAdvanceStep that takes a symbol.+static inline void RansDecAdvanceSymbolStep(RansState* r, RansDecSymbol const* sym, uint32_t scale_bits)+{+    RansDecAdvanceStep(r, sym->start, sym->freq, scale_bits);+}++// Renormalize.+static inline void RansDecRenorm(RansState* r, uint8_t** pptr)+{+    // renormalize+    uint32_t x = *r;++    if (x < RANS_BYTE_L) {+        uint8_t* ptr = *pptr;+        do x = (x << 8) | *ptr++; while (x < RANS_BYTE_L);+        *pptr = ptr;+    }++    *r = x;+}++#endif // RANS_BYTE_HEADER
+ htslib-1.3.2/cram/rANS_static.c view
@@ -0,0 +1,868 @@+/*+ * Copyright (c) 2014 Genome Research Ltd.+ * Author(s): James Bonfield+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions are met:+ *+ *    1. Redistributions of source code must retain the above copyright notice,+ *       this list of conditions and the following disclaimer.+ *+ *    2. Redistributions in binary form must reproduce the above+ *       copyright notice, this list of conditions and the following+ *       disclaimer in the documentation and/or other materials provided+ *       with the distribution.+ *+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+ *       Institute nor the names of its contributors may be used to endorse+ *       or promote products derived from this software without specific+ *       prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++/*+ * Author: James Bonfield, Wellcome Trust Sanger Institute. 2014+ */++#include <config.h>++#include <stdint.h>+#include <stdlib.h>+#include <stdio.h>+#include <unistd.h>+#include <assert.h>+#include <string.h>+#include <sys/time.h>++#include "cram/rANS_static.h"+#include "cram/rANS_byte.h"++#define TF_SHIFT 12+#define TOTFREQ (1<<TF_SHIFT)++#define ABS(a) ((a)>0?(a):-(a))+#ifndef BLK_SIZE+#  define BLK_SIZE 1024*1024+#endif++// Room to allow for expanded BLK_SIZE on worst case compression.+#define BLK_SIZE2 ((int)(1.05*BLK_SIZE))++/*-----------------------------------------------------------------------------+ * Memory to memory compression functions.+ *+ * These are original versions without any manual loop unrolling. They+ * are easier to understand, but can be up to 2x slower.+ */++unsigned char *rans_compress_O0(unsigned char *in, unsigned int in_size,+				unsigned int *out_size) {+    unsigned char *out_buf = malloc(1.05*in_size + 257*257*3 + 9);+    unsigned char *cp, *out_end;+    RansEncSymbol syms[256];+    RansState rans0, rans1, rans2, rans3;+    uint8_t* ptr;+    int F[256] = {0}, i, j, tab_size, rle, x, fsum = 0;+    int m = 0, M = 0;+    uint64_t tr;++    if (!out_buf)+	return NULL;++    ptr = out_end = out_buf + (int)(1.05*in_size) + 257*257*3 + 9;++    // Compute statistics+    for (i = 0; i < in_size; i++) {+	F[in[i]]++;+    }+    tr = ((uint64_t)TOTFREQ<<31)/in_size + (1<<30)/in_size;++    // Normalise so T[i] == TOTFREQ+    for (m = M = j = 0; j < 256; j++) {+	if (!F[j])+	    continue;++	if (m < F[j])+	    m = F[j], M = j;++	if ((F[j] = (F[j]*tr)>>31) == 0)+	    F[j] = 1;+	fsum += F[j];+    }++    fsum++;+    if (fsum < TOTFREQ)+	F[M] += TOTFREQ-fsum;+    else+	F[M] -= fsum-TOTFREQ;++    //printf("F[%d]=%d\n", M, F[M]);+    assert(F[M]>0);++    // Encode statistics.+    cp = out_buf+9;++    for (x = rle = j = 0; j < 256; j++) {+	if (F[j]) {+	    // j+	    if (rle) {+		rle--;+	    } else {+		*cp++ = j;+		if (!rle && j && F[j-1])  {+		    for(rle=j+1; rle<256 && F[rle]; rle++)+			;+		    rle -= j+1;+		    *cp++ = rle;+		}+		//fprintf(stderr, "%d: %d %d\n", j, rle, N[j]);+	    }++	    // F[j]+	    if (F[j]<128) {+		*cp++ = F[j];+	    } else {+		*cp++ = 128 | (F[j]>>8);+		*cp++ = F[j]&0xff;+	    }+	    RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);+	    x += F[j];+	}+    }+    *cp++ = 0;++    //write(1, out_buf+4, cp-(out_buf+4));+    tab_size = cp-out_buf;++    RansEncInit(&rans0);+    RansEncInit(&rans1);+    RansEncInit(&rans2);+    RansEncInit(&rans3);++    switch (i=(in_size&3)) {+    case 3: RansEncPutSymbol(&rans2, &ptr, &syms[in[in_size-(i-2)]]);+    case 2: RansEncPutSymbol(&rans1, &ptr, &syms[in[in_size-(i-1)]]);+    case 1: RansEncPutSymbol(&rans0, &ptr, &syms[in[in_size-(i-0)]]);+    case 0:+	break;+    }+    for (i=(in_size &~3); i>0; i-=4) {+	RansEncSymbol *s3 = &syms[in[i-1]];+	RansEncSymbol *s2 = &syms[in[i-2]];+	RansEncSymbol *s1 = &syms[in[i-3]];+	RansEncSymbol *s0 = &syms[in[i-4]];++	RansEncPutSymbol(&rans3, &ptr, s3);+	RansEncPutSymbol(&rans2, &ptr, s2);+	RansEncPutSymbol(&rans1, &ptr, s1);+	RansEncPutSymbol(&rans0, &ptr, s0);+    }++    RansEncFlush(&rans3, &ptr);+    RansEncFlush(&rans2, &ptr);+    RansEncFlush(&rans1, &ptr);+    RansEncFlush(&rans0, &ptr);++    // Finalise block size and return it+    *out_size = (out_end - ptr) + tab_size;++    cp = out_buf;++    *cp++ = 0; // order+    *cp++ = ((*out_size-9)>> 0) & 0xff;+    *cp++ = ((*out_size-9)>> 8) & 0xff;+    *cp++ = ((*out_size-9)>>16) & 0xff;+    *cp++ = ((*out_size-9)>>24) & 0xff;++    *cp++ = (in_size>> 0) & 0xff;+    *cp++ = (in_size>> 8) & 0xff;+    *cp++ = (in_size>>16) & 0xff;+    *cp++ = (in_size>>24) & 0xff;++    memmove(out_buf + tab_size, ptr, out_end-ptr);++    return out_buf;+}++typedef struct {+    struct {+	int F;+	int C;+    } fc[256];+    unsigned char *R;+} ari_decoder;++unsigned char *rans_uncompress_O0(unsigned char *in, unsigned int in_size,+				  unsigned int *out_size) {+    /* Load in the static tables */+    unsigned char *cp = in + 9;+    int i, j, x, out_sz, in_sz, rle;+    char *out_buf;+    ari_decoder D;+    RansDecSymbol syms[256];++    memset(&D, 0, sizeof(D));++    if (*in++ != 0) // Order-0 check+	return NULL;+    +    in_sz  = ((in[0])<<0) | ((in[1])<<8) | ((in[2])<<16) | ((in[3])<<24);+    out_sz = ((in[4])<<0) | ((in[5])<<8) | ((in[6])<<16) | ((in[7])<<24);+    if (in_sz != in_size-9)+	return NULL;++    out_buf = malloc(out_sz);+    if (!out_buf)+	return NULL;++    //fprintf(stderr, "out_sz=%d\n", out_sz);++    // Precompute reverse lookup of frequency.+    rle = x = 0;+    j = *cp++;+    do {+	if ((D.fc[j].F = *cp++) >= 128) {+	    D.fc[j].F &= ~128;+	    D.fc[j].F = ((D.fc[j].F & 127) << 8) | *cp++;+	}+	D.fc[j].C = x;++	RansDecSymbolInit(&syms[j], D.fc[j].C, D.fc[j].F);++	/* Build reverse lookup table */+	if (!D.R) D.R = (unsigned char *)malloc(TOTFREQ);+	memset(&D.R[x], j, D.fc[j].F);++	x += D.fc[j].F;++	if (!rle && j+1 == *cp) {+	    j = *cp++;+	    rle = *cp++;+	} else if (rle) {+	    rle--;+	    j++;+	} else {+	    j = *cp++;+	}+    } while(j);++    assert(x < TOTFREQ);++    RansState rans0, rans1, rans2, rans3;+    uint8_t *ptr = cp;+    RansDecInit(&rans0, &ptr);+    RansDecInit(&rans1, &ptr);+    RansDecInit(&rans2, &ptr);+    RansDecInit(&rans3, &ptr);++    int out_end = (out_sz&~3);++    RansState R[4];+    R[0] = rans0;+    R[1] = rans1;+    R[2] = rans2;+    R[3] = rans3;+    uint32_t mask = (1u << TF_SHIFT)-1;++    for (i=0; i < out_end; i+=4) {+	uint32_t m[4] = {R[0] & mask,+			 R[1] & mask,+			 R[2] & mask,+			 R[3] & mask};+	uint8_t c[4] = {D.R[m[0]],+			D.R[m[1]],+			D.R[m[2]],+			D.R[m[3]]};+	out_buf[i+0] = c[0];+	out_buf[i+1] = c[1];+	out_buf[i+2] = c[2];+	out_buf[i+3] = c[3];++	// RansDecAdvanceSymbolStep(&R[0], &syms[c[0]], TF_SHIFT);+	// RansDecAdvanceSymbolStep(&R[1], &syms[c[1]], TF_SHIFT);+	// RansDecAdvanceSymbolStep(&R[2], &syms[c[2]], TF_SHIFT);+	// RansDecAdvanceSymbolStep(&R[3], &syms[c[3]], TF_SHIFT);+	R[0] = syms[c[0]].freq * (R[0]>>TF_SHIFT);+	R[1] = syms[c[1]].freq * (R[1]>>TF_SHIFT);+	R[2] = syms[c[2]].freq * (R[2]>>TF_SHIFT);+	R[3] = syms[c[3]].freq * (R[3]>>TF_SHIFT);++	R[0] += m[0] - syms[c[0]].start;+	R[1] += m[1] - syms[c[1]].start;+	R[2] += m[2] - syms[c[2]].start;+	R[3] += m[3] - syms[c[3]].start;++	RansDecRenorm(&R[0], &ptr);+	RansDecRenorm(&R[1], &ptr);+	RansDecRenorm(&R[2], &ptr);+	RansDecRenorm(&R[3], &ptr);+    }++    rans0 = R[0];+    rans1 = R[1];+    rans2 = R[2];+    rans3 = R[3];++    switch(out_sz&3) {+	unsigned char c;+    case 0:+	break;+    case 1:+	c = D.R[RansDecGet(&rans0, TF_SHIFT)];+	RansDecAdvanceSymbol(&rans0, &ptr, &syms[c], TF_SHIFT);+	out_buf[out_end] = c;+	break;++    case 2:+	c = D.R[RansDecGet(&rans0, TF_SHIFT)];+	RansDecAdvanceSymbol(&rans0, &ptr, &syms[c], TF_SHIFT);+	out_buf[out_end] = c;++	c = D.R[RansDecGet(&rans1, TF_SHIFT)];+	RansDecAdvanceSymbol(&rans1, &ptr, &syms[c], TF_SHIFT);+	out_buf[out_end+1] = c;+	break;++    case 3:+	c = D.R[RansDecGet(&rans0, TF_SHIFT)];+	RansDecAdvanceSymbol(&rans0, &ptr, &syms[c], TF_SHIFT);+	out_buf[out_end] = c;++	c = D.R[RansDecGet(&rans1, TF_SHIFT)];+	RansDecAdvanceSymbol(&rans1, &ptr, &syms[c], TF_SHIFT);+	out_buf[out_end+1] = c;++	c = D.R[RansDecGet(&rans2, TF_SHIFT)];+	RansDecAdvanceSymbol(&rans2, &ptr, &syms[c], TF_SHIFT);+	out_buf[out_end+2] = c;+	break;+    }++    *out_size = out_sz;++    if (D.R) free(D.R);++    return (unsigned char *)out_buf;+}++unsigned char *rans_compress_O1(unsigned char *in, unsigned int in_size,+				unsigned int *out_size) {+    unsigned char *out_buf = NULL, *out_end, *cp;+    unsigned int last_i, tab_size, rle_i, rle_j;+    RansEncSymbol (*syms)[256] = NULL;  /* syms[256][256] */+    int (*F)[256] = NULL;               /* F[256][256]    */+    int *T = NULL;                      /* T[256]         */+    int i, j;+    unsigned char c;++    if (in_size < 4)+	return rans_compress_O0(in, in_size, out_size);++    syms = malloc(256 * sizeof(*syms));+    if (!syms) goto cleanup;+    F = calloc(256, sizeof(*F));+    if (!F) goto cleanup;+    T = calloc(256, sizeof(*T));+    if (!T) goto cleanup;+    out_buf = malloc(1.05*in_size + 257*257*3 + 9);+    if (!out_buf) goto cleanup;++    out_end = out_buf + (int)(1.05*in_size) + 257*257*3 + 9;+    cp = out_buf+9;++    //for (last = 0, i=in_size-1; i>=0; i--) {+    //	F[last][c = in[i]]++;+    //	T[last]++;+    //	last = c;+    //}++    for (last_i=i=0; i<in_size; i++) {+	F[last_i][c = in[i]]++;+	T[last_i]++;+	last_i = c;+    }+    F[0][in[1*(in_size>>2)]]++;+    F[0][in[2*(in_size>>2)]]++;+    F[0][in[3*(in_size>>2)]]++;+    T[0]+=3;++    // Normalise so T[i] == TOTFREQ+    for (rle_i = i = 0; i < 256; i++) {+	int t2, m, M;+	unsigned int x;++	if (T[i] == 0)+	    continue;++	//uint64_t p = (TOTFREQ * TOTFREQ) / t;+	double p = ((double)TOTFREQ)/T[i];+	for (t2 = m = M = j = 0; j < 256; j++) {+	    if (!F[i][j])+		continue;++	    if (m < F[i][j])+		m = F[i][j], M = j;++	    //if ((F[i][j] = (F[i][j] * p) / TOTFREQ) == 0)+	    if ((F[i][j] *= p) == 0)+		F[i][j] = 1;+	    t2 += F[i][j];+	}++	t2++;+	if (t2 < TOTFREQ)+	    F[i][M] += TOTFREQ-t2;+	else+	    F[i][M] -= t2-TOTFREQ;++	// Store frequency table+	// i+	if (rle_i) {+	    rle_i--;+	} else {+	    *cp++ = i;+	    // FIXME: could use order-0 statistics to observe which alphabet+	    // symbols are present and base RLE on that ordering instead.+	    if (i && T[i-1]) {+		for(rle_i=i+1; rle_i<256 && T[rle_i]; rle_i++)+		    ;+		rle_i -= i+1;+		*cp++ = rle_i;+	    }+	}++	int *F_i_ = F[i];+	x = 0;+	rle_j = 0;+	for (j = 0; j < 256; j++) {+	    if (F_i_[j]) {+		//fprintf(stderr, "F[%d][%d]=%d, x=%d\n", i, j, F_i_[j], x);++		// j+		if (rle_j) {+		    rle_j--;+		} else {+		    *cp++ = j;+		    if (!rle_j && j && F_i_[j-1]) {+			for(rle_j=j+1; rle_j<256 && F_i_[rle_j]; rle_j++)+			    ;+			rle_j -= j+1;+			*cp++ = rle_j;+		    }+		}++		// F_i_[j]+		if (F_i_[j]<128) {+		    *cp++ = F_i_[j];+		} else {+		    *cp++ = 128 | (F_i_[j]>>8);+		    *cp++ = F_i_[j]&0xff;+		}++		RansEncSymbolInit(&syms[i][j], x, F_i_[j], TF_SHIFT);+		x += F_i_[j];+	    }+	}+	*cp++ = 0;+    }+    *cp++ = 0;++    //write(1, out_buf+4, cp-(out_buf+4));+    tab_size = cp - out_buf;+    assert(tab_size < 257*257*3);++    RansState rans0, rans1, rans2, rans3;+    RansEncInit(&rans0);+    RansEncInit(&rans1);+    RansEncInit(&rans2);+    RansEncInit(&rans3);++    uint8_t* ptr = out_end;++    int isz4 = in_size>>2;+    int i0 = 1*isz4-2;+    int i1 = 2*isz4-2;+    int i2 = 3*isz4-2;+    int i3 = 4*isz4-2;++    unsigned char l0 = in[i0+1];+    unsigned char l1 = in[i1+1];+    unsigned char l2 = in[i2+1];+    unsigned char l3 = in[i3+1];++    // Deal with the remainder+    l3 = in[in_size-1];+    for (i3 = in_size-2; i3 > 4*isz4-2; i3--) {+	unsigned char c3 = in[i3];+	RansEncPutSymbol(&rans3, &ptr, &syms[c3][l3]);+	l3 = c3;+    }++    for (; i0 >= 0; i0--, i1--, i2--, i3--) {+	unsigned char c0, c1, c2, c3;+	RansEncSymbol *s3 = &syms[c3 = in[i3]][l3];+	RansEncSymbol *s2 = &syms[c2 = in[i2]][l2];+	RansEncSymbol *s1 = &syms[c1 = in[i1]][l1];+	RansEncSymbol *s0 = &syms[c0 = in[i0]][l0];++	RansEncPutSymbol(&rans3, &ptr, s3);+	RansEncPutSymbol(&rans2, &ptr, s2);+	RansEncPutSymbol(&rans1, &ptr, s1);+	RansEncPutSymbol(&rans0, &ptr, s0);++	l0 = c0;+	l1 = c1;+	l2 = c2;+	l3 = c3;+    }++    RansEncPutSymbol(&rans3, &ptr, &syms[0][l3]);+    RansEncPutSymbol(&rans2, &ptr, &syms[0][l2]);+    RansEncPutSymbol(&rans1, &ptr, &syms[0][l1]);+    RansEncPutSymbol(&rans0, &ptr, &syms[0][l0]);++    RansEncFlush(&rans3, &ptr);+    RansEncFlush(&rans2, &ptr);+    RansEncFlush(&rans1, &ptr);+    RansEncFlush(&rans0, &ptr);++    *out_size = (out_end - ptr) + tab_size;++    cp = out_buf;+    *cp++ = 1; // order++    *cp++ = ((*out_size-9)>> 0) & 0xff;+    *cp++ = ((*out_size-9)>> 8) & 0xff;+    *cp++ = ((*out_size-9)>>16) & 0xff;+    *cp++ = ((*out_size-9)>>24) & 0xff;++    *cp++ = (in_size>> 0) & 0xff;+    *cp++ = (in_size>> 8) & 0xff;+    *cp++ = (in_size>>16) & 0xff;+    *cp++ = (in_size>>24) & 0xff;++    memmove(out_buf + tab_size, ptr, out_end-ptr);++ cleanup:+    free(syms);+    free(F);+    free(T);++    return out_buf;+}++unsigned char *rans_uncompress_O1(unsigned char *in, unsigned int in_size,+				  unsigned int *out_size) {+    /* Load in the static tables */+    unsigned char *cp = in + 9;+    int i, j = -999, x, out_sz, in_sz, rle_i, rle_j;+    char *out_buf = NULL;+    ari_decoder *D = NULL;              /* D[256] */+    RansDecSymbol (*syms)[256] = NULL;  /* syms[256][256] */++    if (*in++ != 1) // Order-1 check+	return NULL;++    in_sz  = ((in[0])<<0) | ((in[1])<<8) | ((in[2])<<16) | ((in[3])<<24);+    out_sz = ((in[4])<<0) | ((in[5])<<8) | ((in[6])<<16) | ((in[7])<<24);+    if (in_sz != in_size-9)+	return NULL;++    D = calloc(256, sizeof(*D));+    if (!D) goto cleanup;+    syms = malloc(256 * sizeof(*syms));+    if (!syms) goto cleanup;++    //fprintf(stderr, "out_sz=%d\n", out_sz);++    //i = *cp++;+    rle_i = 0;+    i = *cp++;+    do {+	rle_j = x = 0;+	j = *cp++;+	do {+	    if ((D[i].fc[j].F = *cp++) >= 128) {+		D[i].fc[j].F &= ~128;+		D[i].fc[j].F = ((D[i].fc[j].F & 127) << 8) | *cp++;+	    }+	    D[i].fc[j].C = x;++	    //fprintf(stderr, "i=%d j=%d F=%d C=%d\n", i, j, D[i].fc[j].F, D[i].fc[j].C);++	    if (!D[i].fc[j].F)+		D[i].fc[j].F = TOTFREQ;++	    RansDecSymbolInit(&syms[i][j], D[i].fc[j].C, D[i].fc[j].F);++	    /* Build reverse lookup table */+	    if (!D[i].R) {+                D[i].R = (unsigned char *)malloc(TOTFREQ);+                if (!D[i].R)+                    goto cleanup;+            }+	    memset(&D[i].R[x], j, D[i].fc[j].F);++	    x += D[i].fc[j].F;+	    assert(x <= TOTFREQ);++	    if (!rle_j && j+1 == *cp) {+		j = *cp++;+		rle_j = *cp++;+	    } else if (rle_j) {+		rle_j--;+		j++;+	    } else {+		j = *cp++;+	    }+	} while(j);++	if (!rle_i && i+1 == *cp) {+	    i = *cp++;+	    rle_i = *cp++;+	} else if (rle_i) {+	    rle_i--;+	    i++;+	} else {+	    i = *cp++;+	}+    } while (i);++    // Precompute reverse lookup of frequency.++    RansState rans0, rans1, rans2, rans3;+    uint8_t *ptr = cp;+    RansDecInit(&rans0, &ptr);+    RansDecInit(&rans1, &ptr);+    RansDecInit(&rans2, &ptr);+    RansDecInit(&rans3, &ptr);++    int isz4 = out_sz>>2;+    int l0 = 0;+    int l1 = 0;+    int l2 = 0;+    int l3 = 0;+    int i4[] = {0*isz4, 1*isz4, 2*isz4, 3*isz4};++    RansState R[4];+    R[0] = rans0;+    R[1] = rans1;+    R[2] = rans2;+    R[3] = rans3;++    /* Allocate output buffer */+    out_buf = malloc(out_sz);+    if (!out_buf) goto cleanup;++    for (; i4[0] < isz4; i4[0]++, i4[1]++, i4[2]++, i4[3]++) {+	uint32_t m[4] = {R[0] & ((1u << TF_SHIFT)-1),+			 R[1] & ((1u << TF_SHIFT)-1),+			 R[2] & ((1u << TF_SHIFT)-1),+			 R[3] & ((1u << TF_SHIFT)-1)};++	uint8_t c[4] = {D[l0].R[m[0]],+			D[l1].R[m[1]],+			D[l2].R[m[2]],+			D[l3].R[m[3]]};++	out_buf[i4[0]] = c[0];+	out_buf[i4[1]] = c[1];+	out_buf[i4[2]] = c[2];+	out_buf[i4[3]] = c[3];++	//RansDecAdvanceSymbolStep(&R[0], &syms[l0][c[0]], TF_SHIFT);+	//RansDecAdvanceSymbolStep(&R[1], &syms[l1][c[1]], TF_SHIFT);+	//RansDecAdvanceSymbolStep(&R[2], &syms[l2][c[2]], TF_SHIFT);+	//RansDecAdvanceSymbolStep(&R[3], &syms[l3][c[3]], TF_SHIFT);++	R[0] = syms[l0][c[0]].freq * (R[0]>>TF_SHIFT);+	R[1] = syms[l1][c[1]].freq * (R[1]>>TF_SHIFT);+	R[2] = syms[l2][c[2]].freq * (R[2]>>TF_SHIFT);+	R[3] = syms[l3][c[3]].freq * (R[3]>>TF_SHIFT);++	R[0] += m[0] - syms[l0][c[0]].start;+	R[1] += m[1] - syms[l1][c[1]].start;+	R[2] += m[2] - syms[l2][c[2]].start;+	R[3] += m[3] - syms[l3][c[3]].start;++	RansDecRenorm(&R[0], &ptr);+	RansDecRenorm(&R[1], &ptr);+	RansDecRenorm(&R[2], &ptr);+	RansDecRenorm(&R[3], &ptr);++	l0 = c[0];+	l1 = c[1];+	l2 = c[2];+	l3 = c[3];+    }++    rans0 = R[0];+    rans1 = R[1];+    rans2 = R[2];+    rans3 = R[3];++    // Remainder+    for (; i4[3] < out_sz; i4[3]++) {+	unsigned char c3 = D[l3].R[RansDecGet(&rans3, TF_SHIFT)];+	out_buf[i4[3]] = c3;+	RansDecAdvanceSymbol(&rans3, &ptr, &syms[l3][c3], TF_SHIFT);+	l3 = c3;+    }++    *out_size = out_sz;++ cleanup:+    if (D) {+        for (i = 0; i < 256; i++)+            if (D[i].R) free(D[i].R);+        free(D);+    }+    free(syms);++    return (unsigned char *)out_buf;+}++/*-----------------------------------------------------------------------------+ * Simple interface to the order-0 vs order-1 encoders and decoders.+ */+unsigned char *rans_compress(unsigned char *in, unsigned int in_size,+			     unsigned int *out_size, int order) {+    return order+	? rans_compress_O1(in, in_size, out_size)+	: rans_compress_O0(in, in_size, out_size);+}++unsigned char *rans_uncompress(unsigned char *in, unsigned int in_size,+			       unsigned int *out_size) {+    /* Both rans_uncompress functions need to be able to read at least 9+       bytes. */+    if (in_size < 9)+        return NULL;+    return in[0]+	? rans_uncompress_O1(in, in_size, out_size)+	: rans_uncompress_O0(in, in_size, out_size);+}+++#ifdef TEST_MAIN+/*-----------------------------------------------------------------------------+ * Main.+ *+ * This is a simple command line tool for testing order-0 and order-1+ * compression using the rANS codec. Simply compile with+ *+ * gcc -DTEST_MAIN -O3 -I. cram/rANS_static.c -o cram/rANS_static+ *+ * Usage: cram/rANS_static -o0 < file    > file.o0+ *        cram/rANS_static -d  < file.o0 > file2+ *+ *        cram/rANS_static -o1 < file    > file.o1+ *        cram/rANS_static -d  < file.o1 > file2+ */+int main(int argc, char **argv) {+    int opt, order = 0;+    unsigned char in_buf[BLK_SIZE2+257*257*3];+    int decode = 0;+    FILE *infp = stdin, *outfp = stdout;+    struct timeval tv1, tv2;+    size_t bytes = 0;++    extern char *optarg;+    extern int optind;++    while ((opt = getopt(argc, argv, "o:d")) != -1) {+	switch (opt) {+	case 'o':+	    order = atoi(optarg);+	    break;++	case 'd':+	    decode = 1;+	    break;+	}+    }++    order = order ? 1 : 0; // Only support O(0) and O(1)++    if (optind < argc) {+	if (!(infp = fopen(argv[optind], "rb"))) {+	    perror(argv[optind]);+	    return 1;+	}+	optind++;+    }++    if (optind < argc) {+	if (!(outfp = fopen(argv[optind], "wb"))) {+	    perror(argv[optind]);+	    return 1;+	}+	optind++;+    }++    gettimeofday(&tv1, NULL);++    if (decode) {+	// Only used in some test implementations of RC_GetFreq()+	//RC_init();+	//RC_init2();++	for (;;) {+	    uint32_t in_size, out_size;+	    unsigned char *out;++	    if (9 != fread(in_buf, 1, 9, infp))+		break;+	    in_size = *(int *)&in_buf[1];+	    if (in_size != fread(in_buf+9, 1, in_size, infp)) {+		fprintf(stderr, "Truncated input\n");+		exit(1);+	    }+	    out = rans_uncompress(in_buf, in_size+9, &out_size);+	    if (!out)+		abort();++	    fwrite(out, 1, out_size, outfp);+	    free(out);++	    bytes += out_size;+	}+    } else {+	for (;;) {+	    uint32_t in_size, out_size;+	    unsigned char *out;++	    in_size = fread(in_buf, 1, BLK_SIZE, infp);+	    if (in_size <= 0)+		break;++	    out = rans_compress(in_buf, in_size, &out_size, order);++	    fwrite(out, 1, out_size, outfp);+	    free(out);++	    bytes += in_size;+	}+    }++    gettimeofday(&tv2, NULL);++    fprintf(stderr, "Took %ld microseconds, %5.1f MB/s\n",+	    (long)(tv2.tv_sec - tv1.tv_sec)*1000000 ++	    tv2.tv_usec - tv1.tv_usec,+	    (double)bytes / ((long)(tv2.tv_sec - tv1.tv_sec)*1000000 ++			     tv2.tv_usec - tv1.tv_usec));+    return 0;+}+#endif
+ htslib-1.3.2/cram/rANS_static.h view
@@ -0,0 +1,51 @@+/*+ * Copyright (c) 2014 Genome Research Ltd.+ * Author(s): James Bonfield+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions are met:+ *+ *    1. Redistributions of source code must retain the above copyright notice,+ *       this list of conditions and the following disclaimer.+ *+ *    2. Redistributions in binary form must reproduce the above+ *       copyright notice, this list of conditions and the following+ *       disclaimer in the documentation and/or other materials provided+ *       with the distribution.+ *+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+ *       Institute nor the names of its contributors may be used to endorse+ *       or promote products derived from this software without specific+ *       prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */+++#ifndef RANS_STATIC_H+#define RANS_STATIC_H++#ifdef __cplusplus+extern "C" {+#endif++unsigned char *rans_compress(unsigned char *in, unsigned int in_size,+			     unsigned int *out_size, int order);+unsigned char *rans_uncompress(unsigned char *in, unsigned int in_size,+			       unsigned int *out_size);++#ifdef __cplusplus+}+#endif++#endif /* RANS_STATIC_H */
+ htslib-1.3.2/cram/sam_header.c view
@@ -0,0 +1,1268 @@+/*+Copyright (c) 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <string.h>+#include <assert.h>++#include "cram/sam_header.h"+#include "cram/string_alloc.h"++static void sam_hdr_error(char *msg, char *line, int len, int lno) {+    int j;+    +    for (j = 0; j < len && line[j] != '\n'; j++)+	;+    fprintf(stderr, "%s at line %d: \"%.*s\"\n", msg, lno, j, line);+}++void sam_hdr_dump(SAM_hdr *hdr) {+    khint_t k;+    int i;++    printf("===DUMP===\n");+    for (k = kh_begin(hdr->h); k != kh_end(hdr->h); k++) {+	SAM_hdr_type *t1, *t2;+	char c[2];++	if (!kh_exist(hdr->h, k))+	    continue;++	t1 = t2 = kh_val(hdr->h, k);+	c[0] = kh_key(hdr->h, k)>>8;+	c[1] = kh_key(hdr->h, k)&0xff;+	printf("Type %.2s, count %d\n", c, t1->prev->order+1);++	do {+	    SAM_hdr_tag *tag;+	    printf(">>>%d ", t1->order);+	    for (tag = t1->tag; tag; tag=tag->next) {+		printf("\"%.2s\":\"%.*s\"\t",+		       tag->str, tag->len-3, tag->str+3);+	    }+	    putchar('\n');+	    t1 = t1->next;+	} while (t1 != t2);+    }++    /* Dump out PG chains */+    printf("\n@PG chains:\n");+    for (i = 0; i < hdr->npg_end; i++) {+	int j;+	printf("  %d:", i);+	for (j = hdr->pg_end[i]; j != -1; j = hdr->pg[j].prev_id) {+	    printf("%s%d(%.*s)", +		   j == hdr->pg_end[i] ? " " : "->",+		   j, hdr->pg[j].name_len, hdr->pg[j].name);+	}+	printf("\n");+    }++    puts("===END DUMP===");+}++/* Updates the hash tables in the SAM_hdr structure.+ *+ * Returns 0 on success;+ *        -1 on failure+ */+static int sam_hdr_update_hashes(SAM_hdr *sh,+				 int type,+				 SAM_hdr_type *h_type) {+    /* Add to reference hash? */+    if ((type>>8) == 'S' && (type&0xff) == 'Q') {+	SAM_hdr_tag *tag;+	SAM_SQ *new_ref;+	int nref = sh->nref;++	new_ref = realloc(sh->ref, (sh->nref+1)*sizeof(*sh->ref));+	if (!new_ref)+	    return -1;+	sh->ref = new_ref;++	tag = h_type->tag;+	sh->ref[nref].name = NULL;+	sh->ref[nref].len  = 0;+	sh->ref[nref].ty = h_type;+	sh->ref[nref].tag  = tag;++	while (tag) {+	    if (tag->str[0] == 'S' && tag->str[1] == 'N') {+		if (!(sh->ref[nref].name = malloc(tag->len)))+		    return -1;+		strncpy(sh->ref[nref].name, tag->str+3, tag->len-3);+		sh->ref[nref].name[tag->len-3] = 0;+	    } else if (tag->str[0] == 'L' && tag->str[1] == 'N') {+		sh->ref[nref].len = atoi(tag->str+3);+	    }+	    tag = tag->next;+	}++	if (sh->ref[nref].name) {+	    khint_t k;+	    int r;+	    k = kh_put(m_s2i, sh->ref_hash, sh->ref[nref].name, &r);+	    if (-1 == r) return -1;+	    kh_val(sh->ref_hash, k) = nref;+	} else {+	    return -1; // SN should be present, according to spec.+	}++	sh->nref++;+    }++    /* Add to read-group hash? */+    if ((type>>8) == 'R' && (type&0xff) == 'G') {+	SAM_hdr_tag *tag;+	SAM_RG *new_rg;+	int nrg = sh->nrg;++	new_rg = realloc(sh->rg, (sh->nrg+1)*sizeof(*sh->rg));+	if (!new_rg)+	    return -1;+	sh->rg = new_rg;++	tag = h_type->tag;+	sh->rg[nrg].name = NULL;+	sh->rg[nrg].name_len = 0;+	sh->rg[nrg].ty   = h_type;+	sh->rg[nrg].tag  = tag;+	sh->rg[nrg].id   = nrg;++	while (tag) {+	    if (tag->str[0] == 'I' && tag->str[1] == 'D') {+		if (!(sh->rg[nrg].name = malloc(tag->len)))+		    return -1;+		strncpy(sh->rg[nrg].name, tag->str+3, tag->len-3);+		sh->rg[nrg].name[tag->len-3] = 0;+		sh->rg[nrg].name_len = strlen(sh->rg[nrg].name);+	    }+	    tag = tag->next;+	}++	if (sh->rg[nrg].name) {+	    khint_t k;+	    int r;+	    k = kh_put(m_s2i, sh->rg_hash, sh->rg[nrg].name, &r);+	    if (-1 == r) return -1;+	    kh_val(sh->rg_hash, k) = nrg;+	} else {+	    return -1; // ID should be present, according to spec.+	}++	sh->nrg++;+    }++    /* Add to program hash? */+    if ((type>>8) == 'P' && (type&0xff) == 'G') {+	SAM_hdr_tag *tag;+	SAM_PG *new_pg;+	int npg = sh->npg;++	new_pg = realloc(sh->pg, (sh->npg+1)*sizeof(*sh->pg));+	if (!new_pg)+	    return -1;+	sh->pg = new_pg;++	tag = h_type->tag;+	sh->pg[npg].name = NULL;+	sh->pg[npg].name_len = 0;+	sh->pg[npg].ty  = h_type;+	sh->pg[npg].tag  = tag;+	sh->pg[npg].id   = npg;+	sh->pg[npg].prev_id = -1;++	while (tag) {+	    if (tag->str[0] == 'I' && tag->str[1] == 'D') {+		if (!(sh->pg[npg].name = malloc(tag->len)))+		    return -1;+		strncpy(sh->pg[npg].name, tag->str+3, tag->len-3);+		sh->pg[npg].name[tag->len-3] = 0;+		sh->pg[npg].name_len = strlen(sh->pg[npg].name);+	    } else if (tag->str[0] == 'P' && tag->str[1] == 'P') {+		// Resolve later if needed+		khint_t k;+		char tmp = tag->str[tag->len]; tag->str[tag->len] = 0;+		k = kh_get(m_s2i, sh->pg_hash, tag->str+3);+		tag->str[tag->len] = tmp;++		if (k != kh_end(sh->pg_hash)) {+		    int p_id = kh_val(sh->pg_hash, k);+		    sh->pg[npg].prev_id = sh->pg[p_id].id;++		    /* Unmark previous entry as a PG termination */+		    if (sh->npg_end > 0 &&+			sh->pg_end[sh->npg_end-1] == p_id) {+			sh->npg_end--;+		    } else {+			int i;+			for (i = 0; i < sh->npg_end; i++) {+			    if (sh->pg_end[i] == p_id) {+				memmove(&sh->pg_end[i], &sh->pg_end[i+1],+					(sh->npg_end-i-1)*sizeof(*sh->pg_end));+				sh->npg_end--;+			    }+			}+		    }+		} else {+		    sh->pg[npg].prev_id = -1;+		}+	    }+	    tag = tag->next;+	}++	if (sh->pg[npg].name) {+	    khint_t k;+	    int r;+	    k = kh_put(m_s2i, sh->pg_hash, sh->pg[npg].name, &r);+	    if (-1 == r) return -1;+	    kh_val(sh->pg_hash, k) = npg;+	} else {+	    return -1; // ID should be present, according to spec.+	}++	/* Add to npg_end[] array. Remove later if we find a PP line */+	if (sh->npg_end >= sh->npg_end_alloc) {+	    int *new_pg_end;+	    int  new_alloc = sh->npg_end_alloc ? sh->npg_end_alloc*2 : 4;++	    new_pg_end = realloc(sh->pg_end, new_alloc * sizeof(int));+	    if (!new_pg_end)+		return -1;+	    sh->npg_end_alloc = new_alloc;+	    sh->pg_end = new_pg_end;+	}+	sh->pg_end[sh->npg_end++] = npg;++	sh->npg++;+    }++    return 0;+}++/*+ * Appends a formatted line to an existing SAM header.+ * Line is a full SAM header record, eg "@SQ\tSN:foo\tLN:100", with+ * optional new-line. If it contains more than 1 line then multiple lines+ * will be added in order.+ *+ * Input text is of maximum length len or as terminated earlier by a NUL.+ * Len may be 0 if unknown, in which case lines must be NUL-terminated.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int sam_hdr_add_lines(SAM_hdr *sh, const char *lines, int len) {+    int i, lno, text_offset;+    char *hdr;++    if (!len)+	len = strlen(lines);++    text_offset = ks_len(&sh->text);+    if (EOF == kputsn(lines, len, &sh->text))+	return -1;+    hdr = ks_str(&sh->text) + text_offset;++    for (i = 0, lno = 1; i < len && hdr[i] != '\0'; i++, lno++) {+	khint32_t type;+	khint_t k;++	int l_start = i, new;+	SAM_hdr_type *h_type;+	SAM_hdr_tag *h_tag, *last;++	if (hdr[i] != '@') {+	    int j;+	    for (j = i; j < len && hdr[j] != '\0' && hdr[j] != '\n'; j++)+		;+	    sam_hdr_error("Header line does not start with '@'",+			  &hdr[l_start], len - l_start, lno);+	    return -1;+	}++	type = (hdr[i+1]<<8) | hdr[i+2];+	if (hdr[i+1] < 'A' || hdr[i+1] > 'z' ||+	    hdr[i+2] < 'A' || hdr[i+2] > 'z') {+	    sam_hdr_error("Header line does not have a two character key",+			  &hdr[l_start], len - l_start, lno);+	    return -1;+	}++	i += 3;+	if (hdr[i] == '\n')+	    continue;++	// Add the header line type+	if (!(h_type = pool_alloc(sh->type_pool)))+	    return -1;+	if (-1 == (k = kh_put(sam_hdr, sh->h, type, &new)))+	    return -1;++	// Form the ring, either with self or other lines of this type+	if (!new) {+	    SAM_hdr_type *t = kh_val(sh->h, k), *p;+	    p = t->prev;+	    +	    assert(p->next == t);+	    p->next = h_type;+	    h_type->prev = p;++	    t->prev = h_type;+	    h_type->next = t;+	    h_type->order = p->order+1;+	} else {+	    kh_val(sh->h, k) = h_type;+	    h_type->prev = h_type->next = h_type;+	    h_type->order = 0;+	}++	// Parse the tags on this line+	last = NULL;+	if ((type>>8) == 'C' && (type&0xff) == 'O') {+	    int j;+	    if (hdr[i] != '\t') {+		sam_hdr_error("Missing tab",+			      &hdr[l_start], len - l_start, lno);+		return -1;+	    }++	    for (j = ++i; j < len && hdr[j] != '\0' && hdr[j] != '\n'; j++)+		;++	    if (!(h_type->tag = h_tag = pool_alloc(sh->tag_pool)))+		return -1;+	    h_tag->str = string_ndup(sh->str_pool, &hdr[i], j-i);+	    h_tag->len = j-i;+	    h_tag->next = NULL;+	    if (!h_tag->str)+		return -1;++	    i = j;++	} else {+	    do {+		int j;+		if (hdr[i] != '\t') {+		    sam_hdr_error("Missing tab",+				  &hdr[l_start], len - l_start, lno);+		    return -1;+		}++		for (j = ++i; j < len && hdr[j] != '\0' && hdr[j] != '\n' && hdr[j] != '\t'; j++)+		    ;+	    +		if (!(h_tag = pool_alloc(sh->tag_pool)))+		    return -1;+		h_tag->str = string_ndup(sh->str_pool, &hdr[i], j-i);+		h_tag->len = j-i;+		h_tag->next = NULL;+		if (!h_tag->str)+		    return -1;++		if (h_tag->len < 3 || h_tag->str[2] != ':') {+		    sam_hdr_error("Malformed key:value pair",+				  &hdr[l_start], len - l_start, lno);+		    return -1;+		}+	    +		if (last)+		    last->next = h_tag;+		else+		    h_type->tag = h_tag;++		last = h_tag;+		i = j;+	    } while (i < len && hdr[i] != '\0' && hdr[i] != '\n');+	}++	/* Update RG/SQ hashes */+	if (-1 == sam_hdr_update_hashes(sh, type, h_type))+	    return -1;+    }++    return 0;+}++/*+ * Adds a single line to a SAM header.+ * Specify type and one or more key,value pairs, ending with the NULL key.+ * Eg. sam_hdr_add(h, "SQ", "ID", "foo", "LN", "100", NULL).+ *+ * Returns index for specific entry on success (eg 2nd SQ, 4th RG)+ *        -1 on failure+ */+int sam_hdr_add(SAM_hdr *sh, const char *type, ...) {+    va_list args;+    va_start(args, type);+    return sam_hdr_vadd(sh, type, args, NULL);+}++/* + * sam_hdr_add with a va_list interface.+ *+ * Note: this function invokes va_arg at least once, making the value+ * of ap indeterminate after the return.  The caller should call+ * va_start/va_end before/after calling this function or use va_copy.+ */+int sam_hdr_vadd(SAM_hdr *sh, const char *type, va_list ap, ...) {+    va_list args;+    SAM_hdr_type *h_type;+    SAM_hdr_tag *h_tag, *last;+    int new;+    khint32_t type_i = (type[0]<<8) | type[1], k;++    if (EOF == kputc_('@', &sh->text))+	return -1;+    if (EOF == kputsn(type, 2, &sh->text))+	return -1;++    if (!(h_type = pool_alloc(sh->type_pool)))+	return -1;+    if (-1 == (k = kh_put(sam_hdr, sh->h, type_i, &new)))+	return -1;++    // Form the ring, either with self or other lines of this type+    if (!new) {+	SAM_hdr_type *t = kh_val(sh->h, k), *p;+	p = t->prev;+	    +	assert(p->next == t);+	p->next = h_type;+	h_type->prev = p;++	t->prev = h_type;+	h_type->next = t;+	h_type->order = p->order + 1;+    } else {+	kh_val(sh->h, k) = h_type;+	h_type->prev = h_type->next = h_type;+	h_type->order = 0;+    }++    last = NULL;++    // Any ... varargs+    va_start(args, ap);+    for (;;) {+	char *k, *v;+	int idx;+	+	if (!(k = (char *)va_arg(args, char *)))+	    break;+	v = va_arg(args, char *);++	if (EOF == kputc_('\t', &sh->text))+	    return -1;++	if (!(h_tag = pool_alloc(sh->tag_pool)))+	    return -1;+	idx = ks_len(&sh->text);+	+	if (EOF == kputs(k, &sh->text))+	    return -1;+	if (EOF == kputc_(':', &sh->text))+	    return -1;+	if (EOF == kputs(v, &sh->text))+	    return -1;++	h_tag->len = ks_len(&sh->text) - idx;+	h_tag->str = string_ndup(sh->str_pool,+				 ks_str(&sh->text) + idx,+				 h_tag->len);+	h_tag->next = NULL;+	if (!h_tag->str)+	    return -1;++	if (last)+	    last->next = h_tag;+	else+	    h_type->tag = h_tag;+	+	last = h_tag;+    }+    va_end(args);++    // Plus the specified va_list params+    for (;;) {+	char *k, *v;+	int idx;+	+	if (!(k = (char *)va_arg(ap, char *)))+	    break;+	v = va_arg(ap, char *);++	if (EOF == kputc_('\t', &sh->text))+	    return -1;++	if (!(h_tag = pool_alloc(sh->tag_pool)))+	    return -1;+	idx = ks_len(&sh->text);+	+	if (EOF == kputs(k, &sh->text))+	    return -1;+	if (EOF == kputc_(':', &sh->text))+	    return -1;+	if (EOF == kputs(v, &sh->text))+	    return -1;++	h_tag->len = ks_len(&sh->text) - idx;+	h_tag->str = string_ndup(sh->str_pool,+				 ks_str(&sh->text) + idx,+				 h_tag->len);+	h_tag->next = NULL;+	if (!h_tag->str)+	    return -1;++	if (last)+	    last->next = h_tag;+	else+	    h_type->tag = h_tag;+	+	last = h_tag;+    }+    va_end(ap);++    if (EOF == kputc('\n', &sh->text))+	return -1;++    int itype = (type[0]<<8) | type[1];+    if (-1 == sam_hdr_update_hashes(sh, itype, h_type))+	return -1;++    return h_type->order;+}++/*+ * Returns the first header item matching 'type'. If ID is non-NULL it checks+ * for the tag ID: and compares against the specified ID.+ *+ * Returns NULL if no type/ID is found+ */+SAM_hdr_type *sam_hdr_find(SAM_hdr *hdr, char *type,+			   char *ID_key, char *ID_value) {+    SAM_hdr_type *t1, *t2;+    int itype = (type[0]<<8)|(type[1]);+    khint_t k;++    /* Special case for types we have prebuilt hashes on */+    if (ID_key) {+	if (type[0]   == 'S' && type[1]   == 'Q' &&+	    ID_key[0] == 'S' && ID_key[1] == 'N') {+	    k = kh_get(m_s2i, hdr->ref_hash, ID_value);+	    return k != kh_end(hdr->ref_hash)+		? hdr->ref[kh_val(hdr->ref_hash, k)].ty+		: NULL;+	}++	if (type[0]   == 'R' && type[1]   == 'G' &&+	    ID_key[0] == 'I' && ID_key[1] == 'D') {+	    k = kh_get(m_s2i, hdr->rg_hash, ID_value);+	    return k != kh_end(hdr->rg_hash)+		? hdr->rg[kh_val(hdr->rg_hash, k)].ty+		: NULL;+	}++	if (type[0]   == 'P' && type[1]   == 'G' &&+	    ID_key[0] == 'I' && ID_key[1] == 'D') {+	    k = kh_get(m_s2i, hdr->pg_hash, ID_value);+	    return k != kh_end(hdr->pg_hash)+		? hdr->pg[kh_val(hdr->pg_hash, k)].ty+		: NULL;+	}+    }++    k = kh_get(sam_hdr, hdr->h, itype);+    if (k == kh_end(hdr->h))+	return NULL;+    +    if (!ID_key)+	return kh_val(hdr->h, k);++    t1 = t2 = kh_val(hdr->h, k);+    do {+	SAM_hdr_tag *tag;+	for (tag = t1->tag; tag; tag = tag->next) {+	    if (tag->str[0] == ID_key[0] && tag->str[1] == ID_key[1]) {+		char *cp1 = tag->str+3;+		char *cp2 = ID_value;+		while (*cp1 && *cp1 == *cp2)+		    cp1++, cp2++;+		if (*cp2 || *cp1)+		    continue;+		return t1;+	    }+	}+	t1 = t1->next;+    } while (t1 != t2);++    return NULL;+}++/*+ * As per SAM_hdr_type, but returns a complete line of formatted text+ * for a specific head type/ID combination. If ID is NULL then it returns+ * the first line of the specified type.+ *+ * The returned string is malloced and should be freed by the calling+ * function with free().+ *+ * Returns NULL if no type/ID is found.+ */+char *sam_hdr_find_line(SAM_hdr *hdr, char *type,+			char *ID_key, char *ID_value) {+    SAM_hdr_type *ty = sam_hdr_find(hdr, type, ID_key, ID_value);+    kstring_t ks = KS_INITIALIZER;+    SAM_hdr_tag *tag;+    int r = 0;++    if (!ty)+	return NULL;++    // Paste together the line from the hashed copy+    r |= (kputc_('@', &ks) == EOF);+    r |= (kputs(type, &ks) == EOF);+    for (tag = ty->tag; tag; tag = tag->next) {+	r |= (kputc_('\t', &ks) == EOF);+	r |= (kputsn(tag->str, tag->len, &ks) == EOF);+    }++    if (r) {+	KS_FREE(&ks);+	return NULL;+    }++    return ks_str(&ks);+}+++/*+ * Looks for a specific key in a single sam header line.+ * If prev is non-NULL it also fills this out with the previous tag, to+ * permit use in key removal. *prev is set to NULL when the tag is the first+ * key in the list. When a tag isn't found, prev (if non NULL) will be the last+ * tag in the existing list.+ *+ * Returns the tag pointer on success+ *         NULL on failure+ */+SAM_hdr_tag *sam_hdr_find_key(SAM_hdr *sh,+			      SAM_hdr_type *type,+			      char *key,+			      SAM_hdr_tag **prev) {+    SAM_hdr_tag *tag, *p = NULL;++    for (tag = type->tag; tag; p = tag, tag = tag->next) {+	if (tag->str[0] == key[0] && tag->str[1] == key[1]) {+	    if (prev)+		*prev = p;+	    return tag;+	}+    }++    if (prev)+	*prev = p;++    return NULL;+}+++/*+ * Adds or updates tag key,value pairs in a header line.+ * Eg for adding M5 tags to @SQ lines or updating sort order for the+ * @HD line (although use the sam_hdr_sort_order() function for+ * HD manipulation, which is a wrapper around this funuction).+ *+ * Specify multiple key,value pairs ending in NULL.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int sam_hdr_update(SAM_hdr *hdr, SAM_hdr_type *type, ...) {+    va_list ap;++    va_start(ap, type);+    +    for (;;) {+	char *k, *v;+	int idx;+	SAM_hdr_tag *tag, *prev;++	if (!(k = (char *)va_arg(ap, char *)))+	    break;+	v = va_arg(ap, char *);++	tag = sam_hdr_find_key(hdr, type, k, &prev);+	if (!tag) {+	    if (!(tag = pool_alloc(hdr->tag_pool)))+		return -1;+	    if (prev)+		prev->next = tag;+	    else+		type->tag = tag;++	    tag->next = NULL;+	}++	idx = ks_len(&hdr->text);+	if (ksprintf(&hdr->text, "%2.2s:%s", k, v) < 0)+	    return -1;+	tag->len = ks_len(&hdr->text) - idx;+	tag->str = string_ndup(hdr->str_pool,+			       ks_str(&hdr->text) + idx,+			       tag->len);+	if (!tag->str)+	    return -1;+    }++    va_end(ap);++    return 0;+}++#define K(a) (((a)[0]<<8)|((a)[1]))++/*+ * Returns the sort order:+ */+enum sam_sort_order sam_hdr_sort_order(SAM_hdr *hdr) {+    return hdr->sort_order;+}++static enum sam_sort_order sam_hdr_parse_sort_order(SAM_hdr *hdr) {+    khint_t k;+    enum sam_sort_order so;++    so = ORDER_UNKNOWN;+    k = kh_get(sam_hdr, hdr->h, K("HD"));+    if (k != kh_end(hdr->h)) {+	SAM_hdr_type *ty = kh_val(hdr->h, k);+	SAM_hdr_tag *tag;+        for (tag = ty->tag; tag; tag = tag->next) {+	    if (tag->str[0] == 'S' && tag->str[1] == 'O') {+		if (strcmp(tag->str+3, "unsorted") == 0)+		    so = ORDER_UNSORTED;+		else if (strcmp(tag->str+3, "queryname") == 0)+		    so = ORDER_NAME;+		else if (strcmp(tag->str+3, "coordinate") == 0)+		    so = ORDER_COORD;+		else if (strcmp(tag->str+3, "unknown") != 0)+		    fprintf(stderr, "Unknown sort order field: %s\n",+			    tag->str+3);+	    }+	}+    }++    return so;+}+++/*+ * Reconstructs the kstring from the header hash table.+ * Returns 0 on success+ *        -1 on failure+ */+int sam_hdr_rebuild(SAM_hdr *hdr) {+    /* Order: HD then others */+    kstring_t ks = KS_INITIALIZER;+    khint_t k;+++    k = kh_get(sam_hdr, hdr->h, K("HD"));+    if (k != kh_end(hdr->h)) {+	SAM_hdr_type *ty = kh_val(hdr->h, k);+	SAM_hdr_tag *tag;+	if (EOF == kputs("@HD", &ks))+	    return -1;+	for (tag = ty->tag; tag; tag = tag->next) {+	    if (EOF == kputc_('\t', &ks))+		return -1;+	    if (EOF == kputsn_(tag->str, tag->len, &ks))+		return -1;+	}+	if (EOF == kputc('\n', &ks))+	    return -1;+    }++    for (k = kh_begin(hdr->h); k != kh_end(hdr->h); k++) {+	SAM_hdr_type *t1, *t2;++	if (!kh_exist(hdr->h, k))+	    continue;++	if (kh_key(hdr->h, k) == K("HD"))+	    continue;++	t1 = t2 = kh_val(hdr->h, k);+	do {+	    SAM_hdr_tag *tag;+	    char c[2];++	    if (EOF == kputc_('@', &ks))+		return -1;+	    c[0] = kh_key(hdr->h, k)>>8;+	    c[1] = kh_key(hdr->h, k)&0xff;+	    if (EOF == kputsn_(c, 2, &ks))+		return -1;+	    for (tag = t1->tag; tag; tag=tag->next) {+		if (EOF == kputc_('\t', &ks))+		    return -1;+		if (EOF == kputsn_(tag->str, tag->len, &ks))+		    return -1;+	    }+	    if (EOF == kputc('\n', &ks))+		return -1;+	    t1 = t1->next;+	} while (t1 != t2);+    }++    if (ks_str(&hdr->text))+	KS_FREE(&hdr->text);++    hdr->text = ks;++    return 0;+}+++/*+ * Creates an empty SAM header, ready to be populated.+ * + * Returns a SAM_hdr struct on success (free with sam_hdr_free())+ *         NULL on failure+ */+SAM_hdr *sam_hdr_new() {+    SAM_hdr *sh = calloc(1, sizeof(*sh));++    if (!sh)+	return NULL;+    +    sh->h = kh_init(sam_hdr);+    if (!sh->h)+	goto err;++    sh->ID_cnt = 1;+    sh->ref_count = 1;++    sh->nref = 0;+    sh->ref  = NULL;+    if (!(sh->ref_hash = kh_init(m_s2i)))+	goto err;++    sh->nrg = 0;+    sh->rg  = NULL;+    if (!(sh->rg_hash = kh_init(m_s2i)))+	goto err;++    sh->npg = 0;+    sh->pg  = NULL;+    sh->npg_end = sh->npg_end_alloc = 0;+    sh->pg_end = NULL;+    if (!(sh->pg_hash = kh_init(m_s2i)))+	goto err;++    KS_INIT(&sh->text);++    if (!(sh->tag_pool = pool_create(sizeof(SAM_hdr_tag))))+	goto err;++    if (!(sh->type_pool = pool_create(sizeof(SAM_hdr_type))))+	goto err;++    if (!(sh->str_pool = string_pool_create(8192)))+	goto err;++    return sh;++ err:+    if (sh->h)+	kh_destroy(sam_hdr, sh->h);++    if (sh->tag_pool)+	pool_destroy(sh->tag_pool);++    if (sh->type_pool)+	pool_destroy(sh->type_pool);++    if (sh->str_pool)+	string_pool_destroy(sh->str_pool);++    free(sh);++    return NULL;+}+++/*+ * Tokenises a SAM header into a hash table.+ * Also extracts a few bits on specific data types, such as @RG lines.+ *+ * Returns a SAM_hdr struct on success (free with sam_hdr_free())+ *         NULL on failure+ */+SAM_hdr *sam_hdr_parse_(const char *hdr, int len) {+    /* Make an empty SAM_hdr */+    SAM_hdr *sh;+    +    sh = sam_hdr_new();+    if (NULL == sh) return NULL;++    if (NULL == hdr) return sh; // empty header is permitted++    /* Parse the header, line by line */+    if (-1 == sam_hdr_add_lines(sh, hdr, len)) {+	sam_hdr_free(sh);+	return NULL;+    }++    /* Obtain sort order */+    sh->sort_order = sam_hdr_parse_sort_order(sh);++    //sam_hdr_dump(sh);+    //sam_hdr_add(sh, "RG", "ID", "foo", "SM", "bar", NULL);+    //sam_hdr_rebuild(sh);+    //printf(">>%s<<", ks_str(sh->text));++    //parse_references(sh);+    //parse_read_groups(sh);++    sam_hdr_link_pg(sh);+    //sam_hdr_dump(sh);++    return sh;+}++/*+ * Produces a duplicate copy of hdr and returns it.+ * Returns NULL on failure+ */+SAM_hdr *sam_hdr_dup(SAM_hdr *hdr) {+    if (-1 == sam_hdr_rebuild(hdr))+	return NULL;++    return sam_hdr_parse_(sam_hdr_str(hdr), sam_hdr_length(hdr));+}++/*! Increments a reference count on hdr.+ *+ * This permits multiple files to share the same header, all calling+ * sam_hdr_free when done, without causing errors for other open  files.+ */+void sam_hdr_incr_ref(SAM_hdr *hdr) {+    hdr->ref_count++;+}++/*! Increments a reference count on hdr.+ *+ * This permits multiple files to share the same header, all calling+ * sam_hdr_free when done, without causing errors for other open  files.+ *+ * If the reference count hits zero then the header is automatically+ * freed. This makes it a synonym for sam_hdr_free().+ */+void sam_hdr_decr_ref(SAM_hdr *hdr) {+    sam_hdr_free(hdr);+}++/*! Deallocates all storage used by a SAM_hdr struct.+ *+ * This also decrements the header reference count. If after decrementing + * it is still non-zero then the header is assumed to be in use by another+ * caller and the free is not done.+ *+ * This is a synonym for sam_hdr_dec_ref().+ */+void sam_hdr_free(SAM_hdr *hdr) {+    if (!hdr)+	return;++    if (--hdr->ref_count > 0)+	return;++    if (ks_str(&hdr->text))+	KS_FREE(&hdr->text);++    if (hdr->h)+	kh_destroy(sam_hdr, hdr->h);++    if (hdr->ref_hash)+	kh_destroy(m_s2i, hdr->ref_hash);++    if (hdr->ref) {+	int i;+	for (i = 0; i < hdr->nref; i++)+	    if (hdr->ref[i].name)+		free(hdr->ref[i].name);+	free(hdr->ref);+    }++    if (hdr->rg_hash)+	kh_destroy(m_s2i, hdr->rg_hash);++    if (hdr->rg) {+	int i;+	for (i = 0; i < hdr->nrg; i++)+	    if (hdr->rg[i].name)+		free(hdr->rg[i].name);+	free(hdr->rg);+    }++    if (hdr->pg_hash)+	kh_destroy(m_s2i, hdr->pg_hash);++    if (hdr->pg) {+	int i;+	for (i = 0; i < hdr->npg; i++)+	    if (hdr->pg[i].name)+		free(hdr->pg[i].name);+	free(hdr->pg);+    }++    if (hdr->pg_end)+	free(hdr->pg_end);++    if (hdr->type_pool)+	pool_destroy(hdr->type_pool);++    if (hdr->tag_pool)+	pool_destroy(hdr->tag_pool);++    if (hdr->str_pool)+	string_pool_destroy(hdr->str_pool);++    free(hdr);+}++int sam_hdr_length(SAM_hdr *hdr) {+    return ks_len(&hdr->text);+}++char *sam_hdr_str(SAM_hdr *hdr) {+    return ks_str(&hdr->text);+}++/*+ * Looks up a reference sequence by name and returns the numerical ID.+ * Returns -1 if unknown reference.+ */+int sam_hdr_name2ref(SAM_hdr *hdr, const char *ref) {+    khint_t k = kh_get(m_s2i, hdr->ref_hash, ref);+    return k == kh_end(hdr->ref_hash) ? -1 : kh_val(hdr->ref_hash, k);+}++/*+ * Looks up a read-group by name and returns a pointer to the start of the+ * associated tag list.+ *+ * Returns NULL on failure+ */+SAM_RG *sam_hdr_find_rg(SAM_hdr *hdr, const char *rg) {+    khint_t k = kh_get(m_s2i, hdr->rg_hash, rg);+    return k == kh_end(hdr->rg_hash)+	? NULL+	: &hdr->rg[kh_val(hdr->rg_hash, k)];+}+++/*+ * Fixes any PP links in @PG headers.+ * If the entries are in order then this doesn't need doing, but incase+ * our header is out of order this goes through the sh->pg[] array+ * setting the prev_id field.+ *+ * Note we can have multiple complete chains. This code should identify the+ * tails of these chains as these are the entries we have to link to in+ * subsequent PP records.+ *+ * Returns 0 on sucess+ *        -1 on failure (indicating broken PG/PP records)+ */+int sam_hdr_link_pg(SAM_hdr *hdr) {+    int i, j, ret = 0;++    hdr->npg_end_alloc = hdr->npg;+    hdr->pg_end = realloc(hdr->pg_end, hdr->npg * sizeof(*hdr->pg_end));+    if (!hdr->pg_end)+	return -1;++    for (i = 0; i < hdr->npg; i++)+	hdr->pg_end[i] = i;++    for (i = 0; i < hdr->npg; i++) {+	khint_t k;+	SAM_hdr_tag *tag;+	char tmp;++	for (tag = hdr->pg[i].tag; tag; tag = tag->next) {+	    if (tag->str[0] == 'P' && tag->str[1] == 'P')+		break;+	}+	if (!tag) {+	    /* Chain start points */+	    continue;+	}++	tmp = tag->str[tag->len]; tag->str[tag->len] = 0;+	k = kh_get(m_s2i, hdr->pg_hash, tag->str+3);+	tag->str[tag->len] = tmp;++	if (k == kh_end(hdr->pg_hash)) {+	    ret = -1;+	    continue;+	}++	hdr->pg[i].prev_id = hdr->pg[kh_val(hdr->pg_hash, k)].id;+	hdr->pg_end[kh_val(hdr->pg_hash, k)] = -1;+    }++    for (i = j = 0; i < hdr->npg; i++) {+	if (hdr->pg_end[i] != -1)+	    hdr->pg_end[j++] = hdr->pg_end[i];+    }+    hdr->npg_end = j;++    return ret;+}++/*+ * Returns a unique ID from a base name.+ *+ * The value returned is valid until the next call to+ * this function.+ */+const char *sam_hdr_PG_ID(SAM_hdr *sh, const char *name) {+    khint_t k = kh_get(m_s2i, sh->pg_hash, name);+    if (k == kh_end(sh->pg_hash))+	return name;++    do {+	sprintf(sh->ID_buf, "%.1000s.%d", name, sh->ID_cnt++);+	k = kh_get(m_s2i, sh->pg_hash, sh->ID_buf);+    } while (k != kh_end(sh->pg_hash));++    return sh->ID_buf;+}++/*+ * Add an @PG line.+ *+ * If we wish complete control over this use sam_hdr_add() directly. This+ * function uses that, but attempts to do a lot of tedious house work for+ * you too.+ *+ * - It will generate a suitable ID if the supplied one clashes.+ * - It will generate multiple @PG records if we have multiple PG chains.+ *+ * Call it as per sam_hdr_add() with a series of key,value pairs ending+ * in NULL.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int sam_hdr_add_PG(SAM_hdr *sh, const char *name, ...) {+    va_list args;++    if (sh->npg_end) {+	/* Copy ends array to avoid us looping while modifying it */+	int *end = malloc(sh->npg_end * sizeof(int));+	int i, nends = sh->npg_end;++	if (!end)+	    return -1;++	memcpy(end, sh->pg_end, nends * sizeof(*end));++	for (i = 0; i < nends; i++) {+	    va_start(args, name);+	    if (-1 == sam_hdr_vadd(sh, "PG", args,+				   "ID", sam_hdr_PG_ID(sh, name),+				   "PN", name,+				   "PP", sh->pg[end[i]].name,+				   NULL)) {+		free(end);+		return  -1;+	    }+	    va_end(args);+	}++	free(end);+    } else {+	va_start(args, name);+	if (-1 == sam_hdr_vadd(sh, "PG", args,+			       "ID", sam_hdr_PG_ID(sh, name),+			       "PN", name,+			       NULL))+	    return -1;+	va_end(args);+    }++    //sam_hdr_dump(sh);++    return 0;+}++/*+ * A function to help with construction of CL tags in @PG records.+ * Takes an argc, argv pair and returns a single space-separated string.+ * This string should be deallocated by the calling function.+ * + * Returns malloced char * on success+ *         NULL on failure+ */+char *stringify_argv(int argc, char *argv[]) {+    char *str, *cp;+    size_t nbytes = 1;+    int i, j;++    /* Allocate */+    for (i = 0; i < argc; i++) {+	nbytes += strlen(argv[i]) + 1;+    }+    if (!(str = malloc(nbytes)))+	return NULL;++    /* Copy */+    cp = str;+    for (i = 0; i < argc; i++) {+	j = 0;+	while (argv[i][j]) {+	    if (argv[i][j] == '\t')+		*cp++ = ' ';+	    else+		*cp++ = argv[i][j];+	    j++;+	}+	*cp++ = ' ';+    }+    *cp++ = 0;++    return str;+}
+ htslib-1.3.2/cram/sam_header.h view
@@ -0,0 +1,459 @@+/*+Copyright (c) 2013-2014 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*! \file+ * SAM header parsing.+ *+ * These functions can be shared between SAM, BAM and CRAM file+ * formats as all three internally use the same string encoding for+ * header fields.+ */ ++/*+ * TODO.+ *+ * - Sort order (parse to struct, enum type, updating funcs)+ * - Removal of lines.+ * - Updating of lines+ */++#ifndef _SAM_HDR_H_+#define _SAM_HDR_H_++#include <stdarg.h>++#include "cram/string_alloc.h"+#include "cram/pooled_alloc.h"++#include "htslib/khash.h"+#include "htslib/kstring.h"++#ifdef __cplusplus+extern "C" {+#endif++// For structure assignment. Eg kstring_t s = KS_INITIALIZER;+#define KS_INITIALIZER {0,0,0}++// For initialisation elsewhere. Eg KS_INIT(x->str);+#define KS_INIT(ks) ((ks)->l = 0, (ks)->m = 0, (ks)->s = NULL)++// Frees the string subfield only. Assumes 's' itself is static.+#define KS_FREE(ks) do { if ((ks)->s) free((ks)->s); } while(0)++/*+ * Proposed new SAM header parsing++1 @SQ ID:foo LN:100+2 @SQ ID:bar LN:200+3 @SQ ID:ram LN:300 UR:xyz+4 @RG ID:r ...+5 @RG ID:s ...++Hash table for 2-char @keys without dup entries.+If dup lines, we form a circular linked list. Ie hash keys = {RG, SQ}.++HASH("SQ")--\+            |+    (3) <-> 1 <-> 2 <-> 3 <-> (1)++HASH("RG")--\+            |+    (5) <-> 4 <-> 5 <-> (4)++Items stored in the hash values also form their own linked lists:+Ie SQ->ID(foo)->LN(100)+   SQ->ID(bar)->LN(200)+   SQ->ID(ram)->LN(300)->UR(xyz)+   RG->ID(r)+ */++/*! A single key:value pair on a header line+ *+ * These form a linked list and hold strings. The strings are+ * allocated from a string_alloc_t pool referenced in the master+ * SAM_hdr structure. Do not attempt to free, malloc or manipulate+ * these strings directly.+ */+typedef struct SAM_hdr_tag_s {+    struct SAM_hdr_tag_s *next;+    char *str;+    int   len;+} SAM_hdr_tag;++/*! The parsed version of the SAM header string.+ * + * Each header type (SQ, RG, HD, etc) points to its own SAM_hdr_type+ * struct via the main hash table h in the SAM_hdr struct.+ *+ * These in turn consist of circular bi-directional linked lists (ie+ * rings) to hold the multiple instances of the same header type+ * code. For example if we have 5 \@SQ lines the primary hash table+ * will key on \@SQ pointing to the first SAM_hdr_type and that in turn+ * will be part of a ring of 5 elements.+ *+ * For each SAM_hdr_type structure we also point to a SAM_hdr_tag+ * structure which holds the tokenised attributes; the tab separated+ * key:value pairs per line.+ */+typedef struct SAM_hdr_item_s {+    struct SAM_hdr_item_s *next; // cirular+    struct SAM_hdr_item_s *prev;+    SAM_hdr_tag *tag;            // first tag+    int order;                   // 0 upwards+} SAM_hdr_type;++/*! Parsed \@SQ lines */+typedef struct {+    char *name;+    uint32_t len;+    SAM_hdr_type *ty;+    SAM_hdr_tag  *tag;+} SAM_SQ;++/*! Parsed \@RG lines */+typedef struct {+    char *name;+    SAM_hdr_type *ty;+    SAM_hdr_tag  *tag;+    int name_len;+    int id;           // numerical ID+} SAM_RG;++/*! Parsed \@PG lines */+typedef struct {+    char *name;+    SAM_hdr_type *ty;+    SAM_hdr_tag  *tag;+    int name_len;+    int id;           // numerical ID+    int prev_id;      // -1 if none+} SAM_PG;++/*! Sort order parsed from @HD line */+enum sam_sort_order {+    ORDER_UNKNOWN  =-1,+    ORDER_UNSORTED = 0,+    ORDER_NAME     = 1,+    ORDER_COORD    = 2,+  //ORDER_COLLATE  = 3 // maybe one day!+};++KHASH_MAP_INIT_INT(sam_hdr, SAM_hdr_type*)+KHASH_MAP_INIT_STR(m_s2i, int)++/*! Primary structure for header manipulation+ *+ * The initial header text is held in the text kstring_t, but is also+ * parsed out into SQ, RG and PG arrays. These have a hash table+ * associated with each to allow lookup by ID or SN fields instead of+ * their numeric array indices. Additionally PG has an array to hold+ * the linked list start points (the last in a PP chain).+ *+ * Use the appropriate sam_hdr_* functions to edit the header, and + * call sam_hdr_rebuild() any time the textual form needs to be+ * updated again.+ */+typedef struct {+    kstring_t text;           //!< concatenated text, indexed by SAM_hdr_tag+    khash_t(sam_hdr) *h;+    string_alloc_t *str_pool; //!< Pool of SAM_hdr_tag->str strings+    pool_alloc_t   *type_pool;//!< Pool of SAM_hdr_type structs+    pool_alloc_t   *tag_pool; //!< Pool of SAM_hdr_tag structs++    // @SQ lines / references+    int nref;                 //!< Number of \@SQ lines+    SAM_SQ *ref;              //!< Array of parsed \@SQ lines+    khash_t(m_s2i) *ref_hash; //!< Maps SQ SN field to sq[] index++    // @RG lines / read-groups+    int nrg;                  //!< Number of \@RG lines+    SAM_RG *rg;               //!< Array of parsed \@RG lines+    khash_t(m_s2i) *rg_hash;  //!< Maps RG ID field to rg[] index++    // @PG lines / programs+    int npg;                  //!< Number of \@PG lines+    int npg_end;              //!< Number of terminating \@PG lines+    int npg_end_alloc;        //!< Size of pg_end field+    SAM_PG *pg;		      //!< Array of parsed \@PG lines+    khash_t(m_s2i) *pg_hash;  //!< Maps PG ID field to pg[] index+    int *pg_end;              //!< \@PG chain termination IDs++    // @HD data+    enum sam_sort_order sort_order; //!< @HD SO: field++    // @cond internal+    char ID_buf[1024];  // temporary buffer+    int ID_cnt;+    int ref_count;      // number of uses of this SAM_hdr+    // @endcond+} SAM_hdr;++/*! Creates an empty SAM header, ready to be populated.+ * + * @return+ * Returns a SAM_hdr struct on success (free with sam_hdr_free())+ *         NULL on failure+ */+SAM_hdr *sam_hdr_new(void);++/*! Tokenises a SAM header into a hash table.+ *+ * Also extracts a few bits on specific data types, such as @RG lines.+ *+ * @return+ * Returns a SAM_hdr struct on success (free with sam_hdr_free());+ *         NULL on failure+ */+SAM_hdr *sam_hdr_parse_(const char *hdr, int len);+++/*! Produces a duplicate copy of hdr and returns it.+ * @return+ * Returns NULL on failure+ */+SAM_hdr *sam_hdr_dup(SAM_hdr *hdr);+++/*! Increments a reference count on hdr.+ *+ * This permits multiple files to share the same header, all calling+ * sam_hdr_free when done, without causing errors for other open  files.+ */+void sam_hdr_incr_ref(SAM_hdr *hdr);+++/*! Increments a reference count on hdr.+ *+ * This permits multiple files to share the same header, all calling+ * sam_hdr_free when done, without causing errors for other open  files.+ *+ * If the reference count hits zero then the header is automatically+ * freed. This makes it a synonym for sam_hdr_free().+ */+void sam_hdr_decr_ref(SAM_hdr *hdr);+++/*! Deallocates all storage used by a SAM_hdr struct.+ *+ * This also decrements the header reference count. If after decrementing + * it is still non-zero then the header is assumed to be in use by another+ * caller and the free is not done.+ *+ * This is a synonym for sam_hdr_dec_ref().+ */+void sam_hdr_free(SAM_hdr *hdr);++/*! Returns the current length of the SAM_hdr in text form.+ *+ * Call sam_hdr_rebuild() first if editing has taken place.+ */+int sam_hdr_length(SAM_hdr *hdr);++/*! Returns the string form of the SAM_hdr.+ *+ * Call sam_hdr_rebuild() first if editing has taken place.+ */+char *sam_hdr_str(SAM_hdr *hdr);++/*! Appends a formatted line to an existing SAM header.+ *+ * Line is a full SAM header record, eg "@SQ\tSN:foo\tLN:100", with+ * optional new-line. If it contains more than 1 line then multiple lines+ * will be added in order.+ *+ * Input text is of maximum length len or as terminated earlier by a NUL.+ * Len may be 0 if unknown, in which case lines must be NUL-terminated.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_add_lines(SAM_hdr *sh, const char *lines, int len);++/*! Adds a single line to a SAM header.+ *+ * Specify type and one or more key,value pairs, ending with the NULL key.+ * Eg. sam_hdr_add(h, "SQ", "ID", "foo", "LN", "100", NULL).+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_add(SAM_hdr *sh, const char *type, ...);++/*! Adds a single line to a SAM header.+ *+ * This is much like sam_hdr_add() but with the additional va_list+ * argument. This is followed by specifying type and one or more+ * key,value pairs, ending with the NULL key.+ *+ * Eg. sam_hdr_vadd(h, "SQ", args, "ID", "foo", "LN", "100", NULL).+ *+ * The purpose of the additional va_list parameter is to permit other+ * varargs functions to call this while including their own additional+ * parameters; an example is in sam_hdr_add_PG().+ *+ * Note: this function invokes va_arg at least once, making the value+ * of ap indeterminate after the return.  The caller should call+ * va_start/va_end before/after calling this function or use va_copy.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_vadd(SAM_hdr *sh, const char *type, va_list ap, ...);++/*!+ * @return+ * Returns the first header item matching 'type'. If ID is non-NULL it checks+ * for the tag ID: and compares against the specified ID.+ *+ * Returns NULL if no type/ID is found+ */+SAM_hdr_type *sam_hdr_find(SAM_hdr *hdr, char *type,+			   char *ID_key, char *ID_value);++/*!+ *+ * As per SAM_hdr_type, but returns a complete line of formatted text+ * for a specific head type/ID combination. If ID is NULL then it returns+ * the first line of the specified type.+ *+ * The returned string is malloced and should be freed by the calling+ * function with free().+ *+ * @return+ * Returns NULL if no type/ID is found.+ */+char *sam_hdr_find_line(SAM_hdr *hdr, char *type,+			char *ID_key, char *ID_value);++/*! Looks for a specific key in a single sam header line.+ *+ * If prev is non-NULL it also fills this out with the previous tag, to+ * permit use in key removal. *prev is set to NULL when the tag is the first+ * key in the list. When a tag isn't found, prev (if non NULL) will be the last+ * tag in the existing list.+ *+ * @return+ * Returns the tag pointer on success;+ *         NULL on failure+ */+SAM_hdr_tag *sam_hdr_find_key(SAM_hdr *sh,+			      SAM_hdr_type *type,+			      char *key,+			      SAM_hdr_tag **prev);++/*! Adds or updates tag key,value pairs in a header line.+ *+ * Eg for adding M5 tags to @SQ lines or updating sort order for the+ * @HD line (although use the sam_hdr_sort_order() function for+ * HD manipulation, which is a wrapper around this funuction).+ *+ * Specify multiple key,value pairs ending in NULL.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_update(SAM_hdr *hdr, SAM_hdr_type *type, ...);++/*! Returns the sort order from the @HD SO: field */+enum sam_sort_order sam_hdr_sort_order(SAM_hdr *hdr);++/*! Reconstructs the kstring from the header hash table.+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_rebuild(SAM_hdr *hdr);++/*! Looks up a reference sequence by name and returns the numerical ID.+ * @return+ * Returns -1 if unknown reference.+ */+int sam_hdr_name2ref(SAM_hdr *hdr, const char *ref);++/*! Looks up a read-group by name and returns a pointer to the start of the+ * associated tag list.+ *+ * @return+ * Returns NULL on failure+ */+SAM_RG *sam_hdr_find_rg(SAM_hdr *hdr, const char *rg);++/*! Fixes any PP links in @PG headers.+ *+ * If the entries are in order then this doesn't need doing, but incase+ * our header is out of order this goes through the sh->pg[] array+ * setting the prev_id field.+ *+ * @return+ * Returns 0 on sucess;+ *        -1 on failure (indicating broken PG/PP records)+ */+int sam_hdr_link_pg(SAM_hdr *hdr);+++/*! Add an @PG line.+ *+ * If we wish complete control over this use sam_hdr_add() directly. This+ * function uses that, but attempts to do a lot of tedious house work for+ * you too.+ *+ * - It will generate a suitable ID if the supplied one clashes.+ * - It will generate multiple @PG records if we have multiple PG chains.+ *+ * Call it as per sam_hdr_add() with a series of key,value pairs ending+ * in NULL.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_add_PG(SAM_hdr *sh, const char *name, ...);++/*!+ * A function to help with construction of CL tags in @PG records.+ * Takes an argc, argv pair and returns a single space-separated string.+ * This string should be deallocated by the calling function.+ * + * @return+ * Returns malloced char * on success;+ *         NULL on failure+ */+char *stringify_argv(int argc, char *argv[]);++#ifdef __cplusplus+}+#endif++#endif /* _SAM_HDR_H_ */
+ htslib-1.3.2/cram/string_alloc.c view
@@ -0,0 +1,155 @@+/*+Copyright (c) 2010 Genome Research Ltd.+Author: Andrew Whitwham <aw7@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/+++/* +   A pooled string allocator intended to cut down on the+   memory overhead of many small string allocations.+   +   Andrew Whitwham, September 2010.+*/++#include <config.h>++#include <string.h>+#include <stdlib.h>+#include <stdio.h>++#include "string_alloc.h"++#define MIN_STR_SIZE 1024+++/* creates the string pool. max_length is the initial size+   a single string can be.  Tha max_length can grow as+   needed */++string_alloc_t *string_pool_create(size_t max_length) {+    string_alloc_t *a_str;+    +    if (NULL == (a_str = (string_alloc_t *)malloc(sizeof(*a_str)))) {+    	return NULL;+    }+    +    if (max_length < MIN_STR_SIZE) max_length = MIN_STR_SIZE;+    +    a_str->nstrings    = 0;+    a_str->max_length  = max_length;+    a_str->strings     = NULL;+    +    return a_str;+}+++/* internal function to do the actual memory allocation */++static string_t *new_string_pool(string_alloc_t *a_str) {+    string_t *str;+    +    str = realloc(a_str->strings, (a_str->nstrings + 1) * sizeof(*a_str->strings));+    +    if (NULL == str) return NULL;+    +    a_str->strings = str;+    str = &a_str->strings[a_str->nstrings];+    +    str->str = malloc(a_str->max_length);;+    +    if (NULL == str->str) return NULL;+    +    str->used = 0;+    a_str->nstrings++;+    +    return str;+}+++/* free allocated memory */++void string_pool_destroy(string_alloc_t *a_str) {+    size_t i;+    +    for (i = 0; i < a_str->nstrings; i++) {+    	free(a_str->strings[i].str);+    }+    +    free(a_str->strings);+    free(a_str);+}+++/* allocate space for a string */++char *string_alloc(string_alloc_t *a_str, size_t length) {+    string_t *str;+    char *ret;+    +    if (length <= 0) return NULL;+    +    // add to last string pool if we have space+    if (a_str->nstrings) {+    	str = &a_str->strings[a_str->nstrings - 1];+	+	if (str->used + length < a_str->max_length) {+	    ret = str->str + str->used;+	    str->used += length;+	    return ret;+	}+    }+    +    // increase the max length if needs be+    if (length > a_str->max_length) a_str->max_length = length;+	+    // need a new string pool +    str = new_string_pool(a_str);+    +    if (NULL == str) return NULL;+    +    str->used = length;+    return str->str;+}+++/* equivalent to strdup */++char *string_dup(string_alloc_t *a_str, char *instr) {+    return string_ndup(a_str, instr, strlen(instr));+}++char *string_ndup(string_alloc_t *a_str, char *instr, size_t len) {+    char *str = string_alloc(a_str, len + 1);+    +    if (NULL == str) return NULL;+    +    strncpy(str, instr, len);+    str[len] = 0;+    +    return str;+}
+ htslib-1.3.2/cram/string_alloc.h view
@@ -0,0 +1,68 @@+/*+Copyright (c) 2010 Genome Research Ltd.+Author: Andrew Whitwham <aw7@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _STRING_ALLOC_H_+#define _STRING_ALLOC_H_++#include <stdlib.h>++#ifdef __cplusplus+extern "C" {+#endif++/* + * A pooled string allocator intended to cut down on the+ * memory overhead of many small string allocations.+ *+ * Andrew Whitwham, September 2010.+ */++typedef struct {+    char *str;+    size_t used;+} string_t;++typedef struct {+    size_t max_length;+    size_t nstrings;+    string_t *strings;+} string_alloc_t;++string_alloc_t *string_pool_create(size_t max_length);+void string_pool_destroy(string_alloc_t *a_str);+char *string_alloc(string_alloc_t *a_str, size_t length);+char *string_dup(string_alloc_t *a_str, char *instr);+char *string_ndup(string_alloc_t *a_str, char *instr, size_t len);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/cram/thread_pool.c view
@@ -0,0 +1,757 @@+/*+Copyright (c) 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdlib.h>++#include <signal.h>+#include <errno.h>+#include <stdio.h>+#include <string.h>+#include <sys/time.h>+#include <assert.h>++#include "cram/thread_pool.h"++//#define DEBUG+//#define DEBUG_TIME++#define IN_ORDER++#ifdef DEBUG+static int worker_id(t_pool *p) {+    int i;+    pthread_t s = pthread_self();+    for (i = 0; i < p->tsize; i++) {+	if (pthread_equal(s, p->t[i].tid))+	    return i;+    }+    return -1;+}+#endif++/* ----------------------------------------------------------------------------+ * A queue to hold results from the thread pool.+ *+ * Each thread pool may have jobs of multiple types being queued up and+ * interleaved, so we allow several results queue per pool.+ *+ * The jobs themselves are expected to push their results onto their+ * appropriate results queue.+ */++/*+ * Adds a result to the end of the result queue.+ *+ * Returns 0 on success;+ *        -1 on failure+ */+static int t_pool_add_result(t_pool_job *j, void *data) {+    t_results_queue *q = j->q;+    t_pool_result *r;++#ifdef DEBUG+    fprintf(stderr, "%d: Adding resulting to queue %p, serial %d\n",+	    worker_id(j->p), q, j->serial);+#endif++    /* No results queue is fine if we don't want any results back */+    if (!q)+	return 0;++    if (!(r = malloc(sizeof(*r))))+	return -1;++    r->next = NULL;+    r->data = data;+    r->serial = j->serial;++    pthread_mutex_lock(&q->result_m);+    if (q->result_tail) {+	q->result_tail->next = r;+	q->result_tail = r;+    } else {+	q->result_head = q->result_tail = r;+    }+    q->queue_len++;+    q->pending--;++#ifdef DEBUG+    fprintf(stderr, "%d: Broadcasting result_avail (id %d)\n",+	    worker_id(j->p), r->serial);+#endif+    pthread_cond_signal(&q->result_avail_c);+#ifdef DEBUG+    fprintf(stderr, "%d: Broadcast complete\n", worker_id(j->p));+#endif++    pthread_mutex_unlock(&q->result_m);++    return 0;+}++/* Core of t_pool_next_result() */+static t_pool_result *t_pool_next_result_locked(t_results_queue *q) {+    t_pool_result *r, *last;++    for (last = NULL, r = q->result_head; r; last = r, r = r->next) {+	if (r->serial == q->next_serial)+	    break;+    }++    if (r) {+	if (q->result_head == r)+	    q->result_head = r->next;+	else+	    last->next = r->next;++	if (q->result_tail == r)+	    q->result_tail = last;++	if (!q->result_head)+	    q->result_tail = NULL;++	q->next_serial++;+	q->queue_len--;+    }++    return r;+}++/*+ * Pulls a result off the head of the result queue. Caller should+ * free it (and any internals as appropriate) after use. This doesn't+ * wait for a result to be present.+ *+ * Results will be returned in strict order.+ * + * Returns t_pool_result pointer if a result is ready.+ *         NULL if not.+ */+t_pool_result *t_pool_next_result(t_results_queue *q) {+    t_pool_result *r;++#ifdef DEBUG+    fprintf(stderr, "Requesting next result on queue %p\n", q);+#endif++    pthread_mutex_lock(&q->result_m);+    r = t_pool_next_result_locked(q);+    pthread_mutex_unlock(&q->result_m);++#ifdef DEBUG+    fprintf(stderr, "(q=%p) Found %p\n", q, r);+#endif++    return r;+}++t_pool_result *t_pool_next_result_wait(t_results_queue *q) {+    t_pool_result *r;++#ifdef DEBUG+    fprintf(stderr, "Waiting for result %d...\n", q->next_serial);+#endif++    pthread_mutex_lock(&q->result_m);+    while (!(r = t_pool_next_result_locked(q))) {+	/* Possible race here now avoided via _locked() call, but incase... */+	struct timeval now;+	struct timespec timeout;++	gettimeofday(&now, NULL);+	timeout.tv_sec = now.tv_sec + 10;+	timeout.tv_nsec = now.tv_usec * 1000;++	pthread_cond_timedwait(&q->result_avail_c, &q->result_m, &timeout);+    }+    pthread_mutex_unlock(&q->result_m);++    return r;+}++/*+ * Returns true if there are no items on the finished results queue and+ * also none still pending.+ */+int t_pool_results_queue_empty(t_results_queue *q) {+    int empty;++    pthread_mutex_lock(&q->result_m);+    empty = q->queue_len == 0 && q->pending == 0;+    pthread_mutex_unlock(&q->result_m);++    return empty;+}+++/*+ * Returns the number of completed jobs on the results queue.+ */+int t_pool_results_queue_len(t_results_queue *q) {+    int len;++    pthread_mutex_lock(&q->result_m);+    len = q->queue_len;+    pthread_mutex_unlock(&q->result_m);++    return len;+}++int t_pool_results_queue_sz(t_results_queue *q) {+    int len;++    pthread_mutex_lock(&q->result_m);+    len = q->queue_len + q->pending;+    pthread_mutex_unlock(&q->result_m);++    return len;+}++/*+ * Frees a result 'r' and if free_data is true also frees+ * the internal r->data result too.+ */+void t_pool_delete_result(t_pool_result *r, int free_data) {+    if (!r)+	return;++    if (free_data && r->data)+	free(r->data);++    free(r);+}++/*+ * Initialises a results queue.+ *+ * Results queue pointer on success;+ *         NULL on failure+ */+t_results_queue *t_results_queue_init(void) {+    t_results_queue *q = malloc(sizeof(*q));++    pthread_mutex_init(&q->result_m, NULL);+    pthread_cond_init(&q->result_avail_c, NULL);++    q->result_head = NULL;+    q->result_tail = NULL;+    q->next_serial = 0;+    q->curr_serial = 0;+    q->queue_len   = 0;+    q->pending     = 0;++    return q;+}++/* Deallocates memory for a results queue */+void t_results_queue_destroy(t_results_queue *q) {+#ifdef DEBUG+    fprintf(stderr, "Destroying results queue %p\n", q);+#endif++    if (!q)+	return;++    pthread_mutex_destroy(&q->result_m);+    pthread_cond_destroy(&q->result_avail_c);++    memset(q, 0xbb, sizeof(*q));+    free(q);++#ifdef DEBUG+    fprintf(stderr, "Destroyed results queue %p\n", q);+#endif+}++/* ----------------------------------------------------------------------------+ * The thread pool.+ */++#define TDIFF(t2,t1) ((t2.tv_sec-t1.tv_sec)*1000000 + t2.tv_usec-t1.tv_usec)++/*+ * A worker thread.+ *+ * Each thread waits for the pool to be non-empty.+ * As soon as this applies, one of them succeeds in getting the lock+ * and then executes the job.+ */+static void *t_pool_worker(void *arg) {+    t_pool_worker_t *w = (t_pool_worker_t *)arg;+    t_pool *p = w->p;+    t_pool_job *j;+#ifdef DEBUG_TIME+    struct timeval t1, t2, t3;+#endif++    for (;;) {+	// Pop an item off the pool queue+#ifdef DEBUG_TIME+	gettimeofday(&t1, NULL);+#endif++	pthread_mutex_lock(&p->pool_m);++#ifdef DEBUG_TIME+	gettimeofday(&t2, NULL);+	p->wait_time += TDIFF(t2,t1);+	w->wait_time += TDIFF(t2,t1);+#endif++	// If there is something on the job list and a higher priority+	// thread waiting, let it handle this instead.+//	while (p->head && p->t_stack_top != -1 && p->t_stack_top < w->idx) {+//	    pthread_mutex_unlock(&p->pool_m);+//	    pthread_cond_signal(&p->t[p->t_stack_top].pending_c);+//	    pthread_mutex_lock(&p->pool_m);+//	}++	while (!p->head && !p->shutdown) {+	    p->nwaiting++;++	    if (p->njobs == 0)+		pthread_cond_signal(&p->empty_c);+#ifdef DEBUG_TIME+	    gettimeofday(&t2, NULL);+#endif++#ifdef IN_ORDER+	    // Push this thread to the top of the waiting stack+	    if (p->t_stack_top == -1 || p->t_stack_top > w->idx)+		p->t_stack_top = w->idx;++	    p->t_stack[w->idx] = 1;+	    pthread_cond_wait(&w->pending_c, &p->pool_m);+	    p->t_stack[w->idx] = 0;++	    /* Find new t_stack_top */+	    {+		int i;+		p->t_stack_top = -1;+		for (i = 0; i < p->tsize; i++) {+		    if (p->t_stack[i]) {+			p->t_stack_top = i;+			break;+		    }+		}+	    }+#else+	    pthread_cond_wait(&p->pending_c, &p->pool_m);+#endif++#ifdef DEBUG_TIME+	    gettimeofday(&t3, NULL);+	    p->wait_time += TDIFF(t3,t2);+	    w->wait_time += TDIFF(t3,t2);+#endif+	    p->nwaiting--;+	}++	if (p->shutdown) {+#ifdef DEBUG_TIME+	    p->total_time += TDIFF(t3,t1);+#endif+#ifdef DEBUG+	    fprintf(stderr, "%d: Shutting down\n", worker_id(p));+#endif+	    pthread_mutex_unlock(&p->pool_m);+	    pthread_exit(NULL);+	}++	j = p->head;+	if (!(p->head = j->next))+	    p->tail = NULL;++	if (p->njobs-- >= p->qsize)+	    pthread_cond_signal(&p->full_c);++	if (p->njobs == 0)+	    pthread_cond_signal(&p->empty_c);++	pthread_mutex_unlock(&p->pool_m);+	    +	// We have job 'j' - now execute it.+	t_pool_add_result(j, j->func(j->arg));	+#ifdef DEBUG_TIME+	pthread_mutex_lock(&p->pool_m);+	gettimeofday(&t3, NULL);+	p->total_time += TDIFF(t3,t1);+	pthread_mutex_unlock(&p->pool_m);+#endif+	memset(j, 0xbb, sizeof(*j));+	free(j);+    }++    return NULL;+}++/*+ * Creates a worker pool of length qsize with tsize worker threads.+ *+ * Returns pool pointer on success;+ *         NULL on failure+ */+t_pool *t_pool_init(int qsize, int tsize) {+    int i;+    t_pool *p = malloc(sizeof(*p));+    p->qsize = qsize;+    p->tsize = tsize;+    p->njobs = 0;+    p->nwaiting = 0;+    p->shutdown = 0;+    p->head = p->tail = NULL;+    p->t_stack = NULL;+#ifdef DEBUG_TIME+    p->total_time = p->wait_time = 0;+#endif++    p->t = malloc(tsize * sizeof(p->t[0]));++    pthread_mutex_init(&p->pool_m, NULL);+    pthread_cond_init(&p->empty_c, NULL);+    pthread_cond_init(&p->full_c, NULL);++    pthread_mutex_lock(&p->pool_m);++#ifdef IN_ORDER+    if (!(p->t_stack = malloc(tsize * sizeof(*p->t_stack))))+	return NULL;+    p->t_stack_top = -1;++    for (i = 0; i < tsize; i++) {+	t_pool_worker_t *w = &p->t[i];+	p->t_stack[i] = 0;+	w->p = p;+	w->idx = i;+	w->wait_time = 0;+	pthread_cond_init(&w->pending_c, NULL);+	if (0 != pthread_create(&w->tid, NULL, t_pool_worker, w))+	    return NULL;+    }+#else+    pthread_cond_init(&p->pending_c, NULL);++    for (i = 0; i < tsize; i++) {+	t_pool_worker_t *w = &p->t[i];+	w->p = p;+	w->idx = i;+	pthread_cond_init(&w->pending_c, NULL);+	if (0 != pthread_create(&w->tid, NULL, t_pool_worker, w))+	    return NULL;+    }+#endif++    pthread_mutex_unlock(&p->pool_m);++    return p;+}++/*+ * Adds an item to the work pool.+ *+ * FIXME: Maybe return 1,0,-1 and distinguish between job dispathed vs+ * result returned. Ie rather than blocking on full queue we're permitted+ * to return early on "result available" event too.+ * Caller would then have a while loop around t_pool_dispatch.+ * Or, return -1 and set errno to EAGAIN to indicate job not yet submitted.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int t_pool_dispatch(t_pool *p, t_results_queue *q,+		    void *(*func)(void *arg), void *arg) {+    return t_pool_dispatch2(p, q, func, arg, 0);+}++/*+ * As above but optional non-block flag.+ *+ * nonblock  0 => block if input queue is full+ * nonblock +1 => don't block if input queue is full, but do not add task+ * nonblock -1 => add task regardless of whether queue is full (over-size)+ */+int t_pool_dispatch2(t_pool *p, t_results_queue *q,+		     void *(*func)(void *arg), void *arg, int nonblock) {+    t_pool_job *j;++#ifdef DEBUG+    fprintf(stderr, "Dispatching job for queue %p, serial %d\n", q, q->curr_serial);+#endif++    pthread_mutex_lock(&p->pool_m);++    if (p->njobs >= p->qsize && nonblock == 1) {+	pthread_mutex_unlock(&p->pool_m);+	errno = EAGAIN;+	return -1;+    }++    if (!(j = malloc(sizeof(*j))))+	return -1;+    j->func = func;+    j->arg = arg;+    j->next = NULL;+    j->p = p;+    j->q = q;+    if (q) {+	pthread_mutex_lock(&q->result_m);+	j->serial = q->curr_serial++;+	q->pending++;+	pthread_mutex_unlock(&q->result_m);+    } else {+	j->serial = 0;+    }++    // Check if queue is full+    if (nonblock == 0)+	while (p->njobs >= p->qsize)+	    pthread_cond_wait(&p->full_c, &p->pool_m);++    p->njobs++;+    +//    if (q->curr_serial % 100 == 0)+//	fprintf(stderr, "p->njobs = %d    p->qsize = %d\n", p->njobs, p->qsize);++    if (p->tail) {+	p->tail->next = j;+	p->tail = j;+    } else {+	p->head = p->tail = j;+    }++#ifdef DEBUG+    fprintf(stderr, "Dispatched (serial %d)\n", j->serial);+#endif++    // Let a worker know we have data.+#ifdef IN_ORDER+    // Keep incoming queue at 1 per running thread, so there is always+    // something waiting when they end their current task.  If we go above+    // this signal to start more threads (if available). This has the effect+    // of concentrating jobs to fewer cores when we are I/O bound, which in+    // turn benefits systems with auto CPU frequency scaling.+    if (p->t_stack_top >= 0 && p->njobs > p->tsize - p->nwaiting)+	pthread_cond_signal(&p->t[p->t_stack_top].pending_c);+#else+    pthread_cond_signal(&p->pending_c);+#endif++    pthread_mutex_unlock(&p->pool_m);++    return 0;+}++/*+ * Flushes the pool, but doesn't exit. This simply drains the queue and+ * ensures all worker threads have finished their current task.+ *+ * Returns 0 on success;+ *        -1 on failure+ */+int t_pool_flush(t_pool *p) {+    int i;++#ifdef DEBUG+    fprintf(stderr, "Flushing pool %p\n", p);+#endif++    // Drains the queue+    pthread_mutex_lock(&p->pool_m);++    // Wake up everything for the final sprint!+    for (i = 0; i < p->tsize; i++)+	if (p->t_stack[i])+	    pthread_cond_signal(&p->t[i].pending_c);++    while (p->njobs || p->nwaiting != p->tsize)+	pthread_cond_wait(&p->empty_c, &p->pool_m);++    pthread_mutex_unlock(&p->pool_m);++#ifdef DEBUG+    fprintf(stderr, "Flushed complete for pool %p, njobs=%d, nwaiting=%d\n",+	    p, p->njobs, p->nwaiting);+#endif++    return 0;+}++/*+ * Destroys a thread pool. If 'kill' is true the threads are terminated now,+ * otherwise they are joined into the main thread so they will finish their+ * current work load.+ *+ * Use t_pool_destroy(p,0) after a t_pool_flush(p) on a normal shutdown or+ * t_pool_destroy(p,1) to quickly exit after a fatal error.+ */+void t_pool_destroy(t_pool *p, int kill) {+    int i;+    +#ifdef DEBUG+    fprintf(stderr, "Destroying pool %p, kill=%d\n", p, kill);+#endif++    /* Send shutdown message to worker threads */+    if (!kill) {+	pthread_mutex_lock(&p->pool_m);+	p->shutdown = 1;++#ifdef DEBUG+	fprintf(stderr, "Sending shutdown request\n");+#endif++#ifdef IN_ORDER+	for (i = 0; i < p->tsize; i++)+	    pthread_cond_signal(&p->t[i].pending_c);+#else+	pthread_cond_broadcast(&p->pending_c);+#endif+	pthread_mutex_unlock(&p->pool_m);++#ifdef DEBUG+	fprintf(stderr, "Shutdown complete\n");+#endif+	for (i = 0; i < p->tsize; i++)+	    pthread_join(p->t[i].tid, NULL);+    } else {+	for (i = 0; i < p->tsize; i++)+	    pthread_kill(p->t[i].tid, SIGINT);+    }++    pthread_mutex_destroy(&p->pool_m);+    pthread_cond_destroy(&p->empty_c);+    pthread_cond_destroy(&p->full_c);+#ifdef IN_ORDER+    for (i = 0; i < p->tsize; i++)+	pthread_cond_destroy(&p->t[i].pending_c);+#else+    pthread_cond_destroy(&p->pending_c);+#endif++#ifdef DEBUG_TIME+    fprintf(stderr, "Total time=%f\n", p->total_time / 1000000.0);+    fprintf(stderr, "Wait  time=%f\n", p->wait_time  / 1000000.0);+    fprintf(stderr, "%d%% utilisation\n",+	    (int)(100 - ((100.0 * p->wait_time) / p->total_time + 0.5)));+    for (i = 0; i < p->tsize; i++)+	fprintf(stderr, "%d: Wait time=%f\n", i,+		p->t[i].wait_time / 1000000.0);+#endif++    if (p->t_stack)+	free(p->t_stack);++    free(p->t);+    free(p);++#ifdef DEBUG+    fprintf(stderr, "Destroyed pool %p\n", p);+#endif+}+++/*-----------------------------------------------------------------------------+ * Test app.+ */++#ifdef TEST_MAIN++#include <stdio.h>+#include <math.h>++void *doit(void *arg) {+    int i, k, x = 0;+    int job = *(int *)arg;+    int *res;++    printf("Worker: execute job %d\n", job);++    usleep(random() % 1000000); // to coerce job completion out of order+    if (0) {+	for (k = 0; k < 100; k++) {+	    for (i = 0; i < 100000; i++) {+		x++;+		x += x * sin(i);+		x += x * cos(x);+	    }+	}+	x *= 100;+	x += job;+    } else {+	x = job*job;+    }++    printf("Worker: job %d terminating, x=%d\n", job, x);++    free(arg);++    res = malloc(sizeof(*res));+    *res = x;++    return res;+}++#define NTHREADS 8++int main(int argc, char **argv) {+    t_pool *p = t_pool_init(NTHREADS*2, NTHREADS);+    t_results_queue *q = t_results_queue_init();+    int i;+    t_pool_result *r;++    // Dispatch jobs+    for (i = 0; i < 20; i++) {+	int *ip = malloc(sizeof(*ip));+	*ip = i;+	printf("Submitting %d\n", i);+	t_pool_dispatch(p, q, doit, ip);+	+	// Check for results+	if ((r = t_pool_next_result(q))) {+	    printf("RESULT: %d\n", *(int *)r->data);+	    t_pool_delete_result(r, 1);+	}+    }++    t_pool_flush(p);++    while ((r = t_pool_next_result(q))) {+	printf("RESULT: %d\n", *(int *)r->data);+	t_pool_delete_result(r, 1);+    }++    t_pool_destroy(p, 0);+    t_results_queue_destroy(q);++    return 0;+}+#endif
+ htslib-1.3.2/cram/thread_pool.h view
@@ -0,0 +1,218 @@+/*+Copyright (c) 2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++/*+ * This file implements a thread pool for multi-threading applications.+ * It consists of two distinct interfaces: thread pools an results queues.+ *+ * The pool of threads is given a function pointer and void* data to pass in.+ * This means the pool can run jobs of multiple types, albeit first come+ * first served with no job scheduling.+ *+ * Upon completion, the return value from the function pointer is added to+ * a results queue. We may have multiple queues in use for the one pool.+ *+ * An example: reading from BAM and writing to CRAM with 10 threads. We'll+ * have a pool of 10 threads and two results queues holding decoded BAM blocks+ * and encoded CRAM blocks respectively.+ */++#ifndef _THREAD_POOL_H_+#define _THREAD_POOL_H_++#include <pthread.h>++#ifdef __cplusplus+extern "C" {+#endif++struct t_pool;+struct t_results_queue;++typedef struct t_pool_job {+    void *(*func)(void *arg);+    void *arg;+    struct t_pool_job *next;++    struct t_pool *p;+    struct t_results_queue *q;+    int serial;+} t_pool_job;++typedef struct t_res {+    struct t_res *next;+    int serial; // sequential number for ordering+    void *data; // result itself+} t_pool_result;++struct t_pool;++typedef struct {+    struct t_pool *p;+    int idx;+    pthread_t tid;+    pthread_cond_t  pending_c;+    long long wait_time;+} t_pool_worker_t;++typedef struct t_pool {+    int qsize;    // size of queue+    int njobs;    // pending job count+    int nwaiting; // how many workers waiting for new jobs+    int shutdown; // true if pool is being destroyed++    // queue of pending jobs+    t_pool_job *head, *tail;++    // threads+    int tsize;    // maximum number of jobs+    t_pool_worker_t *t;++    // Mutexes+    pthread_mutex_t pool_m; // used when updating head/tail++    pthread_cond_t  empty_c;+    pthread_cond_t  pending_c; // not empty+    pthread_cond_t  full_c;++    // array of worker IDs free+    int *t_stack, t_stack_top;++    // Debugging to check wait time+    long long total_time, wait_time;+} t_pool;++typedef struct t_results_queue {+    t_pool_result *result_head;+    t_pool_result *result_tail;+    int next_serial;+    int curr_serial;+    int queue_len;  // number of items in queue+    int pending;    // number of pending items (in progress or in pool list)+    pthread_mutex_t result_m;+    pthread_cond_t result_avail_c;+} t_results_queue;+++/*+ * Creates a worker pool of length qsize with tsize worker threads.+ *+ * Returns pool pointer on success;+ *         NULL on failure+ */+t_pool *t_pool_init(int qsize, int tsize);++/*+ * Adds an item to the work pool.+ *+ * FIXME: Maybe return 1,0,-1 and distinguish between job dispathed vs+ * result returned. Ie rather than blocking on full queue we're permitted+ * to return early on "result available" event too.+ * Caller would then have a while loop around t_pool_dispatch.+ * Or, return -1 and set errno to E_AGAIN to indicate job not yet submitted.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int t_pool_dispatch(t_pool *p, t_results_queue *q,+		    void *(*func)(void *arg), void *arg);+int t_pool_dispatch2(t_pool *p, t_results_queue *q,+		     void *(*func)(void *arg), void *arg, int nonblock);++/*+ * Flushes the pool, but doesn't exit. This simply drains the queue and+ * ensures all worker threads have finished their current task.+ *+ * Returns 0 on success;+ *        -1 on failure+ */+int t_pool_flush(t_pool *p);++/*+ * Destroys a thread pool. If 'kill' is true the threads are terminated now,+ * otherwise they are joined into the main thread so they will finish their+ * current work load.+ *+ * Use t_pool_destroy(p,0) after a t_pool_flush(p) on a normal shutdown or+ * t_pool_destroy(p,1) to quickly exit after a fatal error.+ */+void t_pool_destroy(t_pool *p, int kill);++/*+ * Pulls a result off the head of the result queue. Caller should+ * free it (and any internals as appropriate) after use. This doesn't+ * wait for a result to be present.+ *+ * Results will be returned in strict order.+ * + * Returns t_pool_result pointer if a result is ready.+ *         NULL if not.+ */+t_pool_result *t_pool_next_result(t_results_queue *q);+t_pool_result *t_pool_next_result_wait(t_results_queue *q);++/*+ * Frees a result 'r' and if free_data is true also frees+ * the internal r->data result too.+ */+void t_pool_delete_result(t_pool_result *r, int free_data);++/*+ * Initialises a results queue.+ *+ * Results queue pointer on success;+ *         NULL on failure+ */+t_results_queue *t_results_queue_init(void);++/* Deallocates memory for a results queue */+void t_results_queue_destroy(t_results_queue *q);++/*+ * Returns true if there are no items on the finished results queue and+ * also none still pending.+ */+int t_pool_results_queue_empty(t_results_queue *q);++/*+ * Returns the number of completed jobs on the results queue.+ */+int t_pool_results_queue_len(t_results_queue *q);++/*+ * Returns the number of completed jobs plus the number queued up to run.+ */+int t_pool_results_queue_sz(t_results_queue *q);++#ifdef __cplusplus+}+#endif++#endif /* _THREAD_POOL_H_ */
+ htslib-1.3.2/cram/vlen.c view
@@ -0,0 +1,430 @@+/*+Author: James Bonfield (jkb@sanger.ac.uk)++Copyright (c) 1995-1996 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1 Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2 Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++/*+Copyright (c) 2004, 2009, 2011-2012 Genome Research Ltd.++Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdio.h>+#include <stdlib.h>+#include <stdarg.h>+#include <sys/types.h>+#include <string.h>++#include "cram/vlen.h"+#include "cram/os.h"++#ifndef MAX+#define MAX(a,b) ((a)>(b)?(a):(b))+#endif++#ifndef ABS+#define ABS(a) ((a)>0?(a):-(a))+#endif++/* #define DEBUG_printf(a,n) printf(a,n) */+#define DEBUG_printf(a,n)++/*+ * vlen: 27/10/95 written by James Bonfield, jkb@mrc-lmb.cam.ac.uk+ *+ * Given sprintf style of arguments this routine returns the maximum+ * size of buffer needed to allocate to use with sprintf. It errs on+ * the side of caution by being simplistic in its approach: we assume+ * all numbers are of maximum length.+ *+ * Handles the usual type conversions (%[%diuaxXcfeEgGpns]), but not+ * the 'wide' character conversions (%C and %S).+ * Precision is handled in the correct formats, including %*.*+ * notations.+ * Additionally, some of the more dubious (but probably illegal) cases+ * are supported (eg "%10%" will expand to "         %" on many+ * systems).+ *+ * We also assume that the largest integer and larger pointer are 64+ * bits, which at least covers the machines we'll need it for.+ */+int flen(char *fmt, ...)+{+    va_list args;++    va_start(args, fmt);+    return vflen(fmt, args);+}++int vflen(char *fmt, va_list ap)+{+    int len = 0;+    char *cp, c;+    long long l;+    int i;+    double d; ++    /*+     * This code modifies 'ap', but we do not know if va_list is a structure+     * or a pointer to an array so we do not know if it is a local variable+     * or not.+     * C99 gets around this by defining va_copy() to make copies of ap, but+     * this does not exist on all systems.+     * For now, I just assume that when va_list is a pointer the system also+     * provides a va_copy macro to work around this problem. The only system+     * I have seen needing this so far was Linux on AMD64.+     */+#if defined(HAVE_VA_COPY)+    va_list ap_local;+    va_copy(ap_local, ap);+#    define ap ap_local+#endif++    for(cp = fmt; *cp; cp++) {+	switch(*cp) {++	/* A format specifier */+	case '%': {+	    char *endp;+	    long conv_len1=0, conv_len2=0, conv_len=0;+	    signed int arg_size;++	    /* Firstly, strip the modifier flags (+-#0 and [space]) */+	    for(; (c=*++cp);) {+		if ('#' == c)+		    len+=2; /* Worst case of "0x" */+		else if ('-' == c || '+' == c || ' ' == c)+		    len++;+		else+		    break;+	    }++	    /* Width specifier */+	    l = strtol(cp, &endp, 10);+	    if (endp != cp) {+		cp = endp;+		conv_len = conv_len1 = l;+	    } else if (*cp == '*') {+		conv_len = conv_len1 = (int)va_arg(ap, int);+		cp++;+	    }++	    /* Precision specifier */+	    if ('.' == *cp) {+		cp++;+		conv_len2 = strtol(cp, &endp, 10);+		if (endp != cp) {+		    cp = endp;+		} else if (*cp == '*') {+		    conv_len2 = (int)va_arg(ap, int);+		    cp++;+		}+		conv_len = MAX(conv_len1, conv_len2);+	    }++	    /* Short/long identifier */+	    if ('h' == *cp) {+		arg_size = -1; /* short */+		cp++;+	    } else if ('l' == *cp) {+		arg_size = 1; /* long */+		cp++;+		if ('l' == *cp) {+		    arg_size = 2; /* long long */+		    cp++;+		}+	    } else {+		arg_size = 0; /* int */+	    }++	    /* The actual type */+	    switch (*cp) {+	    case '%':+		/*+		 * Not real ANSI I suspect, but we'll allow for the+		 * completely daft "%10%" example.+		 */+		len += MAX(conv_len1, 1);+		break;++	    case 'd':+	    case 'i':+	    case 'u':+	    case 'a':+	    case 'x':+	    case 'X':+		/* Remember: char and short are sent as int on the stack */+		if (arg_size == -1)+		    l = (long)va_arg(ap, int);+		else if (arg_size == 1)+		    l = va_arg(ap, long); +		else if (arg_size == 2)+		    l = va_arg(ap, long long); +		else +		    l = (long)va_arg(ap, int);++		DEBUG_printf("%d", l);++		/*+		 * No number can be more than 24 characters so we'll take+		 * the max of conv_len and 24 (23 is len(2^64) in octal).+		 * All that work above and we then go and estimate ;-),+		 * but it's needed incase someone does %500d.+		 */+		len += MAX(conv_len, 23);+		break;++	    case 'c':+		i = va_arg(ap, int);+		DEBUG_printf("%c", i);+		/*+		 * Note that %10c and %.10c act differently.+		 * Besides, I think precision is not really allowed for %c.+		 */+		len += MAX(conv_len1, i>=0x80 ?MB_CUR_MAX :1);+		break;++	    case 'f':+		d = va_arg(ap, double);+		DEBUG_printf("%f", d);+		/*+		 * Maybe "Inf" or "NaN", but we'll not worry about that.+		 * Again, err on side of caution and take max of conv_len+		 * and max length of a double. The worst case I can+		 * think of is 317 characters (-1[308 zeros].000000)+		 * without using precision codes. That's horrid. I+		 * cheat and either use 317 or 15 depending on how+		 * large the number is as I reckon 99% of floats+		 * aren't that long.+		 */+		l = (ABS(d) > 1000000) ? 317 : 15;+		l = MAX(l, conv_len1 + 2);+		if (conv_len2) l += conv_len2 - 6;+		len += l;+		break;++	    case 'e':+	    case 'E':+	    case 'g':+	    case 'G':+		d = va_arg(ap, double);+		DEBUG_printf("%g", d);+		/*+		 * Maybe "Inf" or "NaN", but we'll not worry about that+		 * Again, err on side of caution and take max of conv_len+		 * and max length of a double (which defaults to only+		 * '-' + 6 + '.' + 'E[+-]xxx' == 13.+		 */+		len += MAX(conv_len, 13);+		break;++	    case 'p':+		l = (long)va_arg(ap, void *);+		/*+		 * Max pointer is 64bits == 16 chars (on alpha),+		 * == 20 with + "0x".+		 */+		DEBUG_printf("%p", (void *)l);+		len += MAX(conv_len, 20);+		break;++	    case 'n':+		/* produces no output */+		break;++	    case 's': {+		char *s = (char *)va_arg(ap, char *);+		DEBUG_printf("%s", s);++		if (!conv_len2) {+		    len += MAX(conv_len, (int)strlen(s));+		} else {+		    len += conv_len;+		}+		break;+	    }++	    default:+		/* wchar_t types of 'C' and 'S' aren't supported */+		DEBUG_printf("Arg is %c\n", *cp);+	    }+	    +	}++	case '\0':+	    break;++	default:+	    DEBUG_printf("%c", *cp);+	    len++;+	}+    }++    va_end(ap);++    return len+1; /* one for the null character */+}++#if 0+int main() {+    int l;+    char buf[10000];++    sprintf(buf, "d: %d\n", 500);+    l = flen("d: %d\n", 500);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "");+    l = flen("");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%s\n","test");+    l = flen("%s\n", "test");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%c\n", 'a');+    l = flen("%c\n", 'a');+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%31.30f\n", -9999.99);+    l = flen("%31.30f\n", -9999.99);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%f\n", -1e308);+    l = flen("%f\n", -1e308);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%.9f\n", -1e308);+    l = flen("%.9f\n", -1e308);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%10.20f\n", -1.999222333);+    l = flen("%10.20f\n", -1.999222333);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%#g\n", -3.14159265358e-222);+    l = flen("%#g\n", -3.1415927e-222);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%e\n", -123456789123456789.1);+    l = flen("%e\n", -123456789123456789.1);+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%c %f %d %s %c %g %ld %s\n", 'a', 3.1, 9, "one", 'b', 4.2, 9, "two");+    l = flen("%c %f %d %s %c %g %ld %s\n", 'a', 3.1, 9, "one", 'b', 4.2, 9, "two");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%*.*e %*c\n", 10, 5, 9.0, 20, 'x');+    l = flen("%*.*e %*c\n", 10, 5, 9.0, 20, 'x');+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%10c\n", 'z');+    l = flen("%10c\n", 'z');+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%.10c\n", 'z');+    l = flen("%.10c\n", 'z');+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%10d\n", 'z');+    l = flen("%10d\n", 'z');+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%.10d\n", 'z');+    l = flen("%.10d\n", 'z');+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%10%\n");+    l = flen("%10%\n");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%.10%\n");+    l = flen("%.10%\n");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%s\n", "0123456789");+    l = flen("%s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%5s\n", "0123456789");+    l = flen("%5s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%50s\n", "0123456789");+    l = flen("%50s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%.5s\n", "0123456789");+    l = flen("%.5s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%.50s\n", "0123456789");+    l = flen("%.50s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%5.50s\n", "0123456789");+    l = flen("%5.50s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    sprintf(buf, "%50.5s\n", "0123456789");+    l = flen("%50.5s\n", "0123456789");+    printf("%d %d\n\n", strlen(buf), l);++    return 0;+}+#endif
+ htslib-1.3.2/cram/vlen.h view
@@ -0,0 +1,48 @@+/*+Author: James Bonfield (jkb@sanger.ac.uk)++Copyright (c) 1995-1996 MEDICAL RESEARCH COUNCIL+All rights reserved++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1 Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2 Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF +MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written +permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _VLEN_H_+#define _VLEN_H_++#ifdef __cplusplus+extern "C" {+#endif++extern int vflen(char *fmt, va_list ap);+extern int flen(char *fmt, ...);++#ifdef __cplusplus+}+#endif++#endif /* _VLEN_H_ */
+ htslib-1.3.2/cram/zfio.c view
@@ -0,0 +1,183 @@+/*+Copyright (c) 2009-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#include <config.h>++#include <stdlib.h>+#include <unistd.h>++#include "cram/os.h"+#include "cram/zfio.h"++/* ------------------------------------------------------------------------ */+/* Some wrappers around FILE * vs gzFile *, allowing for either */++/*+ * gzopen() works on both compressed and uncompressed data, but it has+ * a significant performance hit even for uncompressed data (tested as+ * 25s using FILE* to 46s via gzOpen and 66s via gzOpen when gzipped).+ *+ * Hence we use our own wrapper 'zfp' which is a FILE* when uncompressed+ * and gzFile* when compressed. This also means we could hide bzopen in+ * there too if desired.+ */++off_t zftello(zfp *zf) {+    return zf->fp ? ftello(zf->fp) : -1;+}++int zfseeko(zfp *zf, off_t offset, int whence) {+    return zf->fp ? fseeko(zf->fp, offset, whence) : -1;+}+++/*+ * A wrapper for either fgets or gzgets depending on what has been+ * opened.+ */+char *zfgets(char *line, int size, zfp *zf) {+    if (zf->fp)+	return fgets(line, size, zf->fp);+    else+	return gzgets(zf->gz, line, size);+}++/*+ * A wrapper for either fputs or gzputs depending on what has been+ * opened.+ */+int zfputs(char *line, zfp *zf) {+    if (zf->fp)+	return fputs(line, zf->fp);+    else+	return gzputs(zf->gz, line) ? 0 : EOF;+}++/*+ * Peeks at and returns the next character without consuming it from the+ * input. (Ie a combination of getc and ungetc).+ */+int zfpeek(zfp *zf) {+    int c;++    if (zf->fp) {+	c = getc(zf->fp);+	if (c != EOF)+	    ungetc(c, zf->fp);+    } else {+	c = gzgetc(zf->gz);+	if (c != EOF)+	    gzungetc(c, zf->gz);+    }++    return c;+}++/* A replacement for either feof of gzeof */+int zfeof(zfp *zf) {+    return zf->fp ? feof(zf->fp) : gzeof(zf->gz);+}++/* A replacement for either fopen or gzopen */+zfp *zfopen(const char *path, const char *mode) {+    char path2[1024];+    zfp *zf;++    if (!(zf = (zfp *)malloc(sizeof(*zf))))+	return NULL;+    zf->fp = NULL;+    zf->gz = NULL;++    /* Try normal fopen */+    if (mode[0] != 'z' && mode[1] != 'z' &&+	NULL != (zf->fp = fopen(path, mode))) {+	unsigned char magic[2];+	if (2 != fread(magic, 1, 2, zf->fp)) {+	    free(zf);+	    return NULL;+	}+	if (!(magic[0] == 0x1f &&+	      magic[1] == 0x8b)) {+	    fseeko(zf->fp, 0, SEEK_SET);+	    return zf;+	}++	fclose(zf->fp);+	zf->fp = NULL;+    }++#ifdef HAVE_POPEN+    /*+     * I've no idea why, by gzgets is VERY slow, maybe because it handles+     * arbitrary seeks.+     * popen to gzip -cd is 3 times faster though.+     */+    if (*mode == 'w') {+    } else {+	if (access(path, R_OK) == 0) {+	    sprintf(path2, "gzip -cd < %.*s", 1000, path);+	    if (NULL != (zf->fp = popen(path2, "r")))+		return zf;+	}+	+	sprintf(path2, "gzip -cd < %.*s.gz", 1000, path);+	if (NULL != (zf->fp = popen(path2, "r")))+	    return zf;++	printf("Failed on %s\n", path);+    } else {+	sprintf(path2, "gzip > %.*s", 1000, path);+	if (NULL != (zf->fp = popen(path2, "w")))+	    return zf;+	}+	+	printf("Failed on %s\n", path);+    }+#else+    /* Gzopen instead */+    if ((zf->gz = gzopen(path, mode)))+	return zf;++    sprintf(path2, "%.*s.gz", 1020, path);+    if ((zf->gz = gzopen(path2, mode)))+	return zf;+#endif++    perror(path);++    free(zf);+    return NULL;+}++int zfclose(zfp *zf) {+    int r = (zf->fp) ? fclose(zf->fp) : gzclose(zf->gz);+    free(zf);+    return r;+}
+ htslib-1.3.2/cram/zfio.h view
@@ -0,0 +1,62 @@+/*+Copyright (c) 2009-2013 Genome Research Ltd.+Author: James Bonfield <jkb@sanger.ac.uk>++Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met:++   1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer.++   2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution.++   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger+Institute nor the names of its contributors may be used to endorse or promote+products derived from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*/++#ifndef _ZFIO_H_+#define _ZFIO_H_++#include <stdio.h>+#include <zlib.h>++#ifdef __cplusplus+extern "C" {+#endif++/*+ * Either a gzFile or a FILE.+ */+typedef struct {+    FILE   *fp;+    gzFile  gz;+} zfp;++off_t zftello(zfp *zf);+int zfseeko(zfp *zf, off_t offset, int whence);+char *zfgets(char *line, int size, zfp *zf);+int zfputs(char *line, zfp *zf);+zfp *zfopen(const char *path, const char *mode);+int zfclose(zfp *zf);+int zfpeek(zfp *zf);+int zfeof(zfp *zf);++#ifdef __cplusplus+}+#endif++#endif /* _ZFIO_H_ */
+ htslib-1.3.2/faidx.c view
@@ -0,0 +1,525 @@+/*  faidx.c -- FASTA random access.++    Copyright (C) 2008, 2009, 2013-2016 Genome Research Ltd.+    Portions copyright (C) 2011 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <ctype.h>+#include <string.h>+#include <stdlib.h>+#include <stdio.h>+#include <inttypes.h>+#include <errno.h>++#include "htslib/bgzf.h"+#include "htslib/faidx.h"+#include "htslib/hfile.h"+#include "htslib/khash.h"+#include "htslib/kstring.h"++typedef struct {+    int32_t line_len, line_blen;+    int64_t len;+    uint64_t offset;+} faidx1_t;+KHASH_MAP_INIT_STR(s, faidx1_t)++struct __faidx_t {+    BGZF *bgzf;+    int n, m;+    char **name;+    khash_t(s) *hash;+};++#ifndef kroundup32+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))+#endif++static inline int fai_insert_index(faidx_t *idx, const char *name, int len, int line_len, int line_blen, uint64_t offset)+{+    if (!name) {+        fprintf(stderr, "[fai_build_core] malformed line\n");+        return -1;+    }++    char *name_key = strdup(name);+    int absent;+    khint_t k = kh_put(s, idx->hash, name_key, &absent);+    faidx1_t *v = &kh_value(idx->hash, k);++    if (! absent) {+        fprintf(stderr, "[fai_build_core] ignoring duplicate sequence \"%s\" at byte offset %"PRIu64"\n", name, offset);+        free(name_key);+        return 0;+    }++    if (idx->n == idx->m) {+        char **tmp;+        idx->m = idx->m? idx->m<<1 : 16;+        if (!(tmp = (char**)realloc(idx->name, sizeof(char*) * idx->m))) {+            fprintf(stderr, "[fai_build_core] out of memory\n");+            return -1;+        }+        idx->name = tmp;+    }+    idx->name[idx->n++] = name_key;+    v->len = len;+    v->line_len = line_len;+    v->line_blen = line_blen;+    v->offset = offset;++    return 0;+}++faidx_t *fai_build_core(BGZF *bgzf)+{+    kstring_t name = { 0, 0, NULL };+    int c;+    int line_len, line_blen, state;+    int l1, l2;+    faidx_t *idx;+    uint64_t offset;+    int64_t len;++    idx = (faidx_t*)calloc(1, sizeof(faidx_t));+    idx->hash = kh_init(s);+    len = line_len = line_blen = -1; state = 0; l1 = l2 = -1; offset = 0;+    while ( (c=bgzf_getc(bgzf))>=0 ) {+        if (c == '\n') { // an empty line+            if (state == 1) {+                offset = bgzf_utell(bgzf);+                continue;+            } else if ((state == 0 && len < 0) || state == 2) continue;+            else if (state == 0) { state = 2; continue; }+        }+        if (c == '>') { // fasta header+            if (len >= 0) {+                if (fai_insert_index(idx, name.s, len, line_len, line_blen, offset) != 0)+                    goto fail;+            }++            name.l = 0;+            while ((c = bgzf_getc(bgzf)) >= 0)+                if (! isspace(c)) kputc_(c, &name);+                else if (name.l > 0 || c == '\n') break;+            kputsn("", 0, &name);++            if ( c<0 ) {+                fprintf(stderr, "[fai_build_core] the last entry has no sequence\n");+                goto fail;+            }+            if (c != '\n') while ( (c=bgzf_getc(bgzf))>=0 && c != '\n');+            state = 1; len = 0;+            offset = bgzf_utell(bgzf);+        } else {+            if (state == 3) {+                fprintf(stderr, "[fai_build_core] inlined empty line is not allowed in sequence '%s'.\n", name.s);+                goto fail;+            }+            if (state == 2) state = 3;+            l1 = l2 = 0;+            do {+                ++l1;+                if (isgraph(c)) ++l2;+            } while ( (c=bgzf_getc(bgzf))>=0 && c != '\n');+            if (state == 3 && l2) {+                fprintf(stderr, "[fai_build_core] different line length in sequence '%s'.\n", name.s);+                goto fail;+            }+            ++l1; len += l2;+            if (state == 1) line_len = l1, line_blen = l2, state = 0;+            else if (state == 0) {+                if (l1 != line_len || l2 != line_blen) state = 2;+            }+        }+    }++    if (len >= 0) {+        if (fai_insert_index(idx, name.s, len, line_len, line_blen, offset) != 0)+            goto fail;+    } else {+        goto fail;+    }++    free(name.s);+    return idx;++fail:+    free(name.s);+    fai_destroy(idx);+    return NULL;+}++void fai_save(const faidx_t *fai, FILE *fp)+{+    khint_t k;+    int i;+    for (i = 0; i < fai->n; ++i) {+        faidx1_t x;+        k = kh_get(s, fai->hash, fai->name[i]);+        x = kh_value(fai->hash, k);+#ifdef _WIN32+        fprintf(fp, "%s\t%d\t%ld\t%d\t%d\n", fai->name[i], (int)x.len, (long)x.offset, (int)x.line_blen, (int)x.line_len);+#else+        fprintf(fp, "%s\t%d\t%lld\t%d\t%d\n", fai->name[i], (int)x.len, (long long)x.offset, (int)x.line_blen, (int)x.line_len);+#endif+    }+}++static faidx_t *fai_read(FILE *fp, const char *fname)+{+    faidx_t *fai;+    char *buf, *p;+    int len, line_len, line_blen;+#ifdef _WIN32+    long offset;+#else+    long long offset;+#endif+    fai = (faidx_t*)calloc(1, sizeof(faidx_t));+    fai->hash = kh_init(s);+    buf = (char*)calloc(0x10000, 1);+    while (fgets(buf, 0x10000, fp)) {+        for (p = buf; *p && isgraph(*p); ++p);+        *p = 0; ++p;+#ifdef _WIN32+        sscanf(p, "%d%ld%d%d", &len, &offset, &line_blen, &line_len);+#else+        sscanf(p, "%d%lld%d%d", &len, &offset, &line_blen, &line_len);+#endif+        if (fai_insert_index(fai, buf, len, line_len, line_blen, offset) != 0) {+            free(buf);+            return NULL;+        }+    }+    free(buf);+    if (ferror(fp)) {+        fprintf(stderr, "[fai_load] error while reading \"%s\": %s\n", fname, strerror(errno));+        fai_destroy(fai);+        return NULL;+    }+    return fai;+}++void fai_destroy(faidx_t *fai)+{+    int i;+    for (i = 0; i < fai->n; ++i) free(fai->name[i]);+    free(fai->name);+    kh_destroy(s, fai->hash);+    if (fai->bgzf) bgzf_close(fai->bgzf);+    free(fai);+}++int fai_build(const char *fn)+{+    char *str;+    BGZF *bgzf;+    FILE *fp;+    faidx_t *fai;+    str = (char*)calloc(strlen(fn) + 5, 1);+    sprintf(str, "%s.fai", fn);+    bgzf = bgzf_open(fn, "r");+    if ( !bgzf ) {+        fprintf(stderr, "[fai_build] fail to open the FASTA file %s\n",fn);+        free(str);+        return -1;+    }+    if ( bgzf->is_compressed ) bgzf_index_build_init(bgzf);+    fai = fai_build_core(bgzf);+    if ( !fai )+    {+        if ( bgzf->is_compressed && bgzf->is_gzip ) fprintf(stderr,"Cannot index files compressed with gzip, please use bgzip\n");+        bgzf_close(bgzf);+        free(str);+        return -1;+    }+    if ( bgzf->is_compressed ) {+        if (bgzf_index_dump(bgzf, fn, ".gzi") < 0) {+            fprintf(stderr, "[fai_build] fail to make bgzf index %s.gzi\n", fn);+            fai_destroy(fai); free(str);+            return -1;+        }+    }+    if (bgzf_close(bgzf) < 0) {+        fprintf(stderr, "[fai_build] Error on closing %s\n", fn);+        fai_destroy(fai); free(str);+        return -1;+    }+    fp = fopen(str, "wb");+    if ( !fp ) {+        fprintf(stderr, "[fai_build] fail to write FASTA index %s\n",str);+        fai_destroy(fai); free(str);+        return -1;+    }+    fai_save(fai, fp);+    fclose(fp);+    free(str);+    fai_destroy(fai);+    return 0;+}++static FILE *download_and_open(const char *fn)+{+    const int buf_size = 1 * 1024 * 1024;+    uint8_t *buf;+    FILE *fp;+    hFILE *fp_remote;+    const char *url = fn;+    const char *p;+    int l = strlen(fn);+    for (p = fn + l - 1; p >= fn; --p)+        if (*p == '/') break;+    fn = p + 1;++    // First try to open a local copy+    fp = fopen(fn, "r");+    if (fp)+        return fp;++    // If failed, download from remote and open+    fp_remote = hopen(url, "rb");+    if (fp_remote == 0) {+        fprintf(stderr, "[download_from_remote] fail to open remote file %s\n",url);+        return NULL;+    }+    if ((fp = fopen(fn, "wb")) == 0) {+        fprintf(stderr, "[download_from_remote] fail to create file in the working directory %s\n",fn);+        hclose_abruptly(fp_remote);+        return NULL;+    }+    buf = (uint8_t*)calloc(buf_size, 1);+    while ((l = hread(fp_remote, buf, buf_size)) > 0)+        fwrite(buf, 1, l, fp);+    free(buf);+    fclose(fp);+    if (hclose(fp_remote) != 0)+        fprintf(stderr, "[download_from_remote] fail to close remote file %s\n", url);++    return fopen(fn, "r");+}++faidx_t *fai_load(const char *fn)+{+    char *str;+    FILE *fp;+    faidx_t *fai;+    str = (char*)calloc(strlen(fn) + 5, 1);+    sprintf(str, "%s.fai", fn);++    if (hisremote(str))+    {+        fp = download_and_open(str);+        if ( !fp )+        {+            fprintf(stderr, "[fai_load] failed to open remote FASTA index %s\n", str);+            free(str);+            return 0;+        }+    }+    else+        fp = fopen(str, "rb");++    if (fp == 0) {+        fprintf(stderr, "[fai_load] build FASTA index.\n");+        if (fai_build(fn) < 0) {+            free(str);+            return 0;+        }+        fp = fopen(str, "rb");+        if (fp == 0) {+            fprintf(stderr, "[fai_load] failed to open FASTA index: %s\n", strerror(errno));+            free(str);+            return 0;+        }+    }++    fai = fai_read(fp, str);+    fclose(fp);+    free(str);+    if (fai == NULL) {+        return NULL;+    }++    fai->bgzf = bgzf_open(fn, "rb");+    if (fai->bgzf == 0) {+        fprintf(stderr, "[fai_load] fail to open FASTA file.\n");+        return 0;+    }+    if ( fai->bgzf->is_compressed==1 )+    {+        if ( bgzf_index_load(fai->bgzf, fn, ".gzi") < 0 )+        {+            fprintf(stderr, "[fai_load] failed to load .gzi index: %s[.gzi]\n", fn);+            fai_destroy(fai);+            return NULL;+        }+    }+    return fai;+}++char *fai_fetch(const faidx_t *fai, const char *str, int *len)+{+    char *s;+    int c, i, l, k, name_end;+    khiter_t iter;+    faidx1_t val;+    khash_t(s) *h;+    int beg, end;++    beg = end = -1;+    h = fai->hash;+    name_end = l = strlen(str);+    s = (char*)malloc(l+1);+    // remove space+    for (i = k = 0; i < l; ++i)+        if (!isspace(str[i])) s[k++] = str[i];+    s[k] = 0; l = k;+    // determine the sequence name+    for (i = l - 1; i >= 0; --i) if (s[i] == ':') break; // look for colon from the end+    if (i >= 0) name_end = i;+    if (name_end < l) { // check if this is really the end+        int n_hyphen = 0;+        for (i = name_end + 1; i < l; ++i) {+            if (s[i] == '-') ++n_hyphen;+            else if (!isdigit(s[i]) && s[i] != ',') break;+        }+        if (i < l || n_hyphen > 1) name_end = l; // malformated region string; then take str as the name+        s[name_end] = 0;+        iter = kh_get(s, h, s);+        if (iter == kh_end(h)) { // cannot find the sequence name+            iter = kh_get(s, h, str); // try str as the name+            if (iter == kh_end(h)) {+                *len = 0;+            free(s); return 0;+            } else s[name_end] = ':', name_end = l;+        }+    } else iter = kh_get(s, h, str);+    if(iter == kh_end(h)) {+        fprintf(stderr, "[fai_fetch] Warning - Reference %s not found in FASTA file, returning empty sequence\n", str);+        free(s);+        *len = -2;+        return 0;+    };+    val = kh_value(h, iter);+    // parse the interval+    if (name_end < l) {+        for (i = k = name_end + 1; i < l; ++i)+            if (s[i] != ',') s[k++] = s[i];+        s[k] = 0;+        beg = atoi(s + name_end + 1);+        for (i = name_end + 1; i != k; ++i) if (s[i] == '-') break;+        end = i < k? atoi(s + i + 1) : val.len;+        if (beg > 0) --beg;+    } else beg = 0, end = val.len;+    if (beg >= val.len) beg = val.len;+    if (end >= val.len) end = val.len;+    if (beg > end) beg = end;+    free(s);++    // now retrieve the sequence+    int ret = bgzf_useek(fai->bgzf, val.offset + beg / val.line_blen * val.line_len + beg % val.line_blen, SEEK_SET);+    if ( ret<0 )+    {+        *len = -1;+        fprintf(stderr, "[fai_fetch] Error: fai_fetch failed. (Seeking in a compressed, .gzi unindexed, file?)\n");+        return NULL;+    }+    l = 0;+    s = (char*)malloc(end - beg + 2);+    while ( (c=bgzf_getc(fai->bgzf))>=0 && l < end - beg )+        if (isgraph(c)) s[l++] = c;+    s[l] = '\0';+    *len = l;+    return s;+}++int faidx_fetch_nseq(const faidx_t *fai)+{+    return fai->n;+}++int faidx_nseq(const faidx_t *fai)+{+    return fai->n;+}++const char *faidx_iseq(const faidx_t *fai, int i)+{+    return fai->name[i];+}++int faidx_seq_len(const faidx_t *fai, const char *seq)+{+    khint_t k = kh_get(s, fai->hash, seq);+    if ( k == kh_end(fai->hash) ) return -1;+    return kh_val(fai->hash, k).len;+}++char *faidx_fetch_seq(const faidx_t *fai, const char *c_name, int p_beg_i, int p_end_i, int *len)+{+    int l, c;+    khiter_t iter;+    faidx1_t val;+    char *seq=NULL;++    // Adjust position+    iter = kh_get(s, fai->hash, c_name);+    if (iter == kh_end(fai->hash))+    {+        *len = -2;+        fprintf(stderr, "[fai_fetch_seq] The sequence \"%s\" not found\n", c_name);+        return NULL;+    }+    val = kh_value(fai->hash, iter);+    if(p_end_i < p_beg_i) p_beg_i = p_end_i;+    if(p_beg_i < 0) p_beg_i = 0;+    else if(val.len <= p_beg_i) p_beg_i = val.len - 1;+    if(p_end_i < 0) p_end_i = 0;+    else if(val.len <= p_end_i) p_end_i = val.len - 1;++    // Now retrieve the sequence+    int ret = bgzf_useek(fai->bgzf, val.offset + p_beg_i / val.line_blen * val.line_len + p_beg_i % val.line_blen, SEEK_SET);+    if ( ret<0 )+    {+        *len = -1;+        fprintf(stderr, "[fai_fetch_seq] Error: fai_fetch failed. (Seeking in a compressed, .gzi unindexed, file?)\n");+        return NULL;+    }+    l = 0;+    seq = (char*)malloc(p_end_i - p_beg_i + 2);+    while ( (c=bgzf_getc(fai->bgzf))>=0 && l < p_end_i - p_beg_i + 1)+        if (isgraph(c)) seq[l++] = c;+    seq[l] = '\0';+    *len = l;+    return seq;+}++int faidx_has_seq(const faidx_t *fai, const char *seq)+{+    khiter_t iter = kh_get(s, fai->hash, seq);+    if (iter == kh_end(fai->hash)) return 0;+    return 1;+}+
+ htslib-1.3.2/hfile.c view
@@ -0,0 +1,749 @@+/*  hfile.c -- buffered low-level input/output streams.++    Copyright (C) 2013-2016 Genome Research Ltd.++    Author: John Marshall <jm18@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <errno.h>++#include <pthread.h>++#include "htslib/hfile.h"+#include "hfile_internal.h"++#ifndef ENOTSUP+#define ENOTSUP EINVAL+#endif+#ifndef EOVERFLOW+#define EOVERFLOW ERANGE+#endif+#ifndef EPROTONOSUPPORT+#define EPROTONOSUPPORT ENOSYS+#endif++/* hFILE fields are used as follows:++   char *buffer;     // Pointer to the start of the I/O buffer+   char *begin;      // First not-yet-read character / unused position+   char *end;        // First unfilled/unfillable position+   char *limit;      // Pointer to the first position past the buffer++   const hFILE_backend *backend;  // Methods to refill/flush I/O buffer++   off_t offset;     // Offset within the stream of buffer position 0+   unsigned at_eof:1;// For reading, whether EOF has been seen+   int has_errno;    // Error number from the last failure on this stream++For reading, begin is the first unread character in the buffer and end is the+first unfilled position:++   -----------ABCDEFGHIJKLMNO---------------+   ^buffer    ^begin         ^end           ^limit++For writing, begin is the first unused position and end is unused so remains+equal to buffer:++   ABCDEFGHIJKLMNOPQRSTUVWXYZ---------------+   ^buffer                   ^begin         ^limit+   ^end++Thus if begin > end then there is a non-empty write buffer, if begin < end+then there is a non-empty read buffer, and if begin == end then both buffers+are empty.  In all cases, the stream's file position indicator corresponds+to the position pointed to by begin.  */++hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity)+{+    hFILE *fp = (hFILE *) malloc(struct_size);+    if (fp == NULL) goto error;++    if (capacity == 0) capacity = 32768;+    // FIXME For now, clamp input buffer sizes so mpileup doesn't eat memory+    if (strchr(mode, 'r') && capacity > 32768) capacity = 32768;++    fp->buffer = (char *) malloc(capacity);+    if (fp->buffer == NULL) goto error;++    fp->begin = fp->end = fp->buffer;+    fp->limit = &fp->buffer[capacity];++    fp->offset = 0;+    fp->at_eof = 0;+    fp->has_errno = 0;+    return fp;++error:+    hfile_destroy(fp);+    return NULL;+}++void hfile_destroy(hFILE *fp)+{+    int save = errno;+    if (fp) free(fp->buffer);+    free(fp);+    errno = save;+}++static inline int writebuffer_is_nonempty(hFILE *fp)+{+    return fp->begin > fp->end;+}++/* Refills the read buffer from the backend (once, so may only partially+   fill the buffer), returning the number of additional characters read+   (which might be 0), or negative when an error occurred.  */+static ssize_t refill_buffer(hFILE *fp)+{+    ssize_t n;++    // Move any unread characters to the start of the buffer+    if (fp->begin > fp->buffer) {+        fp->offset += fp->begin - fp->buffer;+        memmove(fp->buffer, fp->begin, fp->end - fp->begin);+        fp->end = &fp->buffer[fp->end - fp->begin];+        fp->begin = fp->buffer;+    }++    // Read into the available buffer space at fp->[end,limit)+    if (fp->at_eof || fp->end == fp->limit) n = 0;+    else {+        n = fp->backend->read(fp, fp->end, fp->limit - fp->end);+        if (n < 0) { fp->has_errno = errno; return n; }+        else if (n == 0) fp->at_eof = 1;+    }++    fp->end += n;+    return n;+}++/* Called only from hgetc(), when our buffer is empty.  */+int hgetc2(hFILE *fp)+{+    return (refill_buffer(fp) > 0)? (unsigned char) *(fp->begin++) : EOF;+}++ssize_t hpeek(hFILE *fp, void *buffer, size_t nbytes)+{+    size_t n = fp->end - fp->begin;+    while (n < nbytes) {+        ssize_t ret = refill_buffer(fp);+        if (ret < 0) return ret;+        else if (ret == 0) break;+        else n += ret;+    }++    if (n > nbytes) n = nbytes;+    memcpy(buffer, fp->begin, n);+    return n;+}++/* Called only from hread(); when called, our buffer is empty and nread bytes+   have already been placed in the destination buffer.  */+ssize_t hread2(hFILE *fp, void *destv, size_t nbytes, size_t nread)+{+    const size_t capacity = fp->limit - fp->buffer;+    char *dest = (char *) destv;+    dest += nread, nbytes -= nread;++    // Read large requests directly into the destination buffer+    while (nbytes * 2 >= capacity && !fp->at_eof) {+        ssize_t n = fp->backend->read(fp, dest, nbytes);+        if (n < 0) { fp->has_errno = errno; return n; }+        else if (n == 0) fp->at_eof = 1;+        fp->offset += n;+        dest += n, nbytes -= n;+        nread += n;+    }++    while (nbytes > 0 && !fp->at_eof) {+        size_t n;+        ssize_t ret = refill_buffer(fp);+        if (ret < 0) return ret;++        n = fp->end - fp->begin;+        if (n > nbytes) n = nbytes;+        memcpy(dest, fp->begin, n);+        fp->begin += n;+        dest += n, nbytes -= n;+        nread += n;+    }++    return nread;+}++/* Flushes the write buffer, fp->[buffer,begin), out through the backend+   returning 0 on success or negative if an error occurred.  */+static ssize_t flush_buffer(hFILE *fp)+{+    const char *buffer = fp->buffer;+    while (buffer < fp->begin) {+        ssize_t n = fp->backend->write(fp, buffer, fp->begin - buffer);+        if (n < 0) { fp->has_errno = errno; return n; }+        buffer += n;+        fp->offset += n;+    }++    fp->begin = fp->buffer;  // Leave the buffer empty+    return 0;+}++int hflush(hFILE *fp)+{+    if (flush_buffer(fp) < 0) return EOF;+    if (fp->backend->flush) {+        if (fp->backend->flush(fp) < 0) { fp->has_errno = errno; return EOF; }+    }+    return 0;+}++/* Called only from hputc(), when our buffer is already full.  */+int hputc2(int c, hFILE *fp)+{+    if (flush_buffer(fp) < 0) return EOF;+    *(fp->begin++) = c;+    return c;+}++/* Called only from hwrite() and hputs2(); when called, our buffer is full and+   ncopied bytes from the source have already been copied to our buffer.  */+ssize_t hwrite2(hFILE *fp, const void *srcv, size_t totalbytes, size_t ncopied)+{+    const char *src = (const char *) srcv;+    ssize_t ret;+    const size_t capacity = fp->limit - fp->buffer;+    size_t remaining = totalbytes - ncopied;+    src += ncopied;++    ret = flush_buffer(fp);+    if (ret < 0) return ret;++    // Write large blocks out directly from the source buffer+    while (remaining * 2 >= capacity) {+        ssize_t n = fp->backend->write(fp, src, remaining);+        if (n < 0) { fp->has_errno = errno; return n; }+        fp->offset += n;+        src += n, remaining -= n;+    }++    // Just buffer any remaining characters+    memcpy(fp->begin, src, remaining);+    fp->begin += remaining;++    return totalbytes;+}++/* Called only from hputs(), when our buffer is already full.  */+int hputs2(const char *text, size_t totalbytes, size_t ncopied, hFILE *fp)+{+    return (hwrite2(fp, text, totalbytes, ncopied) >= 0)? 0 : EOF;+}++off_t hseek(hFILE *fp, off_t offset, int whence)+{+    off_t curpos, pos;++    if (writebuffer_is_nonempty(fp)) {+        int ret = flush_buffer(fp);+        if (ret < 0) return ret;+    }++    curpos = htell(fp);++    // Relative offsets are given relative to the hFILE's stream position,+    // which may differ from the backend's physical position due to buffering+    // read-ahead.  Correct for this by converting to an absolute position.+    if (whence == SEEK_CUR) {+        if (curpos + offset < 0) {+            // Either a negative offset resulted in a position before the+            // start of the file, or we overflowed when given a positive offset+            fp->has_errno = errno = (offset < 0)? EINVAL : EOVERFLOW;+            return -1;+        }++        whence = SEEK_SET;+        offset = curpos + offset;+    }++    // TODO Avoid seeking if the desired position is within our read buffer++    pos = fp->backend->seek(fp, offset, whence);+    if (pos < 0) { fp->has_errno = errno; return pos; }++    // Seeking succeeded, so discard any non-empty read buffer+    fp->begin = fp->end = fp->buffer;+    fp->at_eof = 0;++    fp->offset = pos;+    return pos;+}++int hclose(hFILE *fp)+{+    int err = fp->has_errno;++    if (writebuffer_is_nonempty(fp) && hflush(fp) < 0) err = fp->has_errno;+    if (fp->backend->close(fp) < 0) err = errno;+    hfile_destroy(fp);++    if (err) {+        errno = err;+        return EOF;+    }+    else return 0;+}++void hclose_abruptly(hFILE *fp)+{+    int save = errno;+    if (fp->backend->close(fp) < 0) { /* Ignore subsequent errors */ }+    hfile_destroy(fp);+    errno = save;+}+++/***************************+ * File descriptor backend *+ ***************************/++#ifndef _WIN32+#include <sys/socket.h>+#include <sys/stat.h>+#define HAVE_STRUCT_STAT_ST_BLKSIZE+#else+#include <winsock2.h>+#define HAVE_CLOSESOCKET+#define HAVE_SETMODE+#endif+#include <fcntl.h>+#include <unistd.h>++/* For Unix, it doesn't matter whether a file descriptor is a socket.+   However Windows insists on send()/recv() and its own closesocket()+   being used when fd happens to be a socket.  */++typedef struct {+    hFILE base;+    int fd;+    unsigned is_socket:1;+} hFILE_fd;++static ssize_t fd_read(hFILE *fpv, void *buffer, size_t nbytes)+{+    hFILE_fd *fp = (hFILE_fd *) fpv;+    ssize_t n;+    do {+        n = fp->is_socket? recv(fp->fd, buffer, nbytes, 0)+                         : read(fp->fd, buffer, nbytes);+    } while (n < 0 && errno == EINTR);+    return n;+}++static ssize_t fd_write(hFILE *fpv, const void *buffer, size_t nbytes)+{+    hFILE_fd *fp = (hFILE_fd *) fpv;+    ssize_t n;+    do {+        n = fp->is_socket?  send(fp->fd, buffer, nbytes, 0)+                         : write(fp->fd, buffer, nbytes);+    } while (n < 0 && errno == EINTR);+    return n;+}++static off_t fd_seek(hFILE *fpv, off_t offset, int whence)+{+    hFILE_fd *fp = (hFILE_fd *) fpv;+    return lseek(fp->fd, offset, whence);+}++static int fd_flush(hFILE *fpv)+{+    hFILE_fd *fp = (hFILE_fd *) fpv;+    int ret;+    do {+#ifdef HAVE_FDATASYNC+        ret = fdatasync(fp->fd);+#else+        ret = fsync(fp->fd);+#endif+        // Ignore invalid-for-fsync(2) errors due to being, e.g., a pipe,+        // and operation-not-supported errors (Mac OS X)+        if (ret < 0 && (errno == EINVAL || errno == ENOTSUP)) ret = 0;+    } while (ret < 0 && errno == EINTR);+    return ret;+}++static int fd_close(hFILE *fpv)+{+    hFILE_fd *fp = (hFILE_fd *) fpv;+    int ret;+    do {+#ifdef HAVE_CLOSESOCKET+        ret = fp->is_socket? closesocket(fp->fd) : close(fp->fd);+#else+        ret = close(fp->fd);+#endif+    } while (ret < 0 && errno == EINTR);+    return ret;+}++static const struct hFILE_backend fd_backend =+{+    fd_read, fd_write, fd_seek, fd_flush, fd_close+};++static size_t blksize(int fd)+{+#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE+    struct stat sbuf;+    if (fstat(fd, &sbuf) != 0) return 0;+    return sbuf.st_blksize;+#else+    return 0;+#endif+}++static hFILE *hopen_fd(const char *filename, const char *mode)+{+    hFILE_fd *fp = NULL;+    int fd = open(filename, hfile_oflags(mode), 0666);+    if (fd < 0) goto error;++    fp = (hFILE_fd *) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));+    if (fp == NULL) goto error;++    fp->fd = fd;+    fp->is_socket = 0;+    fp->base.backend = &fd_backend;+    return &fp->base;++error:+    if (fd >= 0) { int save = errno; (void) close(fd); errno = save; }+    hfile_destroy((hFILE *) fp);+    return NULL;+}++hFILE *hdopen(int fd, const char *mode)+{+    hFILE_fd *fp = (hFILE_fd*) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));+    if (fp == NULL) return NULL;++    fp->fd = fd;+    fp->is_socket = (strchr(mode, 's') != NULL);+    fp->base.backend = &fd_backend;+    return &fp->base;+}++static hFILE *hopen_fd_fileuri(const char *url, const char *mode)+{+    if (strncmp(url, "file://localhost/", 17) == 0) url += 16;+    else if (strncmp(url, "file:///", 8) == 0) url += 7;+    else { errno = EPROTONOSUPPORT; return NULL; }++    return hopen_fd(url, mode);+}++static hFILE *hopen_fd_stdinout(const char *mode)+{+    int fd = (strchr(mode, 'r') != NULL)? STDIN_FILENO : STDOUT_FILENO;+#if defined HAVE_SETMODE && defined O_BINARY+    if (setmode(fd, O_BINARY) < 0) return NULL;+#endif+    return hdopen(fd, mode);+}++int hfile_oflags(const char *mode)+{+    int rdwr = 0, flags = 0;+    const char *s;+    for (s = mode; *s; s++)+        switch (*s) {+        case 'r': rdwr = O_RDONLY;  break;+        case 'w': rdwr = O_WRONLY; flags |= O_CREAT | O_TRUNC;  break;+        case 'a': rdwr = O_WRONLY; flags |= O_CREAT | O_APPEND;  break;+        case '+': rdwr = O_RDWR;  break;+#ifdef O_CLOEXEC+        case 'e': flags |= O_CLOEXEC;  break;+#endif+#ifdef O_EXCL+        case 'x': flags |= O_EXCL;  break;+#endif+        default:  break;+        }++#ifdef O_BINARY+    flags |= O_BINARY;+#endif++    return rdwr | flags;+}+++/*********************+ * In-memory backend *+ *********************/++typedef struct {+    hFILE base;+    const char *buffer;+    size_t length, pos;+} hFILE_mem;++static ssize_t mem_read(hFILE *fpv, void *buffer, size_t nbytes)+{+    hFILE_mem *fp = (hFILE_mem *) fpv;+    size_t avail = fp->length - fp->pos;+    if (nbytes > avail) nbytes = avail;+    memcpy(buffer, fp->buffer + fp->pos, nbytes);+    fp->pos += nbytes;+    return nbytes;+}++static off_t mem_seek(hFILE *fpv, off_t offset, int whence)+{+    hFILE_mem *fp = (hFILE_mem *) fpv;+    size_t absoffset = (offset >= 0)? offset : -offset;+    size_t origin;++    switch (whence) {+    case SEEK_SET: origin = 0; break;+    case SEEK_CUR: origin = fp->pos; break;+    case SEEK_END: origin = fp->length; break;+    default: errno = EINVAL; return -1;+    }++    if ((offset  < 0 && absoffset > origin) ||+        (offset >= 0 && absoffset > fp->length - origin)) {+        errno = EINVAL;+        return -1;+    }++    fp->pos = origin + offset;+    return fp->pos;+}++static int mem_close(hFILE *fpv)+{+    return 0;+}++static const struct hFILE_backend mem_backend =+{+    mem_read, NULL, mem_seek, NULL, mem_close+};++static hFILE *hopen_mem(const char *data, const char *mode)+{+    if (strncmp(data, "data:", 5) == 0) data += 5;++    // TODO Implement write modes, which will require memory allocation+    if (strchr(mode, 'r') == NULL) { errno = EINVAL; return NULL; }++    hFILE_mem *fp = (hFILE_mem *) hfile_init(sizeof (hFILE_mem), mode, 0);+    if (fp == NULL) return NULL;++    fp->buffer = data;+    fp->length = strlen(data);+    fp->pos = 0;+    fp->base.backend = &mem_backend;+    return &fp->base;+}+++/*****************************************+ * Plugin and hopen() backend dispatcher *+ *****************************************/++#include "hts_internal.h"+#include "htslib/khash.h"++KHASH_MAP_INIT_STR(scheme_string, const struct hFILE_scheme_handler *);+static khash_t(scheme_string) *schemes = NULL;++struct hFILE_plugin_list {+    struct hFILE_plugin plugin;+    struct hFILE_plugin_list *next;+};++static struct hFILE_plugin_list *plugins = NULL;+static pthread_mutex_t plugins_lock = PTHREAD_MUTEX_INITIALIZER;++static void hfile_exit()+{+    pthread_mutex_lock(&plugins_lock);++    kh_destroy(scheme_string, schemes);++    while (plugins != NULL) {+        struct hFILE_plugin_list *p = plugins;+        if (p->plugin.destroy) p->plugin.destroy();+#ifdef ENABLE_PLUGINS+        if (p->plugin.obj) close_plugin(p->plugin.obj);+#endif+        plugins = p->next;+        free(p);+    }++    pthread_mutex_unlock(&plugins_lock);+    pthread_mutex_destroy(&plugins_lock);+}++void hfile_add_scheme_handler(const char *scheme,+                              const struct hFILE_scheme_handler *handler)+{+    int absent;+    khint_t k = kh_put(scheme_string, schemes, scheme, &absent);+    if (absent || handler->priority > kh_value(schemes, k)->priority) {+        kh_value(schemes, k) = handler;+    }+}++static int init_add_plugin(void *obj, int (*init)(struct hFILE_plugin *),+                           const char *pluginname)+{+    struct hFILE_plugin_list *p = malloc (sizeof (struct hFILE_plugin_list));+    if (p == NULL) abort();++    p->plugin.api_version = 1;+    p->plugin.obj = obj;+    p->plugin.name = NULL;+    p->plugin.destroy = NULL;++    int ret = (*init)(&p->plugin);++    if (ret != 0) {+        if (hts_verbose >= 4)+            fprintf(stderr, "[W::load_hfile_plugins] "+                    "initialisation failed for plugin \"%s\": %d\n",+                    pluginname, ret);+        free(p);+        return ret;+    }++    if (hts_verbose >= 5)+        fprintf(stderr, "[M::load_hfile_plugins] loaded \"%s\"\n", pluginname);++    p->next = plugins, plugins = p;+    return 0;+}++static void load_hfile_plugins()+{+    static const struct hFILE_scheme_handler+        data = { hopen_mem, hfile_always_local, "built-in", 80 },+        file = { hopen_fd_fileuri, hfile_always_local, "built-in", 80 };++    schemes = kh_init(scheme_string);+    if (schemes == NULL) abort();++    hfile_add_scheme_handler("data", &data);+    hfile_add_scheme_handler("file", &file);+    init_add_plugin(NULL, hfile_plugin_init_net, "knetfile");++#ifdef ENABLE_PLUGINS+    struct hts_path_itr path;+    const char *pluginname;+    hts_path_itr_setup(&path, NULL, NULL, "hfile_", 6, NULL, 0);+    while ((pluginname = hts_path_itr_next(&path)) != NULL) {+        void *obj;+        int (*init)(struct hFILE_plugin *) = (int (*)(struct hFILE_plugin *))+            load_plugin(&obj, pluginname, "hfile_plugin_init");++        if (init) {+            if (init_add_plugin(obj, init, pluginname) != 0)+                close_plugin(obj);+        }+    }+#else++#ifdef HAVE_IRODS+    init_add_plugin(NULL, hfile_plugin_init_irods, "iRODS");+#endif+#ifdef HAVE_LIBCURL+    init_add_plugin(NULL, hfile_plugin_init_libcurl, "libcurl");+#endif++#endif++    // In the unlikely event atexit() fails, it's better to succeed here and+    // carry on; then eventually when the program exits, we'll merely close+    // down the plugins uncleanly, as if we had aborted.+    (void) atexit(hfile_exit);+}++/* A filename like "foo:bar" in which we don't recognise the scheme is+   either an ordinary file or an indication of a missing or broken plugin.+   Try to open it as an ordinary file; but if there's no such file, set+   errno distinctively to make the plugin issue apparent.  */+static hFILE *hopen_unknown_scheme(const char *fname, const char *mode)+{+    hFILE *fp = hopen_fd(fname, mode);+    if (fp == NULL && errno == ENOENT) errno = EPROTONOSUPPORT;+    return fp;+}++/* Returns the appropriate handler, or NULL if the string isn't an URL.  */+static const struct hFILE_scheme_handler *find_scheme_handler(const char *s)+{+    static const struct hFILE_scheme_handler unknown_scheme =+        { hopen_unknown_scheme, hfile_always_local, "built-in", 0 };++    char scheme[12];+    int i;++    for (i = 0; i < sizeof scheme; i++)+        if (isalnum_c(s[i]) || s[i] == '+' || s[i] == '-' || s[i] == '.')+            scheme[i] = tolower_c(s[i]);+        else if (s[i] == ':') break;+        else return NULL;++    if (i == 0 || i >= sizeof scheme) return NULL;+    scheme[i] = '\0';++    pthread_mutex_lock(&plugins_lock);+    if (! schemes) load_hfile_plugins();+    pthread_mutex_unlock(&plugins_lock);++    khint_t k = kh_get(scheme_string, schemes, scheme);+    return (k != kh_end(schemes))? kh_value(schemes, k) : &unknown_scheme;+}++hFILE *hopen(const char *fname, const char *mode)+{+    const struct hFILE_scheme_handler *handler = find_scheme_handler(fname);+    if (handler) return handler->open(fname, mode);+    else if (strcmp(fname, "-") == 0) return hopen_fd_stdinout(mode);+    else return hopen_fd(fname, mode);+}++int hfile_always_local (const char *fname) { return 0; }+int hfile_always_remote(const char *fname) { return 1; }++int hisremote(const char *fname)+{+    const struct hFILE_scheme_handler *handler = find_scheme_handler(fname);+    return handler? handler->isremote(fname) : 0;+}
+ htslib-1.3.2/hfile_internal.h view
@@ -0,0 +1,139 @@+/*  hfile_internal.h -- internal parts of low-level input/output streams.++    Copyright (C) 2013-2015 Genome Research Ltd.++    Author: John Marshall <jm18@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HFILE_INTERNAL_H+#define HFILE_INTERNAL_H++#include "htslib/hfile.h"++#ifdef __cplusplus+extern "C" {+#endif++struct hFILE_backend {+    /* As per read(2), returning the number of bytes read (possibly 0) or+       negative (and setting errno) on errors.  Front-end code will call this+       repeatedly if necessary to attempt to get the desired byte count.  */+    ssize_t (*read)(hFILE *fp, void *buffer, size_t nbytes) HTS_RESULT_USED;++    /* As per write(2), returning the number of bytes written or negative (and+       setting errno) on errors.  Front-end code will call this repeatedly if+       necessary until the desired block is written or an error occurs.  */+    ssize_t (*write)(hFILE *fp, const void *buffer, size_t nbytes)+        HTS_RESULT_USED;++    /* As per lseek(2), returning the resulting offset within the stream or+       negative (and setting errno) on errors.  */+    off_t (*seek)(hFILE *fp, off_t offset, int whence) HTS_RESULT_USED;++    /* Performs low-level flushing, if any, e.g., fsync(2); for writing streams+       only.  Returns 0 for success or negative (and sets errno) on errors. */+    int (*flush)(hFILE *fp) HTS_RESULT_USED;++    /* Closes the underlying stream (for output streams, the buffer will+       already have been flushed), returning 0 for success or negative (and+       setting errno) on errors, as per close(2).  */+    int (*close)(hFILE *fp) HTS_RESULT_USED;+};++/* May be called by hopen_*() functions to decode a fopen()-style mode into+   open(2)-style flags.  */+int hfile_oflags(const char *mode);++/* Must be called by hopen_*() functions to allocate the hFILE struct and set+   up its base.  Capacity is a suggested buffer size (e.g., via fstat(2))+   or 0 for a default-sized buffer.  */+hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity);++/* May be called by hopen_*() functions to undo the effects of hfile_init()+   in the event opening the stream subsequently fails.  (This is safe to use+   even if fp is NULL.  This takes care to preserve errno.)  */+void hfile_destroy(hFILE *fp);+++struct hFILE_scheme_handler {+    /* Opens a stream when dispatched by hopen(); should call hfile_init()+       to malloc a struct "derived" from hFILE and initialise it appropriately,+       including setting base.backend to its own backend vector.  */+    hFILE *(*open)(const char *filename, const char *mode) HTS_RESULT_USED;++    /* Returns whether the URL denotes remote storage when dispatched by+       hisremote().  For simple cases, use one of hfile_always_*() below.  */+    int (*isremote)(const char *filename) HTS_RESULT_USED;++    /* The name of the plugin or other code providing this handler.  */+    const char *provider;++    /* If multiple handlers are registered for the same scheme, the one with+       the highest priority is used; range is 0 (lowest) to 100 (highest).  */+    int priority;+};++/* May be used as an isremote() function in simple cases.  */+extern int hfile_always_local (const char *fname);+extern int hfile_always_remote(const char *fname);++/* Should be called by plugins for each URL scheme they wish to handle.  */+void hfile_add_scheme_handler(const char *scheme,+                              const struct hFILE_scheme_handler *handler);++struct hFILE_plugin {+    /* On entry, HTSlib's plugin API version (currently 1).  */+    int api_version;++    /* On entry, the plugin's handle as returned by dlopen() etc.  */+    void *obj;++    /* The plugin should fill this in with its (human-readable) name.  */+    const char *name;++    /* The plugin may wish to fill in a function to be called on closing.  */+    void (*destroy)(void);+};++#ifdef ENABLE_PLUGINS+#define PLUGIN_GLOBAL(identifier,suffix) identifier++/* Plugins must define an entry point with this signature.  */+extern int hfile_plugin_init(struct hFILE_plugin *self);++#else+#define PLUGIN_GLOBAL(identifier,suffix) identifier##suffix++/* Only plugins distributed within the HTSlib source that might be built+   even with --disable-plugins need to use PLUGIN_GLOBAL and be listed here;+   others can simply define hfile_plugin_init().  */++extern int hfile_plugin_init_irods(struct hFILE_plugin *self);+extern int hfile_plugin_init_libcurl(struct hFILE_plugin *self);+#endif++/* This one is never built as a separate plugin.  */+extern int hfile_plugin_init_net(struct hFILE_plugin *self);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/hfile_net.c view
@@ -0,0 +1,112 @@+/*  hfile_net.c -- network backend for low-level input/output streams.++    Copyright (C) 2013-2015 Genome Research Ltd.++    Author: John Marshall <jm18@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <stdlib.h>+#include <errno.h>++#include "hfile_internal.h"++#include "htslib/knetfile.h"++typedef struct {+    hFILE base;+    knetFile *netfp;+} hFILE_net;++static int net_inited = 0;++#ifdef _WIN32+static void net_exit(void)+{+    knet_win32_destroy();+}+#endif++static int net_init(void)+{+#ifdef _WIN32+    if (knet_win32_init() != 0) return -1;++    // In the unlikely event atexit() fails, it's better to succeed here and+    // carry on and do the I/O; then eventually when the program exits, we'll+    // merely have failed to clean up properly, as if we had aborted.+    (void) atexit(net_exit);+#endif++    net_inited = 1;+    return 0;+}++static ssize_t net_read(hFILE *fpv, void *buffer, size_t nbytes)+{+    hFILE_net *fp = (hFILE_net *) fpv;+    return knet_read(fp->netfp, buffer, nbytes);+}++static off_t net_seek(hFILE *fpv, off_t offset, int whence)+{+    hFILE_net *fp = (hFILE_net *) fpv;+    return knet_seek(fp->netfp, offset, whence);+}++static int net_close(hFILE *fpv)+{+    hFILE_net *fp = (hFILE_net *) fpv;+    return knet_close(fp->netfp);+}++static const struct hFILE_backend net_backend =+{+    net_read, NULL, net_seek, NULL, net_close+};++hFILE *hopen_net(const char *filename, const char *mode)+{+    hFILE_net *fp;++    // Do any networking initialisation if this is the first use.+    if (! net_inited) { if (net_init() < 0) return NULL; }++    fp = (hFILE_net *) hfile_init(sizeof (hFILE_net), mode, 0);+    if (fp == NULL) return NULL;++    fp->netfp = knet_open(filename, mode);+    if (fp->netfp == NULL) { hfile_destroy((hFILE *) fp); return NULL; }++    fp->base.backend = &net_backend;+    return &fp->base;+}++int hfile_plugin_init_net(struct hFILE_plugin *self)+{+    static const struct hFILE_scheme_handler handler =+        { hopen_net, hfile_always_remote, "knetfile", 0 };++    self->name = "knetfile";+    hfile_add_scheme_handler("http", &handler);+    hfile_add_scheme_handler("ftp",  &handler);+    return 0;+}
+ htslib-1.3.2/hts.c view
@@ -0,0 +1,2065 @@+/*  hts.c -- format-neutral I/O, indexing, and iterator API functions.++    Copyright (C) 2008, 2009, 2012-2016 Genome Research Ltd.+    Copyright (C) 2012, 2013 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <zlib.h>+#include <stdio.h>+#include <string.h>+#include <stdlib.h>+#include <limits.h>+#include <fcntl.h>+#include <errno.h>+#include <sys/stat.h>+#include "htslib/bgzf.h"+#include "htslib/hts.h"+#include "cram/cram.h"+#include "htslib/hfile.h"+#include "version.h"+#include "hts_internal.h"++#include "htslib/kseq.h"+#define KS_BGZF 1+#if KS_BGZF+    // bgzf now supports gzip-compressed files, the gzFile branch can be removed+    KSTREAM_INIT2(, BGZF*, bgzf_read, 65536)+#else+    KSTREAM_INIT2(, gzFile, gzread, 16384)+#endif++#include "htslib/khash.h"+KHASH_INIT2(s2i,, kh_cstr_t, int64_t, 1, kh_str_hash_func, kh_str_hash_equal)++int hts_verbose = 3;++const char *hts_version()+{+    return HTS_VERSION;+}++const unsigned char seq_nt16_table[256] = {+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+     1, 2, 4, 8, 15,15,15,15, 15,15,15,15, 15, 0 /*=*/,15,15,+    15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,+    15,15, 5, 6,  8,15, 7, 9, 15,10,15,15, 15,15,15,15,+    15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,+    15,15, 5, 6,  8,15, 7, 9, 15,10,15,15, 15,15,15,15,++    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15+};++const char seq_nt16_str[] = "=ACMGRSVTWYHKDBN";++const int seq_nt16_int[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 };++/**********************+ *** Basic file I/O ***+ **********************/++static enum htsFormatCategory format_category(enum htsExactFormat fmt)+{+    switch (fmt) {+    case bam:+    case sam:+    case cram:+        return sequence_data;++    case vcf:+    case bcf:+        return variant_data;++    case bai:+    case crai:+    case csi:+    case gzi:+    case tbi:+        return index_file;++    case bed:+        return region_list;++    case unknown_format:+    case binary_format:+    case text_format:+    case format_maximum:+        break;+    }++    return unknown_category;+}++// Decompress up to ten or so bytes by peeking at the file, which must be+// positioned at the start of a GZIP block.+static size_t decompress_peek(hFILE *fp, unsigned char *dest, size_t destsize)+{+    // Typically at most a couple of hundred bytes of input are required+    // to get a few bytes of output from inflate(), so hopefully this buffer+    // size suffices in general.+    unsigned char buffer[512];+    z_stream zs;+    ssize_t npeek = hpeek(fp, buffer, sizeof buffer);++    if (npeek < 0) return 0;++    zs.zalloc = NULL;+    zs.zfree = NULL;+    zs.next_in = buffer;+    zs.avail_in = npeek;+    zs.next_out = dest;+    zs.avail_out = destsize;+    if (inflateInit2(&zs, 31) != Z_OK) return 0;++    while (zs.total_out < destsize)+        if (inflate(&zs, Z_SYNC_FLUSH) != Z_OK) break;++    destsize = zs.total_out;+    inflateEnd(&zs);++    return destsize;+}++// Parse "x.y" text, taking care because the string is not NUL-terminated+// and filling in major/minor only when the digits are followed by a delimiter,+// so we don't misread "1.10" as "1.1" due to reaching the end of the buffer.+static void+parse_version(htsFormat *fmt, const unsigned char *u, const unsigned char *ulim)+{+    const char *s    = (const char *) u;+    const char *slim = (const char *) ulim;+    short v;++    fmt->version.major = fmt->version.minor = -1;++    for (v = 0; s < slim && isdigit_c(*s); s++)+        v = 10 * v + *s - '0';++    if (s < slim) {+        fmt->version.major = v;+        if (*s == '.') {+            s++;+            for (v = 0; s < slim && isdigit_c(*s); s++)+                v = 10 * v + *s - '0';+            if (s < slim)+                fmt->version.minor = v;+        }+        else+            fmt->version.minor = 0;+    }+}++int hts_detect_format(hFILE *hfile, htsFormat *fmt)+{+    unsigned char s[21];+    ssize_t len = hpeek(hfile, s, 18);+    if (len < 0) return -1;++    if (len >= 2 && s[0] == 0x1f && s[1] == 0x8b) {+        // The stream is either gzip-compressed or BGZF-compressed.+        // Determine which, and decompress the first few bytes.+        fmt->compression = (len >= 18 && (s[3] & 4) &&+                            memcmp(&s[12], "BC\2\0", 4) == 0)? bgzf : gzip;+        len = decompress_peek(hfile, s, sizeof s);+    }+    else {+        fmt->compression = no_compression;+        len = hpeek(hfile, s, sizeof s);+    }+    if (len < 0) return -1;++    fmt->compression_level = -1;+    fmt->specific = NULL;++    if (len >= 6 && memcmp(s,"CRAM",4) == 0 && s[4]>=1 && s[4]<=3 && s[5]<=1) {+        fmt->category = sequence_data;+        fmt->format = cram;+        fmt->version.major = s[4], fmt->version.minor = s[5];+        fmt->compression = custom;+        return 0;+    }+    else if (len >= 4 && s[3] <= '\4') {+        if (memcmp(s, "BAM\1", 4) == 0) {+            fmt->category = sequence_data;+            fmt->format = bam;+            // TODO Decompress enough to pick version from @HD-VN header+            fmt->version.major = 1, fmt->version.minor = -1;+            return 0;+        }+        else if (memcmp(s, "BAI\1", 4) == 0) {+            fmt->category = index_file;+            fmt->format = bai;+            fmt->version.major = -1, fmt->version.minor = -1;+            return 0;+        }+        else if (memcmp(s, "BCF\4", 4) == 0) {+            fmt->category = variant_data;+            fmt->format = bcf;+            fmt->version.major = 1, fmt->version.minor = -1;+            return 0;+        }+        else if (memcmp(s, "BCF\2", 4) == 0) {+            fmt->category = variant_data;+            fmt->format = bcf;+            fmt->version.major = s[3];+            fmt->version.minor = (len >= 5 && s[4] <= 2)? s[4] : 0;+            return 0;+        }+        else if (memcmp(s, "CSI\1", 4) == 0) {+            fmt->category = index_file;+            fmt->format = csi;+            fmt->version.major = 1, fmt->version.minor = -1;+            return 0;+        }+        else if (memcmp(s, "TBI\1", 4) == 0) {+            fmt->category = index_file;+            fmt->format = tbi;+            fmt->version.major = -1, fmt->version.minor = -1;+            return 0;+        }+    }+    else if (len >= 16 && memcmp(s, "##fileformat=VCF", 16) == 0) {+        fmt->category = variant_data;+        fmt->format = vcf;+        if (len >= 21 && s[16] == 'v')+            parse_version(fmt, &s[17], &s[len]);+        else+            fmt->version.major = fmt->version.minor = -1;+        return 0;+    }+    else if (len >= 4 && s[0] == '@' &&+             (memcmp(s, "@HD\t", 4) == 0 || memcmp(s, "@SQ\t", 4) == 0 ||+              memcmp(s, "@RG\t", 4) == 0 || memcmp(s, "@PG\t", 4) == 0)) {+        fmt->category = sequence_data;+        fmt->format = sam;+        // @HD-VN is not guaranteed to be the first tag, but then @HD is+        // not guaranteed to be present at all...+        if (len >= 9 && memcmp(s, "@HD\tVN:", 7) == 0)+            parse_version(fmt, &s[7], &s[len]);+        else+            fmt->version.major = 1, fmt->version.minor = -1;+        return 0;+    }+    else {+        // Various possibilities for tab-delimited text:+        // .crai   (gzipped tab-delimited six columns: seqid 5*number)+        // .bed    ([3..12] tab-delimited columns)+        // .bedpe  (>= 10 tab-delimited columns)+        // .sam    (tab-delimited >= 11 columns: seqid number seqid...)+        // FIXME For now, assume it's SAM+        fmt->category = sequence_data;+        fmt->format = sam;+        fmt->version.major = 1, fmt->version.minor = -1;+        return 0;+    }++    fmt->category = unknown_category;+    fmt->format = unknown_format;+    fmt->version.major = fmt->version.minor = -1;+    fmt->compression = no_compression;+    return 0;+}++char *hts_format_description(const htsFormat *format)+{+    kstring_t str = { 0, 0, NULL };++    switch (format->format) {+    case sam:   kputs("SAM", &str); break;+    case bam:   kputs("BAM", &str); break;+    case cram:  kputs("CRAM", &str); break;+    case vcf:   kputs("VCF", &str); break;+    case bcf:+        if (format->version.major == 1) kputs("Legacy BCF", &str);+        else kputs("BCF", &str);+        break;+    case bai:   kputs("BAI", &str); break;+    case crai:  kputs("CRAI", &str); break;+    case csi:   kputs("CSI", &str); break;+    case tbi:   kputs("Tabix", &str); break;+    default:    kputs("unknown", &str); break;+    }++    if (format->version.major >= 0) {+        kputs(" version ", &str);+        kputw(format->version.major, &str);+        if (format->version.minor >= 0) {+            kputc('.', &str);+            kputw(format->version.minor, &str);+        }+    }++    switch (format->compression) {+    case custom: kputs(" compressed", &str); break;+    case gzip:   kputs(" gzip-compressed", &str); break;+    case bgzf:+        switch (format->format) {+        case bam:+        case bcf:+        case csi:+        case tbi:+            // These are by definition BGZF, so just use the generic term+            kputs(" compressed", &str);+            break;+        default:+            kputs(" BGZF-compressed", &str);+            break;+        }+        break;+    default: break;+    }++    switch (format->category) {+    case sequence_data: kputs(" sequence", &str); break;+    case variant_data:  kputs(" variant calling", &str); break;+    case index_file:    kputs(" index", &str); break;+    case region_list:   kputs(" genomic region", &str); break;+    default: break;+    }++    if (format->compression == no_compression)+        switch (format->format) {+        case sam:+        case crai:+        case vcf:+        case bed:+            kputs(" text", &str);+            break;++        default:+            kputs(" data", &str);+            break;+        }+    else+        kputs(" data", &str);++    return ks_release(&str);+}++htsFile *hts_open_format(const char *fn, const char *mode, const htsFormat *fmt)+{+    char smode[102], *cp, *cp2, *mode_c;+    htsFile *fp = NULL;+    hFILE *hfile;+    char fmt_code = '\0';++    strncpy(smode, mode, 100);+    smode[100]=0;+    if ((cp = strchr(smode, ',')))+        *cp = '\0';++    // Migrate format code (b or c) to the end of the smode buffer.+    for (cp2 = cp = smode; *cp; cp++) {+        if (*cp == 'b')+            fmt_code = 'b';+        else if (*cp == 'c')+            fmt_code = 'c';+        else+            *cp2++ = *cp;+    }+    mode_c = cp2;+    *cp2++ = fmt_code;+    *cp2++ = 0;+    *cp2++ = 0;++    // Set or reset the format code if opts->format is used+    if (fmt && fmt->format != unknown_format)+        *mode_c = "\0g\0\0b\0c\0\0b\0g\0\0"[fmt->format];++    hfile = hopen(fn, smode);+    if (hfile == NULL) goto error;++    fp = hts_hopen(hfile, fn, smode);+    if (fp == NULL) goto error;++    if (fmt && fmt->specific)+        if (hts_opt_apply(fp, fmt->specific) != 0)+            goto error;++    return fp;++error:+    if (hts_verbose >= 2)+        fprintf(stderr, "[E::%s] fail to open file '%s'\n", __func__, fn);++    if (hfile)+        hclose_abruptly(hfile);++    return NULL;+}++htsFile *hts_open(const char *fn, const char *mode) {+    return hts_open_format(fn, mode, NULL);+}++/*+ * Splits str into a prefix, delimiter ('\0' or delim), and suffix, writing+ * the prefix in lowercase into buf and returning a pointer to the suffix.+ * On return, buf is always NUL-terminated; thus assumes that the "keyword"+ * prefix should be one of several known values of maximum length buflen-2.+ * (If delim is not found, returns a pointer to the '\0'.)+ */+static const char *+scan_keyword(const char *str, char delim, char *buf, size_t buflen)+{+    size_t i = 0;+    while (*str && *str != delim) {+        if (i < buflen-1) buf[i++] = tolower_c(*str);+        str++;+    }++    buf[i] = '\0';+    return *str? str+1 : str;+}++/*+ * Parses arg and appends it to the option list.+ *+ * Returns 0 on success;+ *        -1 on failure.+ */+int hts_opt_add(hts_opt **opts, const char *c_arg) {+    hts_opt *o, *t;+    char *val;++    if (!c_arg)+        return -1;++    if (!(o =  malloc(sizeof(*o))))+        return -1;++    if (!(o->arg = strdup(c_arg))) {+        free(o);+        return -1;+    }++    if (!(val = strchr(o->arg, '=')))+        val = "1"; // assume boolean+    else+        *val++ = '\0';++    if (strcmp(o->arg, "decode_md") == 0 ||+        strcmp(o->arg, "DECODE_MD") == 0)+        o->opt = CRAM_OPT_DECODE_MD, o->val.i = atoi(val);++    else if (strcmp(o->arg, "verbosity") == 0 ||+             strcmp(o->arg, "VERBOSITY") == 0)+        o->opt = CRAM_OPT_VERBOSITY, o->val.i = atoi(val);++    else if (strcmp(o->arg, "seqs_per_slice") == 0 ||+             strcmp(o->arg, "SEQS_PER_SLICE") == 0)+        o->opt = CRAM_OPT_SEQS_PER_SLICE, o->val.i = atoi(val);++    else if (strcmp(o->arg, "slices_per_container") == 0 ||+             strcmp(o->arg, "SLICES_PER_CONTAINER") == 0)+        o->opt = CRAM_OPT_SLICES_PER_CONTAINER, o->val.i = atoi(val);++    else if (strcmp(o->arg, "embed_ref") == 0 ||+             strcmp(o->arg, "EMBED_REF") == 0)+        o->opt = CRAM_OPT_EMBED_REF, o->val.i = atoi(val);++    else if (strcmp(o->arg, "no_ref") == 0 ||+             strcmp(o->arg, "NO_REF") == 0)+        o->opt = CRAM_OPT_NO_REF, o->val.i = atoi(val);++    else if (strcmp(o->arg, "ignore_md5") == 0 ||+             strcmp(o->arg, "IGNORE_MD5") == 0)+        o->opt = CRAM_OPT_IGNORE_MD5, o->val.i = atoi(val);++    else if (strcmp(o->arg, "use_bzip2") == 0 ||+             strcmp(o->arg, "USE_BZIP2") == 0)+        o->opt = CRAM_OPT_USE_BZIP2, o->val.i = atoi(val);++    else if (strcmp(o->arg, "use_rans") == 0 ||+             strcmp(o->arg, "USE_RANS") == 0)+        o->opt = CRAM_OPT_USE_RANS, o->val.i = atoi(val);++    else if (strcmp(o->arg, "use_lzma") == 0 ||+             strcmp(o->arg, "USE_LZMA") == 0)+        o->opt = CRAM_OPT_USE_LZMA, o->val.i = atoi(val);++    else if (strcmp(o->arg, "reference") == 0 ||+             strcmp(o->arg, "REFERENCE") == 0)+        o->opt = CRAM_OPT_REFERENCE, o->val.s = val;++    else if (strcmp(o->arg, "version") == 0 ||+             strcmp(o->arg, "VERSION") == 0)+        o->opt = CRAM_OPT_VERSION, o->val.s =val;++    else if (strcmp(o->arg, "multi_seq_per_slice") == 0 ||+             strcmp(o->arg, "MULTI_SEQ_PER_SLICE") == 0)+        o->opt = CRAM_OPT_MULTI_SEQ_PER_SLICE, o->val.i = atoi(val);++    else if (strcmp(o->arg, "nthreads") == 0 ||+             strcmp(o->arg, "NTHREADS") == 0)+        o->opt = HTS_OPT_NTHREADS, o->val.i = atoi(val);++    else if (strcmp(o->arg, "required_fields") == 0 ||+             strcmp(o->arg, "REQUIRED_FIELDS") == 0)+        o->opt = CRAM_OPT_REQUIRED_FIELDS, o->val.i = strtol(val, NULL, 0);++    else {+        fprintf(stderr, "Unknown option '%s'\n", o->arg);+        free(o->arg);+        free(o);+        return -1;+    }++    o->next = NULL;++    // Append; assumes small list.+    if (*opts) {+        t = *opts;+        while (t->next)+            t = t->next;+        t->next = o;+    } else {+        *opts = o;+    }++    return 0;+}++/*+ * Applies an hts_opt option list to a given htsFile.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int hts_opt_apply(htsFile *fp, hts_opt *opts) {+    hts_opt *last = NULL;++    for (; opts;  opts = (last=opts)->next)+        if (hts_set_opt(fp,  opts->opt,  opts->val) != 0)+            return -1;++    return 0;+}++/*+ * Frees an hts_opt list.+ */+void hts_opt_free(hts_opt *opts) {+    hts_opt *last = NULL;+    while (opts) {+        opts = (last=opts)->next;+        free(last->arg);+        free(last);+    }+}+++/*+ * Tokenise options as (key(=value)?,)*(key(=value)?)?+ * NB: No provision for ',' appearing in the value!+ * Add backslashing rules?+ *+ * This could be used as part of a general command line option parser or+ * as a string concatenated onto the file open mode.+ *+ * Returns 0 on success+ *        -1 on failure.+ */+int hts_parse_opt_list(htsFormat *fmt, const char *str) {+    while (str && *str) {+        const char *str_start;+        int len;+        char arg[8001];++        while (*str && *str == ',')+            str++;++        for (str_start = str; *str && *str != ','; str++);+        len = str - str_start;++        // Produce a nul terminated copy of the option+        strncpy(arg, str_start, len < 8000 ? len : 8000);+        arg[len < 8000 ? len : 8000] = '\0';++        if (hts_opt_add((hts_opt **)&fmt->specific, arg) != 0)+            return -1;++        if (*str)+            str++;+    }++    return 0;+}++/*+ * Accepts a string file format (sam, bam, cram, vcf, bam) optionally+ * followed by a comma separated list of key=value options and splits+ * these up into the fields of htsFormat struct.+ *+ * format is assumed to be already initialised, either to blank+ * "unknown" values or via previous hts_opt_add calls.+ *+ * Returns 0 on success+ *        -1 on failure.+ */+int hts_parse_format(htsFormat *format, const char *str) {+    char fmt[8];+    const char *cp = scan_keyword(str, ',', fmt, sizeof fmt);++    format->version.minor = 0; // unknown+    format->version.major = 0; // unknown++    if (strcmp(fmt, "sam") == 0) {+        format->category          = sequence_data;+        format->format            = sam;+        format->compression       = no_compression;;+        format->compression_level = 0;+    } else if (strcmp(fmt, "bam") == 0) {+        format->category          = sequence_data;+        format->format            = bam;+        format->compression       = bgzf;+        format->compression_level = -1;+    } else if (strcmp(fmt, "cram") == 0) {+        format->category          = sequence_data;+        format->format            = cram;+        format->compression       = custom;+        format->compression_level = -1;+    } else if (strcmp(fmt, "vcf") == 0) {+        format->category          = variant_data;+        format->format            = vcf;+        format->compression       = no_compression;;+        format->compression_level = 0;+    } else if (strcmp(fmt, "bcf") == 0) {+        format->category          = variant_data;+        format->format            = bcf;+        format->compression       = bgzf;+        format->compression_level = -1;+    } else {+        return -1;+    }++    return hts_parse_opt_list(format, cp);+}+++/*+ * Tokenise options as (key(=value)?,)*(key(=value)?)?+ * NB: No provision for ',' appearing in the value!+ * Add backslashing rules?+ *+ * This could be used as part of a general command line option parser or+ * as a string concatenated onto the file open mode.+ *+ * Returns 0 on success+ *        -1 on failure.+ */+static int hts_process_opts(htsFile *fp, const char *opts) {+    htsFormat fmt;++    fmt.specific = NULL;+    if (hts_parse_opt_list(&fmt, opts) != 0)+        return -1;++    if (hts_opt_apply(fp, fmt.specific) != 0) {+        hts_opt_free(fmt.specific);+        return -1;+    }++    hts_opt_free(fmt.specific);++    return 0;+}+++htsFile *hts_hopen(struct hFILE *hfile, const char *fn, const char *mode)+{+    htsFile *fp = (htsFile*)calloc(1, sizeof(htsFile));+    char simple_mode[101], *cp, *opts;+    simple_mode[100] = '\0';++    if (fp == NULL) goto error;++    fp->fn = strdup(fn);+    fp->is_be = ed_is_big();++    // Split mode into simple_mode,opts strings+    if ((cp = strchr(mode, ','))) {+        strncpy(simple_mode, mode, cp-mode <= 100 ? cp-mode : 100);+        simple_mode[cp-mode] = '\0';+        opts = cp+1;+    } else {+        strncpy(simple_mode, mode, 100);+        opts = NULL;+    }++    if (strchr(simple_mode, 'r')) {+        if (hts_detect_format(hfile, &fp->format) < 0) goto error;+    }+    else if (strchr(simple_mode, 'w') || strchr(simple_mode, 'a')) {+        htsFormat *fmt = &fp->format;+        fp->is_write = 1;++        if (strchr(simple_mode, 'b')) fmt->format = binary_format;+        else if (strchr(simple_mode, 'c')) fmt->format = cram;+        else fmt->format = text_format;++        if (strchr(simple_mode, 'z')) fmt->compression = bgzf;+        else if (strchr(simple_mode, 'g')) fmt->compression = gzip;+        else if (strchr(simple_mode, 'u')) fmt->compression = no_compression;+        else {+            // No compression mode specified, set to the default for the format+            switch (fmt->format) {+            case binary_format: fmt->compression = bgzf; break;+            case cram: fmt->compression = custom; break;+            case text_format: fmt->compression = no_compression; break;+            default: abort();+            }+        }++        // Fill in category (if determinable; e.g. 'b' could be BAM or BCF)+        fmt->category = format_category(fmt->format);++        fmt->version.major = fmt->version.minor = -1;+        fmt->compression_level = -1;+        fmt->specific = NULL;+    }+    else goto error;++    switch (fp->format.format) {+    case binary_format:+    case bam:+    case bcf:+        fp->fp.bgzf = bgzf_hopen(hfile, simple_mode);+        if (fp->fp.bgzf == NULL) goto error;+        fp->is_bin = 1;+        break;++    case cram:+        fp->fp.cram = cram_dopen(hfile, fn, simple_mode);+        if (fp->fp.cram == NULL) goto error;+        if (!fp->is_write)+            cram_set_option(fp->fp.cram, CRAM_OPT_DECODE_MD, 1);+        fp->is_cram = 1;+        break;++    case text_format:+    case sam:+    case vcf:+        if (!fp->is_write) {+        #if KS_BGZF+            BGZF *gzfp = bgzf_hopen(hfile, simple_mode);+        #else+            // TODO Implement gzip hFILE adaptor+            hclose(hfile); // This won't work, especially for stdin+            gzFile gzfp = strcmp(fn, "-")? gzopen(fn, "rb") : gzdopen(fileno(stdin), "rb");+        #endif+            if (gzfp) fp->fp.voidp = ks_init(gzfp);+            else goto error;+        }+        else if (fp->format.compression != no_compression) {+            fp->fp.bgzf = bgzf_hopen(hfile, simple_mode);+            if (fp->fp.bgzf == NULL) goto error;+        }+        else+            fp->fp.hfile = hfile;+        break;++    default:+        goto error;+    }++    if (opts)+        hts_process_opts(fp, opts);++    return fp;++error:+    if (hts_verbose >= 2)+        fprintf(stderr, "[E::%s] fail to open file '%s'\n", __func__, fn);++    if (fp) {+        free(fp->fn);+        free(fp->fn_aux);+        free(fp);+    }+    return NULL;+}++int hts_close(htsFile *fp)+{+    int ret, save;++    switch (fp->format.format) {+    case binary_format:+    case bam:+    case bcf:+        ret = bgzf_close(fp->fp.bgzf);+        break;++    case cram:+        if (!fp->is_write) {+            switch (cram_eof(fp->fp.cram)) {+            case 2:+                fprintf(stderr, "[W::%s] EOF marker is absent. The input is probably truncated.\n", __func__);+                break;+            case 0:  /* not at EOF, but may not have wanted all seqs */+            default: /* case 1, expected EOF */+                break;+            }+        }+        ret = cram_close(fp->fp.cram);+        break;++    case text_format:+    case sam:+    case vcf:+        if (!fp->is_write) {+        #if KS_BGZF+            BGZF *gzfp = ((kstream_t*)fp->fp.voidp)->f;+            ret = bgzf_close(gzfp);+        #else+            gzFile gzfp = ((kstream_t*)fp->fp.voidp)->f;+            ret = gzclose(gzfp);+        #endif+            ks_destroy((kstream_t*)fp->fp.voidp);+        }+        else if (fp->format.compression != no_compression)+            ret = bgzf_close(fp->fp.bgzf);+        else+            ret = hclose(fp->fp.hfile);+        break;++    default:+        ret = -1;+        break;+    }++    save = errno;+    free(fp->fn);+    free(fp->fn_aux);+    free(fp->line.s);+    free(fp);+    errno = save;+    return ret;+}++const htsFormat *hts_get_format(htsFile *fp)+{+    return fp? &fp->format : NULL;+}++const char *hts_format_file_extension(const htsFormat *format) {+    if (!format)+        return "?";++    switch (format->format) {+    case sam:  return "sam";+    case bam:  return "bam";+    case bai:  return "bai";+    case cram: return "cram";+    case crai: return "crai";+    case vcf:  return "vcf";+    case bcf:  return "bcf";+    case csi:  return "csi";+    case gzi:  return "gzi";+    case tbi:  return "tbi";+    case bed:  return "bed";+    default:   return "?";+    }+}++int hts_set_opt(htsFile *fp, enum hts_fmt_option opt, ...) {+    int r;+    va_list args;++    if (opt == HTS_OPT_NTHREADS) {+        va_start(args, opt);+        int nthreads = va_arg(args, int);+        va_end(args);+        return hts_set_threads(fp, nthreads);+    }++    if (fp->format.format != cram)+        return 0;++    va_start(args, opt);+    r = cram_set_voption(fp->fp.cram, opt, args);+    va_end(args);++    return r;+}++int hts_set_threads(htsFile *fp, int n)+{+    if (fp->format.compression == bgzf) {+        return bgzf_mt(fp->fp.bgzf, n, 256);+    } else if (fp->format.format == cram) {+        return hts_set_opt(fp, CRAM_OPT_NTHREADS, n);+    }+    else return 0;+}++int hts_set_fai_filename(htsFile *fp, const char *fn_aux)+{+    free(fp->fn_aux);+    if (fn_aux) {+        fp->fn_aux = strdup(fn_aux);+        if (fp->fn_aux == NULL) return -1;+    }+    else fp->fn_aux = NULL;++    if (fp->format.format == cram)+        if (cram_set_option(fp->fp.cram, CRAM_OPT_REFERENCE, fp->fn_aux))+            return -1;++    return 0;+}++// For VCF/BCF backward sweeper. Not exposing these functions because their+// future is uncertain. Things will probably have to change with hFILE...+BGZF *hts_get_bgzfp(htsFile *fp)+{+    if ( fp->is_bin )+        return fp->fp.bgzf;+    else+        return ((kstream_t*)fp->fp.voidp)->f;+}+int hts_useek(htsFile *fp, long uoffset, int where)+{+    if ( fp->is_bin )+        return bgzf_useek(fp->fp.bgzf, uoffset, where);+    else+    {+        ks_rewind((kstream_t*)fp->fp.voidp);+        ((kstream_t*)fp->fp.voidp)->seek_pos = uoffset;+        return bgzf_useek(((kstream_t*)fp->fp.voidp)->f, uoffset, where);+    }+}+long hts_utell(htsFile *fp)+{+    if ( fp->is_bin )+        return bgzf_utell(fp->fp.bgzf);+    else+        return ((kstream_t*)fp->fp.voidp)->seek_pos;+}++int hts_getline(htsFile *fp, int delimiter, kstring_t *str)+{+    int ret, dret;+    ret = ks_getuntil((kstream_t*)fp->fp.voidp, delimiter, str, &dret);+    ++fp->lineno;+    return ret;+}++char **hts_readlist(const char *string, int is_file, int *_n)+{+    int m = 0, n = 0, dret;+    char **s = 0;+    if ( is_file )+    {+#if KS_BGZF+        BGZF *fp = bgzf_open(string, "r");+#else+        gzFile fp = gzopen(string, "r");+#endif+        if ( !fp ) return NULL;++        kstream_t *ks;+        kstring_t str;+        str.s = 0; str.l = str.m = 0;+        ks = ks_init(fp);+        while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0)+        {+            if (str.l == 0) continue;+            n++;+            hts_expand(char*,n,m,s);+            s[n-1] = strdup(str.s);+        }+        ks_destroy(ks);+#if KS_BGZF+        bgzf_close(fp);+#else+        gzclose(fp);+#endif+        free(str.s);+    }+    else+    {+        const char *q = string, *p = string;+        while ( 1 )+        {+            if (*p == ',' || *p == 0)+            {+                n++;+                hts_expand(char*,n,m,s);+                s[n-1] = (char*)calloc(p - q + 1, 1);+                strncpy(s[n-1], q, p - q);+                q = p + 1;+            }+            if ( !*p ) break;+            p++;+        }+    }+    s = (char**)realloc(s, n * sizeof(char*));+    *_n = n;+    return s;+}++char **hts_readlines(const char *fn, int *_n)+{+    int m = 0, n = 0, dret;+    char **s = 0;+#if KS_BGZF+    BGZF *fp = bgzf_open(fn, "r");+#else+    gzFile fp = gzopen(fn, "r");+#endif+    if ( fp ) { // read from file+        kstream_t *ks;+        kstring_t str;+        str.s = 0; str.l = str.m = 0;+        ks = ks_init(fp);+        while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0) {+            if (str.l == 0) continue;+            if (m == n) {+                m = m? m<<1 : 16;+                s = (char**)realloc(s, m * sizeof(char*));+            }+            s[n++] = strdup(str.s);+        }+        ks_destroy(ks);+        #if KS_BGZF+            bgzf_close(fp);+        #else+            gzclose(fp);+        #endif+        s = (char**)realloc(s, n * sizeof(char*));+        free(str.s);+    } else if (*fn == ':') { // read from string+        const char *q, *p;+        for (q = p = fn + 1;; ++p)+            if (*p == ',' || *p == 0) {+                if (m == n) {+                    m = m? m<<1 : 16;+                    s = (char**)realloc(s, m * sizeof(char*));+                }+                s[n] = (char*)calloc(p - q + 1, 1);+                strncpy(s[n++], q, p - q);+                q = p + 1;+                if (*p == 0) break;+            }+    } else return 0;+    s = (char**)realloc(s, n * sizeof(char*));+    *_n = n;+    return s;+}++// DEPRECATED: To be removed in a future HTSlib release+int hts_file_type(const char *fname)+{+    int len = strlen(fname);+    if ( !strcasecmp(".vcf.gz",fname+len-7) ) return FT_VCF_GZ;+    if ( !strcasecmp(".vcf",fname+len-4) ) return FT_VCF;+    if ( !strcasecmp(".bcf",fname+len-4) ) return FT_BCF_GZ;+    if ( !strcmp("-",fname) ) return FT_STDIN;++    hFILE *f = hopen(fname, "r");+    if (f == NULL) return 0;++    htsFormat fmt;+    if (hts_detect_format(f, &fmt) < 0) { hclose_abruptly(f); return 0; }+    if (hclose(f) < 0) return 0;++    switch (fmt.format) {+    case vcf: return (fmt.compression == no_compression)? FT_VCF : FT_VCF_GZ;+    case bcf: return (fmt.compression == no_compression)? FT_BCF : FT_BCF_GZ;+    default:  return 0;+    }+}++/****************+ *** Indexing ***+ ****************/++#define HTS_MIN_MARKER_DIST 0x10000++// Finds the special meta bin+//  ((1<<(3 * n_lvls + 3)) - 1) / 7 + 1+#define META_BIN(idx) ((idx)->n_bins + 1)++#define pair64_lt(a,b) ((a).u < (b).u)++#include "htslib/ksort.h"+KSORT_INIT(_off, hts_pair64_t, pair64_lt)++typedef struct {+    int32_t m, n;+    uint64_t loff;+    hts_pair64_t *list;+} bins_t;++#include "htslib/khash.h"+KHASH_MAP_INIT_INT(bin, bins_t)+typedef khash_t(bin) bidx_t;++typedef struct {+    int32_t n, m;+    uint64_t *offset;+} lidx_t;++struct __hts_idx_t {+    int fmt, min_shift, n_lvls, n_bins;+    uint32_t l_meta;+    int32_t n, m;+    uint64_t n_no_coor;+    bidx_t **bidx;+    lidx_t *lidx;+    uint8_t *meta;+    struct {+        uint32_t last_bin, save_bin;+        int last_coor, last_tid, save_tid, finished;+        uint64_t last_off, save_off;+        uint64_t off_beg, off_end;+        uint64_t n_mapped, n_unmapped;+    } z; // keep internal states+};++static inline void insert_to_b(bidx_t *b, int bin, uint64_t beg, uint64_t end)+{+    khint_t k;+    bins_t *l;+    int absent;+    k = kh_put(bin, b, bin, &absent);+    l = &kh_value(b, k);+    if (absent) {+        l->m = 1; l->n = 0;+        l->list = (hts_pair64_t*)calloc(l->m, sizeof(hts_pair64_t));+    }+    if (l->n == l->m) {+        l->m <<= 1;+        l->list = (hts_pair64_t*)realloc(l->list, l->m * sizeof(hts_pair64_t));+    }+    l->list[l->n].u = beg;+    l->list[l->n++].v = end;+}++static inline void insert_to_l(lidx_t *l, int64_t _beg, int64_t _end, uint64_t offset, int min_shift)+{+    int i, beg, end;+    beg = _beg >> min_shift;+    end = (_end - 1) >> min_shift;+    if (l->m < end + 1) {+        int old_m = l->m;+        l->m = end + 1;+        kroundup32(l->m);+        l->offset = (uint64_t*)realloc(l->offset, l->m * sizeof(uint64_t));+        memset(l->offset + old_m, 0xff, 8 * (l->m - old_m)); // fill l->offset with (uint64_t)-1+    }+    if (beg == end) { // to save a loop in this case+        if (l->offset[beg] == (uint64_t)-1) l->offset[beg] = offset;+    } else {+        for (i = beg; i <= end; ++i)+            if (l->offset[i] == (uint64_t)-1) l->offset[i] = offset;+    }+    if (l->n < end + 1) l->n = end + 1;+}++hts_idx_t *hts_idx_init(int n, int fmt, uint64_t offset0, int min_shift, int n_lvls)+{+    hts_idx_t *idx;+    idx = (hts_idx_t*)calloc(1, sizeof(hts_idx_t));+    if (idx == NULL) return NULL;+    idx->fmt = fmt;+    idx->min_shift = min_shift;+    idx->n_lvls = n_lvls;+    idx->n_bins = ((1<<(3 * n_lvls + 3)) - 1) / 7;+    idx->z.save_bin = idx->z.save_tid = idx->z.last_tid = idx->z.last_bin = 0xffffffffu;+    idx->z.save_off = idx->z.last_off = idx->z.off_beg = idx->z.off_end = offset0;+    idx->z.last_coor = 0xffffffffu;+    if (n) {+        idx->n = idx->m = n;+        idx->bidx = (bidx_t**)calloc(n, sizeof(bidx_t*));+        if (idx->bidx == NULL) { free(idx); return NULL; }+        idx->lidx = (lidx_t*) calloc(n, sizeof(lidx_t));+        if (idx->lidx == NULL) { free(idx->bidx); free(idx); return NULL; }+    }+    return idx;+}++static void update_loff(hts_idx_t *idx, int i, int free_lidx)+{+    bidx_t *bidx = idx->bidx[i];+    lidx_t *lidx = &idx->lidx[i];+    khint_t k;+    int l;+    uint64_t offset0 = 0;+    if (bidx) {+        k = kh_get(bin, bidx, META_BIN(idx));+        if (k != kh_end(bidx))+            offset0 = kh_val(bidx, k).list[0].u;+        for (l = 0; l < lidx->n && lidx->offset[l] == (uint64_t)-1; ++l)+            lidx->offset[l] = offset0;+    } else l = 1;+    for (; l < lidx->n; ++l) // fill missing values+        if (lidx->offset[l] == (uint64_t)-1)+            lidx->offset[l] = lidx->offset[l-1];+    if (bidx == 0) return;+    for (k = kh_begin(bidx); k != kh_end(bidx); ++k) // set loff+        if (kh_exist(bidx, k))+        {+            if ( kh_key(bidx, k) < idx->n_bins )+            {+                int bot_bin = hts_bin_bot(kh_key(bidx, k), idx->n_lvls);+                // disable linear index if bot_bin out of bounds+                kh_val(bidx, k).loff = bot_bin < lidx->n ? lidx->offset[bot_bin] : 0;+            }+            else+                kh_val(bidx, k).loff = 0;+        }+    if (free_lidx) {+        free(lidx->offset);+        lidx->m = lidx->n = 0;+        lidx->offset = 0;+    }+}++static void compress_binning(hts_idx_t *idx, int i)+{+    bidx_t *bidx = idx->bidx[i];+    khint_t k;+    int l, m;+    if (bidx == 0) return;+    // merge a bin to its parent if the bin is too small+    for (l = idx->n_lvls; l > 0; --l) {+        unsigned start = hts_bin_first(l);+        for (k = kh_begin(bidx); k != kh_end(bidx); ++k) {+            bins_t *p, *q;+            if (!kh_exist(bidx, k) || kh_key(bidx, k) >= idx->n_bins || kh_key(bidx, k) < start) continue;+            p = &kh_value(bidx, k);+            if (l < idx->n_lvls && p->n > 1) ks_introsort(_off, p->n, p->list);+            if ((p->list[p->n - 1].v>>16) - (p->list[0].u>>16) < HTS_MIN_MARKER_DIST) {+                khint_t kp;+                kp = kh_get(bin, bidx, hts_bin_parent(kh_key(bidx, k)));+                if (kp == kh_end(bidx)) continue;+                q = &kh_val(bidx, kp);+                if (q->n + p->n > q->m) {+                    q->m = q->n + p->n;+                    kroundup32(q->m);+                    q->list = (hts_pair64_t*)realloc(q->list, q->m * sizeof(hts_pair64_t));+                }+                memcpy(q->list + q->n, p->list, p->n * sizeof(hts_pair64_t));+                q->n += p->n;+                free(p->list);+                kh_del(bin, bidx, k);+            }+        }+    }+    k = kh_get(bin, bidx, 0);+    if (k != kh_end(bidx)) ks_introsort(_off, kh_val(bidx, k).n, kh_val(bidx, k).list);+    // merge adjacent chunks that start from the same BGZF block+    for (k = kh_begin(bidx); k != kh_end(bidx); ++k) {+        bins_t *p;+        if (!kh_exist(bidx, k) || kh_key(bidx, k) >= idx->n_bins) continue;+        p = &kh_value(bidx, k);+        for (l = 1, m = 0; l < p->n; ++l) {+            if (p->list[m].v>>16 >= p->list[l].u>>16) {+                if (p->list[m].v < p->list[l].v) p->list[m].v = p->list[l].v;+            } else p->list[++m] = p->list[l];+        }+        p->n = m + 1;+    }+}++void hts_idx_finish(hts_idx_t *idx, uint64_t final_offset)+{+    int i;+    if (idx == NULL || idx->z.finished) return; // do not run this function on an empty index or multiple times+    if (idx->z.save_tid >= 0) {+        insert_to_b(idx->bidx[idx->z.save_tid], idx->z.save_bin, idx->z.save_off, final_offset);+        insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx), idx->z.off_beg, final_offset);+        insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx), idx->z.n_mapped, idx->z.n_unmapped);+    }+    for (i = 0; i < idx->n; ++i) {+        update_loff(idx, i, (idx->fmt == HTS_FMT_CSI));+        compress_binning(idx, i);+    }+    idx->z.finished = 1;+}++int hts_idx_push(hts_idx_t *idx, int tid, int beg, int end, uint64_t offset, int is_mapped)+{+    int bin;+    if (tid<0) beg = -1, end = 0;+    if (tid >= idx->m) { // enlarge the index+        int32_t oldm = idx->m;+        idx->m = idx->m? idx->m<<1 : 2;+        idx->bidx = (bidx_t**)realloc(idx->bidx, idx->m * sizeof(bidx_t*));+        idx->lidx = (lidx_t*) realloc(idx->lidx, idx->m * sizeof(lidx_t));+        memset(&idx->bidx[oldm], 0, (idx->m - oldm) * sizeof(bidx_t*));+        memset(&idx->lidx[oldm], 0, (idx->m - oldm) * sizeof(lidx_t));+    }+    if (idx->n < tid + 1) idx->n = tid + 1;+    if (idx->z.finished) return 0;+    if (idx->z.last_tid != tid || (idx->z.last_tid >= 0 && tid < 0)) { // change of chromosome+        if ( tid>=0 && idx->n_no_coor )+        {+            if (hts_verbose >= 1) fprintf(stderr,"[E::%s] NO_COOR reads not in a single block at the end %d %d\n", __func__, tid,idx->z.last_tid);+            return -1;+        }+        if (tid>=0 && idx->bidx[tid] != 0)+        {+            if (hts_verbose >= 1) fprintf(stderr, "[E::%s] chromosome blocks not continuous\n", __func__);+            return -1;+        }+        idx->z.last_tid = tid;+        idx->z.last_bin = 0xffffffffu;+    } else if (tid >= 0 && idx->z.last_coor > beg) { // test if positions are out of order+        if (hts_verbose >= 1) fprintf(stderr, "[E::%s] unsorted positions\n", __func__);+        return -1;+    }+    if ( tid>=0 )+    {+        if (idx->bidx[tid] == 0) idx->bidx[tid] = kh_init(bin);+        if ( is_mapped)+            insert_to_l(&idx->lidx[tid], beg, end, idx->z.last_off, idx->min_shift); // last_off points to the start of the current record+    }+    else idx->n_no_coor++;+    bin = hts_reg2bin(beg, end, idx->min_shift, idx->n_lvls);+    if ((int)idx->z.last_bin != bin) { // then possibly write the binning index+        if (idx->z.save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record+            insert_to_b(idx->bidx[idx->z.save_tid], idx->z.save_bin, idx->z.save_off, idx->z.last_off);+        if (idx->z.last_bin == 0xffffffffu && idx->z.save_bin != 0xffffffffu) { // change of chr; keep meta information+            idx->z.off_end = idx->z.last_off;+            insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx), idx->z.off_beg, idx->z.off_end);+            insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx), idx->z.n_mapped, idx->z.n_unmapped);+            idx->z.n_mapped = idx->z.n_unmapped = 0;+            idx->z.off_beg = idx->z.off_end;+        }+        idx->z.save_off = idx->z.last_off;+        idx->z.save_bin = idx->z.last_bin = bin;+        idx->z.save_tid = tid;+    }+    if (is_mapped) ++idx->z.n_mapped;+    else ++idx->z.n_unmapped;+    idx->z.last_off = offset;+    idx->z.last_coor = beg;+    return 0;+}++void hts_idx_destroy(hts_idx_t *idx)+{+    khint_t k;+    int i;+    if (idx == 0) return;++    // For HTS_FMT_CRAI, idx actually points to a different type -- see sam.c+    if (idx->fmt == HTS_FMT_CRAI) {+        hts_cram_idx_t *cidx = (hts_cram_idx_t *) idx;+        cram_index_free(cidx->cram);+        free(cidx);+        return;+    }++    for (i = 0; i < idx->m; ++i) {+        bidx_t *bidx = idx->bidx[i];+        free(idx->lidx[i].offset);+        if (bidx == 0) continue;+        for (k = kh_begin(bidx); k != kh_end(bidx); ++k)+            if (kh_exist(bidx, k))+                free(kh_value(bidx, k).list);+        kh_destroy(bin, bidx);+    }+    free(idx->bidx); free(idx->lidx); free(idx->meta);+    free(idx);+}++// The optimizer eliminates these ed_is_big() calls; still it would be good to+// TODO Determine endianness at configure- or compile-time++static inline ssize_t HTS_RESULT_USED idx_write_int32(BGZF *fp, int32_t x)+{+    if (ed_is_big()) x = ed_swap_4(x);+    return bgzf_write(fp, &x, sizeof x);+}++static inline ssize_t HTS_RESULT_USED idx_write_uint32(BGZF *fp, uint32_t x)+{+    if (ed_is_big()) x = ed_swap_4(x);+    return bgzf_write(fp, &x, sizeof x);+}++static inline ssize_t HTS_RESULT_USED idx_write_uint64(BGZF *fp, uint64_t x)+{+    if (ed_is_big()) x = ed_swap_8(x);+    return bgzf_write(fp, &x, sizeof x);+}++static inline void swap_bins(bins_t *p)+{+    int i;+    for (i = 0; i < p->n; ++i) {+        ed_swap_8p(&p->list[i].u);+        ed_swap_8p(&p->list[i].v);+    }+}++static int hts_idx_save_core(const hts_idx_t *idx, BGZF *fp, int fmt)+{+    int32_t i, j;++    #define check(ret) if ((ret) < 0) return -1++    check(idx_write_int32(fp, idx->n));+    if (fmt == HTS_FMT_TBI && idx->l_meta)+        check(bgzf_write(fp, idx->meta, idx->l_meta));++    for (i = 0; i < idx->n; ++i) {+        khint_t k;+        bidx_t *bidx = idx->bidx[i];+        lidx_t *lidx = &idx->lidx[i];+        // write binning index+        check(idx_write_int32(fp, bidx? kh_size(bidx) : 0));+        if (bidx)+            for (k = kh_begin(bidx); k != kh_end(bidx); ++k)+                if (kh_exist(bidx, k)) {+                    bins_t *p = &kh_value(bidx, k);+                    check(idx_write_uint32(fp, kh_key(bidx, k)));+                    if (fmt == HTS_FMT_CSI) check(idx_write_uint64(fp, p->loff));+                    //int j;for(j=0;j<p->n;++j)fprintf(stderr,"%d,%llx,%d,%llx:%llx\n",kh_key(bidx,k),kh_val(bidx, k).loff,j,p->list[j].u,p->list[j].v);+                    check(idx_write_int32(fp, p->n));+                    for (j = 0; j < p->n; ++j) {+                        check(idx_write_uint64(fp, p->list[j].u));+                        check(idx_write_uint64(fp, p->list[j].v));+                    }+                }++        // write linear index+        if (fmt != HTS_FMT_CSI) {+            check(idx_write_int32(fp, lidx->n));+            for (j = 0; j < lidx->n; ++j)+                check(idx_write_uint64(fp, lidx->offset[j]));+        }+    }++    check(idx_write_uint64(fp, idx->n_no_coor));+    return 0;+    #undef check+}++int hts_idx_save(const hts_idx_t *idx, const char *fn, int fmt)+{+    int ret, save;+    char *fnidx = (char*)calloc(1, strlen(fn) + 5);+    if (fnidx == NULL) return -1;++    strcpy(fnidx, fn);+    switch (fmt) {+    case HTS_FMT_BAI: strcat(fnidx, ".bai"); break;+    case HTS_FMT_CSI: strcat(fnidx, ".csi"); break;+    case HTS_FMT_TBI: strcat(fnidx, ".tbi"); break;+    default: abort();+    }++    ret = hts_idx_save_as(idx, fn, fnidx, fmt);+    save = errno;+    free(fnidx);+    errno = save;+    return ret;+}++int hts_idx_save_as(const hts_idx_t *idx, const char *fn, const char *fnidx, int fmt)+{+    BGZF *fp;++    #define check(ret) if ((ret) < 0) goto fail++    if (fnidx == NULL) return hts_idx_save(idx, fn, fmt);++    fp = bgzf_open(fnidx, (fmt == HTS_FMT_BAI)? "wu" : "w");+    if (fp == NULL) return -1;++    if (fmt == HTS_FMT_CSI) {+        check(bgzf_write(fp, "CSI\1", 4));+        check(idx_write_int32(fp, idx->min_shift));+        check(idx_write_int32(fp, idx->n_lvls));+        check(idx_write_uint32(fp, idx->l_meta));+        if (idx->l_meta) check(bgzf_write(fp, idx->meta, idx->l_meta));+    } else if (fmt == HTS_FMT_TBI) {+        check(bgzf_write(fp, "TBI\1", 4));+    } else if (fmt == HTS_FMT_BAI) {+        check(bgzf_write(fp, "BAI\1", 4));+    } else abort();++    check(hts_idx_save_core(idx, fp, fmt));++    return bgzf_close(fp);+    #undef check++fail:+    bgzf_close(fp);+    return -1;+}++static int hts_idx_load_core(hts_idx_t *idx, BGZF *fp, int fmt)+{+    int32_t i, n, is_be;+    is_be = ed_is_big();+    if (idx == NULL) return -4;+    for (i = 0; i < idx->n; ++i) {+        bidx_t *h;+        lidx_t *l = &idx->lidx[i];+        uint32_t key;+        int j, absent;+        bins_t *p;+        h = idx->bidx[i] = kh_init(bin);+        if (bgzf_read(fp, &n, 4) != 4) return -1;+        if (is_be) ed_swap_4p(&n);+        for (j = 0; j < n; ++j) {+            khint_t k;+            if (bgzf_read(fp, &key, 4) != 4) return -1;+            if (is_be) ed_swap_4p(&key);+            k = kh_put(bin, h, key, &absent);+            if (absent <= 0) return -3; // Duplicate bin number+            p = &kh_val(h, k);+            if (fmt == HTS_FMT_CSI) {+                if (bgzf_read(fp, &p->loff, 8) != 8) return -1;+                if (is_be) ed_swap_8p(&p->loff);+            } else p->loff = 0;+            if (bgzf_read(fp, &p->n, 4) != 4) return -1;+            if (is_be) ed_swap_4p(&p->n);+            p->m = p->n;+            p->list = (hts_pair64_t*)malloc(p->m * sizeof(hts_pair64_t));+            if (p->list == NULL) return -2;+            if (bgzf_read(fp, p->list, p->n<<4) != p->n<<4) return -1;+            if (is_be) swap_bins(p);+        }+        if (fmt != HTS_FMT_CSI) { // load linear index+            int j;+            if (bgzf_read(fp, &l->n, 4) != 4) return -1;+            if (is_be) ed_swap_4p(&l->n);+            l->m = l->n;+            l->offset = (uint64_t*)malloc(l->n * sizeof(uint64_t));+            if (l->offset == NULL) return -2;+            if (bgzf_read(fp, l->offset, l->n << 3) != l->n << 3) return -1;+            if (is_be) for (j = 0; j < l->n; ++j) ed_swap_8p(&l->offset[j]);+            for (j = 1; j < l->n; ++j) // fill missing values; may happen given older samtools and tabix+                if (l->offset[j] == 0) l->offset[j] = l->offset[j-1];+            update_loff(idx, i, 1);+        }+    }+    if (bgzf_read(fp, &idx->n_no_coor, 8) != 8) idx->n_no_coor = 0;+    if (is_be) ed_swap_8p(&idx->n_no_coor);+    return 0;+}++static hts_idx_t *hts_idx_load_local(const char *fn)+{+    uint8_t magic[4];+    int i, is_be;+    hts_idx_t *idx = NULL;+    uint8_t *meta = NULL;+    BGZF *fp = bgzf_open(fn, "r");+    if (fp == NULL) return NULL;+    is_be = ed_is_big();+    if (bgzf_read(fp, magic, 4) != 4) goto fail;++    if (memcmp(magic, "CSI\1", 4) == 0) {+        uint32_t x[3], n;+        if (bgzf_read(fp, x, 12) != 12) goto fail;+        if (is_be) for (i = 0; i < 3; ++i) ed_swap_4p(&x[i]);+        if (x[2]) {+            if ((meta = (uint8_t*)malloc(x[2])) == NULL) goto fail;+            if (bgzf_read(fp, meta, x[2]) != x[2]) goto fail;+        }+        if (bgzf_read(fp, &n, 4) != 4) goto fail;+        if (is_be) ed_swap_4p(&n);+        if ((idx = hts_idx_init(n, HTS_FMT_CSI, 0, x[0], x[1])) == NULL) goto fail;+        idx->l_meta = x[2];+        idx->meta = meta;+        meta = NULL;+        if (hts_idx_load_core(idx, fp, HTS_FMT_CSI) < 0) goto fail;+    }+    else if (memcmp(magic, "TBI\1", 4) == 0) {+        uint32_t x[8];+        if (bgzf_read(fp, x, 32) != 32) goto fail;+        if (is_be) for (i = 0; i < 8; ++i) ed_swap_4p(&x[i]);+        if ((idx = hts_idx_init(x[0], HTS_FMT_TBI, 0, 14, 5)) == NULL) goto fail;+        idx->l_meta = 28 + x[7];+        if ((idx->meta = (uint8_t*)malloc(idx->l_meta)) == NULL) goto fail;+        memcpy(idx->meta, &x[1], 28);+        if (bgzf_read(fp, idx->meta + 28, x[7]) != x[7]) goto fail;+        if (hts_idx_load_core(idx, fp, HTS_FMT_TBI) < 0) goto fail;+    }+    else if (memcmp(magic, "BAI\1", 4) == 0) {+        uint32_t n;+        if (bgzf_read(fp, &n, 4) != 4) goto fail;+        if (is_be) ed_swap_4p(&n);+        idx = hts_idx_init(n, HTS_FMT_BAI, 0, 14, 5);+        if (hts_idx_load_core(idx, fp, HTS_FMT_BAI) < 0) goto fail;+    }+    else { errno = EINVAL; goto fail; }++    bgzf_close(fp);+    return idx;++fail:+    bgzf_close(fp);+    hts_idx_destroy(idx);+    free(meta);+    return NULL;+}++void hts_idx_set_meta(hts_idx_t *idx, int l_meta, uint8_t *meta, int is_copy)+{+    if (idx->meta) free(idx->meta);+    idx->l_meta = l_meta;+    if (is_copy) {+        idx->meta = (uint8_t*)malloc(l_meta);+        memcpy(idx->meta, meta, l_meta);+    } else idx->meta = meta;+}++uint8_t *hts_idx_get_meta(hts_idx_t *idx, int *l_meta)+{+    *l_meta = idx->l_meta;+    return idx->meta;+}++const char **hts_idx_seqnames(const hts_idx_t *idx, int *n, hts_id2name_f getid, void *hdr)+{+    if ( !idx->n )+    {+        *n = 0;+        return NULL;+    }++    int tid = 0, i;+    const char **names = (const char**) calloc(idx->n,sizeof(const char*));+    for (i=0; i<idx->n; i++)+    {+        bidx_t *bidx = idx->bidx[i];+        if ( !bidx ) continue;+        names[tid++] = getid(hdr,i);+    }+    *n = tid;+    return names;+}++int hts_idx_get_stat(const hts_idx_t* idx, int tid, uint64_t* mapped, uint64_t* unmapped)+{+    if ( idx->fmt == HTS_FMT_CRAI ) {+        *mapped = 0; *unmapped = 0;+        return -1;+    }++    bidx_t *h = idx->bidx[tid];+    khint_t k = kh_get(bin, h, META_BIN(idx));+    if (k != kh_end(h)) {+        *mapped = kh_val(h, k).list[1].u;+        *unmapped = kh_val(h, k).list[1].v;+        return 0;+    } else {+        *mapped = 0; *unmapped = 0;+        return -1;+    }+}++uint64_t hts_idx_get_n_no_coor(const hts_idx_t* idx)+{+    return idx->n_no_coor;+}++/****************+ *** Iterator ***+ ****************/++static inline int reg2bins(int64_t beg, int64_t end, hts_itr_t *itr, int min_shift, int n_lvls)+{+    int l, t, s = min_shift + (n_lvls<<1) + n_lvls;+    if (beg >= end) return 0;+    if (end >= 1LL<<s) end = 1LL<<s;+    for (--end, l = 0, t = 0; l <= n_lvls; s -= 3, t += 1<<((l<<1)+l), ++l) {+        int b, e, n, i;+        b = t + (beg>>s); e = t + (end>>s); n = e - b + 1;+        if (itr->bins.n + n > itr->bins.m) {+            itr->bins.m = itr->bins.n + n;+            kroundup32(itr->bins.m);+            itr->bins.a = (int*)realloc(itr->bins.a, sizeof(int) * itr->bins.m);+        }+        for (i = b; i <= e; ++i) itr->bins.a[itr->bins.n++] = i;+    }+    return itr->bins.n;+}++hts_itr_t *hts_itr_query(const hts_idx_t *idx, int tid, int beg, int end, hts_readrec_func *readrec)+{+    int i, n_off, l, bin;+    hts_pair64_t *off;+    khint_t k;+    bidx_t *bidx;+    uint64_t min_off;+    hts_itr_t *iter = 0;+    if (tid < 0) {+        int finished0 = 0;+        uint64_t off0 = (uint64_t)-1;+        khint_t k;+        switch (tid) {+        case HTS_IDX_START:+            // Find the smallest offset, note that sequence ids may not be ordered sequentially+            for (i=0; i<idx->n; i++)+            {+                bidx = idx->bidx[i];+                k = kh_get(bin, bidx, META_BIN(idx));+                if (k == kh_end(bidx)) continue;+                if ( off0 > kh_val(bidx, k).list[0].u ) off0 = kh_val(bidx, k).list[0].u;+            }+            if ( off0==(uint64_t)-1 && idx->n_no_coor ) off0 = 0; // only no-coor reads in this bam+            break;++        case HTS_IDX_NOCOOR:+            if ( idx->n>0 )+            {+                bidx = idx->bidx[idx->n - 1];+                k = kh_get(bin, bidx, META_BIN(idx));+                if (k != kh_end(bidx)) off0 = kh_val(bidx, k).list[0].v;+            }+            if ( off0==(uint64_t)-1 && idx->n_no_coor ) off0 = 0; // only no-coor reads in this bam+            break;++        case HTS_IDX_REST:+            off0 = 0;+            break;++        case HTS_IDX_NONE:+            finished0 = 1;+            off0 = 0;+            break;++        default:+            return 0;+        }+        if (off0 != (uint64_t)-1) {+            iter = (hts_itr_t*)calloc(1, sizeof(hts_itr_t));+            iter->read_rest = 1;+            iter->finished = finished0;+            iter->curr_off = off0;+            iter->readrec = readrec;+            return iter;+        } else return 0;+    }++    if (beg < 0) beg = 0;+    if (end < beg) return 0;+    if (tid >= idx->n || (bidx = idx->bidx[tid]) == NULL) return 0;++    iter = (hts_itr_t*)calloc(1, sizeof(hts_itr_t));+    iter->tid = tid, iter->beg = beg, iter->end = end; iter->i = -1;+    iter->readrec = readrec;++    // compute min_off+    bin = hts_bin_first(idx->n_lvls) + (beg>>idx->min_shift);+    do {+        int first;+        k = kh_get(bin, bidx, bin);+        if (k != kh_end(bidx)) break;+        first = (hts_bin_parent(bin)<<3) + 1;+        if (bin > first) --bin;+        else bin = hts_bin_parent(bin);+    } while (bin);+    if (bin == 0) k = kh_get(bin, bidx, bin);+    min_off = k != kh_end(bidx)? kh_val(bidx, k).loff : 0;+    // retrieve bins+    reg2bins(beg, end, iter, idx->min_shift, idx->n_lvls);+    for (i = n_off = 0; i < iter->bins.n; ++i)+        if ((k = kh_get(bin, bidx, iter->bins.a[i])) != kh_end(bidx))+            n_off += kh_value(bidx, k).n;+    if (n_off == 0) return iter;+    off = (hts_pair64_t*)calloc(n_off, sizeof(hts_pair64_t));+    for (i = n_off = 0; i < iter->bins.n; ++i) {+        if ((k = kh_get(bin, bidx, iter->bins.a[i])) != kh_end(bidx)) {+            int j;+            bins_t *p = &kh_value(bidx, k);+            for (j = 0; j < p->n; ++j)+                if (p->list[j].v > min_off) off[n_off++] = p->list[j];+        }+    }+    if (n_off == 0) {+        free(off); return iter;+    }+    ks_introsort(_off, n_off, off);+    // resolve completely contained adjacent blocks+    for (i = 1, l = 0; i < n_off; ++i)+        if (off[l].v < off[i].v) off[++l] = off[i];+    n_off = l + 1;+    // resolve overlaps between adjacent blocks; this may happen due to the merge in indexing+    for (i = 1; i < n_off; ++i)+        if (off[i-1].v >= off[i].u) off[i-1].v = off[i].u;+    // merge adjacent blocks+    for (i = 1, l = 0; i < n_off; ++i) {+        if (off[l].v>>16 == off[i].u>>16) off[l].v = off[i].v;+        else off[++l] = off[i];+    }+    n_off = l + 1;+    iter->n_off = n_off; iter->off = off;+    return iter;+}++void hts_itr_destroy(hts_itr_t *iter)+{+    if (iter) { free(iter->off); free(iter->bins.a); free(iter); }+}++static inline long long push_digit(long long i, char c)+{+    // ensure subtraction occurs first, avoiding overflow for >= MAX-48 or so+    int digit = c - '0';+    return 10 * i + digit;+}++long long hts_parse_decimal(const char *str, char **strend, int flags)+{+    long long n = 0;+    int decimals = 0, e = 0, lost = 0;+    char sign = '+', esign = '+';+    const char *s;++    while (isspace_c(*str)) str++;+    s = str;++    if (*s == '+' || *s == '-') sign = *s++;+    while (*s)+        if (isdigit_c(*s)) n = push_digit(n, *s++);+        else if (*s == ',' && (flags & HTS_PARSE_THOUSANDS_SEP)) s++;+        else break;++    if (*s == '.') {+        s++;+        while (isdigit_c(*s)) decimals++, n = push_digit(n, *s++);+    }++    if (*s == 'E' || *s == 'e') {+        s++;+        if (*s == '+' || *s == '-') esign = *s++;+        while (isdigit_c(*s)) e = push_digit(e, *s++);+        if (esign == '-') e = -e;+    }++    e -= decimals;+    while (e > 0) n *= 10, e--;+    while (e < 0) lost += n % 10, n /= 10, e++;++    if (lost > 0 && hts_verbose >= 3)+        fprintf(stderr, "[W::%s] discarding fractional part of %.*s\n",+                __func__, (int)(s - str), str);++    if (strend) *strend = (char *) s;+    else if (*s && hts_verbose >= 2)+        fprintf(stderr, "[W::%s] ignoring unknown characters after %.*s[%s]\n",+                __func__, (int)(s - str), str, s);++    return (sign == '+')? n : -n;+}++const char *hts_parse_reg(const char *s, int *beg, int *end)+{+    char *hyphen;+    const char *colon = strrchr(s, ':');+    if (colon == NULL) {+        *beg = 0; *end = INT_MAX;+        return s + strlen(s);+    }++    *beg = hts_parse_decimal(colon+1, &hyphen, HTS_PARSE_THOUSANDS_SEP) - 1;+    if (*beg < 0) *beg = 0;++    if (*hyphen == '\0') *end = INT_MAX;+    else if (*hyphen == '-') *end = hts_parse_decimal(hyphen+1, NULL, HTS_PARSE_THOUSANDS_SEP);+    else return NULL;++    if (*beg >= *end) return NULL;+    return colon;+}++hts_itr_t *hts_itr_querys(const hts_idx_t *idx, const char *reg, hts_name2id_f getid, void *hdr, hts_itr_query_func *itr_query, hts_readrec_func *readrec)+{+    int tid, beg, end;+    const char *q;++    if (strcmp(reg, ".") == 0)+        return itr_query(idx, HTS_IDX_START, 0, 0, readrec);+    else if (strcmp(reg, "*") == 0)+        return itr_query(idx, HTS_IDX_NOCOOR, 0, 0, readrec);++    q = hts_parse_reg(reg, &beg, &end);+    if (q) {+        char *tmp = (char*)alloca(q - reg + 1);+        strncpy(tmp, reg, q - reg);+        tmp[q - reg] = 0;+        tid = getid(hdr, tmp);+    }+    else {+        // not parsable as a region, but possibly a sequence named "foo:a"+        tid = getid(hdr, reg);+        beg = 0; end = INT_MAX;+    }++    if (tid < 0) return NULL;+    return itr_query(idx, tid, beg, end, readrec);+}++int hts_itr_next(BGZF *fp, hts_itr_t *iter, void *r, void *data)+{+    int ret, tid, beg, end;+    if (iter == NULL || iter->finished) return -1;+    if (iter->read_rest) {+        if (iter->curr_off) { // seek to the start+            if (bgzf_seek(fp, iter->curr_off, SEEK_SET) < 0) return -1;+            iter->curr_off = 0; // only seek once+        }+        ret = iter->readrec(fp, data, r, &tid, &beg, &end);+        if (ret < 0) iter->finished = 1;+        iter->curr_tid = tid;+        iter->curr_beg = beg;+        iter->curr_end = end;+        return ret;+    }+    if (iter->off == 0) return -1;+    for (;;) {+        if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk+            if (iter->i == iter->n_off - 1) { ret = -1; break; } // no more chunks+            if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek+                if (bgzf_seek(fp, iter->off[iter->i+1].u, SEEK_SET) < 0) return -1;+                iter->curr_off = bgzf_tell(fp);+            }+            ++iter->i;+        }+        if ((ret = iter->readrec(fp, data, r, &tid, &beg, &end)) >= 0) {+            iter->curr_off = bgzf_tell(fp);+            if (tid != iter->tid || beg >= iter->end) { // no need to proceed+                ret = -1; break;+            } else if (end > iter->beg && iter->end > beg) {+                iter->curr_tid = tid;+                iter->curr_beg = beg;+                iter->curr_end = end;+                return ret;+            }+        } else break; // end of file or error+    }+    iter->finished = 1;+    return ret;+}++/**********************+ *** Retrieve index ***+ **********************/++static char *test_and_fetch(const char *fn)+{+    FILE *fp;+    if (hisremote(fn)) {+        const int buf_size = 1 * 1024 * 1024;+        hFILE *fp_remote;+        uint8_t *buf;+        int l;+        const char *p;+        for (p = fn + strlen(fn) - 1; p >= fn; --p)+            if (*p == '/') break;+        ++p; // p now points to the local file name+        // Attempt to open local file first+        if ((fp = fopen((char*)p, "rb")) != 0)+        {+            fclose(fp);+            return (char*)p;+        }+        // Attempt to open remote file. Stay quiet on failure, it is OK to fail when trying first .csi then .tbi index.+        if ((fp_remote = hopen(fn, "r")) == 0) return 0;+        if ((fp = fopen(p, "w")) == 0) {+            if (hts_verbose >= 1) fprintf(stderr, "[E::%s] fail to create file '%s' in the working directory\n", __func__, p);+            hclose_abruptly(fp_remote);+            return 0;+        }+        if (hts_verbose >= 3) fprintf(stderr, "[M::%s] downloading file '%s' to local directory\n", __func__, fn);+        buf = (uint8_t*)calloc(buf_size, 1);+        while ((l = hread(fp_remote, buf, buf_size)) > 0) fwrite(buf, 1, l, fp);+        free(buf);+        fclose(fp);+        if (hclose(fp_remote) != 0) fprintf(stderr, "[E::%s] fail to close remote file '%s'\n", __func__, fn);+        return (char*)p;+    } else {+        if ((fp = fopen(fn, "rb")) == 0) return 0;+        fclose(fp);+        return (char*)fn;+    }+}++char *hts_idx_getfn(const char *fn, const char *ext)+{+    int i, l_fn, l_ext;+    char *fnidx, *ret;+    l_fn = strlen(fn); l_ext = strlen(ext);+    fnidx = (char*)calloc(l_fn + l_ext + 1, 1);+    strcpy(fnidx, fn); strcpy(fnidx + l_fn, ext);+    if ((ret = test_and_fetch(fnidx)) == 0) {+        for (i = l_fn - 1; i > 0; --i)+            if (fnidx[i] == '.') break;+        strcpy(fnidx + i, ext);+        ret = test_and_fetch(fnidx);+    }+    if (ret == 0) {+        free(fnidx);+        return 0;+    }+    l_fn = strlen(ret);+    memmove(fnidx, ret, l_fn + 1);+    return fnidx;+}++hts_idx_t *hts_idx_load(const char *fn, int fmt)+{+    char *fnidx;+    hts_idx_t *idx;+    fnidx = hts_idx_getfn(fn, ".csi");+    if (! fnidx) fnidx = hts_idx_getfn(fn, fmt == HTS_FMT_BAI? ".bai" : ".tbi");+    if (fnidx == 0) return 0;++    idx = hts_idx_load2(fn, fnidx);+    free(fnidx);+    return idx;+}++hts_idx_t *hts_idx_load2(const char *fn, const char *fnidx)+{+    // Check that the index file is up to date, the main file might have changed+    struct stat stat_idx,stat_main;+    if ( !stat(fn, &stat_main) && !stat(fnidx, &stat_idx) )+    {+        if ( stat_idx.st_mtime < stat_main.st_mtime )+            fprintf(stderr, "Warning: The index file is older than the data file: %s\n", fnidx);+    }++    return hts_idx_load_local(fnidx);+}
+ htslib-1.3.2/hts_internal.h view
@@ -0,0 +1,90 @@+/*  hts_internal.h -- internal functions; not part of the public API.++    Copyright (C) 2015-2016 Genome Research Ltd.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_HTS_INTERNAL_H+#define HTSLIB_HTS_INTERNAL_H++#include <stddef.h>+#include <ctype.h>++#include "htslib/hts.h"++#ifdef __cplusplus+extern "C" {+#endif++// The <ctype.h> functions operate on ints such as are returned by fgetc(),+// i.e., characters represented as unsigned-char-valued ints, or EOF.+// To operate on plain chars (and to avoid warnings on some platforms),+// technically one must cast to unsigned char everywhere (see CERT STR37-C)+// or less painfully use these *_c() functions that operate on plain chars+// (but not EOF, which must be considered separately where it is applicable).+// TODO We may eventually wish to implement these functions directly without+// using their <ctype.h> equivalents, and thus make them immune to locales.+static inline int isalnum_c(char c) { return isalnum((unsigned char) c); }+static inline int isalpha_c(char c) { return isalpha((unsigned char) c); }+static inline int isdigit_c(char c) { return isdigit((unsigned char) c); }+static inline int isgraph_c(char c) { return isgraph((unsigned char) c); }+static inline int islower_c(char c) { return islower((unsigned char) c); }+static inline int isprint_c(char c) { return isprint((unsigned char) c); }+static inline int isspace_c(char c) { return isspace((unsigned char) c); }+static inline int isupper_c(char c) { return isupper((unsigned char) c); }+static inline char tolower_c(char c) { return tolower((unsigned char) c); }+static inline char toupper_c(char c) { return toupper((unsigned char) c); }+++struct cram_fd;++char *hts_idx_getfn(const char *fn, const char *ext);++// The CRAM implementation stores the loaded index within the cram_fd rather+// than separately as is done elsewhere in htslib.  So if p is a pointer to+// an hts_idx_t with p->fmt == HTS_FMT_CRAI, then it actually points to an+// hts_cram_idx_t and should be cast accordingly.+typedef struct hts_cram_idx_t {+    int fmt;+    struct cram_fd *cram;+} hts_cram_idx_t;+++struct hts_path_itr {+    kstring_t path, entry;+    void *dirv;  // DIR * privately+    const char *pathdir, *prefix, *suffix;+    size_t prefix_len, suffix_len, entry_dir_l;+};++void hts_path_itr_setup(struct hts_path_itr *itr, const char *path,+    const char *builtin_path, const char *prefix, size_t prefix_len,+    const char *suffix, size_t suffix_len);++const char *hts_path_itr_next(struct hts_path_itr *itr);++void *load_plugin(void **pluginp, const char *filename, const char *symbol);+void *plugin_sym(void *plugin, const char *name, const char **errmsg);+void close_plugin(void *plugin);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/bgzf.h view
@@ -0,0 +1,335 @@+/* The MIT License++   Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology+                 2011, 2012 Attractive Chaos <attractor@live.co.uk>+   Copyright (C) 2009, 2013, 2014 Genome Research Ltd++   Permission is hereby granted, free of charge, to any person obtaining a copy+   of this software and associated documentation files (the "Software"), to deal+   in the Software without restriction, including without limitation the rights+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+   copies of the Software, and to permit persons to whom the Software is+   furnished to do so, subject to the following conditions:++   The above copyright notice and this permission notice shall be included in+   all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+   THE SOFTWARE.+*/++/* The BGZF library was originally written by Bob Handsaker from the Broad+ * Institute. It was later improved by the SAMtools developers. */++#ifndef HTSLIB_BGZF_H+#define HTSLIB_BGZF_H++#include <stdint.h>+#include <stdio.h>+#include <zlib.h>+#include <sys/types.h>++#include "hts_defs.h"++#ifdef __cplusplus+extern "C" {+#endif++#define BGZF_BLOCK_SIZE     0xff00 // make sure compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE+#define BGZF_MAX_BLOCK_SIZE 0x10000++#define BGZF_ERR_ZLIB   1+#define BGZF_ERR_HEADER 2+#define BGZF_ERR_IO     4+#define BGZF_ERR_MISUSE 8++struct hFILE;+struct bgzf_mtaux_t;+typedef struct __bgzidx_t bgzidx_t;++struct BGZF {+    unsigned errcode:16, is_write:2, is_be:2;+    signed compress_level:9;+    unsigned is_compressed:2, is_gzip:1;+    int cache_size;+    int block_length, block_offset;+    int64_t block_address, uncompressed_address;+    void *uncompressed_block, *compressed_block;+    void *cache; // a pointer to a hash table+    struct hFILE *fp; // actual file handle+    struct bgzf_mtaux_t *mt; // only used for multi-threading+    bgzidx_t *idx;      // BGZF index+    int idx_build_otf;  // build index on the fly, set by bgzf_index_build_init()+    z_stream *gz_stream;// for gzip-compressed files+};+#ifndef HTS_BGZF_TYPEDEF+typedef struct BGZF BGZF;+#define HTS_BGZF_TYPEDEF+#endif++#ifndef KSTRING_T+#define KSTRING_T kstring_t+typedef struct __kstring_t {+    size_t l, m;+    char *s;+} kstring_t;+#endif++    /******************+     * Basic routines *+     ******************/++    /**+     * Open an existing file descriptor for reading or writing.+     *+     * @param fd    file descriptor+     * @param mode  mode matching /[rwag][u0-9]+/: 'r' for reading, 'w' for+     *              writing, 'a' for appending, 'g' for gzip rather than BGZF+     *              compression (with 'w' only), and digit specifies the zlib+     *              compression level.+     *              Note that there is a distinction between 'u' and '0': the+     *              first yields plain uncompressed output whereas the latter+     *              outputs uncompressed data wrapped in the zlib format.+     * @return      BGZF file handler; 0 on error+     */+    BGZF* bgzf_dopen(int fd, const char *mode);++    #define bgzf_fdopen(fd, mode) bgzf_dopen((fd), (mode)) // for backward compatibility++    /**+     * Open the specified file for reading or writing.+     */+    BGZF* bgzf_open(const char* path, const char *mode);++    /**+     * Open an existing hFILE stream for reading or writing.+     */+    BGZF* bgzf_hopen(struct hFILE *fp, const char *mode);++    /**+     * Close the BGZF and free all associated resources.+     *+     * @param fp    BGZF file handler+     * @return      0 on success and -1 on error+     */+    int bgzf_close(BGZF *fp);++    /**+     * Read up to _length_ bytes from the file storing into _data_.+     *+     * @param fp     BGZF file handler+     * @param data   data array to read into+     * @param length size of data to read+     * @return       number of bytes actually read; 0 on end-of-file and -1 on error+     */+    ssize_t bgzf_read(BGZF *fp, void *data, size_t length) HTS_RESULT_USED;++    /**+     * Write _length_ bytes from _data_ to the file.  If no I/O errors occur,+     * the complete _length_ bytes will be written (or queued for writing).+     *+     * @param fp     BGZF file handler+     * @param data   data array to write+     * @param length size of data to write+     * @return       number of bytes written (i.e., _length_); negative on error+     */+    ssize_t bgzf_write(BGZF *fp, const void *data, size_t length) HTS_RESULT_USED;++    /**+     * Read up to _length_ bytes directly from the underlying stream without+     * decompressing.  Bypasses BGZF blocking, so must be used with care in+     * specialised circumstances only.+     *+     * @param fp     BGZF file handler+     * @param data   data array to read into+     * @param length number of raw bytes to read+     * @return       number of bytes actually read; 0 on end-of-file and -1 on error+     */+    ssize_t bgzf_raw_read(BGZF *fp, void *data, size_t length) HTS_RESULT_USED;++    /**+     * Write _length_ bytes directly to the underlying stream without+     * compressing.  Bypasses BGZF blocking, so must be used with care+     * in specialised circumstances only.+     *+     * @param fp     BGZF file handler+     * @param data   data array to write+     * @param length number of raw bytes to write+     * @return       number of bytes actually written; -1 on error+     */+    ssize_t bgzf_raw_write(BGZF *fp, const void *data, size_t length) HTS_RESULT_USED;++    /**+     * Write the data in the buffer to the file.+     *+     * @param fp     BGZF file handle+     * @return       0 on success and -1 on error+     */+    int bgzf_flush(BGZF *fp) HTS_RESULT_USED;++    /**+     * Return a virtual file pointer to the current location in the file.+     * No interpetation of the value should be made, other than a subsequent+     * call to bgzf_seek can be used to position the file at the same point.+     * Return value is non-negative on success.+     */+    #define bgzf_tell(fp) (((fp)->block_address << 16) | ((fp)->block_offset & 0xFFFF))++    /**+     * Set the file to read from the location specified by _pos_.+     *+     * @param fp     BGZF file handler+     * @param pos    virtual file offset returned by bgzf_tell()+     * @param whence must be SEEK_SET+     * @return       0 on success and -1 on error+     */+    int64_t bgzf_seek(BGZF *fp, int64_t pos, int whence) HTS_RESULT_USED;++    /**+     * Check if the BGZF end-of-file (EOF) marker is present+     *+     * @param fp    BGZF file handler opened for reading+     * @return      1 if the EOF marker is present and correct;+     *              2 if it can't be checked, e.g., because fp isn't seekable;+     *              0 if the EOF marker is absent;+     *              -1 (with errno set) on error+     */+    int bgzf_check_EOF(BGZF *fp);++    /**+     * Check if a file is in the BGZF format+     *+     * @param fn    file name+     * @return      1 if _fn_ is BGZF; 0 if not or on I/O error+     */+     int bgzf_is_bgzf(const char *fn);++    /*********************+     * Advanced routines *+     *********************/++    /**+     * Set the cache size. Only effective when compiled with -DBGZF_CACHE.+     *+     * @param fp    BGZF file handler+     * @param size  size of cache in bytes; 0 to disable caching (default)+     */+    void bgzf_set_cache_size(BGZF *fp, int size);++    /**+     * Flush the file if the remaining buffer size is smaller than _size_+     * @return      0 if flushing succeeded or was not needed; negative on error+     */+    int bgzf_flush_try(BGZF *fp, ssize_t size) HTS_RESULT_USED;++    /**+     * Read one byte from a BGZF file. It is faster than bgzf_read()+     * @param fp     BGZF file handler+     * @return       byte read; -1 on end-of-file or error+     */+    int bgzf_getc(BGZF *fp);++    /**+     * Read one line from a BGZF file. It is faster than bgzf_getc()+     *+     * @param fp     BGZF file handler+     * @param delim  delimitor+     * @param str    string to write to; must be initialized+     * @return       length of the string; 0 on end-of-file; negative on error+     */+    int bgzf_getline(BGZF *fp, int delim, kstring_t *str);++    /**+     * Read the next BGZF block.+     */+    int bgzf_read_block(BGZF *fp) HTS_RESULT_USED;++    /**+     * Enable multi-threading (only effective on writing and when the+     * library was compiled with -DBGZF_MT)+     *+     * @param fp          BGZF file handler; must be opened for writing+     * @param n_threads   #threads used for writing+     * @param n_sub_blks  #blocks processed by each thread; a value 64-256 is recommended+     */+    int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks);++    /**+     * Compress a single BGZF block.+     *+     * @param dst    output buffer (must have size >= BGZF_MAX_BLOCK_SIZE)+     * @param dlen   size of output buffer; updated on return to the number+     *               of bytes actually written to dst+     * @param src    buffer to be compressed+     * @param slen   size of data to compress (must be <= BGZF_BLOCK_SIZE)+     * @param level  compression level+     * @return       0 on success and negative on error+     */+    int bgzf_compress(void *dst, size_t *dlen, const void *src, size_t slen, int level);++    /*******************+     * bgzidx routines *+     *******************/++    /**+     *  Position BGZF at the uncompressed offset+     *+     *  @param fp           BGZF file handler; must be opened for reading+     *  @param uoffset      file offset in the uncompressed data+     *  @param where        SEEK_SET supported atm+     *+     *  Returns 0 on success and -1 on error.+     */+    int bgzf_useek(BGZF *fp, long uoffset, int where) HTS_RESULT_USED;++    /**+     *  Position in uncompressed BGZF+     *+     *  @param fp           BGZF file handler; must be opened for reading+     *+     *  Returns the current offset on success and -1 on error.+     */+    long bgzf_utell(BGZF *fp);++    /**+     * Tell BGZF to build index while compressing.+     *+     * @param fp          BGZF file handler; can be opened for reading or writing.+     *+     * Returns 0 on success and -1 on error.+     */+    int bgzf_index_build_init(BGZF *fp);++    /**+     * Load BGZF index+     *+     * @param fp          BGZF file handler+     * @param bname       base name+     * @param suffix      suffix to add to bname (can be NULL)+     *+     * Returns 0 on success and -1 on error.+     */+    int bgzf_index_load(BGZF *fp, const char *bname, const char *suffix);++    /**+     * Save BGZF index+     *+     * @param fp          BGZF file handler+     * @param bname       base name+     * @param suffix      suffix to add to bname (can be NULL)+     *+     * Returns 0 on success and -1 on error.+     */+    int bgzf_index_dump(BGZF *fp,+                        const char *bname, const char *suffix) HTS_RESULT_USED;++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/config.h view
+ htslib-1.3.2/htslib/cram.h view
@@ -0,0 +1,492 @@+/*  cram.h -- public CRAM-specific API functions.++    Copyright (C) 2015 Genome Research Ltd.++    Author: James Bonfield <jkb@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++/*! \file+ * CRAM interface.+ *+ * Consider using the higher level hts_*() API for programs that wish to+ * be file format agnostic (see htslib/hts.h).+ *+ * This API should be used for CRAM specific code. The specifics of the+ * public API are implemented in cram_io.h, cram_encode.h and cram_decode.h+ * although these should not be included directly (use this file instead).+ */++#ifndef HTSLIB_CRAM_H+#define HTSLIB_CRAM_H++#include <stdarg.h>+#include <stdint.h>+#include <sys/types.h>++#include "hts.h"++#ifdef __cplusplus+extern "C" {+#endif++#ifndef _CRAM_STRUCTS_H_+enum cram_block_method {+    ERROR    = -1,+    RAW      = 0,+    GZIP     = 1,+    BZIP2    = 2,+    LZMA     = 3,+    RANS     = 4,  // Generic; either order+    RANS0    = 4,+    RANS1    = 10, // Not externalised; stored as RANS (generic)+    GZIP_RLE = 11, // NB: not externalised in CRAM+};++enum cram_content_type {+    CT_ERROR           = -1,+    FILE_HEADER        = 0,+    COMPRESSION_HEADER = 1,+    MAPPED_SLICE       = 2,+    UNMAPPED_SLICE     = 3, // CRAM V1.0 only+    EXTERNAL           = 4,+    CORE               = 5,+};++// Opaque data types, see cram_structs for the fully fledged versions.+typedef struct SAM_hdr SAM_hdr;+typedef struct cram_file_def cram_file_def;+typedef struct cram_fd cram_fd;+typedef struct cram_container cram_container;+typedef struct cram_block cram_block;+typedef struct cram_slice cram_slice;+typedef struct cram_metrics cram_metrics;+typedef struct cram_block_slice_hdr cram_block_slice_hdr;+typedef struct cram_block_compression_hdr cram_block_compression_hdr;+typedef struct refs_t refs_t; // need this?++struct hFILE;+#endif++// Accessor functions++/*+ *-----------------------------------------------------------------------------+ * cram_fd+ */+SAM_hdr *cram_fd_get_header(cram_fd *fd);+void cram_fd_set_header(cram_fd *fd, SAM_hdr *hdr);++int cram_fd_get_version(cram_fd *fd);+void cram_fd_set_version(cram_fd *fd, int vers);++int cram_major_vers(cram_fd *fd);+int cram_minor_vers(cram_fd *fd);++struct hFILE *cram_fd_get_fp(cram_fd *fd);+void cram_fd_set_fp(cram_fd *fd, struct hFILE *fp);+++/*+ *-----------------------------------------------------------------------------+ * cram_container+ */+int32_t cram_container_get_length(cram_container *c);+void cram_container_set_length(cram_container *c, int32_t length);+int32_t cram_container_get_num_blocks(cram_container *c);+void cram_container_set_num_blocks(cram_container *c, int32_t num_blocks);+int32_t *cram_container_get_landmarks(cram_container *c, int32_t *num_landmarks);+void cram_container_set_landmarks(cram_container *c, int32_t num_landmarks,+                                  int32_t *landmarks);++/* Returns true if the container is empty (EOF marker) */+int cram_container_is_empty(cram_fd *fd);+++/*+ *-----------------------------------------------------------------------------+ * cram_block+ */+int32_t cram_block_get_content_id(cram_block *b);+int32_t cram_block_get_comp_size(cram_block *b);+int32_t cram_block_get_uncomp_size(cram_block *b);+int32_t cram_block_get_crc32(cram_block *b);+void *  cram_block_get_data(cram_block *b);++enum cram_content_type cram_block_get_content_type(cram_block *b);++void cram_block_set_content_id(cram_block *b, int32_t id);+void cram_block_set_comp_size(cram_block *b, int32_t size);+void cram_block_set_uncomp_size(cram_block *b, int32_t size);+void cram_block_set_crc32(cram_block *b, int32_t crc);+void cram_block_set_data(cram_block *b, void *data);++int cram_block_append(cram_block *b, void *data, int size);+void cram_block_update_size(cram_block *b);++// Offset is known as "size" internally, but it can be confusing.+size_t cram_block_get_offset(cram_block *b);+void cram_block_set_offset(cram_block *b, size_t offset);++/*+ * Computes the size of a cram block, including the block+ * header itself.+ */+uint32_t cram_block_size(cram_block *b);++/*+ * Renumbers RG numbers in a cram compression header.+ *+ * CRAM stores RG as the Nth number in the header, rather than a+ * string holding the ID: tag.  This is smaller in space, but means+ * "samtools cat" to join files together that contain single but+ * different RG lines needs a way of renumbering them.+ *+ * The file descriptor is expected to be immediately after the+ * cram_container structure (ie before the cram compression header).+ * Due to the nature of the CRAM format, this needs to read and write+ * the blocks itself.  Note that there may be multiple slices within+ * the container, meaning multiple compression headers to manipulate.+ * Changing RG may change the size of the compression header and+ * therefore the length field in the container.  Hence we rewrite all+ * blocks just incase and also emit the adjusted container.+ *+ * The current implementation can only cope with renumbering a single+ * RG (and only then if it is using HUFFMAN or BETA codecs).  In+ * theory it *may* be possible to renumber multiple RGs if they use+ * HUFFMAN to the CORE block or use an external block unshared by any+ * other data series.  So we have an API that can be upgraded to+ * support this, but do not implement it for now.  An example+ * implementation of RG as an EXTERNAL block would be to find that+ * block and rewrite it, returning the number of blocks consumed.+ *+ * Returns 0 on success;+ *        -1 if unable to edit;+ *        -2 on other errors (eg I/O).+ */+int cram_transcode_rg(cram_fd *in, cram_fd *out,+                      cram_container *c,+                      int nrg, int *in_rg, int *out_rg);++/*+ * Copies the blocks representing the next num_slice slices from a+ * container from 'in' to 'out'.  It is expected that the file pointer+ * is just after the read of the cram_container and cram compression+ * header.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_copy_slice(cram_fd *in, cram_fd *out, int32_t num_slice);++/*+ *-----------------------------------------------------------------------------+ * SAM_hdr+ */++/*! Tokenises a SAM header into a hash table.+ *+ * Also extracts a few bits on specific data types, such as @RG lines.+ *+ * @return+ * Returns a SAM_hdr struct on success (free with sam_hdr_free());+ *         NULL on failure+ */+SAM_hdr *sam_hdr_parse_(const char *hdr, int len);+++/*+ *-----------------------------------------------------------------------------+ * cram_io basics+ */++/**@{ ----------------------------------------------------------------------+ * CRAM blocks - the dynamically growable data block. We have code to+ * create, update, (un)compress and read/write.+ *+ * These are derived from the deflate_interlaced.c blocks, but with the+ * CRAM extension of content types and IDs.+ */++/*! Allocates a new cram_block structure with a specified content_type and+ * id.+ *+ * @return+ * Returns block pointer on success;+ *         NULL on failure+ */+cram_block *cram_new_block(enum cram_content_type content_type,+                           int content_id);++/*! Reads a block from a cram file.+ *+ * @return+ * Returns cram_block pointer on success;+ *         NULL on failure+ */+cram_block *cram_read_block(cram_fd *fd);++/*! Writes a CRAM block.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_write_block(cram_fd *fd, cram_block *b);++/*! Frees a CRAM block, deallocating internal data too.+ */+void cram_free_block(cram_block *b);++/*! Uncompresses a CRAM block, if compressed.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_uncompress_block(cram_block *b);++/*! Compresses a block.+ *+ * Compresses a block using one of two different zlib strategies. If we only+ * want one choice set strat2 to be -1.+ *+ * The logic here is that sometimes Z_RLE does a better job than Z_FILTERED+ * or Z_DEFAULT_STRATEGY on quality data. If so, we'd rather use it as it is+ * significantly faster.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_compress_block(cram_fd *fd, cram_block *b, cram_metrics *metrics,+                        int method, int level);++/**@}*/+/**@{ ----------------------------------------------------------------------+ * Containers+ */++/*! Creates a new container, specifying the maximum number of slices+ * and records permitted.+ *+ * @return+ * Returns cram_container ptr on success;+ *         NULL on failure+ */+cram_container *cram_new_container(int nrec, int nslice);+void cram_free_container(cram_container *c);++/*! Reads a container header.+ *+ * @return+ * Returns cram_container on success;+ *         NULL on failure or no container left (fd->err == 0).+ */+cram_container *cram_read_container(cram_fd *fd);++/*! Writes a container structure.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_write_container(cram_fd *fd, cram_container *h);++/*+ * Stores the container structure in dat and returns *size as the+ * number of bytes written to dat[].  The input size of dat is also+ * held in *size and should be initialised to cram_container_size(c).+ *+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_store_container(cram_fd *fd, cram_container *c, char *dat, int *size);++int cram_container_size(cram_container *c);++/**@}*/+/**@{ ----------------------------------------------------------------------+ * The top-level cram opening, closing and option handling+ */++/*! Opens a CRAM file for read (mode "rb") or write ("wb").+ *+ * The filename may be "-" to indicate stdin or stdout.+ *+ * @return+ * Returns file handle on success;+ *         NULL on failure.+ */+cram_fd *cram_open(const char *filename, const char *mode);++/*! Opens an existing stream for reading or writing.+ *+ * @return+ * Returns file handle on success;+ *         NULL on failure.+ */+cram_fd *cram_dopen(struct hFILE *fp, const char *filename, const char *mode);++/*! Closes a CRAM file.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_close(cram_fd *fd);++/*+ * Seek within a CRAM file.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_seek(cram_fd *fd, off_t offset, int whence);++/*+ * Flushes a CRAM file.+ * Useful for when writing to stdout without wishing to close the stream.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int cram_flush(cram_fd *fd);++/*! Checks for end of file on a cram_fd stream.+ *+ * @return+ * Returns 0 if not at end of file+ *         1 if we hit an expected EOF (end of range or EOF block)+ *         2 for other EOF (end of stream without EOF block)+ */+int cram_eof(cram_fd *fd);++/*! Sets options on the cram_fd.+ *+ * See CRAM_OPT_* definitions in hts.h.+ * Use this immediately after opening.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_set_option(cram_fd *fd, enum hts_fmt_option opt, ...);++/*! Sets options on the cram_fd.+ *+ * See CRAM_OPT_* definitions in hts.h.+ * Use this immediately after opening.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_set_voption(cram_fd *fd, enum hts_fmt_option opt, va_list args);++/*!+ * Attaches a header to a cram_fd.+ *+ * This should be used when creating a new cram_fd for writing where+ * we have an SAM_hdr already constructed (eg from a file we've read+ * in).+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int cram_set_header(cram_fd *fd, SAM_hdr *hdr);+++/* As int32_decoded/encode, but from/to blocks instead of cram_fd */+int int32_put_blk(cram_block *b, int32_t val);++/**@}*/+/**@{ -------------------------------------------------------------------*/+/*! Deallocates all storage used by a SAM_hdr struct.+ *+ * This also decrements the header reference count. If after decrementing+ * it is still non-zero then the header is assumed to be in use by another+ * caller and the free is not done.+ *+ * This is a synonym for sam_hdr_dec_ref().+ */+void sam_hdr_free(SAM_hdr *hdr);++/*! Returns the current length of the SAM_hdr in text form.+ *+ * Call sam_hdr_rebuild() first if editing has taken place.+ */+int sam_hdr_length(SAM_hdr *hdr);++/*! Returns the string form of the SAM_hdr.+ *+ * Call sam_hdr_rebuild() first if editing has taken place.+ */+char *sam_hdr_str(SAM_hdr *hdr);++/*! Appends a formatted line to an existing SAM header.+ *+ * Line is a full SAM header record, eg "@SQ\tSN:foo\tLN:100", with+ * optional new-line. If it contains more than 1 line then multiple lines+ * will be added in order.+ *+ * Len is the length of the text data, or 0 if unknown (in which case+ * it should be null terminated).+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */++/*! Add an @PG line.+ *+ * If we wish complete control over this use sam_hdr_add() directly. This+ * function uses that, but attempts to do a lot of tedious house work for+ * you too.+ *+ * - It will generate a suitable ID if the supplied one clashes.+ * - It will generate multiple @PG records if we have multiple PG chains.+ *+ * Call it as per sam_hdr_add() with a series of key,value pairs ending+ * in NULL.+ *+ * @return+ * Returns 0 on success;+ *        -1 on failure+ */+int sam_hdr_add_PG(SAM_hdr *sh, const char *name, ...);++/*!+ * A function to help with construction of CL tags in @PG records.+ * Takes an argc, argv pair and returns a single space-separated string.+ * This string should be deallocated by the calling function.+ *+ * @return+ * Returns malloced char * on success;+ *         NULL on failure+ */+char *stringify_argv(int argc, char *argv[]);+/**@}*/++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/faidx.h view
@@ -0,0 +1,137 @@+/* faidx.h -- FASTA random access.++   Copyright (C) 2008, 2009, 2013, 2014 Genome Research Ltd.++   Author: Heng Li <lh3@sanger.ac.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#ifndef HTSLIB_FAIDX_H+#define HTSLIB_FAIDX_H++#include "hts_defs.h"++#ifdef __cplusplus+extern "C" {+#endif++/*!+  @header++  Index FASTA files and extract subsequence.++  The fai file index columns are:+    - chromosome name+    - chromosome length: number of bases+    - offset: number of bytes to skip to get to the first base+        from the beginning of the file, including the length+        of the sequence description string (">chr ..\n")+    - line length: number of bases per line (excluding \n)+    - binary line length: number of bytes, including \n++  @copyright The Wellcome Trust Sanger Institute.+ */++struct __faidx_t;+typedef struct __faidx_t faidx_t;++    /*!+      @abstract   Build index for a FASTA or bgzip-compressed FASTA file.+      @param  fn  FASTA file name+      @return     0 on success; or -1 on failure+      @discussion File "fn.fai" will be generated.+     */+    int fai_build(const char *fn) HTS_RESULT_USED;++    /*!+      @abstract    Destroy a faidx_t struct.+      @param  fai  Pointer to the struct to be destroyed+     */+    void fai_destroy(faidx_t *fai);++    /*!+      @abstract   Load index from "fn.fai".+      @param  fn  File name of the FASTA file+     */+    faidx_t *fai_load(const char *fn);++    /*!+      @abstract    Fetch the sequence in a region.+      @param  fai  Pointer to the faidx_t struct+      @param  reg  Region in the format "chr2:20,000-30,000"+      @param  len  Length of the region; -2 if seq not present, -1 general error+      @return      Pointer to the sequence; null on failure++      @discussion The returned sequence is allocated by malloc family+      and should be destroyed by end users by calling free() on it.+     */+    char *fai_fetch(const faidx_t *fai, const char *reg, int *len);++    /*!+      @abstract    Fetch the number of sequences.+      @param  fai  Pointer to the faidx_t struct+      @return      The number of sequences+     */+    int faidx_fetch_nseq(const faidx_t *fai) HTS_DEPRECATED("Please use faidx_nseq instead");++    /*!+      @abstract    Fetch the sequence in a region.+      @param  fai  Pointer to the faidx_t struct+      @param  c_name Region name+      @param  p_beg_i  Beginning position number (zero-based)+      @param  p_end_i  End position number (zero-based)+      @param  len  Length of the region; -2 if c_name not present, -1 general error+      @return      Pointer to the sequence; null on failure++      @discussion The returned sequence is allocated by malloc family+      and should be destroyed by end users by calling free() on it.+     */+    char *faidx_fetch_seq(const faidx_t *fai, const char *c_name, int p_beg_i, int p_end_i, int *len);++    /*!+      @abstract    Query if sequence is present+      @param  fai  Pointer to the faidx_t struct+      @param  seq  Sequence name+      @return      1 if present or 0 if absent+     */+    int faidx_has_seq(const faidx_t *fai, const char *seq);++    /*!+      @abstract    Return number of sequences in fai index+     */+    int faidx_nseq(const faidx_t *fai);++    /*!+      @abstract    Return name of i-th sequence+     */+    const char *faidx_iseq(const faidx_t *fai, int i);++    /*!+      @abstract    Return sequence length, -1 if not present+     */+    int faidx_seq_len(const faidx_t *fai, const char *seq);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/hfile.h view
@@ -0,0 +1,215 @@+/*  hfile.h -- buffered low-level input/output streams.++    Copyright (C) 2013-2015 Genome Research Ltd.++    Author: John Marshall <jm18@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_HFILE_H+#define HTSLIB_HFILE_H++#include <string.h>++#include <sys/types.h>++#include "hts_defs.h"++#ifdef __cplusplus+extern "C" {+#endif++/* These fields are declared here solely for the benefit of the inline functions+   below.  They may change in future releases.  User code should not use them+   directly; you should imagine that hFILE is an opaque incomplete type.  */+struct hFILE_backend;+typedef struct hFILE {+    char *buffer, *begin, *end, *limit;+    const struct hFILE_backend *backend;+    off_t offset;+    unsigned at_eof:1;+    int has_errno;+} hFILE;++/*!+  @abstract  Open the named file or URL as a stream+  @return    An hFILE pointer, or NULL (with errno set) if an error occurred.+  @notes     The usual @c fopen(3) @a mode letters are supported: one of+    @e r (read), @e w (write), @e a (append), optionally followed by any of+    @e + (update), @e e (close on @c exec(2)), @e x (create exclusively).+*/+hFILE *hopen(const char *filename, const char *mode) HTS_RESULT_USED;++/*!+  @abstract  Associate a stream with an existing open file descriptor+  @return    An hFILE pointer, or NULL (with errno set) if an error occurred.+  @notes     For socket descriptors (on Windows), mode should contain 's'.+*/+hFILE *hdopen(int fd, const char *mode) HTS_RESULT_USED;++/*!+  @abstract  Report whether the file name or URL denotes remote storage+  @return    0 if local, 1 if remote.+  @notes     "Remote" means involving e.g. explicit network access, with the+    implication that callers may wish to cache such files' contents locally.+*/+int hisremote(const char *filename) HTS_RESULT_USED;++/*!+  @abstract  Flush (for output streams) and close the stream+  @return    0 if successful, or EOF (with errno set) if an error occurred.+*/+int hclose(hFILE *fp) HTS_RESULT_USED;++/*!+  @abstract  Close the stream, without flushing or propagating errors+  @notes     For use while cleaning up after an error only.  Preserves errno.+*/+void hclose_abruptly(hFILE *fp);++/*!+  @abstract  Return the stream's error indicator+  @return    Non-zero (in fact, an errno value) if an error has occurred.+  @notes     This would be called herror() and return true/false to parallel+    ferror(3), but a networking-related herror(3) function already exists.  */+static inline int herrno(hFILE *fp)+{+    return fp->has_errno;+}++/*!+  @abstract  Clear the stream's error indicator+*/+static inline void hclearerr(hFILE *fp)+{+    fp->has_errno = 0;+}++/*!+  @abstract  Reposition the read/write stream offset+  @return    The resulting offset within the stream (as per lseek(2)),+    or negative if an error occurred.+*/+off_t hseek(hFILE *fp, off_t offset, int whence) HTS_RESULT_USED;++/*!+  @abstract  Report the current stream offset+  @return    The offset within the stream, starting from zero.+*/+static inline off_t htell(hFILE *fp)+{+    return fp->offset + (fp->begin - fp->buffer);+}++/*!+  @abstract  Read one character from the stream+  @return    The character read, or EOF on end-of-file or error+*/+static inline int hgetc(hFILE *fp)+{+    extern int hgetc2(hFILE *);+    return (fp->end > fp->begin)? (unsigned char) *(fp->begin++) : hgetc2(fp);+}++/*!+  @abstract  Peek at characters to be read without removing them from buffers+  @param fp      The file stream+  @param buffer  The buffer to which the peeked bytes will be written+  @param nbytes  The number of bytes to peek at; limited by the size of the+    internal buffer, which could be as small as 4K.+  @return    The number of bytes peeked, which may be less than nbytes if EOF+    is encountered; or negative, if there was an I/O error.+  @notes  The characters peeked at remain in the stream's internal buffer,+    and will be returned by later hread() etc calls.+*/+ssize_t hpeek(hFILE *fp, void *buffer, size_t nbytes) HTS_RESULT_USED;++/*!+  @abstract  Read a block of characters from the file+  @return    The number of bytes read, or negative if an error occurred.+  @notes     The full nbytes requested will be returned, except as limited+    by EOF or I/O errors.+*/+static inline ssize_t HTS_RESULT_USED+hread(hFILE *fp, void *buffer, size_t nbytes)+{+    extern ssize_t hread2(hFILE *, void *, size_t, size_t);++    size_t n = fp->end - fp->begin;+    if (n > nbytes) n = nbytes;+    memcpy(buffer, fp->begin, n);+    fp->begin += n;+    return (n == nbytes)? (ssize_t) n : hread2(fp, buffer, nbytes, n);+}++/*!+  @abstract  Write a character to the stream+  @return    The character written, or EOF if an error occurred.+*/+static inline int hputc(int c, hFILE *fp)+{+    extern int hputc2(int, hFILE *);+    if (fp->begin < fp->limit) *(fp->begin++) = c;+    else c = hputc2(c, fp);+    return c;+}++/*!+  @abstract  Write a string to the stream+  @return    0 if successful, or EOF if an error occurred.+*/+static inline int hputs(const char *text, hFILE *fp)+{+    extern int hputs2(const char *, size_t, size_t, hFILE *);++    size_t nbytes = strlen(text), n = fp->limit - fp->begin;+    if (n > nbytes) n = nbytes;+    memcpy(fp->begin, text, n);+    fp->begin += n;+    return (n == nbytes)? 0 : hputs2(text, nbytes, n, fp);+}++/*!+  @abstract  Write a block of characters to the file+  @return    Either nbytes, or negative if an error occurred.+  @notes     In the absence of I/O errors, the full nbytes will be written.+*/+static inline ssize_t HTS_RESULT_USED+hwrite(hFILE *fp, const void *buffer, size_t nbytes)+{+    extern ssize_t hwrite2(hFILE *, const void *, size_t, size_t);++    size_t n = fp->limit - fp->begin;+    if (n > nbytes) n = nbytes;+    memcpy(fp->begin, buffer, n);+    fp->begin += n;+    return (n==nbytes)? (ssize_t) n : hwrite2(fp, buffer, nbytes, n);+}++/*!+  @abstract  For writing streams, flush buffered output to the underlying stream+  @return    0 if successful, or EOF if an error occurred.+*/+int hflush(hFILE *fp) HTS_RESULT_USED;++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/hts.h view
@@ -0,0 +1,639 @@+/*  hts.h -- format-neutral I/O, indexing, and iterator API functions.++    Copyright (C) 2012-2015 Genome Research Ltd.+    Copyright (C) 2012 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_HTS_H+#define HTSLIB_HTS_H++#include <stddef.h>+#include <stdint.h>++#include "hts_defs.h"++#ifdef __cplusplus+extern "C" {+#endif++#ifndef HTS_BGZF_TYPEDEF+typedef struct BGZF BGZF;+#define HTS_BGZF_TYPEDEF+#endif+struct cram_fd;+struct hFILE;++#ifndef KSTRING_T+#define KSTRING_T kstring_t+typedef struct __kstring_t {+    size_t l, m;+    char *s;+} kstring_t;+#endif++#ifndef kroundup32+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))+#endif++/**+ * hts_expand()  - expands memory block pointed to by $ptr;+ * hts_expand0()   the latter sets the newly allocated part to 0.+ *+ * @param n     requested number of elements of type type_t+ * @param m     size of memory allocated+ */+#define hts_expand(type_t, n, m, ptr) if ((n) > (m)) { \+        (m) = (n); kroundup32(m); \+        (ptr) = (type_t*)realloc((ptr), (m) * sizeof(type_t)); \+    }+#define hts_expand0(type_t, n, m, ptr) if ((n) > (m)) { \+        int t = (m); (m) = (n); kroundup32(m); \+        (ptr) = (type_t*)realloc((ptr), (m) * sizeof(type_t)); \+        memset(((type_t*)ptr)+t,0,sizeof(type_t)*((m)-t)); \+    }++/************+ * File I/O *+ ************/++// Add new entries only at the end (but before the *_maximum entry)+// of these enums, as their numbering is part of the htslib ABI.++enum htsFormatCategory {+    unknown_category,+    sequence_data,    // Sequence data -- SAM, BAM, CRAM, etc+    variant_data,     // Variant calling data -- VCF, BCF, etc+    index_file,       // Index file associated with some data file+    region_list,      // Coordinate intervals or regions -- BED, etc+    category_maximum = 32767+};++enum htsExactFormat {+    unknown_format,+    binary_format, text_format,+    sam, bam, bai, cram, crai, vcf, bcf, csi, gzi, tbi, bed,+    format_maximum = 32767+};++enum htsCompression {+    no_compression, gzip, bgzf, custom,+    compression_maximum = 32767+};++typedef struct htsFormat {+    enum htsFormatCategory category;+    enum htsExactFormat format;+    struct { short major, minor; } version;+    enum htsCompression compression;+    short compression_level;  // currently unused+    void *specific;  // format specific options; see struct hts_opt.+} htsFormat;++// Maintainers note htsFile cannot be an opaque structure because some of its+// fields are part of libhts.so's ABI (hence these fields must not be moved):+//  - fp is used in the public sam_itr_next()/etc macros+//  - is_bin is used directly in samtools <= 1.1 and bcftools <= 1.1+//  - is_write and is_cram are used directly in samtools <= 1.1+//  - fp is used directly in samtools (up to and including current develop)+//  - line is used directly in bcftools (up to and including current develop)+typedef struct {+    uint32_t is_bin:1, is_write:1, is_be:1, is_cram:1, dummy:28;+    int64_t lineno;+    kstring_t line;+    char *fn, *fn_aux;+    union {+        BGZF *bgzf;+        struct cram_fd *cram;+        struct hFILE *hfile;+        void *voidp;+    } fp;+    htsFormat format;+} htsFile;++// REQUIRED_FIELDS+enum sam_fields {+    SAM_QNAME = 0x00000001,+    SAM_FLAG  = 0x00000002,+    SAM_RNAME = 0x00000004,+    SAM_POS   = 0x00000008,+    SAM_MAPQ  = 0x00000010,+    SAM_CIGAR = 0x00000020,+    SAM_RNEXT = 0x00000040,+    SAM_PNEXT = 0x00000080,+    SAM_TLEN  = 0x00000100,+    SAM_SEQ   = 0x00000200,+    SAM_QUAL  = 0x00000400,+    SAM_AUX   = 0x00000800,+    SAM_RGAUX = 0x00001000,+};++// Mostly CRAM only, but this could also include other format options+enum hts_fmt_option {+    // CRAM specific+    CRAM_OPT_DECODE_MD,+    CRAM_OPT_PREFIX,+    CRAM_OPT_VERBOSITY,  // make general+    CRAM_OPT_SEQS_PER_SLICE,+    CRAM_OPT_SLICES_PER_CONTAINER,+    CRAM_OPT_RANGE,+    CRAM_OPT_VERSION,    // rename to cram_version?+    CRAM_OPT_EMBED_REF,+    CRAM_OPT_IGNORE_MD5,+    CRAM_OPT_REFERENCE,  // make general+    CRAM_OPT_MULTI_SEQ_PER_SLICE,+    CRAM_OPT_NO_REF,+    CRAM_OPT_USE_BZIP2,+    CRAM_OPT_SHARED_REF,+    CRAM_OPT_NTHREADS,   // deprecated, use HTS_OPT_NTHREADS+    CRAM_OPT_THREAD_POOL,// make general+    CRAM_OPT_USE_LZMA,+    CRAM_OPT_USE_RANS,+    CRAM_OPT_REQUIRED_FIELDS,++    // General purpose+    HTS_OPT_COMPRESSION_LEVEL = 100,+    HTS_OPT_NTHREADS,+};++// For backwards compatibility+#define cram_option hts_fmt_option++typedef struct hts_opt {+    char *arg;                // string form, strdup()ed+    enum hts_fmt_option opt;  // tokenised key+    union {                   // ... and value+        int i;+        char *s;+    } val;+    struct hts_opt *next;+} hts_opt;++#define HTS_FILE_OPTS_INIT {{0},0}++/**********************+ * Exported functions *+ **********************/++/*+ * Parses arg and appends it to the option list.+ *+ * Returns 0 on success;+ *        -1 on failure.+ */+int hts_opt_add(hts_opt **opts, const char *c_arg);++/*+ * Applies an hts_opt option list to a given htsFile.+ *+ * Returns 0 on success+ *        -1 on failure+ */+int hts_opt_apply(htsFile *fp, hts_opt *opts);++/*+ * Frees an hts_opt list.+ */+void hts_opt_free(hts_opt *opts);++/*+ * Accepts a string file format (sam, bam, cram, vcf, bam) optionally+ * followed by a comma separated list of key=value options and splits+ * these up into the fields of htsFormat struct.+ *+ * Returns 0 on success+ *        -1 on failure.+ */+int hts_parse_format(htsFormat *opt, const char *str);++/*+ * Tokenise options as (key(=value)?,)*(key(=value)?)?+ * NB: No provision for ',' appearing in the value!+ * Add backslashing rules?+ *+ * This could be used as part of a general command line option parser or+ * as a string concatenated onto the file open mode.+ *+ * Returns 0 on success+ *        -1 on failure.+ */+int hts_parse_opt_list(htsFormat *opt, const char *str);++extern int hts_verbose;++/*! @abstract Table for converting a nucleotide character to 4-bit encoding.+The input character may be either an IUPAC ambiguity code, '=' for 0, or+'0'/'1'/'2'/'3' for a result of 1/2/4/8.  The result is encoded as 1/2/4/8+for A/C/G/T or combinations of these bits for ambiguous bases.+*/+extern const unsigned char seq_nt16_table[256];++/*! @abstract Table for converting a 4-bit encoded nucleotide to an IUPAC+ambiguity code letter (or '=' when given 0).+*/+extern const char seq_nt16_str[];++/*! @abstract Table for converting a 4-bit encoded nucleotide to about 2 bits.+Returns 0/1/2/3 for 1/2/4/8 (i.e., A/C/G/T), or 4 otherwise (0 or ambiguous).+*/+extern const int seq_nt16_int[];++/*!+  @abstract  Get the htslib version number+  @return    For released versions, a string like "N.N[.N]"; or git describe+  output if using a library built within a Git repository.+*/+const char *hts_version(void);++/*!+  @abstract    Determine format by peeking at the start of a file+  @param fp    File opened for reading, positioned at the beginning+  @param fmt   Format structure that will be filled out on return+  @return      0 for success, or negative if an error occurred.+*/+int hts_detect_format(struct hFILE *fp, htsFormat *fmt);++/*!+  @abstract    Get a human-readable description of the file format+  @param fmt   Format structure holding type, version, compression, etc.+  @return      Description string, to be freed by the caller after use.+*/+char *hts_format_description(const htsFormat *format);++/*!+  @abstract       Open a SAM/BAM/CRAM/VCF/BCF/etc file+  @param fn       The file name or "-" for stdin/stdout+  @param mode     Mode matching / [rwa][bceguxz0-9]* /+  @discussion+      With 'r' opens for reading; any further format mode letters are ignored+      as the format is detected by checking the first few bytes or BGZF blocks+      of the file.  With 'w' or 'a' opens for writing or appending, with format+      specifier letters:+        b  binary format (BAM, BCF, etc) rather than text (SAM, VCF, etc)+        c  CRAM format+        g  gzip compressed+        u  uncompressed+        z  bgzf compressed+        [0-9]  zlib compression level+      and with non-format option letters (for any of 'r'/'w'/'a'):+        e  close the file on exec(2) (opens with O_CLOEXEC, where supported)+        x  create the file exclusively (opens with O_EXCL, where supported)+      Note that there is a distinction between 'u' and '0': the first yields+      plain uncompressed output whereas the latter outputs uncompressed data+      wrapped in the zlib format.+  @example+      [rw]b  .. compressed BCF, BAM, FAI+      [rw]bu .. uncompressed BCF+      [rw]z  .. compressed VCF+      [rw]   .. uncompressed VCF+*/+htsFile *hts_open(const char *fn, const char *mode);++/*!+  @abstract       Open a SAM/BAM/CRAM/VCF/BCF/etc file+  @param fn       The file name or "-" for stdin/stdout+  @param mode     Open mode, as per hts_open()+  @param fmt      Optional format specific parameters+  @discussion+      See hts_open() for description of fn and mode.+      // TODO Update documentation for s/opts/fmt/+      Opts contains a format string (sam, bam, cram, vcf, bcf) which will,+      if defined, override mode.  Opts also contains a linked list of hts_opt+      structures to apply to the open file handle.  These can contain things+      like pointers to the reference or information on compression levels,+      block sizes, etc.+*/+htsFile *hts_open_format(const char *fn, const char *mode, const htsFormat *fmt);++/*!+  @abstract       Open an existing stream as a SAM/BAM/CRAM/VCF/BCF/etc file+  @param fn       The already-open file handle+  @param mode     Open mode, as per hts_open()+*/+htsFile *hts_hopen(struct hFILE *fp, const char *fn, const char *mode);++/*!+  @abstract  Close a file handle, flushing buffered data for output streams+  @param fp  The file handle to be closed+  @return    0 for success, or negative if an error occurred.+*/+int hts_close(htsFile *fp);++/*!+  @abstract  Returns the file's format information+  @param fp  The file handle+  @return    Read-only pointer to the file's htsFormat.+*/+const htsFormat *hts_get_format(htsFile *fp);++/*!+  @ abstract      Returns a string containing the file format extension.+  @ param format  Format structure containing the file type.+  @ return        A string ("sam", "bam", etc) or "?" for unknown formats.+ */+const char *hts_format_file_extension(const htsFormat *format);++/*!+  @abstract  Sets a specified CRAM option on the open file handle.+  @param fp  The file handle open the open file.+  @param opt The CRAM_OPT_* option.+  @param ... Optional arguments, dependent on the option used.+  @return    0 for success, or negative if an error occurred.+*/+int hts_set_opt(htsFile *fp, enum hts_fmt_option opt, ...);++int hts_getline(htsFile *fp, int delimiter, kstring_t *str);+char **hts_readlines(const char *fn, int *_n);+/*!+    @abstract       Parse comma-separated list or read list from a file+    @param list     File name or comma-separated list+    @param is_file+    @param _n       Size of the output array (number of items read)+    @return         NULL on failure or pointer to newly allocated array of+                    strings+*/+char **hts_readlist(const char *fn, int is_file, int *_n);++/*!+  @abstract  Create extra threads to aid compress/decompression for this file+  @param fp  The file handle+  @param n   The number of worker threads to create+  @return    0 for success, or negative if an error occurred.+  @notes     THIS THREADING API IS LIKELY TO CHANGE IN FUTURE.+*/+int hts_set_threads(htsFile *fp, int n);++/*!+  @abstract  Set .fai filename for a file opened for reading+  @return    0 for success, negative on failure+  @discussion+      Called before *_hdr_read(), this provides the name of a .fai file+      used to provide a reference list if the htsFile contains no @SQ headers.+*/+int hts_set_fai_filename(htsFile *fp, const char *fn_aux);++/************+ * Indexing *+ ************/++/*!+These HTS_IDX_* macros are used as special tid values for hts_itr_query()/etc,+producing iterators operating as follows:+ - HTS_IDX_NOCOOR iterates over unmapped reads sorted at the end of the file+ - HTS_IDX_START  iterates over the entire file+ - HTS_IDX_REST   iterates from the current position to the end of the file+ - HTS_IDX_NONE   always returns "no more alignment records"+When one of these special tid values is used, beg and end are ignored.+When REST or NONE is used, idx is also ignored and may be NULL.+*/+#define HTS_IDX_NOCOOR (-2)+#define HTS_IDX_START  (-3)+#define HTS_IDX_REST   (-4)+#define HTS_IDX_NONE   (-5)++#define HTS_FMT_CSI 0+#define HTS_FMT_BAI 1+#define HTS_FMT_TBI 2+#define HTS_FMT_CRAI 3++struct __hts_idx_t;+typedef struct __hts_idx_t hts_idx_t;++typedef struct {+    uint64_t u, v;+} hts_pair64_t;++typedef int hts_readrec_func(BGZF *fp, void *data, void *r, int *tid, int *beg, int *end);++typedef struct {+    uint32_t read_rest:1, finished:1, dummy:29;+    int tid, beg, end, n_off, i;+    int curr_tid, curr_beg, curr_end;+    uint64_t curr_off;+    hts_pair64_t *off;+    hts_readrec_func *readrec;+    struct {+        int n, m;+        int *a;+    } bins;+} hts_itr_t;++    #define hts_bin_first(l) (((1<<(((l)<<1) + (l))) - 1) / 7)+    #define hts_bin_parent(l) (((l) - 1) >> 3)++    hts_idx_t *hts_idx_init(int n, int fmt, uint64_t offset0, int min_shift, int n_lvls);+    void hts_idx_destroy(hts_idx_t *idx);+    int hts_idx_push(hts_idx_t *idx, int tid, int beg, int end, uint64_t offset, int is_mapped);+    void hts_idx_finish(hts_idx_t *idx, uint64_t final_offset);++/// Save an index to a file+/** @param idx  Index to be written+    @param fn   Input BAM/BCF/etc filename, to which .bai/.csi/etc will be added+    @param fmt  One of the HTS_FMT_* index formats+    @return  0 if successful, or negative if an error occurred.+*/+int hts_idx_save(const hts_idx_t *idx, const char *fn, int fmt) HTS_RESULT_USED;++/// Save an index to a specific file+/** @param idx    Index to be written+    @param fn     Input BAM/BCF/etc filename+    @param fnidx  Output filename, or NULL to add .bai/.csi/etc to @a fn+    @param fmt    One of the HTS_FMT_* index formats+    @return  0 if successful, or negative if an error occurred.+*/+int hts_idx_save_as(const hts_idx_t *idx, const char *fn, const char *fnidx, int fmt) HTS_RESULT_USED;++/// Load an index file+/** @param fn   BAM/BCF/etc filename, to which .bai/.csi/etc will be added or+                the extension substituted, to search for an existing index file+    @param fmt  One of the HTS_FMT_* index formats+    @return  The index, or NULL if an error occurred.+*/+hts_idx_t *hts_idx_load(const char *fn, int fmt);++/// Load a specific index file+/** @param fn     Input BAM/BCF/etc filename+    @param fnidx  The input index filename+    @return  The index, or NULL if an error occurred.+*/+hts_idx_t *hts_idx_load2(const char *fn, const char *fnidx);++    uint8_t *hts_idx_get_meta(hts_idx_t *idx, int *l_meta);+    void hts_idx_set_meta(hts_idx_t *idx, int l_meta, uint8_t *meta, int is_copy);++    int hts_idx_get_stat(const hts_idx_t* idx, int tid, uint64_t* mapped, uint64_t* unmapped);+    uint64_t hts_idx_get_n_no_coor(const hts_idx_t* idx);+++#define HTS_PARSE_THOUSANDS_SEP 1  ///< Ignore ',' separators within numbers++/// Parse a numeric string+/** The number may be expressed in scientific notation, and optionally may+    contain commas in the integer part (before any decimal point or E notation).+    @param str     String to be parsed+    @param strend  If non-NULL, set on return to point to the first character+                   in @a str after those forming the parsed number+    @param flags   Or'ed-together combination of HTS_PARSE_* flags+    @return  Converted value of the parsed number.++    When @a strend is NULL, a warning will be printed (if hts_verbose is 2+    or more) if there are any trailing characters after the number.+*/+long long hts_parse_decimal(const char *str, char **strend, int flags);++/// Parse a "CHR:START-END"-style region string+/** @param str  String to be parsed+    @param beg  Set on return to the 0-based start of the region+    @param end  Set on return to the 1-based end of the region+    @return  Pointer to the colon or '\0' after the reference sequence name,+             or NULL if @a str could not be parsed.+*/+const char *hts_parse_reg(const char *str, int *beg, int *end);++    hts_itr_t *hts_itr_query(const hts_idx_t *idx, int tid, int beg, int end, hts_readrec_func *readrec);+    void hts_itr_destroy(hts_itr_t *iter);++    typedef int (*hts_name2id_f)(void*, const char*);+    typedef const char *(*hts_id2name_f)(void*, int);+    typedef hts_itr_t *hts_itr_query_func(const hts_idx_t *idx, int tid, int beg, int end, hts_readrec_func *readrec);++    hts_itr_t *hts_itr_querys(const hts_idx_t *idx, const char *reg, hts_name2id_f getid, void *hdr, hts_itr_query_func *itr_query, hts_readrec_func *readrec);+    int hts_itr_next(BGZF *fp, hts_itr_t *iter, void *r, void *data) HTS_RESULT_USED;+    const char **hts_idx_seqnames(const hts_idx_t *idx, int *n, hts_id2name_f getid, void *hdr); // free only the array, not the values++    /**+     * hts_file_type() - Convenience function to determine file type+     * DEPRECATED:  This function has been replaced by hts_detect_format().+     * It and these FT_* macros will be removed in a future HTSlib release.+     */+    #define FT_UNKN   0+    #define FT_GZ     1+    #define FT_VCF    2+    #define FT_VCF_GZ (FT_GZ|FT_VCF)+    #define FT_BCF    (1<<2)+    #define FT_BCF_GZ (FT_GZ|FT_BCF)+    #define FT_STDIN  (1<<3)+    int hts_file_type(const char *fname);+++    /**********************+     * MD5 implementation *+     **********************/++    struct hts_md5_context;+    typedef struct hts_md5_context hts_md5_context;++    /*! @abstract   Intialises an MD5 context.+     *  @discussion+     *    The expected use is to allocate an hts_md5_context using+     *    hts_md5_init().  This pointer is then passed into one or more calls+     *    of hts_md5_update() to compute successive internal portions of the+     *    MD5 sum, which can then be externalised as a full 16-byte MD5sum+     *    calculation by calling hts_md5_final().  This can then be turned+     *    into ASCII via hts_md5_hex().+     *+     *    To dealloate any resources created by hts_md5_init() call the+     *    hts_md5_destroy() function.+     *+     *  @return     hts_md5_context pointer on success, NULL otherwise.+     */+    hts_md5_context *hts_md5_init(void);++    /*! @abstract Updates the context with the MD5 of the data. */+    void hts_md5_update(hts_md5_context *ctx, const void *data, unsigned long size);++    /*! @abstract Computes the final 128-bit MD5 hash from the given context */+    void hts_md5_final(unsigned char *digest, hts_md5_context *ctx);++    /*! @abstract Resets an md5_context to the initial state, as returned+     *            by hts_md5_init().+     */+    void hts_md5_reset(hts_md5_context *ctx);++    /*! @abstract Converts a 128-bit MD5 hash into a 33-byte nul-termninated+     *            hex string.+     */+    void hts_md5_hex(char *hex, const unsigned char *digest);++    /*! @abstract Deallocates any memory allocated by hts_md5_init. */+    void hts_md5_destroy(hts_md5_context *ctx);+++static inline int hts_reg2bin(int64_t beg, int64_t end, int min_shift, int n_lvls)+{+    int l, s = min_shift, t = ((1<<((n_lvls<<1) + n_lvls)) - 1) / 7;+    for (--end, l = n_lvls; l > 0; --l, s += 3, t -= 1<<((l<<1)+l))+        if (beg>>s == end>>s) return t + (beg>>s);+    return 0;+}++static inline int hts_bin_bot(int bin, int n_lvls)+{+    int l, b;+    for (l = 0, b = bin; b; ++l, b = hts_bin_parent(b)); // compute the level of bin+    return (bin - hts_bin_first(l)) << (n_lvls - l) * 3;+}++/**************+ * Endianness *+ **************/++static inline int ed_is_big(void)+{+    long one= 1;+    return !(*((char *)(&one)));+}+static inline uint16_t ed_swap_2(uint16_t v)+{+    return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8));+}+static inline void *ed_swap_2p(void *x)+{+    *(uint16_t*)x = ed_swap_2(*(uint16_t*)x);+    return x;+}+static inline uint32_t ed_swap_4(uint32_t v)+{+    v = ((v & 0x0000FFFFU) << 16) | (v >> 16);+    return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);+}+static inline void *ed_swap_4p(void *x)+{+    *(uint32_t*)x = ed_swap_4(*(uint32_t*)x);+    return x;+}+static inline uint64_t ed_swap_8(uint64_t v)+{+    v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);+    v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);+    return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);+}+static inline void *ed_swap_8p(void *x)+{+    *(uint64_t*)x = ed_swap_8(*(uint64_t*)x);+    return x;+}++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/hts_defs.h view
@@ -0,0 +1,72 @@+/*  hts_defs.h -- Miscellaneous definitions.++    Copyright (C) 2013-2015 Genome Research Ltd.++    Author: John Marshall <jm18@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_HTS_DEFS_H+#define HTSLIB_HTS_DEFS_H++#ifdef __clang__+#ifdef __has_attribute+#define HTS_COMPILER_HAS(attribute) __has_attribute(attribute)+#endif++#elif defined __GNUC__+#define HTS_GCC_AT_LEAST(major, minor) \+    (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))+#endif++#ifndef HTS_COMPILER_HAS+#define HTS_COMPILER_HAS(attribute) 0+#endif+#ifndef HTS_GCC_AT_LEAST+#define HTS_GCC_AT_LEAST(major, minor) 0+#endif++#if HTS_COMPILER_HAS(__noreturn__) || HTS_GCC_AT_LEAST(3,0)+#define HTS_NORETURN __attribute__ ((__noreturn__))+#else+#define HTS_NORETURN+#endif++// GCC introduced warn_unused_result in 3.4 but added -Wno-unused-result later+#if HTS_COMPILER_HAS(__warn_unused_result__) || HTS_GCC_AT_LEAST(4,5)+#define HTS_RESULT_USED __attribute__ ((__warn_unused_result__))+#else+#define HTS_RESULT_USED+#endif++#if HTS_COMPILER_HAS(__unused__) || HTS_GCC_AT_LEAST(3,0)+#define HTS_UNUSED __attribute__ ((__unused__))+#else+#define HTS_UNUSED+#endif++#if HTS_COMPILER_HAS(__deprecated__) || HTS_GCC_AT_LEAST(4,5)+#define HTS_DEPRECATED(message) __attribute__ ((__deprecated__ (message)))+#elif HTS_GCC_AT_LEAST(3,1)+#define HTS_DEPRECATED(message) __attribute__ ((__deprecated__))+#else+#define HTS_DEPRECATED(message)+#endif++#endif
+ htslib-1.3.2/htslib/kbitset.h view
@@ -0,0 +1,160 @@+/* The MIT License++   Copyright (C) 2015 Genome Research Ltd.++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#ifndef KBITSET_H+#define KBITSET_H++/* Example of using kbitset_t, which represents a subset of {0,..., N-1},+   where N is the size specified in kbs_init().++	kbitset_t *bset = kbs_init(100);+	kbs_insert(bset, 5);+	kbs_insert(bset, 68);+	kbs_delete(bset, 37);+	// ...++	if (kbs_exists(bset, 68)) printf("68 present\n");++	kbitset_iter itr;+	int i;+	kbs_start(&itr);+	while ((i = kbs_next(bset, &itr)) >= 0)+		printf("%d present\n", i);++	kbs_destroy(bset);++   Example of declaring a kbitset_t-using function in a header file, so that+   only source files that actually use process() need to include <kbitset.h>:++	struct kbitset_t;+	void process(struct kbitset_t *bset);+*/++#include <limits.h>+#include <stdlib.h>+#include <string.h>++#define KBS_ELTBITS (CHAR_BIT * sizeof (unsigned long))+#define KBS_ELT(i)  ((i) / KBS_ELTBITS)+#define KBS_MASK(i) (1UL << ((i) % KBS_ELTBITS))++typedef struct kbitset_t {+	size_t n;+	unsigned long b[1];+} kbitset_t;++// Initialise a bit set capable of holding ni integers, 0 <= i < ni.+// The set returned is empty if fill == 0, or all of [0,ni) otherwise.+static inline kbitset_t *kbs_init2(size_t ni, int fill)+{+	size_t n = (ni + KBS_ELTBITS-1) / KBS_ELTBITS;+	kbitset_t *bs =+		(kbitset_t *) malloc(sizeof(kbitset_t) + n * sizeof(unsigned long));+	if (bs == NULL) return NULL;+	bs->n = n;+	memset(bs->b, fill? ~0 : 0, n * sizeof (unsigned long));+	bs->b[n] = ~0UL;+	return bs;+}++// Initialise an empty bit set capable of holding ni integers, 0 <= i < ni.+static inline kbitset_t *kbs_init(size_t ni)+{+	return kbs_init2(ni, 0);+}++// Destroy a bit set.+static inline void kbs_destroy(kbitset_t *bs)+{+	free(bs);+}++// Reset the bit set to empty.+static inline void kbs_clear(kbitset_t *bs)+{+	memset(bs->b, 0, bs->n * sizeof (unsigned long));+}++// Reset the bit set to all of [0,ni).+static inline void kbs_insert_all(kbitset_t *bs)+{+	memset(bs->b, ~0, bs->n * sizeof (unsigned long));+}++// Insert an element into the bit set.+static inline void kbs_insert(kbitset_t *bs, int i)+{+	bs->b[KBS_ELT(i)] |= KBS_MASK(i);+}++// Remove an element from the bit set.+static inline void kbs_delete(kbitset_t *bs, int i)+{+	bs->b[KBS_ELT(i)] &= ~KBS_MASK(i);+}++// Test whether the bit set contains the element.+static inline int kbs_exists(const kbitset_t *bs, int i)+{+	return (bs->b[KBS_ELT(i)] & KBS_MASK(i)) != 0;+}++typedef struct kbitset_iter_t {+	unsigned long mask;+	size_t elt;+	int i;+} kbitset_iter_t;++// Initialise or reset a bit set iterator.+static inline void kbs_start(kbitset_iter_t *itr)+{+	itr->mask = 1;+	itr->elt = 0;+	itr->i = 0;+}++// Return the next element contained in the bit set, or -1 if there are no more.+static inline int kbs_next(const kbitset_t *bs, kbitset_iter_t *itr)+{+	unsigned long b = bs->b[itr->elt];++	for (;;) {+		if (itr->mask == 0) {+			while ((b = bs->b[++itr->elt]) == 0) itr->i += KBS_ELTBITS;+			if (itr->elt == bs->n) return -1;+			itr->mask = 1;+		}++		if (b & itr->mask) break;++		itr->i++;+		itr->mask <<= 1;+	}++	itr->mask <<= 1;+	return itr->i++;+}++#endif
+ htslib-1.3.2/htslib/kfunc.h view
@@ -0,0 +1,83 @@+/* The MIT License++   Copyright (C) 2010, 2013 Genome Research Ltd.+   Copyright (C) 2011 Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#ifndef HTSLIB_KFUNC_H+#define HTSLIB_KFUNC_H++#ifdef __cplusplus+extern "C" {+#endif++/* Log gamma function+ * \log{\Gamma(z)}+ * AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245+ */+double kf_lgamma(double z);++/* complementary error function+ * \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} dt+ * AS66, 2nd algorithm, http://lib.stat.cmu.edu/apstat/66+ */+double kf_erfc(double x);++/* The following computes regularized incomplete gamma functions.+ * Formulas are taken from Wiki, with additional input from Numerical+ * Recipes in C (for modified Lentz's algorithm) and AS245+ * (http://lib.stat.cmu.edu/apstat/245).+ *+ * A good online calculator is available at:+ *+ *   http://www.danielsoper.com/statcalc/calc23.aspx+ *+ * It calculates upper incomplete gamma function, which equals+ * kf_gammaq(s,z)*tgamma(s).+ */++double kf_gammap(double s, double z);+double kf_gammaq(double s, double z);++/* Regularized incomplete beta function. The method is taken from+ * Numerical Recipe in C, 2nd edition, section 6.4. The following web+ * page calculates the incomplete beta function, which equals+ * kf_betai(a,b,x) * gamma(a) * gamma(b) / gamma(a+b):+ *+ *   http://www.danielsoper.com/statcalc/calc36.aspx+ */+double kf_betai(double a, double b, double x);++/*+ *    n11  n12  | n1_+ *    n21  n22  | n2_+ *   -----------+----+ *    n_1  n_2  | n+ */+double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/khash.h view
@@ -0,0 +1,627 @@+/* The MIT License++   Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++/*+  An example:++#include "khash.h"+KHASH_MAP_INIT_INT(32, char)+int main() {+	int ret, is_missing;+	khiter_t k;+	khash_t(32) *h = kh_init(32);+	k = kh_put(32, h, 5, &ret);+	kh_value(h, k) = 10;+	k = kh_get(32, h, 10);+	is_missing = (k == kh_end(h));+	k = kh_get(32, h, 5);+	kh_del(32, h, k);+	for (k = kh_begin(h); k != kh_end(h); ++k)+		if (kh_exist(h, k)) kh_value(h, k) = 1;+	kh_destroy(32, h);+	return 0;+}+*/++/*+  2013-05-02 (0.2.8):++	* Use quadratic probing. When the capacity is power of 2, stepping function+	  i*(i+1)/2 guarantees to traverse each bucket. It is better than double+	  hashing on cache performance and is more robust than linear probing.++	  In theory, double hashing should be more robust than quadratic probing.+	  However, my implementation is probably not for large hash tables, because+	  the second hash function is closely tied to the first hash function,+	  which reduce the effectiveness of double hashing.++	Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php++  2011-12-29 (0.2.7):++    * Minor code clean up; no actual effect.++  2011-09-16 (0.2.6):++	* The capacity is a power of 2. This seems to dramatically improve the+	  speed for simple keys. Thank Zilong Tan for the suggestion. Reference:++	   - http://code.google.com/p/ulib/+	   - http://nothings.org/computer/judy/++	* Allow to optionally use linear probing which usually has better+	  performance for random input. Double hashing is still the default as it+	  is more robust to certain non-random input.++	* Added Wang's integer hash function (not used by default). This hash+	  function is more robust to certain non-random input.++  2011-02-14 (0.2.5):++    * Allow to declare global functions.++  2009-09-26 (0.2.4):++    * Improve portability++  2008-09-19 (0.2.3):++	* Corrected the example+	* Improved interfaces++  2008-09-11 (0.2.2):++	* Improved speed a little in kh_put()++  2008-09-10 (0.2.1):++	* Added kh_clear()+	* Fixed a compiling error++  2008-09-02 (0.2.0):++	* Changed to token concatenation which increases flexibility.++  2008-08-31 (0.1.2):++	* Fixed a bug in kh_get(), which has not been tested previously.++  2008-08-31 (0.1.1):++	* Added destructor+*/+++#ifndef __AC_KHASH_H+#define __AC_KHASH_H++/*!+  @header++  Generic hash table library.+ */++#define AC_VERSION_KHASH_H "0.2.8"++#include <stdlib.h>+#include <string.h>+#include <limits.h>++/* compiler specific configuration */++#if UINT_MAX == 0xffffffffu+typedef unsigned int khint32_t;+#elif ULONG_MAX == 0xffffffffu+typedef unsigned long khint32_t;+#endif++#if ULONG_MAX == ULLONG_MAX+typedef unsigned long khint64_t;+#else+typedef unsigned long long khint64_t;+#endif++#ifndef kh_inline+#ifdef _MSC_VER+#define kh_inline __inline+#else+#define kh_inline inline+#endif+#endif /* kh_inline */++#ifndef klib_unused+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)+#define klib_unused __attribute__ ((__unused__))+#else+#define klib_unused+#endif+#endif /* klib_unused */++typedef khint32_t khint_t;+typedef khint_t khiter_t;++#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))++#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)++#ifndef kroundup32+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))+#endif++#ifndef kcalloc+#define kcalloc(N,Z) calloc(N,Z)+#endif+#ifndef kmalloc+#define kmalloc(Z) malloc(Z)+#endif+#ifndef krealloc+#define krealloc(P,Z) realloc(P,Z)+#endif+#ifndef kfree+#define kfree(P) free(P)+#endif++static const double __ac_HASH_UPPER = 0.77;++#define __KHASH_TYPE(name, khkey_t, khval_t) \+	typedef struct kh_##name##_s { \+		khint_t n_buckets, size, n_occupied, upper_bound; \+		khint32_t *flags; \+		khkey_t *keys; \+		khval_t *vals; \+	} kh_##name##_t;++#define __KHASH_PROTOTYPES(name, khkey_t, khval_t)	 					\+	extern kh_##name##_t *kh_init_##name(void);							\+	extern void kh_destroy_##name(kh_##name##_t *h);					\+	extern void kh_clear_##name(kh_##name##_t *h);						\+	extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); 	\+	extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \+	extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \+	extern void kh_del_##name(kh_##name##_t *h, khint_t x);++#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \+	SCOPE kh_##name##_t *kh_init_##name(void) {							\+		return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t));		\+	}																	\+	SCOPE void kh_destroy_##name(kh_##name##_t *h)						\+	{																	\+		if (h) {														\+			kfree((void *)h->keys); kfree(h->flags);					\+			kfree((void *)h->vals);										\+			kfree(h);													\+		}																\+	}																	\+	SCOPE void kh_clear_##name(kh_##name##_t *h)						\+	{																	\+		if (h && h->flags) {											\+			memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \+			h->size = h->n_occupied = 0;								\+		}																\+	}																	\+	SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) 	\+	{																	\+		if (h->n_buckets) {												\+			khint_t k, i, last, mask, step = 0; \+			mask = h->n_buckets - 1;									\+			k = __hash_func(key); i = k & mask;							\+			last = i; \+			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \+				i = (i + (++step)) & mask; \+				if (i == last) return h->n_buckets;						\+			}															\+			return __ac_iseither(h->flags, i)? h->n_buckets : i;		\+		} else return 0;												\+	}																	\+	SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \+	{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \+		khint32_t *new_flags = 0;										\+		khint_t j = 1;													\+		{																\+			kroundup32(new_n_buckets); 									\+			if (new_n_buckets < 4) new_n_buckets = 4;					\+			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \+			else { /* hash table size to be changed (shrink or expand); rehash */ \+				new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t));	\+				if (!new_flags) return -1;								\+				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \+				if (h->n_buckets < new_n_buckets) {	/* expand */		\+					khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \+					if (!new_keys) { kfree(new_flags); return -1; }		\+					h->keys = new_keys;									\+					if (kh_is_map) {									\+						khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \+						if (!new_vals) { kfree(new_flags); return -1; }	\+						h->vals = new_vals;								\+					}													\+				} /* otherwise shrink */								\+			}															\+		}																\+		if (j) { /* rehashing is needed */								\+			for (j = 0; j != h->n_buckets; ++j) {						\+				if (__ac_iseither(h->flags, j) == 0) {					\+					khkey_t key = h->keys[j];							\+					khval_t val;										\+					khint_t new_mask;									\+					new_mask = new_n_buckets - 1; 						\+					if (kh_is_map) val = h->vals[j];					\+					__ac_set_isdel_true(h->flags, j);					\+					while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \+						khint_t k, i, step = 0; \+						k = __hash_func(key);							\+						i = k & new_mask;								\+						while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \+						__ac_set_isempty_false(new_flags, i);			\+						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \+							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \+							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \+							__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \+						} else { /* write the element and jump out of the loop */ \+							h->keys[i] = key;							\+							if (kh_is_map) h->vals[i] = val;			\+							break;										\+						}												\+					}													\+				}														\+			}															\+			if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \+				h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \+				if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \+			}															\+			kfree(h->flags); /* free the working space */				\+			h->flags = new_flags;										\+			h->n_buckets = new_n_buckets;								\+			h->n_occupied = h->size;									\+			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \+		}																\+		return 0;														\+	}																	\+	SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \+	{																	\+		khint_t x;														\+		if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \+			if (h->n_buckets > (h->size<<1)) {							\+				if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \+					*ret = -1; return h->n_buckets;						\+				}														\+			} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \+				*ret = -1; return h->n_buckets;							\+			}															\+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \+		{																\+			khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \+			x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \+			if (__ac_isempty(h->flags, i)) x = i; /* for speed up */	\+			else {														\+				last = i; \+				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \+					if (__ac_isdel(h->flags, i)) site = i;				\+					i = (i + (++step)) & mask; \+					if (i == last) { x = site; break; }					\+				}														\+				if (x == h->n_buckets) {								\+					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \+					else x = i;											\+				}														\+			}															\+		}																\+		if (__ac_isempty(h->flags, x)) { /* not present at all */		\+			h->keys[x] = key;											\+			__ac_set_isboth_false(h->flags, x);							\+			++h->size; ++h->n_occupied;									\+			*ret = 1;													\+		} else if (__ac_isdel(h->flags, x)) { /* deleted */				\+			h->keys[x] = key;											\+			__ac_set_isboth_false(h->flags, x);							\+			++h->size;													\+			*ret = 2;													\+		} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \+		return x;														\+	}																	\+	SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x)				\+	{																	\+		if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {			\+			__ac_set_isdel_true(h->flags, x);							\+			--h->size;													\+		}																\+	}++#define KHASH_DECLARE(name, khkey_t, khval_t)		 					\+	__KHASH_TYPE(name, khkey_t, khval_t) 								\+	__KHASH_PROTOTYPES(name, khkey_t, khval_t)++#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \+	__KHASH_TYPE(name, khkey_t, khval_t) 								\+	__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)++#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \+	KHASH_INIT2(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)++/* --- BEGIN OF HASH FUNCTIONS --- */++/*! @function+  @abstract     Integer hash function+  @param  key   The integer [khint32_t]+  @return       The hash value [khint_t]+ */+#define kh_int_hash_func(key) (khint32_t)(key)+/*! @function+  @abstract     Integer comparison function+ */+#define kh_int_hash_equal(a, b) ((a) == (b))+/*! @function+  @abstract     64-bit integer hash function+  @param  key   The integer [khint64_t]+  @return       The hash value [khint_t]+ */+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)+/*! @function+  @abstract     64-bit integer comparison function+ */+#define kh_int64_hash_equal(a, b) ((a) == (b))+/*! @function+  @abstract     const char* hash function+  @param  s     Pointer to a null terminated string+  @return       The hash value+ */+static kh_inline khint_t __ac_X31_hash_string(const char *s)+{+	khint_t h = (khint_t)*s;+	if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;+	return h;+}+/*! @function+  @abstract     Another interface to const char* hash function+  @param  key   Pointer to a null terminated string [const char*]+  @return       The hash value [khint_t]+ */+#define kh_str_hash_func(key) __ac_X31_hash_string(key)+/*! @function+  @abstract     Const char* comparison function+ */+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)++static kh_inline khint_t __ac_Wang_hash(khint_t key)+{+    key += ~(key << 15);+    key ^=  (key >> 10);+    key +=  (key << 3);+    key ^=  (key >> 6);+    key += ~(key << 11);+    key ^=  (key >> 16);+    return key;+}+#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key)++/* --- END OF HASH FUNCTIONS --- */++/* Other convenient macros... */++/*!+  @abstract Type of the hash table.+  @param  name  Name of the hash table [symbol]+ */+#define khash_t(name) kh_##name##_t++/*! @function+  @abstract     Initiate a hash table.+  @param  name  Name of the hash table [symbol]+  @return       Pointer to the hash table [khash_t(name)*]+ */+#define kh_init(name) kh_init_##name()++/*! @function+  @abstract     Destroy a hash table.+  @param  name  Name of the hash table [symbol]+  @param  h     Pointer to the hash table [khash_t(name)*]+ */+#define kh_destroy(name, h) kh_destroy_##name(h)++/*! @function+  @abstract     Reset a hash table without deallocating memory.+  @param  name  Name of the hash table [symbol]+  @param  h     Pointer to the hash table [khash_t(name)*]+ */+#define kh_clear(name, h) kh_clear_##name(h)++/*! @function+  @abstract     Resize a hash table.+  @param  name  Name of the hash table [symbol]+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  s     New size [khint_t]+ */+#define kh_resize(name, h, s) kh_resize_##name(h, s)++/*! @function+  @abstract     Insert a key to the hash table.+  @param  name  Name of the hash table [symbol]+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  k     Key [type of keys]+  @param  r     Extra return code: -1 if the operation failed;+                0 if the key is present in the hash table;+                1 if the bucket is empty (never used); 2 if the element in+				the bucket has been deleted [int*]+  @return       Iterator to the inserted element [khint_t]+ */+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)++/*! @function+  @abstract     Retrieve a key from the hash table.+  @param  name  Name of the hash table [symbol]+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  k     Key [type of keys]+  @return       Iterator to the found element, or kh_end(h) if the element is absent [khint_t]+ */+#define kh_get(name, h, k) kh_get_##name(h, k)++/*! @function+  @abstract     Remove a key from the hash table.+  @param  name  Name of the hash table [symbol]+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  k     Iterator to the element to be deleted [khint_t]+ */+#define kh_del(name, h, k) kh_del_##name(h, k)++/*! @function+  @abstract     Test whether a bucket contains data.+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  x     Iterator to the bucket [khint_t]+  @return       1 if containing data; 0 otherwise [int]+ */+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))++/*! @function+  @abstract     Get key given an iterator+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  x     Iterator to the bucket [khint_t]+  @return       Key [type of keys]+ */+#define kh_key(h, x) ((h)->keys[x])++/*! @function+  @abstract     Get value given an iterator+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  x     Iterator to the bucket [khint_t]+  @return       Value [type of values]+  @discussion   For hash sets, calling this results in segfault.+ */+#define kh_val(h, x) ((h)->vals[x])++/*! @function+  @abstract     Alias of kh_val()+ */+#define kh_value(h, x) ((h)->vals[x])++/*! @function+  @abstract     Get the start iterator+  @param  h     Pointer to the hash table [khash_t(name)*]+  @return       The start iterator [khint_t]+ */+#define kh_begin(h) (khint_t)(0)++/*! @function+  @abstract     Get the end iterator+  @param  h     Pointer to the hash table [khash_t(name)*]+  @return       The end iterator [khint_t]+ */+#define kh_end(h) ((h)->n_buckets)++/*! @function+  @abstract     Get the number of elements in the hash table+  @param  h     Pointer to the hash table [khash_t(name)*]+  @return       Number of elements in the hash table [khint_t]+ */+#define kh_size(h) ((h)->size)++/*! @function+  @abstract     Get the number of buckets in the hash table+  @param  h     Pointer to the hash table [khash_t(name)*]+  @return       Number of buckets in the hash table [khint_t]+ */+#define kh_n_buckets(h) ((h)->n_buckets)++/*! @function+  @abstract     Iterate over the entries in the hash table+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  kvar  Variable to which key will be assigned+  @param  vvar  Variable to which value will be assigned+  @param  code  Block of code to execute+ */+#define kh_foreach(h, kvar, vvar, code) { khint_t __i;		\+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\+		if (!kh_exist(h,__i)) continue;						\+		(kvar) = kh_key(h,__i);								\+		(vvar) = kh_val(h,__i);								\+		code;												\+	} }++/*! @function+  @abstract     Iterate over the values in the hash table+  @param  h     Pointer to the hash table [khash_t(name)*]+  @param  vvar  Variable to which value will be assigned+  @param  code  Block of code to execute+ */+#define kh_foreach_value(h, vvar, code) { khint_t __i;		\+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\+		if (!kh_exist(h,__i)) continue;						\+		(vvar) = kh_val(h,__i);								\+		code;												\+	} }++/* More conenient interfaces */++/*! @function+  @abstract     Instantiate a hash set containing integer keys+  @param  name  Name of the hash table [symbol]+ */+#define KHASH_SET_INIT_INT(name)										\+	KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)++/*! @function+  @abstract     Instantiate a hash map containing integer keys+  @param  name  Name of the hash table [symbol]+  @param  khval_t  Type of values [type]+ */+#define KHASH_MAP_INIT_INT(name, khval_t)								\+	KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)++/*! @function+  @abstract     Instantiate a hash map containing 64-bit integer keys+  @param  name  Name of the hash table [symbol]+ */+#define KHASH_SET_INIT_INT64(name)										\+	KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)++/*! @function+  @abstract     Instantiate a hash map containing 64-bit integer keys+  @param  name  Name of the hash table [symbol]+  @param  khval_t  Type of values [type]+ */+#define KHASH_MAP_INIT_INT64(name, khval_t)								\+	KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)++typedef const char *kh_cstr_t;+/*! @function+  @abstract     Instantiate a hash map containing const char* keys+  @param  name  Name of the hash table [symbol]+ */+#define KHASH_SET_INIT_STR(name)										\+	KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)++/*! @function+  @abstract     Instantiate a hash map containing const char* keys+  @param  name  Name of the hash table [symbol]+  @param  khval_t  Type of values [type]+ */+#define KHASH_MAP_INIT_STR(name, khval_t)								\+	KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)++#endif /* __AC_KHASH_H */
+ htslib-1.3.2/htslib/khash_str2int.h view
@@ -0,0 +1,133 @@+/*  khash_str2int.h -- C-string to integer hash table.++    Copyright (C) 2013 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_KHASH_STR2INT_H+#define HTSLIB_KHASH_STR2INT_H++#include <htslib/khash.h>++KHASH_MAP_INIT_STR(str2int, int)++/*+ *  Wrappers for khash dictionaries used by mpileup.+ */++static inline void *khash_str2int_init(void)+{+    return kh_init(str2int);+}++/*+ *  Destroy the hash structure, but not the keys+ */+static inline void khash_str2int_destroy(void *_hash)+{+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    if (hash) kh_destroy(str2int, hash); // Note that strings are not freed.+}++/*+ *  Destroys both the hash structure and the keys+ */+static inline void khash_str2int_destroy_free(void *_hash)+{+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    khint_t k;+    if (hash == 0) return;+    for (k = 0; k < kh_end(hash); ++k)+        if (kh_exist(hash, k)) free((char*)kh_key(hash, k));+    kh_destroy(str2int, hash);+}++/*+ *  Returns 1 if key exists or 0 if not+ */+static inline int khash_str2int_has_key(void *_hash, const char *str)+{+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    khint_t k = kh_get(str2int, hash, str);+    if ( k == kh_end(hash) ) return 0;+    return 1;+}++/*+ *  Returns 0 on success and -1 when the key is not present. On success,+ *  *value is set, unless NULL is passed.+ */+static inline int khash_str2int_get(void *_hash, const char *str, int *value)+{+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    khint_t k;+    if ( !hash ) return -1;+    k = kh_get(str2int, hash, str);+    if ( k == kh_end(hash) ) return -1;+    if ( !value ) return 0;+    *value = kh_val(hash, k);+    return 0;+}++/*+ *  Add a new string to the dictionary, auto-incrementing the value.+ *  On success returns the newly inserted integer id, on error -1+ *  is returned. Note that the key must continue to exist throughout+ *  the whole life of _hash.+ */+static inline int khash_str2int_inc(void *_hash, const char *str)+{+    khint_t k;+    int ret;+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    if ( !hash ) return -1;+    k = kh_put(str2int, hash, str, &ret);+    if (ret == 0) return kh_val(hash, k);+    kh_val(hash, k) = kh_size(hash) - 1;+    return kh_val(hash, k);+}++/*+ *  Set a new key,value pair. On success returns the bin index, on+ *  error -1 is returned. Note that the key must contnue to exist+ *  throughout the whole life of _hash.+ */+static inline int khash_str2int_set(void *_hash, const char *str, int value)+{+    khint_t k;+    int ret;+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    if ( !hash ) return -1;+    k = kh_put(str2int, hash, str, &ret);+    kh_val(hash,k) = value;+    return k;+}++/*+ *  Return the number of keys in the hash table.+ */+static inline int khash_str2int_size(void *_hash)+{+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;+    return kh_size(hash);+}++#endif
+ htslib-1.3.2/htslib/klist.h view
@@ -0,0 +1,135 @@+/* The MIT License++   Copyright (c) 2008-2009, by Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#ifndef _AC_KLIST_H+#define _AC_KLIST_H++#include <stdlib.h>++#ifndef klib_unused+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)+#define klib_unused __attribute__ ((__unused__))+#else+#define klib_unused+#endif+#endif /* klib_unused */++#define KMEMPOOL_INIT2(SCOPE, name, kmptype_t, kmpfree_f)				\+	typedef struct {													\+		size_t cnt, n, max;												\+		kmptype_t **buf;												\+	} kmp_##name##_t;													\+	SCOPE kmp_##name##_t *kmp_init_##name(void) {						\+		return calloc(1, sizeof(kmp_##name##_t));						\+	}																	\+	SCOPE void kmp_destroy_##name(kmp_##name##_t *mp) {					\+		size_t k;														\+		for (k = 0; k < mp->n; ++k) {									\+			kmpfree_f(mp->buf[k]); free(mp->buf[k]);					\+		}																\+		free(mp->buf); free(mp);										\+	}																	\+	SCOPE kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) {				\+		++mp->cnt;														\+		if (mp->n == 0) return calloc(1, sizeof(kmptype_t));			\+		return mp->buf[--mp->n];										\+	}																	\+	SCOPE void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) {		\+		--mp->cnt;														\+		if (mp->n == mp->max) {											\+			mp->max = mp->max? mp->max<<1 : 16;							\+			mp->buf = realloc(mp->buf, sizeof(kmptype_t *) * mp->max);	\+		}																\+		mp->buf[mp->n++] = p;											\+	}++#define KMEMPOOL_INIT(name, kmptype_t, kmpfree_f)						\+	KMEMPOOL_INIT2(static inline klib_unused, name, kmptype_t, kmpfree_f)++#define kmempool_t(name) kmp_##name##_t+#define kmp_init(name) kmp_init_##name()+#define kmp_destroy(name, mp) kmp_destroy_##name(mp)+#define kmp_alloc(name, mp) kmp_alloc_##name(mp)+#define kmp_free(name, mp, p) kmp_free_##name(mp, p)++#define KLIST_INIT2(SCOPE, name, kltype_t, kmpfree_t)					\+	struct __kl1_##name {												\+		kltype_t data;													\+		struct __kl1_##name *next;										\+	};																	\+	typedef struct __kl1_##name kl1_##name;								\+	KMEMPOOL_INIT2(SCOPE, name, kl1_##name, kmpfree_t)					\+	typedef struct {													\+		kl1_##name *head, *tail;										\+		kmp_##name##_t *mp;												\+		size_t size;													\+	} kl_##name##_t;													\+	SCOPE kl_##name##_t *kl_init_##name(void) {							\+		kl_##name##_t *kl = calloc(1, sizeof(kl_##name##_t));			\+		kl->mp = kmp_init(name);										\+		kl->head = kl->tail = kmp_alloc(name, kl->mp);					\+		kl->head->next = 0;												\+		return kl;														\+	}																	\+	SCOPE void kl_destroy_##name(kl_##name##_t *kl) {					\+		kl1_##name *p;													\+		for (p = kl->head; p != kl->tail; p = p->next)					\+			kmp_free(name, kl->mp, p);									\+		kmp_free(name, kl->mp, p);										\+		kmp_destroy(name, kl->mp);										\+		free(kl);														\+	}																	\+	SCOPE kltype_t *kl_pushp_##name(kl_##name##_t *kl) {				\+		kl1_##name *q, *p = kmp_alloc(name, kl->mp);					\+		q = kl->tail; p->next = 0; kl->tail->next = p; kl->tail = p;	\+		++kl->size;														\+		return &q->data;												\+	}																	\+	SCOPE int kl_shift_##name(kl_##name##_t *kl, kltype_t *d) {			\+		kl1_##name *p;													\+		if (kl->head->next == 0) return -1;								\+		--kl->size;														\+		p = kl->head; kl->head = kl->head->next;						\+		if (d) *d = p->data;											\+		kmp_free(name, kl->mp, p);										\+		return 0;														\+	}++#define KLIST_INIT(name, kltype_t, kmpfree_t)							\+	KLIST_INIT2(static inline klib_unused, name, kltype_t, kmpfree_t)++#define kliter_t(name) kl1_##name+#define klist_t(name) kl_##name##_t+#define kl_val(iter) ((iter)->data)+#define kl_next(iter) ((iter)->next)+#define kl_begin(kl) ((kl)->head)+#define kl_end(kl) ((kl)->tail)++#define kl_init(name) kl_init_##name()+#define kl_destroy(name, kl) kl_destroy_##name(kl)+#define kl_pushp(name, kl) kl_pushp_##name(kl)+#define kl_shift(name, kl, d) kl_shift_##name(kl, d)++#endif
+ htslib-1.3.2/htslib/knetfile.h view
@@ -0,0 +1,101 @@+/* The MIT License++   Copyright (c) 2008 by Genome Research Ltd (GRL).+                 2010 by Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#ifndef KNETFILE_H+#define KNETFILE_H++#include <stdint.h>+#include <fcntl.h>++#ifndef _WIN32+#define netread(fd, ptr, len) read(fd, ptr, len)+#define netwrite(fd, ptr, len) write(fd, ptr, len)+#define netclose(fd) close(fd)+#else+#include <winsock2.h>+#define netread(fd, ptr, len) recv(fd, ptr, len, 0)+#define netwrite(fd, ptr, len) send(fd, ptr, len, 0)+#define netclose(fd) closesocket(fd)+#endif++// FIXME: currently I/O is unbuffered++#define KNF_TYPE_LOCAL 1+#define KNF_TYPE_FTP   2+#define KNF_TYPE_HTTP  3++typedef struct knetFile_s {+	int type, fd;+	int64_t offset;+	char *host, *port;++	// the following are for FTP only+	int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready;+	char *response, *retr, *size_cmd;+	int64_t seek_offset; // for lazy seek+    int64_t file_size;++	// the following are for HTTP only+	char *path, *http_host;+} knetFile;++#define knet_tell(fp) ((fp)->offset)+#define knet_fileno(fp) ((fp)->fd)++#ifdef __cplusplus+extern "C" {+#endif++#ifdef _WIN32+	int knet_win32_init();+	void knet_win32_destroy();+#endif++	knetFile *knet_open(const char *fn, const char *mode);++	/* +	   This only works with local files.+	 */+	knetFile *knet_dopen(int fd, const char *mode);++	/*+	  If ->is_ready==0, this routine updates ->fd; otherwise, it simply+	  reads from ->fd.+	 */+	ssize_t knet_read(knetFile *fp, void *buf, size_t len);++	/*+	  This routine only sets ->offset and ->is_ready=0. It does not+	  communicate with the FTP server.+	 */+	off_t knet_seek(knetFile *fp, off_t off, int whence);+	int knet_close(knetFile *fp);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/kseq.h view
@@ -0,0 +1,253 @@+/* The MIT License++   Copyright (c) 2008, 2009, 2011 Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++/* Last Modified: 05MAR2012 */++#ifndef AC_KSEQ_H+#define AC_KSEQ_H++#include <ctype.h>+#include <string.h>+#include <stdlib.h>++#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r+#define KS_SEP_TAB   1 // isspace() && !' '+#define KS_SEP_LINE  2 // line separator: "\n" (Unix) or "\r\n" (Windows)+#define KS_SEP_MAX   2++#define __KS_TYPE(type_t) \+	typedef struct __kstream_t { \+		int begin, end; \+		int is_eof:2, bufsize:30; \+        uint64_t seek_pos; \+		type_t f; \+		unsigned char *buf; \+	} kstream_t;++#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end)+#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0)++#define __KS_BASIC(SCOPE, type_t, __bufsize) \+	SCOPE kstream_t *ks_init(type_t f) \+	{ \+		kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t)); \+		ks->f = f; ks->bufsize = __bufsize; \+		ks->buf = (unsigned char*)malloc(__bufsize); \+		return ks; \+	} \+	SCOPE void ks_destroy(kstream_t *ks) \+	{ \+		if (!ks) return; \+		free(ks->buf); \+		free(ks); \+	}++#define __KS_INLINED(__read) \+	static inline int ks_getc(kstream_t *ks) \+	{ \+		if (ks->is_eof && ks->begin >= ks->end) return -1; \+		if (ks->begin >= ks->end) { \+			ks->begin = 0; \+			ks->end = __read(ks->f, ks->buf, ks->bufsize); \+			if (ks->end == 0) { ks->is_eof = 1; return -1; } \+		} \+        ks->seek_pos++; \+		return (int)ks->buf[ks->begin++]; \+	} \+	static inline int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \+	{ return ks_getuntil2(ks, delimiter, str, dret, 0); }++#ifndef KSTRING_T+#define KSTRING_T kstring_t+typedef struct __kstring_t {+	size_t l, m;+	char *s;+} kstring_t;+#endif++#ifndef kroundup32+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))+#endif++#define __KS_GETUNTIL(SCOPE, __read) \+	SCOPE int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append)  \+	{ \+		int gotany = 0; \+		if (dret) *dret = 0; \+		str->l = append? str->l : 0; \+        uint64_t seek_pos = str->l; \+		for (;;) { \+			int i; \+			if (ks->begin >= ks->end) { \+				if (!ks->is_eof) { \+					ks->begin = 0; \+					ks->end = __read(ks->f, ks->buf, ks->bufsize); \+					if (ks->end == 0) { ks->is_eof = 1; break; } \+				} else break; \+			} \+			if (delimiter == KS_SEP_LINE) {  \+				for (i = ks->begin; i < ks->end; ++i)  \+					if (ks->buf[i] == '\n') break; \+			} else if (delimiter > KS_SEP_MAX) { \+				for (i = ks->begin; i < ks->end; ++i) \+					if (ks->buf[i] == delimiter) break; \+			} else if (delimiter == KS_SEP_SPACE) { \+				for (i = ks->begin; i < ks->end; ++i) \+					if (isspace(ks->buf[i])) break; \+			} else if (delimiter == KS_SEP_TAB) { \+				for (i = ks->begin; i < ks->end; ++i) \+					if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break;  \+			} else i = 0; /* never come to here! */ \+			if (str->m - str->l < (size_t)(i - ks->begin + 1)) { \+				str->m = str->l + (i - ks->begin) + 1; \+				kroundup32(str->m); \+				str->s = (char*)realloc(str->s, str->m); \+			} \+            seek_pos += i - ks->begin; if ( i < ks->end ) seek_pos++; \+			gotany = 1; \+			memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin);  \+			str->l = str->l + (i - ks->begin); \+			ks->begin = i + 1; \+			if (i < ks->end) { \+				if (dret) *dret = ks->buf[i]; \+				break; \+			} \+		} \+		if (!gotany && ks_eof(ks)) return -1; \+        ks->seek_pos += seek_pos; \+		if (str->s == 0) { \+			str->m = 1; \+			str->s = (char*)calloc(1, 1); \+		} else if (delimiter == KS_SEP_LINE && str->l > 1 && str->s[str->l-1] == '\r') --str->l; \+		str->s[str->l] = '\0';											\+		return str->l; \+	}++#define KSTREAM_INIT2(SCOPE, type_t, __read, __bufsize) \+	__KS_TYPE(type_t) \+	__KS_BASIC(SCOPE, type_t, __bufsize) \+	__KS_GETUNTIL(SCOPE, __read) \+	__KS_INLINED(__read)++#define KSTREAM_INIT(type_t, __read, __bufsize) KSTREAM_INIT2(static, type_t, __read, __bufsize)++#define KSTREAM_DECLARE(type_t, __read) \+	__KS_TYPE(type_t) \+	extern int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append); \+	extern kstream_t *ks_init(type_t f); \+	extern void ks_destroy(kstream_t *ks); \+	__KS_INLINED(__read)++/******************+ * FASTA/Q parser *+ ******************/++#define kseq_rewind(ks) ((ks)->last_char = (ks)->f->is_eof = (ks)->f->begin = (ks)->f->end = 0)++#define __KSEQ_BASIC(SCOPE, type_t)										\+	SCOPE kseq_t *kseq_init(type_t fd)									\+	{																	\+		kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t));					\+		s->f = ks_init(fd);												\+		return s;														\+	}																	\+	SCOPE void kseq_destroy(kseq_t *ks)									\+	{																	\+		if (!ks) return;												\+		free(ks->name.s); free(ks->comment.s); free(ks->seq.s);	free(ks->qual.s); \+		ks_destroy(ks->f);												\+		free(ks);														\+	}++/* Return value:+   >=0  length of the sequence (normal)+   -1   end-of-file+   -2   truncated quality string+ */+#define __KSEQ_READ(SCOPE) \+	SCOPE int kseq_read(kseq_t *seq) \+	{ \+		int c; \+		kstream_t *ks = seq->f; \+		if (seq->last_char == 0) { /* then jump to the next header line */ \+			while ((c = ks_getc(ks)) != -1 && c != '>' && c != '@'); \+			if (c == -1) return -1; /* end of file */ \+			seq->last_char = c; \+		} /* else: the first header char has been read in the previous call */ \+		seq->comment.l = seq->seq.l = seq->qual.l = 0; /* reset all members */ \+		if (ks_getuntil(ks, 0, &seq->name, &c) < 0) return -1; /* normal exit: EOF */ \+		if (c != '\n') ks_getuntil(ks, KS_SEP_LINE, &seq->comment, 0); /* read FASTA/Q comment */ \+		if (seq->seq.s == 0) { /* we can do this in the loop below, but that is slower */ \+			seq->seq.m = 256; \+			seq->seq.s = (char*)malloc(seq->seq.m); \+		} \+		while ((c = ks_getc(ks)) != -1 && c != '>' && c != '+' && c != '@') { \+			if (c == '\n') continue; /* skip empty lines */ \+			seq->seq.s[seq->seq.l++] = c; /* this is safe: we always have enough space for 1 char */ \+			ks_getuntil2(ks, KS_SEP_LINE, &seq->seq, 0, 1); /* read the rest of the line */ \+		} \+		if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */	\+		if (seq->seq.l + 1 >= seq->seq.m) { /* seq->seq.s[seq->seq.l] below may be out of boundary */ \+			seq->seq.m = seq->seq.l + 2; \+			kroundup32(seq->seq.m); /* rounded to the next closest 2^k */ \+			seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \+		} \+		seq->seq.s[seq->seq.l] = 0;	/* null terminated string */ \+		if (c != '+') return seq->seq.l; /* FASTA */ \+		if (seq->qual.m < seq->seq.m) {	/* allocate memory for qual in case insufficient */ \+			seq->qual.m = seq->seq.m; \+			seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \+		} \+		while ((c = ks_getc(ks)) != -1 && c != '\n'); /* skip the rest of '+' line */ \+		if (c == -1) return -2; /* error: no quality string */ \+		while (ks_getuntil2(ks, KS_SEP_LINE, &seq->qual, 0, 1) >= 0 && seq->qual.l < seq->seq.l); \+		seq->last_char = 0;	/* we have not come to the next header line */ \+		if (seq->seq.l != seq->qual.l) return -2; /* error: qual string is of a different length */ \+		return seq->seq.l; \+	}++#define __KSEQ_TYPE(type_t)						\+	typedef struct {							\+		kstring_t name, comment, seq, qual;		\+		int last_char;							\+		kstream_t *f;							\+	} kseq_t;++#define KSEQ_INIT2(SCOPE, type_t, __read)		\+	KSTREAM_INIT(type_t, __read, 16384)			\+	__KSEQ_TYPE(type_t)							\+	__KSEQ_BASIC(SCOPE, type_t)					\+	__KSEQ_READ(SCOPE)++#define KSEQ_INIT(type_t, __read) KSEQ_INIT2(static, type_t, __read)++#define KSEQ_DECLARE(type_t) \+	__KS_TYPE(type_t) \+	__KSEQ_TYPE(type_t) \+	extern kseq_t *kseq_init(type_t fd); \+	void kseq_destroy(kseq_t *ks); \+	int kseq_read(kseq_t *seq);++#endif
+ htslib-1.3.2/htslib/ksort.h view
@@ -0,0 +1,285 @@+/* The MIT License++   Copyright (c) 2008 Genome Research Ltd (GRL).++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++/* Contact: Heng Li <lh3@sanger.ac.uk> */++/*+  2012-12-11 (0.1.4):++    * Defined __ks_insertsort_##name as static to compile with C99.++  2008-11-16 (0.1.4):++    * Fixed a bug in introsort() that happens in rare cases.++  2008-11-05 (0.1.3):++    * Fixed a bug in introsort() for complex comparisons.++	* Fixed a bug in mergesort(). The previous version is not stable.++  2008-09-15 (0.1.2):++	* Accelerated introsort. On my Mac (not on another Linux machine),+	  my implementation is as fast as std::sort on random input.++	* Added combsort and in introsort, switch to combsort if the+	  recursion is too deep.++  2008-09-13 (0.1.1):++	* Added k-small algorithm++  2008-09-05 (0.1.0):++	* Initial version++*/++#ifndef AC_KSORT_H+#define AC_KSORT_H++#include <stdlib.h>+#include <string.h>++typedef struct {+	void *left, *right;+	int depth;+} ks_isort_stack_t;++#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; }++#define KSORT_INIT(name, type_t, __sort_lt)								\+	void ks_mergesort_##name(size_t n, type_t array[], type_t temp[])	\+	{																	\+		type_t *a2[2], *a, *b;											\+		int curr, shift;												\+																		\+		a2[0] = array;													\+		a2[1] = temp? temp : (type_t*)malloc(sizeof(type_t) * n);		\+		for (curr = 0, shift = 0; (1ul<<shift) < n; ++shift) {			\+			a = a2[curr]; b = a2[1-curr];								\+			if (shift == 0) {											\+				type_t *p = b, *i, *eb = a + n;							\+				for (i = a; i < eb; i += 2) {							\+					if (i == eb - 1) *p++ = *i;							\+					else {												\+						if (__sort_lt(*(i+1), *i)) {					\+							*p++ = *(i+1); *p++ = *i;					\+						} else {										\+							*p++ = *i; *p++ = *(i+1);					\+						}												\+					}													\+				}														\+			} else {													\+				size_t i, step = 1ul<<shift;							\+				for (i = 0; i < n; i += step<<1) {						\+					type_t *p, *j, *k, *ea, *eb;						\+					if (n < i + step) {									\+						ea = a + n; eb = a;								\+					} else {											\+						ea = a + i + step;								\+						eb = a + (n < i + (step<<1)? n : i + (step<<1)); \+					}													\+					j = a + i; k = a + i + step; p = b + i;				\+					while (j < ea && k < eb) {							\+						if (__sort_lt(*k, *j)) *p++ = *k++;				\+						else *p++ = *j++;								\+					}													\+					while (j < ea) *p++ = *j++;							\+					while (k < eb) *p++ = *k++;							\+				}														\+			}															\+			curr = 1 - curr;											\+		}																\+		if (curr == 1) {												\+			type_t *p = a2[0], *i = a2[1], *eb = array + n;				\+			for (; p < eb; ++i) *p++ = *i;								\+		}																\+		if (temp == 0) free(a2[1]);										\+	}																	\+	void ks_heapadjust_##name(size_t i, size_t n, type_t l[])			\+	{																	\+		size_t k = i;													\+		type_t tmp = l[i];												\+		while ((k = (k << 1) + 1) < n) {								\+			if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k;				\+			if (__sort_lt(l[k], tmp)) break;							\+			l[i] = l[k]; i = k;											\+		}																\+		l[i] = tmp;														\+	}																	\+	void ks_heapmake_##name(size_t lsize, type_t l[])					\+	{																	\+		size_t i;														\+		for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i)				\+			ks_heapadjust_##name(i, lsize, l);							\+	}																	\+	void ks_heapsort_##name(size_t lsize, type_t l[])					\+	{																	\+		size_t i;														\+		for (i = lsize - 1; i > 0; --i) {								\+			type_t tmp;													\+			tmp = *l; *l = l[i]; l[i] = tmp; ks_heapadjust_##name(0, i, l); \+		}																\+	}																	\+	static inline void __ks_insertsort_##name(type_t *s, type_t *t)		\+	{																	\+		type_t *i, *j, swap_tmp;										\+		for (i = s + 1; i < t; ++i)										\+			for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) {			\+				swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp;			\+			}															\+	}																	\+	void ks_combsort_##name(size_t n, type_t a[])						\+	{																	\+		const double shrink_factor = 1.2473309501039786540366528676643; \+		int do_swap;													\+		size_t gap = n;													\+		type_t tmp, *i, *j;												\+		do {															\+			if (gap > 2) {												\+				gap = (size_t)(gap / shrink_factor);					\+				if (gap == 9 || gap == 10) gap = 11;					\+			}															\+			do_swap = 0;												\+			for (i = a; i < a + n - gap; ++i) {							\+				j = i + gap;											\+				if (__sort_lt(*j, *i)) {								\+					tmp = *i; *i = *j; *j = tmp;						\+					do_swap = 1;										\+				}														\+			}															\+		} while (do_swap || gap > 2);									\+		if (gap != 1) __ks_insertsort_##name(a, a + n);					\+	}																	\+	void ks_introsort_##name(size_t n, type_t a[])						\+	{																	\+		int d;															\+		ks_isort_stack_t *top, *stack;									\+		type_t rp, swap_tmp;											\+		type_t *s, *t, *i, *j, *k;										\+																		\+		if (n < 1) return;												\+		else if (n == 2) {												\+			if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \+			return;														\+		}																\+		for (d = 2; 1ul<<d < n; ++d);									\+		stack = (ks_isort_stack_t*)malloc(sizeof(ks_isort_stack_t) * ((sizeof(size_t)*d)+2)); \+		top = stack; s = a; t = a + (n-1); d <<= 1;						\+		while (1) {														\+			if (s < t) {												\+				if (--d == 0) {											\+					ks_combsort_##name(t - s + 1, s);					\+					t = s;												\+					continue;											\+				}														\+				i = s; j = t; k = i + ((j-i)>>1) + 1;					\+				if (__sort_lt(*k, *i)) {								\+					if (__sort_lt(*k, *j)) k = j;						\+				} else k = __sort_lt(*j, *i)? i : j;					\+				rp = *k;												\+				if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; }	\+				for (;;) {												\+					do ++i; while (__sort_lt(*i, rp));					\+					do --j; while (i <= j && __sort_lt(rp, *j));		\+					if (j <= i) break;									\+					swap_tmp = *i; *i = *j; *j = swap_tmp;				\+				}														\+				swap_tmp = *i; *i = *t; *t = swap_tmp;					\+				if (i-s > t-i) {										\+					if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \+					s = t-i > 16? i+1 : t;								\+				} else {												\+					if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \+					t = i-s > 16? i-1 : s;								\+				}														\+			} else {													\+				if (top == stack) {										\+					free(stack);										\+					__ks_insertsort_##name(a, a+n);						\+					return;												\+				} else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \+			}															\+		}																\+	}																	\+	/* This function is adapted from: http://ndevilla.free.fr/median/ */ \+	/* 0 <= kk < n */													\+	type_t ks_ksmall_##name(size_t n, type_t arr[], size_t kk)			\+	{																	\+		type_t *low, *high, *k, *ll, *hh, *mid;							\+		low = arr; high = arr + n - 1; k = arr + kk;					\+		for (;;) {														\+			if (high <= low) return *k;									\+			if (high == low + 1) {										\+				if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \+				return *k;												\+			}															\+			mid = low + (high - low) / 2;								\+			if (__sort_lt(*high, *mid)) KSORT_SWAP(type_t, *mid, *high); \+			if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \+			if (__sort_lt(*low, *mid)) KSORT_SWAP(type_t, *mid, *low);	\+			KSORT_SWAP(type_t, *mid, *(low+1));							\+			ll = low + 1; hh = high;									\+			for (;;) {													\+				do ++ll; while (__sort_lt(*ll, *low));					\+				do --hh; while (__sort_lt(*low, *hh));					\+				if (hh < ll) break;										\+				KSORT_SWAP(type_t, *ll, *hh);							\+			}															\+			KSORT_SWAP(type_t, *low, *hh);								\+			if (hh <= k) low = ll;										\+			if (hh >= k) high = hh - 1;									\+		}																\+	}																	\+	void ks_shuffle_##name(size_t n, type_t a[])						\+	{																	\+		int i, j;														\+		for (i = n; i > 1; --i) {										\+			type_t tmp;													\+			j = (int)(drand48() * i);									\+			tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;					\+		}																\+	}++#define ks_mergesort(name, n, a, t) ks_mergesort_##name(n, a, t)+#define ks_introsort(name, n, a) ks_introsort_##name(n, a)+#define ks_combsort(name, n, a) ks_combsort_##name(n, a)+#define ks_heapsort(name, n, a) ks_heapsort_##name(n, a)+#define ks_heapmake(name, n, a) ks_heapmake_##name(n, a)+#define ks_heapadjust(name, i, n, a) ks_heapadjust_##name(i, n, a)+#define ks_ksmall(name, n, a, k) ks_ksmall_##name(n, a, k)+#define ks_shuffle(name, n, a) ks_shuffle_##name(n, a)++#define ks_lt_generic(a, b) ((a) < (b))+#define ks_lt_str(a, b) (strcmp((a), (b)) < 0)++typedef const char *ksstr_t;++#define KSORT_INIT_GENERIC(type_t) KSORT_INIT(type_t, type_t, ks_lt_generic)+#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str)++#endif
+ htslib-1.3.2/htslib/kstring.h view
@@ -0,0 +1,277 @@+/* The MIT License++   Copyright (C) 2011 by Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#ifndef KSTRING_H+#define KSTRING_H++#include <stdlib.h>+#include <string.h>+#include <stdarg.h>+#include <stdint.h>+#include <stdio.h>++#ifndef kroundup32+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))+#endif++#if defined __GNUC__ && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4))+#define KS_ATTR_PRINTF(fmt, arg) __attribute__((__format__ (__printf__, fmt, arg)))+#else+#define KS_ATTR_PRINTF(fmt, arg)+#endif+++/* kstring_t is a simple non-opaque type whose fields are likely to be+ * used directly by user code (but see also ks_str() and ks_len() below).+ * A kstring_t object is initialised by either of+ *       kstring_t str = { 0, 0, NULL };+ *       kstring_t str; ...; str.l = str.m = 0; str.s = NULL;+ * and either ownership of the underlying buffer should be given away before+ * the object disappears (see ks_release() below) or the kstring_t should be+ * destroyed with  free(str.s);  */+#ifndef KSTRING_T+#define KSTRING_T kstring_t+typedef struct __kstring_t {+	size_t l, m;+	char *s;+} kstring_t;+#endif++typedef struct {+	uint64_t tab[4];+	int sep, finished;+	const char *p; // end of the current token+} ks_tokaux_t;++#ifdef __cplusplus+extern "C" {+#endif++	int kvsprintf(kstring_t *s, const char *fmt, va_list ap) KS_ATTR_PRINTF(2,0);+	int ksprintf(kstring_t *s, const char *fmt, ...) KS_ATTR_PRINTF(2,3);+	int ksplit_core(char *s, int delimiter, int *_max, int **_offsets);+	char *kstrstr(const char *str, const char *pat, int **_prep);+	char *kstrnstr(const char *str, const char *pat, int n, int **_prep);+	void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep);++	/* kstrtok() is similar to strtok_r() except that str is not+	 * modified and both str and sep can be NULL. For efficiency, it is+	 * actually recommended to set both to NULL in the subsequent calls+	 * if sep is not changed. */+	char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux);++	/* kgetline() uses the supplied fgets()-like function to read a "\n"-+	 * or "\r\n"-terminated line from fp.  The line read is appended to the+	 * kstring without its terminator and 0 is returned; EOF is returned at+	 * EOF or on error (determined by querying fp, as per fgets()). */+	typedef char *kgets_func(char *, int, void *);+	int kgetline(kstring_t *s, kgets_func *fgets, void *fp);++#ifdef __cplusplus+}+#endif++static inline int ks_resize(kstring_t *s, size_t size)+{+	if (s->m < size) {+		char *tmp;+		s->m = size;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return -1;+	}+	return 0;+}++static inline char *ks_str(kstring_t *s)+{+	return s->s;+}++static inline size_t ks_len(kstring_t *s)+{+	return s->l;+}++// Give ownership of the underlying buffer away to something else (making+// that something else responsible for freeing it), leaving the kstring_t+// empty and ready to be used again, or ready to go out of scope without+// needing  free(str.s)  to prevent a memory leak.+static inline char *ks_release(kstring_t *s)+{+	char *ss = s->s;+	s->l = s->m = 0;+	s->s = NULL;+	return ss;+}++static inline int kputsn(const char *p, int l, kstring_t *s)+{+	if (s->l + l + 1 >= s->m) {+		char *tmp;+		s->m = s->l + l + 2;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	memcpy(s->s + s->l, p, l);+	s->l += l;+	s->s[s->l] = 0;+	return l;+}++static inline int kputs(const char *p, kstring_t *s)+{+	return kputsn(p, strlen(p), s);+}++static inline int kputc(int c, kstring_t *s)+{+	if (s->l + 1 >= s->m) {+		char *tmp;+		s->m = s->l + 2;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	s->s[s->l++] = c;+	s->s[s->l] = 0;+	return c;+}++static inline int kputc_(int c, kstring_t *s)+{+	if (s->l + 1 > s->m) {+		char *tmp;+		s->m = s->l + 1;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	s->s[s->l++] = c;+	return 1;+}++static inline int kputsn_(const void *p, int l, kstring_t *s)+{+	if (s->l + l > s->m) {+		char *tmp;+		s->m = s->l + l;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	memcpy(s->s + s->l, p, l);+	s->l += l;+	return l;+}++static inline int kputw(int c, kstring_t *s)+{+	char buf[16];+	int i, l = 0;+	unsigned int x = c;+	if (c < 0) x = -x;+	do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);+	if (c < 0) buf[l++] = '-';+	if (s->l + l + 1 >= s->m) {+		char *tmp;+		s->m = s->l + l + 2;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];+	s->s[s->l] = 0;+	return 0;+}++static inline int kputuw(unsigned c, kstring_t *s)+{+	char buf[16];+	int l, i;+	unsigned x;+	if (c == 0) return kputc('0', s);+	for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0';+	if (s->l + l + 1 >= s->m) {+		char *tmp;+		s->m = s->l + l + 2;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];+	s->s[s->l] = 0;+	return 0;+}++static inline int kputl(long c, kstring_t *s)+{+	char buf[32];+	int i, l = 0;+	unsigned long x = c;+	if (c < 0) x = -x;+	do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);+	if (c < 0) buf[l++] = '-';+	if (s->l + l + 1 >= s->m) {+		char *tmp;+		s->m = s->l + l + 2;+		kroundup32(s->m);+		if ((tmp = (char*)realloc(s->s, s->m)))+			s->s = tmp;+		else+			return EOF;+	}+	for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];+	s->s[s->l] = 0;+	return 0;+}++/*+ * Returns 's' split by delimiter, with *n being the number of components;+ *         NULL on failue.+ */+static inline int *ksplit(kstring_t *s, int delimiter, int *n)+{+	int max = 0, *offsets = 0;+	*n = ksplit_core(s->s, delimiter, &max, &offsets);+	return offsets;+}++#endif
+ htslib-1.3.2/htslib/regidx.h view
@@ -0,0 +1,154 @@+/* +    Copyright (C) 2014 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++    Permission is hereby granted, free of charge, to any person obtaining a copy+    of this software and associated documentation files (the "Software"), to deal+    in the Software without restriction, including without limitation the rights+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+    copies of the Software, and to permit persons to whom the Software is+    furnished to do so, subject to the following conditions:+    +    The above copyright notice and this permission notice shall be included in+    all copies or substantial portions of the Software.+    +    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+    THE SOFTWARE.+*/++/*+    Regions indexing with an optional payload. Inspired by samtools/bedidx.c.+    This code is intended as future replacement of bcf_sr_regions_t.++    Example of usage:++        // Init the parser and print regions. In this example the payload is a+        // pointer to a string. For the description of parse_custom and+        // free_custom functions, see regidx_parse_f and regidx_free_f below,+        // and for working example see test/test-regidx.c.+        regidx_t *idx = regidx_init(in_fname,parse_custom,free_custom,sizeof(char*),NULL);++        // Query overlap with chr:from-to+        regitr_t itr;+        if ( regidx_overlap(idx, chr,from,to, &itr) ) printf("There is an overlap!\n");++        while ( REGITR_OVERLAP(itr,from,to) )+        {+            printf("[%d,%d] overlaps with [%d,%d], payload=%s\n", from,to, +                REGITR_START(itr), REGITR_END(itr), REGITR_PAYLOAD(itr,char*));+            itr.i++;+        }++        regidx_destroy(regs);+*/++#ifndef HTSLIB_REGIDX_H+#define HTSLIB_REGIDX_H++#include <stdio.h>+#include <inttypes.h>++#ifdef __cplusplus+extern "C" {+#endif++typedef struct _regidx_t regidx_t;+typedef struct+{+    uint32_t start, end;+}+reg_t;+typedef struct+{+    int i, n;+    reg_t *reg;+    void *payload;+}+regitr_t;++#define REGITR_START(itr) (itr).reg[(itr).i].start+#define REGITR_END(itr)   (itr).reg[(itr).i].end+#define REGITR_PAYLOAD(itr,type_t) ((type_t*)(itr).payload)[(itr).i]+#define REGITR_OVERLAP(itr,from,to) (itr.i < itr.n && REGITR_START(itr)<=to && REGITR_END(itr)>=from )++/*+ *  regidx_parse_f - Function to parse one input line, such as regidx_parse_bed+ *  or regidx_parse_tab below. The function is expected to set `chr_from` and+ *  `chr_to` to point to first and last character of chromosome name and set+ *  coordinates `reg->start` and `reg->end` (0-based, inclusive). If+ *  regidx_init() was called with non-zero payload_size, the `payload` points+ *  to a memory location of the payload_size and `usr` is data passed to+ *  regidx_init(). Any memory allocated by the function will be freed by+ *  regidx_free_f on regidx_destroy().+ *+ *  Return value: 0 on success, -1 to skip a record, -2 on fatal error.+ */+typedef int  (*regidx_parse_f)(const char *line, char **chr_beg, char **chr_end, reg_t *reg, void *payload, void *usr);+typedef void (*regidx_free_f)(void *payload);++int regidx_parse_bed(const char*,char**,char**,reg_t*,void*,void*);   // CHROM,FROM,TO (0-based,right-open)+int regidx_parse_tab(const char*,char**,char**,reg_t*,void*,void*);   // CHROM,POS (1-based, inclusive)++/*+ *  regidx_init() - creates new index+ *  @param fname:  input file name or NULL if regions will be added one-by-one via regidx_insert()+ *  @param parsef: regidx_parse_bed, regidx_parse_tab or see description of regidx_parse_f. If NULL,+ *                 the format will be autodected, currently either regidx_parse_tab (the default) or+ *                 regidx_parse_bed (file must be named 'bed' or 'bed.gz') will be used. Note that+ *                 the exact autodetection algorithm will change.+ *  @param freef:  NULL or see description of regidx_parse_f+ *  @param payload_size: 0 with regidx_parse_bed, regidx_parse_tab or see regidx_parse_f+ *  @param usr:    optional user data passed to regidx_parse_f+ *+ *  Returns index on success or NULL on error.+ */+regidx_t *regidx_init(const char *fname, regidx_parse_f parsef, regidx_free_f freef, size_t payload_size, void *usr);++/*+ *  regidx_destroy() - free memory allocated by regidx_init+ */+void regidx_destroy(regidx_t *idx);++/*+ *  regidx_overlap() - check overlap of the location chr:from-to with regions+ *  @param start,end:   0-based start, end coordinate (inclusive)+ *  @param itr:         pointer to iterator, can be NULL if not needed+ *+ *  Returns 0 if there is no overlap or 1 if overlap is found. The overlapping+ *  regions can be iterated as shown in the example above.+ */+int regidx_overlap(regidx_t *idx, const char *chr, uint32_t start, uint32_t end, regitr_t *itr);++/*+ *  regidx_insert() - add a new region. + *+ *  After last region has been added, call regidx_insert(idx,NULL) to+ *  build the index.+ *+ *  Returns 0 on success or -1 on error.+ */+int regidx_insert(regidx_t *idx, char *line);++/*+ *  regidx_seq_names() - return list of all sequence names+ */+char **regidx_seq_names(regidx_t *idx, int *n);++/*+ *  regidx_seq_nregs() - number of regions+ *  regidx_nregs()  - total number of regions+ */+int regidx_seq_nregs(regidx_t *idx, const char *seq);+int regidx_nregs(regidx_t *idx);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/sam.h view
@@ -0,0 +1,454 @@+/*  sam.h -- SAM and BAM file I/O and manipulation.++    Copyright (C) 2008, 2009, 2013-2014 Genome Research Ltd.+    Copyright (C) 2010, 2012, 2013 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_SAM_H+#define HTSLIB_SAM_H++#include <stdint.h>+#include "hts.h"++#ifdef __cplusplus+extern "C" {+#endif++/**********************+ *** SAM/BAM header ***+ **********************/++/*! @typedef+ @abstract Structure for the alignment header.+ @field n_targets   number of reference sequences+ @field l_text      length of the plain text in the header+ @field target_len  lengths of the reference sequences+ @field target_name names of the reference sequences+ @field text        plain text+ @field sdict       header dictionary+ */++typedef struct {+    int32_t n_targets, ignore_sam_err;+    uint32_t l_text;+    uint32_t *target_len;+    int8_t *cigar_tab;+    char **target_name;+    char *text;+    void *sdict;+} bam_hdr_t;++/****************************+ *** CIGAR related macros ***+ ****************************/++#define BAM_CMATCH      0+#define BAM_CINS        1+#define BAM_CDEL        2+#define BAM_CREF_SKIP   3+#define BAM_CSOFT_CLIP  4+#define BAM_CHARD_CLIP  5+#define BAM_CPAD        6+#define BAM_CEQUAL      7+#define BAM_CDIFF       8+#define BAM_CBACK       9++#define BAM_CIGAR_STR   "MIDNSHP=XB"+#define BAM_CIGAR_SHIFT 4+#define BAM_CIGAR_MASK  0xf+#define BAM_CIGAR_TYPE  0x3C1A7++#define bam_cigar_op(c) ((c)&BAM_CIGAR_MASK)+#define bam_cigar_oplen(c) ((c)>>BAM_CIGAR_SHIFT)+#define bam_cigar_opchr(c) (BAM_CIGAR_STR[bam_cigar_op(c)])+#define bam_cigar_gen(l, o) ((l)<<BAM_CIGAR_SHIFT|(o))++/* bam_cigar_type returns a bit flag with:+ *   bit 1 set if the cigar operation consumes the query+ *   bit 2 set if the cigar operation consumes the reference+ *+ * For reference, the unobfuscated truth table for this function is:+ * BAM_CIGAR_TYPE  QUERY  REFERENCE+ * --------------------------------+ * BAM_CMATCH      1      1+ * BAM_CINS        1      0+ * BAM_CDEL        0      1+ * BAM_CREF_SKIP   0      1+ * BAM_CSOFT_CLIP  1      0+ * BAM_CHARD_CLIP  0      0+ * BAM_CPAD        0      0+ * BAM_CEQUAL      1      1+ * BAM_CDIFF       1      1+ * BAM_CBACK       0      0+ * --------------------------------+ */+#define bam_cigar_type(o) (BAM_CIGAR_TYPE>>((o)<<1)&3) // bit 1: consume query; bit 2: consume reference++/*! @abstract the read is paired in sequencing, no matter whether it is mapped in a pair */+#define BAM_FPAIRED        1+/*! @abstract the read is mapped in a proper pair */+#define BAM_FPROPER_PAIR   2+/*! @abstract the read itself is unmapped; conflictive with BAM_FPROPER_PAIR */+#define BAM_FUNMAP         4+/*! @abstract the mate is unmapped */+#define BAM_FMUNMAP        8+/*! @abstract the read is mapped to the reverse strand */+#define BAM_FREVERSE      16+/*! @abstract the mate is mapped to the reverse strand */+#define BAM_FMREVERSE     32+/*! @abstract this is read1 */+#define BAM_FREAD1        64+/*! @abstract this is read2 */+#define BAM_FREAD2       128+/*! @abstract not primary alignment */+#define BAM_FSECONDARY   256+/*! @abstract QC failure */+#define BAM_FQCFAIL      512+/*! @abstract optical or PCR duplicate */+#define BAM_FDUP        1024+/*! @abstract supplementary alignment */+#define BAM_FSUPPLEMENTARY 2048++/*************************+ *** Alignment records ***+ *************************/++/*! @typedef+ @abstract Structure for core alignment information.+ @field  tid     chromosome ID, defined by bam_hdr_t+ @field  pos     0-based leftmost coordinate+ @field  bin     bin calculated by bam_reg2bin()+ @field  qual    mapping quality+ @field  l_qname length of the query name+ @field  flag    bitwise flag+ @field  n_cigar number of CIGAR operations+ @field  l_qseq  length of the query sequence (read)+ @field  mtid    chromosome ID of next read in template, defined by bam_hdr_t+ @field  mpos    0-based leftmost coordinate of next read in template+ */+typedef struct {+    int32_t tid;+    int32_t pos;+    uint32_t bin:16, qual:8, l_qname:8;+    uint32_t flag:16, n_cigar:16;+    int32_t l_qseq;+    int32_t mtid;+    int32_t mpos;+    int32_t isize;+} bam1_core_t;++/*! @typedef+ @abstract Structure for one alignment.+ @field  core       core information about the alignment+ @field  l_data     current length of bam1_t::data+ @field  m_data     maximum length of bam1_t::data+ @field  data       all variable-length data, concatenated; structure: qname-cigar-seq-qual-aux++ @discussion Notes:++ 1. qname is zero tailing and core.l_qname includes the tailing '\0'.+ 2. l_qseq is calculated from the total length of an alignment block+ on reading or from CIGAR.+ 3. cigar data is encoded 4 bytes per CIGAR operation.+ 4. seq is nybble-encoded according to bam_nt16_table.+ */+typedef struct {+    bam1_core_t core;+    int l_data, m_data;+    uint8_t *data;+#ifndef BAM_NO_ID+    uint64_t id;+#endif+} bam1_t;++/*! @function+ @abstract  Get whether the query is on the reverse strand+ @param  b  pointer to an alignment+ @return    boolean true if query is on the reverse strand+ */+#define bam_is_rev(b) (((b)->core.flag&BAM_FREVERSE) != 0)+/*! @function+ @abstract  Get whether the query's mate is on the reverse strand+ @param  b  pointer to an alignment+ @return    boolean true if query's mate on the reverse strand+ */+#define bam_is_mrev(b) (((b)->core.flag&BAM_FMREVERSE) != 0)+/*! @function+ @abstract  Get the name of the query+ @param  b  pointer to an alignment+ @return    pointer to the name string, null terminated+ */+#define bam_get_qname(b) ((char*)(b)->data)+/*! @function+ @abstract  Get the CIGAR array+ @param  b  pointer to an alignment+ @return    pointer to the CIGAR array++ @discussion In the CIGAR array, each element is a 32-bit integer. The+ lower 4 bits gives a CIGAR operation and the higher 28 bits keep the+ length of a CIGAR.+ */+#define bam_get_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname))+/*! @function+ @abstract  Get query sequence+ @param  b  pointer to an alignment+ @return    pointer to sequence++ @discussion Each base is encoded in 4 bits: 1 for A, 2 for C, 4 for G,+ 8 for T and 15 for N. Two bases are packed in one byte with the base+ at the higher 4 bits having smaller coordinate on the read. It is+ recommended to use bam_seqi() macro to get the base.+ */+#define bam_get_seq(b)   ((b)->data + ((b)->core.n_cigar<<2) + (b)->core.l_qname)+/*! @function+ @abstract  Get query quality+ @param  b  pointer to an alignment+ @return    pointer to quality string+ */+#define bam_get_qual(b)  ((b)->data + ((b)->core.n_cigar<<2) + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1))+/*! @function+ @abstract  Get auxiliary data+ @param  b  pointer to an alignment+ @return    pointer to the concatenated auxiliary data+ */+#define bam_get_aux(b)   ((b)->data + ((b)->core.n_cigar<<2) + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1) + (b)->core.l_qseq)+/*! @function+ @abstract  Get length of auxiliary data+ @param  b  pointer to an alignment+ @return    length of the concatenated auxiliary data+ */+#define bam_get_l_aux(b) ((b)->l_data - ((b)->core.n_cigar<<2) - (b)->core.l_qname - (b)->core.l_qseq - (((b)->core.l_qseq + 1)>>1))+/*! @function+ @abstract  Get a base on read+ @param  s  Query sequence returned by bam_get_seq()+ @param  i  The i-th position, 0-based+ @return    4-bit integer representing the base.+ */+#define bam_seqi(s, i) ((s)[(i)>>1] >> ((~(i)&1)<<2) & 0xf)++/**************************+ *** Exported functions ***+ **************************/++    /***************+     *** BAM I/O ***+     ***************/++    bam_hdr_t *bam_hdr_init(void);+    bam_hdr_t *bam_hdr_read(BGZF *fp);+    int bam_hdr_write(BGZF *fp, const bam_hdr_t *h) HTS_RESULT_USED;+    void bam_hdr_destroy(bam_hdr_t *h);+    int bam_name2id(bam_hdr_t *h, const char *ref);+    bam_hdr_t* bam_hdr_dup(const bam_hdr_t *h0);++    bam1_t *bam_init1(void);+    void bam_destroy1(bam1_t *b);+    int bam_read1(BGZF *fp, bam1_t *b) HTS_RESULT_USED;+    int bam_write1(BGZF *fp, const bam1_t *b) HTS_RESULT_USED;+    bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc);+    bam1_t *bam_dup1(const bam1_t *bsrc);++    int bam_cigar2qlen(int n_cigar, const uint32_t *cigar);+    int bam_cigar2rlen(int n_cigar, const uint32_t *cigar);++    /*!+      @abstract Calculate the rightmost base position of an alignment on the+      reference genome.++      @param  b  pointer to an alignment+      @return    the coordinate of the first base after the alignment, 0-based++      @discussion For a mapped read, this is just b->core.pos + bam_cigar2rlen.+      For an unmapped read (either according to its flags or if it has no cigar+      string), we return b->core.pos + 1 by convention.+    */+    int32_t bam_endpos(const bam1_t *b);++    int   bam_str2flag(const char *str);    /** returns negative value on error */+    char *bam_flag2str(int flag);   /** The string must be freed by the user */++    /*************************+     *** BAM/CRAM indexing ***+     *************************/++    // These BAM iterator functions work only on BAM files.  To work with either+    // BAM or CRAM files use the sam_index_load() & sam_itr_*() functions.+    #define bam_itr_destroy(iter) hts_itr_destroy(iter)+    #define bam_itr_queryi(idx, tid, beg, end) sam_itr_queryi(idx, tid, beg, end)+    #define bam_itr_querys(idx, hdr, region) sam_itr_querys(idx, hdr, region)+    #define bam_itr_next(htsfp, itr, r) hts_itr_next((htsfp)->fp.bgzf, (itr), (r), 0)++// Load/build .csi or .bai BAM index file.  Does not work with CRAM.+// It is recommended to use the sam_index_* functions below instead.+#define bam_index_load(fn) hts_idx_load((fn), HTS_FMT_BAI)+#define bam_index_build(fn, min_shift) (sam_index_build((fn), (min_shift)))++/// Load a BAM (.csi or .bai) or CRAM (.crai) index file+/** @param fp  File handle of the data file whose index is being opened+    @param fn  BAM/CRAM/etc filename to search alongside for the index file+    @return  The index, or NULL if an error occurred.+*/+hts_idx_t *sam_index_load(htsFile *fp, const char *fn);++/// Load a specific BAM (.csi or .bai) or CRAM (.crai) index file+/** @param fp     File handle of the data file whose index is being opened+    @param fn     BAM/CRAM/etc data file filename+    @param fnidx  Index filename, or NULL to search alongside @a fn+    @return  The index, or NULL if an error occurred.+*/+hts_idx_t *sam_index_load2(htsFile *fp, const char *fn, const char *fnidx);++/// Generate and save an index file+/** @param fn        Input BAM/etc filename, to which .csi/etc will be added+    @param min_shift Positive to generate CSI, or 0 to generate BAI+    @return  0 if successful, or negative if an error occurred (usually -1; or+             -2: opening fn failed; -3: format not indexable)+*/+int sam_index_build(const char *fn, int min_shift) HTS_RESULT_USED;++/// Generate and save an index to a specific file+/** @param fn        Input BAM/CRAM/etc filename+    @param fnidx     Output filename, or NULL to add .bai/.csi/etc to @a fn+    @param min_shift Positive to generate CSI, or 0 to generate BAI+    @return  0 if successful, or negative if an error occurred.+*/+int sam_index_build2(const char *fn, const char *fnidx, int min_shift) HTS_RESULT_USED;++    #define sam_itr_destroy(iter) hts_itr_destroy(iter)+    hts_itr_t *sam_itr_queryi(const hts_idx_t *idx, int tid, int beg, int end);+    hts_itr_t *sam_itr_querys(const hts_idx_t *idx, bam_hdr_t *hdr, const char *region);+    #define sam_itr_next(htsfp, itr, r) hts_itr_next((htsfp)->fp.bgzf, (itr), (r), (htsfp))++    /***************+     *** SAM I/O ***+     ***************/++    #define sam_open(fn, mode) (hts_open((fn), (mode)))+    #define sam_open_format(fn, mode, fmt) (hts_open_format((fn), (mode), (fmt)))+    #define sam_close(fp) hts_close(fp)++    int sam_open_mode(char *mode, const char *fn, const char *format);++    // A version of sam_open_mode that can handle ,key=value options.+    // The format string is allocated and returned, to be freed by the caller.+    // Prefix should be "r" or "w",+    char *sam_open_mode_opts(const char *fn,+                             const char *mode,+                             const char *format);++    typedef htsFile samFile;+    bam_hdr_t *sam_hdr_parse(int l_text, const char *text);+    bam_hdr_t *sam_hdr_read(samFile *fp);+    int sam_hdr_write(samFile *fp, const bam_hdr_t *h) HTS_RESULT_USED;++    int sam_parse1(kstring_t *s, bam_hdr_t *h, bam1_t *b) HTS_RESULT_USED;+    int sam_format1(const bam_hdr_t *h, const bam1_t *b, kstring_t *str) HTS_RESULT_USED;+    int sam_read1(samFile *fp, bam_hdr_t *h, bam1_t *b) HTS_RESULT_USED;+    int sam_write1(samFile *fp, const bam_hdr_t *h, const bam1_t *b) HTS_RESULT_USED;++    /*************************************+     *** Manipulating auxiliary fields ***+     *************************************/++    uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]);+    int32_t bam_aux2i(const uint8_t *s);+    double bam_aux2f(const uint8_t *s);+    char bam_aux2A(const uint8_t *s);+    char *bam_aux2Z(const uint8_t *s);++    void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data);+    int bam_aux_del(bam1_t *b, uint8_t *s);++/**************************+ *** Pileup and Mpileup ***+ **************************/++#if !defined(BAM_NO_PILEUP)++/*! @typedef+ @abstract Structure for one alignment covering the pileup position.+ @field  b          pointer to the alignment+ @field  qpos       position of the read base at the pileup site, 0-based+ @field  indel      indel length; 0 for no indel, positive for ins and negative for del+ @field  level      the level of the read in the "viewer" mode+ @field  is_del     1 iff the base on the padded read is a deletion+ @field  is_head    ???+ @field  is_tail    ???+ @field  is_refskip ???+ @field  aux        ???++ @discussion See also bam_plbuf_push() and bam_lplbuf_push(). The+ difference between the two functions is that the former does not+ set bam_pileup1_t::level, while the later does. Level helps the+ implementation of alignment viewers, but calculating this has some+ overhead.+ */+typedef struct {+    bam1_t *b;+    int32_t qpos;+    int indel, level;+    uint32_t is_del:1, is_head:1, is_tail:1, is_refskip:1, aux:28;+} bam_pileup1_t;++typedef int (*bam_plp_auto_f)(void *data, bam1_t *b);++struct __bam_plp_t;+typedef struct __bam_plp_t *bam_plp_t;++struct __bam_mplp_t;+typedef struct __bam_mplp_t *bam_mplp_t;++    /**+     *  bam_plp_init() - sets an iterator over multiple+     *  @func:      see mplp_func in bam_plcmd.c in samtools for an example. Expected return+     *              status: 0 on success, -1 on end, < -1 on non-recoverable errors+     *  @data:      user data to pass to @func+     */+    bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data);+    void bam_plp_destroy(bam_plp_t iter);+    int bam_plp_push(bam_plp_t iter, const bam1_t *b);+    const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);+    const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);+    void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt);+    void bam_plp_reset(bam_plp_t iter);++    bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data);+    /**+     *  bam_mplp_init_overlaps() - if called, mpileup will detect overlapping+     *  read pairs and for each base pair set the base quality of the+     *  lower-quality base to zero, thus effectively discarding it from+     *  calling. If the two bases are identical, the quality of the other base+     *  is increased to the sum of their qualities (capped at 200), otherwise+     *  it is multiplied by 0.8.+     */+    void bam_mplp_init_overlaps(bam_mplp_t iter);+    void bam_mplp_destroy(bam_mplp_t iter);+    void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt);+    int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp);++#endif // ~!defined(BAM_NO_PILEUP)++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/synced_bcf_reader.h view
@@ -0,0 +1,302 @@+/*  synced_bcf_reader.h -- stream through multiple VCF files.++    Copyright (C) 2012-2014 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++/*+    The synced_bcf_reader allows to keep multiple VCFs open and stream them+    using the next_line iterator in a seamless matter without worrying about+    chromosomes and synchronizing the sites. This is used by vcfcheck to+    compare multiple VCFs simultaneously and is used also for merging,+    creating intersections, etc.++    The synced_bcf_reader also provides API for reading indexed BCF/VCF,+    hiding differences in BCF/VCF opening, indexing and reading.+++    Example of usage:++        bcf_srs_t *sr = bcf_sr_init();+        for (i=0; i<nfiles; i++)+            bcf_sr_add_reader(sr,files[i]);+        while ( bcf_sr_next_line(sr) )+        {+            for (i=0; i<nfiles; i++)+            {+                bcf1_t *line = bcf_sr_get_line(sr,i);+                ...+            }+        }+        bcf_sr_destroy(sr);+*/++#ifndef HTSLIB_SYNCED_BCF_READER_H+#define HTSLIB_SYNCED_BCF_READER_H++#include "hts.h"+#include "vcf.h"+#include "tbx.h"++#ifdef __cplusplus+extern "C" {+#endif++// How should be treated sites with the same position but different alleles+#define COLLAPSE_NONE   0   // require the exact same set of alleles in all files+#define COLLAPSE_SNPS   1   // allow different alleles, as long as they all are SNPs+#define COLLAPSE_INDELS 2   // the same as above, but with indels+#define COLLAPSE_ANY    4   // any combination of alleles can be returned by bcf_sr_next_line()+#define COLLAPSE_SOME   8   // at least some of the ALTs must match+#define COLLAPSE_BOTH  (COLLAPSE_SNPS|COLLAPSE_INDELS)++typedef struct _bcf_sr_regions_t+{+    // for reading from tabix-indexed file (big data)+    tbx_t *tbx;             // tabix index+    hts_itr_t *itr;         // tabix iterator+    kstring_t line;         // holder of the current line, set only when reading from tabix-indexed files+    htsFile *file;+    char *fname;+    int is_bin;             // is open in binary mode (tabix access)+    char **als;             // parsed alleles if targets_als set and _regions_match_alleles called+    kstring_t als_str;      // block of parsed alleles+    int nals, mals;         // number of set alleles and the size of allocated array+    int als_type;           // alleles type, currently VCF_SNP or VCF_INDEL++    // user handler to deal with skipped regions without a counterpart in VCFs+    void (*missed_reg_handler)(struct _bcf_sr_regions_t *, void *);+    void *missed_reg_data;++    // for in-memory regions (small data)+    struct _region_t *regs; // the regions++    // shared by both tabix-index and in-memory regions+    void *seq_hash;         // keys: sequence names, values: index to seqs+    char **seq_names;       // sequence names+    int nseqs;              // number of sequences (chromosomes) in the file+    int iseq;               // current position: chr name, index to snames+    int start, end;         // current position: start, end of the region (0-based)+    int prev_seq, prev_start;+}+bcf_sr_regions_t;++typedef struct+{+    htsFile *file;+    tbx_t *tbx_idx;+    hts_idx_t *bcf_idx;+    bcf_hdr_t *header;+    hts_itr_t *itr;+    char *fname;+    bcf1_t **buffer;                // cached VCF records. First is the current record synced across the reader+    int nbuffer, mbuffer;           // number of cached records (including the current record); number of allocated records+    int nfilter_ids, *filter_ids;   // -1 for ".", otherwise filter id as returned by bcf_hdr_id2int+    int *samples, n_smpl;   // list of columns in the order consistent with bcf_srs_t.samples+}+bcf_sr_t;++typedef enum+{+    open_failed, not_bgzf, idx_load_failed, file_type_error, api_usage_error,+    header_error, no_eof+}+bcf_sr_error;++typedef struct+{+    // Parameters controlling the logic+    int collapse;       // How should the duplicate sites be treated. One of the COLLAPSE_* types above.+    char *apply_filters;    // If set, sites where none of the FILTER strings is listed+                            // will be skipped. Active only at the time of+                            // initialization, that is during the add_reader()+                            // calls. Therefore, each reader can be initialized with different+                            // filters.+    int require_index;  // Some tools do not need random access+    int max_unpack;     // When reading VCFs and knowing some fields will not be needed, boost performance of vcf_parse1+    int *has_line;      // Corresponds to return value of bcf_sr_next_line but is not limited by sizeof(int). Use bcf_sr_has_line macro to query.+    bcf_sr_error errnum;++    // Auxiliary data+    bcf_sr_t *readers;+    int nreaders;+    int streaming;      // reading mode: index-jumping or streaming+    int explicit_regs;  // was the list of regions se by bcf_sr_set_regions or guessed from tabix index?+    char **samples; // List of samples+    bcf_sr_regions_t *regions, *targets;    // see bcf_sr_set_[targets|regions] for description+    int targets_als;    // subset to targets not only by position but also by alleles?+    int targets_exclude;+    kstring_t tmps;+    int n_smpl;+}+bcf_srs_t;++/** Init bcf_srs_t struct */+bcf_srs_t *bcf_sr_init(void);++/** Destroy  bcf_srs_t struct */+void bcf_sr_destroy(bcf_srs_t *readers);++char *bcf_sr_strerror(int errnum);+++/**+ *  bcf_sr_add_reader() - open new reader+ *  @readers: holder of the open readers+ *  @fname:   the VCF file+ *+ *  Returns 1 if the call succeeded, or 0 on error.+ *+ *  See also the bcf_srs_t data structure for parameters controlling+ *  the reader's logic.+ */+int bcf_sr_add_reader(bcf_srs_t *readers, const char *fname);+void bcf_sr_remove_reader(bcf_srs_t *files, int i);++/**+ * bcf_sr_next_line() - the iterator+ * @readers:    holder of the open readers+ *+ * Returns the number of readers which have the current line+ * (bcf_sr_t.buffer[0]) set at this position. Use the bcf_sr_has_line macro to+ * determine which of the readers are set.+ */+int bcf_sr_next_line(bcf_srs_t *readers);+#define bcf_sr_has_line(readers, i) (readers)->has_line[i]+#define bcf_sr_get_line(_readers, i) ((_readers)->has_line[i] ? ((_readers)->readers[i].buffer[0]) : NULL)+#define bcf_sr_swap_line(_readers, i, lieu) { bcf1_t *tmp = lieu; lieu = (_readers)->readers[i].buffer[0]; (_readers)->readers[i].buffer[0] = tmp; }+#define bcf_sr_region_done(_readers,i) (!(_readers)->has_line[i] && !(_readers)->readers[i].nbuffer ? 1 : 0)+#define bcf_sr_get_header(_readers, i) (_readers)->readers[i].header+#define bcf_sr_get_reader(_readers, i) &((_readers)->readers[i])++/**+ *  bcf_sr_seek() - set all readers to selected position+ *  @seq:  sequence name; NULL to seek to start+ *  @pos:  0-based coordinate+ */+int bcf_sr_seek(bcf_srs_t *readers, const char *seq, int pos);++/**+ * bcf_sr_set_samples() - sets active samples+ * @readers: holder of the open readers+ * @samples: this can be one of: file name with one sample per line;+ *           or column-separated list of samples; or '-' for a list of+ *           samples shared by all files. If first character is the+ *           exclamation mark, all but the listed samples are included.+ * @is_file: 0: list of samples; 1: file with sample names+ *+ * Returns 1 if the call succeeded, or 0 on error.+ */+int bcf_sr_set_samples(bcf_srs_t *readers, const char *samples, int is_file);++/**+ *  bcf_sr_set_targets(), bcf_sr_set_regions() - init targets/regions+ *  @readers:   holder of the open readers+ *  @targets:   list of regions, one-based and inclusive.+ *  @is_fname:  0: targets is a comma-separated list of regions (chr,chr:from-to)+ *              1: targets is a tabix indexed file with a list of regions+ *              (<chr,pos> or <chr,from,to>)+ *+ *  Returns 0 if the call succeeded, or -1 on error.+ *+ *  Both functions behave the same way, unlisted positions will be skipped by+ *  bcf_sr_next_line(). However, there is an important difference: regions use+ *  index to jump to desired positions while targets streams the whole files+ *  and merely skip unlisted positions.+ *+ *  Moreover, bcf_sr_set_targets() accepts an optional parameter $alleles which+ *  is intepreted as a 1-based column index in the tab-delimited file where+ *  alleles are listed. This in principle enables to perform the COLLAPSE_*+ *  logic also with tab-delimited files. However, the current implementation+ *  considers the alleles merely as a suggestion for prioritizing one of possibly+ *  duplicate VCF lines. It is up to the caller to examine targets->als if+ *  perfect match is sought after. Note that the duplicate positions in targets+ *  file are currently not supported.+ *  Targets (but not regions) can be prefixed with "^" to request logical complement,+ *  for example "^X,Y,MT" indicates that sequences X, Y and MT should be skipped.+ */+int bcf_sr_set_targets(bcf_srs_t *readers, const char *targets, int is_file, int alleles);+int bcf_sr_set_regions(bcf_srs_t *readers, const char *regions, int is_file);++++/*+ *  bcf_sr_regions_init()+ *  @regions:   regions can be either a comma-separated list of regions+ *              (chr|chr:pos|chr:from-to|chr:from-) or VCF, BED, or+ *              tab-delimited file (the default). Uncompressed files+ *              are stored in memory while bgzip-compressed and tabix-indexed+ *              region files are streamed.+ *  @is_file:   0: regions is a comma-separated list of regions+ *                  (chr|chr:pos|chr:from-to|chr:from-)+ *              1: VCF, BED or tab-delimited file+ *  @chr, from, to:+ *              Column indexes of chromosome, start position and end position+ *              in the tab-delimited file. The positions are 1-based and+ *              inclusive.+ *              These parameters are ignored when reading from VCF, BED or+ *              tabix-indexed files. When end position column is not present,+ *              supply 'from' in place of 'to'. When 'to' is negative, first+ *              abs(to) will be attempted and if that fails, 'from' will be used+ *              instead.+ */+bcf_sr_regions_t *bcf_sr_regions_init(const char *regions, int is_file, int chr, int from, int to);+void bcf_sr_regions_destroy(bcf_sr_regions_t *regions);++/*+ *  bcf_sr_regions_seek() - seek to the chromosome block+ *+ *  Returns 0 on success or -1 on failure. Sets reg->seq appropriately and+ *  reg->start,reg->end to -1.+ */+int bcf_sr_regions_seek(bcf_sr_regions_t *regions, const char *chr);++/*+ *  bcf_sr_regions_next() - retrieves next region. Returns 0 on success and -1+ *  when all regions have been read. The fields reg->seq, reg->start and+ *  reg->end are filled with the genomic coordinates on succes or with+ *  NULL,-1,-1 when no region is available. The coordinates are 0-based,+ *  inclusive.+ */+int bcf_sr_regions_next(bcf_sr_regions_t *reg);++/*+ *  bcf_sr_regions_overlap() - checks if the interval <start,end> overlaps any of+ *  the regions, the coordinates are 0-based, inclusive. The coordinate queries+ *  must come in ascending order.+ *+ *  Returns 0 if the position is in regions; -1 if the position is not in the+ *  regions and more regions exist; -2 if not in the regions and there are no more+ *  regions left.+ */+int bcf_sr_regions_overlap(bcf_sr_regions_t *reg, const char *seq, int start, int end);++/*+ *  bcf_sr_regions_flush() - calls repeatedly regs->missed_reg_handler() until+ *  all remaining records are processed.+ */+void bcf_sr_regions_flush(bcf_sr_regions_t *regs);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/tbx.h view
@@ -0,0 +1,79 @@+/*  tbx.h -- tabix API functions.++    Copyright (C) 2009, 2012-2015 Genome Research Ltd.+    Copyright (C) 2010, 2012 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_TBX_H+#define HTSLIB_TBX_H++#include "hts.h"++#ifdef __cplusplus+extern "C" {+#endif++#define TBX_MAX_SHIFT 31++#define TBX_GENERIC 0+#define TBX_SAM     1+#define TBX_VCF     2+#define TBX_UCSC    0x10000++typedef struct {+    int32_t preset;+    int32_t sc, bc, ec; // seq col., beg col. and end col.+    int32_t meta_char, line_skip;+} tbx_conf_t;++typedef struct {+    tbx_conf_t conf;+    hts_idx_t *idx;+    void *dict;+} tbx_t;++extern tbx_conf_t tbx_conf_gff, tbx_conf_bed, tbx_conf_psltbl, tbx_conf_sam, tbx_conf_vcf;++    #define tbx_itr_destroy(iter) hts_itr_destroy(iter)+    #define tbx_itr_queryi(tbx, tid, beg, end) hts_itr_query((tbx)->idx, (tid), (beg), (end), tbx_readrec)+    #define tbx_itr_querys(tbx, s) hts_itr_querys((tbx)->idx, (s), (hts_name2id_f)(tbx_name2id), (tbx), hts_itr_query, tbx_readrec)+    #define tbx_itr_next(htsfp, tbx, itr, r) hts_itr_next(hts_get_bgzfp(htsfp), (itr), (r), (tbx))+    #define tbx_bgzf_itr_next(bgzfp, tbx, itr, r) hts_itr_next((bgzfp), (itr), (r), (tbx))++    int tbx_name2id(tbx_t *tbx, const char *ss);++    /* Internal helper function used by tbx_itr_next() */+    BGZF *hts_get_bgzfp(htsFile *fp);+    int tbx_readrec(BGZF *fp, void *tbxv, void *sv, int *tid, int *beg, int *end);++    int tbx_index_build(const char *fn, int min_shift, const tbx_conf_t *conf);+    int tbx_index_build2(const char *fn, const char *fnidx, int min_shift, const tbx_conf_t *conf);+    tbx_t *tbx_index_load(const char *fn);+    tbx_t *tbx_index_load2(const char *fn, const char *fnidx);+    const char **tbx_seqnames(tbx_t *tbx, int *n);  // free the array but not the values+    void tbx_destroy(tbx_t *tbx);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/vcf.h view
@@ -0,0 +1,914 @@+/*  vcf.h -- VCF/BCF API functions.++    Copyright (C) 2012, 2013 Broad Institute.+    Copyright (C) 2012-2014 Genome Research Ltd.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++/*+    todo:+        - make the function names consistent+        - provide calls to abstract away structs as much as possible+ */++#ifndef HTSLIB_VCF_H+#define HTSLIB_VCF_H++#include <stdint.h>+#include <limits.h>+#include <assert.h>+#include "hts.h"+#include "kstring.h"+#include "hts_defs.h"++#ifdef __cplusplus+extern "C" {+#endif++/*****************+ * Header struct *+ *****************/++#define BCF_HL_FLT  0 // header line+#define BCF_HL_INFO 1+#define BCF_HL_FMT  2+#define BCF_HL_CTG  3+#define BCF_HL_STR  4 // structured header line TAG=<A=..,B=..>+#define BCF_HL_GEN  5 // generic header line++#define BCF_HT_FLAG 0 // header type+#define BCF_HT_INT  1+#define BCF_HT_REAL 2+#define BCF_HT_STR  3++#define BCF_VL_FIXED 0 // variable length+#define BCF_VL_VAR   1+#define BCF_VL_A     2+#define BCF_VL_G     3+#define BCF_VL_R     4++/* === Dictionary ===++   The header keeps three dictonaries. The first keeps IDs in the+   "FILTER/INFO/FORMAT" lines, the second keeps the sequence names and lengths+   in the "contig" lines and the last keeps the sample names. bcf_hdr_t::dict[]+   is the actual hash table, which is opaque to the end users. In the hash+   table, the key is the ID or sample name as a C string and the value is a+   bcf_idinfo_t struct. bcf_hdr_t::id[] points to key-value pairs in the hash+   table in the order that they appear in the VCF header. bcf_hdr_t::n[] is the+   size of the hash table or, equivalently, the length of the id[] arrays.+*/++#define BCF_DT_ID       0 // dictionary type+#define BCF_DT_CTG      1+#define BCF_DT_SAMPLE   2++// Complete textual representation of a header line+typedef struct {+    int type;       // One of the BCF_HL_* type+    char *key;      // The part before '=', i.e. FILTER/INFO/FORMAT/contig/fileformat etc.+    char *value;    // Set only for generic lines, NULL for FILTER/INFO, etc.+    int nkeys;              // Number of structured fields+    char **keys, **vals;    // The key=value pairs+} bcf_hrec_t;++typedef struct {+    uint32_t info[3];  // stores Number:20, var:4, Type:4, ColType:4 in info[0..2]+                       // for BCF_HL_FLT,INFO,FMT and contig length in info[0] for BCF_HL_CTG+    bcf_hrec_t *hrec[3];+    int id;+} bcf_idinfo_t;++typedef struct {+    const char *key;+    const bcf_idinfo_t *val;+} bcf_idpair_t;++// Note that bcf_hdr_t structs must always be created via bcf_hdr_init()+typedef struct {+    int32_t n[3];           // n:the size of the dictionary block in use, (allocated size, m, is below to preserve ABI)+    bcf_idpair_t *id[3];+    void *dict[3];          // ID dictionary, contig dict and sample dict+    char **samples;+    bcf_hrec_t **hrec;+    int nhrec, dirty;+    int ntransl, *transl[2];    // for bcf_translate()+    int nsamples_ori;           // for bcf_hdr_set_samples()+    uint8_t *keep_samples;+    kstring_t mem;+    int32_t m[3];          // m: allocated size of the dictionary block in use (see n above)+} bcf_hdr_t;++extern uint8_t bcf_type_shift[];++/**************+ * VCF record *+ **************/++#define BCF_BT_NULL     0+#define BCF_BT_INT8     1+#define BCF_BT_INT16    2+#define BCF_BT_INT32    3+#define BCF_BT_FLOAT    5+#define BCF_BT_CHAR     7++#define VCF_REF   0+#define VCF_SNP   1+#define VCF_MNP   2+#define VCF_INDEL 4+#define VCF_OTHER 8++typedef struct {+    int type, n;    // variant type and the number of bases affected, negative for deletions+} variant_t;++typedef struct {+    int id;             // id: numeric tag id, the corresponding string is bcf_hdr_t::id[BCF_DT_ID][$id].key+    int n, size, type;  // n: number of values per-sample; size: number of bytes per-sample; type: one of BCF_BT_* types+    uint8_t *p;         // same as vptr and vptr_* in bcf_info_t below+    uint32_t p_len;+    uint32_t p_off:31, p_free:1;+} bcf_fmt_t;++typedef struct {+    int key;        // key: numeric tag id, the corresponding string is bcf_hdr_t::id[BCF_DT_ID][$key].key+    int type, len;  // type: one of BCF_BT_* types; len: vector length, 1 for scalars+    union {+        int32_t i; // integer value+        float f;   // float value+    } v1; // only set if $len==1; for easier access+    uint8_t *vptr;          // pointer to data array in bcf1_t->shared.s, excluding the size+type and tag id bytes+    uint32_t vptr_len;      // length of the vptr block or, when set, of the vptr_mod block, excluding offset+    uint32_t vptr_off:31,   // vptr offset, i.e., the size of the INFO key plus size+type bytes+            vptr_free:1;    // indicates that vptr-vptr_off must be freed; set only when modified and the new+                            //    data block is bigger than the original+} bcf_info_t;+++#define BCF1_DIRTY_ID  1+#define BCF1_DIRTY_ALS 2+#define BCF1_DIRTY_FLT 4+#define BCF1_DIRTY_INF 8++typedef struct {+    int m_fmt, m_info, m_id, m_als, m_allele, m_flt; // allocated size (high-water mark); do not change+    int n_flt;  // Number of FILTER fields+    int *flt;   // FILTER keys in the dictionary+    char *id, *als;     // ID and REF+ALT block (\0-seperated)+    char **allele;      // allele[0] is the REF (allele[] pointers to the als block); all null terminated+    bcf_info_t *info;   // INFO+    bcf_fmt_t *fmt;     // FORMAT and individual sample+    variant_t *var;     // $var and $var_type set only when set_variant_types called+    int n_var, var_type;+    int shared_dirty;   // if set, shared.s must be recreated on BCF output+    int indiv_dirty;    // if set, indiv.s must be recreated on BCF output+} bcf_dec_t;+++#define BCF_ERR_CTG_UNDEF 1+#define BCF_ERR_TAG_UNDEF 2+#define BCF_ERR_NCOLS     4+#define BCF_ERR_LIMITS    8+#define BCF_ERR_CHAR     16+#define BCF_ERR_CTG_INVALID   32+#define BCF_ERR_TAG_INVALID   64++/*+    The bcf1_t structure corresponds to one VCF/BCF line. Reading from VCF file+    is slower because the string is first to be parsed, packed into BCF line+    (done in vcf_parse), then unpacked into internal bcf1_t structure. If it+    is known in advance that some of the fields will not be required (notably+    the sample columns), parsing of these can be skipped by setting max_unpack+    appropriately.+    Similarly, it is fast to output a BCF line because the columns (kept in+    shared.s, indiv.s, etc.) are written directly by bcf_write, whereas a VCF+    line must be formatted in vcf_format.+ */+typedef struct {+    int32_t rid;  // CHROM+    int32_t pos;  // POS+    int32_t rlen; // length of REF+    float qual;   // QUAL+    uint32_t n_info:16, n_allele:16;+    uint32_t n_fmt:8, n_sample:24;+    kstring_t shared, indiv;+    bcf_dec_t d; // lazy evaluation: $d is not generated by bcf_read(), but by explicitly calling bcf_unpack()+    int max_unpack;         // Set to BCF_UN_STR, BCF_UN_FLT, or BCF_UN_INFO to boost performance of vcf_parse when some of the fields won't be needed+    int unpacked;           // remember what has been unpacked to allow calling bcf_unpack() repeatedly without redoing the work+    int unpack_size[3];     // the original block size of ID, REF+ALT and FILTER+    int errcode;    // one of BCF_ERR_* codes+} bcf1_t;++/*******+ * API *+ *******/++    /***********************************************************************+     *  BCF and VCF I/O+     *+     *  A note about naming conventions: htslib internally represents VCF+     *  records as bcf1_t data structures, therefore most functions are+     *  prefixed with bcf_. There are a few exceptions where the functions must+     *  be aware of both BCF and VCF worlds, such as bcf_parse vs vcf_parse. In+     *  these cases, functions prefixed with bcf_ are more general and work+     *  with both BCF and VCF.+     *+     ***********************************************************************/++    /** These macros are defined only for consistency with other parts of htslib */+    #define bcf_init1()         bcf_init()+    #define bcf_read1(fp,h,v)   bcf_read((fp),(h),(v))+    #define vcf_read1(fp,h,v)   vcf_read((fp),(h),(v))+    #define bcf_write1(fp,h,v)  bcf_write((fp),(h),(v))+    #define vcf_write1(fp,h,v)  vcf_write((fp),(h),(v))+    #define bcf_destroy1(v)     bcf_destroy(v)+    #define bcf_empty1(v)       bcf_empty(v)+    #define vcf_parse1(s,h,v)   vcf_parse((s),(h),(v))+    #define bcf_clear1(v)       bcf_clear(v)+    #define vcf_format1(h,v,s)  vcf_format((h),(v),(s))++    /**+     *  bcf_hdr_init() - create an empty BCF header.+     *  @param mode    "r" or "w"+     *+     *  When opened for writing, the mandatory fileFormat and+     *  FILTER=PASS lines are added automatically.+     */+    bcf_hdr_t *bcf_hdr_init(const char *mode);++    /** Destroy a BCF header struct */+    void bcf_hdr_destroy(bcf_hdr_t *h);++    /** Initialize a bcf1_t object; equivalent to calloc(1, sizeof(bcf1_t)) */+    bcf1_t *bcf_init(void);++    /** Deallocate a bcf1_t object */+    void bcf_destroy(bcf1_t *v);++    /**+     *  Same as bcf_destroy() but frees only the memory allocated by bcf1_t,+     *  not the bcf1_t object itself.+     */+    void bcf_empty(bcf1_t *v);++    /**+     *  Make the bcf1_t object ready for next read. Intended mostly for+     *  internal use, the user should rarely need to call this function+     *  directly.+     */+    void bcf_clear(bcf1_t *v);+++    /** bcf_open and vcf_open mode: please see hts_open() in hts.h */+    typedef htsFile vcfFile;+    #define bcf_open(fn, mode) hts_open((fn), (mode))+    #define vcf_open(fn, mode) hts_open((fn), (mode))+    #define bcf_close(fp) hts_close(fp)+    #define vcf_close(fp) hts_close(fp)++    /** Reads VCF or BCF header */+    bcf_hdr_t *bcf_hdr_read(htsFile *fp);++    /**+     *  bcf_hdr_set_samples() - for more efficient VCF parsing when only one/few samples are needed+     *  @samples: samples to include or exclude from file or as a comma-separated string.+     *              LIST|FILE   .. select samples in list/file+     *              ^LIST|FILE  .. exclude samples from list/file+     *              -           .. include all samples+     *              NULL        .. exclude all samples+     *  @is_file: @samples is a file (1) or a comma-separated list (0)+     *+     *  The bottleneck of VCF reading is parsing of genotype fields. If the+     *  reader knows in advance that only subset of samples is needed (possibly+     *  no samples at all), the performance of bcf_read() can be significantly+     *  improved by calling bcf_hdr_set_samples after bcf_hdr_read().+     *  The function bcf_read() will subset the VCF/BCF records automatically+     *  with the notable exception when reading records via bcf_itr_next().+     *  In this case, bcf_subset_format() must be called explicitly, because+     *  bcf_readrec() does not see the header.+     *+     *  Returns 0 on success, -1 on error or a positive integer if the list+     *  contains samples not present in the VCF header. In such a case, the+     *  return value is the index of the offending sample.+     */+    int bcf_hdr_set_samples(bcf_hdr_t *hdr, const char *samples, int is_file);+    int bcf_subset_format(const bcf_hdr_t *hdr, bcf1_t *rec);+++    /** Writes VCF or BCF header */+    int bcf_hdr_write(htsFile *fp, bcf_hdr_t *h);++    /**+     * Parse VCF line contained in kstring and populate the bcf1_t struct+     * The line must not end with \n or \r characters.+     */+    int vcf_parse(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v);++    /** The opposite of vcf_parse. It should rarely be called directly, see vcf_write */+    int vcf_format(const bcf_hdr_t *h, const bcf1_t *v, kstring_t *s);++    /**+     *  bcf_read() - read next VCF or BCF record+     *+     *  Returns -1 on critical errors, 0 otherwise. On errors which are not+     *  critical for reading, such as missing header definitions, v->errcode is+     *  set to one of BCF_ERR* code and must be checked before calling+     *  vcf_write().+     */+    int bcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v);++    /**+     *  bcf_unpack() - unpack/decode a BCF record (fills the bcf1_t::d field)+     *+     *  Note that bcf_unpack() must be called even when reading VCF. It is safe+     *  to call the function repeatedly, it will not unpack the same field+     *  twice.+     */+    #define BCF_UN_STR  1       // up to ALT inclusive+    #define BCF_UN_FLT  2       // up to FILTER+    #define BCF_UN_INFO 4       // up to INFO+    #define BCF_UN_SHR  (BCF_UN_STR|BCF_UN_FLT|BCF_UN_INFO) // all shared information+    #define BCF_UN_FMT  8                           // unpack format and each sample+    #define BCF_UN_IND  BCF_UN_FMT                  // a synonymo of BCF_UN_FMT+    #define BCF_UN_ALL  (BCF_UN_SHR|BCF_UN_FMT)     // everything+    int bcf_unpack(bcf1_t *b, int which);++    /*+     *  bcf_dup() - create a copy of BCF record.+     *+     *  Note that bcf_unpack() must be called on the returned copy as if it was+     *  obtained from bcf_read(). Also note that bcf_dup() calls bcf_sync1(src)+     *  internally to reflect any changes made by bcf_update_* functions.+     */+    bcf1_t *bcf_dup(bcf1_t *src);+    bcf1_t *bcf_copy(bcf1_t *dst, bcf1_t *src);++    /**+     *  bcf_write() - write one VCF or BCF record. The type is determined at the open() call.+     */+    int bcf_write(htsFile *fp, bcf_hdr_t *h, bcf1_t *v);++    /**+     *  The following functions work only with VCFs and should rarely be called+     *  directly. Usually one wants to use their bcf_* alternatives, which work+     *  transparently with both VCFs and BCFs.+     */+    bcf_hdr_t *vcf_hdr_read(htsFile *fp);+    int vcf_hdr_write(htsFile *fp, const bcf_hdr_t *h);+    int vcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v);+    int vcf_write(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v);++    /** Helper function for the bcf_itr_next() macro; internal use, ignore it */+    int bcf_readrec(BGZF *fp, void *null, void *v, int *tid, int *beg, int *end);++++    /**************************************************************************+     *  Header querying and manipulation routines+     **************************************************************************/++    /** Create a new header using the supplied template */+    bcf_hdr_t *bcf_hdr_dup(const bcf_hdr_t *hdr);++    /**+     *  Copy header lines from src to dst if not already present in dst. See also bcf_translate().+     *  Returns 0 on success or sets a bit on error:+     *      1 .. conflicting definitions of tag length+     *      // todo+     */+    int bcf_hdr_combine(bcf_hdr_t *dst, const bcf_hdr_t *src) HTS_DEPRECATED("Please use bcf_hdr_merge instead");++    /**+     *  bcf_hdr_merge() - copy header lines from src to dst, see also bcf_translate()+     *  @param dst: the destination header to be merged into, NULL on the first pass+     *  @param src: the source header+     *+     *  Notes:+     *      - use as:+     *          bcf_hdr_t *dst = NULL;+     *          for (i=0; i<nsrc; i++) dst = bcf_hdr_merge(dst,src[i]);+     *+     *      - bcf_hdr_merge() replaces bcf_hdr_combine() which had a problem when+     *      combining multiple BCF headers. The current bcf_hdr_combine()+     *      does not have this problem, but became slow when used for many files.+     */+    bcf_hdr_t *bcf_hdr_merge(bcf_hdr_t *dst, const bcf_hdr_t *src);++    /**+     *  bcf_hdr_add_sample() - add a new sample.+     *  @param sample:  sample name to be added+     */+    int bcf_hdr_add_sample(bcf_hdr_t *hdr, const char *sample);++    /** Read VCF header from a file and update the header */+    int bcf_hdr_set(bcf_hdr_t *hdr, const char *fname);++    /** Returns formatted header (newly allocated string) and its length,+     *  excluding the terminating \0. If is_bcf parameter is unset, IDX+     *  fields are discarded.+     */+    char *bcf_hdr_fmt_text(const bcf_hdr_t *hdr, int is_bcf, int *len);++    /** Append new VCF header line, returns 0 on success */+    int bcf_hdr_append(bcf_hdr_t *h, const char *line);+    int bcf_hdr_printf(bcf_hdr_t *h, const char *format, ...);++    /** VCF version, e.g. VCFv4.2 */+    const char *bcf_hdr_get_version(const bcf_hdr_t *hdr);+    void bcf_hdr_set_version(bcf_hdr_t *hdr, const char *version);++    /**+     *  bcf_hdr_remove() - remove VCF header tag+     *  @param type:      one of BCF_HL_*+     *  @param key:       tag name or NULL to remove all tags of the given type+     */+    void bcf_hdr_remove(bcf_hdr_t *h, int type, const char *key);++    /**+     *  bcf_hdr_subset() - creates a new copy of the header removing unwanted samples+     *  @param n:        number of samples to keep+     *  @param samples:  names of the samples to keep+     *  @param imap:     mapping from index in @samples to the sample index in the original file+     *+     *  Sample names not present in h0 are ignored. The number of unmatched samples can be checked+     *  by comparing n and bcf_hdr_nsamples(out_hdr).+     *  This function can be used to reorder samples.+     *  See also bcf_subset() which subsets individual records.+     */+    bcf_hdr_t *bcf_hdr_subset(const bcf_hdr_t *h0, int n, char *const* samples, int *imap);++    /** Creates a list of sequence names. It is up to the caller to free the list (but not the sequence names) */+    const char **bcf_hdr_seqnames(const bcf_hdr_t *h, int *nseqs);++    /** Get number of samples */+    #define bcf_hdr_nsamples(hdr) (hdr)->n[BCF_DT_SAMPLE]+++    /** The following functions are for internal use and should rarely be called directly */+    int bcf_hdr_parse(bcf_hdr_t *hdr, char *htxt);+    int bcf_hdr_sync(bcf_hdr_t *h);+    bcf_hrec_t *bcf_hdr_parse_line(const bcf_hdr_t *h, const char *line, int *len);+    void bcf_hrec_format(const bcf_hrec_t *hrec, kstring_t *str);+    int bcf_hdr_add_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec);+    /**+     *  bcf_hdr_get_hrec() - get header line info+     *  @param type:  one of the BCF_HL_* types: FLT,INFO,FMT,CTG,STR,GEN+     *  @param key:   the header key for generic lines (e.g. "fileformat"), any field+     *                  for structured lines, typically "ID".+     *  @param value: the value which pairs with key. Can be be NULL for BCF_HL_GEN+     *  @param str_class: the class of BCF_HL_STR line (e.g. "ALT" or "SAMPLE"), otherwise NULL+     */+    bcf_hrec_t *bcf_hdr_get_hrec(const bcf_hdr_t *hdr, int type, const char *key, const char *value, const char *str_class);+    bcf_hrec_t *bcf_hrec_dup(bcf_hrec_t *hrec);+    void bcf_hrec_add_key(bcf_hrec_t *hrec, const char *str, int len);+    void bcf_hrec_set_val(bcf_hrec_t *hrec, int i, const char *str, int len, int is_quoted);+    int bcf_hrec_find_key(bcf_hrec_t *hrec, const char *key);+    void hrec_add_idx(bcf_hrec_t *hrec, int idx);+    void bcf_hrec_destroy(bcf_hrec_t *hrec);++++    /**************************************************************************+     *  Individual record querying and manipulation routines+     **************************************************************************/++    /** See the description of bcf_hdr_subset() */+    int bcf_subset(const bcf_hdr_t *h, bcf1_t *v, int n, int *imap);++    /**+     *  bcf_translate() - translate tags ids to be consistent with different header. This function+     *                    is useful when lines from multiple VCF need to be combined.+     *  @dst_hdr:   the destination header, to be used in bcf_write(), see also bcf_hdr_combine()+     *  @src_hdr:   the source header, used in bcf_read()+     *  @src_line:  line obtained by bcf_read()+     */+    int bcf_translate(const bcf_hdr_t *dst_hdr, bcf_hdr_t *src_hdr, bcf1_t *src_line);++    /**+     *  bcf_get_variant_type[s]()  - returns one of VCF_REF, VCF_SNP, etc+     */+    int bcf_get_variant_types(bcf1_t *rec);+    int bcf_get_variant_type(bcf1_t *rec, int ith_allele);+    int bcf_is_snp(bcf1_t *v);++    /**+     *  bcf_update_filter() - sets the FILTER column+     *  @flt_ids:  The filter IDs to set, numeric IDs returned by bcf_hdr_id2int(hdr, BCF_DT_ID, "PASS")+     *  @n:        Number of filters. If n==0, all filters are removed+     */+    int bcf_update_filter(const bcf_hdr_t *hdr, bcf1_t *line, int *flt_ids, int n);+    /**+     *  bcf_add_filter() - adds to the FILTER column+     *  @flt_id:   filter ID to add, numeric ID returned by bcf_hdr_id2int(hdr, BCF_DT_ID, "PASS")+     *+     *  If flt_id is PASS, all existing filters are removed first. If other than PASS, existing PASS is removed.+     */+    int bcf_add_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id);+    /**+     *  bcf_remove_filter() - removes from the FILTER column+     *  @flt_id:   filter ID to remove, numeric ID returned by bcf_hdr_id2int(hdr, BCF_DT_ID, "PASS")+     *  @pass:     when set to 1 and no filters are present, set to PASS+     */+    int bcf_remove_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id, int pass);+    /**+     *  Returns 1 if present, 0 if absent, or -1 if filter does not exist. "PASS" and "." can be used interchangeably.+     */+    int bcf_has_filter(const bcf_hdr_t *hdr, bcf1_t *line, char *filter);+    /**+     *  bcf_update_alleles() and bcf_update_alleles_str() - update REF and ALLT column+     *  @alleles:           Array of alleles+     *  @nals:              Number of alleles+     *  @alleles_string:    Comma-separated alleles, starting with the REF allele+     */+    int bcf_update_alleles(const bcf_hdr_t *hdr, bcf1_t *line, const char **alleles, int nals);+    int bcf_update_alleles_str(const bcf_hdr_t *hdr, bcf1_t *line, const char *alleles_string);++    /**+      *  bcf_update_id() - sets new ID string+      *  bcf_add_id() - adds to the ID string checking for duplicates+      */+    int bcf_update_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id);+    int bcf_add_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id);++    /*+     *  bcf_update_info_*() - functions for updating INFO fields+     *  @hdr:       the BCF header+     *  @line:      VCF line to be edited+     *  @key:       the INFO tag to be updated+     *  @values:    pointer to the array of values. Pass NULL to remove the tag.+     *  @n:         number of values in the array. When set to 0, the INFO tag is removed+     *+     *  The @string in bcf_update_info_flag() is optional, @n indicates whether+     *  the flag is set or removed.+     *+     *  Returns 0 on success or negative value on error.+     */+    #define bcf_update_info_int32(hdr,line,key,values,n)   bcf_update_info((hdr),(line),(key),(values),(n),BCF_HT_INT)+    #define bcf_update_info_float(hdr,line,key,values,n)   bcf_update_info((hdr),(line),(key),(values),(n),BCF_HT_REAL)+    #define bcf_update_info_flag(hdr,line,key,string,n)    bcf_update_info((hdr),(line),(key),(string),(n),BCF_HT_FLAG)+    #define bcf_update_info_string(hdr,line,key,string)    bcf_update_info((hdr),(line),(key),(string),1,BCF_HT_STR)+    int bcf_update_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type);++    /*+     *  bcf_update_format_*() - functions for updating FORMAT fields+     *  @values:    pointer to the array of values, the same number of elements+     *              is expected for each sample. Missing values must be padded+     *              with bcf_*_missing or bcf_*_vector_end values.+     *  @n:         number of values in the array. If n==0, existing tag is removed.+     *+     *  The function bcf_update_format_string() is a higher-level (slower) variant of+     *  bcf_update_format_char(). The former accepts array of \0-terminated strings+     *  whereas the latter requires that the strings are collapsed into a single array+     *  of fixed-length strings. In case of strings with variable length, shorter strings+     *  can be \0-padded. Note that the collapsed strings passed to bcf_update_format_char()+     *  are not \0-terminated.+     *+     *  Returns 0 on success or negative value on error.+     */+    #define bcf_update_format_int32(hdr,line,key,values,n) bcf_update_format((hdr),(line),(key),(values),(n),BCF_HT_INT)+    #define bcf_update_format_float(hdr,line,key,values,n) bcf_update_format((hdr),(line),(key),(values),(n),BCF_HT_REAL)+    #define bcf_update_format_char(hdr,line,key,values,n) bcf_update_format((hdr),(line),(key),(values),(n),BCF_HT_STR)+    #define bcf_update_genotypes(hdr,line,gts,n) bcf_update_format((hdr),(line),"GT",(gts),(n),BCF_HT_INT)     // See bcf_gt_ macros below+    int bcf_update_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const char **values, int n);+    int bcf_update_format(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type);++    // Macros for setting genotypes correctly, for use with bcf_update_genotypes only; idx corresponds+    // to VCF's GT (1-based index to ALT or 0 for the reference allele) and val is the opposite, obtained+    // from bcf_get_genotypes() below.+    #define bcf_gt_phased(idx)      (((idx)+1)<<1|1)+    #define bcf_gt_unphased(idx)    (((idx)+1)<<1)+    #define bcf_gt_missing          0 +    #define bcf_gt_is_missing(val)  ((val)>>1 ? 0 : 1)+    #define bcf_gt_is_phased(idx)   ((idx)&1)+    #define bcf_gt_allele(val)      (((val)>>1)-1)++    /** Conversion between alleles indexes to Number=G genotype index (assuming diploid, all 0-based) */+    #define bcf_alleles2gt(a,b) ((a)>(b)?((a)*((a)+1)/2+(b)):((b)*((b)+1)/2+(a)))+    static inline void bcf_gt2alleles(int igt, int *a, int *b)+    {+        int k = 0, dk = 1;+        while ( k<igt ) { dk++; k += dk; }+        *b = dk - 1; *a = igt - k + *b;+    }++    /**+     * bcf_get_fmt() - returns pointer to FORMAT's field data+     * @header: for access to BCF_DT_ID dictionary+     * @line:   VCF line obtained from vcf_parse1+     * @fmt:    one of GT,PL,...+     *+     * Returns bcf_fmt_t* if the call succeeded, or returns NULL when the field+     * is not available.+     */+    bcf_fmt_t *bcf_get_fmt(const bcf_hdr_t *hdr, bcf1_t *line, const char *key);+    bcf_info_t *bcf_get_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key);++    /**+     * bcf_get_*_id() - returns pointer to FORMAT/INFO field data given the header index instead of the string ID+     * @line: VCF line obtained from vcf_parse1+     * @id:  The header index for the tag, obtained from bcf_hdr_id2int()+     * +     * Returns bcf_fmt_t* / bcf_info_t*. These functions do not check if the index is valid +     * as their goal is to avoid the header lookup.+     */+    bcf_fmt_t *bcf_get_fmt_id(bcf1_t *line, const int id);+    bcf_info_t *bcf_get_info_id(bcf1_t *line, const int id);++    /**+     *  bcf_get_info_*() - get INFO values, integers or floats+     *  @hdr:       BCF header+     *  @line:      BCF record+     *  @tag:       INFO tag to retrieve+     *  @dst:       *dst is pointer to a memory location, can point to NULL+     *  @ndst:      pointer to the size of allocated memory+     *+     *  Returns negative value on error or the number of written values on+     *  success. bcf_get_info_string() returns on success the number of+     *  characters written excluding the null-terminating byte. bcf_get_info_flag()+     *  returns 1 when flag is set or 0 if not.+     *+     *  List of return codes:+     *      -1 .. no such INFO tag defined in the header+     *      -2 .. clash between types defined in the header and encountered in the VCF record+     *      -3 .. tag is not present in the VCF record+     */+    #define bcf_get_info_int32(hdr,line,tag,dst,ndst)  bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_INT)+    #define bcf_get_info_float(hdr,line,tag,dst,ndst)  bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_REAL)+    #define bcf_get_info_string(hdr,line,tag,dst,ndst) bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_STR)+    #define bcf_get_info_flag(hdr,line,tag,dst,ndst)   bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_FLAG)+    int bcf_get_info_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type);++    /**+     *  bcf_get_format_*() - same as bcf_get_info*() above+     *+     *  The function bcf_get_format_string() is a higher-level (slower) variant of bcf_get_format_char().+     *  see the description of bcf_update_format_string() and bcf_update_format_char() above.+     *  Unlike other bcf_get_format__*() functions, bcf_get_format_string() allocates two arrays:+     *  a single block of \0-terminated strings collapsed into a single array and an array of pointers+     *  to these strings. Both arrays must be cleaned by the user.+     *+     *  Returns negative value on error or the number of written values on success.+     *+     *  Example:+     *      int ndst = 0; char **dst = NULL;+     *      if ( bcf_get_format_string(hdr, line, "XX", &dst, &ndst) > 0 )+     *          for (i=0; i<bcf_hdr_nsamples(hdr); i++) printf("%s\n", dst[i]);+     *      free(dst[0]); free(dst);+     *+     *  Example:+     *      int ngt, *gt_arr = NULL, ngt_arr = 0;+     *      ngt = bcf_get_genotypes(hdr, line, &gt_arr, &ngt_arr);+     */+    #define bcf_get_format_int32(hdr,line,tag,dst,ndst)  bcf_get_format_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_INT)+    #define bcf_get_format_float(hdr,line,tag,dst,ndst)  bcf_get_format_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_REAL)+    #define bcf_get_format_char(hdr,line,tag,dst,ndst)   bcf_get_format_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_STR)+    #define bcf_get_genotypes(hdr,line,dst,ndst)         bcf_get_format_values(hdr,line,"GT",(void**)(dst),ndst,BCF_HT_INT)+    int bcf_get_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, char ***dst, int *ndst);+    int bcf_get_format_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type);++++    /**************************************************************************+     *  Helper functions+     **************************************************************************/++    /**+     *  bcf_hdr_id2int() - Translates string into numeric ID+     *  bcf_hdr_int2id() - Translates numeric ID into string+     *  @type:     one of BCF_DT_ID, BCF_DT_CTG, BCF_DT_SAMPLE+     *  @id:       tag name, such as: PL, DP, GT, etc.+     *+     *  Returns -1 if string is not in dictionary, otherwise numeric ID which identifies+     *  fields in BCF records.+     */+    int bcf_hdr_id2int(const bcf_hdr_t *hdr, int type, const char *id);+    #define bcf_hdr_int2id(hdr,type,int_id) ((hdr)->id[type][int_id].key)++    /**+     *  bcf_hdr_name2id() - Translates sequence names (chromosomes) into numeric ID+     *  bcf_hdr_id2name() - Translates numeric ID to sequence name+     */+    static inline int bcf_hdr_name2id(const bcf_hdr_t *hdr, const char *id) { return bcf_hdr_id2int(hdr, BCF_DT_CTG, id); }+    static inline const char *bcf_hdr_id2name(const bcf_hdr_t *hdr, int rid) { return hdr->id[BCF_DT_CTG][rid].key; }+    static inline const char *bcf_seqname(const bcf_hdr_t *hdr, bcf1_t *rec) { return hdr->id[BCF_DT_CTG][rec->rid].key; }++    /**+     *  bcf_hdr_id2*() - Macros for accessing bcf_idinfo_t+     *  @type:      one of BCF_HL_FLT, BCF_HL_INFO, BCF_HL_FMT+     *  @int_id:    return value of bcf_hdr_id2int, must be >=0+     *+     *  The returned values are:+     *     bcf_hdr_id2length   ..  whether the number of values is fixed or variable, one of BCF_VL_*+     *     bcf_hdr_id2number   ..  the number of values, 0xfffff for variable length fields+     *     bcf_hdr_id2type     ..  the field type, one of BCF_HT_*+     *     bcf_hdr_id2coltype  ..  the column type, one of BCF_HL_*+     *+     *  Notes: Prior to using the macros, the presence of the info should be+     *  tested with bcf_hdr_idinfo_exists().+     */+    #define bcf_hdr_id2length(hdr,type,int_id)  ((hdr)->id[BCF_DT_ID][int_id].val->info[type]>>8 & 0xf)+    #define bcf_hdr_id2number(hdr,type,int_id)  ((hdr)->id[BCF_DT_ID][int_id].val->info[type]>>12)+    #define bcf_hdr_id2type(hdr,type,int_id)    ((hdr)->id[BCF_DT_ID][int_id].val->info[type]>>4 & 0xf)+    #define bcf_hdr_id2coltype(hdr,type,int_id) ((hdr)->id[BCF_DT_ID][int_id].val->info[type] & 0xf)+    #define bcf_hdr_idinfo_exists(hdr,type,int_id)  ((int_id<0 || bcf_hdr_id2coltype(hdr,type,int_id)==0xf) ? 0 : 1)+    #define bcf_hdr_id2hrec(hdr,dict_type,col_type,int_id)    ((hdr)->id[(dict_type)==BCF_DT_CTG?BCF_DT_CTG:BCF_DT_ID][int_id].val->hrec[(dict_type)==BCF_DT_CTG?0:(col_type)])++    void bcf_fmt_array(kstring_t *s, int n, int type, void *data);+    uint8_t *bcf_fmt_sized_array(kstring_t *s, uint8_t *ptr);++    void bcf_enc_vchar(kstring_t *s, int l, const char *a);+    void bcf_enc_vint(kstring_t *s, int n, int32_t *a, int wsize);+    void bcf_enc_vfloat(kstring_t *s, int n, float *a);+++    /**************************************************************************+     *  BCF index+     *+     *  Note that these functions work with BCFs only. See synced_bcf_reader.h+     *  which provides (amongst other things) an API to work transparently with+     *  both indexed BCFs and VCFs.+     **************************************************************************/++    #define bcf_itr_destroy(iter) hts_itr_destroy(iter)+    #define bcf_itr_queryi(idx, tid, beg, end) hts_itr_query((idx), (tid), (beg), (end), bcf_readrec)+    #define bcf_itr_querys(idx, hdr, s) hts_itr_querys((idx), (s), (hts_name2id_f)(bcf_hdr_name2id), (hdr), hts_itr_query, bcf_readrec)+    #define bcf_itr_next(htsfp, itr, r) hts_itr_next((htsfp)->fp.bgzf, (itr), (r), 0)+    #define bcf_index_load(fn) hts_idx_load(fn, HTS_FMT_CSI)+    #define bcf_index_seqnames(idx, hdr, nptr) hts_idx_seqnames((idx),(nptr),(hts_id2name_f)(bcf_hdr_id2name),(hdr))++    hts_idx_t *bcf_index_load2(const char *fn, const char *fnidx);+    int bcf_index_build(const char *fn, int min_shift);+    int bcf_index_build2(const char *fn, const char *fnidx, int min_shift);++/*******************+ * Typed value I/O *+ *******************/++/*+    Note that in contrast with BCFv2.1 specification, HTSlib implementation+    allows missing values in vectors. For integer types, the values 0x80,+    0x8000, 0x80000000 are interpreted as missing values and 0x81, 0x8001,+    0x80000001 as end-of-vector indicators.  Similarly for floats, the value of+    0x7F800001 is interpreted as a missing value and 0x7F800002 as an+    end-of-vector indicator.+    Note that the end-of-vector byte is not part of the vector.++    This trial BCF version (v2.2) is compatible with the VCF specification and+    enables to handle correctly vectors with different ploidy in presence of+    missing values.+ */+#define bcf_int8_vector_end  (INT8_MIN+1)+#define bcf_int16_vector_end (INT16_MIN+1)+#define bcf_int32_vector_end (INT32_MIN+1)+#define bcf_str_vector_end   0+#define bcf_int8_missing     INT8_MIN+#define bcf_int16_missing    INT16_MIN+#define bcf_int32_missing    INT32_MIN+#define bcf_str_missing      0x07+extern uint32_t bcf_float_vector_end;+extern uint32_t bcf_float_missing;+static inline void bcf_float_set(float *ptr, uint32_t value)+{+    union { uint32_t i; float f; } u;+    u.i = value;+    *ptr = u.f;+}+#define bcf_float_set_vector_end(x) bcf_float_set(&(x),bcf_float_vector_end)+#define bcf_float_set_missing(x)    bcf_float_set(&(x),bcf_float_missing)+static inline int bcf_float_is_missing(float f)+{+    union { uint32_t i; float f; } u;+    u.f = f;+    return u.i==bcf_float_missing ? 1 : 0;+}+static inline int bcf_float_is_vector_end(float f)+{+    union { uint32_t i; float f; } u;+    u.f = f;+    return u.i==bcf_float_vector_end ? 1 : 0;+}++static inline void bcf_format_gt(bcf_fmt_t *fmt, int isample, kstring_t *str)+{+    #define BRANCH(type_t, missing, vector_end) { \+        type_t *ptr = (type_t*) (fmt->p + isample*fmt->size); \+        int i; \+        for (i=0; i<fmt->n && ptr[i]!=vector_end; i++) \+        { \+            if ( i ) kputc("/|"[ptr[i]&1], str); \+            if ( !(ptr[i]>>1) ) kputc('.', str); \+            else kputw((ptr[i]>>1) - 1, str); \+        } \+        if (i == 0) kputc('.', str); \+    }+    switch (fmt->type) {+        case BCF_BT_INT8:  BRANCH(int8_t,  bcf_int8_missing, bcf_int8_vector_end); break;+        case BCF_BT_INT16: BRANCH(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;+        case BCF_BT_INT32: BRANCH(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;+        case BCF_BT_NULL:  kputc('.', str); break;+        default: fprintf(stderr,"FIXME: type %d in bcf_format_gt?\n", fmt->type); abort(); break;+    }+    #undef BRANCH+}++static inline void bcf_enc_size(kstring_t *s, int size, int type)+{+    if (size >= 15) {+        kputc(15<<4|type, s);+        if (size >= 128) {+            if (size >= 32768) {+                int32_t x = size;+                kputc(1<<4|BCF_BT_INT32, s);+                kputsn((char*)&x, 4, s);+            } else {+                int16_t x = size;+                kputc(1<<4|BCF_BT_INT16, s);+                kputsn((char*)&x, 2, s);+            }+        } else {+            kputc(1<<4|BCF_BT_INT8, s);+            kputc(size, s);+        }+    } else kputc(size<<4|type, s);+}++static inline int bcf_enc_inttype(long x)+{+    if (x <= INT8_MAX && x > bcf_int8_missing) return BCF_BT_INT8;+    if (x <= INT16_MAX && x > bcf_int16_missing) return BCF_BT_INT16;+    return BCF_BT_INT32;+}++static inline void bcf_enc_int1(kstring_t *s, int32_t x)+{+    if (x == bcf_int32_vector_end) {+        bcf_enc_size(s, 1, BCF_BT_INT8);+        kputc(bcf_int8_vector_end, s);+    } else if (x == bcf_int32_missing) {+        bcf_enc_size(s, 1, BCF_BT_INT8);+        kputc(bcf_int8_missing, s);+    } else if (x <= INT8_MAX && x > bcf_int8_missing) {+        bcf_enc_size(s, 1, BCF_BT_INT8);+        kputc(x, s);+    } else if (x <= INT16_MAX && x > bcf_int16_missing) {+        int16_t z = x;+        bcf_enc_size(s, 1, BCF_BT_INT16);+        kputsn((char*)&z, 2, s);+    } else {+        int32_t z = x;+        bcf_enc_size(s, 1, BCF_BT_INT32);+        kputsn((char*)&z, 4, s);+    }+}++static inline int32_t bcf_dec_int1(const uint8_t *p, int type, uint8_t **q)+{+    if (type == BCF_BT_INT8) {+        *q = (uint8_t*)p + 1;+        return *(int8_t*)p;+    } else if (type == BCF_BT_INT16) {+        *q = (uint8_t*)p + 2;+        return *(int16_t*)p;+    } else {+        *q = (uint8_t*)p + 4;+        return *(int32_t*)p;+    }+}++static inline int32_t bcf_dec_typed_int1(const uint8_t *p, uint8_t **q)+{+    return bcf_dec_int1(p + 1, *p&0xf, q);+}++static inline int32_t bcf_dec_size(const uint8_t *p, uint8_t **q, int *type)+{+    *type = *p & 0xf;+    if (*p>>4 != 15) {+        *q = (uint8_t*)p + 1;+        return *p>>4;+    } else return bcf_dec_typed_int1(p + 1, q);+}++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/vcf_sweep.h view
@@ -0,0 +1,47 @@+/*  vcf_sweep.h -- forward/reverse sweep API.++    Copyright (C) 2013 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_VCF_SWEEP_H+#define HTSLIB_VCF_SWEEP_H++#include "hts.h"+#include "vcf.h"++#ifdef __cplusplus+extern "C" {+#endif++typedef struct _bcf_sweep_t bcf_sweep_t;++bcf_sweep_t *bcf_sweep_init(const char *fname);+void bcf_sweep_destroy(bcf_sweep_t *sw);+bcf_hdr_t *bcf_sweep_hdr(bcf_sweep_t *sw);+bcf1_t *bcf_sweep_fwd(bcf_sweep_t *sw);+bcf1_t *bcf_sweep_bwd(bcf_sweep_t *sw);++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/htslib/vcfutils.h view
@@ -0,0 +1,134 @@+/*  vcfutils.h -- allele-related utility functions.++    Copyright (C) 2012, 2013, 2015 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#ifndef HTSLIB_VCFUTILS_H+#define HTSLIB_VCFUTILS_H++#include "vcf.h"++#ifdef __cplusplus+extern "C" {+#endif++struct kbitset_t;++/**+ *  bcf_trim_alleles() - remove ALT alleles unused in genotype fields+ *  @header:  for access to BCF_DT_ID dictionary+ *  @line:    VCF line obtain from vcf_parse1+ *+ *  Returns the number of removed alleles on success or negative+ *  on error:+ *      -1 .. some allele index is out of bounds+ */+int bcf_trim_alleles(const bcf_hdr_t *header, bcf1_t *line);++/**+ *  bcf_remove_alleles() - remove ALT alleles according to bitmask @mask+ *  @header:  for access to BCF_DT_ID dictionary+ *  @line:    VCF line obtained from vcf_parse1+ *  @mask:    alleles to remove+ * + *  If you have more than 31 alleles, then the integer bit mask will+ *  overflow, so use bcf_remove_allele_set instead+ */+void bcf_remove_alleles(const bcf_hdr_t *header, bcf1_t *line, int mask);++/**+ *  bcf_remove_allele_set() - remove ALT alleles according to bitset @rm_set+ *  @header:  for access to BCF_DT_ID dictionary+ *  @line:    VCF line obtained from vcf_parse1+ *  @rm_set:  pointer to kbitset_t object with bits set for allele+ *            indexes to remove+ *  + *  Number=A,R,G INFO and FORMAT fields will be updated accordingly.+ */+void bcf_remove_allele_set(const bcf_hdr_t *header, bcf1_t *line, const struct kbitset_t *rm_set);++/**+ *  bcf_calc_ac() - calculate the number of REF and ALT alleles+ *  @header:  for access to BCF_DT_ID dictionary+ *  @line:    VCF line obtained from vcf_parse1+ *  @ac:      array of length line->n_allele+ *  @which:   determine if INFO/AN,AC and indv fields be used+ *+ *  Returns 1 if the call succeeded, or 0 if the value could not+ *  be determined.+ *+ *  The value of @which determines if existing INFO/AC,AN can be+ *  used (BCF_UN_INFO) and and if indv fields can be splitted+ *  (BCF_UN_FMT).+ */+int bcf_calc_ac(const bcf_hdr_t *header, bcf1_t *line, int *ac, int which);+++/**+ * bcf_gt_type() - determines type of the genotype+ * @fmt_ptr:  the GT format field as set for example by set_fmt_ptr+ * @isample:  sample index (starting from 0)+ * @ial:      index of the 1st non-reference allele (starting from 1)+ * @jal:      index of the 2nd non-reference allele (starting from 1)+ *+ * Returns the type of the genotype (one of GT_HOM_RR, GT_HET_RA,+ * GT_HOM_AA, GT_HET_AA, GT_HAPL_R, GT_HAPL_A or GT_UNKN). If $ial+ * is not NULL and the genotype has one or more non-reference+ * alleles, $ial will be set. In case of GT_HET_AA, $ial is the+ * position of the allele which appeared first in ALT. If $jal is+ * not null and the genotype is GT_HET_AA, $jal will be set and is+ * the position of the second allele in ALT.+ */+#define GT_HOM_RR 0 // note: the actual value of GT_* matters, used in dosage r2 calculation+#define GT_HOM_AA 1+#define GT_HET_RA 2+#define GT_HET_AA 3+#define GT_HAPL_R 4+#define GT_HAPL_A 5+#define GT_UNKN   6+int bcf_gt_type(bcf_fmt_t *fmt_ptr, int isample, int *ial, int *jal);++static inline int bcf_acgt2int(char c)+{+    if ( (int)c>96 ) c -= 32;+    if ( c=='A' ) return 0;+    if ( c=='C' ) return 1;+    if ( c=='G' ) return 2;+    if ( c=='T' ) return 3;+    return -1;+}++#define bcf_int2acgt(i) "ACGT"[i]++/**+  * bcf_ij2G() - common task: allele indexes to Number=G index (diploid)+  * @i,j:  allele indexes, 0-based, i<=j+  *+  * Returns index to the Number=G diploid array+  */+#define bcf_ij2G(i, j) ((j)*((j)+1)/2+(i))++#ifdef __cplusplus+}+#endif++#endif
+ htslib-1.3.2/kfunc.c view
@@ -0,0 +1,282 @@+/* The MIT License++   Copyright (C) 2010, 2013 Genome Research Ltd.+   Copyright (C) 2011 Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#include <config.h>++#include <math.h>+#include <stdlib.h>+#include "htslib/kfunc.h"++/* Log gamma function+ * \log{\Gamma(z)}+ * AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245+ */+double kf_lgamma(double z)+{+	double x = 0;+	x += 0.1659470187408462e-06 / (z+7);+	x += 0.9934937113930748e-05 / (z+6);+	x -= 0.1385710331296526     / (z+5);+	x += 12.50734324009056      / (z+4);+	x -= 176.6150291498386      / (z+3);+	x += 771.3234287757674      / (z+2);+	x -= 1259.139216722289      / (z+1);+	x += 676.5203681218835      / z;+	x += 0.9999999999995183;+	return log(x) - 5.58106146679532777 - z + (z-0.5) * log(z+6.5);+}++/* complementary error function+ * \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} dt+ * AS66, 2nd algorithm, http://lib.stat.cmu.edu/apstat/66+ */+double kf_erfc(double x)+{+	const double p0 = 220.2068679123761;+	const double p1 = 221.2135961699311;+	const double p2 = 112.0792914978709;+	const double p3 = 33.912866078383;+	const double p4 = 6.37396220353165;+	const double p5 = .7003830644436881;+	const double p6 = .03526249659989109;+	const double q0 = 440.4137358247522;+	const double q1 = 793.8265125199484;+	const double q2 = 637.3336333788311;+	const double q3 = 296.5642487796737;+	const double q4 = 86.78073220294608;+	const double q5 = 16.06417757920695;+	const double q6 = 1.755667163182642;+	const double q7 = .08838834764831844;+	double expntl, z, p;+	z = fabs(x) * M_SQRT2;+	if (z > 37.) return x > 0.? 0. : 2.;+	expntl = exp(z * z * - .5);+	if (z < 10. / M_SQRT2) // for small z+	    p = expntl * ((((((p6 * z + p5) * z + p4) * z + p3) * z + p2) * z + p1) * z + p0)+			/ (((((((q7 * z + q6) * z + q5) * z + q4) * z + q3) * z + q2) * z + q1) * z + q0);+	else p = expntl / 2.506628274631001 / (z + 1. / (z + 2. / (z + 3. / (z + 4. / (z + .65)))));+	return x > 0.? 2. * p : 2. * (1. - p);+}++/* The following computes regularized incomplete gamma functions.+ * Formulas are taken from Wiki, with additional input from Numerical+ * Recipes in C (for modified Lentz's algorithm) and AS245+ * (http://lib.stat.cmu.edu/apstat/245).+ *+ * A good online calculator is available at:+ *+ *   http://www.danielsoper.com/statcalc/calc23.aspx+ *+ * It calculates upper incomplete gamma function, which equals+ * kf_gammaq(s,z)*tgamma(s).+ */++#define KF_GAMMA_EPS 1e-14+#define KF_TINY 1e-290++// regularized lower incomplete gamma function, by series expansion+static double _kf_gammap(double s, double z)+{+	double sum, x;+	int k;+	for (k = 1, sum = x = 1.; k < 100; ++k) {+		sum += (x *= z / (s + k));+		if (x / sum < KF_GAMMA_EPS) break;+	}+	return exp(s * log(z) - z - kf_lgamma(s + 1.) + log(sum));+}+// regularized upper incomplete gamma function, by continued fraction+static double _kf_gammaq(double s, double z)+{+	int j;+	double C, D, f;+	f = 1. + z - s; C = f; D = 0.;+	// Modified Lentz's algorithm for computing continued fraction+	// See Numerical Recipes in C, 2nd edition, section 5.2+	for (j = 1; j < 100; ++j) {+		double a = j * (s - j), b = (j<<1) + 1 + z - s, d;+		D = b + a * D;+		if (D < KF_TINY) D = KF_TINY;+		C = b + a / C;+		if (C < KF_TINY) C = KF_TINY;+		D = 1. / D;+		d = C * D;+		f *= d;+		if (fabs(d - 1.) < KF_GAMMA_EPS) break;+	}+	return exp(s * log(z) - z - kf_lgamma(s) - log(f));+}++double kf_gammap(double s, double z)+{+	return z <= 1. || z < s? _kf_gammap(s, z) : 1. - _kf_gammaq(s, z);+}++double kf_gammaq(double s, double z)+{+	return z <= 1. || z < s? 1. - _kf_gammap(s, z) : _kf_gammaq(s, z);+}++/* Regularized incomplete beta function. The method is taken from+ * Numerical Recipe in C, 2nd edition, section 6.4. The following web+ * page calculates the incomplete beta function, which equals+ * kf_betai(a,b,x) * gamma(a) * gamma(b) / gamma(a+b):+ *+ *   http://www.danielsoper.com/statcalc/calc36.aspx+ */+static double kf_betai_aux(double a, double b, double x)+{+	double C, D, f;+	int j;+	if (x == 0.) return 0.;+	if (x == 1.) return 1.;+	f = 1.; C = f; D = 0.;+	// Modified Lentz's algorithm for computing continued fraction+	for (j = 1; j < 200; ++j) {+		double aa, d;+		int m = j>>1;+		aa = (j&1)? -(a + m) * (a + b + m) * x / ((a + 2*m) * (a + 2*m + 1))+			: m * (b - m) * x / ((a + 2*m - 1) * (a + 2*m));+		D = 1. + aa * D;+		if (D < KF_TINY) D = KF_TINY;+		C = 1. + aa / C;+		if (C < KF_TINY) C = KF_TINY;+		D = 1. / D;+		d = C * D;+		f *= d;+		if (fabs(d - 1.) < KF_GAMMA_EPS) break;+	}+	return exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b) + a * log(x) + b * log(1.-x)) / a / f;+}+double kf_betai(double a, double b, double x)+{+	return x < (a + 1.) / (a + b + 2.)? kf_betai_aux(a, b, x) : 1. - kf_betai_aux(b, a, 1. - x);+}++#ifdef KF_MAIN+#include <stdio.h>+int main(int argc, char *argv[])+{+	double x = 5.5, y = 3;+	double a, b;+	printf("erfc(%lg): %lg, %lg\n", x, erfc(x), kf_erfc(x));+	printf("upper-gamma(%lg,%lg): %lg\n", x, y, kf_gammaq(y, x)*tgamma(y));+	a = 2; b = 2; x = 0.5;+	printf("incomplete-beta(%lg,%lg,%lg): %lg\n", a, b, x, kf_betai(a, b, x) / exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b)));+	return 0;+}+#endif+++// log\binom{n}{k}+static double lbinom(int n, int k)+{+    if (k == 0 || n == k) return 0;+    return lgamma(n+1) - lgamma(k+1) - lgamma(n-k+1);+}++// n11  n12  | n1_+// n21  n22  | n2_+//-----------+----+// n_1  n_2  | n++// hypergeometric distribution+static double hypergeo(int n11, int n1_, int n_1, int n)+{+    return exp(lbinom(n1_, n11) + lbinom(n-n1_, n_1-n11) - lbinom(n, n_1));+}++typedef struct {+    int n11, n1_, n_1, n;+    double p;+} hgacc_t;++// incremental version of hypergenometric distribution+static double hypergeo_acc(int n11, int n1_, int n_1, int n, hgacc_t *aux)+{+    if (n1_ || n_1 || n) {+        aux->n11 = n11; aux->n1_ = n1_; aux->n_1 = n_1; aux->n = n;+    } else { // then only n11 changed; the rest fixed+        if (n11%11 && n11 + aux->n - aux->n1_ - aux->n_1) {+            if (n11 == aux->n11 + 1) { // incremental+                aux->p *= (double)(aux->n1_ - aux->n11) / n11+                    * (aux->n_1 - aux->n11) / (n11 + aux->n - aux->n1_ - aux->n_1);+                aux->n11 = n11;+                return aux->p;+            }+            if (n11 == aux->n11 - 1) { // incremental+                aux->p *= (double)aux->n11 / (aux->n1_ - n11)+                    * (aux->n11 + aux->n - aux->n1_ - aux->n_1) / (aux->n_1 - n11);+                aux->n11 = n11;+                return aux->p;+            }+        }+        aux->n11 = n11;+    }+    aux->p = hypergeo(aux->n11, aux->n1_, aux->n_1, aux->n);+    return aux->p;+}++double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two)+{+    int i, j, max, min;+    double p, q, left, right;+    hgacc_t aux;+    int n1_, n_1, n;++    n1_ = n11 + n12; n_1 = n11 + n21; n = n11 + n12 + n21 + n22; // calculate n1_, n_1 and n+    max = (n_1 < n1_) ? n_1 : n1_; // max n11, for right tail+    min = n1_ + n_1 - n;    // not sure why n11-n22 is used instead of min(n_1,n1_)+    if (min < 0) min = 0; // min n11, for left tail+    *two = *_left = *_right = 1.;+    if (min == max) return 1.; // no need to do test+    q = hypergeo_acc(n11, n1_, n_1, n, &aux); // the probability of the current table+    // left tail+    p = hypergeo_acc(min, 0, 0, 0, &aux);+    for (left = 0., i = min + 1; p < 0.99999999 * q && i<=max; ++i) // loop until underflow+        left += p, p = hypergeo_acc(i, 0, 0, 0, &aux);+    --i;+    if (p < 1.00000001 * q) left += p;+    else --i;+    // right tail+    p = hypergeo_acc(max, 0, 0, 0, &aux);+    for (right = 0., j = max - 1; p < 0.99999999 * q && j>=0; --j) // loop until underflow+        right += p, p = hypergeo_acc(j, 0, 0, 0, &aux);+    ++j;+    if (p < 1.00000001 * q) right += p;+    else ++j;+    // two-tail+    *two = left + right;+    if (*two > 1.) *two = 1.;+    // adjust left and right+    if (abs(i - n11) < abs(j - n11)) right = 1. - left + q;+    else left = 1.0 - right + q;+    *_left = left; *_right = right;+    return q;+}+++
+ htslib-1.3.2/knetfile.c view
@@ -0,0 +1,634 @@+/* The MIT License++   Copyright (c) 2008 by Genome Research Ltd (GRL).+                 2010 by Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++/* Probably I will not do socket programming in the next few years and+   therefore I decide to heavily annotate this file, for Linux and+   Windows as well.  -ac */++#include <config.h>++#include <time.h>+#include <stdio.h>+#include <ctype.h>+#include <stdlib.h>+#include <string.h>+#include <errno.h>+#include <unistd.h>+#include <sys/types.h>++#ifndef _WIN32+#include <netdb.h>+#include <arpa/inet.h>+#include <sys/socket.h>+#endif++#include "htslib/knetfile.h"++/* In winsock.h, the type of a socket is SOCKET, which is: "typedef+ * u_int SOCKET". An invalid SOCKET is: "(SOCKET)(~0)", or signed+ * integer -1. In knetfile.c, I use "int" for socket type+ * throughout. This should be improved to avoid confusion.+ *+ * In Linux/Mac, recv() and read() do almost the same thing. You can see+ * in the header file that netread() is simply an alias of read(). In+ * Windows, however, they are different and using recv() is mandatory.+ */++/* This function tests if the file handler is ready for reading (or+ * writing if is_read==0). */+static int socket_wait(int fd, int is_read)+{+	fd_set fds, *fdr = 0, *fdw = 0;+	struct timeval tv;+	int ret;+	tv.tv_sec = 5; tv.tv_usec = 0; // 5 seconds time out+	FD_ZERO(&fds);+	FD_SET(fd, &fds);+	if (is_read) fdr = &fds;+	else fdw = &fds;+	ret = select(fd+1, fdr, fdw, 0, &tv);+#ifndef _WIN32+	if (ret == -1) perror("select");+#else+	if (ret == 0)+		fprintf(stderr, "select time-out\n");+	else if (ret == SOCKET_ERROR)+		fprintf(stderr, "select: %d\n", WSAGetLastError());+#endif+	return ret;+}++#ifndef _WIN32+/* This function does not work with Windows due to the lack of+ * getaddrinfo() in winsock. It is addapted from an example in "Beej's+ * Guide to Network Programming" (http://beej.us/guide/bgnet/). */+static int socket_connect(const char *host, const char *port)+{+#define __err_connect(func) do { perror(func); freeaddrinfo(res); return -1; } while (0)++	int ai_err, on = 1, fd;+	struct linger lng = { 0, 0 };+	struct addrinfo hints, *res = 0;+	memset(&hints, 0, sizeof(struct addrinfo));+	hints.ai_family = AF_UNSPEC;+	hints.ai_socktype = SOCK_STREAM;+	/* In Unix/Mac, getaddrinfo() is the most convenient way to get+	 * server information. */+	if ((ai_err = getaddrinfo(host, port, &hints, &res)) != 0) { fprintf(stderr, "can't resolve %s:%s: %s\n", host, port, gai_strerror(ai_err)); return -1; }+	if ((fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) __err_connect("socket");+	/* The following two setsockopt() are used by ftplib+	 * (http://nbpfaus.net/~pfau/ftplib/). I am not sure if they+	 * necessary. */+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) __err_connect("setsockopt");+	if (setsockopt(fd, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) __err_connect("setsockopt");+	if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) __err_connect("connect");+	freeaddrinfo(res);+	return fd;+}+#else+/* MinGW's printf has problem with "%lld" */+char *int64tostr(char *buf, int64_t x)+{+	int cnt;+	int i = 0;+	do {+		buf[i++] = '0' + x % 10;+		x /= 10;+	} while (x);+	buf[i] = 0;+	for (cnt = i, i = 0; i < cnt/2; ++i) {+		int c = buf[i]; buf[i] = buf[cnt-i-1]; buf[cnt-i-1] = c;+	}+	return buf;+}++int64_t strtoint64(const char *buf)+{+	int64_t x;+	for (x = 0; *buf != '\0'; ++buf)+		x = x * 10 + ((int64_t) *buf - 48);+	return x;+}+/* In windows, the first thing is to establish the TCP connection. */+int knet_win32_init()+{+	WSADATA wsaData;+	return WSAStartup(MAKEWORD(2, 2), &wsaData);+}+void knet_win32_destroy()+{+	WSACleanup();+}+/* A slightly modfied version of the following function also works on+ * Mac (and presummably Linux). However, this function is not stable on+ * my Mac. It sometimes works fine but sometimes does not. Therefore for+ * non-Windows OS, I do not use this one. */+static SOCKET socket_connect(const char *host, const char *port)+{+#define __err_connect(func)										\+	do {														\+		fprintf(stderr, "%s: %d\n", func, WSAGetLastError());	\+		return -1;												\+	} while (0)++	int on = 1;+	SOCKET fd;+	struct linger lng = { 0, 0 };+	struct sockaddr_in server;+	struct hostent *hp = 0;+	// open socket+	if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) __err_connect("socket");+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)) == -1) __err_connect("setsockopt");+	if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char*)&lng, sizeof(lng)) == -1) __err_connect("setsockopt");+	// get host info+	if (isalpha(host[0])) hp = gethostbyname(host);+	else {+		struct in_addr addr;+		addr.s_addr = inet_addr(host);+		hp = gethostbyaddr((char*)&addr, 4, AF_INET);+	}+	if (hp == 0) __err_connect("gethost");+	// connect+	server.sin_addr.s_addr = *((unsigned long*)hp->h_addr);+	server.sin_family= AF_INET;+	server.sin_port = htons(atoi(port));+	if (connect(fd, (struct sockaddr*)&server, sizeof(server)) != 0) __err_connect("connect");+	// freehostent(hp); // strangely in MSDN, hp is NOT freed (memory leak?!)+	return fd;+}+#endif++static off_t my_netread(int fd, void *buf, off_t len)+{+	off_t rest = len, curr, l = 0;+	/* recv() and read() may not read the required length of data with+	 * one call. They have to be called repeatedly. */+	while (rest) {+		if (socket_wait(fd, 1) <= 0) break; // socket is not ready for reading+		curr = netread(fd, (void*)((char*)buf + l), rest);+		/* According to the glibc manual, section 13.2, a zero returned+		 * value indicates end-of-file (EOF), which should mean that+		 * read() will not return zero if EOF has not been met but data+		 * are not immediately available. */+		if (curr == 0) break;+		l += curr; rest -= curr;+	}+	return l;+}++/*************************+ * FTP specific routines *+ *************************/++static int kftp_get_response(knetFile *ftp)+{+#ifndef _WIN32+	unsigned char c;+#else+	char c;+#endif+	int n = 0;+	char *p;+	if (socket_wait(ftp->ctrl_fd, 1) <= 0) return 0;+	while (netread(ftp->ctrl_fd, &c, 1)) { // FIXME: this is *VERY BAD* for unbuffered I/O+		//fputc(c, stderr);+		if (n >= ftp->max_response) {+			ftp->max_response = ftp->max_response? ftp->max_response<<1 : 256;+			ftp->response = (char*)realloc(ftp->response, ftp->max_response);+		}+		ftp->response[n++] = c;+		if (c == '\n') {+			if (n >= 4 && isdigit(ftp->response[0]) && isdigit(ftp->response[1]) && isdigit(ftp->response[2])+				&& ftp->response[3] != '-') break;+			n = 0;+			continue;+		}+	}+	if (n < 2) return -1;+	ftp->response[n-2] = 0;+	return strtol(ftp->response, &p, 0);+}++static int kftp_send_cmd(knetFile *ftp, const char *cmd, int is_get)+{+	if (socket_wait(ftp->ctrl_fd, 0) <= 0) return -1; // socket is not ready for writing+    int len = strlen(cmd);+	if ( netwrite(ftp->ctrl_fd, cmd, len) != len ) return -1;+	return is_get? kftp_get_response(ftp) : 0;+}++static int kftp_pasv_prep(knetFile *ftp)+{+	char *p;+	int v[6];+	kftp_send_cmd(ftp, "PASV\r\n", 1);+	for (p = ftp->response; *p && *p != '('; ++p);+	if (*p != '(') return -1;+	++p;+	sscanf(p, "%d,%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]);+	memcpy(ftp->pasv_ip, v, 4 * sizeof(int));+	ftp->pasv_port = (v[4]<<8&0xff00) + v[5];+	return 0;+}+++static int kftp_pasv_connect(knetFile *ftp)+{+	char host[80], port[10];+	if (ftp->pasv_port == 0) {+		fprintf(stderr, "[kftp_pasv_connect] kftp_pasv_prep() is not called before hand.\n");+		return -1;+	}+	sprintf(host, "%d.%d.%d.%d", ftp->pasv_ip[0], ftp->pasv_ip[1], ftp->pasv_ip[2], ftp->pasv_ip[3]);+	sprintf(port, "%d", ftp->pasv_port);+	ftp->fd = socket_connect(host, port);+	if (ftp->fd == -1) return -1;+	return 0;+}++int kftp_connect(knetFile *ftp)+{+	ftp->ctrl_fd = socket_connect(ftp->host, ftp->port);+	if (ftp->ctrl_fd == -1) return -1;+	kftp_get_response(ftp);+	kftp_send_cmd(ftp, "USER anonymous\r\n", 1);+	kftp_send_cmd(ftp, "PASS kftp@\r\n", 1);+	kftp_send_cmd(ftp, "TYPE I\r\n", 1);+	return 0;+}++int kftp_reconnect(knetFile *ftp)+{+	if (ftp->ctrl_fd != -1) {+		netclose(ftp->ctrl_fd);+		ftp->ctrl_fd = -1;+	}+	netclose(ftp->fd);+	ftp->fd = -1;+	return kftp_connect(ftp);+}++// initialize ->type, ->host, ->retr and ->size+knetFile *kftp_parse_url(const char *fn, const char *mode)+{+	knetFile *fp;+	char *p;+	int l;+	if (strstr(fn, "ftp://") != fn) return 0;+	for (p = (char*)fn + 6; *p && *p != '/'; ++p);+	if (*p != '/') return 0;+	l = p - fn - 6;+	fp = (knetFile*)calloc(1, sizeof(knetFile));+	fp->type = KNF_TYPE_FTP;+	fp->fd = -1;+	/* the Linux/Mac version of socket_connect() also recognizes a port+	 * like "ftp", but the Windows version does not. */+	fp->port = strdup("21");+	fp->host = (char*)calloc(l + 1, 1);+	if (strchr(mode, 'c')) fp->no_reconnect = 1;+	strncpy(fp->host, fn + 6, l);+	fp->retr = (char*)calloc(strlen(p) + 8, 1);+	sprintf(fp->retr, "RETR %s\r\n", p);+    fp->size_cmd = (char*)calloc(strlen(p) + 8, 1);+    sprintf(fp->size_cmd, "SIZE %s\r\n", p);+	fp->seek_offset = 0;+	return fp;+}+// place ->fd at offset off+int kftp_connect_file(knetFile *fp)+{+	int ret;+	long long file_size;+	if (fp->fd != -1) {+		netclose(fp->fd);+		if (fp->no_reconnect) kftp_get_response(fp);+	}+	kftp_pasv_prep(fp);+    kftp_send_cmd(fp, fp->size_cmd, 1);+#ifndef _WIN32+    // If the file does not exist, the response will be "550 Could not get file+    // size". Be silent on failure, hts_idx_load can be trying the existence of .csi or .tbi.+    if ( sscanf(fp->response,"%*d %lld", &file_size) != 1 ) return -1;+#else+	const char *p = fp->response;+	while (*p != ' ') ++p;+	while (*p < '0' || *p > '9') ++p;+	file_size = strtoint64(p);+#endif+	fp->file_size = file_size;+	if (fp->offset>=0) {+		char tmp[32];+#ifndef _WIN32+		sprintf(tmp, "REST %lld\r\n", (long long)fp->offset);+#else+		strcpy(tmp, "REST ");+		int64tostr(tmp + 5, fp->offset);+		strcat(tmp, "\r\n");+#endif+		kftp_send_cmd(fp, tmp, 1);+	}+	kftp_send_cmd(fp, fp->retr, 0);+	kftp_pasv_connect(fp);+	ret = kftp_get_response(fp);+	if (ret != 150) {+		fprintf(stderr, "[kftp_connect_file] %s\n", fp->response);+		netclose(fp->fd);+		fp->fd = -1;+		return -1;+	}+	fp->is_ready = 1;+	return 0;+}+++/**************************+ * HTTP specific routines *+ **************************/++knetFile *khttp_parse_url(const char *fn, const char *mode)+{+	knetFile *fp;+	char *p, *proxy, *q;+	int l;+	if (strstr(fn, "http://") != fn) return 0;+	// set ->http_host+	for (p = (char*)fn + 7; *p && *p != '/'; ++p);+	l = p - fn - 7;+	fp = (knetFile*)calloc(1, sizeof(knetFile));+	fp->http_host = (char*)calloc(l + 1, 1);+	strncpy(fp->http_host, fn + 7, l);+	fp->http_host[l] = 0;+	for (q = fp->http_host; *q && *q != ':'; ++q);+	if (*q == ':') *q++ = 0;+	// get http_proxy+	proxy = getenv("http_proxy");+	// set ->host, ->port and ->path+	if (proxy == 0) {+		fp->host = strdup(fp->http_host); // when there is no proxy, server name is identical to http_host name.+		fp->port = strdup(*q? q : "80");+		fp->path = strdup(*p? p : "/");+	} else {+		fp->host = (strstr(proxy, "http://") == proxy)? strdup(proxy + 7) : strdup(proxy);+		for (q = fp->host; *q && *q != ':'; ++q);+		if (*q == ':') *q++ = 0; +		fp->port = strdup(*q? q : "80");+		fp->path = strdup(fn);+	}+	fp->type = KNF_TYPE_HTTP;+	fp->ctrl_fd = fp->fd = -1;+	fp->seek_offset = 0;+	return fp;+}++int khttp_connect_file(knetFile *fp)+{+	int ret, l = 0;+	char *buf, *p;+	if (fp->fd != -1) netclose(fp->fd);+	fp->fd = socket_connect(fp->host, fp->port);+	buf = (char*)calloc(0x10000, 1); // FIXME: I am lazy... But in principle, 64KB should be large enough.+	l += sprintf(buf + l, "GET %s HTTP/1.0\r\nHost: %s\r\n", fp->path, fp->http_host);+    l += sprintf(buf + l, "Range: bytes=%lld-\r\n", (long long)fp->offset);+	l += sprintf(buf + l, "\r\n");+	if ( netwrite(fp->fd, buf, l) != l ) { free(buf); return -1; }+	l = 0;+	while (netread(fp->fd, buf + l, 1)) { // read HTTP header; FIXME: bad efficiency+		if (buf[l] == '\n' && l >= 3)+			if (strncmp(buf + l - 3, "\r\n\r\n", 4) == 0) break;+		++l;+	}+	buf[l] = 0;+	if (l < 14) { // prematured header+		free(buf);+		netclose(fp->fd);+		fp->fd = -1;+		return -1;+	}+	ret = strtol(buf + 8, &p, 0); // HTTP return code+	if (ret == 200 && fp->offset>0) { // 200 (complete result); then skip beginning of the file+		off_t rest = fp->offset;+		while (rest) {+			off_t l = rest < 0x10000? rest : 0x10000;+			rest -= my_netread(fp->fd, buf, l);+		}+	} else if (ret != 206 && ret != 200) {+		// failed to open file+		free(buf);+		netclose(fp->fd);+		switch (ret) {+		case 401: errno = EPERM; break;+		case 403: errno = EACCES; break;+		case 404: errno = ENOENT; break;+		case 407: errno = EPERM; break;+		case 408: errno = ETIMEDOUT; break;+		case 410: errno = ENOENT; break;+		case 503: errno = EAGAIN; break;+		case 504: errno = ETIMEDOUT; break;+		default:  errno = (ret >= 400 && ret < 500)? EINVAL : EIO; break;+		}+		fp->fd = -1;+		return -1;+	}+	free(buf);+	fp->is_ready = 1;+	return 0;+}++/********************+ * Generic routines *+ ********************/++knetFile *knet_open(const char *fn, const char *mode)+{+	knetFile *fp = 0;+	if (mode[0] != 'r') {+		fprintf(stderr, "[kftp_open] only mode \"r\" is supported.\n");+		return 0;+	}+	if (strstr(fn, "ftp://") == fn) {+		fp = kftp_parse_url(fn, mode);+		if (fp == 0) return 0;+		if (kftp_connect(fp) == -1) {+			knet_close(fp);+			return 0;+		}+		kftp_connect_file(fp);+	} else if (strstr(fn, "http://") == fn) {+		fp = khttp_parse_url(fn, mode);+		if (fp == 0) return 0;+		khttp_connect_file(fp);+	} else { // local file+#ifdef _WIN32+		/* In windows, O_BINARY is necessary. In Linux/Mac, O_BINARY may+		 * be undefined on some systems, although it is defined on my+		 * Mac and the Linux I have tested on. */+		int fd = open(fn, O_RDONLY | O_BINARY);+#else		+		int fd = open(fn, O_RDONLY);+#endif+		if (fd == -1) {+			perror("open");+			return 0;+		}+		fp = (knetFile*)calloc(1, sizeof(knetFile));+		fp->type = KNF_TYPE_LOCAL;+		fp->fd = fd;+		fp->ctrl_fd = -1;+	}+	if (fp && fp->fd == -1) {+		knet_close(fp);+		return 0;+	}+	return fp;+}++knetFile *knet_dopen(int fd, const char *mode)+{+	knetFile *fp = (knetFile*)calloc(1, sizeof(knetFile));+	fp->type = KNF_TYPE_LOCAL;+	fp->fd = fd;+	return fp;+}++ssize_t knet_read(knetFile *fp, void *buf, size_t len)+{+	off_t l = 0;+	if (fp->fd == -1) return 0;+	if (fp->type == KNF_TYPE_FTP) {+		if (fp->is_ready == 0) {+			if (!fp->no_reconnect) kftp_reconnect(fp);+			kftp_connect_file(fp);+		}+	} else if (fp->type == KNF_TYPE_HTTP) {+		if (fp->is_ready == 0)+			khttp_connect_file(fp);+	}+	if (fp->type == KNF_TYPE_LOCAL) { // on Windows, the following block is necessary; not on UNIX+		size_t rest = len;+		ssize_t curr;+		while (rest) {+			do {+				curr = read(fp->fd, (void*)((char*)buf + l), rest);+			} while (curr < 0 && EINTR == errno);+			if (curr < 0) return -1;+			if (curr == 0) break;+			l += curr; rest -= curr;+		}+	} else l = my_netread(fp->fd, buf, len);+	fp->offset += l;+	return l;+}++off_t knet_seek(knetFile *fp, off_t off, int whence)+{+	if (whence == SEEK_SET && off == fp->offset) return 0;+	if (fp->type == KNF_TYPE_LOCAL) {+		/* Be aware that lseek() returns the offset after seeking, while fseek() returns zero on success. */+		off_t offset = lseek(fp->fd, off, whence);+		if (offset == -1) return -1;+		fp->offset = offset;+		return fp->offset;+	} else if (fp->type == KNF_TYPE_FTP) {+		if (whence == SEEK_CUR) fp->offset += off;+		else if (whence == SEEK_SET) fp->offset = off;+		else if (whence == SEEK_END) fp->offset = fp->file_size + off;+		else return -1;+		fp->is_ready = 0;+		return fp->offset;+	} else if (fp->type == KNF_TYPE_HTTP) {+		if (whence == SEEK_END) { // FIXME: can we allow SEEK_END in future?+			fprintf(stderr, "[knet_seek] SEEK_END is not supported for HTTP. Offset is unchanged.\n");+			errno = ESPIPE;+			return -1;+		}+		if (whence == SEEK_CUR) fp->offset += off;+		else if (whence == SEEK_SET) fp->offset = off;+		else return -1;+		fp->is_ready = 0;+		return fp->offset;+	}+	errno = EINVAL;+	fprintf(stderr,"[knet_seek] %s\n", strerror(errno));+	return -1;+}++int knet_close(knetFile *fp)+{+	if (fp == 0) return 0;+	if (fp->ctrl_fd != -1) netclose(fp->ctrl_fd); // FTP specific+	if (fp->fd != -1) {+		/* On Linux/Mac, netclose() is an alias of close(), but on+		 * Windows, it is an alias of closesocket(). */+		if (fp->type == KNF_TYPE_LOCAL) close(fp->fd);+		else netclose(fp->fd);+	}+	free(fp->host); free(fp->port);+	free(fp->response); free(fp->retr); // FTP specific+	free(fp->path); free(fp->http_host); // HTTP specific+	free(fp);+	return 0;+}++#ifdef KNETFILE_MAIN+int main(void)+{+	char *buf;+	knetFile *fp;+	int type = 4, l;+#ifdef _WIN32+	knet_win32_init();+#endif+	buf = calloc(0x100000, 1);+	if (type == 0) {+		fp = knet_open("knetfile.c", "r");+		knet_seek(fp, 1000, SEEK_SET);+	} else if (type == 1) { // NCBI FTP, large file+		fp = knet_open("ftp://ftp.ncbi.nih.gov/1000genomes/ftp/data/NA12878/alignment/NA12878.chrom6.SLX.SRP000032.2009_06.bam", "r");+		knet_seek(fp, 2500000000ll, SEEK_SET);+		l = knet_read(fp, buf, 255);+	} else if (type == 2) {+		fp = knet_open("ftp://ftp.sanger.ac.uk/pub4/treefam/tmp/index.shtml", "r");+		knet_seek(fp, 1000, SEEK_SET);+	} else if (type == 3) {+		fp = knet_open("http://www.sanger.ac.uk/Users/lh3/index.shtml", "r");+		knet_seek(fp, 1000, SEEK_SET);+	} else if (type == 4) {+		fp = knet_open("http://www.sanger.ac.uk/Users/lh3/ex1.bam", "r");+		knet_read(fp, buf, 10000);+		knet_seek(fp, 20000, SEEK_SET);+		knet_seek(fp, 10000, SEEK_SET);+		l = knet_read(fp, buf+10000, 10000000) + 10000;+	}+	if (type != 4 && type != 1) {+		knet_read(fp, buf, 255);+		buf[255] = 0;+		printf("%s\n", buf);+	} else write(fileno(stdout), buf, l);+	knet_close(fp);+	free(buf);+	return 0;+}+#endif
+ htslib-1.3.2/kstring.c view
@@ -0,0 +1,276 @@+/* The MIT License++   Copyright (C) 2011 by Attractive Chaos <attractor@live.co.uk>++   Permission is hereby granted, free of charge, to any person obtaining+   a copy of this software and associated documentation files (the+   "Software"), to deal in the Software without restriction, including+   without limitation the rights to use, copy, modify, merge, publish,+   distribute, sublicense, and/or sell copies of the Software, and to+   permit persons to whom the Software is furnished to do so, subject to+   the following conditions:++   The above copyright notice and this permission notice shall be+   included in all copies or substantial portions of the Software.++   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+   SOFTWARE.+*/++#include <config.h>++#include <stdarg.h>+#include <stdio.h>+#include <ctype.h>+#include <string.h>+#include <stdint.h>+#include "htslib/kstring.h"++int kvsprintf(kstring_t *s, const char *fmt, va_list ap)+{+	va_list args;+	int l;+	va_copy(args, ap);+	l = vsnprintf(s->s + s->l, s->m - s->l, fmt, args); // This line does not work with glibc 2.0. See `man snprintf'.+	va_end(args);+	if (l + 1 > s->m - s->l) {+		s->m = s->l + l + 2;+		kroundup32(s->m);+		s->s = (char*)realloc(s->s, s->m);+		va_copy(args, ap);+		l = vsnprintf(s->s + s->l, s->m - s->l, fmt, args);+		va_end(args);+	}+	s->l += l;+	return l;+}++int ksprintf(kstring_t *s, const char *fmt, ...)+{+	va_list ap;+	int l;+	va_start(ap, fmt);+	l = kvsprintf(s, fmt, ap);+	va_end(ap);+	return l;+}++char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux)+{+	const char *p, *start;+	if (sep) { // set up the table+		if (str == 0 && (aux->tab[0]&1)) return 0; // no need to set up if we have finished+		aux->finished = 0;+		if (sep[1]) {+			aux->sep = -1;+			aux->tab[0] = aux->tab[1] = aux->tab[2] = aux->tab[3] = 0;+			for (p = sep; *p; ++p) aux->tab[*p>>6] |= 1ull<<(*p&0x3f);+		} else aux->sep = sep[0];+	}+	if (aux->finished) return 0;+	else if (str) aux->p = str - 1, aux->finished = 0;+	if (aux->sep < 0) {+		for (p = start = aux->p + 1; *p; ++p)+			if (aux->tab[*p>>6]>>(*p&0x3f)&1) break;+	} else {+		for (p = start = aux->p + 1; *p; ++p)+			if (*p == aux->sep) break;+	}+	aux->p = p; // end of token+	if (*p == 0) aux->finished = 1; // no more tokens+	return (char*)start;+}++// s MUST BE a null terminated string; l = strlen(s)+int ksplit_core(char *s, int delimiter, int *_max, int **_offsets)+{+	int i, n, max, last_char, last_start, *offsets, l;+	n = 0; max = *_max; offsets = *_offsets;+	l = strlen(s);+	+#define __ksplit_aux do {						\+		if (_offsets) {						\+			s[i] = 0;					\+			if (n == max) {					\+				int *tmp;				\+				max = max? max<<1 : 2;			\+				if ((tmp = (int*)realloc(offsets, sizeof(int) * max))) {  \+					offsets = tmp;			\+				} else	{				\+					free(offsets);			\+					*_offsets = NULL;		\+					return 0;			\+				}					\+			}						\+			offsets[n++] = last_start;			\+		} else ++n;						\+	} while (0)++	for (i = 0, last_char = last_start = 0; i <= l; ++i) {+		if (delimiter == 0) {+			if (isspace(s[i]) || s[i] == 0) {+				if (isgraph(last_char)) __ksplit_aux; // the end of a field+			} else {+				if (isspace(last_char) || last_char == 0) last_start = i;+			}+		} else {+			if (s[i] == delimiter || s[i] == 0) {+				if (last_char != 0 && last_char != delimiter) __ksplit_aux; // the end of a field+			} else {+				if (last_char == delimiter || last_char == 0) last_start = i;+			}+		}+		last_char = s[i];+	}+	*_max = max; *_offsets = offsets;+	return n;+}++int kgetline(kstring_t *s, kgets_func *fgets_fn, void *fp)+{+	size_t l0 = s->l;++	while (s->l == l0 || s->s[s->l-1] != '\n') {+		if (s->m - s->l < 200) ks_resize(s, s->m + 200);+		if (fgets_fn(s->s + s->l, s->m - s->l, fp) == NULL) break;+		s->l += strlen(s->s + s->l);+	}++	if (s->l == l0) return EOF;++	if (s->l > l0 && s->s[s->l-1] == '\n') {+		s->l--;+		if (s->l > l0 && s->s[s->l-1] == '\r') s->l--;+	}+	s->s[s->l] = '\0';+	return 0;+}++/**********************+ * Boyer-Moore search *+ **********************/++typedef unsigned char ubyte_t;++// reference: http://www-igm.univ-mlv.fr/~lecroq/string/node14.html+static int *ksBM_prep(const ubyte_t *pat, int m)+{+	int i, *suff, *prep, *bmGs, *bmBc;+	prep = (int*)calloc(m + 256, sizeof(int));+	bmGs = prep; bmBc = prep + m;+	{ // preBmBc()+		for (i = 0; i < 256; ++i) bmBc[i] = m;+		for (i = 0; i < m - 1; ++i) bmBc[pat[i]] = m - i - 1;+	}+	suff = (int*)calloc(m, sizeof(int));+	{ // suffixes()+		int f = 0, g;+		suff[m - 1] = m;+		g = m - 1;+		for (i = m - 2; i >= 0; --i) {+			if (i > g && suff[i + m - 1 - f] < i - g)+				suff[i] = suff[i + m - 1 - f];+			else {+				if (i < g) g = i;+				f = i;+				while (g >= 0 && pat[g] == pat[g + m - 1 - f]) --g;+				suff[i] = f - g;+			}+		}+	}+	{ // preBmGs()+		int j = 0;+		for (i = 0; i < m; ++i) bmGs[i] = m;+		for (i = m - 1; i >= 0; --i)+			if (suff[i] == i + 1)+				for (; j < m - 1 - i; ++j)+					if (bmGs[j] == m)+						bmGs[j] = m - 1 - i;+		for (i = 0; i <= m - 2; ++i)+			bmGs[m - 1 - suff[i]] = m - 1 - i;+	}+	free(suff);+	return prep;+}++void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep)+{+	int i, j, *prep = 0, *bmGs, *bmBc;+	const ubyte_t *str, *pat;+	str = (const ubyte_t*)_str; pat = (const ubyte_t*)_pat;+	prep = (_prep == 0 || *_prep == 0)? ksBM_prep(pat, m) : *_prep;+	if (_prep && *_prep == 0) *_prep = prep;+	bmGs = prep; bmBc = prep + m;+	j = 0;+	while (j <= n - m) {+		for (i = m - 1; i >= 0 && pat[i] == str[i+j]; --i);+		if (i >= 0) {+			int max = bmBc[str[i+j]] - m + 1 + i;+			if (max < bmGs[i]) max = bmGs[i];+			j += max;+		} else return (void*)(str + j);+	}+	if (_prep == 0) free(prep);+	return 0;+}++char *kstrstr(const char *str, const char *pat, int **_prep)+{+	return (char*)kmemmem(str, strlen(str), pat, strlen(pat), _prep);+}++char *kstrnstr(const char *str, const char *pat, int n, int **_prep)+{+	return (char*)kmemmem(str, n, pat, strlen(pat), _prep);+}++/***********************+ * The main() function *+ ***********************/++#ifdef KSTRING_MAIN+#include <stdio.h>+int main()+{+	kstring_t *s;+	int *fields, n, i;+	ks_tokaux_t aux;+	char *p;+	s = (kstring_t*)calloc(1, sizeof(kstring_t));+	// test ksprintf()+	ksprintf(s, " abcdefg:    %d ", 100);+	printf("'%s'\n", s->s);+	// test ksplit()+	fields = ksplit(s, 0, &n);+	for (i = 0; i < n; ++i)+		printf("field[%d] = '%s'\n", i, s->s + fields[i]);+	// test kstrtok()+	s->l = 0;+	for (p = kstrtok("ab:cde:fg/hij::k", ":/", &aux); p; p = kstrtok(0, 0, &aux)) {+		kputsn(p, aux.p - p, s);+		kputc('\n', s);+	}+	printf("%s", s->s);+	// free+	free(s->s); free(s); free(fields);++	{+		static char *str = "abcdefgcdgcagtcakcdcd";+		static char *pat = "cd";+		char *ret, *s = str;+		int *prep = 0;+		while ((ret = kstrstr(s, pat, &prep)) != 0) {+			printf("match: %s\n", ret);+			s = ret + prep[0];+		}+		free(prep);+	}+	return 0;+}+#endif
+ htslib-1.3.2/md5.c view
@@ -0,0 +1,386 @@+/*+ * Trivial amendments by James Bonfield <jkb@sanger.ac.uk> to provide an+ * HTSlib interface. 2015.+ *+ * Externally our API uses an opaque hts_md5_context structure.+ *+ * Internally either this gets defined and used with the routines here+ * or it remains incomplete and is cast to the OpenSSL MD5_CTX structure+ * and used by routines from OpenSSL.+ */++/*+ * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.+ * MD5 Message-Digest Algorithm (RFC 1321).+ *+ * Homepage:+ * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5+ *+ * Author:+ * Alexander Peslyak, better known as Solar Designer <solar at openwall.com>+ *+ * This software was written by Alexander Peslyak in 2001.  No copyright is+ * claimed, and the software is hereby placed in the public domain.+ * In case this attempt to disclaim copyright and place the software in the+ * public domain is deemed null and void, then the software is+ * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the+ * general public under the following terms:+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted.+ *+ * There's ABSOLUTELY NO WARRANTY, express or implied.+ *+ * (This is a heavily cut-down "BSD license".)+ *+ * This differs from Colin Plumb's older public domain implementation in that+ * no exactly 32-bit integer data type is required (any 32-bit or wider+ * unsigned integer data type will do), there's no compile-time endianness+ * configuration, and the function prototypes match OpenSSL's.  No code from+ * Colin Plumb's implementation has been reused; this comment merely compares+ * the properties of the two independent implementations.+ *+ * The primary goals of this implementation are portability and ease of use.+ * It is meant to be fast, but not as fast as possible.  Some known+ * optimizations are not included to reduce source code size and avoid+ * compile-time configuration.+ */++#include <config.h>++#include <stdlib.h>+#include "htslib/hts.h"++#ifndef HAVE_OPENSSL++#include <string.h>++/* Any 32-bit or wider unsigned integer data type will do */+typedef unsigned int hts_md5_u32plus;++struct hts_md5_context {+	hts_md5_u32plus lo, hi;+	hts_md5_u32plus a, b, c, d;+	unsigned char buffer[64];+	hts_md5_u32plus block[16];+};++/*+ * The basic MD5 functions.+ *+ * F and G are optimized compared to their RFC 1321 definitions for+ * architectures that lack an AND-NOT instruction, just like in Colin Plumb's+ * implementation.+ */+#define F(x, y, z)			((z) ^ ((x) & ((y) ^ (z))))+#define G(x, y, z)			((y) ^ ((z) & ((x) ^ (y))))+#define H(x, y, z)			(((x) ^ (y)) ^ (z))+#define H2(x, y, z)			((x) ^ ((y) ^ (z)))+#define I(x, y, z)			((y) ^ ((x) | ~(z)))++/*+ * The MD5 transformation for all four rounds.+ */+#define STEP(f, a, b, c, d, x, t, s) \+	(a) += f((b), (c), (d)) + (x) + (t); \+	(a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \+	(a) += (b);++/*+ * SET reads 4 input bytes in little-endian byte order and stores them+ * in a properly aligned word in host byte order.+ *+ * The check for little-endian architectures that tolerate unaligned+ * memory accesses is just an optimization.  Nothing will break if it+ * doesn't work.+ */+#if defined(__i386__) || defined(__x86_64__) || defined(__vax__)+#define SET(n) \+	(*(hts_md5_u32plus *)&ptr[(n) * 4])+#define GET(n) \+	SET(n)+#else+#define SET(n) \+	(ctx->block[(n)] = \+	(hts_md5_u32plus)ptr[(n) * 4] | \+	((hts_md5_u32plus)ptr[(n) * 4 + 1] << 8) | \+	((hts_md5_u32plus)ptr[(n) * 4 + 2] << 16) | \+	((hts_md5_u32plus)ptr[(n) * 4 + 3] << 24))+#define GET(n) \+	(ctx->block[(n)])+#endif++/*+ * This processes one or more 64-byte data blocks, but does NOT update+ * the bit counters.  There are no alignment requirements.+ */+static const void *body(hts_md5_context *ctx, const void *data, unsigned long size)+{+	const unsigned char *ptr;+	hts_md5_u32plus a, b, c, d;+	hts_md5_u32plus saved_a, saved_b, saved_c, saved_d;++	ptr = (const unsigned char *)data;++	a = ctx->a;+	b = ctx->b;+	c = ctx->c;+	d = ctx->d;++	do {+		saved_a = a;+		saved_b = b;+		saved_c = c;+		saved_d = d;++/* Round 1 */+		STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)+		STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)+		STEP(F, c, d, a, b, SET(2), 0x242070db, 17)+		STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)+		STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)+		STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)+		STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)+		STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)+		STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)+		STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)+		STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)+		STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)+		STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)+		STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)+		STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)+		STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)++/* Round 2 */+		STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)+		STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)+		STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)+		STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)+		STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)+		STEP(G, d, a, b, c, GET(10), 0x02441453, 9)+		STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)+		STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)+		STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)+		STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)+		STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)+		STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)+		STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)+		STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)+		STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)+		STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)++/* Round 3 */+		STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)+		STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11)+		STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)+		STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23)+		STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)+		STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11)+		STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)+		STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23)+		STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)+		STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11)+		STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)+		STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23)+		STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)+		STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11)+		STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)+		STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23)++/* Round 4 */+		STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)+		STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)+		STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)+		STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)+		STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)+		STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)+		STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)+		STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)+		STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)+		STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)+		STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)+		STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)+		STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)+		STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)+		STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)+		STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)++		a += saved_a;+		b += saved_b;+		c += saved_c;+		d += saved_d;++		ptr += 64;+	} while (size -= 64);++	ctx->a = a;+	ctx->b = b;+	ctx->c = c;+	ctx->d = d;++	return ptr;+}++void hts_md5_reset(hts_md5_context *ctx)+{+	ctx->a = 0x67452301;+	ctx->b = 0xefcdab89;+	ctx->c = 0x98badcfe;+	ctx->d = 0x10325476;++	ctx->lo = 0;+	ctx->hi = 0;+}++void hts_md5_update(hts_md5_context *ctx, const void *data, unsigned long size)+{+	hts_md5_u32plus saved_lo;+	unsigned long used, available;++	saved_lo = ctx->lo;+	if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)+		ctx->hi++;+	ctx->hi += size >> 29;++	used = saved_lo & 0x3f;++	if (used) {+		available = 64 - used;++		if (size < available) {+			memcpy(&ctx->buffer[used], data, size);+			return;+		}++		memcpy(&ctx->buffer[used], data, available);+		data = (const unsigned char *)data + available;+		size -= available;+		body(ctx, ctx->buffer, 64);+	}++	if (size >= 64) {+		data = body(ctx, data, size & ~(unsigned long)0x3f);+		size &= 0x3f;+	}++	memcpy(ctx->buffer, data, size);+}++void hts_md5_final(unsigned char *result, hts_md5_context *ctx)+{+	unsigned long used, available;++	used = ctx->lo & 0x3f;++	ctx->buffer[used++] = 0x80;++	available = 64 - used;++	if (available < 8) {+		memset(&ctx->buffer[used], 0, available);+		body(ctx, ctx->buffer, 64);+		used = 0;+		available = 64;+	}++	memset(&ctx->buffer[used], 0, available - 8);++	ctx->lo <<= 3;+	ctx->buffer[56] = ctx->lo;+	ctx->buffer[57] = ctx->lo >> 8;+	ctx->buffer[58] = ctx->lo >> 16;+	ctx->buffer[59] = ctx->lo >> 24;+	ctx->buffer[60] = ctx->hi;+	ctx->buffer[61] = ctx->hi >> 8;+	ctx->buffer[62] = ctx->hi >> 16;+	ctx->buffer[63] = ctx->hi >> 24;++	body(ctx, ctx->buffer, 64);++	result[0] = ctx->a;+	result[1] = ctx->a >> 8;+	result[2] = ctx->a >> 16;+	result[3] = ctx->a >> 24;+	result[4] = ctx->b;+	result[5] = ctx->b >> 8;+	result[6] = ctx->b >> 16;+	result[7] = ctx->b >> 24;+	result[8] = ctx->c;+	result[9] = ctx->c >> 8;+	result[10] = ctx->c >> 16;+	result[11] = ctx->c >> 24;+	result[12] = ctx->d;+	result[13] = ctx->d >> 8;+	result[14] = ctx->d >> 16;+	result[15] = ctx->d >> 24;++	memset(ctx, 0, sizeof(*ctx));+}+++hts_md5_context *hts_md5_init(void)+{+    hts_md5_context *ctx = malloc(sizeof(*ctx));+    if (!ctx)+        return NULL;++    hts_md5_reset(ctx);+    return ctx;+}++#else++#include <openssl/md5.h>+#include <assert.h>++/*+ * Wrappers around the OpenSSL libcrypto.so MD5 implementation.+ *+ * These are here to ensure they end up in the symbol table of the+ * library regardless of the static inline in the headers.+ */+hts_md5_context *hts_md5_init(void)+{+    MD5_CTX *ctx = malloc(sizeof(*ctx));+    if (!ctx)+        return NULL;++    MD5_Init(ctx);++    return (hts_md5_context *)ctx;+}++void hts_md5_reset(hts_md5_context *ctx)+{+    MD5_Init((MD5_CTX *)ctx);+}++void hts_md5_update(hts_md5_context *ctx, const void *data, unsigned long size)+{+    MD5_Update((MD5_CTX *)ctx, data, size);+}++void hts_md5_final(unsigned char *result, hts_md5_context *ctx)+{+    MD5_Final(result, (MD5_CTX *)ctx);+}++#endif++void hts_md5_destroy(hts_md5_context *ctx)+{+    if (!ctx)+        return;++    free(ctx);+}++void hts_md5_hex(char *hex, const unsigned char *digest)+{+    int i;+    for (i = 0; i < 16; i++) {+        hex[i*2+0] = "0123456789abcdef"[(digest[i]>>4)&0xf];+        hex[i*2+1] = "0123456789abcdef"[digest[i]&0xf];+    }+    hex[32] = 0;+}
+ htslib-1.3.2/regidx.c view
@@ -0,0 +1,340 @@+/* +    Copyright (C) 2014 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++    Permission is hereby granted, free of charge, to any person obtaining a copy+    of this software and associated documentation files (the "Software"), to deal+    in the Software without restriction, including without limitation the rights+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+    copies of the Software, and to permit persons to whom the Software is+    furnished to do so, subject to the following conditions:+    +    The above copyright notice and this permission notice shall be included in+    all copies or substantial portions of the Software.+    +    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+    THE SOFTWARE.+*/++#include <config.h>++#include "htslib/hts.h"+#include "htslib/kstring.h"+#include "htslib/kseq.h"+#include "htslib/khash_str2int.h"+#include "htslib/regidx.h"++#define LIDX_SHIFT 13   // number of insignificant index bits++// List of regions for one chromosome+typedef struct+{+    int *idx, nidx;+    int nregs, mregs;   // n:used, m:alloced+    reg_t *regs;+    void *payload;+}+reglist_t;++// Container of all sequences+struct _regidx_t+{+    int nseq, mseq;     // n:used, m:alloced+    reglist_t *seq;     // regions for each sequence+    void *seq2regs;     // hash for fast lookup from chr name to regions+    char **seq_names;+    regidx_free_f free;     // function to free any data allocated by regidx_parse_f+    regidx_parse_f parse;   // parse one input line+    void *usr;              // user data to pass to regidx_parse_f++    // temporary data for index initialization+    kstring_t str;+    int rid_prev, start_prev, end_prev;+    int payload_size;+    void *payload;+};++int regidx_seq_nregs(regidx_t *idx, const char *seq)+{+    int iseq;+    if ( khash_str2int_get(idx->seq2regs, seq, &iseq)!=0 ) return 0; // no such sequence+    return idx->seq[iseq].nregs;+}++int regidx_nregs(regidx_t *idx)+{+    int i, nregs = 0;+    for (i=0; i<idx->nseq; i++) nregs += idx->seq[i].nregs;+    return nregs;+}++char **regidx_seq_names(regidx_t *idx, int *n)+{+    *n = idx->nseq;+    return idx->seq_names;+}++int _regidx_build_index(regidx_t *idx)+{+    int iseq;+    for (iseq=0; iseq<idx->nseq; iseq++)+    {+        reglist_t *list = &idx->seq[iseq];+        int j,k, imax = 0;   // max index bin+        for (j=0; j<list->nregs; j++)+        {+            int ibeg = list->regs[j].start >> LIDX_SHIFT;+            int iend = list->regs[j].end >> LIDX_SHIFT;+            if ( imax < iend + 1 )+            {+                int old_imax = imax; +                imax = iend + 1;+                kroundup32(imax);+                list->idx = (int*) realloc(list->idx, imax*sizeof(int));+                for (k=old_imax; k<imax; k++) list->idx[k] = -1;+            }+            if ( ibeg==iend )+            {+                if ( list->idx[ibeg]<0 ) list->idx[ibeg] = j;+            }+            else+            {+                for (k=ibeg; k<=iend; k++)+                    if ( list->idx[k]<0 ) list->idx[k] = j;+            }+            list->nidx = iend + 1;+        }+    }+    return 0;+}++int regidx_insert(regidx_t *idx, char *line)+{+    if ( !line )+        return _regidx_build_index(idx);++    char *chr_from, *chr_to;+    reg_t reg;+    int ret = idx->parse(line,&chr_from,&chr_to,&reg,idx->payload,idx->usr);+    if ( ret==-2 ) return -1;   // error+    if ( ret==-1 ) return 0;    // skip the line++    int rid;+    idx->str.l = 0;+    kputsn(chr_from, chr_to-chr_from+1, &idx->str);+    if ( khash_str2int_get(idx->seq2regs, idx->str.s, &rid)!=0 )+    {+        idx->nseq++;+        int m_prev = idx->mseq;+        hts_expand0(reglist_t,idx->nseq,idx->mseq,idx->seq);+        hts_expand0(char*,idx->nseq,m_prev,idx->seq_names);+        idx->seq_names[idx->nseq-1] = strdup(idx->str.s);+        rid = khash_str2int_inc(idx->seq2regs, idx->seq_names[idx->nseq-1]);+    }++    reglist_t *list = &idx->seq[rid];+    list->nregs++;+    int m_prev = list->mregs;+    hts_expand(reg_t,list->nregs,list->mregs,list->regs);+    list->regs[list->nregs-1] = reg;+    if ( idx->payload_size )+    {+        if ( m_prev < list->mregs ) list->payload = realloc(list->payload,idx->payload_size*list->mregs);+        memcpy(list->payload + idx->payload_size*(list->nregs-1), idx->payload, idx->payload_size);+    }++    if ( idx->rid_prev==rid )+    {+        if ( idx->start_prev > reg.start || (idx->start_prev==reg.start && idx->end_prev>reg.end) ) +        { +            fprintf(stderr,"The regions are not sorted: %s:%d-%d is before %s:%d-%d\n", +                idx->str.s,idx->start_prev+1,idx->end_prev+1,idx->str.s,reg.start+1,reg.end+1); +            return -1;+        }+    }+    idx->rid_prev = rid;+    idx->start_prev = reg.start;+    idx->end_prev = reg.end;+    return 0;+}++regidx_t *regidx_init(const char *fname, regidx_parse_f parser, regidx_free_f free_f, size_t payload_size, void *usr_dat)+{+    if ( !parser )+    {+        if ( !fname ) parser = regidx_parse_tab;+        else+        {+            int len = strlen(fname);+            if ( len>=7 && !strcasecmp(".bed.gz",fname+len-7) )+                parser = regidx_parse_bed;+            else if ( len>=8 && !strcasecmp(".bed.bgz",fname+len-8) )+                parser = regidx_parse_bed;+            else if ( len>=4 && !strcasecmp(".bed",fname+len-4) )+                parser = regidx_parse_bed;+            else+                parser = regidx_parse_tab;+        }+    }++    regidx_t *idx = (regidx_t*) calloc(1,sizeof(regidx_t));+    idx->free  = free_f;+    idx->parse = parser;+    idx->usr   = usr_dat;+    idx->seq2regs = khash_str2int_init();+    idx->rid_prev   = -1;+    idx->start_prev = -1;+    idx->end_prev   = -1;+    idx->payload_size = payload_size;+    if ( payload_size ) idx->payload = malloc(payload_size);++    if ( !fname ) return idx;+    +    kstring_t str = {0,0,0};++    htsFile *fp = hts_open(fname,"r");+    if ( !fp ) goto error;++    while ( hts_getline(fp, KS_SEP_LINE, &str) > 0 )+    {+        if ( regidx_insert(idx, str.s) ) goto error;+    }+    regidx_insert(idx, NULL);+    +    free(str.s);+    hts_close(fp);+    return idx;++error:+    free(str.s);+    if ( fp ) hts_close(fp);+    regidx_destroy(idx);+    return NULL;+}++void regidx_destroy(regidx_t *idx)+{+    int i, j;+    for (i=0; i<idx->nseq; i++)+    {+        reglist_t *list = &idx->seq[i];+        if ( idx->free )+        {+            for (j=0; j<list->nregs; j++)+                idx->free(list->payload + idx->payload_size*j);+        }+        free(list->payload);+        free(list->regs);+        free(list->idx);+    }+    free(idx->seq_names);+    free(idx->seq);+    free(idx->str.s);+    free(idx->payload);+    khash_str2int_destroy_free(idx->seq2regs);+    free(idx);+}++int regidx_overlap(regidx_t *idx, const char *chr, uint32_t from, uint32_t to, regitr_t *itr)+{+    if ( itr ) itr->i = itr->n = 0;++    int iseq;+    if ( khash_str2int_get(idx->seq2regs, chr, &iseq)!=0 ) return 0; // no such sequence++    reglist_t *list = &idx->seq[iseq];+    if ( !list->nregs ) return 0;++    int i, ibeg = from>>LIDX_SHIFT; +    int ireg = ibeg < list->nidx ? list->idx[ibeg] : list->idx[ list->nidx - 1 ];+    if ( ireg < 0 )+    {+        // linear search; if slow, replace with binary search+        if ( ibeg > list->nidx ) ibeg = list->nidx;+        for (i=ibeg - 1; i>=0; i--)+            if ( list->idx[i] >=0 ) break;+        ireg = i>=0 ? list->idx[i] : 0;+    }+    for (i=ireg; i<list->nregs; i++)+    {+        if ( list->regs[i].start > to ) return 0;   // no match+        if ( list->regs[i].end >= from && list->regs[i].start <= to ) break; // found+    }++    if ( i>=list->nregs ) return 0;   // no match++    if ( !itr ) return 1;++    itr->i = 0;+    itr->n = list->nregs - i;+    itr->reg = &idx->seq[iseq].regs[i];+    if ( idx->payload_size )+        itr->payload = idx->seq[iseq].payload + i*idx->payload_size;+    else+        itr->payload = NULL;++    return 1;+}++int regidx_parse_bed(const char *line, char **chr_beg, char **chr_end, reg_t *reg, void *payload, void *usr)+{+    char *ss = (char*) line;+    while ( *ss && isspace(*ss) ) ss++;+    if ( !*ss ) return -1;      // skip blank lines+    if ( *ss=='#' ) return -1;  // skip comments+    +    char *se = ss;+    while ( *se && !isspace(*se) ) se++;+    if ( !*se ) { fprintf(stderr,"Could not parse bed line: %s\n", line); return -2; }++    *chr_beg = ss;+    *chr_end = se-1;++    ss = se+1;+    reg->start = hts_parse_decimal(ss, &se, 0);+    if ( ss==se ) { fprintf(stderr,"Could not parse bed line: %s\n", line); return -2; }++    ss = se+1;+    reg->end = hts_parse_decimal(ss, &se, 0) - 1;+    if ( ss==se ) { fprintf(stderr,"Could not parse bed line: %s\n", line); return -2; }+    +    return 0;+}++int regidx_parse_tab(const char *line, char **chr_beg, char **chr_end, reg_t *reg, void *payload, void *usr)+{+    char *ss = (char*) line;+    while ( *ss && isspace(*ss) ) ss++;+    if ( !*ss ) return -1;      // skip blank lines+    if ( *ss=='#' ) return -1;  // skip comments+    +    char *se = ss;+    while ( *se && !isspace(*se) ) se++;+    if ( !*se ) { fprintf(stderr,"Could not parse bed line: %s\n", line); return -2; }++    *chr_beg = ss;+    *chr_end = se-1;++    ss = se+1;+    reg->start = hts_parse_decimal(ss, &se, 0) - 1;+    if ( ss==se ) { fprintf(stderr,"Could not parse bed line: %s\n", line); return -2; }++    if ( !se[0] || !se[1] )+        reg->end = reg->start;+    else+    {+        ss = se+1;+        reg->end = hts_parse_decimal(ss, &se, 0);+        if ( ss==se ) reg->end = reg->start;+        else reg->end--;+    }+    +    return 0;+}+
+ htslib-1.3.2/sam.c view
@@ -0,0 +1,2072 @@+/*  sam.c -- SAM and BAM file I/O and manipulation.++    Copyright (C) 2008-2010, 2012-2016 Genome Research Ltd.+    Copyright (C) 2010, 2012, 2013 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <errno.h>+#include <zlib.h>+#include "htslib/sam.h"+#include "htslib/bgzf.h"+#include "cram/cram.h"+#include "hts_internal.h"+#include "htslib/hfile.h"++#include "htslib/khash.h"+KHASH_DECLARE(s2i, kh_cstr_t, int64_t)++typedef khash_t(s2i) sdict_t;++/**********************+ *** BAM header I/O ***+ **********************/++bam_hdr_t *bam_hdr_init()+{+    return (bam_hdr_t*)calloc(1, sizeof(bam_hdr_t));+}++void bam_hdr_destroy(bam_hdr_t *h)+{+    int32_t i;+    if (h == NULL) return;+    if (h->target_name) {+        for (i = 0; i < h->n_targets; ++i)+            free(h->target_name[i]);+        free(h->target_name);+        free(h->target_len);+    }+    free(h->text); free(h->cigar_tab);+    if (h->sdict) kh_destroy(s2i, (sdict_t*)h->sdict);+    free(h);+}++bam_hdr_t *bam_hdr_dup(const bam_hdr_t *h0)+{+    if (h0 == NULL) return NULL;+    bam_hdr_t *h;+    if ((h = bam_hdr_init()) == NULL) return NULL;+    // copy the simple data+    h->n_targets = h0->n_targets;+    h->ignore_sam_err = h0->ignore_sam_err;+    h->l_text = h0->l_text;+    // Then the pointery stuff+    h->cigar_tab = NULL;+    h->sdict = NULL;+    // TODO Check for memory allocation failures+    h->text = (char*)calloc(h->l_text + 1, 1);+    memcpy(h->text, h0->text, h->l_text);+    h->target_len = (uint32_t*)calloc(h->n_targets, sizeof(uint32_t));+    h->target_name = (char**)calloc(h->n_targets, sizeof(char*));+    int i;+    for (i = 0; i < h->n_targets; ++i) {+        h->target_len[i] = h0->target_len[i];+        h->target_name[i] = strdup(h0->target_name[i]);+    }+    return h;+}+++static bam_hdr_t *hdr_from_dict(sdict_t *d)+{+    bam_hdr_t *h;+    khint_t k;+    h = bam_hdr_init();+    h->sdict = d;+    h->n_targets = kh_size(d);+    // TODO Check for memory allocation failures+    h->target_len = (uint32_t*)malloc(sizeof(uint32_t) * h->n_targets);+    h->target_name = (char**)malloc(sizeof(char*) * h->n_targets);+    for (k = kh_begin(d); k != kh_end(d); ++k) {+        if (!kh_exist(d, k)) continue;+        h->target_name[kh_val(d, k)>>32] = (char*)kh_key(d, k);+        h->target_len[kh_val(d, k)>>32]  = kh_val(d, k)<<32>>32;+        kh_val(d, k) >>= 32;+    }+    return h;+}++bam_hdr_t *bam_hdr_read(BGZF *fp)+{+    bam_hdr_t *h;+    char buf[4];+    int magic_len, has_EOF;+    int32_t i, name_len, num_names = 0;+    size_t bufsize;+    ssize_t bytes;+    // check EOF+    has_EOF = bgzf_check_EOF(fp);+    if (has_EOF < 0) {+        perror("[W::bam_hdr_read] bgzf_check_EOF");+    } else if (has_EOF == 0 && hts_verbose >= 2)+        fprintf(stderr, "[W::%s] EOF marker is absent. The input is probably truncated.\n", __func__);+    // read "BAM1"+    magic_len = bgzf_read(fp, buf, 4);+    if (magic_len != 4 || strncmp(buf, "BAM\1", 4)) {+        if (hts_verbose >= 1) fprintf(stderr, "[E::%s] invalid BAM binary header\n", __func__);+        return 0;+    }+    h = bam_hdr_init();+    if (!h) goto nomem;++    // read plain text and the number of reference sequences+    bytes = bgzf_read(fp, &h->l_text, 4);+    if (bytes != 4) goto read_err;+    if (fp->is_be) ed_swap_4p(&h->l_text);++    bufsize = ((size_t) h->l_text) + 1;+    if (bufsize < h->l_text) goto nomem; // so large that adding 1 overflowed+    h->text = (char*)malloc(bufsize);+    if (!h->text) goto nomem;+    h->text[h->l_text] = 0; // make sure it is NULL terminated+    bytes = bgzf_read(fp, h->text, h->l_text);+    if (bytes != h->l_text) goto read_err;++    bytes = bgzf_read(fp, &h->n_targets, 4);+    if (bytes != 4) goto read_err;+    if (fp->is_be) ed_swap_4p(&h->n_targets);++    if (h->n_targets < 0) goto invalid;++    // read reference sequence names and lengths+    if (h->n_targets > 0) {+        h->target_name = (char**)calloc(h->n_targets, sizeof(char*));+        if (!h->target_name) goto nomem;+        h->target_len = (uint32_t*)calloc(h->n_targets, sizeof(uint32_t));+        if (!h->target_len) goto nomem;+    }+    else {+        h->target_name = NULL;+        h->target_len = NULL;+    }++    for (i = 0; i != h->n_targets; ++i) {+        bytes = bgzf_read(fp, &name_len, 4);+        if (bytes != 4) goto read_err;+        if (fp->is_be) ed_swap_4p(&name_len);+        if (name_len <= 0) goto invalid;++        h->target_name[i] = (char*)malloc(name_len);+        if (!h->target_name[i]) goto nomem;+        num_names++;++        bytes = bgzf_read(fp, h->target_name[i], name_len);+        if (bytes != name_len) goto read_err;++        if (h->target_name[i][name_len - 1] != '\0') {+            /* Fix missing NUL-termination.  Is this being too nice?+               We could alternatively bail out with an error. */+            char *new_name;+            if (name_len == INT32_MAX) goto invalid;+            new_name = realloc(h->target_name[i], name_len + 1);+            if (new_name == NULL) goto nomem;+            h->target_name[i] = new_name;+            h->target_name[i][name_len] = '\0';+        }++        bytes = bgzf_read(fp, &h->target_len[i], 4);+        if (bytes != 4) goto read_err;+        if (fp->is_be) ed_swap_4p(&h->target_len[i]);+    }+    return h;++ nomem:+    if (hts_verbose >= 1) fprintf(stderr, "[E::%s] out of memory\n", __func__);+    goto clean;++ read_err:+    if (hts_verbose >= 1) {+        if (bytes < 0) {+            fprintf(stderr, "[E::%s] error reading BGZF stream\n", __func__);+        } else {+            fprintf(stderr, "[E::%s] truncated bam header\n", __func__);+        }+    }+    goto clean;++ invalid:+    if (hts_verbose >= 1) {+        fprintf(stderr, "[E::%s] invalid BAM binary header\n", __func__);+    }++ clean:+    if (h != NULL) {+        h->n_targets = num_names; // ensure we free only allocated target_names+        bam_hdr_destroy(h);+    }+    return NULL;+}++int bam_hdr_write(BGZF *fp, const bam_hdr_t *h)+{+    char buf[4];+    int32_t i, name_len, x;+    // write "BAM1"+    strncpy(buf, "BAM\1", 4);+    if (bgzf_write(fp, buf, 4) < 0) return -1;+    // write plain text and the number of reference sequences+    if (fp->is_be) {+        x = ed_swap_4(h->l_text);+        if (bgzf_write(fp, &x, 4) < 0) return -1;+        if (h->l_text) {+            if (bgzf_write(fp, h->text, h->l_text) < 0) return -1;+        }+        x = ed_swap_4(h->n_targets);+        if (bgzf_write(fp, &x, 4) < 0) return -1;+    } else {+        if (bgzf_write(fp, &h->l_text, 4) < 0) return -1;+        if (h->l_text) {+            if (bgzf_write(fp, h->text, h->l_text) < 0) return -1;+        }+        if (bgzf_write(fp, &h->n_targets, 4) < 0) return -1;+    }+    // write sequence names and lengths+    for (i = 0; i != h->n_targets; ++i) {+        char *p = h->target_name[i];+        name_len = strlen(p) + 1;+        if (fp->is_be) {+            x = ed_swap_4(name_len);+            if (bgzf_write(fp, &x, 4) < 0) return -1;+        } else {+            if (bgzf_write(fp, &name_len, 4) < 0) return -1;+        }+        if (bgzf_write(fp, p, name_len) < 0) return -1;+        if (fp->is_be) {+            x = ed_swap_4(h->target_len[i]);+            if (bgzf_write(fp, &x, 4) < 0) return -1;+        } else {+            if (bgzf_write(fp, &h->target_len[i], 4) < 0) return -1;+        }+    }+    if (bgzf_flush(fp) < 0) return -1;+    return 0;+}++int bam_name2id(bam_hdr_t *h, const char *ref)+{+    sdict_t *d = (sdict_t*)h->sdict;+    khint_t k;+    if (h->sdict == 0) {+        int i, absent;+        d = kh_init(s2i);+        for (i = 0; i < h->n_targets; ++i) {+            k = kh_put(s2i, d, h->target_name[i], &absent);+            kh_val(d, k) = i;+        }+        h->sdict = d;+    }+    k = kh_get(s2i, d, ref);+    return k == kh_end(d)? -1 : kh_val(d, k);+}++/*************************+ *** BAM alignment I/O ***+ *************************/++bam1_t *bam_init1()+{+    return (bam1_t*)calloc(1, sizeof(bam1_t));+}++void bam_destroy1(bam1_t *b)+{+    if (b == 0) return;+    free(b->data); free(b);+}++bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc)+{+    uint8_t *data = bdst->data;+    int m_data = bdst->m_data;   // backup data and m_data+    if (m_data < bsrc->l_data) { // double the capacity+        m_data = bsrc->l_data; kroundup32(m_data);+        data = (uint8_t*)realloc(data, m_data);+    }+    memcpy(data, bsrc->data, bsrc->l_data); // copy var-len data+    *bdst = *bsrc; // copy the rest+    // restore the backup+    bdst->m_data = m_data;+    bdst->data = data;+    return bdst;+}++bam1_t *bam_dup1(const bam1_t *bsrc)+{+    if (bsrc == NULL) return NULL;+    bam1_t *bdst = bam_init1();+    if (bdst == NULL) return NULL;+    return bam_copy1(bdst, bsrc);+}++int bam_cigar2qlen(int n_cigar, const uint32_t *cigar)+{+    int k, l;+    for (k = l = 0; k < n_cigar; ++k)+        if (bam_cigar_type(bam_cigar_op(cigar[k]))&1)+            l += bam_cigar_oplen(cigar[k]);+    return l;+}++int bam_cigar2rlen(int n_cigar, const uint32_t *cigar)+{+    int k, l;+    for (k = l = 0; k < n_cigar; ++k)+        if (bam_cigar_type(bam_cigar_op(cigar[k]))&2)+            l += bam_cigar_oplen(cigar[k]);+    return l;+}++int32_t bam_endpos(const bam1_t *b)+{+    if (!(b->core.flag & BAM_FUNMAP) && b->core.n_cigar > 0)+        return b->core.pos + bam_cigar2rlen(b->core.n_cigar, bam_get_cigar(b));+    else+        return b->core.pos + 1;+}++static inline int aux_type2size(uint8_t type)+{+    switch (type) {+    case 'A': case 'c': case 'C':+        return 1;+    case 's': case 'S':+        return 2;+    case 'i': case 'I': case 'f':+        return 4;+    case 'd':+        return 8;+    case 'Z': case 'H': case 'B':+        return type;+    default:+        return 0;+    }+}++static void swap_data(const bam1_core_t *c, int l_data, uint8_t *data, int is_host)+{+    uint8_t *s;+    uint32_t *cigar = (uint32_t*)(data + c->l_qname);+    uint32_t i, n;+    s = data + c->n_cigar*4 + c->l_qname + c->l_qseq + (c->l_qseq + 1)/2;+    for (i = 0; i < c->n_cigar; ++i) ed_swap_4p(&cigar[i]);+    while (s < data + l_data) {+        int size;+        s += 2; // skip key+        size = aux_type2size(*s); ++s; // skip type+        switch (size) {+        case 1: ++s; break;+        case 2: ed_swap_2p(s); s += 2; break;+        case 4: ed_swap_4p(s); s += 4; break;+        case 8: ed_swap_8p(s); s += 8; break;+        case 'Z':+        case 'H':+            while (*s) ++s;+            ++s;+            break;+        case 'B':+            size = aux_type2size(*s); ++s;+            if (is_host) memcpy(&n, s, 4), ed_swap_4p(s);+            else ed_swap_4p(s), memcpy(&n, s, 4);+            s += 4;+            switch (size) {+            case 1: s += n; break;+            case 2: for (i = 0; i < n; ++i, s += 2) ed_swap_2p(s); break;+            case 4: for (i = 0; i < n; ++i, s += 4) ed_swap_4p(s); break;+            case 8: for (i = 0; i < n; ++i, s += 8) ed_swap_8p(s); break;+            }+            break;+        }+    }+}++int bam_read1(BGZF *fp, bam1_t *b)+{+    bam1_core_t *c = &b->core;+    int32_t block_len, ret, i;+    uint32_t x[8];+    if ((ret = bgzf_read(fp, &block_len, 4)) != 4) {+        if (ret == 0) return -1; // normal end-of-file+        else return -2; // truncated+    }+    if (bgzf_read(fp, x, 32) != 32) return -3;+    if (fp->is_be) {+        ed_swap_4p(&block_len);+        for (i = 0; i < 8; ++i) ed_swap_4p(x + i);+    }+    c->tid = x[0]; c->pos = x[1];+    c->bin = x[2]>>16; c->qual = x[2]>>8&0xff; c->l_qname = x[2]&0xff;+    c->flag = x[3]>>16; c->n_cigar = x[3]&0xffff;+    c->l_qseq = x[4];+    c->mtid = x[5]; c->mpos = x[6]; c->isize = x[7];+    b->l_data = block_len - 32;+    if (b->l_data < 0 || c->l_qseq < 0 || c->l_qname < 1) return -4;+    if ((char *)bam_get_aux(b) - (char *)b->data > b->l_data)+        return -4;+    if (b->m_data < b->l_data) {+        b->m_data = b->l_data;+        kroundup32(b->m_data);+        b->data = (uint8_t*)realloc(b->data, b->m_data);+        if (!b->data)+            return -4;+    }+    if (bgzf_read(fp, b->data, b->l_data) != b->l_data) return -4;+    //b->l_aux = b->l_data - c->n_cigar * 4 - c->l_qname - c->l_qseq - (c->l_qseq+1)/2;+    if (fp->is_be) swap_data(c, b->l_data, b->data, 0);+    return 4 + block_len;+}++int bam_write1(BGZF *fp, const bam1_t *b)+{+    const bam1_core_t *c = &b->core;+    uint32_t x[8], block_len = b->l_data + 32, y;+    int i, ok;+    x[0] = c->tid;+    x[1] = c->pos;+    x[2] = (uint32_t)c->bin<<16 | c->qual<<8 | c->l_qname;+    x[3] = (uint32_t)c->flag<<16 | c->n_cigar;+    x[4] = c->l_qseq;+    x[5] = c->mtid;+    x[6] = c->mpos;+    x[7] = c->isize;+    ok = (bgzf_flush_try(fp, 4 + block_len) >= 0);+    if (fp->is_be) {+        for (i = 0; i < 8; ++i) ed_swap_4p(x + i);+        y = block_len;+        if (ok) ok = (bgzf_write(fp, ed_swap_4p(&y), 4) >= 0);+        swap_data(c, b->l_data, b->data, 1);+    } else {+        if (ok) ok = (bgzf_write(fp, &block_len, 4) >= 0);+    }+    if (ok) ok = (bgzf_write(fp, x, 32) >= 0);+    if (ok) ok = (bgzf_write(fp, b->data, b->l_data) >= 0);+    if (fp->is_be) swap_data(c, b->l_data, b->data, 0);+    return ok? 4 + block_len : -1;+}++/********************+ *** BAM indexing ***+ ********************/++static hts_idx_t *bam_index(BGZF *fp, int min_shift)+{+    int n_lvls, i, fmt, ret;+    bam1_t *b;+    hts_idx_t *idx;+    bam_hdr_t *h;+    h = bam_hdr_read(fp);+    if (h == NULL) return NULL;+    if (min_shift > 0) {+        int64_t max_len = 0, s;+        for (i = 0; i < h->n_targets; ++i)+            if (max_len < h->target_len[i]) max_len = h->target_len[i];+        max_len += 256;+        for (n_lvls = 0, s = 1<<min_shift; max_len > s; ++n_lvls, s <<= 3);+        fmt = HTS_FMT_CSI;+    } else min_shift = 14, n_lvls = 5, fmt = HTS_FMT_BAI;+    idx = hts_idx_init(h->n_targets, fmt, bgzf_tell(fp), min_shift, n_lvls);+    bam_hdr_destroy(h);+    b = bam_init1();+    while ((ret = bam_read1(fp, b)) >= 0) {+        ret = hts_idx_push(idx, b->core.tid, b->core.pos, bam_endpos(b), bgzf_tell(fp), !(b->core.flag&BAM_FUNMAP));+        if (ret < 0) goto err; // unsorted+    }+    if (ret < -1) goto err; // corrupted BAM file++    hts_idx_finish(idx, bgzf_tell(fp));+    bam_destroy1(b);+    return idx;++err:+    bam_destroy1(b);+    hts_idx_destroy(idx);+    return NULL;+}++int sam_index_build2(const char *fn, const char *fnidx, int min_shift)+{+    hts_idx_t *idx;+    htsFile *fp;+    int ret = 0;++    if ((fp = hts_open(fn, "r")) == 0) return -2;+    switch (fp->format.format) {+    case cram:+        ret = cram_index_build(fp->fp.cram, fn, fnidx);+        break;++    case bam:+        idx = bam_index(fp->fp.bgzf, min_shift);+        if (idx) {+            ret = hts_idx_save_as(idx, fn, fnidx, (min_shift > 0)? HTS_FMT_CSI : HTS_FMT_BAI);+            hts_idx_destroy(idx);+        }+        else ret = -1;+        break;++    default:+        ret = -3;+        break;+    }+    hts_close(fp);++    return ret;+}++int sam_index_build(const char *fn, int min_shift)+{+    return sam_index_build2(fn, NULL, min_shift);+}++// Provide bam_index_build() symbol for binary compability with earlier HTSlib+#undef bam_index_build+int bam_index_build(const char *fn, int min_shift)+{+    return sam_index_build2(fn, NULL, min_shift);+}++static int bam_readrec(BGZF *fp, void *ignored, void *bv, int *tid, int *beg, int *end)+{+    bam1_t *b = bv;+    int ret;+    if ((ret = bam_read1(fp, b)) >= 0) {+        *tid = b->core.tid;+        *beg = b->core.pos;+        *end = bam_endpos(b);+    }+    return ret;+}++// This is used only with read_rest=1 iterators, so need not set tid/beg/end.+static int cram_readrec(BGZF *ignored, void *fpv, void *bv, int *tid, int *beg, int *end)+{+    htsFile *fp = fpv;+    bam1_t *b = bv;+    return cram_get_bam_seq(fp->fp.cram, &b);+}++// This is used only with read_rest=1 iterators, so need not set tid/beg/end.+static int sam_bam_cram_readrec(BGZF *bgzfp, void *fpv, void *bv, int *tid, int *beg, int *end)+{+    htsFile *fp = fpv;+    bam1_t *b = bv;+    switch (fp->format.format) {+    case bam:   return bam_read1(bgzfp, b);+    case cram:  return cram_get_bam_seq(fp->fp.cram, &b);+    default:+        // TODO Need headers available to implement this for SAM files+        fprintf(stderr, "[sam_bam_cram_readrec] Not implemented for SAM files -- Exiting\n");+        abort();+    }+}++hts_idx_t *sam_index_load2(htsFile *fp, const char *fn, const char *fnidx)+{+    switch (fp->format.format) {+    case bam:+        return fnidx? hts_idx_load2(fn, fnidx) : hts_idx_load(fn, HTS_FMT_BAI);++    case cram: {+        if (cram_index_load(fp->fp.cram, fn, fnidx) < 0) return NULL;+        // Cons up a fake "index" just pointing at the associated cram_fd:+        hts_cram_idx_t *idx = malloc(sizeof (hts_cram_idx_t));+        if (idx == NULL) return NULL;+        idx->fmt = HTS_FMT_CRAI;+        idx->cram = fp->fp.cram;+        return (hts_idx_t *) idx;+        }++    default:+        return NULL; // TODO Would use tbx_index_load if it returned hts_idx_t+    }+}++hts_idx_t *sam_index_load(htsFile *fp, const char *fn)+{+    return sam_index_load2(fp, fn, NULL);+}++static hts_itr_t *cram_itr_query(const hts_idx_t *idx, int tid, int beg, int end, hts_readrec_func *readrec)+{+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;+    hts_itr_t *iter = (hts_itr_t *) calloc(1, sizeof(hts_itr_t));+    if (iter == NULL) return NULL;++    // Cons up a dummy iterator for which hts_itr_next() will simply invoke+    // the readrec function:+    iter->read_rest = 1;+    iter->off = NULL;+    iter->bins.a = NULL;+    iter->readrec = readrec;++    if (tid >= 0 || tid == HTS_IDX_NOCOOR) {+        cram_range r = { tid == HTS_IDX_NOCOOR ? -1 : tid, beg+1, end };+        int ret = cram_set_option(cidx->cram, CRAM_OPT_RANGE, &r);++        iter->curr_off = 0;+        // The following fields are not required by hts_itr_next(), but are+        // filled in in case user code wants to look at them.+        iter->tid = tid;+        iter->beg = beg;+        iter->end = end;++        switch (ret) {+        case 0:+            break;++        case -2:+            // No data vs this ref, so mark iterator as completed.+            // Same as HTS_IDX_NONE.+            iter->finished = 1;+            break;++        default:+            free(iter);+            return NULL;+        }+    }+    else switch (tid) {+    case HTS_IDX_REST:+        iter->curr_off = 0;+        break;+    case HTS_IDX_NONE:+        iter->curr_off = 0;+        iter->finished = 1;+        break;+    default:+        fprintf(stderr, "[cram_itr_query] tid=%d not implemented for CRAM files -- Exiting\n", tid);+        abort();+        break;+    }++    return iter;+}++hts_itr_t *sam_itr_queryi(const hts_idx_t *idx, int tid, int beg, int end)+{+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;+    if (idx == NULL)+        return hts_itr_query(NULL, tid, beg, end, sam_bam_cram_readrec);+    else if (cidx->fmt == HTS_FMT_CRAI)+        return cram_itr_query(idx, tid, beg, end, cram_readrec);+    else+        return hts_itr_query(idx, tid, beg, end, bam_readrec);+}++static int cram_name2id(void *fdv, const char *ref)+{+    cram_fd *fd = (cram_fd *) fdv;+    return sam_hdr_name2ref(fd->header, ref);+}++hts_itr_t *sam_itr_querys(const hts_idx_t *idx, bam_hdr_t *hdr, const char *region)+{+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;+    if (cidx->fmt == HTS_FMT_CRAI)+        return hts_itr_querys(idx, region, cram_name2id, cidx->cram, cram_itr_query, cram_readrec);+    else+        return hts_itr_querys(idx, region, (hts_name2id_f)(bam_name2id), hdr, hts_itr_query, bam_readrec);+}++/**********************+ *** SAM header I/O ***+ **********************/++#include "htslib/kseq.h"+#include "htslib/kstring.h"++bam_hdr_t *sam_hdr_parse(int l_text, const char *text)+{+    const char *q, *r, *p;+    khash_t(s2i) *d;+    d = kh_init(s2i);+    for (p = text; *p; ++p) {+        if (strncmp(p, "@SQ\t", 4) == 0) {+            char *sn = 0;+            int ln = -1;+            for (q = p + 4;; ++q) {+                if (strncmp(q, "SN:", 3) == 0) {+                    q += 3;+                    for (r = q; *r != '\t' && *r != '\n' && *r != '\0'; ++r);+                    sn = (char*)calloc(r - q + 1, 1);+                    strncpy(sn, q, r - q);+                    q = r;+                } else if (strncmp(q, "LN:", 3) == 0)+                    ln = strtol(q + 3, (char**)&q, 10);+                while (*q != '\t' && *q != '\n' && *q != '\0') ++q;+                if (*q == '\0' || *q == '\n') break;+            }+            p = q;+            if (sn && ln >= 0) {+                khint_t k;+                int absent;+                k = kh_put(s2i, d, sn, &absent);+                if (!absent) {+                    if (hts_verbose >= 2)+                        fprintf(stderr, "[W::%s] duplicated sequence '%s'\n", __func__, sn);+                    free(sn);+                } else kh_val(d, k) = (int64_t)(kh_size(d) - 1)<<32 | ln;+            }+        }+        while (*p != '\0' && *p != '\n') ++p;+    }+    return hdr_from_dict(d);+}++bam_hdr_t *sam_hdr_read(htsFile *fp)+{+    switch (fp->format.format) {+    case bam:+        return bam_hdr_read(fp->fp.bgzf);++    case cram:+        return cram_header_to_bam(fp->fp.cram->header);++    case sam: {+        kstring_t str;+        bam_hdr_t *h;+        int has_SQ = 0;+        str.l = str.m = 0; str.s = 0;+        while (hts_getline(fp, KS_SEP_LINE, &fp->line) >= 0) {+            if (fp->line.s[0] != '@') break;+            if (fp->line.l > 3 && strncmp(fp->line.s,"@SQ",3) == 0) has_SQ = 1;+            kputsn(fp->line.s, fp->line.l, &str);+            kputc('\n', &str);+        }+        if (! has_SQ && fp->fn_aux) {+            char line[2048];+            FILE *f = fopen(fp->fn_aux, "r");+            if (f == NULL) return NULL;+            while (fgets(line, sizeof line, f)) {+                const char *name = strtok(line, "\t");+                const char *length = strtok(NULL, "\t");+                ksprintf(&str, "@SQ\tSN:%s\tLN:%s\n", name, length);+            }+            fclose(f);+        }+        if (str.l == 0) kputsn("", 0, &str);+        h = sam_hdr_parse(str.l, str.s);+        h->l_text = str.l; h->text = str.s;+        return h;+        }++    default:+        abort();+    }+}++int sam_hdr_write(htsFile *fp, const bam_hdr_t *h)+{+    switch (fp->format.format) {+    case binary_format:+        fp->format.category = sequence_data;+        fp->format.format = bam;+        /* fall-through */+    case bam:+        if (bam_hdr_write(fp->fp.bgzf, h) < 0) return -1;+        break;++    case cram: {+        cram_fd *fd = fp->fp.cram;+        SAM_hdr *hdr = bam_header_to_cram((bam_hdr_t *)h);+        if (! hdr) return -1;+        if (cram_set_header(fd, hdr) < 0) return -1;+        if (fp->fn_aux)+            cram_load_reference(fd, fp->fn_aux);+        if (cram_write_SAM_hdr(fd, fd->header) < 0) return -1;+        }+        break;++    case text_format:+        fp->format.category = sequence_data;+        fp->format.format = sam;+        /* fall-through */+    case sam: {+        char *p;+        hputs(h->text, fp->fp.hfile);+        p = strstr(h->text, "@SQ\t"); // FIXME: we need a loop to make sure "@SQ\t" does not match something unwanted!!!+        if (p == 0) {+            int i;+            for (i = 0; i < h->n_targets; ++i) {+                fp->line.l = 0;+                kputsn("@SQ\tSN:", 7, &fp->line); kputs(h->target_name[i], &fp->line);+                kputsn("\tLN:", 4, &fp->line); kputw(h->target_len[i], &fp->line); kputc('\n', &fp->line);+                if ( hwrite(fp->fp.hfile, fp->line.s, fp->line.l) != fp->line.l ) return -1;+            }+        }+        if ( hflush(fp->fp.hfile) != 0 ) return -1;+        }+        break;++    default:+        abort();+    }+    return 0;+}++/**********************+ *** SAM record I/O ***+ **********************/++int sam_parse1(kstring_t *s, bam_hdr_t *h, bam1_t *b)+{+#define _read_token(_p) (_p); for (; *(_p) && *(_p) != '\t'; ++(_p)); if (*(_p) != '\t') goto err_ret; *(_p)++ = 0+#define _read_token_aux(_p) (_p); for (; *(_p) && *(_p) != '\t'; ++(_p)); *(_p)++ = 0 // this is different in that it does not test *(_p)=='\t'+#define _get_mem(type_t, _x, _s, _l) ks_resize((_s), (_s)->l + (_l)); *(_x) = (type_t*)((_s)->s + (_s)->l); (_s)->l += (_l)+#define _parse_err(cond, msg) do { if ((cond) && hts_verbose >= 1) { fprintf(stderr, "[E::%s] " msg "\n", __func__); goto err_ret; } } while (0)+#define _parse_warn(cond, msg) if ((cond) && hts_verbose >= 2) fprintf(stderr, "[W::%s] " msg "\n", __func__)++    uint8_t *t;+    char *p = s->s, *q;+    int i;+    kstring_t str;+    bam1_core_t *c = &b->core;++    str.l = b->l_data = 0;+    str.s = (char*)b->data; str.m = b->m_data;+    memset(c, 0, 32);+    if (h->cigar_tab == 0) {+        h->cigar_tab = (int8_t*) malloc(128);+        for (i = 0; i < 128; ++i)+            h->cigar_tab[i] = -1;+        for (i = 0; BAM_CIGAR_STR[i]; ++i)+            h->cigar_tab[(int)BAM_CIGAR_STR[i]] = i;+    }+    // qname+    q = _read_token(p);+    _parse_warn(p - q <= 1, "empty query name");+    _parse_err(p - q > 255, "query name too long");+    kputsn_(q, p - q, &str);+    c->l_qname = p - q;+    // flag+    c->flag = strtol(p, &p, 0);+    if (*p++ != '\t') goto err_ret; // malformated flag+    // chr+    q = _read_token(p);+    if (strcmp(q, "*")) {+        _parse_err(h->n_targets == 0, "missing SAM header");+        c->tid = bam_name2id(h, q);+        _parse_warn(c->tid < 0, "urecognized reference name; treated as unmapped");+    } else c->tid = -1;+    // pos+    c->pos = strtol(p, &p, 10) - 1;+    if (*p++ != '\t') goto err_ret;+    if (c->pos < 0 && c->tid >= 0) {+        _parse_warn(1, "mapped query cannot have zero coordinate; treated as unmapped");+        c->tid = -1;+    }+    if (c->tid < 0) c->flag |= BAM_FUNMAP;+    // mapq+    c->qual = strtol(p, &p, 10);+    if (*p++ != '\t') goto err_ret;+    // cigar+    if (*p != '*') {+        uint32_t *cigar;+        size_t n_cigar = 0;+        for (q = p; *p && *p != '\t'; ++p)+            if (!isdigit_c(*p)) ++n_cigar;+        if (*p++ != '\t') goto err_ret;+        _parse_err(n_cigar == 0, "no CIGAR operations");+        _parse_err(n_cigar >= 65536, "too many CIGAR operations");+        c->n_cigar = n_cigar;+        _get_mem(uint32_t, &cigar, &str, c->n_cigar * sizeof(uint32_t));+        for (i = 0; i < c->n_cigar; ++i, ++q) {+            int op;+            cigar[i] = strtol(q, &q, 10)<<BAM_CIGAR_SHIFT;+            op = (uint8_t)*q >= 128? -1 : h->cigar_tab[(int)*q];+            _parse_err(op < 0, "unrecognized CIGAR operator");+            cigar[i] |= op;+        }+        // can't use bam_endpos() directly as some fields not yet set up+        i = (!(c->flag&BAM_FUNMAP))? bam_cigar2rlen(c->n_cigar, cigar) : 1;+    } else {+        _parse_warn(!(c->flag&BAM_FUNMAP), "mapped query must have a CIGAR; treated as unmapped");+        c->flag |= BAM_FUNMAP;+        q = _read_token(p);+        i = 1;+    }+    c->bin = hts_reg2bin(c->pos, c->pos + i, 14, 5);+    // mate chr+    q = _read_token(p);+    if (strcmp(q, "=") == 0) {+        c->mtid = c->tid;+    } else if (strcmp(q, "*") == 0) {+        c->mtid = -1;+    } else {+        c->mtid = bam_name2id(h, q);+        _parse_warn(c->mtid < 0, "urecognized mate reference name; treated as unmapped");+    }+    // mpos+    c->mpos = strtol(p, &p, 10) - 1;+    if (*p++ != '\t') goto err_ret;+    if (c->mpos < 0 && c->mtid >= 0) {+        _parse_warn(1, "mapped mate cannot have zero coordinate; treated as unmapped");+        c->mtid = -1;+    }+    // tlen+    c->isize = strtol(p, &p, 10);+    if (*p++ != '\t') goto err_ret;+    // seq+    q = _read_token(p);+    if (strcmp(q, "*")) {+        c->l_qseq = p - q - 1;+        i = bam_cigar2qlen(c->n_cigar, (uint32_t*)(str.s + c->l_qname));+        _parse_err(c->n_cigar && i != c->l_qseq, "CIGAR and query sequence are of different length");+        i = (c->l_qseq + 1) >> 1;+        _get_mem(uint8_t, &t, &str, i);+        memset(t, 0, i);+        for (i = 0; i < c->l_qseq; ++i)+            t[i>>1] |= seq_nt16_table[(int)q[i]] << ((~i&1)<<2);+    } else c->l_qseq = 0;+    // qual+    q = _read_token_aux(p);+    _get_mem(uint8_t, &t, &str, c->l_qseq);+    if (strcmp(q, "*")) {+        _parse_err(p - q - 1 != c->l_qseq, "SEQ and QUAL are of different length");+        for (i = 0; i < c->l_qseq; ++i) t[i] = q[i] - 33;+    } else memset(t, 0xff, c->l_qseq);+    // aux+    // Note that (like the bam1_core_t fields) this aux data in b->data is+    // stored in host endianness; so there is no byte swapping needed here.+    while (p < s->s + s->l) {+        uint8_t type;+        q = _read_token_aux(p); // FIXME: can be accelerated for long 'B' arrays+        _parse_err(p - q - 1 < 6, "incomplete aux field");+        kputsn_(q, 2, &str);+        q += 3; type = *q++; ++q; // q points to value+        if (type == 'A' || type == 'a' || type == 'c' || type == 'C') {+            kputc_('A', &str);+            kputc_(*q, &str);+        } else if (type == 'i' || type == 'I') {+            if (*q == '-') {+                long x = strtol(q, &q, 10);+                if (x >= INT8_MIN) {+                    kputc_('c', &str); kputc_(x, &str);+                } else if (x >= INT16_MIN) {+                    int16_t y = x;+                    kputc_('s', &str); kputsn_((char*)&y, 2, &str);+                } else {+                    int32_t y = x;+                    kputc_('i', &str); kputsn_(&y, 4, &str);+                }+            } else {+                unsigned long x = strtoul(q, &q, 10);+                if (x <= UINT8_MAX) {+                    kputc_('C', &str); kputc_(x, &str);+                } else if (x <= UINT16_MAX) {+                    uint16_t y = x;+                    kputc_('S', &str); kputsn_(&y, 2, &str);+                } else {+                    uint32_t y = x;+                    kputc_('I', &str); kputsn_(&y, 4, &str);+                }+            }+        } else if (type == 'f') {+            float x;+            x = strtod(q, &q);+            kputc_('f', &str); kputsn_(&x, 4, &str);+        } else if (type == 'd') {+            double x;+            x = strtod(q, &q);+            kputc_('d', &str); kputsn_(&x, 8, &str);+        } else if (type == 'Z' || type == 'H') {+            kputc_(type, &str);kputsn_(q, p - q, &str); // note that this include the trailing NULL+        } else if (type == 'B') {+            int32_t n;+            char *r;+            _parse_err(p - q - 1 < 3, "incomplete B-typed aux field");+            type = *q++; // q points to the first ',' following the typing byte+            for (r = q, n = 0; *r; ++r)+                if (*r == ',') ++n;+            kputc_('B', &str); kputc_(type, &str); kputsn_(&n, 4, &str);+            // FIXME: to evaluate which is faster: a) aligned array and then memmove(); b) unaligned array; c) kputsn_()+            if (type == 'c')      while (q + 1 < p) { int8_t   x = strtol(q + 1, &q, 0); kputc_(x, &str); }+            else if (type == 'C') while (q + 1 < p) { uint8_t  x = strtoul(q + 1, &q, 0); kputc_(x, &str); }+            else if (type == 's') while (q + 1 < p) { int16_t  x = strtol(q + 1, &q, 0); kputsn_(&x, 2, &str); }+            else if (type == 'S') while (q + 1 < p) { uint16_t x = strtoul(q + 1, &q, 0); kputsn_(&x, 2, &str); }+            else if (type == 'i') while (q + 1 < p) { int32_t  x = strtol(q + 1, &q, 0); kputsn_(&x, 4, &str); }+            else if (type == 'I') while (q + 1 < p) { uint32_t x = strtoul(q + 1, &q, 0); kputsn_(&x, 4, &str); }+            else if (type == 'f') while (q + 1 < p) { float    x = strtod(q + 1, &q);    kputsn_(&x, 4, &str); }+            else _parse_err(1, "unrecognized type");+        } else _parse_err(1, "unrecognized type");+    }+    b->data = (uint8_t*)str.s; b->l_data = str.l; b->m_data = str.m;+    return 0;++#undef _parse_warn+#undef _parse_err+#undef _get_mem+#undef _read_token_aux+#undef _read_token+err_ret:+    b->data = (uint8_t*)str.s; b->l_data = str.l; b->m_data = str.m;+    return -2;+}++int sam_read1(htsFile *fp, bam_hdr_t *h, bam1_t *b)+{+    switch (fp->format.format) {+    case bam: {+        int r = bam_read1(fp->fp.bgzf, b);+        if (r >= 0) {+            if (b->core.tid  >= h->n_targets || b->core.tid  < -1 ||+                b->core.mtid >= h->n_targets || b->core.mtid < -1)+                return -3;+        }+        return r;+        }++    case cram: {+        int ret = cram_get_bam_seq(fp->fp.cram, &b);+        return ret >= 0+            ? ret+            : (cram_eof(fp->fp.cram) ? -1 : -2);+    }++    case sam: {+        int ret;+err_recover:+        if (fp->line.l == 0) {+            ret = hts_getline(fp, KS_SEP_LINE, &fp->line);+            if (ret < 0) return -1;+        }+        ret = sam_parse1(&fp->line, h, b);+        fp->line.l = 0;+        if (ret < 0) {+            if (hts_verbose >= 1)+                fprintf(stderr, "[W::%s] parse error at line %lld\n", __func__, (long long)fp->lineno);+            if (h->ignore_sam_err) goto err_recover;+        }+        return ret;+        }++    default:+        abort();+    }+}++int sam_format1(const bam_hdr_t *h, const bam1_t *b, kstring_t *str)+{+    int i;+    uint8_t *s;+    const bam1_core_t *c = &b->core;++    str->l = 0;+    kputsn(bam_get_qname(b), c->l_qname-1, str); kputc('\t', str); // query name+    kputw(c->flag, str); kputc('\t', str); // flag+    if (c->tid >= 0) { // chr+        kputs(h->target_name[c->tid] , str);+        kputc('\t', str);+    } else kputsn("*\t", 2, str);+    kputw(c->pos + 1, str); kputc('\t', str); // pos+    kputw(c->qual, str); kputc('\t', str); // qual+    if (c->n_cigar) { // cigar+        uint32_t *cigar = bam_get_cigar(b);+        for (i = 0; i < c->n_cigar; ++i) {+            kputw(bam_cigar_oplen(cigar[i]), str);+            kputc(bam_cigar_opchr(cigar[i]), str);+        }+    } else kputc('*', str);+    kputc('\t', str);+    if (c->mtid < 0) kputsn("*\t", 2, str); // mate chr+    else if (c->mtid == c->tid) kputsn("=\t", 2, str);+    else {+        kputs(h->target_name[c->mtid], str);+        kputc('\t', str);+    }+    kputw(c->mpos + 1, str); kputc('\t', str); // mate pos+    kputw(c->isize, str); kputc('\t', str); // template len+    if (c->l_qseq) { // seq and qual+        uint8_t *s = bam_get_seq(b);+        for (i = 0; i < c->l_qseq; ++i) kputc("=ACMGRSVTWYHKDBN"[bam_seqi(s, i)], str);+        kputc('\t', str);+        s = bam_get_qual(b);+        if (s[0] == 0xff) kputc('*', str);+        else for (i = 0; i < c->l_qseq; ++i) kputc(s[i] + 33, str);+    } else kputsn("*\t*", 3, str);++    // FIXME change "s+N <= b->data+b->l_data" to "b->data+b->l_data - s >= N"+    // (or equivalent) everywhere to avoid looking past the end of the array+    s = bam_get_aux(b); // aux+    while (s+4 <= b->data + b->l_data) {+        uint8_t type, key[2];+        key[0] = s[0]; key[1] = s[1];+        s += 2; type = *s++;+        kputc('\t', str); kputsn((char*)key, 2, str); kputc(':', str);+        if (type == 'A') {+            kputsn("A:", 2, str);+            kputc(*s, str);+            ++s;+        } else if (type == 'C') {+            kputsn("i:", 2, str);+            kputw(*s, str);+            ++s;+        } else if (type == 'c') {+            kputsn("i:", 2, str);+            kputw(*(int8_t*)s, str);+            ++s;+        } else if (type == 'S') {+            if (s+2 <= b->data + b->l_data) {+                kputsn("i:", 2, str);+                kputw(*(uint16_t*)s, str);+                s += 2;+            } else return -1;+        } else if (type == 's') {+            if (s+2 <= b->data + b->l_data) {+                kputsn("i:", 2, str);+                kputw(*(int16_t*)s, str);+                s += 2;+            } else return -1;+        } else if (type == 'I') {+            if (s+4 <= b->data + b->l_data) {+                kputsn("i:", 2, str);+                kputuw(*(uint32_t*)s, str);+                s += 4;+            } else return -1;+        } else if (type == 'i') {+            if (s+4 <= b->data + b->l_data) {+                kputsn("i:", 2, str);+                kputw(*(int32_t*)s, str);+                s += 4;+            } else return -1;+        } else if (type == 'f') {+            if (s+4 <= b->data + b->l_data) {+                ksprintf(str, "f:%g", *(float*)s);+                s += 4;+            } else return -1;++        } else if (type == 'd') {+            if (s+8 <= b->data + b->l_data) {+                ksprintf(str, "d:%g", *(double*)s);+                s += 8;+            } else return -1;+        } else if (type == 'Z' || type == 'H') {+            kputc(type, str); kputc(':', str);+            while (s < b->data + b->l_data && *s) kputc(*s++, str);+            if (s >= b->data + b->l_data)+                return -1;+            ++s;+        } else if (type == 'B') {+            uint8_t sub_type = *(s++);+            int sub_type_size = aux_type2size(sub_type);+            uint32_t n;+            if (sub_type_size == 0 || b->data + b->l_data - s < 4)+                return -1;+            memcpy(&n, s, 4);+            s += 4; // now points to the start of the array+            if ((b->data + b->l_data - s) / sub_type_size < n)+                return -1;+            kputsn("B:", 2, str); kputc(sub_type, str); // write the typing+            for (i = 0; i < n; ++i) { // FIXME: for better performance, put the loop after "if"+                kputc(',', str);+                if ('c' == sub_type)      { kputw(*(int8_t*)s, str); ++s; }+                else if ('C' == sub_type) { kputw(*(uint8_t*)s, str); ++s; }+                else if ('s' == sub_type) { kputw(*(int16_t*)s, str); s += 2; }+                else if ('S' == sub_type) { kputw(*(uint16_t*)s, str); s += 2; }+                else if ('i' == sub_type) { kputw(*(int32_t*)s, str); s += 4; }+                else if ('I' == sub_type) { kputuw(*(uint32_t*)s, str); s += 4; }+                else if ('f' == sub_type) { ksprintf(str, "%g", *(float*)s); s += 4; }+                else return -1;+            }+        }+    }+    return str->l;+}++int sam_write1(htsFile *fp, const bam_hdr_t *h, const bam1_t *b)+{+    switch (fp->format.format) {+    case binary_format:+        fp->format.category = sequence_data;+        fp->format.format = bam;+        /* fall-through */+    case bam:+        return bam_write1(fp->fp.bgzf, b);++    case cram:+        return cram_put_bam_seq(fp->fp.cram, (bam1_t *)b);++    case text_format:+        fp->format.category = sequence_data;+        fp->format.format = sam;+        /* fall-through */+    case sam:+        if (sam_format1(h, b, &fp->line) < 0) return -1;+        kputc('\n', &fp->line);+        if ( hwrite(fp->fp.hfile, fp->line.s, fp->line.l) != fp->line.l ) return -1;+        return fp->line.l;++    default:+        abort();+    }+}++/************************+ *** Auxiliary fields ***+ ************************/++void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data)+{+    int ori_len = b->l_data;+    b->l_data += 3 + len;+    if (b->m_data < b->l_data) {+        b->m_data = b->l_data;+        kroundup32(b->m_data);+        b->data = (uint8_t*)realloc(b->data, b->m_data);+    }+    b->data[ori_len] = tag[0]; b->data[ori_len + 1] = tag[1];+    b->data[ori_len + 2] = type;+    memcpy(b->data + ori_len + 3, data, len);+}++static inline uint8_t *skip_aux(uint8_t *s)+{+    int size = aux_type2size(*s); ++s; // skip type+    uint32_t n;+    switch (size) {+    case 'Z':+    case 'H':+        while (*s) ++s;+        return s + 1;+    case 'B':+        size = aux_type2size(*s); ++s;+        memcpy(&n, s, 4); s += 4;+        return s + size * n;+    case 0:+        abort();+        break;+    default:+        return s + size;+    }+}++uint8_t *bam_aux_get(const bam1_t *b, const char tag[2])+{+    uint8_t *s;+    int y = tag[0]<<8 | tag[1];+    s = bam_get_aux(b);+    while (s < b->data + b->l_data) {+        int x = (int)s[0]<<8 | s[1];+        s += 2;+        if (x == y) return s;+        s = skip_aux(s);+    }+    return 0;+}+// s MUST BE returned by bam_aux_get()+int bam_aux_del(bam1_t *b, uint8_t *s)+{+    uint8_t *p, *aux;+    int l_aux = bam_get_l_aux(b);+    aux = bam_get_aux(b);+    p = s - 2;+    s = skip_aux(s);+    memmove(p, s, l_aux - (s - aux));+    b->l_data -= s - p;+    return 0;+}++int32_t bam_aux2i(const uint8_t *s)+{+    int type;+    type = *s++;+    if (type == 'c') return (int32_t)*(int8_t*)s;+    else if (type == 'C') return (int32_t)*(uint8_t*)s;+    else if (type == 's') return (int32_t)*(int16_t*)s;+    else if (type == 'S') return (int32_t)*(uint16_t*)s;+    else if (type == 'i' || type == 'I') return *(int32_t*)s;+    else return 0;+}++double bam_aux2f(const uint8_t *s)+{+    int type;+    type = *s++;+    if (type == 'd') return *(double*)s;+    else if (type == 'f') return *(float*)s;+    else return 0.0;+}++char bam_aux2A(const uint8_t *s)+{+    int type;+    type = *s++;+    if (type == 'A') return *(char*)s;+    else return 0;+}++char *bam_aux2Z(const uint8_t *s)+{+    int type;+    type = *s++;+    if (type == 'Z' || type == 'H') return (char*)s;+    else return 0;+}++int sam_open_mode(char *mode, const char *fn, const char *format)+{+    // TODO Parse "bam5" etc for compression level+    if (format == NULL) {+        // Try to pick a format based on the filename extension+        const char *ext = fn? strrchr(fn, '.') : NULL;+        if (ext == NULL || strchr(ext, '/')) return -1;+        return sam_open_mode(mode, fn, ext+1);+    }+    else if (strcmp(format, "bam") == 0) strcpy(mode, "b");+    else if (strcmp(format, "cram") == 0) strcpy(mode, "c");+    else if (strcmp(format, "sam") == 0) strcpy(mode, "");+    else return -1;++    return 0;+}++// A version of sam_open_mode that can handle ,key=value options.+// The format string is allocated and returned, to be freed by the caller.+// Prefix should be "r" or "w",+char *sam_open_mode_opts(const char *fn,+                         const char *mode,+                         const char *format)+{+    char *mode_opts = malloc((format ? strlen(format) : 1) ++                             (mode   ? strlen(mode)   : 1) + 12);+    char *opts, *cp;+    int format_len;++    if (!mode_opts)+        return NULL;++    strcpy(mode_opts, mode ? mode : "r");+    cp = mode_opts + strlen(mode_opts);++    if (format == NULL) {+        // Try to pick a format based on the filename extension+        const char *ext = fn? strrchr(fn, '.') : NULL;+        if (ext == NULL || strchr(ext, '/')) {+            free(mode_opts);+            return NULL;+        }+        return sam_open_mode(cp, fn, ext+1)+            ? (free(mode_opts), NULL)+            : mode_opts;+    }++    if ((opts = strchr(format, ','))) {+        format_len = opts-format;+    } else {+        opts="";+        format_len = strlen(format);+    }++    if (strncmp(format, "bam", format_len) == 0) {+        *cp++ = 'b';+    } else if (strncmp(format, "cram", format_len) == 0) {+        *cp++ = 'c';+    } else if (strncmp(format, "cram2", format_len) == 0) {+        *cp++ = 'c';+        strcpy(cp, ",VERSION=2.1");+        cp += 12;+    } else if (strncmp(format, "cram3", format_len) == 0) {+        *cp++ = 'c';+        strcpy(cp, ",VERSION=3.0");+        cp += 12;+    } else if (strncmp(format, "sam", format_len) == 0) {+        ; // format mode=""+    } else {+        free(mode_opts);+        return NULL;+    }++    strcpy(cp, opts);++    return mode_opts;+}++#define STRNCMP(a,b,n) (strncasecmp((a),(b),(n)) || strlen(a)!=(n))+int bam_str2flag(const char *str)+{+    char *end, *beg = (char*) str;+    long int flag = strtol(str, &end, 0);+    if ( end!=str ) return flag;    // the conversion was successful+    flag = 0;+    while ( *str )+    {+        end = beg;+        while ( *end && *end!=',' ) end++;+        if ( !STRNCMP("PAIRED",beg,end-beg) ) flag |= BAM_FPAIRED;+        else if ( !STRNCMP("PROPER_PAIR",beg,end-beg) ) flag |= BAM_FPROPER_PAIR;+        else if ( !STRNCMP("UNMAP",beg,end-beg) ) flag |= BAM_FUNMAP;+        else if ( !STRNCMP("MUNMAP",beg,end-beg) ) flag |= BAM_FMUNMAP;+        else if ( !STRNCMP("REVERSE",beg,end-beg) ) flag |= BAM_FREVERSE;+        else if ( !STRNCMP("MREVERSE",beg,end-beg) ) flag |= BAM_FMREVERSE;+        else if ( !STRNCMP("READ1",beg,end-beg) ) flag |= BAM_FREAD1;+        else if ( !STRNCMP("READ2",beg,end-beg) ) flag |= BAM_FREAD2;+        else if ( !STRNCMP("SECONDARY",beg,end-beg) ) flag |= BAM_FSECONDARY;+        else if ( !STRNCMP("QCFAIL",beg,end-beg) ) flag |= BAM_FQCFAIL;+        else if ( !STRNCMP("DUP",beg,end-beg) ) flag |= BAM_FDUP;+        else if ( !STRNCMP("SUPPLEMENTARY",beg,end-beg) ) flag |= BAM_FSUPPLEMENTARY;+        else return -1;+        if ( !*end ) break;+        beg = end + 1;+    }+    return flag;+}++char *bam_flag2str(int flag)+{+    kstring_t str = {0,0,0};+    if ( flag&BAM_FPAIRED ) ksprintf(&str,"%s%s", str.l?",":"","PAIRED");+    if ( flag&BAM_FPROPER_PAIR ) ksprintf(&str,"%s%s", str.l?",":"","PROPER_PAIR");+    if ( flag&BAM_FUNMAP ) ksprintf(&str,"%s%s", str.l?",":"","UNMAP");+    if ( flag&BAM_FMUNMAP ) ksprintf(&str,"%s%s", str.l?",":"","MUNMAP");+    if ( flag&BAM_FREVERSE ) ksprintf(&str,"%s%s", str.l?",":"","REVERSE");+    if ( flag&BAM_FMREVERSE ) ksprintf(&str,"%s%s", str.l?",":"","MREVERSE");+    if ( flag&BAM_FREAD1 ) ksprintf(&str,"%s%s", str.l?",":"","READ1");+    if ( flag&BAM_FREAD2 ) ksprintf(&str,"%s%s", str.l?",":"","READ2");+    if ( flag&BAM_FSECONDARY ) ksprintf(&str,"%s%s", str.l?",":"","SECONDARY");+    if ( flag&BAM_FQCFAIL ) ksprintf(&str,"%s%s", str.l?",":"","QCFAIL");+    if ( flag&BAM_FDUP ) ksprintf(&str,"%s%s", str.l?",":"","DUP");+    if ( flag&BAM_FSUPPLEMENTARY ) ksprintf(&str,"%s%s", str.l?",":"","SUPPLEMENTARY");+    if ( str.l == 0 ) kputsn("", 0, &str);+    return str.s;+}+++/**************************+ *** Pileup and Mpileup ***+ **************************/++#if !defined(BAM_NO_PILEUP)++#include <assert.h>++/*******************+ *** Memory pool ***+ *******************/++typedef struct {+    int k, x, y, end;+} cstate_t;++static cstate_t g_cstate_null = { -1, 0, 0, 0 };++typedef struct __linkbuf_t {+    bam1_t b;+    int32_t beg, end;+    cstate_t s;+    struct __linkbuf_t *next;+} lbnode_t;++typedef struct {+    int cnt, n, max;+    lbnode_t **buf;+} mempool_t;++static mempool_t *mp_init(void)+{+    mempool_t *mp;+    mp = (mempool_t*)calloc(1, sizeof(mempool_t));+    return mp;+}+static void mp_destroy(mempool_t *mp)+{+    int k;+    for (k = 0; k < mp->n; ++k) {+        free(mp->buf[k]->b.data);+        free(mp->buf[k]);+    }+    free(mp->buf);+    free(mp);+}+static inline lbnode_t *mp_alloc(mempool_t *mp)+{+    ++mp->cnt;+    if (mp->n == 0) return (lbnode_t*)calloc(1, sizeof(lbnode_t));+    else return mp->buf[--mp->n];+}+static inline void mp_free(mempool_t *mp, lbnode_t *p)+{+    --mp->cnt; p->next = 0; // clear lbnode_t::next here+    if (mp->n == mp->max) {+        mp->max = mp->max? mp->max<<1 : 256;+        mp->buf = (lbnode_t**)realloc(mp->buf, sizeof(lbnode_t*) * mp->max);+    }+    mp->buf[mp->n++] = p;+}++/**********************+ *** CIGAR resolver ***+ **********************/++/* s->k: the index of the CIGAR operator that has just been processed.+   s->x: the reference coordinate of the start of s->k+   s->y: the query coordiante of the start of s->k+ */+static inline int resolve_cigar2(bam_pileup1_t *p, int32_t pos, cstate_t *s)+{+#define _cop(c) ((c)&BAM_CIGAR_MASK)+#define _cln(c) ((c)>>BAM_CIGAR_SHIFT)++    bam1_t *b = p->b;+    bam1_core_t *c = &b->core;+    uint32_t *cigar = bam_get_cigar(b);+    int k;+    // determine the current CIGAR operation+//  fprintf(stderr, "%s\tpos=%d\tend=%d\t(%d,%d,%d)\n", bam_get_qname(b), pos, s->end, s->k, s->x, s->y);+    if (s->k == -1) { // never processed+        if (c->n_cigar == 1) { // just one operation, save a loop+          if (_cop(cigar[0]) == BAM_CMATCH || _cop(cigar[0]) == BAM_CEQUAL || _cop(cigar[0]) == BAM_CDIFF) s->k = 0, s->x = c->pos, s->y = 0;+        } else { // find the first match or deletion+            for (k = 0, s->x = c->pos, s->y = 0; k < c->n_cigar; ++k) {+                int op = _cop(cigar[k]);+                int l = _cln(cigar[k]);+                if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CEQUAL || op == BAM_CDIFF) break;+                else if (op == BAM_CREF_SKIP) s->x += l;+                else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l;+            }+            assert(k < c->n_cigar);+            s->k = k;+        }+    } else { // the read has been processed before+        int op, l = _cln(cigar[s->k]);+        if (pos - s->x >= l) { // jump to the next operation+            assert(s->k < c->n_cigar); // otherwise a bug: this function should not be called in this case+            op = _cop(cigar[s->k+1]);+            if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) { // jump to the next without a loop+              if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l;+                s->x += l;+                ++s->k;+            } else { // find the next M/D/N/=/X+              if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l;+                s->x += l;+                for (k = s->k + 1; k < c->n_cigar; ++k) {+                    op = _cop(cigar[k]), l = _cln(cigar[k]);+                    if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) break;+                    else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l;+                }+                s->k = k;+            }+            assert(s->k < c->n_cigar); // otherwise a bug+        } // else, do nothing+    }+    { // collect pileup information+        int op, l;+        op = _cop(cigar[s->k]); l = _cln(cigar[s->k]);+        p->is_del = p->indel = p->is_refskip = 0;+        if (s->x + l - 1 == pos && s->k + 1 < c->n_cigar) { // peek the next operation+            int op2 = _cop(cigar[s->k+1]);+            int l2 = _cln(cigar[s->k+1]);+            if (op2 == BAM_CDEL) p->indel = -(int)l2;+            else if (op2 == BAM_CINS) p->indel = l2;+            else if (op2 == BAM_CPAD && s->k + 2 < c->n_cigar) { // no working for adjacent padding+                int l3 = 0;+                for (k = s->k + 2; k < c->n_cigar; ++k) {+                    op2 = _cop(cigar[k]); l2 = _cln(cigar[k]);+                    if (op2 == BAM_CINS) l3 += l2;+                    else if (op2 == BAM_CDEL || op2 == BAM_CMATCH || op2 == BAM_CREF_SKIP || op2 == BAM_CEQUAL || op2 == BAM_CDIFF) break;+                }+                if (l3 > 0) p->indel = l3;+            }+        }+        if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {+            p->qpos = s->y + (pos - s->x);+        } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) {+            p->is_del = 1; p->qpos = s->y; // FIXME: distinguish D and N!!!!!+            p->is_refskip = (op == BAM_CREF_SKIP);+        } // cannot be other operations; otherwise a bug+        p->is_head = (pos == c->pos); p->is_tail = (pos == s->end);+    }+    return 1;+}++/***********************+ *** Pileup iterator ***+ ***********************/++// Dictionary of overlapping reads+KHASH_MAP_INIT_STR(olap_hash, lbnode_t *)+typedef khash_t(olap_hash) olap_hash_t;++struct __bam_plp_t {+    mempool_t *mp;+    lbnode_t *head, *tail;+    int32_t tid, pos, max_tid, max_pos;+    int is_eof, max_plp, error, maxcnt;+    uint64_t id;+    bam_pileup1_t *plp;+    // for the "auto" interface only+    bam1_t *b;+    bam_plp_auto_f func;+    void *data;+    olap_hash_t *overlaps;+};++bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data)+{+    bam_plp_t iter;+    iter = (bam_plp_t)calloc(1, sizeof(struct __bam_plp_t));+    iter->mp = mp_init();+    iter->head = iter->tail = mp_alloc(iter->mp);+    iter->max_tid = iter->max_pos = -1;+    iter->maxcnt = 8000;+    if (func) {+        iter->func = func;+        iter->data = data;+        iter->b = bam_init1();+    }+    return iter;+}++void bam_plp_init_overlaps(bam_plp_t iter)+{+    iter->overlaps = kh_init(olap_hash);  // hash for tweaking quality of bases in overlapping reads+}++void bam_plp_destroy(bam_plp_t iter)+{+    lbnode_t *p, *pnext;+    if ( iter->overlaps ) kh_destroy(olap_hash, iter->overlaps);+    for (p = iter->head; p != NULL; p = pnext) {+        pnext = p->next;+        mp_free(iter->mp, p);+    }+    mp_destroy(iter->mp);+    if (iter->b) bam_destroy1(iter->b);+    free(iter->plp);+    free(iter);+}+++//---------------------------------+//---  Tweak overlapping reads+//---------------------------------++/**+ *  cigar_iref2iseq_set()  - find the first CMATCH setting the ref and the read index+ *  cigar_iref2iseq_next() - get the next CMATCH base+ *  @cigar:       pointer to current cigar block (rw)+ *  @cigar_max:   pointer just beyond the last cigar block+ *  @icig:        position within the current cigar block (rw)+ *  @iseq:        position in the sequence (rw)+ *  @iref:        position with respect to the beginning of the read (iref_pos - b->core.pos) (rw)+ *+ *  Returns BAM_CMATCH or -1 when there is no more cigar to process or the requested position is not covered.+ */+static inline int cigar_iref2iseq_set(uint32_t **cigar, uint32_t *cigar_max, int *icig, int *iseq, int *iref)+{+    int pos = *iref;+    if ( pos < 0 ) return -1;+    *icig = 0;+    *iseq = 0;+    *iref = 0;+    while ( *cigar<cigar_max )+    {+        int cig  = (**cigar) & BAM_CIGAR_MASK;+        int ncig = (**cigar) >> BAM_CIGAR_SHIFT;++        if ( cig==BAM_CSOFT_CLIP ) { (*cigar)++; *iseq += ncig; *icig = 0; continue; }+        if ( cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) { (*cigar)++; *icig = 0; continue; }+        if ( cig==BAM_CMATCH || cig==BAM_CEQUAL || cig==BAM_CDIFF )+        {+            pos -= ncig;+            if ( pos < 0 ) { *icig = ncig + pos; *iseq += *icig; *iref += *icig; return BAM_CMATCH; }+            (*cigar)++; *iseq += ncig; *icig = 0; *iref += ncig;+            continue;+        }+        if ( cig==BAM_CINS ) { (*cigar)++; *iseq += ncig; *icig = 0; continue; }+        if ( cig==BAM_CDEL || cig==BAM_CREF_SKIP )+        {+            pos -= ncig;+            if ( pos<0 ) pos = 0;+            (*cigar)++; *icig = 0; *iref += ncig;+            continue;+        }+        fprintf(stderr,"todo: cigar %d\n", cig);+        assert(0);+    }+    *iseq = -1;+    return -1;+}+static inline int cigar_iref2iseq_next(uint32_t **cigar, uint32_t *cigar_max, int *icig, int *iseq, int *iref)+{+    while ( *cigar < cigar_max )+    {+        int cig  = (**cigar) & BAM_CIGAR_MASK;+        int ncig = (**cigar) >> BAM_CIGAR_SHIFT;++        if ( cig==BAM_CMATCH || cig==BAM_CEQUAL || cig==BAM_CDIFF )+        {+            if ( *icig >= ncig - 1 ) { *icig = 0;  (*cigar)++; continue; }+            (*iseq)++; (*icig)++; (*iref)++;+            return BAM_CMATCH;+        }+        if ( cig==BAM_CDEL || cig==BAM_CREF_SKIP ) { (*cigar)++; (*iref) += ncig; *icig = 0; continue; }+        if ( cig==BAM_CINS ) { (*cigar)++; *iseq += ncig; *icig = 0; continue; }+        if ( cig==BAM_CSOFT_CLIP ) { (*cigar)++; *iseq += ncig; *icig = 0; continue; }+        if ( cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) { (*cigar)++; *icig = 0; continue; }+        fprintf(stderr,"todo: cigar %d\n", cig);+        assert(0);+    }+    *iseq = -1;+    *iref = -1;+    return -1;+}++static void tweak_overlap_quality(bam1_t *a, bam1_t *b)+{+    uint32_t *a_cigar = bam_get_cigar(a), *a_cigar_max = a_cigar + a->core.n_cigar;+    uint32_t *b_cigar = bam_get_cigar(b), *b_cigar_max = b_cigar + b->core.n_cigar;+    int a_icig = 0, a_iseq = 0;+    int b_icig = 0, b_iseq = 0;+    uint8_t *a_qual = bam_get_qual(a), *b_qual = bam_get_qual(b);+    uint8_t *a_seq  = bam_get_seq(a), *b_seq = bam_get_seq(b);++    int iref   = b->core.pos;+    int a_iref = iref - a->core.pos;+    int b_iref = iref - b->core.pos;+    int a_ret = cigar_iref2iseq_set(&a_cigar, a_cigar_max, &a_icig, &a_iseq, &a_iref);+    if ( a_ret<0 ) return;  // no overlap+    int b_ret = cigar_iref2iseq_set(&b_cigar, b_cigar_max, &b_icig, &b_iseq, &b_iref);+    if ( b_ret<0 ) return;  // no overlap++    #if DBG+        fprintf(stderr,"tweak %s  n_cigar=%d %d  .. %d-%d vs %d-%d\n", bam_get_qname(a), a->core.n_cigar, b->core.n_cigar,+            a->core.pos+1,a->core.pos+bam_cigar2rlen(a->core.n_cigar,bam_get_cigar(a)), b->core.pos+1, b->core.pos+bam_cigar2rlen(b->core.n_cigar,bam_get_cigar(b)));+    #endif++    while ( 1 )+    {+        // Increment reference position+        while ( a_iref>=0 && a_iref < iref - a->core.pos )+            a_ret = cigar_iref2iseq_next(&a_cigar, a_cigar_max, &a_icig, &a_iseq, &a_iref);+        if ( a_ret<0 ) break;   // done+        if ( iref < a_iref + a->core.pos ) iref = a_iref + a->core.pos;++        while ( b_iref>=0 && b_iref < iref - b->core.pos )+            b_ret = cigar_iref2iseq_next(&b_cigar, b_cigar_max, &b_icig, &b_iseq, &b_iref);+        if ( b_ret<0 ) break;   // done+        if ( iref < b_iref + b->core.pos ) iref = b_iref + b->core.pos;++        iref++;+        if ( a_iref+a->core.pos != b_iref+b->core.pos ) continue;   // only CMATCH positions, don't know what to do with indels++        if ( bam_seqi(a_seq,a_iseq) == bam_seqi(b_seq,b_iseq) )+        {+            #if DBG+                fprintf(stderr,"%c",seq_nt16_str[bam_seqi(a_seq,a_iseq)]);+            #endif+            // we are very confident about this base+            int qual = a_qual[a_iseq] + b_qual[b_iseq];+            a_qual[a_iseq] = qual>200 ? 200 : qual;+            b_qual[b_iseq] = 0;+        }+        else+        {+            if ( a_qual[a_iseq] >= b_qual[b_iseq] )+            {+                #if DBG+                    fprintf(stderr,"[%c/%c]",seq_nt16_str[bam_seqi(a_seq,a_iseq)],tolower(seq_nt16_str[bam_seqi(b_seq,b_iseq)]));+                #endif+                a_qual[a_iseq] = 0.8 * a_qual[a_iseq];  // not so confident about a_qual anymore given the mismatch+                b_qual[b_iseq] = 0;+            }+            else+            {+                #if DBG+                    fprintf(stderr,"[%c/%c]",tolower(seq_nt16_str[bam_seqi(a_seq,a_iseq)]),seq_nt16_str[bam_seqi(b_seq,b_iseq)]);+                #endif+                b_qual[b_iseq] = 0.8 * b_qual[b_iseq];+                a_qual[a_iseq] = 0;+            }+        }+    }+    #if DBG+        fprintf(stderr,"\n");+    #endif+}++// Fix overlapping reads. Simple soft-clipping did not give good results.+// Lowering qualities of unwanted bases is more selective and works better.+//+static void overlap_push(bam_plp_t iter, lbnode_t *node)+{+    if ( !iter->overlaps ) return;++    // mapped mates and paired reads only+    if ( node->b.core.flag&BAM_FMUNMAP || !(node->b.core.flag&BAM_FPROPER_PAIR) ) return;++    // no overlap possible, unless some wild cigar+    if ( abs(node->b.core.isize) >= 2*node->b.core.l_qseq ) return;++    khiter_t kitr = kh_get(olap_hash, iter->overlaps, bam_get_qname(&node->b));+    if ( kitr==kh_end(iter->overlaps) )+    {+        int ret;+        kitr = kh_put(olap_hash, iter->overlaps, bam_get_qname(&node->b), &ret);+        kh_value(iter->overlaps, kitr) = node;+    }+    else+    {+        lbnode_t *a = kh_value(iter->overlaps, kitr);+        tweak_overlap_quality(&a->b, &node->b);+        kh_del(olap_hash, iter->overlaps, kitr);+        assert(a->end-1 == a->s.end);+        a->end = bam_endpos(&a->b);+        a->s.end = a->end - 1;+    }+}++static void overlap_remove(bam_plp_t iter, const bam1_t *b)+{+    if ( !iter->overlaps ) return;++    khiter_t kitr;+    if ( b )+    {+        kitr = kh_get(olap_hash, iter->overlaps, bam_get_qname(b));+        if ( kitr!=kh_end(iter->overlaps) )+            kh_del(olap_hash, iter->overlaps, kitr);+    }+    else+    {+        // remove all+        for (kitr = kh_begin(iter->overlaps); kitr<kh_end(iter->overlaps); kitr++)+            if ( kh_exist(iter->overlaps, kitr) ) kh_del(olap_hash, iter->overlaps, kitr);+    }+}++++// Prepares next pileup position in bam records collected by bam_plp_auto -> user func -> bam_plp_push. Returns+// pointer to the piled records if next position is ready or NULL if there is not enough records in the+// buffer yet (the current position is still the maximum position across all buffered reads).+const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp)+{+    if (iter->error) { *_n_plp = -1; return 0; }+    *_n_plp = 0;+    if (iter->is_eof && iter->head == iter->tail) return 0;+    while (iter->is_eof || iter->max_tid > iter->tid || (iter->max_tid == iter->tid && iter->max_pos > iter->pos)) {+        int n_plp = 0;+        // write iter->plp at iter->pos+        lbnode_t **pptr = &iter->head;+        while (*pptr != iter->tail) {+            lbnode_t *p = *pptr;+            if (p->b.core.tid < iter->tid || (p->b.core.tid == iter->tid && p->end <= iter->pos)) { // then remove+                overlap_remove(iter, &p->b);+                *pptr = p->next; mp_free(iter->mp, p);+            }+            else {+                if (p->b.core.tid == iter->tid && p->beg <= iter->pos) { // here: p->end > pos; then add to pileup+                    if (n_plp == iter->max_plp) { // then double the capacity+                        iter->max_plp = iter->max_plp? iter->max_plp<<1 : 256;+                        iter->plp = (bam_pileup1_t*)realloc(iter->plp, sizeof(bam_pileup1_t) * iter->max_plp);+                    }+                    iter->plp[n_plp].b = &p->b;+                    if (resolve_cigar2(iter->plp + n_plp, iter->pos, &p->s)) ++n_plp; // actually always true...+                }+                pptr = &(*pptr)->next;+            }+        }+        *_n_plp = n_plp; *_tid = iter->tid; *_pos = iter->pos;+        // update iter->tid and iter->pos+        if (iter->head != iter->tail) {+            if (iter->tid > iter->head->b.core.tid) {+                fprintf(stderr, "[%s] unsorted input. Pileup aborts.\n", __func__);+                iter->error = 1;+                *_n_plp = -1;+                return 0;+            }+        }+        if (iter->tid < iter->head->b.core.tid) { // come to a new reference sequence+            iter->tid = iter->head->b.core.tid; iter->pos = iter->head->beg; // jump to the next reference+        } else if (iter->pos < iter->head->beg) { // here: tid == head->b.core.tid+            iter->pos = iter->head->beg; // jump to the next position+        } else ++iter->pos; // scan contiguously+        // return+        if (n_plp) return iter->plp;+        if (iter->is_eof && iter->head == iter->tail) break;+    }+    return 0;+}++int bam_plp_push(bam_plp_t iter, const bam1_t *b)+{+    if (iter->error) return -1;+    if (b) {+        if (b->core.tid < 0) { overlap_remove(iter, b); return 0; }+        // Skip only unmapped reads here, any additional filtering must be done in iter->func+        if (b->core.flag & BAM_FUNMAP) { overlap_remove(iter, b); return 0; }+        if (iter->tid == b->core.tid && iter->pos == b->core.pos && iter->mp->cnt > iter->maxcnt)+        {+            overlap_remove(iter, b);+            return 0;+        }+        bam_copy1(&iter->tail->b, b);+        overlap_push(iter, iter->tail);+#ifndef BAM_NO_ID+        iter->tail->b.id = iter->id++;+#endif+        iter->tail->beg = b->core.pos;+        iter->tail->end = bam_endpos(b);+        iter->tail->s = g_cstate_null; iter->tail->s.end = iter->tail->end - 1; // initialize cstate_t+        if (b->core.tid < iter->max_tid) {+            fprintf(stderr, "[bam_pileup_core] the input is not sorted (chromosomes out of order)\n");+            iter->error = 1;+            return -1;+        }+        if ((b->core.tid == iter->max_tid) && (iter->tail->beg < iter->max_pos)) {+            fprintf(stderr, "[bam_pileup_core] the input is not sorted (reads out of order)\n");+            iter->error = 1;+            return -1;+        }+        iter->max_tid = b->core.tid; iter->max_pos = iter->tail->beg;+        if (iter->tail->end > iter->pos || iter->tail->b.core.tid > iter->tid) {+            iter->tail->next = mp_alloc(iter->mp);+            iter->tail = iter->tail->next;+        }+    } else iter->is_eof = 1;+    return 0;+}++const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp)+{+    const bam_pileup1_t *plp;+    if (iter->func == 0 || iter->error) { *_n_plp = -1; return 0; }+    if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp;+    else { // no pileup line can be obtained; read alignments+        *_n_plp = 0;+        if (iter->is_eof) return 0;+        int ret;+        while ( (ret=iter->func(iter->data, iter->b)) >= 0) {+            if (bam_plp_push(iter, iter->b) < 0) {+                *_n_plp = -1;+                return 0;+            }+            if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp;+            // otherwise no pileup line can be returned; read the next alignment.+        }+        if ( ret < -1 ) { iter->error = ret; *_n_plp = -1; return 0; }+        bam_plp_push(iter, 0);+        if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp;+        return 0;+    }+}++void bam_plp_reset(bam_plp_t iter)+{+    overlap_remove(iter, NULL);+    iter->max_tid = iter->max_pos = -1;+    iter->tid = iter->pos = 0;+    iter->is_eof = 0;+    while (iter->head != iter->tail) {+        lbnode_t *p = iter->head;+        iter->head = p->next;+        mp_free(iter->mp, p);+    }+}++void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt)+{+    iter->maxcnt = maxcnt;+}++/************************+ *** Mpileup iterator ***+ ************************/++struct __bam_mplp_t {+    int n;+    uint64_t min, *pos;+    bam_plp_t *iter;+    int *n_plp;+    const bam_pileup1_t **plp;+};++bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data)+{+    int i;+    bam_mplp_t iter;+    iter = (bam_mplp_t)calloc(1, sizeof(struct __bam_mplp_t));+    iter->pos = (uint64_t*)calloc(n, sizeof(uint64_t));+    iter->n_plp = (int*)calloc(n, sizeof(int));+    iter->plp = (const bam_pileup1_t**)calloc(n, sizeof(bam_pileup1_t*));+    iter->iter = (bam_plp_t*)calloc(n, sizeof(bam_plp_t));+    iter->n = n;+    iter->min = (uint64_t)-1;+    for (i = 0; i < n; ++i) {+        iter->iter[i] = bam_plp_init(func, data[i]);+        iter->pos[i] = iter->min;+    }+    return iter;+}++void bam_mplp_init_overlaps(bam_mplp_t iter)+{+    int i;+    for (i = 0; i < iter->n; ++i)+        bam_plp_init_overlaps(iter->iter[i]);+}++void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt)+{+    int i;+    for (i = 0; i < iter->n; ++i)+        iter->iter[i]->maxcnt = maxcnt;+}++void bam_mplp_destroy(bam_mplp_t iter)+{+    int i;+    for (i = 0; i < iter->n; ++i) bam_plp_destroy(iter->iter[i]);+    free(iter->iter); free(iter->pos); free(iter->n_plp); free(iter->plp);+    free(iter);+}++int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp)+{+    int i, ret = 0;+    uint64_t new_min = (uint64_t)-1;+    for (i = 0; i < iter->n; ++i) {+        if (iter->pos[i] == iter->min) {+            int tid, pos;+            iter->plp[i] = bam_plp_auto(iter->iter[i], &tid, &pos, &iter->n_plp[i]);+            if ( iter->iter[i]->error ) return -1;+            iter->pos[i] = iter->plp[i] ? (uint64_t)tid<<32 | pos : 0;+        }+        if (iter->plp[i] && iter->pos[i] < new_min) new_min = iter->pos[i];+    }+    iter->min = new_min;+    if (new_min == (uint64_t)-1) return 0;+    *_tid = new_min>>32; *_pos = (uint32_t)new_min;+    for (i = 0; i < iter->n; ++i) {+        if (iter->pos[i] == iter->min) { // FIXME: valgrind reports "uninitialised value(s) at this line"+            n_plp[i] = iter->n_plp[i], plp[i] = iter->plp[i];+            ++ret;+        } else n_plp[i] = 0, plp[i] = 0;+    }+    return ret;+}++#endif // ~!defined(BAM_NO_PILEUP)
+ htslib-1.3.2/synced_bcf_reader.c view
@@ -0,0 +1,1284 @@+/*  synced_bcf_reader.c -- stream through multiple VCF files.++    Copyright (C) 2012-2014 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <stdio.h>+#include <unistd.h>+#include <string.h>+#include <limits.h>+#include <errno.h>+#include <ctype.h>+#include <sys/stat.h>+#include "htslib/synced_bcf_reader.h"+#include "htslib/kseq.h"+#include "htslib/khash_str2int.h"+#include "htslib/bgzf.h"++#define MAX_CSI_COOR 0x7fffffff     // maximum indexable coordinate of .csi++typedef struct+{+    uint32_t start, end;+}+region1_t;++typedef struct _region_t+{+    region1_t *regs;+    int nregs, mregs, creg;+}+region_t;++static void _regions_add(bcf_sr_regions_t *reg, const char *chr, int start, int end);+static bcf_sr_regions_t *_regions_init_string(const char *str);+static int _regions_match_alleles(bcf_sr_regions_t *reg, int als_idx, bcf1_t *rec);++char *bcf_sr_strerror(int errnum)+{+    switch (errnum)+    {+        case open_failed: +            return strerror(errno); break;+        case not_bgzf:+            return "not compressed with bgzip"; break;+        case idx_load_failed:+            return "could not load index"; break;+        case file_type_error:+            return "unknown file type"; break;+        case api_usage_error:+            return "API usage error"; break;+        case header_error:+            return "could not parse header"; break;+        case no_eof:+            return "no BGZF EOF marker; file may be truncated"; break;+        default: return ""; +    }+}++static int *init_filters(bcf_hdr_t *hdr, const char *filters, int *nfilters)+{+    kstring_t str = {0,0,0};+    const char *tmp = filters, *prev = filters;+    int nout = 0, *out = NULL;+    while ( 1 )+    {+        if ( *tmp==',' || !*tmp )+        {+            out = (int*) realloc(out, (nout+1)*sizeof(int));+            if ( tmp-prev==1 && *prev=='.' )+            {+                out[nout] = -1;+                nout++;+            }+            else+            {+                str.l = 0;+                kputsn(prev, tmp-prev, &str);+                out[nout] = bcf_hdr_id2int(hdr, BCF_DT_ID, str.s);+                if ( out[nout]>=0 ) nout++;+            }+            if ( !*tmp ) break;+            prev = tmp+1;+        }+        tmp++;+    }+    if ( str.m ) free(str.s);+    *nfilters = nout;+    return out;+}++int bcf_sr_set_regions(bcf_srs_t *readers, const char *regions, int is_file)+{+    assert( !readers->regions );+    if ( readers->nreaders )+    {+        fprintf(stderr,"[%s:%d %s] Error: bcf_sr_set_regions() must be called before bcf_sr_add_reader()\n", __FILE__,__LINE__,__FUNCTION__);+        return -1;+    }+    readers->regions = bcf_sr_regions_init(regions,is_file,0,1,-2);+    if ( !readers->regions ) return -1;+    readers->explicit_regs = 1;+    readers->require_index = 1;+    return 0;+}+int bcf_sr_set_targets(bcf_srs_t *readers, const char *targets, int is_file, int alleles)+{+    assert( !readers->targets );+    if ( targets[0]=='^' )+    {+        readers->targets_exclude = 1;+        targets++;+    }+    readers->targets = bcf_sr_regions_init(targets,is_file,0,1,-2);+    if ( !readers->targets ) return -1;+    readers->targets_als = alleles;+    return 0;+}++int bcf_sr_add_reader(bcf_srs_t *files, const char *fname)+{+    htsFile* file_ptr = hts_open(fname, "r");+    if ( ! file_ptr ) {+        files->errnum = open_failed;+        return 0;+    }++    files->has_line = (int*) realloc(files->has_line, sizeof(int)*(files->nreaders+1));+    files->has_line[files->nreaders] = 0;+    files->readers  = (bcf_sr_t*) realloc(files->readers, sizeof(bcf_sr_t)*(files->nreaders+1));+    bcf_sr_t *reader = &files->readers[files->nreaders++];+    memset(reader,0,sizeof(bcf_sr_t));++    reader->file = file_ptr;++    files->errnum = 0;++    if ( reader->file->format.compression==bgzf )+    {+        BGZF *bgzf = hts_get_bgzfp(reader->file);+        if ( bgzf && bgzf_check_EOF(bgzf) == 0 ) {+            files->errnum = no_eof;+            fprintf(stderr,"[%s] Warning: no BGZF EOF marker; file may be truncated.\n", fname);+        }+    }++    if ( files->require_index )+    {+        if ( reader->file->format.format==vcf )+        {+            if ( reader->file->format.compression!=bgzf )+            {+                files->errnum = not_bgzf;+                return 0;+            }++            reader->tbx_idx = tbx_index_load(fname);+            if ( !reader->tbx_idx )+            {+                files->errnum = idx_load_failed;+                return 0;+            }++            reader->header = bcf_hdr_read(reader->file);+        }+        else if ( reader->file->format.format==bcf )+        {+            if ( reader->file->format.compression!=bgzf )+            {+                files->errnum = not_bgzf;+                return 0;+            }++            reader->header = bcf_hdr_read(reader->file);++            reader->bcf_idx = bcf_index_load(fname);+            if ( !reader->bcf_idx )+            {+                files->errnum = idx_load_failed;+                return 0;+            }+        }+        else+        {+            files->errnum = file_type_error;+            return 0;+        }+    }+    else+    {+        if ( reader->file->format.format==bcf || reader->file->format.format==vcf )+        {+            reader->header = bcf_hdr_read(reader->file);+        }+        else+        {+            files->errnum = file_type_error;+            return 0;+        }+        files->streaming = 1;+    }+    if ( files->streaming && files->nreaders>1 )+    {+        files->errnum = api_usage_error;+        fprintf(stderr,"[%s:%d %s] Error: %d readers, yet require_index not set\n", __FILE__,__LINE__,__FUNCTION__,files->nreaders);+        return 0;+    }+    if ( files->streaming && files->regions )+    {+        files->errnum = api_usage_error;+        fprintf(stderr,"[%s:%d %s] Error: cannot tabix-jump in streaming mode\n", __FILE__,__LINE__,__FUNCTION__);+        return 0;+    }+    if ( !reader->header )+    {+        files->errnum = header_error;+        return 0;+    }++    reader->fname = strdup(fname);+    if ( files->apply_filters )+        reader->filter_ids = init_filters(reader->header, files->apply_filters, &reader->nfilter_ids);++    // Update list of chromosomes+    if ( !files->explicit_regs && !files->streaming )+    {+        int n,i;+        const char **names = reader->tbx_idx ? tbx_seqnames(reader->tbx_idx, &n) : bcf_hdr_seqnames(reader->header, &n);+        for (i=0; i<n; i++)+        {+            if ( !files->regions )+                files->regions = _regions_init_string(names[i]);+            else+                _regions_add(files->regions, names[i], -1, -1);+        }+        free(names);+    }++    return 1;+}++bcf_srs_t *bcf_sr_init(void)+{+    bcf_srs_t *files = (bcf_srs_t*) calloc(1,sizeof(bcf_srs_t));+    return files;+}++static void bcf_sr_destroy1(bcf_sr_t *reader)+{+    free(reader->fname);+    if ( reader->tbx_idx ) tbx_destroy(reader->tbx_idx);+    if ( reader->bcf_idx ) hts_idx_destroy(reader->bcf_idx);+    bcf_hdr_destroy(reader->header);+    hts_close(reader->file);+    if ( reader->itr ) tbx_itr_destroy(reader->itr);+    int j;+    for (j=0; j<reader->mbuffer; j++)+        bcf_destroy1(reader->buffer[j]);+    free(reader->buffer);+    free(reader->samples);+    free(reader->filter_ids);+}+void bcf_sr_destroy(bcf_srs_t *files)+{+    int i;+    for (i=0; i<files->nreaders; i++)+        bcf_sr_destroy1(&files->readers[i]);+    free(files->has_line);+    free(files->readers);+    for (i=0; i<files->n_smpl; i++) free(files->samples[i]);+    free(files->samples);+    if (files->targets) bcf_sr_regions_destroy(files->targets);+    if (files->regions) bcf_sr_regions_destroy(files->regions);+    if ( files->tmps.m ) free(files->tmps.s);+    free(files);+}++void bcf_sr_remove_reader(bcf_srs_t *files, int i)+{+    assert( !files->samples );  // not ready for this yet+    bcf_sr_destroy1(&files->readers[i]);+    if ( i+1 < files->nreaders )+    {+        memmove(&files->readers[i], &files->readers[i+1], (files->nreaders-i-1)*sizeof(bcf_sr_t));+        memmove(&files->has_line[i], &files->has_line[i+1], (files->nreaders-i-1)*sizeof(int));+    }+    files->nreaders--;+}+++/*+   Removes duplicate records from the buffer. The meaning of "duplicate" is+   controlled by the $collapse variable, which can cause that from multiple+   <indel|snp|any> lines only the first is considered and the rest is ignored.+   The removal is done by setting the redundant lines' positions to -1 and+   moving these lines at the end of the buffer.+ */+static void collapse_buffer(bcf_srs_t *files, bcf_sr_t *reader)+{+    int irec,jrec, has_snp=0, has_indel=0, has_any=0;+    for (irec=1; irec<=reader->nbuffer; irec++)+    {+        bcf1_t *line = reader->buffer[irec];+        if ( line->pos != reader->buffer[1]->pos ) break;+        if ( files->collapse&COLLAPSE_ANY )+        {+            if ( !has_any ) has_any = 1;+            else line->pos = -1;+        }+        int line_type = bcf_get_variant_types(line);+        if ( files->collapse&COLLAPSE_SNPS && line_type&(VCF_SNP|VCF_MNP) )+        {+            if ( !has_snp ) has_snp = 1;+            else line->pos = -1;+        }+        if ( files->collapse&COLLAPSE_INDELS && line_type&VCF_INDEL )+        {+            if ( !has_indel ) has_indel = 1;+            else line->pos = -1;+        }+    }+    bcf1_t *tmp;+    irec = jrec = 1;+    while ( irec<=reader->nbuffer && jrec<=reader->nbuffer )+    {+        if ( reader->buffer[irec]->pos != -1 ) { irec++; continue; }+        if ( jrec<=irec ) jrec = irec+1;+        while ( jrec<=reader->nbuffer && reader->buffer[jrec]->pos==-1 ) jrec++;+        if ( jrec<=reader->nbuffer )+        {+            tmp = reader->buffer[irec]; reader->buffer[irec] = reader->buffer[jrec]; reader->buffer[jrec] = tmp;+        }+    }+    reader->nbuffer = irec - 1;+}++void debug_buffer(FILE *fp, bcf_sr_t *reader)+{+    int j;+    for (j=0; j<=reader->nbuffer; j++)+    {+        bcf1_t *line = reader->buffer[j];+        fprintf(fp,"\t%p\t%s%s\t%s:%d\t%s ", line,reader->fname,j==0?"*":" ",reader->header->id[BCF_DT_CTG][line->rid].key,line->pos+1,line->n_allele?line->d.allele[0]:"");+        int k;+        for (k=1; k<line->n_allele; k++) fprintf(fp," %s", line->d.allele[k]);+        fprintf(fp,"\n");+    }+}++void debug_buffers(FILE *fp, bcf_srs_t *files)+{+    int i;+    for (i=0; i<files->nreaders; i++)+    {+        fprintf(fp, "has_line: %d\t%s\n", bcf_sr_has_line(files,i),files->readers[i].fname);+        debug_buffer(fp, &files->readers[i]);+    }+    fprintf(fp,"\n");+}++static inline int has_filter(bcf_sr_t *reader, bcf1_t *line)+{+    int i, j;+    if ( !line->d.n_flt )+    {+        for (j=0; j<reader->nfilter_ids; j++)+            if ( reader->filter_ids[j]<0 ) return 1;+        return 0;+    }+    for (i=0; i<line->d.n_flt; i++)+    {+        for (j=0; j<reader->nfilter_ids; j++)+            if ( line->d.flt[i]==reader->filter_ids[j] ) return 1;+    }+    return 0;+}++static int _reader_seek(bcf_sr_t *reader, const char *seq, int start, int end)+{+    if ( end>=MAX_CSI_COOR )+    {+        fprintf(stderr,"The coordinate is out of csi index limit: %d\n", end+1);+        exit(1);+    }+    if ( reader->itr )+    {+        hts_itr_destroy(reader->itr);+        reader->itr = NULL;+    }+    reader->nbuffer = 0;+    if ( reader->tbx_idx )+    {+        int tid = tbx_name2id(reader->tbx_idx, seq);+        if ( tid==-1 ) return -1;    // the sequence not present in this file+        reader->itr = tbx_itr_queryi(reader->tbx_idx,tid,start,end+1);+    }+    else+    {+        int tid = bcf_hdr_name2id(reader->header, seq);+        if ( tid==-1 ) return -1;    // the sequence not present in this file+        reader->itr = bcf_itr_queryi(reader->bcf_idx,tid,start,end+1);+    }+    if ( !reader->itr ) fprintf(stderr,"Could not seek: %s:%d-%d\n",seq,start+1,end+1);+    assert(reader->itr);+    return 0;+}++/*+ *  _readers_next_region() - jumps to next region if necessary+ *  Returns 0 on success or -1 when there are no more regions left+ */+static int _readers_next_region(bcf_srs_t *files)+{+    // Need to open new chromosome? Check number of lines in all readers' buffers+    int i, eos = 0;+    for (i=0; i<files->nreaders; i++)+        if ( !files->readers[i].itr && !files->readers[i].nbuffer ) eos++;++    if ( eos!=files->nreaders )+    {+        // Some of the readers still has buffered lines+        return 0;+    }++    // No lines in the buffer, need to open new region or quit+    if ( bcf_sr_regions_next(files->regions)<0 ) return -1;++    for (i=0; i<files->nreaders; i++)+        _reader_seek(&files->readers[i],files->regions->seq_names[files->regions->iseq],files->regions->start,files->regions->end);++    return 0;+}++/*+ *  _reader_fill_buffer() - buffers all records with the same coordinate+ */+static void _reader_fill_buffer(bcf_srs_t *files, bcf_sr_t *reader)+{+    // Return if the buffer is full: the coordinate of the last buffered record differs+    if ( reader->nbuffer && reader->buffer[reader->nbuffer]->pos != reader->buffer[1]->pos ) return;++    // No iterator (sequence not present in this file) and not streaming+    if ( !reader->itr && !files->streaming ) return;++    // Fill the buffer with records starting at the same position+    int i, ret = 0;+    while (1)+    {+        if ( reader->nbuffer+1 >= reader->mbuffer )+        {+            // Increase buffer size+            reader->mbuffer += 8;+            reader->buffer = (bcf1_t**) realloc(reader->buffer, sizeof(bcf1_t*)*reader->mbuffer);+            for (i=8; i>0; i--)     // initialize+            {+                reader->buffer[reader->mbuffer-i] = bcf_init1();+                reader->buffer[reader->mbuffer-i]->max_unpack = files->max_unpack;+                reader->buffer[reader->mbuffer-i]->pos = -1;    // for rare cases when VCF starts from 1+            }+        }+        if ( files->streaming )+        {+            if ( reader->file->format.format==vcf )+            {+                if ( (ret=hts_getline(reader->file, KS_SEP_LINE, &files->tmps)) < 0 ) break;   // no more lines+                int ret = vcf_parse1(&files->tmps, reader->header, reader->buffer[reader->nbuffer+1]);+                if ( ret<0 ) break;+            }+            else if ( reader->file->format.format==bcf )+            {+                if ( (ret=bcf_read1(reader->file, reader->header, reader->buffer[reader->nbuffer+1])) < 0 ) break; // no more lines+            }+            else+            {+                fprintf(stderr,"[%s:%d %s] fixme: not ready for this\n", __FILE__,__LINE__,__FUNCTION__);+                exit(1);+            }+        }+        else if ( reader->tbx_idx )+        {+            if ( (ret=tbx_itr_next(reader->file, reader->tbx_idx, reader->itr, &files->tmps)) < 0 ) break;  // no more lines+            vcf_parse1(&files->tmps, reader->header, reader->buffer[reader->nbuffer+1]);+        }+        else+        {+            if ( (ret=bcf_itr_next(reader->file, reader->itr, reader->buffer[reader->nbuffer+1])) < 0 ) break; // no more lines+            bcf_subset_format(reader->header,reader->buffer[reader->nbuffer+1]);+        }++        // apply filter+        if ( !reader->nfilter_ids )+            bcf_unpack(reader->buffer[reader->nbuffer+1], BCF_UN_STR);+        else+        {+            bcf_unpack(reader->buffer[reader->nbuffer+1], BCF_UN_STR|BCF_UN_FLT);+            if ( !has_filter(reader, reader->buffer[reader->nbuffer+1]) ) continue;+        }+        reader->nbuffer++;++        if ( reader->buffer[reader->nbuffer]->pos != reader->buffer[1]->pos ) break;    // the buffer is full+    }+    if ( ret<0 )+    {+        // done for this region+        tbx_itr_destroy(reader->itr);+        reader->itr = NULL;+    }+    if ( files->collapse && reader->nbuffer>=2 && reader->buffer[1]->pos==reader->buffer[2]->pos )+        collapse_buffer(files, reader);+}++/*+ *  _readers_shift_buffer() - removes the first line and all subsequent lines with the same position+ */+static void _reader_shift_buffer(bcf_sr_t *reader)+{+    int i;+    for (i=2; i<=reader->nbuffer; i++)+        if ( reader->buffer[i]->pos!=reader->buffer[1]->pos ) break;+    if ( i<=reader->nbuffer )+    {+        // A record with a different position follows, swap it. Because of the reader's logic,+        // only one such line can be present.+        bcf1_t *tmp = reader->buffer[1]; reader->buffer[1] = reader->buffer[i]; reader->buffer[i] = tmp;+        reader->nbuffer = 1;+    }+    else+        reader->nbuffer = 0;    // no other line+}++/*+ *  _reader_match_alleles() - from multiple buffered lines selects the one which+ *  corresponds best to the template line. The logic is controlled by COLLAPSE_*+ *  Returns 0 on success or -1 when no good matching line is found.+ */+static int _reader_match_alleles(bcf_srs_t *files, bcf_sr_t *reader, bcf1_t *tmpl)+{+    int i, irec = -1;++    // if no template given, use the first available record+    if ( !tmpl )+        irec = 1;+    else+    {+        int tmpl_type = bcf_get_variant_types(tmpl);+        for (i=1; i<=reader->nbuffer; i++)+        {+            bcf1_t *line = reader->buffer[i];+            if ( line->pos != reader->buffer[1]->pos ) break;  // done with this reader++            // Easiest case: matching by position only+            if ( files->collapse&COLLAPSE_ANY ) { irec=i; break; }++            int line_type = bcf_get_variant_types(line);++            // No matter what the alleles are, as long as they are both SNPs+            if ( files->collapse&COLLAPSE_SNPS && tmpl_type&VCF_SNP && line_type&VCF_SNP ) { irec=i; break; }+            // ... or indels+            if ( files->collapse&COLLAPSE_INDELS && tmpl_type&VCF_INDEL && line_type&VCF_INDEL ) { irec=i; break; }++            // More thorough checking: REFs must match+            if ( tmpl->rlen != line->rlen ) continue;  // different length+            if ( !tmpl->d.allele || !line->d.allele ) continue;   // one of the lines is empty, someone is swapped buffered lines?!+            if ( strcmp(tmpl->d.allele[0], line->d.allele[0]) ) continue; // the strings do not match++            int ial,jal;+            if ( files->collapse==COLLAPSE_NONE )+            {+                // Exact match, all alleles must be identical+                if ( tmpl->n_allele!=line->n_allele ) continue;   // different number of alleles, skip++                int nmatch = 1; // REF has been already checked+                for (ial=1; ial<tmpl->n_allele; ial++)+                {+                    for (jal=1; jal<line->n_allele; jal++)+                        if ( !strcmp(tmpl->d.allele[ial], line->d.allele[jal]) ) { nmatch++; break; }+                }+                if ( nmatch==tmpl->n_allele ) { irec=i; break; }    // found: exact match+                continue;+            }++            if ( line->n_allele==1 && tmpl->n_allele==1 ) { irec=i; break; }    // both sites are non-variant++            // COLLAPSE_SOME: at least some ALTs must match+            for (ial=1; ial<tmpl->n_allele; ial++)+            {+                for (jal=1; jal<line->n_allele; jal++)+                    if ( !strcmp(tmpl->d.allele[ial], line->d.allele[jal]) ) { irec=i; break; }+                if ( irec>=1 ) break;+            }+            if ( irec>=1 ) break;+        }+        if ( irec==-1 ) return -1;  // no matching line was found+    }++    // Set the selected line (irec) as active: set it to buffer[0], move the remaining lines forward+    // and put the old bcf1_t record at the end.+    bcf1_t *tmp = reader->buffer[0];+    reader->buffer[0] = reader->buffer[irec];+    for (i=irec+1; i<=reader->nbuffer; i++) reader->buffer[i-1] = reader->buffer[i];+    reader->buffer[ reader->nbuffer ] = tmp;+    reader->nbuffer--;++    return 0;+}++int _reader_next_line(bcf_srs_t *files)+{+    int i, min_pos = INT_MAX;++    // Loop until next suitable line is found or all readers have finished+    while ( 1 )+    {+        // Get all readers ready for the next region.+        if ( files->regions && _readers_next_region(files)<0 ) break;++        // Fill buffers+        const char *chr = NULL;+        for (i=0; i<files->nreaders; i++)+        {+            _reader_fill_buffer(files, &files->readers[i]);++            // Update the minimum coordinate+            if ( !files->readers[i].nbuffer ) continue;+            if ( min_pos > files->readers[i].buffer[1]->pos )+            {+                min_pos = files->readers[i].buffer[1]->pos;+                chr = bcf_seqname(files->readers[i].header, files->readers[i].buffer[1]);+            }+        }+        if ( min_pos==INT_MAX )+        {+            if ( !files->regions ) break;+            continue;+        }++        // Skip this position if not present in targets+        if ( files->targets )+        {+            int ret = bcf_sr_regions_overlap(files->targets, chr, min_pos, min_pos);+            if ( (!files->targets_exclude && ret<0) || (files->targets_exclude && !ret) )+            {+                // Remove all lines with this position from the buffer+                for (i=0; i<files->nreaders; i++)+                    if ( files->readers[i].nbuffer && files->readers[i].buffer[1]->pos==min_pos )+                        _reader_shift_buffer(&files->readers[i]);+                min_pos = INT_MAX;+                continue;+            }+        }++        break;  // done: min_pos is set+    }++    // There can be records with duplicate positions. Set the active line intelligently so that+    // the alleles match.+    int nret = 0;   // number of readers sharing the position+    bcf1_t *first = NULL;   // record which will be used for allele matching+    for (i=0; i<files->nreaders; i++)+    {+        files->has_line[i] = 0;++        // Skip readers with no records at this position+        if ( !files->readers[i].nbuffer || files->readers[i].buffer[1]->pos!=min_pos ) continue;++        // Until now buffer[0] of all reader was empty and the lines started at buffer[1].+        // Now lines which are ready to be output will be moved to buffer[0].+        if ( _reader_match_alleles(files, &files->readers[i], first) < 0 ) continue;+        if ( !first ) first = files->readers[i].buffer[0];++        nret++;+        files->has_line[i] = 1;+    }+    return nret;+}++int bcf_sr_next_line(bcf_srs_t *files)+{+    if ( !files->targets_als )+        return _reader_next_line(files);++    while (1)+    {+        int i, ret = _reader_next_line(files);+        if ( !ret ) return ret;++        for (i=0; i<files->nreaders; i++)+            if ( files->has_line[i] ) break;++        if ( _regions_match_alleles(files->targets, files->targets_als-1, files->readers[i].buffer[0]) ) return ret;++        // Check if there are more duplicate lines in the buffers. If not, return this line as if it+        // matched the targets, even if there is a type mismatch+        for (i=0; i<files->nreaders; i++)+        {+            if ( !files->has_line[i] ) continue;+            if ( files->readers[i].nbuffer==0 || files->readers[i].buffer[1]->pos!=files->readers[i].buffer[0]->pos ) continue;+            break;+        }+        if ( i==files->nreaders ) return ret;   // no more lines left, output even if target alleles are not of the same type+    }+}++static void bcf_sr_seek_start(bcf_srs_t *readers)+{+    bcf_sr_regions_t *reg = readers->regions;+    int i;+    for (i=0; i<reg->nseqs; i++)+        reg->regs[i].creg = -1;+    reg->iseq = 0;+}+++int bcf_sr_seek(bcf_srs_t *readers, const char *seq, int pos)+{+    if ( !readers->regions ) return 0;+    if ( !seq && !pos )+    {+        // seek to start+        bcf_sr_seek_start(readers);+        return 0;+    }+    bcf_sr_regions_overlap(readers->regions, seq, pos, pos);+    int i, nret = 0;+    for (i=0; i<readers->nreaders; i++)+    {+        nret += _reader_seek(&readers->readers[i],seq,pos,MAX_CSI_COOR-1);+    }+    return nret;+}++int bcf_sr_set_samples(bcf_srs_t *files, const char *fname, int is_file)+{+    int i, j, nsmpl, free_smpl = 0;+    char **smpl = NULL;++    void *exclude = (fname[0]=='^') ? khash_str2int_init() : NULL;+    if ( exclude || strcmp("-",fname) ) // "-" stands for all samples+    {+        smpl = hts_readlist(fname, is_file, &nsmpl);+        if ( !smpl )+        {+            fprintf(stderr,"Could not read the file: \"%s\"\n", fname);+            return 0;+        }+        if ( exclude )+        {+            for (i=0; i<nsmpl; i++)+                khash_str2int_inc(exclude, smpl[i]);+        }+        free_smpl = 1;+    }+    if ( !smpl )+    {+        smpl  = files->readers[0].header->samples;   // intersection of all samples+        nsmpl = bcf_hdr_nsamples(files->readers[0].header);+    }++    files->samples = NULL;+    files->n_smpl  = 0;+    for (i=0; i<nsmpl; i++)+    {+        if ( exclude && khash_str2int_has_key(exclude,smpl[i])  ) continue;++        int n_isec = 0;+        for (j=0; j<files->nreaders; j++)+        {+            if ( bcf_hdr_id2int(files->readers[j].header, BCF_DT_SAMPLE, smpl[i])<0 ) break;+            n_isec++;+        }+        if ( n_isec!=files->nreaders )+        {+            fprintf(stderr,"Warning: The sample \"%s\" was not found in %s, skipping\n", smpl[i], files->readers[n_isec].fname);+            continue;+        }++        files->samples = (char**) realloc(files->samples, (files->n_smpl+1)*sizeof(const char*));+        files->samples[files->n_smpl++] = strdup(smpl[i]);+    }++    if ( exclude ) khash_str2int_destroy(exclude);+    if ( free_smpl )+    {+        for (i=0; i<nsmpl; i++) free(smpl[i]);+        free(smpl);+    }++    if ( !files->n_smpl )+    {+        if ( files->nreaders>1 )+            fprintf(stderr,"No samples in common.\n");+        return 0;+    }+    for (i=0; i<files->nreaders; i++)+    {+        bcf_sr_t *reader = &files->readers[i];+        reader->samples  = (int*) malloc(sizeof(int)*files->n_smpl);+        reader->n_smpl   = files->n_smpl;+        for (j=0; j<files->n_smpl; j++)+            reader->samples[j] = bcf_hdr_id2int(reader->header, BCF_DT_SAMPLE, files->samples[j]);+    }+    return 1;+}++// Add a new region into a list sorted by start,end. On input the coordinates+// are 1-based, stored 0-based, inclusive.+static void _regions_add(bcf_sr_regions_t *reg, const char *chr, int start, int end)+{+    if ( start==-1 && end==-1 )+    {+        start = 0; end = MAX_CSI_COOR-1;+    }+    else+    {+        start--; end--; // store 0-based coordinates+    }++    if ( !reg->seq_hash )+         reg->seq_hash = khash_str2int_init();++    int iseq;+    if ( khash_str2int_get(reg->seq_hash, chr, &iseq)<0 )+    {+        // the chromosome block does not exist+        iseq = reg->nseqs++;+        reg->seq_names = (char**) realloc(reg->seq_names,sizeof(char*)*reg->nseqs);+        reg->regs = (region_t*) realloc(reg->regs,sizeof(region_t)*reg->nseqs);+        memset(&reg->regs[reg->nseqs-1],0,sizeof(region_t));+        reg->seq_names[iseq] = strdup(chr);+        reg->regs[iseq].creg = -1;+        khash_str2int_set(reg->seq_hash,reg->seq_names[iseq],iseq);+    }++    region_t *creg = &reg->regs[iseq];++    // the regions may not be sorted on input: binary search+    int i, min = 0, max = creg->nregs - 1;+    while ( min<=max )+    {+        i = (max+min)/2;+        if ( start < creg->regs[i].start ) max = i - 1;+        else if ( start > creg->regs[i].start ) min = i + 1;+        else break;+    }+    if ( min>max || creg->regs[i].start!=start || creg->regs[i].end!=end )+    {+        // no such region, insert a new one just after max+        hts_expand(region1_t,creg->nregs+1,creg->mregs,creg->regs);+        if ( ++max < creg->nregs )+            memmove(&creg->regs[max+1],&creg->regs[max],(creg->nregs - max)*sizeof(region1_t));+        creg->regs[max].start = start;+        creg->regs[max].end   = end;+        creg->nregs++;+    }+}++// File name or a list of genomic locations. If file name, NULL is returned.+static bcf_sr_regions_t *_regions_init_string(const char *str)+{+    bcf_sr_regions_t *reg = (bcf_sr_regions_t *) calloc(1, sizeof(bcf_sr_regions_t));+    reg->start = reg->end = -1;+    reg->prev_start = reg->prev_seq = -1;++    kstring_t tmp = {0,0,0};+    const char *sp = str, *ep = str;+    int from, to;+    while ( 1 )+    {+        while ( *ep && *ep!=',' && *ep!=':' ) ep++;+        tmp.l = 0;+        kputsn(sp,ep-sp,&tmp);+        if ( *ep==':' )+        {+            sp = ep+1;+            from = hts_parse_decimal(sp,(char**)&ep,0);+            if ( sp==ep )+            {+                fprintf(stderr,"[%s:%d %s] Could not parse the region(s): %s\n", __FILE__,__LINE__,__FUNCTION__,str);+                free(reg); free(tmp.s); return NULL;+            }+            if ( !*ep || *ep==',' )+            {+                _regions_add(reg, tmp.s, from, from);+                sp = ep;+                continue;+            }+            if ( *ep!='-' )+            {+                fprintf(stderr,"[%s:%d %s] Could not parse the region(s): %s\n", __FILE__,__LINE__,__FUNCTION__,str);+                free(reg); free(tmp.s); return NULL;+            }+            ep++;+            sp = ep;+            to = hts_parse_decimal(sp,(char**)&ep,0);+            if ( *ep && *ep!=',' )+            {+                fprintf(stderr,"[%s:%d %s] Could not parse the region(s): %s\n", __FILE__,__LINE__,__FUNCTION__,str);+                free(reg); free(tmp.s); return NULL;+            }+            if ( sp==ep ) to = MAX_CSI_COOR-1;+            _regions_add(reg, tmp.s, from, to);+            if ( !*ep ) break;+            sp = ep;+        }+        else+        {+            if ( tmp.l ) _regions_add(reg, tmp.s, -1, -1);+            if ( !*ep ) break;+            sp = ++ep;+        }+    }+    free(tmp.s);+    return reg;+}++// ichr,ifrom,ito are 0-based;+// returns -1 on error, 0 if the line is a comment line, 1 on success+static int _regions_parse_line(char *line, int ichr,int ifrom,int ito, char **chr,char **chr_end,int *from,int *to)+{+    *chr_end = NULL;++    if ( line[0]=='#' ) return 0;++    int k,l;    // index of the start and end column of the tab-delimited file+    if ( ifrom <= ito )+        k = ifrom, l = ito;+    else+        l = ifrom, k = ito;++    int i;+    char *se = line, *ss = NULL; // start and end+    char *tmp;+    for (i=0; i<=k && *se; i++)+    {+        ss = i==0 ? se++ : ++se;+        while (*se && *se!='\t') se++;+    }+    if ( i<=k ) return -1;+    if ( k==l )+    {+        *from = *to = hts_parse_decimal(ss, &tmp, 0);+        if ( tmp==ss ) return -1;+    }+    else+    {+        if ( k==ifrom )+            *from = hts_parse_decimal(ss, &tmp, 0);+        else+            *to = hts_parse_decimal(ss, &tmp, 0);+        if ( ss==tmp ) return -1;++        for (i=k; i<l && *se; i++)+        {+            ss = ++se;+            while (*se && *se!='\t') se++;+        }+        if ( i<l ) return -1;+        if ( k==ifrom )+            *to = hts_parse_decimal(ss, &tmp, 0);+        else+            *from = hts_parse_decimal(ss, &tmp, 0);+        if ( ss==tmp ) return -1;+    }++    ss = se = line;+    for (i=0; i<=ichr && *se; i++)+    {+        if ( i>0 ) ss = ++se;+        while (*se && *se!='\t') se++;+    }+    if ( i<=ichr ) return -1;+    *chr_end = se;+    *chr = ss;+    return 1;+}++bcf_sr_regions_t *bcf_sr_regions_init(const char *regions, int is_file, int ichr, int ifrom, int ito)+{+    bcf_sr_regions_t *reg;+    if ( !is_file ) return _regions_init_string(regions);++    reg = (bcf_sr_regions_t *) calloc(1, sizeof(bcf_sr_regions_t));+    reg->start = reg->end = -1;+    reg->prev_start = reg->prev_seq = -1;++    reg->file = hts_open(regions, "rb");+    if ( !reg->file )+    {+        fprintf(stderr,"[%s:%d %s] Could not open file: %s\n", __FILE__,__LINE__,__FUNCTION__,regions);+        free(reg);+        return NULL;+    }++    reg->tbx = tbx_index_load(regions);+    if ( !reg->tbx )+    {+        int len = strlen(regions);+        int is_bed  = strcasecmp(".bed",regions+len-4) ? 0 : 1;+        if ( !is_bed && !strcasecmp(".bed.gz",regions+len-7) ) is_bed = 1;++        if ( reg->file->format.format==vcf ) ito = 1;++        // read the whole file, tabix index is not present+        while ( hts_getline(reg->file, KS_SEP_LINE, &reg->line) > 0 )+        {+            char *chr, *chr_end;+            int from, to, ret;+            ret = _regions_parse_line(reg->line.s, ichr,ifrom,abs(ito), &chr,&chr_end,&from,&to);+            if ( ret < 0 )+            {+                if ( ito<0 )+                    ret = _regions_parse_line(reg->line.s, ichr,ifrom,ifrom, &chr,&chr_end,&from,&to);+                if ( ret<0 )+                {+                    fprintf(stderr,"[%s:%d] Could not parse the file %s, using the columns %d,%d[,%d]\n", __FILE__,__LINE__,regions,ichr+1,ifrom+1,ito+1);+                    hts_close(reg->file); reg->file = NULL; free(reg);+                    return NULL;+                }+            }+            if ( !ret ) continue;+            if ( is_bed ) from++;+            *chr_end = 0;+            _regions_add(reg, chr, from, to);+            *chr_end = '\t';+        }+        hts_close(reg->file); reg->file = NULL;+        if ( !reg->nseqs ) { free(reg); return NULL; }+        return reg;+    }++    reg->seq_names = (char**) tbx_seqnames(reg->tbx, &reg->nseqs);+    if ( !reg->seq_hash )+        reg->seq_hash = khash_str2int_init();+    int i;+    for (i=0; i<reg->nseqs; i++)+    {+        khash_str2int_set(reg->seq_hash,reg->seq_names[i],i);+    }+    reg->fname  = strdup(regions);+    reg->is_bin = 1;+    return reg;+}++void bcf_sr_regions_destroy(bcf_sr_regions_t *reg)+{+    int i;+    free(reg->fname);+    if ( reg->itr ) tbx_itr_destroy(reg->itr);+    if ( reg->tbx ) tbx_destroy(reg->tbx);+    if ( reg->file ) hts_close(reg->file);+    if ( reg->als ) free(reg->als);+    if ( reg->als_str.s ) free(reg->als_str.s);+    free(reg->line.s);+    if ( reg->regs )+    {+         // free only in-memory names, tbx names are const+        for (i=0; i<reg->nseqs; i++)+        {+            free(reg->seq_names[i]);+            free(reg->regs[i].regs);+        }+    }+    free(reg->regs);+    free(reg->seq_names);+    khash_str2int_destroy(reg->seq_hash);+    free(reg);+}++int bcf_sr_regions_seek(bcf_sr_regions_t *reg, const char *seq)+{+    reg->iseq = reg->start = reg->end = -1;+    if ( khash_str2int_get(reg->seq_hash, seq, &reg->iseq) < 0 ) return -1;  // sequence seq not in regions++    // using in-memory regions+    if ( reg->regs )+    {+        reg->regs[reg->iseq].creg = -1;+        return 0;+    }++    // reading regions from tabix+    if ( reg->itr ) tbx_itr_destroy(reg->itr);+    reg->itr = tbx_itr_querys(reg->tbx, seq);+    if ( reg->itr ) return 0;++    return -1;+}++int bcf_sr_regions_next(bcf_sr_regions_t *reg)+{+    if ( reg->iseq<0 ) return -1;+    reg->start = reg->end = -1;+    reg->nals = 0;++    // using in-memory regions+    if ( reg->regs )+    {+        while ( reg->iseq < reg->nseqs )+        {+            reg->regs[reg->iseq].creg++;+            if ( reg->regs[reg->iseq].creg < reg->regs[reg->iseq].nregs ) break;+            reg->iseq++;+        }+        if ( reg->iseq >= reg->nseqs ) { reg->iseq = -1; return -1; } // no more regions left+        region1_t *creg = &reg->regs[reg->iseq].regs[reg->regs[reg->iseq].creg];+        reg->start = creg->start;+        reg->end   = creg->end;+        return 0;+    }++    // reading from tabix+    char *chr, *chr_end;+    int ichr = 0, ifrom = 1, ito = 2, is_bed = 0, from, to;+    if ( reg->tbx )+    {+        ichr   = reg->tbx->conf.sc-1;+        ifrom  = reg->tbx->conf.bc-1;+        ito    = reg->tbx->conf.ec-1;+        if ( ito<0 ) ito = ifrom;+        is_bed = reg->tbx->conf.preset==TBX_UCSC ? 1 : 0;+    }++    int ret = 0;+    while ( !ret )+    {+        if ( reg->itr )+        {+            // tabix index present, reading a chromosome block+            ret = tbx_itr_next(reg->file, reg->tbx, reg->itr, &reg->line);+            if ( ret<0 ) { reg->iseq = -1; return -1; }+        }+        else+        {+            if ( reg->is_bin )+            {+                // Waited for seek which never came. Reopen in text mode and stream+                // through the regions, otherwise hts_getline would fail+                hts_close(reg->file);+                reg->file = hts_open(reg->fname, "r");+                if ( !reg->file )+                {+                    fprintf(stderr,"[%s:%d %s] Could not open file: %s\n", __FILE__,__LINE__,__FUNCTION__,reg->fname);+                    reg->file = NULL;+                    bcf_sr_regions_destroy(reg);+                    return -1;+                }+                reg->is_bin = 0;+            }++            // tabix index absent, reading the whole file+            ret = hts_getline(reg->file, KS_SEP_LINE, &reg->line);+            if ( ret<0 ) { reg->iseq = -1; return -1; }+        }+        ret = _regions_parse_line(reg->line.s, ichr,ifrom,ito, &chr,&chr_end,&from,&to);+        if ( ret<0 )+        {+            fprintf(stderr,"[%s:%d] Could not parse the file %s, using the columns %d,%d,%d\n", __FILE__,__LINE__,reg->fname,ichr+1,ifrom+1,ito+1);+            return -1;+        }+    }+    if ( is_bed ) from++;++    *chr_end = 0;+    if ( khash_str2int_get(reg->seq_hash, chr, &reg->iseq)<0 )+    {+        fprintf(stderr,"Broken tabix index? The sequence \"%s\" not in dictionary [%s]\n", chr,reg->line.s);+        exit(1);+    }+    *chr_end = '\t';++    reg->start = from - 1;+    reg->end   = to - 1;+    return 0;+}++static int _regions_match_alleles(bcf_sr_regions_t *reg, int als_idx, bcf1_t *rec)+{+    if ( reg->regs )+    {+        // payload is not supported for in-memory regions, switch to regidx instead in future+        fprintf(stderr,"Error: Compressed and indexed targets file is required\n");+        exit(1);+    }++    int i = 0, max_len = 0;+    if ( !reg->nals )+    {+        char *ss = reg->line.s;+        while ( i<als_idx && *ss )+        {+            if ( *ss=='\t' ) i++;+            ss++;+        }+        char *se = ss;+        reg->nals = 1;+        while ( *se && *se!='\t' )+        {+            if ( *se==',' ) reg->nals++;+            se++;+        }+        ks_resize(&reg->als_str, se-ss+1+reg->nals);+        reg->als_str.l = 0;+        hts_expand(char*,reg->nals,reg->mals,reg->als);+        reg->nals = 0;++        se = ss;+        while ( *(++se) )+        {+            if ( *se=='\t' ) break;+            if ( *se!=',' ) continue;+            reg->als[reg->nals] = &reg->als_str.s[reg->als_str.l];+            kputsn(ss,se-ss,&reg->als_str);+            if ( &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals] > max_len ) max_len = &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals];+            reg->als_str.l++;+            reg->nals++;+            ss = ++se;+        }+        reg->als[reg->nals] = &reg->als_str.s[reg->als_str.l];+        kputsn(ss,se-ss,&reg->als_str);+        if ( &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals] > max_len ) max_len = &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals];+        reg->nals++;+        reg->als_type = max_len > 1 ? VCF_INDEL : VCF_SNP;  // this is a simplified check, see vcf.c:bcf_set_variant_types+    }+    int type = bcf_get_variant_types(rec);+    if ( reg->als_type & VCF_INDEL )+        return type & VCF_INDEL ? 1 : 0;+    return !(type & VCF_INDEL) ? 1 : 0;+}++int bcf_sr_regions_overlap(bcf_sr_regions_t *reg, const char *seq, int start, int end)+{+    int iseq;+    if ( khash_str2int_get(reg->seq_hash, seq, &iseq)<0 ) return -1;    // no such sequence++    if ( reg->prev_seq==-1 || iseq!=reg->prev_seq || reg->prev_start > start ) // new chromosome or after a seek+    {+        // flush regions left on previous chromosome+        if ( reg->missed_reg_handler && reg->prev_seq!=-1 && reg->iseq!=-1 )+            bcf_sr_regions_flush(reg);++        bcf_sr_regions_seek(reg, seq);+        reg->start = reg->end = -1;+    }+    if ( reg->prev_seq==iseq && reg->iseq!=iseq ) return -2;    // no more regions on this chromosome+    reg->prev_seq = reg->iseq;+    reg->prev_start = start;++    while ( iseq==reg->iseq && reg->end < start )+    {+        if ( bcf_sr_regions_next(reg) < 0 ) return -2;  // no more regions left+        if ( reg->iseq != iseq ) return -1; // does not overlap any regions+        if ( reg->missed_reg_handler && reg->end < start ) reg->missed_reg_handler(reg, reg->missed_reg_data);+    }+    if ( reg->start <= end ) return 0;    // region overlap+    return -1;  // no overlap+}++void bcf_sr_regions_flush(bcf_sr_regions_t *reg)+{+    if ( !reg->missed_reg_handler || reg->prev_seq==-1 ) return;+    while ( !bcf_sr_regions_next(reg) ) reg->missed_reg_handler(reg, reg->missed_reg_data);+    return;+}+
+ htslib-1.3.2/tbx.c view
@@ -0,0 +1,333 @@+/*  tbx.c -- tabix API functions.++    Copyright (C) 2009, 2010, 2012-2015 Genome Research Ltd.+    Copyright (C) 2010-2012 Broad Institute.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <stdlib.h>+#include <string.h>+#include <ctype.h>+#include <stdio.h>+#include <assert.h>+#include "htslib/tbx.h"+#include "htslib/bgzf.h"++#include "htslib/khash.h"+KHASH_DECLARE(s2i, kh_cstr_t, int64_t)++tbx_conf_t tbx_conf_gff = { 0, 1, 4, 5, '#', 0 };+tbx_conf_t tbx_conf_bed = { TBX_UCSC, 1, 2, 3, '#', 0 };+tbx_conf_t tbx_conf_psltbl = { TBX_UCSC, 15, 17, 18, '#', 0 };+tbx_conf_t tbx_conf_sam = { TBX_SAM, 3, 4, 0, '@', 0 };+tbx_conf_t tbx_conf_vcf = { TBX_VCF, 1, 2, 0, '#', 0 };++typedef struct {+    int64_t beg, end;+    char *ss, *se;+    int tid;+} tbx_intv_t;++static inline int get_tid(tbx_t *tbx, const char *ss, int is_add)+{+    khint_t k;+    khash_t(s2i) *d;+    if (tbx->dict == 0) tbx->dict = kh_init(s2i);+    d = (khash_t(s2i)*)tbx->dict;+    if (is_add) {+        int absent;+        k = kh_put(s2i, d, ss, &absent);+        if (absent) {+            kh_key(d, k) = strdup(ss);+            kh_val(d, k) = kh_size(d) - 1;+        }+    } else k = kh_get(s2i, d, ss);+    return k == kh_end(d)? -1 : kh_val(d, k);+}++int tbx_name2id(tbx_t *tbx, const char *ss)+{+    return get_tid(tbx, ss, 0);+}++int tbx_parse1(const tbx_conf_t *conf, int len, char *line, tbx_intv_t *intv)+{+    int i, b = 0, id = 1, ncols = 0;+    char *s;+    intv->ss = intv->se = 0; intv->beg = intv->end = -1;+    for (i = 0; i <= len; ++i) {+        if (line[i] == '\t' || line[i] == 0) {+            ++ncols;+            if (id == conf->sc) {+                intv->ss = line + b; intv->se = line + i;+            } else if (id == conf->bc) {+                // here ->beg is 0-based.+                intv->beg = intv->end = strtol(line + b, &s, 0);+                if ( s==line+b ) return -1; // expected int+                if (!(conf->preset&TBX_UCSC)) --intv->beg;+                else ++intv->end;+                if (intv->beg < 0) intv->beg = 0;+                if (intv->end < 1) intv->end = 1;+            } else {+                if ((conf->preset&0xffff) == TBX_GENERIC) {+                    if (id == conf->ec)+                    {+                        intv->end = strtol(line + b, &s, 0);+                        if ( s==line+b ) return -1; // expected int+                    }+                } else if ((conf->preset&0xffff) == TBX_SAM) {+                    if (id == 6) { // CIGAR+                        int l = 0, op;+                        char *t;+                        for (s = line + b; s < line + i;) {+                            long x = strtol(s, &t, 10);+                            op = toupper(*t);+                            if (op == 'M' || op == 'D' || op == 'N') l += x;+                            s = t + 1;+                        }+                        if (l == 0) l = 1;+                        intv->end = intv->beg + l;+                    }+                } else if ((conf->preset&0xffff) == TBX_VCF) {+                    if (id == 4) {+                        if (b < i) intv->end = intv->beg + (i - b);+                    } else if (id == 8) { // look for "END="+                        int c = line[i];+                        line[i] = 0;+                        s = strstr(line + b, "END=");+                        if (s == line + b) s += 4;+                        else if (s) {+                            s = strstr(line + b, ";END=");+                            if (s) s += 5;+                        }+                        if (s) intv->end = strtol(s, &s, 0);+                        line[i] = c;+                    }+                }+            }+            b = i + 1;+            ++id;+        }+    }+    if (intv->ss == 0 || intv->se == 0 || intv->beg < 0 || intv->end < 0) return -1;+    return 0;+}++static inline int get_intv(tbx_t *tbx, kstring_t *str, tbx_intv_t *intv, int is_add)+{+    if (tbx_parse1(&tbx->conf, str->l, str->s, intv) == 0) {+        int c = *intv->se;+        *intv->se = '\0'; intv->tid = get_tid(tbx, intv->ss, is_add); *intv->se = c;+        return (intv->tid >= 0 && intv->beg >= 0 && intv->end >= 0)? 0 : -1;+    } else {+        char *type = NULL;+        switch (tbx->conf.preset&0xffff)+        {+            case TBX_SAM: type = "TBX_SAM"; break;+            case TBX_VCF: type = "TBX_VCF"; break;+            case TBX_UCSC: type = "TBX_UCSC"; break;+            default: type = "TBX_GENERIC"; break;+        }+        fprintf(stderr, "[E::%s] failed to parse %s, was wrong -p [type] used?\nThe offending line was: \"%s\"\n", __func__, type, str->s);+        return -1;+    }+}++int tbx_readrec(BGZF *fp, void *tbxv, void *sv, int *tid, int *beg, int *end)+{+    tbx_t *tbx = (tbx_t *) tbxv;+    kstring_t *s = (kstring_t *) sv;+    int ret;+    if ((ret = bgzf_getline(fp, '\n', s)) >= 0) {+        tbx_intv_t intv;+        get_intv(tbx, s, &intv, 0);+        *tid = intv.tid; *beg = intv.beg; *end = intv.end;+    }+    return ret;+}++void tbx_set_meta(tbx_t *tbx)+{+    int i, l = 0, l_nm;+    uint32_t x[7];+    char **name;+    uint8_t *meta;+    khint_t k;+    khash_t(s2i) *d = (khash_t(s2i)*)tbx->dict;++    memcpy(x, &tbx->conf, 24);+    name = (char**)malloc(sizeof(char*) * kh_size(d));+    for (k = kh_begin(d), l = 0; k != kh_end(d); ++k) {+        if (!kh_exist(d, k)) continue;+        name[kh_val(d, k)] = (char*)kh_key(d, k);+        l += strlen(kh_key(d, k)) + 1; // +1 to include '\0'+    }+    l_nm = x[6] = l;+    meta = (uint8_t*)malloc(l_nm + 28);+    if (ed_is_big())+        for (i = 0; i < 7; ++i)+            x[i] = ed_swap_4(x[i]);+    memcpy(meta, x, 28);+    for (l = 28, i = 0; i < (int)kh_size(d); ++i) {+        int x = strlen(name[i]) + 1;+        memcpy(meta + l, name[i], x);+        l += x;+    }+    free(name);+    hts_idx_set_meta(tbx->idx, l, meta, 0);+}++tbx_t *tbx_index(BGZF *fp, int min_shift, const tbx_conf_t *conf)+{+    tbx_t *tbx;+    kstring_t str;+    int ret, first = 0, n_lvls, fmt;+    int64_t lineno = 0;+    uint64_t last_off = 0;+    tbx_intv_t intv;++    str.s = 0; str.l = str.m = 0;+    tbx = (tbx_t*)calloc(1, sizeof(tbx_t));+    tbx->conf = *conf;+    if (min_shift > 0) n_lvls = (TBX_MAX_SHIFT - min_shift + 2) / 3, fmt = HTS_FMT_CSI;+    else min_shift = 14, n_lvls = 5, fmt = HTS_FMT_TBI;+    while ((ret = bgzf_getline(fp, '\n', &str)) >= 0) {+        ++lineno;+        if (lineno <= tbx->conf.line_skip || str.s[0] == tbx->conf.meta_char) {+            last_off = bgzf_tell(fp);+            continue;+        }+        if (first == 0) {+            tbx->idx = hts_idx_init(0, fmt, last_off, min_shift, n_lvls);+            first = 1;+        }+        get_intv(tbx, &str, &intv, 1);+        ret = hts_idx_push(tbx->idx, intv.tid, intv.beg, intv.end, bgzf_tell(fp), 1);+        if (ret < 0)+        {+            free(str.s);+            tbx_destroy(tbx);+            return NULL;+        }+    }+    if ( !tbx->idx ) tbx->idx = hts_idx_init(0, fmt, last_off, min_shift, n_lvls);   // empty file+    if ( !tbx->dict ) tbx->dict = kh_init(s2i);+    hts_idx_finish(tbx->idx, bgzf_tell(fp));+    tbx_set_meta(tbx);+    free(str.s);+    return tbx;+}++void tbx_destroy(tbx_t *tbx)+{+    khash_t(s2i) *d = (khash_t(s2i)*)tbx->dict;+    if (d != NULL)+    {+        khint_t k;+        for (k = kh_begin(d); k != kh_end(d); ++k)+            if (kh_exist(d, k)) free((char*)kh_key(d, k));+    }+    hts_idx_destroy(tbx->idx);+    kh_destroy(s2i, d);+    free(tbx);+}++int tbx_index_build2(const char *fn, const char *fnidx, int min_shift, const tbx_conf_t *conf)+{+    tbx_t *tbx;+    BGZF *fp;+    int ret;+    if ( bgzf_is_bgzf(fn)!=1 ) { fprintf(stderr,"Not a BGZF file: %s\n", fn); return -1; }+    if ((fp = bgzf_open(fn, "r")) == 0) return -1;+    if ( !fp->is_compressed ) { bgzf_close(fp); return -1; }+    tbx = tbx_index(fp, min_shift, conf);+    bgzf_close(fp);+    if ( !tbx ) return -1;+    ret = hts_idx_save_as(tbx->idx, fn, fnidx, min_shift > 0? HTS_FMT_CSI : HTS_FMT_TBI);+    tbx_destroy(tbx);+    return ret;+}++int tbx_index_build(const char *fn, int min_shift, const tbx_conf_t *conf)+{+    return tbx_index_build2(fn, NULL, min_shift, conf);+}++tbx_t *tbx_index_load2(const char *fn, const char *fnidx)+{+    tbx_t *tbx;+    uint8_t *meta;+    char *nm, *p;+    uint32_t x[7];+    int l_meta, l_nm;+    tbx = (tbx_t*)calloc(1, sizeof(tbx_t));+    tbx->idx = fnidx? hts_idx_load2(fn, fnidx) : hts_idx_load(fn, HTS_FMT_TBI);+    if ( !tbx->idx )+    {+        free(tbx);+        return NULL;+    }+    meta = hts_idx_get_meta(tbx->idx, &l_meta);+    if ( !meta )+    {+        free(tbx);+        return NULL;+    }+    memcpy(x, meta, 28);+    memcpy(&tbx->conf, x, 24);+    p = nm = (char*)meta + 28;+    l_nm = x[6];+    for (; p - nm < l_nm; p += strlen(p) + 1) get_tid(tbx, p, 1);+    return tbx;+}++tbx_t *tbx_index_load(const char *fn)+{+    return tbx_index_load2(fn, NULL);+}++const char **tbx_seqnames(tbx_t *tbx, int *n)+{+    khash_t(s2i) *d = (khash_t(s2i)*)tbx->dict;+    if (d == NULL)+    {+        *n = 0;+        return NULL;+    }+    int tid, m = kh_size(d);+    const char **names = (const char**) calloc(m,sizeof(const char*));+    khint_t k;+    for (k=kh_begin(d); k<kh_end(d); k++)+    {+        if ( !kh_exist(d,k) ) continue;+        tid = kh_val(d,k);+        assert( tid<m );+        names[tid] = kh_key(d,k);+    }+    // sanity check: there should be no gaps+    for (tid=0; tid<m; tid++)+        assert(names[tid]);+    *n = m;+    return names;+}+
+ htslib-1.3.2/vcf.c view
@@ -0,0 +1,3483 @@+/*  vcf.c -- VCF/BCF API functions.++    Copyright (C) 2012, 2013 Broad Institute.+    Copyright (C) 2012-2016 Genome Research Ltd.+    Portions copyright (C) 2014 Intel Corporation.++    Author: Heng Li <lh3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include <zlib.h>+#include <stdio.h>+#include <ctype.h>+#include <assert.h>+#include <string.h>+#include <stdlib.h>+#include <limits.h>+#include "htslib/kstring.h"+#include "htslib/bgzf.h"+#include "htslib/vcf.h"+#include "htslib/tbx.h"+#include "htslib/hfile.h"+#include "htslib/khash_str2int.h"+#include "hts_internal.h"++#include "htslib/khash.h"+KHASH_MAP_INIT_STR(vdict, bcf_idinfo_t)+typedef khash_t(vdict) vdict_t;++#include "htslib/kseq.h"+KSTREAM_DECLARE(gzFile, gzread)++uint32_t bcf_float_missing    = 0x7F800001;+uint32_t bcf_float_vector_end = 0x7F800002;+uint8_t bcf_type_shift[] = { 0, 0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };+static bcf_idinfo_t bcf_idinfo_def = { .info = { 15, 15, 15 }, .hrec = { NULL, NULL, NULL}, .id = -1 };++static const char *dump_char(char *buffer, char c)+{+    switch (c) {+    case '\n': strcpy(buffer, "\\n"); break;+    case '\r': strcpy(buffer, "\\r"); break;+    case '\t': strcpy(buffer, "\\t"); break;+    case '\'':+    case '\"':+    case '\\':+        sprintf(buffer, "\\%c", c);+        break;+    default:+        if (isprint_c(c)) sprintf(buffer, "%c", c);+        else sprintf(buffer, "\\x%02X", (unsigned char) c);+        break;+    }+    return buffer;+}++/*************************+ *** VCF header parser ***+ *************************/++int bcf_hdr_add_sample(bcf_hdr_t *h, const char *s)+{+    if ( !s ) return 0;++    const char *ss = s;+    while ( !*ss && isspace(*ss) ) ss++;+    if ( !*ss )+    {+        fprintf(stderr,"[E::%s] Empty sample name: trailing spaces/tabs in the header line?\n", __func__);+        abort();+    }++    vdict_t *d = (vdict_t*)h->dict[BCF_DT_SAMPLE];+    int ret;+    char *sdup = strdup(s);+    int k = kh_put(vdict, d, sdup, &ret);+    if (ret) { // absent+        kh_val(d, k) = bcf_idinfo_def;+        kh_val(d, k).id = kh_size(d) - 1;+    } else {+        if (hts_verbose >= 2)+        {+            fprintf(stderr, "[E::%s] Duplicated sample name '%s'\n", __func__, s);+            abort();+        }+        free(sdup);+        return -1;+    }+    int n = kh_size(d);+    h->samples = (char**) realloc(h->samples,sizeof(char*)*n);+    h->samples[n-1] = sdup;+    h->dirty = 1;+    return 0;+}++int bcf_hdr_parse_sample_line(bcf_hdr_t *h, const char *str)+{+    int ret = 0;+    int i = 0;+    const char *p, *q;+    // add samples+    for (p = q = str;; ++q) {+        if (*q != '\t' && *q != 0 && *q != '\n') continue;+        if (++i > 9) {+            char *s = (char*)malloc(q - p + 1);+            strncpy(s, p, q - p);+            s[q - p] = 0;+            if ( bcf_hdr_add_sample(h,s) < 0 ) ret = -1;+            free(s);+        }+        if (*q == 0 || *q == '\n') break;+        p = q + 1;+    }+    bcf_hdr_add_sample(h,NULL);+    return ret;+}++int bcf_hdr_sync(bcf_hdr_t *h)+{+    int i;+    for (i = 0; i < 3; i++)+    {+        vdict_t *d = (vdict_t*)h->dict[i];+        khint_t k;+        if ( h->n[i] < kh_size(d) )+        {+            // this should be true only for i=2, BCF_DT_SAMPLE+            h->n[i] = kh_size(d);+            h->id[i] = (bcf_idpair_t*) realloc(h->id[i], kh_size(d)*sizeof(bcf_idpair_t));+        }+        for (k=kh_begin(d); k<kh_end(d); k++)+        {+            if (!kh_exist(d,k)) continue;+            h->id[i][kh_val(d,k).id].key = kh_key(d,k);+            h->id[i][kh_val(d,k).id].val = &kh_val(d,k);+        }+    }+    h->dirty = 0;+    return 0;+}++void bcf_hrec_destroy(bcf_hrec_t *hrec)+{+    free(hrec->key);+    if ( hrec->value ) free(hrec->value);+    int i;+    for (i=0; i<hrec->nkeys; i++)+    {+        free(hrec->keys[i]);+        free(hrec->vals[i]);+    }+    free(hrec->keys);+    free(hrec->vals);+    free(hrec);+}++// Copies all fields except IDX.+bcf_hrec_t *bcf_hrec_dup(bcf_hrec_t *hrec)+{+    bcf_hrec_t *out = (bcf_hrec_t*) calloc(1,sizeof(bcf_hrec_t));+    out->type = hrec->type;+    if ( hrec->key ) out->key = strdup(hrec->key);+    if ( hrec->value ) out->value = strdup(hrec->value);+    out->nkeys = hrec->nkeys;+    out->keys = (char**) malloc(sizeof(char*)*hrec->nkeys);+    out->vals = (char**) malloc(sizeof(char*)*hrec->nkeys);+    int i, j = 0;+    for (i=0; i<hrec->nkeys; i++)+    {+        if ( hrec->keys[i] && !strcmp("IDX",hrec->keys[i]) ) continue;+        if ( hrec->keys[i] ) out->keys[j] = strdup(hrec->keys[i]);+        if ( hrec->vals[i] ) out->vals[j] = strdup(hrec->vals[i]);+        j++;+    }+    if ( i!=j ) out->nkeys -= i-j;   // IDX was omitted+    return out;+}++void bcf_hrec_debug(FILE *fp, bcf_hrec_t *hrec)+{+    fprintf(fp, "key=[%s] value=[%s]", hrec->key, hrec->value?hrec->value:"");+    int i;+    for (i=0; i<hrec->nkeys; i++)+        fprintf(fp, "\t[%s]=[%s]", hrec->keys[i],hrec->vals[i]);+    fprintf(fp, "\n");+}++void bcf_header_debug(bcf_hdr_t *hdr)+{+    int i, j;+    for (i=0; i<hdr->nhrec; i++)+    {+        if ( !hdr->hrec[i]->value )+        {+            fprintf(stderr, "##%s=<", hdr->hrec[i]->key);+            fprintf(stderr,"%s=%s", hdr->hrec[i]->keys[0], hdr->hrec[i]->vals[0]);+            for (j=1; j<hdr->hrec[i]->nkeys; j++)+                fprintf(stderr,",%s=%s", hdr->hrec[i]->keys[j], hdr->hrec[i]->vals[j]);+            fprintf(stderr,">\n");+        }+        else+            fprintf(stderr,"##%s=%s\n", hdr->hrec[i]->key,hdr->hrec[i]->value);+    }+}++void bcf_hrec_add_key(bcf_hrec_t *hrec, const char *str, int len)+{+    int n = ++hrec->nkeys;+    hrec->keys = (char**) realloc(hrec->keys, sizeof(char*)*n);+    hrec->vals = (char**) realloc(hrec->vals, sizeof(char*)*n);+    assert( len );+    hrec->keys[n-1] = (char*) malloc((len+1)*sizeof(char));+    memcpy(hrec->keys[n-1],str,len);+    hrec->keys[n-1][len] = 0;+    hrec->vals[n-1] = NULL;+}++void bcf_hrec_set_val(bcf_hrec_t *hrec, int i, const char *str, int len, int is_quoted)+{+    if ( !str ) { hrec->vals[i] = NULL; return; }+    if ( hrec->vals[i] ) free(hrec->vals[i]);+    if ( is_quoted )+    {+        hrec->vals[i] = (char*) malloc((len+3)*sizeof(char));+        hrec->vals[i][0] = '"';+        memcpy(&hrec->vals[i][1],str,len);+        hrec->vals[i][len+1] = '"';+        hrec->vals[i][len+2] = 0;+    }+    else+    {+        hrec->vals[i] = (char*) malloc((len+1)*sizeof(char));+        memcpy(hrec->vals[i],str,len);+        hrec->vals[i][len] = 0;+    }+}++void hrec_add_idx(bcf_hrec_t *hrec, int idx)+{+    int n = ++hrec->nkeys;+    hrec->keys = (char**) realloc(hrec->keys, sizeof(char*)*n);+    hrec->vals = (char**) realloc(hrec->vals, sizeof(char*)*n);+    hrec->keys[n-1] = strdup("IDX");+    kstring_t str = {0,0,0};+    kputw(idx, &str);+    hrec->vals[n-1] = str.s;+}++int bcf_hrec_find_key(bcf_hrec_t *hrec, const char *key)+{+    int i;+    for (i=0; i<hrec->nkeys; i++)+        if ( !strcasecmp(key,hrec->keys[i]) ) return i;+    return -1;+}++static inline int is_escaped(const char *min, const char *str)+{+    int n = 0;+    while ( --str>=min && *str=='\\' ) n++;+    return n%2;+}++bcf_hrec_t *bcf_hdr_parse_line(const bcf_hdr_t *h, const char *line, int *len)+{+    const char *p = line;+    if (p[0] != '#' || p[1] != '#') { *len = 0; return NULL; }+    p += 2;++    const char *q = p;+    while ( *q && *q!='=' ) q++;+    int n = q-p;+    if ( *q!='=' || !n ) { *len = q-line+1; return NULL; } // wrong format++    bcf_hrec_t *hrec = (bcf_hrec_t*) calloc(1,sizeof(bcf_hrec_t));+    hrec->key = (char*) malloc(sizeof(char)*(n+1));+    memcpy(hrec->key,p,n);+    hrec->key[n] = 0;++    p = ++q;+    if ( *p!='<' ) // generic field, e.g. ##samtoolsVersion=0.1.18-r579+    {+        while ( *q && *q!='\n' ) q++;+        hrec->value = (char*) malloc((q-p+1)*sizeof(char));+        memcpy(hrec->value, p, q-p);+        hrec->value[q-p] = 0;+        *len = q-line+1;+        return hrec;+    }++    // structured line, e.g.+    // ##INFO=<ID=PV1,Number=1,Type=Float,Description="P-value for baseQ bias">+    // ##PEDIGREE=<Name_0=G0-ID,Name_1=G1-ID,Name_3=GN-ID>+    int nopen = 1;+    while ( *q && *q!='\n' && nopen>0 )+    {+        p = ++q;+        while ( *q && *q==' ' ) { p++; q++; }+        // ^[A-Za-z_][0-9A-Za-z_.]*$+        if (p==q && *q && (isalpha(*q) || *q=='_'))+        {+            q++;+            while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;+        }+        n = q-p;+        int m = 0;+        while ( *q && *q==' ' ) { q++; m++; }+        if ( *q!='=' || !n )+        {+            // wrong format+            while ( *q && *q!='\n' ) q++;+            kstring_t tmp = {0,0,0};+            kputsn(line,q-line,&tmp);+            fprintf(stderr,"Could not parse the header line: \"%s\"\n", tmp.s);+            free(tmp.s);+            *len = q-line+1;+            bcf_hrec_destroy(hrec);+            return NULL;+        }+        bcf_hrec_add_key(hrec, p, q-p-m);+        p = ++q;+        while ( *q && *q==' ' ) { p++; q++; }+        int quoted = *p=='"' ? 1 : 0;+        if ( quoted ) p++, q++;+        while ( *q && *q != '\n' )+        {+            if ( quoted ) { if ( *q=='"' && !is_escaped(p,q) ) break; }+            else+            {+                if ( *q=='<' ) nopen++;+                if ( *q=='>' ) nopen--;+                if ( !nopen ) break;+                if ( *q==',' && nopen==1 ) break;+            }+            q++;+        }+        const char *r = q;+        while ( r > p && r[-1] == ' ' ) r--;+        bcf_hrec_set_val(hrec, hrec->nkeys-1, p, r-p, quoted);+        if ( quoted && *q=='"' ) q++;+        if ( *q=='>' ) { nopen--; q++; }+    }++    // Skip trailing spaces+    while ( *q && *q==' ' ) { q++; }++    *len = q-line+1;+    return hrec;+}++static int bcf_hdr_set_idx(bcf_hdr_t *hdr, int dict_type, const char *tag, bcf_idinfo_t *idinfo)+{+    // If available, preserve existing IDX+    if ( idinfo->id==-1 )+        idinfo->id = hdr->n[dict_type]++;+    else if ( idinfo->id < hdr->n[dict_type] && hdr->id[dict_type][idinfo->id].key )+    {+        fprintf(stderr,"[%s:%d %s] Conflicting IDX=%d lines in the header dictionary, the new tag is %s\n", __FILE__,__LINE__,__FUNCTION__, idinfo->id, tag);+        exit(1);+    }++    if ( idinfo->id >= hdr->n[dict_type] ) hdr->n[dict_type] = idinfo->id+1;+    hts_expand0(bcf_idpair_t,hdr->n[dict_type],hdr->m[dict_type],hdr->id[dict_type]);++    // NB: the next kh_put call can invalidate the idinfo pointer, therefore+    // we leave it unassigned here. It myst be set explicitly in bcf_hdr_sync.+    hdr->id[dict_type][idinfo->id].key = tag;++    return 0;+}++// returns: 1 when hdr needs to be synced, 0 otherwise+int bcf_hdr_register_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec)+{+    // contig+    int i,j, ret;+    khint_t k;+    char *str;+    if ( !strcmp(hrec->key, "contig") )+    {+        hrec->type = BCF_HL_CTG;++        // Get the contig ID ($str) and length ($j)+        i = bcf_hrec_find_key(hrec,"length");+        if ( i<0 ) j = 0;+        else if ( sscanf(hrec->vals[i],"%d",&j)!=1 ) return 0;++        i = bcf_hrec_find_key(hrec,"ID");+        if ( i<0 ) return 0;+        str = strdup(hrec->vals[i]);++        // Register in the dictionary+        vdict_t *d = (vdict_t*)hdr->dict[BCF_DT_CTG];+        khint_t k = kh_get(vdict, d, str);+        if ( k != kh_end(d) ) { free(str); return 0; }    // already present+        k = kh_put(vdict, d, str, &ret);++        int idx = bcf_hrec_find_key(hrec,"IDX");+        if ( idx!=-1 )+        {+            char *tmp = hrec->vals[idx];+            idx = strtol(hrec->vals[idx], &tmp, 10);+            if ( *tmp || idx < 0 )+            {+                fprintf(stderr,"[%s:%d %s] Error parsing the IDX tag, skipping.\n", __FILE__,__LINE__,__FUNCTION__);+                return 0;+            }+        }++        kh_val(d, k) = bcf_idinfo_def;+        kh_val(d, k).id = idx;+        kh_val(d, k).info[0] = j;+        kh_val(d, k).hrec[0] = hrec;+        bcf_hdr_set_idx(hdr, BCF_DT_CTG, kh_key(d,k), &kh_val(d,k));+        if ( idx==-1 ) hrec_add_idx(hrec, kh_val(d,k).id);++        return 1;+    }++    if ( !strcmp(hrec->key, "INFO") ) hrec->type = BCF_HL_INFO;+    else if ( !strcmp(hrec->key, "FILTER") ) hrec->type = BCF_HL_FLT;+    else if ( !strcmp(hrec->key, "FORMAT") ) hrec->type = BCF_HL_FMT;+    else if ( hrec->nkeys>0 ) { hrec->type = BCF_HL_STR; return 1; }+    else return 0;++    // INFO/FILTER/FORMAT+    char *id = NULL;+    int type = -1, num = -1, var = -1, idx = -1;+    for (i=0; i<hrec->nkeys; i++)+    {+        if ( !strcmp(hrec->keys[i], "ID") ) id = hrec->vals[i];+        else if ( !strcmp(hrec->keys[i], "IDX") )+        {+            char *tmp = hrec->vals[i];+            idx = strtol(hrec->vals[i], &tmp, 10);+            if ( *tmp || idx < 0 )+            {+                fprintf(stderr,"[%s:%d %s] Error parsing the IDX tag, skipping.\n", __FILE__,__LINE__,__FUNCTION__);+                return 0;+            }+        }+        else if ( !strcmp(hrec->keys[i], "Type") )+        {+            if ( !strcmp(hrec->vals[i], "Integer") ) type = BCF_HT_INT;+            else if ( !strcmp(hrec->vals[i], "Float") ) type = BCF_HT_REAL;+            else if ( !strcmp(hrec->vals[i], "String") ) type = BCF_HT_STR;+            else if ( !strcmp(hrec->vals[i], "Character") ) type = BCF_HT_STR;+            else if ( !strcmp(hrec->vals[i], "Flag") ) type = BCF_HT_FLAG;+            else+            {+                if (hts_verbose >= 2) fprintf(stderr, "[E::%s] The type \"%s\" is not supported, assuming \"String\"\n", __func__, hrec->vals[i]);+                type = BCF_HT_STR;+            }+        }+        else if ( !strcmp(hrec->keys[i], "Number") )+        {+            if ( !strcmp(hrec->vals[i],"A") ) var = BCF_VL_A;+            else if ( !strcmp(hrec->vals[i],"R") ) var = BCF_VL_R;+            else if ( !strcmp(hrec->vals[i],"G") ) var = BCF_VL_G;+            else if ( !strcmp(hrec->vals[i],".") ) var = BCF_VL_VAR;+            else+            {+                sscanf(hrec->vals[i],"%d",&num);+                var = BCF_VL_FIXED;+            }+            if (var != BCF_VL_FIXED) num = 0xfffff;+        }+    }+    uint32_t info = (uint32_t)num<<12 | var<<8 | type<<4 | hrec->type;++    if ( !id ) return 0;+    str = strdup(id);++    vdict_t *d = (vdict_t*)hdr->dict[BCF_DT_ID];+    k = kh_get(vdict, d, str);+    if ( k != kh_end(d) ) +    {+        // already present+        free(str);+        if ( kh_val(d, k).hrec[info&0xf] ) return 0;+        kh_val(d, k).info[info&0xf] = info;+        kh_val(d, k).hrec[info&0xf] = hrec;+        if ( idx==-1 ) hrec_add_idx(hrec, kh_val(d, k).id);+        return 1;+    }+    k = kh_put(vdict, d, str, &ret);+    kh_val(d, k) = bcf_idinfo_def;+    kh_val(d, k).info[info&0xf] = info;+    kh_val(d, k).hrec[info&0xf] = hrec;+    kh_val(d, k).id = idx;+    bcf_hdr_set_idx(hdr, BCF_DT_ID, kh_key(d,k), &kh_val(d,k));+    if ( idx==-1 ) hrec_add_idx(hrec, kh_val(d,k).id);++    return 1;+}++int bcf_hdr_add_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec)+{+    if ( !hrec ) return 0;++    hrec->type = BCF_HL_GEN;+    if ( !bcf_hdr_register_hrec(hdr,hrec) )+    {+        // If one of the hashed field, then it is already present+        if ( hrec->type != BCF_HL_GEN )+        {+            bcf_hrec_destroy(hrec);+            return 0;+        }++        // Is one of the generic fields and already present?+        int i;+        for (i=0; i<hdr->nhrec; i++)+        {+            if ( hdr->hrec[i]->type!=BCF_HL_GEN ) continue;+            if ( !strcmp(hdr->hrec[i]->key,hrec->key) && !strcmp(hrec->key,"fileformat") ) break;+            if ( !strcmp(hdr->hrec[i]->key,hrec->key) && !strcmp(hdr->hrec[i]->value,hrec->value) ) break;+        }+        if ( i<hdr->nhrec )+        {+            bcf_hrec_destroy(hrec);+            return 0;+        }+    }++    // New record, needs to be added+    int n = ++hdr->nhrec;+    hdr->hrec = (bcf_hrec_t**) realloc(hdr->hrec, n*sizeof(bcf_hrec_t*));+    hdr->hrec[n-1] = hrec;+    hdr->dirty = 1;++    return hrec->type==BCF_HL_GEN ? 0 : 1;+}++/*+ *  Note that while querying of FLT,INFO,FMT,CTG lines is fast (the keys are hashed),+ *  the STR,GEN lines are searched for linearly in a linked list of all header lines.+ *  This may become a problem for VCFs with huge headers, we might need to build a+ *  dictionary for these lines as well.+ */+bcf_hrec_t *bcf_hdr_get_hrec(const bcf_hdr_t *hdr, int type, const char *key, const char *value, const char *str_class)+{+    int i;+    if ( type==BCF_HL_GEN )+    {+        for (i=0; i<hdr->nhrec; i++)+        {+            if ( hdr->hrec[i]->type!=type ) continue;+            if ( strcmp(hdr->hrec[i]->key,key) ) continue;+            if ( !value || !strcmp(hdr->hrec[i]->value,value) ) return hdr->hrec[i];+        }+        return NULL;+    }+    else if ( type==BCF_HL_STR )+    {+        for (i=0; i<hdr->nhrec; i++)+        {+            if ( hdr->hrec[i]->type!=type ) continue;+            if ( strcmp(hdr->hrec[i]->key,str_class) ) continue;+            int j = bcf_hrec_find_key(hdr->hrec[i],key);+            if ( j>=0 && !strcmp(hdr->hrec[i]->vals[j],value) ) return hdr->hrec[i];+        }+        return NULL;+    }+    vdict_t *d = type==BCF_HL_CTG ? (vdict_t*)hdr->dict[BCF_DT_CTG] : (vdict_t*)hdr->dict[BCF_DT_ID];+    khint_t k = kh_get(vdict, d, value);+    if ( k == kh_end(d) ) return NULL;+    return kh_val(d, k).hrec[type==BCF_HL_CTG?0:type];+}++void bcf_hdr_check_sanity(bcf_hdr_t *hdr)+{+    static int PL_warned = 0, GL_warned = 0;++    if ( !PL_warned )+    {+        int id = bcf_hdr_id2int(hdr, BCF_DT_ID, "PL");+        if ( bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,id) && bcf_hdr_id2length(hdr,BCF_HL_FMT,id)!=BCF_VL_G )+        {+            if (hts_verbose >= 2) fprintf(stderr,"[W::%s] PL should be declared as Number=G\n", __func__);+            PL_warned = 1;+        }+    }+    if ( !GL_warned )+    {+        int id = bcf_hdr_id2int(hdr, BCF_DT_ID, "GL");+        if ( bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,id) && bcf_hdr_id2length(hdr,BCF_HL_FMT,id)!=BCF_VL_G )+        {+            if (hts_verbose >= 2) fprintf(stderr,"[W::%s] GL should be declared as Number=G\n", __func__);+            GL_warned = 1;+        }+    }+}++int bcf_hdr_parse(bcf_hdr_t *hdr, char *htxt)+{+    int len, needs_sync = 0;+    char *p = htxt;++    // Check sanity: "fileformat" string must come as first+    bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr,p,&len);+    if ( !hrec || !hrec->key || strcasecmp(hrec->key,"fileformat") )+        fprintf(stderr, "[W::%s] The first line should be ##fileformat; is the VCF/BCF header broken?\n", __func__);+    needs_sync += bcf_hdr_add_hrec(hdr, hrec);++    // The filter PASS must appear first in the dictionary+    hrec = bcf_hdr_parse_line(hdr,"##FILTER=<ID=PASS,Description=\"All filters passed\">",&len);+    needs_sync += bcf_hdr_add_hrec(hdr, hrec);++    // Parse the whole header+    while ( (hrec=bcf_hdr_parse_line(hdr,p,&len)) )+    {+        needs_sync += bcf_hdr_add_hrec(hdr, hrec);+        p += len;+    }+    int ret = bcf_hdr_parse_sample_line(hdr,p);+    bcf_hdr_sync(hdr);+    bcf_hdr_check_sanity(hdr);+    return ret;+}++int bcf_hdr_append(bcf_hdr_t *hdr, const char *line)+{+    int len;+    bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr, (char*) line, &len);+    if ( !hrec ) return -1;+    bcf_hdr_add_hrec(hdr, hrec);+    return 0;+}++void bcf_hdr_remove(bcf_hdr_t *hdr, int type, const char *key)+{+    int i = 0;+    bcf_hrec_t *hrec;+    if ( !key )+    {+        while ( i<hdr->nhrec )+        {+            if ( hdr->hrec[i]->type!=type ) { i++; continue; }+            hrec = hdr->hrec[i];++            if ( type==BCF_HL_FLT || type==BCF_HL_INFO || type==BCF_HL_FMT || type== BCF_HL_CTG )+            {+                int j = bcf_hrec_find_key(hdr->hrec[i], "ID");+                if ( j>0 )+                {+                    vdict_t *d = type==BCF_HL_CTG ? (vdict_t*)hdr->dict[BCF_DT_CTG] : (vdict_t*)hdr->dict[BCF_DT_ID];+                    khint_t k = kh_get(vdict, d, hdr->hrec[i]->vals[j]);+                    kh_val(d, k).hrec[type==BCF_HL_CTG?0:type] = NULL;+                }+            }++            hdr->dirty = 1;+            hdr->nhrec--;+            if ( i < hdr->nhrec )+                memmove(&hdr->hrec[i],&hdr->hrec[i+1],(hdr->nhrec-i)*sizeof(bcf_hrec_t*));+            bcf_hrec_destroy(hrec);+        }+        return;+    }+    while (1)+    {+        if ( type==BCF_HL_FLT || type==BCF_HL_INFO || type==BCF_HL_FMT || type== BCF_HL_CTG )+        {+            hrec = bcf_hdr_get_hrec(hdr, type, "ID", key, NULL);+            if ( !hrec ) return;++            for (i=0; i<hdr->nhrec; i++)+                if ( hdr->hrec[i]==hrec ) break;+            assert( i<hdr->nhrec );++            vdict_t *d = type==BCF_HL_CTG ? (vdict_t*)hdr->dict[BCF_DT_CTG] : (vdict_t*)hdr->dict[BCF_DT_ID];+            khint_t k = kh_get(vdict, d, key);+            kh_val(d, k).hrec[type==BCF_HL_CTG?0:type] = NULL;+        }+        else+        {+            for (i=0; i<hdr->nhrec; i++)+            {+                if ( hdr->hrec[i]->type!=type ) continue;+                if ( type==BCF_HL_GEN )+                {+                    if ( !strcmp(hdr->hrec[i]->key,key) ) break;+                }+                else+                {+                    // not all structured lines have ID, we could be more sophisticated as in bcf_hdr_get_hrec()+                    int j = bcf_hrec_find_key(hdr->hrec[i], "ID");+                    if ( j>=0 && !strcmp(hdr->hrec[i]->vals[j],key) ) break;+                }+            }+            if ( i==hdr->nhrec ) return;+            hrec = hdr->hrec[i];+        }++        hdr->nhrec--;+        if ( i < hdr->nhrec )+            memmove(&hdr->hrec[i],&hdr->hrec[i+1],(hdr->nhrec-i)*sizeof(bcf_hrec_t*));+        bcf_hrec_destroy(hrec);+        hdr->dirty = 1;+    }+}++int bcf_hdr_printf(bcf_hdr_t *hdr, const char *fmt, ...)+{+    va_list ap;+    va_start(ap, fmt);+    int n = vsnprintf(NULL, 0, fmt, ap) + 2;+    va_end(ap);++    char *line = (char*)malloc(n);+    va_start(ap, fmt);+    vsnprintf(line, n, fmt, ap);+    va_end(ap);++    int ret = bcf_hdr_append(hdr, line);++    free(line);+    return ret;+}+++/**********************+ *** BCF header I/O ***+ **********************/++const char *bcf_hdr_get_version(const bcf_hdr_t *hdr)+{+    bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, BCF_HL_GEN, "fileformat", NULL, NULL);+    if ( !hrec )+    {+        fprintf(stderr,"No version string found, assuming VCFv4.2\n");+        return "VCFv4.2";+    }+    return hrec->value;+}++void bcf_hdr_set_version(bcf_hdr_t *hdr, const char *version)+{+    bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, BCF_HL_GEN, "fileformat", NULL, NULL);+    if ( !hrec )+    {+        int len;+        kstring_t str = {0,0,0};+        ksprintf(&str,"##fileformat=%s", version);+        hrec = bcf_hdr_parse_line(hdr, str.s, &len);+        free(str.s);+    }+    else+    {+        free(hrec->value);+        hrec->value = strdup(version);+    }+    hdr->dirty = 1;+}++bcf_hdr_t *bcf_hdr_init(const char *mode)+{+    int i;+    bcf_hdr_t *h;+    h = (bcf_hdr_t*)calloc(1, sizeof(bcf_hdr_t));+    if (!h) return NULL;+    for (i = 0; i < 3; ++i)+        if ((h->dict[i] = kh_init(vdict)) == NULL) goto fail;+    if ( strchr(mode,'w') )+    {+        bcf_hdr_append(h, "##fileformat=VCFv4.2");+        // The filter PASS must appear first in the dictionary+        bcf_hdr_append(h, "##FILTER=<ID=PASS,Description=\"All filters passed\">");+    }+    return h;++ fail:+    for (i = 0; i < 3; ++i)+        kh_destroy(vdict, h->dict[i]);+    free(h);+    return NULL;+}++void bcf_hdr_destroy(bcf_hdr_t *h)+{+    int i;+    khint_t k;+    for (i = 0; i < 3; ++i) {+        vdict_t *d = (vdict_t*)h->dict[i];+        if (d == 0) continue;+        for (k = kh_begin(d); k != kh_end(d); ++k)+            if (kh_exist(d, k)) free((char*)kh_key(d, k));+        kh_destroy(vdict, d);+        free(h->id[i]);+    }+    for (i=0; i<h->nhrec; i++)+        bcf_hrec_destroy(h->hrec[i]);+    if (h->nhrec) free(h->hrec);+    if (h->samples) free(h->samples);+    free(h->keep_samples);+    free(h->transl[0]); free(h->transl[1]);+    free(h->mem.s);+    free(h);+}++bcf_hdr_t *bcf_hdr_read(htsFile *hfp)+{+    if (hfp->format.format == vcf)+        return vcf_hdr_read(hfp);++    BGZF *fp = hfp->fp.bgzf;+    uint8_t magic[5];+    bcf_hdr_t *h;+    h = bcf_hdr_init("r");+    if (!h) {+        fprintf(stderr, "[E::%s] failed to allocate bcf header\n", __func__);+        return NULL;+    }+    if (bgzf_read(fp, magic, 5) != 5)+    {+        fprintf(stderr,"[%s:%d %s] Failed to read the header (reading BCF in text mode?)\n", __FILE__,__LINE__,__FUNCTION__);+        bcf_hdr_destroy(h);+        return NULL;+    }+    if (strncmp((char*)magic, "BCF\2\2", 5) != 0)+    {+        if (!strncmp((char*)magic, "BCF", 3))+            fprintf(stderr,"[%s:%d %s] invalid BCF2 magic string: only BCFv2.2 is supported.\n", __FILE__,__LINE__,__FUNCTION__);+        else if (hts_verbose >= 2)+            fprintf(stderr, "[E::%s] invalid BCF2 magic string\n", __func__);+        bcf_hdr_destroy(h);+        return NULL;+    }+    int hlen;+    char *htxt = NULL;+    if (bgzf_read(fp, &hlen, 4) != 4) goto fail;+    htxt = (char*)malloc(hlen);+    if (!htxt) goto fail;+    if (bgzf_read(fp, htxt, hlen) != hlen) goto fail;+    bcf_hdr_parse(h, htxt);  // FIXME: Does this return anything meaningful?+    free(htxt);+    return h;+ fail:+    if (hts_verbose >= 2) {+        fprintf(stderr, "[E::%s] failed to read BCF header\n", __func__);+    }+    free(htxt);+    bcf_hdr_destroy(h);+    return NULL;+}++int bcf_hdr_write(htsFile *hfp, bcf_hdr_t *h)+{+    if ( h->dirty ) bcf_hdr_sync(h);+    if (hfp->format.format == vcf || hfp->format.format == text_format)+        return vcf_hdr_write(hfp, h);++    int hlen;+    char *htxt = bcf_hdr_fmt_text(h, 1, &hlen);+    hlen++; // include the \0 byte++    BGZF *fp = hfp->fp.bgzf;+    if ( bgzf_write(fp, "BCF\2\2", 5) !=5 ) return -1;+    if ( bgzf_write(fp, &hlen, 4) !=4 ) return -1;+    if ( bgzf_write(fp, htxt, hlen) != hlen ) return -1;++    free(htxt);+    return 0;+}++/********************+ *** BCF site I/O ***+ ********************/++bcf1_t *bcf_init()+{+    bcf1_t *v;+    v = (bcf1_t*)calloc(1, sizeof(bcf1_t));+    return v;+}++void bcf_clear(bcf1_t *v)+{+    int i;+    for (i=0; i<v->d.m_info; i++)+    {+        if ( v->d.info[i].vptr_free )+        {+            free(v->d.info[i].vptr - v->d.info[i].vptr_off);+            v->d.info[i].vptr_free = 0;+        }+    }+    for (i=0; i<v->d.m_fmt; i++)+    {+        if ( v->d.fmt[i].p_free )+        {+            free(v->d.fmt[i].p - v->d.fmt[i].p_off);+            v->d.fmt[i].p_free = 0;+        }+    }+    v->rid = v->pos = v->rlen = v->unpacked = 0;+    bcf_float_set_missing(v->qual);+    v->n_info = v->n_allele = v->n_fmt = v->n_sample = 0;+    v->shared.l = v->indiv.l = 0;+    v->d.var_type = -1;+    v->d.shared_dirty = 0;+    v->d.indiv_dirty  = 0;+    v->d.n_flt = 0;+    v->errcode = 0;+    if (v->d.m_als) v->d.als[0] = 0;+    if (v->d.m_id) v->d.id[0] = 0;+}++void bcf_empty(bcf1_t *v)+{+    bcf_clear1(v);+    free(v->d.id);+    free(v->d.als);+    free(v->d.allele); free(v->d.flt); free(v->d.info); free(v->d.fmt);+    if (v->d.var ) free(v->d.var);+    free(v->shared.s); free(v->indiv.s);+}++void bcf_destroy(bcf1_t *v)+{+    bcf_empty1(v);+    free(v);+}++static inline int bcf_read1_core(BGZF *fp, bcf1_t *v)+{+    uint32_t x[8];+    int ret;+    if ((ret = bgzf_read(fp, x, 32)) != 32) {+        if (ret == 0) return -1;+        return -2;+    }+    bcf_clear1(v);+    x[0] -= 24; // to exclude six 32-bit integers+    ks_resize(&v->shared, x[0]);+    ks_resize(&v->indiv, x[1]);+    memcpy(v, x + 2, 16);+    v->n_allele = x[6]>>16; v->n_info = x[6]&0xffff;+    v->n_fmt = x[7]>>24; v->n_sample = x[7]&0xffffff;+    v->shared.l = x[0], v->indiv.l = x[1];+    // silent fix of broken BCFs produced by earlier versions of bcf_subset, prior to and including bd6ed8b4+    if ( (!v->indiv.l || !v->n_sample) && v->n_fmt ) v->n_fmt = 0;++    if (bgzf_read(fp, v->shared.s, v->shared.l) != v->shared.l) return -1;+    if (bgzf_read(fp, v->indiv.s, v->indiv.l) != v->indiv.l) return -1;+    return 0;+}++#define bit_array_size(n) ((n)/8+1)+#define bit_array_set(a,i)   ((a)[(i)/8] |=   1 << ((i)%8))+#define bit_array_clear(a,i) ((a)[(i)/8] &= ~(1 << ((i)%8)))+#define bit_array_test(a,i)  ((a)[(i)/8] &   (1 << ((i)%8)))++static inline uint8_t *bcf_unpack_fmt_core1(uint8_t *ptr, int n_sample, bcf_fmt_t *fmt);+int bcf_subset_format(const bcf_hdr_t *hdr, bcf1_t *rec)+{+    if ( !hdr->keep_samples ) return 0;+    if ( !bcf_hdr_nsamples(hdr) )+    {+        rec->indiv.l = rec->n_sample = 0;+        return 0;+    }++    int i, j;+    uint8_t *ptr = (uint8_t*)rec->indiv.s, *dst = NULL, *src;+    bcf_dec_t *dec = &rec->d;+    hts_expand(bcf_fmt_t, rec->n_fmt, dec->m_fmt, dec->fmt);+    for (i=0; i<dec->m_fmt; ++i) dec->fmt[i].p_free = 0;++    for (i=0; i<rec->n_fmt; i++)+    {+        ptr = bcf_unpack_fmt_core1(ptr, rec->n_sample, &dec->fmt[i]);+        src = dec->fmt[i].p - dec->fmt[i].size;+        if ( dst )+        {+            memmove(dec->fmt[i-1].p + dec->fmt[i-1].p_len, dec->fmt[i].p - dec->fmt[i].p_off, dec->fmt[i].p_off);+            dec->fmt[i].p = dec->fmt[i-1].p + dec->fmt[i-1].p_len + dec->fmt[i].p_off;+        }+        dst = dec->fmt[i].p;+        for (j=0; j<hdr->nsamples_ori; j++)+        {+            src += dec->fmt[i].size;+            if ( !bit_array_test(hdr->keep_samples,j) ) continue;+            memmove(dst, src, dec->fmt[i].size);+            dst += dec->fmt[i].size;+        }+        rec->indiv.l -= dec->fmt[i].p_len - (dst - dec->fmt[i].p);+        dec->fmt[i].p_len = dst - dec->fmt[i].p;+    }+    rec->unpacked |= BCF_UN_FMT;++    rec->n_sample = bcf_hdr_nsamples(hdr);+    return 0;+}++int bcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v)+{+    if (fp->format.format == vcf) return vcf_read(fp,h,v);+    int ret = bcf_read1_core(fp->fp.bgzf, v);+    if ( ret!=0 || !h->keep_samples ) return ret;+    return bcf_subset_format(h,v);+}++int bcf_readrec(BGZF *fp, void *null, void *vv, int *tid, int *beg, int *end)+{+    bcf1_t *v = (bcf1_t *) vv;+    int ret;+    if ((ret = bcf_read1_core(fp, v)) >= 0)+        *tid = v->rid, *beg = v->pos, *end = v->pos + v->rlen;+    return ret;+}++static inline void bcf1_sync_id(bcf1_t *line, kstring_t *str)+{+    // single typed string+    if ( line->d.id && strcmp(line->d.id, ".") ) bcf_enc_vchar(str, strlen(line->d.id), line->d.id);+    else bcf_enc_size(str, 0, BCF_BT_CHAR);+}+static inline void bcf1_sync_alleles(bcf1_t *line, kstring_t *str)+{+    // list of typed strings+    int i;+    for (i=0; i<line->n_allele; i++)+        bcf_enc_vchar(str, strlen(line->d.allele[i]), line->d.allele[i]);+    if ( !line->rlen && line->n_allele ) line->rlen = strlen(line->d.allele[0]);+}+static inline void bcf1_sync_filter(bcf1_t *line, kstring_t *str)+{+    // typed vector of integers+    if ( line->d.n_flt ) bcf_enc_vint(str, line->d.n_flt, line->d.flt, -1);+    else bcf_enc_vint(str, 0, 0, -1);+}++static inline void bcf1_sync_info(bcf1_t *line, kstring_t *str)+{+    // pairs of typed vectors+    int i, irm = -1;+    for (i=0; i<line->n_info; i++)+    {+        bcf_info_t *info = &line->d.info[i];+        if ( !info->vptr )+        {+            // marked for removal+            if ( irm < 0 ) irm = i;+            continue;+        }+        kputsn_(info->vptr - info->vptr_off, info->vptr_len + info->vptr_off, str);+        if ( irm >=0 )+        {+            bcf_info_t tmp = line->d.info[irm]; line->d.info[irm] = line->d.info[i]; line->d.info[i] = tmp;+            while ( irm<=i && line->d.info[irm].vptr ) irm++;+        }+    }+    if ( irm>=0 ) line->n_info = irm;+}++static int bcf1_sync(bcf1_t *line)+{+    char *shared_ori = line->shared.s;+    size_t prev_len;++    kstring_t tmp = {0,0,0};+    if ( !line->shared.l )+    {+        // New line created via API, BCF data blocks do not exist. Get it ready for BCF output+        tmp = line->shared;+        bcf1_sync_id(line, &tmp);+        line->unpack_size[0] = tmp.l; prev_len = tmp.l;++        bcf1_sync_alleles(line, &tmp);+        line->unpack_size[1] = tmp.l - prev_len; prev_len = tmp.l;++        bcf1_sync_filter(line, &tmp);+        line->unpack_size[2] = tmp.l - prev_len;++        bcf1_sync_info(line, &tmp);+        line->shared = tmp;+    }+    else if ( line->d.shared_dirty )+    {+        // The line was edited, update the BCF data block.+        +        if ( !(line->unpacked & BCF_UN_STR) ) bcf_unpack(line,BCF_UN_STR);+        +        // ptr_ori points to the original unchanged BCF data.+        uint8_t *ptr_ori = (uint8_t *) line->shared.s;++        // ID: single typed string+        if ( line->d.shared_dirty & BCF1_DIRTY_ID )+            bcf1_sync_id(line, &tmp);+        else+            kputsn_(ptr_ori, line->unpack_size[0], &tmp);+        ptr_ori += line->unpack_size[0];+        line->unpack_size[0] = tmp.l; prev_len = tmp.l;++        // REF+ALT: list of typed strings+        if ( line->d.shared_dirty & BCF1_DIRTY_ALS )+            bcf1_sync_alleles(line, &tmp);+        else+        {+            kputsn_(ptr_ori, line->unpack_size[1], &tmp);+            if ( !line->rlen && line->n_allele ) line->rlen = strlen(line->d.allele[0]);+        }+        ptr_ori += line->unpack_size[1];+        line->unpack_size[1] = tmp.l - prev_len; prev_len = tmp.l;++        if ( line->unpacked & BCF_UN_FLT )+        {+            // FILTER: typed vector of integers+            if ( line->d.shared_dirty & BCF1_DIRTY_FLT )+                bcf1_sync_filter(line, &tmp);+            else if ( line->d.n_flt )+                kputsn_(ptr_ori, line->unpack_size[2], &tmp);+            else+                bcf_enc_vint(&tmp, 0, 0, -1);+            ptr_ori += line->unpack_size[2];+            line->unpack_size[2] = tmp.l - prev_len;++            if ( line->unpacked & BCF_UN_INFO )+            {+                // INFO: pairs of typed vectors+                if ( line->d.shared_dirty & BCF1_DIRTY_INF )+                {+                    bcf1_sync_info(line, &tmp);+                    ptr_ori = (uint8_t*)line->shared.s + line->shared.l;+                }+            }+        }++        int size = line->shared.l - (size_t)ptr_ori + (size_t)line->shared.s;+        if ( size ) kputsn_(ptr_ori, size, &tmp);++        free(line->shared.s);+        line->shared = tmp;+    }+    if ( line->shared.s != shared_ori && line->unpacked & BCF_UN_INFO )+    {+        // Reallocated line->shared.s block invalidated line->d.info[].vptr pointers+        size_t off_new = line->unpack_size[0] + line->unpack_size[1] + line->unpack_size[2];+        int i;+        for (i=0; i<line->n_info; i++)+        {+            uint8_t *vptr_free = line->d.info[i].vptr_free ? line->d.info[i].vptr - line->d.info[i].vptr_off : NULL;+            line->d.info[i].vptr = (uint8_t*) line->shared.s + off_new + line->d.info[i].vptr_off;+            off_new += line->d.info[i].vptr_len + line->d.info[i].vptr_off;+            if ( vptr_free )+            {+                free(vptr_free);+                line->d.info[i].vptr_free = 0;+            }+        }+    }++    if ( line->n_sample && line->n_fmt && (!line->indiv.l || line->d.indiv_dirty) )+    {+        // The genotype fields changed or are not present+        tmp.l = tmp.m = 0; tmp.s = NULL;+        int i, irm = -1;+        for (i=0; i<line->n_fmt; i++)+        {+            bcf_fmt_t *fmt = &line->d.fmt[i];+            if ( !fmt->p )+            {+                // marked for removal+                if ( irm < 0 ) irm = i;+                continue;+            }+            kputsn_(fmt->p - fmt->p_off, fmt->p_len + fmt->p_off, &tmp);+            if ( irm >=0 )+            {+                bcf_fmt_t tfmt = line->d.fmt[irm]; line->d.fmt[irm] = line->d.fmt[i]; line->d.fmt[i] = tfmt;+                while ( irm<=i && line->d.fmt[irm].p ) irm++;+            }++        }+        if ( irm>=0 ) line->n_fmt = irm;+        free(line->indiv.s);+        line->indiv = tmp;++        // Reallocated line->indiv.s block invalidated line->d.fmt[].p pointers+        size_t off_new = 0;+        for (i=0; i<line->n_fmt; i++)+        {+            uint8_t *p_free = line->d.fmt[i].p_free ? line->d.fmt[i].p - line->d.fmt[i].p_off : NULL;+            line->d.fmt[i].p = (uint8_t*) line->indiv.s + off_new + line->d.fmt[i].p_off;+            off_new += line->d.fmt[i].p_len + line->d.fmt[i].p_off;+            if ( p_free )+            {+                free(p_free);+                line->d.fmt[i].p_free = 0;+            }+        }+    }+    if ( !line->n_sample ) line->n_fmt = 0;+    line->d.shared_dirty = line->d.indiv_dirty = 0;+    return 0;+}++bcf1_t *bcf_copy(bcf1_t *dst, bcf1_t *src)+{+    bcf1_sync(src);++    bcf_clear(dst);+    dst->rid  = src->rid;+    dst->pos  = src->pos;+    dst->rlen = src->rlen;+    dst->qual = src->qual;+    dst->n_info = src->n_info; dst->n_allele = src->n_allele;+    dst->n_fmt = src->n_fmt; dst->n_sample = src->n_sample;++    dst->shared.m = dst->shared.l = src->shared.l;+    dst->shared.s = (char*) malloc(dst->shared.l);+    memcpy(dst->shared.s,src->shared.s,dst->shared.l);++    dst->indiv.m = dst->indiv.l = src->indiv.l;+    dst->indiv.s = (char*) malloc(dst->indiv.l);+    memcpy(dst->indiv.s,src->indiv.s,dst->indiv.l);++    return dst;+}+bcf1_t *bcf_dup(bcf1_t *src)+{+    bcf1_t *out = bcf_init1();+    return bcf_copy(out, src);+}++int bcf_write(htsFile *hfp, bcf_hdr_t *h, bcf1_t *v)+{+    if ( h->dirty ) bcf_hdr_sync(h);+    if ( bcf_hdr_nsamples(h)!=v->n_sample )+    {+        fprintf(stderr,"[%s:%d %s] Broken VCF record, the number of columns at %s:%d does not match the number of samples (%d vs %d).\n",+                __FILE__,__LINE__,__FUNCTION__,bcf_seqname(h,v),v->pos+1, v->n_sample,bcf_hdr_nsamples(h));+        return -1;+    }++    if ( hfp->format.format == vcf || hfp->format.format == text_format )+        return vcf_write(hfp,h,v);++    if ( v->errcode )+    {+        // vcf_parse1() encountered a new contig or tag, undeclared in the+        // header.  At this point, the header must have been printed,+        // proceeding would lead to a broken BCF file. Errors must be checked+        // and cleared by the caller before we can proceed.+        fprintf(stderr,"[%s:%d %s] Unchecked error (%d), exiting.\n", __FILE__,__LINE__,__FUNCTION__,v->errcode);+        exit(1);+    }+    bcf1_sync(v);   // check if the BCF record was modified++    BGZF *fp = hfp->fp.bgzf;+    uint32_t x[8];+    x[0] = v->shared.l + 24; // to include six 32-bit integers+    x[1] = v->indiv.l;+    memcpy(x + 2, v, 16);+    x[6] = (uint32_t)v->n_allele<<16 | v->n_info;+    x[7] = (uint32_t)v->n_fmt<<24 | v->n_sample;+    if ( bgzf_write(fp, x, 32) != 32 ) return -1;+    if ( bgzf_write(fp, v->shared.s, v->shared.l) != v->shared.l ) return -1;+    if ( bgzf_write(fp, v->indiv.s, v->indiv.l) != v->indiv.l ) return -1;+    return 0;+}++/**********************+ *** VCF header I/O ***+ **********************/++bcf_hdr_t *vcf_hdr_read(htsFile *fp)+{+    kstring_t txt, *s = &fp->line;+    bcf_hdr_t *h;+    h = bcf_hdr_init("r");+    if (!h) {+        fprintf(stderr, "[E::%s] failed to allocate bcf header\n", __func__);+        return NULL;+    }+    txt.l = txt.m = 0; txt.s = 0;+    while (hts_getline(fp, KS_SEP_LINE, s) >= 0) {+        if (s->l == 0) continue;+        if (s->s[0] != '#') {+            if (hts_verbose >= 2)+                fprintf(stderr, "[E::%s] no sample line\n", __func__);+            free(txt.s);+            bcf_hdr_destroy(h);+            return NULL;+        }+        if (s->s[1] != '#' && fp->fn_aux) { // insert contigs here+            int dret;+            gzFile f;+            kstream_t *ks;+            kstring_t tmp;+            tmp.l = tmp.m = 0; tmp.s = 0;+            f = gzopen(fp->fn_aux, "r");+            ks = ks_init(f);+            while (ks_getuntil(ks, 0, &tmp, &dret) >= 0) {+                int c;+                kputs("##contig=<ID=", &txt); kputs(tmp.s, &txt);+                ks_getuntil(ks, 0, &tmp, &dret);+                kputs(",length=", &txt); kputw(atol(tmp.s), &txt);+                kputsn(">\n", 2, &txt);+                if (dret != '\n')+                    while ((c = ks_getc(ks)) != '\n' && c != -1); // skip the rest of the line+            }+            free(tmp.s);+            ks_destroy(ks);+            gzclose(f);+        }+        kputsn(s->s, s->l, &txt);+        kputc('\n', &txt);+        if (s->s[1] != '#') break;+    }+    if ( !txt.s )+    {+        fprintf(stderr,"[%s:%d %s] Could not read the header\n", __FILE__,__LINE__,__FUNCTION__);+        return NULL;+    }+    bcf_hdr_parse(h, txt.s);++    // check tabix index, are all contigs listed in the header? add the missing ones+    tbx_t *idx = tbx_index_load(fp->fn);+    if ( idx )+    {+        int i, n, need_sync = 0;+        const char **names = tbx_seqnames(idx, &n);+        for (i=0; i<n; i++)+        {+            bcf_hrec_t *hrec = bcf_hdr_get_hrec(h, BCF_HL_CTG, "ID", (char*) names[i], NULL);+            if ( hrec ) continue;+            hrec = (bcf_hrec_t*) calloc(1,sizeof(bcf_hrec_t));+            hrec->key = strdup("contig");+            bcf_hrec_add_key(hrec, "ID", strlen("ID"));+            bcf_hrec_set_val(hrec, hrec->nkeys-1, (char*) names[i], strlen(names[i]), 0);+            bcf_hdr_add_hrec(h, hrec);+            need_sync = 1;+        }+        free(names);+        tbx_destroy(idx);+        if ( need_sync )+            bcf_hdr_sync(h);+    }+    free(txt.s);+    return h;+}++int bcf_hdr_set(bcf_hdr_t *hdr, const char *fname)+{+    int i, n;+    char **lines = hts_readlines(fname, &n);+    if ( !lines ) return 1;+    for (i=0; i<n-1; i++)+    {+        int k;+        bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr,lines[i],&k);+        if ( hrec ) bcf_hdr_add_hrec(hdr, hrec);+        free(lines[i]);+    }+    bcf_hdr_parse_sample_line(hdr,lines[n-1]);+    free(lines[n-1]);+    free(lines);+    bcf_hdr_sync(hdr);+    return 0;+}++static void _bcf_hrec_format(const bcf_hrec_t *hrec, int is_bcf, kstring_t *str)+{+    if ( !hrec->value )+    {+        int j, nout = 0;+        ksprintf(str, "##%s=<", hrec->key);+        for (j=0; j<hrec->nkeys; j++)+        {+            // do not output IDX if output is VCF+            if ( !is_bcf && !strcmp("IDX",hrec->keys[j]) ) continue;+            if ( nout ) kputc(',',str);+            ksprintf(str,"%s=%s", hrec->keys[j], hrec->vals[j]);+            nout++;+        }+        ksprintf(str,">\n");+    }+    else+        ksprintf(str,"##%s=%s\n", hrec->key,hrec->value);+}++void bcf_hrec_format(const bcf_hrec_t *hrec, kstring_t *str)+{+    _bcf_hrec_format(hrec,0,str);+}+char *bcf_hdr_fmt_text(const bcf_hdr_t *hdr, int is_bcf, int *len)+{+    int i;+    kstring_t txt = {0,0,0};+    for (i=0; i<hdr->nhrec; i++)+        _bcf_hrec_format(hdr->hrec[i], is_bcf, &txt);++    ksprintf(&txt,"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO");+    if ( bcf_hdr_nsamples(hdr) )+    {+        ksprintf(&txt,"\tFORMAT");+        for (i=0; i<bcf_hdr_nsamples(hdr); i++)+            ksprintf(&txt,"\t%s", hdr->samples[i]);+    }+    ksprintf(&txt,"\n");++    if ( len ) *len = txt.l;+    return txt.s;+}++const char **bcf_hdr_seqnames(const bcf_hdr_t *h, int *n)+{+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_CTG];+    int tid, m = kh_size(d);+    const char **names = (const char**) calloc(m,sizeof(const char*));+    khint_t k;+    for (k=kh_begin(d); k<kh_end(d); k++)+    {+        if ( !kh_exist(d,k) ) continue;+        tid = kh_val(d,k).id;+        assert( tid<m );+        names[tid] = kh_key(d,k);+    }+    // sanity check: there should be no gaps+    for (tid=0; tid<m; tid++)+        assert(names[tid]);+    *n = m;+    return names;+}++int vcf_hdr_write(htsFile *fp, const bcf_hdr_t *h)+{+    int hlen;+    char *htxt = bcf_hdr_fmt_text(h, 0, &hlen);+    while (hlen && htxt[hlen-1] == 0) --hlen; // kill trailing zeros+    int ret;+    if ( fp->format.compression!=no_compression )+        ret = bgzf_write(fp->fp.bgzf, htxt, hlen);+    else+        ret = hwrite(fp->fp.hfile, htxt, hlen);+    free(htxt);+    return ret<0 ? -1 : 0;+}++/***********************+ *** Typed value I/O ***+ ***********************/++void bcf_enc_vint(kstring_t *s, int n, int32_t *a, int wsize)+{+    int32_t max = INT32_MIN + 1, min = INT32_MAX;+    int i;+    if (n == 0) bcf_enc_size(s, 0, BCF_BT_NULL);+    else if (n == 1) bcf_enc_int1(s, a[0]);+    else {+        if (wsize <= 0) wsize = n;+        for (i = 0; i < n; ++i) {+            if (a[i] == bcf_int32_missing || a[i] == bcf_int32_vector_end ) continue;+            if (max < a[i]) max = a[i];+            if (min > a[i]) min = a[i];+        }+        if (max <= INT8_MAX && min > bcf_int8_vector_end) {+            bcf_enc_size(s, wsize, BCF_BT_INT8);+            for (i = 0; i < n; ++i)+                if ( a[i]==bcf_int32_vector_end ) kputc(bcf_int8_vector_end, s);+                else if ( a[i]==bcf_int32_missing ) kputc(bcf_int8_missing, s);+                else kputc(a[i], s);+        } else if (max <= INT16_MAX && min > bcf_int16_vector_end) {+            bcf_enc_size(s, wsize, BCF_BT_INT16);+            for (i = 0; i < n; ++i)+            {+                int16_t x;+                if ( a[i]==bcf_int32_vector_end ) x = bcf_int16_vector_end;+                else if ( a[i]==bcf_int32_missing ) x = bcf_int16_missing;+                else x = a[i];+                kputsn((char*)&x, 2, s);+            }+        } else {+            bcf_enc_size(s, wsize, BCF_BT_INT32);+            for (i = 0; i < n; ++i) {+                int32_t x = a[i];+                kputsn((char*)&x, 4, s);+            }+        }+    }+}++void bcf_enc_vfloat(kstring_t *s, int n, float *a)+{+    bcf_enc_size(s, n, BCF_BT_FLOAT);+    kputsn((char*)a, n << 2, s);+}++void bcf_enc_vchar(kstring_t *s, int l, const char *a)+{+    bcf_enc_size(s, l, BCF_BT_CHAR);+    kputsn(a, l, s);+}++void bcf_fmt_array(kstring_t *s, int n, int type, void *data)+{+    int j = 0;+    if (n == 0) {+        kputc('.', s);+        return;+    }+    if (type == BCF_BT_CHAR)+    {+        char *p = (char*)data;+        for (j = 0; j < n && *p; ++j, ++p)+        {+            if ( *p==bcf_str_missing ) kputc('.', s);+            else kputc(*p, s);+        }+    }+    else+    {+        #define BRANCH(type_t, is_missing, is_vector_end, kprint) { \+            type_t *p = (type_t *) data; \+            for (j=0; j<n; j++) \+            { \+                if ( is_vector_end ) break; \+                if ( j ) kputc(',', s); \+                if ( is_missing ) kputc('.', s); \+                else kprint; \+            } \+        }+        switch (type) {+            case BCF_BT_INT8:  BRANCH(int8_t,  p[j]==bcf_int8_missing,  p[j]==bcf_int8_vector_end,  kputw(p[j], s)); break;+            case BCF_BT_INT16: BRANCH(int16_t, p[j]==bcf_int16_missing, p[j]==bcf_int16_vector_end, kputw(p[j], s)); break;+            case BCF_BT_INT32: BRANCH(int32_t, p[j]==bcf_int32_missing, p[j]==bcf_int32_vector_end, kputw(p[j], s)); break;+            case BCF_BT_FLOAT: BRANCH(float,   bcf_float_is_missing(p[j]), bcf_float_is_vector_end(p[j]), ksprintf(s, "%g", p[j])); break;+            default: fprintf(stderr,"todo: type %d\n", type); exit(1); break;+        }+        #undef BRANCH+    }+}++uint8_t *bcf_fmt_sized_array(kstring_t *s, uint8_t *ptr)+{+    int x, type;+    x = bcf_dec_size(ptr, &ptr, &type);+    bcf_fmt_array(s, x, type, ptr);+    return ptr + (x << bcf_type_shift[type]);+}++/********************+ *** VCF site I/O ***+ ********************/++typedef struct {+    int key, max_m, size, offset;+    uint64_t is_gt:1, max_g:31, max_l:32;+    uint32_t y;+    uint8_t *buf;+} fmt_aux_t;++static inline void align_mem(kstring_t *s)+{+    if (s->l&7) {+        uint64_t zero = 0;+        int l = ((s->l + 7)>>3<<3) - s->l;+        kputsn((char*)&zero, l, s);+    }+}++// p,q is the start and the end of the FORMAT field+#define MAX_N_FMT 255   /* Limited by size of bcf1_t n_fmt field */+static int vcf_parse_format(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v, char *p, char *q)+{+    if ( !bcf_hdr_nsamples(h) ) return 0;++    char *r, *t;+    int j, l, m, g;+    khint_t k;+    ks_tokaux_t aux1;+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_ID];+    kstring_t *mem = (kstring_t*)&h->mem;+    fmt_aux_t fmt[MAX_N_FMT];+    mem->l = 0;++    char *end = s->s + s->l;+    if ( q>=end )+    {+        fprintf(stderr,"[%s:%d %s] Error: FORMAT column with no sample columns starting at %s:%d\n", __FILE__,__LINE__,__FUNCTION__,s->s,v->pos+1);+        return -1;+    }++    // get format information from the dictionary+    v->n_fmt = 0;+    for (j = 0, t = kstrtok(p, ":", &aux1); t; t = kstrtok(0, 0, &aux1), ++j) {+        if (j >= MAX_N_FMT) {+            v->errcode |= BCF_ERR_LIMITS;+            fprintf(stderr,"[E::%s] Error: FORMAT column at %s:%d lists more identifiers than htslib can handle.\n", __func__, bcf_seqname(h,v), v->pos+1);+            return -1;+        }++        *(char*)aux1.p = 0;+        k = kh_get(vdict, d, t);+        if (k == kh_end(d) || kh_val(d, k).info[BCF_HL_FMT] == 15) {+            if (hts_verbose >= 2) fprintf(stderr, "[W::%s] FORMAT '%s' is not defined in the header, assuming Type=String\n", __func__, t);+            kstring_t tmp = {0,0,0};+            int l;+            ksprintf(&tmp, "##FORMAT=<ID=%s,Number=1,Type=String,Description=\"Dummy\">", t);+            bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);+            free(tmp.s);+            if ( bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) ) bcf_hdr_sync((bcf_hdr_t*)h);+            k = kh_get(vdict, d, t);+            v->errcode = BCF_ERR_TAG_UNDEF;+            if (k == kh_end(d)) {+                fprintf(stderr, "[E::%s] Could not add dummy header for FORMAT '%s'\n", __func__, t);+                v->errcode |= BCF_ERR_TAG_INVALID;+                return -1;+            }+        }+        fmt[j].max_l = fmt[j].max_m = fmt[j].max_g = 0;+        fmt[j].key = kh_val(d, k).id;+        fmt[j].is_gt = !strcmp(t, "GT");+        fmt[j].y = h->id[0][fmt[j].key].val->info[BCF_HL_FMT];+        v->n_fmt++;+    }+    // compute max+    int n_sample_ori = -1;+    r = q + 1;  // r: position in the format string+    l = 0, m = g = 1, v->n_sample = 0;  // m: max vector size, l: max field len, g: max number of alleles+    while ( r<end )+    {+        // can we skip some samples?+        if ( h->keep_samples )+        {+            n_sample_ori++;+            if ( !bit_array_test(h->keep_samples,n_sample_ori) )+            {+                while ( *r!='\t' && r<end ) r++;+                if ( *r=='\t' ) { *r = 0; r++; }+                continue;+            }+        }++        // collect fmt stats: max vector size, length, number of alleles+        j = 0;  // j-th format field+        for (;;)+        {+            if ( *r == '\t' ) *r = 0;+            if ( *r == ':' || !*r )  // end of field or end of sample+            {+                if (fmt[j].max_m < m) fmt[j].max_m = m;+                if (fmt[j].max_l < l) fmt[j].max_l = l;+                if (fmt[j].is_gt && fmt[j].max_g < g) fmt[j].max_g = g;+                l = 0, m = g = 1;+                if ( *r==':' ) +                {+                    j++;+                    if ( j>=v->n_fmt ) +                    { +                        fprintf(stderr,"Incorrect number of FORMAT fields at %s:%d\n", h->id[BCF_DT_CTG][v->rid].key,v->pos+1);+                        exit(1); +                    }+                }+                else break;+            }+            else if ( *r== ',' ) m++;+            else if ( fmt[j].is_gt && (*r == '|' || *r == '/') ) g++;+            if ( r>=end ) break;+            r++; l++;+        }+        v->n_sample++;+        if ( v->n_sample == bcf_hdr_nsamples(h) ) break;+        r++;+    }++    // allocate memory for arrays+    for (j = 0; j < v->n_fmt; ++j) {+        fmt_aux_t *f = &fmt[j];+        if ( !f->max_m ) f->max_m = 1;  // omitted trailing format field+        if ((f->y>>4&0xf) == BCF_HT_STR) {+            f->size = f->is_gt? f->max_g << 2 : f->max_l;+        } else if ((f->y>>4&0xf) == BCF_HT_REAL || (f->y>>4&0xf) == BCF_HT_INT) {+            f->size = f->max_m << 2;+        } else+        {+            fprintf(stderr, "[E::%s] the format type %d currently not supported\n", __func__, f->y>>4&0xf);+            abort(); // I do not know how to do with Flag in the genotype fields+        }+        align_mem(mem);+        f->offset = mem->l;+        ks_resize(mem, mem->l + v->n_sample * f->size);+        mem->l += v->n_sample * f->size;+    }+    for (j = 0; j < v->n_fmt; ++j)+        fmt[j].buf = (uint8_t*)mem->s + fmt[j].offset;+    // fill the sample fields; at beginning of the loop, t points to the first char of a format+    n_sample_ori = -1;+    t = q + 1; m = 0;   // m: sample id+    while ( t<end )+    {+        // can we skip some samples?+        if ( h->keep_samples )+        {+            n_sample_ori++;+            if ( !bit_array_test(h->keep_samples,n_sample_ori) )+            {+                while ( *t && t<end ) t++;+                t++;+                continue;+            }+        }+        if ( m == bcf_hdr_nsamples(h) ) break;++        j = 0; // j-th format field, m-th sample+        while ( t < end )+        {+            fmt_aux_t *z = &fmt[j++];+            if ((z->y>>4&0xf) == BCF_HT_STR) {+                if (z->is_gt) { // genotypes+                    int32_t is_phased = 0, *x = (int32_t*)(z->buf + z->size * m);+                    for (l = 0;; ++t) {+                        if (*t == '.') ++t, x[l++] = is_phased;+                        else x[l++] = (strtol(t, &t, 10) + 1) << 1 | is_phased;+#if THOROUGH_SANITY_CHECKS+                        assert( 0 );    // success of strtol,strtod not checked+#endif+                        is_phased = (*t == '|');+                        if (*t != '|' && *t != '/') break;+                    }+                    if ( !l ) x[l++] = 0;   // An empty field, insert missing value+                    for (; l < z->size>>2; ++l) x[l] = bcf_int32_vector_end;+                } else {+                    char *x = (char*)z->buf + z->size * m;+                    for (r = t, l = 0; *t != ':' && *t; ++t) x[l++] = *t;+                    for (; l < z->size; ++l) x[l] = 0;+                }+            } else if ((z->y>>4&0xf) == BCF_HT_INT) {+                int32_t *x = (int32_t*)(z->buf + z->size * m);+                for (l = 0;; ++t) {+                    if (*t == '.') x[l++] = bcf_int32_missing, ++t; // ++t to skip "."+                    else x[l++] = strtol(t, &t, 10);+                    if (*t != ',') break;+                }+                if ( !l ) x[l++] = bcf_int32_missing;+                for (; l < z->size>>2; ++l) x[l] = bcf_int32_vector_end;+            } else if ((z->y>>4&0xf) == BCF_HT_REAL) {+                float *x = (float*)(z->buf + z->size * m);+                for (l = 0;; ++t) {+                    if (*t == '.' && !isdigit(t[1])) bcf_float_set_missing(x[l++]), ++t; // ++t to skip "."+                    else x[l++] = strtod(t, &t);+                    if (*t != ',') break;+                }+                if ( !l ) bcf_float_set_missing(x[l++]);    // An empty field, insert missing value+                for (; l < z->size>>2; ++l) bcf_float_set_vector_end(x[l]);+            } else abort();++            if (*t == '\0') {+                break;+            }+            else if (*t == ':') {+                t++;+            }+            else {+                char buffer[8];+                fprintf(stderr,"[E::%s] Invalid character '%s' in '%s' FORMAT field at %s:%d\n", __FUNCTION__, dump_char(buffer, *t), h->id[BCF_DT_ID][z->key].key, bcf_seqname(h,v), v->pos+1);+                v->errcode |= BCF_ERR_CHAR;+                return -1;+            }+        }++        for (; j < v->n_fmt; ++j) { // fill end-of-vector values+            fmt_aux_t *z = &fmt[j];+            if ((z->y>>4&0xf) == BCF_HT_STR) {+                if (z->is_gt) {+                    int32_t *x = (int32_t*)(z->buf + z->size * m);+                    if (z->size) x[0] = bcf_int32_missing;+                    for (l = 1; l < z->size>>2; ++l) x[l] = bcf_int32_vector_end;+                } else {+                    char *x = (char*)z->buf + z->size * m;+                    if ( z->size ) x[0] = '.';+                    for (l = 1; l < z->size; ++l) x[l] = 0;+                }+            } else if ((z->y>>4&0xf) == BCF_HT_INT) {+                int32_t *x = (int32_t*)(z->buf + z->size * m);+                x[0] = bcf_int32_missing;+                for (l = 1; l < z->size>>2; ++l) x[l] = bcf_int32_vector_end;+            } else if ((z->y>>4&0xf) == BCF_HT_REAL) {+                float *x = (float*)(z->buf + z->size * m);+                bcf_float_set_missing(x[0]);+                for (l = 1; l < z->size>>2; ++l) bcf_float_set_vector_end(x[l]);+            }+        }++        m++; t++;+    }++    // write individual genotype information+    kstring_t *str = &v->indiv;+    int i;+    if (v->n_sample > 0) {+        for (i = 0; i < v->n_fmt; ++i) {+            fmt_aux_t *z = &fmt[i];+            bcf_enc_int1(str, z->key);+            if ((z->y>>4&0xf) == BCF_HT_STR && !z->is_gt) {+                bcf_enc_size(str, z->size, BCF_BT_CHAR);+                kputsn((char*)z->buf, z->size * v->n_sample, str);+            } else if ((z->y>>4&0xf) == BCF_HT_INT || z->is_gt) {+                bcf_enc_vint(str, (z->size>>2) * v->n_sample, (int32_t*)z->buf, z->size>>2);+            } else {+                bcf_enc_size(str, z->size>>2, BCF_BT_FLOAT);+                kputsn((char*)z->buf, z->size * v->n_sample, str);+            }+        }+    }++    if ( v->n_sample!=bcf_hdr_nsamples(h) )+    {+        fprintf(stderr,"[%s:%d %s] Number of columns at %s:%d does not match the number of samples (%d vs %d).\n",+                __FILE__,__LINE__,__FUNCTION__,bcf_seqname(h,v),v->pos+1, v->n_sample,bcf_hdr_nsamples(h));+        v->errcode |= BCF_ERR_NCOLS;+        return -1;+    }+    if ( v->indiv.l > 0xffffffff )+    {+        fprintf(stderr,"[%s:%d %s] The FORMAT at %s:%d is too long...\n",+                __FILE__,__LINE__,__FUNCTION__,bcf_seqname(h,v),v->pos+1);+        v->errcode |= BCF_ERR_LIMITS;++        // Error recovery: return -1 if this is a critical error or 0 if we want to ignore the FORMAT and proceed+        v->n_fmt = 0;+        return -1;+    }++    return 0;+}++int vcf_parse(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v)+{+    int i = 0;+    char *p, *q, *r, *t;+    kstring_t *str;+    khint_t k;+    ks_tokaux_t aux;++    bcf_clear1(v);+    str = &v->shared;+    memset(&aux, 0, sizeof(ks_tokaux_t));+    for (p = kstrtok(s->s, "\t", &aux), i = 0; p; p = kstrtok(0, 0, &aux), ++i) {+        q = (char*)aux.p;+        *q = 0;+        if (i == 0) { // CHROM+            vdict_t *d = (vdict_t*)h->dict[BCF_DT_CTG];+            k = kh_get(vdict, d, p);+            if (k == kh_end(d))+            {+                // Simple error recovery for chromosomes not defined in the header. It will not help when VCF header has+                // been already printed, but will enable tools like vcfcheck to proceed.+                if (hts_verbose >= 2) fprintf(stderr, "[W::%s] contig '%s' is not defined in the header. (Quick workaround: index the file with tabix.)\n", __func__, p);+                kstring_t tmp = {0,0,0};+                int l;+                ksprintf(&tmp, "##contig=<ID=%s>", p);+                bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);+                free(tmp.s);+                if ( bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) ) bcf_hdr_sync((bcf_hdr_t*)h);+                k = kh_get(vdict, d, p);+                v->errcode = BCF_ERR_CTG_UNDEF;+                if (k == kh_end(d)) {+                    fprintf(stderr, "[E::%s] Could not add dummy header for contig '%s'\n", __func__, p);+                    v->errcode |= BCF_ERR_CTG_INVALID;+                    return -1;+                }+            }+            v->rid = kh_val(d, k).id;+        } else if (i == 1) { // POS+            v->pos = atoi(p) - 1;+        } else if (i == 2) { // ID+            if (strcmp(p, ".")) bcf_enc_vchar(str, q - p, p);+            else bcf_enc_size(str, 0, BCF_BT_CHAR);+        } else if (i == 3) { // REF+            bcf_enc_vchar(str, q - p, p);+            v->n_allele = 1, v->rlen = q - p;+        } else if (i == 4) { // ALT+            if (strcmp(p, ".")) {+                for (r = t = p;; ++r) {+                    if (*r == ',' || *r == 0) {+                        bcf_enc_vchar(str, r - t, t);+                        t = r + 1;+                        ++v->n_allele;+                    }+                    if (r == q) break;+                }+            }+        } else if (i == 5) { // QUAL+            if (strcmp(p, ".")) v->qual = atof(p);+            else memcpy(&v->qual, &bcf_float_missing, 4);+            if ( v->max_unpack && !(v->max_unpack>>1) ) return 0; // BCF_UN_STR+        } else if (i == 6) { // FILTER+            if (strcmp(p, ".")) {+                int32_t *a;+                int n_flt = 1, i;+                ks_tokaux_t aux1;+                vdict_t *d = (vdict_t*)h->dict[BCF_DT_ID];+                // count the number of filters+                if (*(q-1) == ';') *(q-1) = 0;+                for (r = p; *r; ++r)+                    if (*r == ';') ++n_flt;+                a = (int32_t*)alloca(n_flt * sizeof(int32_t));+                // add filters+                for (t = kstrtok(p, ";", &aux1), i = 0; t; t = kstrtok(0, 0, &aux1)) {+                    *(char*)aux1.p = 0;+                    k = kh_get(vdict, d, t);+                    if (k == kh_end(d))+                    {+                        // Simple error recovery for FILTERs not defined in the header. It will not help when VCF header has+                        // been already printed, but will enable tools like vcfcheck to proceed.+                        if (hts_verbose >= 2) fprintf(stderr, "[W::%s] FILTER '%s' is not defined in the header\n", __func__, t);+                        kstring_t tmp = {0,0,0};+                        int l;+                        ksprintf(&tmp, "##FILTER=<ID=%s,Description=\"Dummy\">", t);+                        bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);+                        free(tmp.s);+                        if ( bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) ) bcf_hdr_sync((bcf_hdr_t*)h);+                        k = kh_get(vdict, d, t);+                        v->errcode = BCF_ERR_TAG_UNDEF;+                        if (k == kh_end(d)) {+                            fprintf(stderr, "[E::%s] Could not add dummy header for FILTER '%s'\n", __func__, t);+                            v->errcode |= BCF_ERR_TAG_INVALID;+                            return -1;+                        }+                    }+                    a[i++] = kh_val(d, k).id;+                }+                n_flt = i;+                bcf_enc_vint(str, n_flt, a, -1);+            } else bcf_enc_vint(str, 0, 0, -1);+            if ( v->max_unpack && !(v->max_unpack>>2) ) return 0;    // BCF_UN_FLT+        } else if (i == 7) { // INFO+            char *key;+            vdict_t *d = (vdict_t*)h->dict[BCF_DT_ID];+            v->n_info = 0;+            if (strcmp(p, ".")) {+                if (*(q-1) == ';') *(q-1) = 0;+                for (r = key = p;; ++r) {+                    int c;+                    char *val, *end;+                    if (*r != ';' && *r != '=' && *r != 0) continue;+                    val = end = 0;+                    c = *r; *r = 0;+                    if (c == '=') {+                        val = r + 1;+                        for (end = val; *end != ';' && *end != 0; ++end);+                        c = *end; *end = 0;+                    } else end = r;+                    if ( !*key ) { if (c==0) break; r = end; key = r + 1; continue; }  // faulty VCF, ";;" in the INFO+                    k = kh_get(vdict, d, key);+                    if (k == kh_end(d) || kh_val(d, k).info[BCF_HL_INFO] == 15)+                    {+                        if (hts_verbose >= 2) fprintf(stderr, "[W::%s] INFO '%s' is not defined in the header, assuming Type=String\n", __func__, key);+                        kstring_t tmp = {0,0,0};+                        int l;+                        ksprintf(&tmp, "##INFO=<ID=%s,Number=1,Type=String,Description=\"Dummy\">", key);+                        bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);+                        free(tmp.s);+                        if ( bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) ) bcf_hdr_sync((bcf_hdr_t*)h);+                        k = kh_get(vdict, d, key);+                        v->errcode = BCF_ERR_TAG_UNDEF;+                        if (k == kh_end(d)) {+                            fprintf(stderr, "[E::%s] Could not add dummy header for INFO '%s'\n", __func__, key);+                            v->errcode |= BCF_ERR_TAG_INVALID;+                            return -1;+                        }+                    }+                    uint32_t y = kh_val(d, k).info[BCF_HL_INFO];+                    ++v->n_info;+                    bcf_enc_int1(str, kh_val(d, k).id);+                    if (val == 0) {+                        bcf_enc_size(str, 0, BCF_BT_NULL);+                    } else if ((y>>4&0xf) == BCF_HT_FLAG || (y>>4&0xf) == BCF_HT_STR) { // if Flag has a value, treat it as a string+                        bcf_enc_vchar(str, end - val, val);+                    } else { // int/float value/array+                        int i, n_val;+                        char *t, *te;+                        for (t = val, n_val = 1; *t; ++t) // count the number of values+                            if (*t == ',') ++n_val;+                        if ((y>>4&0xf) == BCF_HT_INT) {+                            int32_t *z;+                            z = (int32_t*)alloca(n_val * sizeof(int32_t));+                            for (i = 0, t = val; i < n_val; ++i, ++t)+                            {+                                z[i] = strtol(t, &te, 10);+                                if ( te==t ) // conversion failed+                                {+                                    z[i] = bcf_int32_missing;+                                    while ( *te && *te!=',' ) te++;+                                }+                                t = te;+                            }+                            bcf_enc_vint(str, n_val, z, -1);+                            if (strcmp(key, "END") == 0) v->rlen = z[0] - v->pos;+                        } else if ((y>>4&0xf) == BCF_HT_REAL) {+                            float *z;+                            z = (float*)alloca(n_val * sizeof(float));+                            for (i = 0, t = val; i < n_val; ++i, ++t)+                            {+                                z[i] = strtod(t, &te);+                                if ( te==t ) // conversion failed+                                {+                                    bcf_float_set_missing(z[i]);+                                    while ( *te && *te!=',' ) te++;+                                }+                                t = te;+                            }+                            bcf_enc_vfloat(str, n_val, z);+                        }+                    }+                    if (c == 0) break;+                    r = end;+                    key = r + 1;+                }+            }+            if ( v->max_unpack && !(v->max_unpack>>3) ) return 0;+        } else if (i == 8) // FORMAT+            return vcf_parse_format(s, h, v, p, q);+    }+    return 0;+}++int vcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v)+{+    int ret;+    ret = hts_getline(fp, KS_SEP_LINE, &fp->line);+    if (ret < 0) return -1;+    return vcf_parse1(&fp->line, h, v);+}++static inline uint8_t *bcf_unpack_fmt_core1(uint8_t *ptr, int n_sample, bcf_fmt_t *fmt)+{+    uint8_t *ptr_start = ptr;+    fmt->id = bcf_dec_typed_int1(ptr, &ptr);+    fmt->n = bcf_dec_size(ptr, &ptr, &fmt->type);+    fmt->size = fmt->n << bcf_type_shift[fmt->type];+    fmt->p = ptr;+    fmt->p_off  = ptr - ptr_start;+    fmt->p_free = 0;+    ptr += n_sample * fmt->size;+    fmt->p_len = ptr - fmt->p;+    return ptr;+}++static inline uint8_t *bcf_unpack_info_core1(uint8_t *ptr, bcf_info_t *info)+{+    uint8_t *ptr_start = ptr;+    info->key = bcf_dec_typed_int1(ptr, &ptr);+    info->len = bcf_dec_size(ptr, &ptr, &info->type);+    info->vptr = ptr;+    info->vptr_off  = ptr - ptr_start;+    info->vptr_free = 0;+    info->v1.i = 0;+    if (info->len == 1) {+        if (info->type == BCF_BT_INT8 || info->type == BCF_BT_CHAR) info->v1.i = *(int8_t*)ptr;+        else if (info->type == BCF_BT_INT32) info->v1.i = *(int32_t*)ptr;+        else if (info->type == BCF_BT_FLOAT) info->v1.f = *(float*)ptr;+        else if (info->type == BCF_BT_INT16) info->v1.i = *(int16_t*)ptr;+    }+    ptr += info->len << bcf_type_shift[info->type];+    info->vptr_len = ptr - info->vptr;+    return ptr;+}++int bcf_unpack(bcf1_t *b, int which)+{+    if ( !b->shared.l ) return 0; // Building a new BCF record from scratch+    uint8_t *ptr = (uint8_t*)b->shared.s, *ptr_ori;+    int *offset, i;+    bcf_dec_t *d = &b->d;+    if (which & BCF_UN_FLT) which |= BCF_UN_STR;+    if (which & BCF_UN_INFO) which |= BCF_UN_SHR;+    if ((which&BCF_UN_STR) && !(b->unpacked&BCF_UN_STR))+    {+        kstring_t tmp;++        // ID+        tmp.l = 0; tmp.s = d->id; tmp.m = d->m_id;+        ptr_ori = ptr;+        ptr = bcf_fmt_sized_array(&tmp, ptr);+        b->unpack_size[0] = ptr - ptr_ori;+        kputc('\0', &tmp);+        d->id = tmp.s; d->m_id = tmp.m;++        // REF and ALT are in a single block (d->als) and d->alleles are pointers into this block+        tmp.l = 0; tmp.s = d->als; tmp.m = d->m_als;+        offset = (int*)alloca(b->n_allele * sizeof(int));+        ptr_ori = ptr;+        for (i = 0; i < b->n_allele; ++i) {+            offset[i] = tmp.l;+            ptr = bcf_fmt_sized_array(&tmp, ptr);+            kputc('\0', &tmp);+        }+        b->unpack_size[1] = ptr - ptr_ori;+        d->als = tmp.s; d->m_als = tmp.m;++        hts_expand(char*, b->n_allele, d->m_allele, d->allele); // NM: hts_expand() is a macro+        for (i = 0; i < b->n_allele; ++i)+            d->allele[i] = d->als + offset[i];+        b->unpacked |= BCF_UN_STR;+    }+    if ((which&BCF_UN_FLT) && !(b->unpacked&BCF_UN_FLT)) { // FILTER+        ptr = (uint8_t*)b->shared.s + b->unpack_size[0] + b->unpack_size[1];+        ptr_ori = ptr;+        if (*ptr>>4) {+            int type;+            d->n_flt = bcf_dec_size(ptr, &ptr, &type);+            hts_expand(int, d->n_flt, d->m_flt, d->flt);+            for (i = 0; i < d->n_flt; ++i)+                d->flt[i] = bcf_dec_int1(ptr, type, &ptr);+        } else ++ptr, d->n_flt = 0;+        b->unpack_size[2] = ptr - ptr_ori;+        b->unpacked |= BCF_UN_FLT;+    }+    if ((which&BCF_UN_INFO) && !(b->unpacked&BCF_UN_INFO)) { // INFO+        ptr = (uint8_t*)b->shared.s + b->unpack_size[0] + b->unpack_size[1] + b->unpack_size[2];+        hts_expand(bcf_info_t, b->n_info, d->m_info, d->info);+        for (i = 0; i < d->m_info; ++i) d->info[i].vptr_free = 0;+        for (i = 0; i < b->n_info; ++i)+            ptr = bcf_unpack_info_core1(ptr, &d->info[i]);+        b->unpacked |= BCF_UN_INFO;+    }+    if ((which&BCF_UN_FMT) && b->n_sample && !(b->unpacked&BCF_UN_FMT)) { // FORMAT+        ptr = (uint8_t*)b->indiv.s;+        hts_expand(bcf_fmt_t, b->n_fmt, d->m_fmt, d->fmt);+        for (i = 0; i < d->m_fmt; ++i) d->fmt[i].p_free = 0;+        for (i = 0; i < b->n_fmt; ++i)+            ptr = bcf_unpack_fmt_core1(ptr, b->n_sample, &d->fmt[i]);+        b->unpacked |= BCF_UN_FMT;+    }+    return 0;+}++int vcf_format(const bcf_hdr_t *h, const bcf1_t *v, kstring_t *s)+{+    int i;+    bcf_unpack((bcf1_t*)v, BCF_UN_ALL);+    kputs(h->id[BCF_DT_CTG][v->rid].key, s); // CHROM+    kputc('\t', s); kputw(v->pos + 1, s); // POS+    kputc('\t', s); kputs(v->d.id ? v->d.id : ".", s); // ID+    kputc('\t', s); // REF+    if (v->n_allele > 0) kputs(v->d.allele[0], s);+    else kputc('.', s);+    kputc('\t', s); // ALT+    if (v->n_allele > 1) {+        for (i = 1; i < v->n_allele; ++i) {+            if (i > 1) kputc(',', s);+            kputs(v->d.allele[i], s);+        }+    } else kputc('.', s);+    kputc('\t', s); // QUAL+    if ( bcf_float_is_missing(v->qual) ) kputc('.', s); // QUAL+    else ksprintf(s, "%g", v->qual);+    kputc('\t', s); // FILTER+    if (v->d.n_flt) {+        for (i = 0; i < v->d.n_flt; ++i) {+            if (i) kputc(';', s);+            kputs(h->id[BCF_DT_ID][v->d.flt[i]].key, s);+        }+    } else kputc('.', s);+    kputc('\t', s); // INFO+    if (v->n_info) {+        int first = 1;+        for (i = 0; i < v->n_info; ++i) {+            bcf_info_t *z = &v->d.info[i];+            if ( !z->vptr ) continue;+            if ( !first ) kputc(';', s);+            first = 0;+            kputs(h->id[BCF_DT_ID][z->key].key, s);+            if (z->len <= 0) continue;+            kputc('=', s);+            if (z->len == 1) +            {+                switch (z->type) +                {+                    case BCF_BT_INT8:  if ( z->v1.i==bcf_int8_missing ) kputc('.', s); else kputw(z->v1.i, s); break;+                    case BCF_BT_INT16: if ( z->v1.i==bcf_int16_missing ) kputc('.', s); else kputw(z->v1.i, s); break;+                    case BCF_BT_INT32: if ( z->v1.i==bcf_int32_missing ) kputc('.', s); else kputw(z->v1.i, s); break;+                    case BCF_BT_FLOAT: if ( bcf_float_is_missing(z->v1.f) ) kputc('.', s); else ksprintf(s, "%g", z->v1.f); break;+                    case BCF_BT_CHAR:  kputc(z->v1.i, s); break;+                    default: fprintf(stderr,"todo: type %d\n", z->type); exit(1); break;+                }+            }+            else bcf_fmt_array(s, z->len, z->type, z->vptr);+        }+        if ( first ) kputc('.', s);+    } else kputc('.', s);+    // FORMAT and individual information+    if (v->n_sample)+    {+        int i,j;+        if ( v->n_fmt)+        {+            int gt_i = -1;+            bcf_fmt_t *fmt = v->d.fmt;+            int first = 1;+            for (i = 0; i < (int)v->n_fmt; ++i) {+                if ( !fmt[i].p ) continue;+                kputc(!first ? ':' : '\t', s); first = 0;+                if ( fmt[i].id<0 ) //!bcf_hdr_idinfo_exists(h,BCF_HL_FMT,fmt[i].id) )+                {+                    fprintf(stderr, "[E::%s] invalid BCF, the FORMAT tag id=%d not present in the header.\n", __func__, fmt[i].id);+                    abort();+                }+                kputs(h->id[BCF_DT_ID][fmt[i].id].key, s);+                if (strcmp(h->id[BCF_DT_ID][fmt[i].id].key, "GT") == 0) gt_i = i;+            }+            if ( first ) kputs("\t.", s);+            for (j = 0; j < v->n_sample; ++j) {+                kputc('\t', s);+                first = 1;+                for (i = 0; i < (int)v->n_fmt; ++i) {+                    bcf_fmt_t *f = &fmt[i];+                    if ( !f->p ) continue;+                    if (!first) kputc(':', s);+                    first = 0;+                    if (gt_i == i)+                        bcf_format_gt(f,j,s);+                    else+                        bcf_fmt_array(s, f->n, f->type, f->p + j * f->size);+                }+                if ( first ) kputc('.', s);+            }+        }+        else+            for (j=0; j<=v->n_sample; j++)+                kputs("\t.", s);+    }+    kputc('\n', s);+    return 0;+}++int vcf_write_line(htsFile *fp, kstring_t *line)+{+    int ret;+    if ( line->s[line->l-1]!='\n' ) kputc('\n',line);+    if ( fp->format.compression!=no_compression )+        ret = bgzf_write(fp->fp.bgzf, line->s, line->l);+    else+        ret = hwrite(fp->fp.hfile, line->s, line->l);+    return ret==line->l ? 0 : -1;+}++int vcf_write(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v)+{+    int ret;+    fp->line.l = 0;+    vcf_format1(h, v, &fp->line);+    if ( fp->format.compression!=no_compression )+        ret = bgzf_write(fp->fp.bgzf, fp->line.s, fp->line.l);+    else+        ret = hwrite(fp->fp.hfile, fp->line.s, fp->line.l);+    return ret==fp->line.l ? 0 : -1;+}++/************************+ * Data access routines *+ ************************/++int bcf_hdr_id2int(const bcf_hdr_t *h, int which, const char *id)+{+    khint_t k;+    vdict_t *d = (vdict_t*)h->dict[which];+    k = kh_get(vdict, d, id);+    return k == kh_end(d)? -1 : kh_val(d, k).id;+}+++/********************+ *** BCF indexing ***+ ********************/++hts_idx_t *bcf_index(htsFile *fp, int min_shift)+{+    int n_lvls, i;+    bcf1_t *b;+    hts_idx_t *idx;+    bcf_hdr_t *h;+    int64_t max_len = 0, s;+    h = bcf_hdr_read(fp);+    if ( !h ) return NULL;+    int nids = 0;+    for (i = 0; i < h->n[BCF_DT_CTG]; ++i)+    {+        if ( !h->id[BCF_DT_CTG][i].val ) continue;+        if ( max_len < h->id[BCF_DT_CTG][i].val->info[0] ) max_len = h->id[BCF_DT_CTG][i].val->info[0];+        nids++;+    }+    if ( !max_len ) max_len = ((int64_t)1<<31) - 1;  // In case contig line is broken.+    max_len += 256;+    for (n_lvls = 0, s = 1<<min_shift; max_len > s; ++n_lvls, s <<= 3);+    idx = hts_idx_init(nids, HTS_FMT_CSI, bgzf_tell(fp->fp.bgzf), min_shift, n_lvls);+    b = bcf_init1();+    while (bcf_read1(fp,h, b) >= 0) {+        int ret;+        ret = hts_idx_push(idx, b->rid, b->pos, b->pos + b->rlen, bgzf_tell(fp->fp.bgzf), 1);+        if (ret < 0)+        {+            bcf_destroy1(b);+            hts_idx_destroy(idx);+            return NULL;+        }+    }+    hts_idx_finish(idx, bgzf_tell(fp->fp.bgzf));+    bcf_destroy1(b);+    bcf_hdr_destroy(h);+    return idx;+}++hts_idx_t *bcf_index_load2(const char *fn, const char *fnidx)+{+    return fnidx? hts_idx_load2(fn, fnidx) : bcf_index_load(fn);+}++int bcf_index_build2(const char *fn, const char *fnidx, int min_shift)+{+    htsFile *fp;+    hts_idx_t *idx;+    int ret;+    if ((fp = hts_open(fn, "rb")) == 0) return -1;+    if ( fp->format.compression!=bgzf ) { hts_close(fp); return -1; }+    idx = bcf_index(fp, min_shift);+    hts_close(fp);+    if ( !idx ) return -1;+    ret = hts_idx_save_as(idx, fn, fnidx, HTS_FMT_CSI);+    hts_idx_destroy(idx);+    return ret;+}++int bcf_index_build(const char *fn, int min_shift)+{+    return bcf_index_build2(fn, NULL, min_shift);+}++/*****************+ *** Utilities ***+ *****************/++int bcf_hdr_combine(bcf_hdr_t *dst, const bcf_hdr_t *src)+{+    int i, ndst_ori = dst->nhrec, need_sync = 0, ret = 0;+    for (i=0; i<src->nhrec; i++)+    {+        if ( src->hrec[i]->type==BCF_HL_GEN && src->hrec[i]->value )+        {+            int j;+            for (j=0; j<ndst_ori; j++)+            {+                if ( dst->hrec[j]->type!=BCF_HL_GEN ) continue;++                // Checking only the key part of generic lines, otherwise+                // the VCFs are too verbose. Should we perhaps add a flag+                // to bcf_hdr_combine() and make this optional?+                if ( !strcmp(src->hrec[i]->key,dst->hrec[j]->key) ) break;+            }+            if ( j>=ndst_ori )+                need_sync += bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));+        }+        else if ( src->hrec[i]->type==BCF_HL_STR )+        {+            // NB: we are ignoring fields without ID+            int j = bcf_hrec_find_key(src->hrec[i],"ID");+            if ( j>=0 )+            {+                bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], src->hrec[i]->key);+                if ( !rec )+                    need_sync += bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));+            }+        }+        else+        {+            int j = bcf_hrec_find_key(src->hrec[i],"ID");+            assert( j>=0 ); // this should always be true for valid VCFs++            bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], NULL);+            if ( !rec )+                need_sync += bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));+            else if ( src->hrec[i]->type==BCF_HL_INFO || src->hrec[i]->type==BCF_HL_FMT )+            {+                // Check that both records are of the same type. The bcf_hdr_id2length+                // macro cannot be used here because dst header is not synced yet.+                vdict_t *d_src = (vdict_t*)src->dict[BCF_DT_ID];+                vdict_t *d_dst = (vdict_t*)dst->dict[BCF_DT_ID];+                khint_t k_src  = kh_get(vdict, d_src, src->hrec[i]->vals[0]);+                khint_t k_dst  = kh_get(vdict, d_dst, src->hrec[i]->vals[0]);+                if ( (kh_val(d_src,k_src).info[rec->type]>>8 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>8 & 0xf) )+                {+                    fprintf(stderr,"Warning: trying to combine \"%s\" tag definitions of different lengths\n", src->hrec[i]->vals[0]);+                    ret |= 1;+                }+                if ( (kh_val(d_src,k_src).info[rec->type]>>4 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>4 & 0xf) )+                {+                    fprintf(stderr,"Warning: trying to combine \"%s\" tag definitions of different types\n", src->hrec[i]->vals[0]);+                    ret |= 1;+                }+            }+        }+    }+    if ( need_sync ) bcf_hdr_sync(dst);+    return ret;+}++bcf_hdr_t *bcf_hdr_merge(bcf_hdr_t *dst, const bcf_hdr_t *src)+{+    if ( !dst )+    {+        // this will effectively strip existing IDX attributes from src to become dst+        dst = bcf_hdr_init("r");+        char *htxt = bcf_hdr_fmt_text(src, 0, NULL);+        bcf_hdr_parse(dst, htxt);+        free(htxt);+        return dst;+    }++    int i, ndst_ori = dst->nhrec, need_sync = 0, ret = 0;+    for (i=0; i<src->nhrec; i++)+    {+        if ( src->hrec[i]->type==BCF_HL_GEN && src->hrec[i]->value )+        {+            int j;+            for (j=0; j<ndst_ori; j++)+            {+                if ( dst->hrec[j]->type!=BCF_HL_GEN ) continue;++                // Checking only the key part of generic lines, otherwise+                // the VCFs are too verbose. Should we perhaps add a flag+                // to bcf_hdr_combine() and make this optional?+                if ( !strcmp(src->hrec[i]->key,dst->hrec[j]->key) ) break;+            }+            if ( j>=ndst_ori )+                need_sync += bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));+        }+        else if ( src->hrec[i]->type==BCF_HL_STR )+        {+            // NB: we are ignoring fields without ID+            int j = bcf_hrec_find_key(src->hrec[i],"ID");+            if ( j>=0 )+            {+                bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], src->hrec[i]->key);+                if ( !rec )+                    need_sync += bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));+            }+        }+        else+        {+            int j = bcf_hrec_find_key(src->hrec[i],"ID");+            assert( j>=0 ); // this should always be true for valid VCFs++            bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], NULL);+            if ( !rec )+                need_sync += bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));+            else if ( src->hrec[i]->type==BCF_HL_INFO || src->hrec[i]->type==BCF_HL_FMT )+            {+                // Check that both records are of the same type. The bcf_hdr_id2length+                // macro cannot be used here because dst header is not synced yet.+                vdict_t *d_src = (vdict_t*)src->dict[BCF_DT_ID];+                vdict_t *d_dst = (vdict_t*)dst->dict[BCF_DT_ID];+                khint_t k_src  = kh_get(vdict, d_src, src->hrec[i]->vals[0]);+                khint_t k_dst  = kh_get(vdict, d_dst, src->hrec[i]->vals[0]);+                if ( (kh_val(d_src,k_src).info[rec->type]>>8 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>8 & 0xf) )+                {+                    fprintf(stderr,"Warning: trying to combine \"%s\" tag definitions of different lengths\n", src->hrec[i]->vals[0]);+                    ret |= 1;+                }+                if ( (kh_val(d_src,k_src).info[rec->type]>>4 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>4 & 0xf) )+                {+                    fprintf(stderr,"Warning: trying to combine \"%s\" tag definitions of different types\n", src->hrec[i]->vals[0]);+                    ret |= 1;+                }+            }+        }+    }+    if ( need_sync ) bcf_hdr_sync(dst);+    return dst;+}+int bcf_translate(const bcf_hdr_t *dst_hdr, bcf_hdr_t *src_hdr, bcf1_t *line)+{+    int i;+    if ( line->errcode )+    {+        fprintf(stderr,"[%s:%d %s] Unchecked error (%d), exiting.\n", __FILE__,__LINE__,__FUNCTION__,line->errcode);+        exit(1);+    }+    if ( src_hdr->ntransl==-1 ) return 0;    // no need to translate, all tags have the same id+    if ( !src_hdr->ntransl )  // called for the first time, see what needs translating+    {+        int dict;+        for (dict=0; dict<2; dict++)    // BCF_DT_ID and BCF_DT_CTG+        {+            src_hdr->transl[dict] = (int*) malloc(src_hdr->n[dict]*sizeof(int));+            for (i=0; i<src_hdr->n[dict]; i++)+            {+                if ( !src_hdr->id[dict][i].key ) // gap left after removed BCF header lines+                {+                    src_hdr->transl[dict][i] = -1;+                    continue;+                }+                src_hdr->transl[dict][i] = bcf_hdr_id2int(dst_hdr,dict,src_hdr->id[dict][i].key);+                if ( src_hdr->transl[dict][i]!=-1 && i!=src_hdr->transl[dict][i] ) src_hdr->ntransl++;+            }+        }+        if ( !src_hdr->ntransl )+        {+            free(src_hdr->transl[0]); src_hdr->transl[0] = NULL;+            free(src_hdr->transl[1]); src_hdr->transl[1] = NULL;+            src_hdr->ntransl = -1;+        }+        if ( src_hdr->ntransl==-1 ) return 0;+    }+    bcf_unpack(line,BCF_UN_ALL);++    // CHROM+    if ( src_hdr->transl[BCF_DT_CTG][line->rid] >=0 ) line->rid = src_hdr->transl[BCF_DT_CTG][line->rid];++    // FILTER+    for (i=0; i<line->d.n_flt; i++)+    {+        int src_id = line->d.flt[i];+        if ( src_hdr->transl[BCF_DT_ID][src_id] >=0 )+            line->d.flt[i] = src_hdr->transl[BCF_DT_ID][src_id];+        line->d.shared_dirty |= BCF1_DIRTY_FLT;+    }++    // INFO+    for (i=0; i<line->n_info; i++)+    {+        int src_id = line->d.info[i].key;+        int dst_id = src_hdr->transl[BCF_DT_ID][src_id];+        if ( dst_id<0 ) continue;+        int src_size = src_id>>7 ? ( src_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;+        int dst_size = dst_id>>7 ? ( dst_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;+        if ( src_size==dst_size )   // can overwrite+        {+            line->d.info[i].key = dst_id;+            uint8_t *vptr = line->d.info[i].vptr - line->d.info[i].vptr_off;+            if ( dst_size==BCF_BT_INT8 ) { vptr[1] = (uint8_t)dst_id; }+            else if ( dst_size==BCF_BT_INT16 ) { *(uint16_t*)vptr = (uint16_t)dst_id; }+            else { *(uint32_t*)vptr = (uint32_t)dst_id; }+        }+        else    // must realloc+        {+            bcf_info_t *info = &line->d.info[i];+            assert( !info->vptr_free );+            kstring_t str = {0,0,0};+            bcf_enc_int1(&str, dst_id);+            bcf_enc_size(&str, info->len,info->type);+            info->vptr_off = str.l;+            kputsn((char*)info->vptr, info->vptr_len, &str);+            info->vptr = (uint8_t*)str.s + info->vptr_off;+            info->vptr_free = 1;+            info->key = dst_id;+            line->d.shared_dirty |= BCF1_DIRTY_INF;+        }+    }++    // FORMAT+    for (i=0; i<line->n_fmt; i++)+    {+        int src_id = line->d.fmt[i].id;+        int dst_id = src_hdr->transl[BCF_DT_ID][src_id];+        if ( dst_id<0 ) continue;+        int src_size = src_id>>7 ? ( src_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;+        int dst_size = dst_id>>7 ? ( dst_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;+        if ( src_size==dst_size )   // can overwrite+        {+            line->d.fmt[i].id = dst_id;+            uint8_t *p = line->d.fmt[i].p - line->d.fmt[i].p_off;    // pointer to the vector size (4bits) and BT type (4bits)+            if ( dst_size==BCF_BT_INT8 ) { p[1] = dst_id; }+            else if ( dst_size==BCF_BT_INT16 ) { uint8_t *x = (uint8_t*) &dst_id; p[1] = x[0]; p[2] = x[1]; }+            else { uint8_t *x = (uint8_t*) &dst_id; p[1] = x[0]; p[2] = x[1]; p[3] = x[2]; p[4] = x[3]; }+        }+        else    // must realloc+        {+            bcf_fmt_t *fmt = &line->d.fmt[i];+            assert( !fmt->p_free );+            kstring_t str = {0,0,0};+            bcf_enc_int1(&str, dst_id);+            bcf_enc_size(&str, fmt->n, fmt->type);+            fmt->p_off = str.l;+            kputsn((char*)fmt->p, fmt->p_len, &str);+            fmt->p = (uint8_t*)str.s + fmt->p_off;+            fmt->p_free = 1;+            fmt->id = dst_id;+            line->d.indiv_dirty = 1;+        }+    }+    return 0;+}++bcf_hdr_t *bcf_hdr_dup(const bcf_hdr_t *hdr)+{+    bcf_hdr_t *hout = bcf_hdr_init("r");+    char *htxt = bcf_hdr_fmt_text(hdr, 1, NULL);+    if (!hout) {+        fprintf(stderr, "[E::%s] failed to allocate bcf header\n", __func__);+        free(htxt);+        return NULL;+    }+    bcf_hdr_parse(hout, htxt);+    free(htxt);+    return hout;+}++bcf_hdr_t *bcf_hdr_subset(const bcf_hdr_t *h0, int n, char *const* samples, int *imap)+{+    int hlen;+    void *names_hash = khash_str2int_init();+    char *htxt = bcf_hdr_fmt_text(h0, 1, &hlen);+    kstring_t str;+    bcf_hdr_t *h;+    str.l = str.m = 0; str.s = 0;+    h = bcf_hdr_init("w");+    if (!h) {+        fprintf(stderr, "[E::%s] failed to allocate bcf header\n", __func__);+        free(htxt);+        return NULL;+    }+    bcf_hdr_set_version(h,bcf_hdr_get_version(h0));+    int j;+    for (j=0; j<n; j++) imap[j] = -1;+    if ( bcf_hdr_nsamples(h0) > 0) {+        char *p;+        int i = 0, end = n? 8 : 7;+        while ((p = strstr(htxt, "#CHROM\t")) != 0)+            if (p > htxt && *(p-1) == '\n') break;+        while ((p = strchr(p, '\t')) != 0 && i < end) ++i, ++p;+        if (i != end) {+            free(h); free(str.s);+            return 0; // malformated header+        }+        kputsn(htxt, p - htxt, &str);+        for (i = 0; i < n; ++i) {+            if ( khash_str2int_has_key(names_hash,samples[i]) )+            {+                fprintf(stderr,"[E::bcf_hdr_subset] Duplicate sample name \"%s\".\n", samples[i]);+                free(str.s);+                free(htxt);+                khash_str2int_destroy(names_hash);+                bcf_hdr_destroy(h);+                return NULL;+            }+            imap[i] = bcf_hdr_id2int(h0, BCF_DT_SAMPLE, samples[i]);+            if (imap[i] < 0) continue;+            kputc('\t', &str);+            kputs(samples[i], &str);+            khash_str2int_inc(names_hash,samples[i]);+        }+    } else kputsn(htxt, hlen, &str);+    while (str.l && (!str.s[str.l-1] || str.s[str.l-1]=='\n') ) str.l--; // kill trailing zeros and newlines+    kputc('\n',&str);+    bcf_hdr_parse(h, str.s);+    free(str.s);+    free(htxt);+    khash_str2int_destroy(names_hash);+    return h;+}++int bcf_hdr_set_samples(bcf_hdr_t *hdr, const char *samples, int is_file)+{+    if ( samples && !strcmp("-",samples) ) return 0;            // keep all samples++    hdr->nsamples_ori = bcf_hdr_nsamples(hdr);+    if ( !samples ) { bcf_hdr_nsamples(hdr) = 0; return 0; }    // exclude all samples++    int i, narr = bit_array_size(bcf_hdr_nsamples(hdr));+    hdr->keep_samples = (uint8_t*) calloc(narr,1);+    if ( samples[0]=='^' )+        for (i=0; i<bcf_hdr_nsamples(hdr); i++) bit_array_set(hdr->keep_samples,i);++    int idx, n, ret = 0;+    char **smpls = hts_readlist(samples[0]=='^'?samples+1:samples, is_file, &n);+    if ( !smpls ) return -1;+    for (i=0; i<n; i++)+    {+        idx = bcf_hdr_id2int(hdr,BCF_DT_SAMPLE,smpls[i]);+        if ( idx<0 )+        {+            if ( !ret ) ret = i+1;+            continue;+        }+        assert( idx<bcf_hdr_nsamples(hdr) );+        if (  samples[0]=='^' )+            bit_array_clear(hdr->keep_samples, idx);+        else+            bit_array_set(hdr->keep_samples, idx);+    }+    for (i=0; i<n; i++) free(smpls[i]);+    free(smpls);++    bcf_hdr_nsamples(hdr) = 0;+    for (i=0; i<hdr->nsamples_ori; i++)+        if ( bit_array_test(hdr->keep_samples,i) ) bcf_hdr_nsamples(hdr)++;+    if ( !bcf_hdr_nsamples(hdr) ) { free(hdr->keep_samples); hdr->keep_samples=NULL; }+    else+    {+        char **samples = (char**) malloc(sizeof(char*)*bcf_hdr_nsamples(hdr));+        idx = 0;+        for (i=0; i<hdr->nsamples_ori; i++)+            if ( bit_array_test(hdr->keep_samples,i) ) samples[idx++] = strdup(hdr->samples[i]);+        free(hdr->samples);+        hdr->samples = samples;++        // delete original samples from the dictionary+        vdict_t *d = (vdict_t*)hdr->dict[BCF_DT_SAMPLE];+        int k;+        for (k = kh_begin(d); k != kh_end(d); ++k)+            if (kh_exist(d, k)) free((char*)kh_key(d, k));+        kh_destroy(vdict, d);++        // add the subset back+        hdr->dict[BCF_DT_SAMPLE] = d = kh_init(vdict);+        for (i=0; i<bcf_hdr_nsamples(hdr); i++)+        {+            int ignore, k = kh_put(vdict, d, hdr->samples[i], &ignore);+            kh_val(d, k) = bcf_idinfo_def;+            kh_val(d, k).id = kh_size(d) - 1;+        }+        bcf_hdr_sync(hdr);+    }++    return ret;+}++int bcf_subset(const bcf_hdr_t *h, bcf1_t *v, int n, int *imap)+{+    kstring_t ind;+    ind.s = 0; ind.l = ind.m = 0;+    if (n) {+        bcf_fmt_t *fmt;+        int i, j;+        fmt = (bcf_fmt_t*)alloca(v->n_fmt * sizeof(bcf_fmt_t));+        uint8_t *ptr = (uint8_t*)v->indiv.s;+        for (i = 0; i < v->n_fmt; ++i)+            ptr = bcf_unpack_fmt_core1(ptr, v->n_sample, &fmt[i]);+        for (i = 0; i < (int)v->n_fmt; ++i) {+            bcf_fmt_t *f = &fmt[i];+            bcf_enc_int1(&ind, f->id);+            bcf_enc_size(&ind, f->n, f->type);+            for (j = 0; j < n; ++j)+                if (imap[j] >= 0) kputsn((char*)(f->p + imap[j] * f->size), f->size, &ind);+        }+        for (i = j = 0; j < n; ++j) if (imap[j] >= 0) ++i;+        v->n_sample = i;+    } else v->n_sample = 0;+    if ( !v->n_sample ) v->n_fmt = 0;+    free(v->indiv.s);+    v->indiv = ind;+    v->unpacked &= ~BCF_UN_FMT;    // only BCF is ready for output, VCF will need to unpack again+    return 0;+}++int bcf_is_snp(bcf1_t *v)+{+    int i;+    bcf_unpack(v, BCF_UN_STR);+    for (i = 0; i < v->n_allele; ++i)+    {+        if ( v->d.allele[i][1]==0 ) continue;++        // mpileup's <X> allele, see also below. This is not completely satisfactory,+        // a general library is here narrowly tailored to fit samtools.+        if ( v->d.allele[i][0]=='<' && v->d.allele[i][1]=='X' && v->d.allele[i][2]=='>' ) continue;+        if ( v->d.allele[i][0]=='<' && v->d.allele[i][1]=='*' && v->d.allele[i][2]=='>' ) continue;++        break;+    }+    return i == v->n_allele;+}++static void bcf_set_variant_type(const char *ref, const char *alt, variant_t *var)+{+    // The most frequent case+    if ( !ref[1] && !alt[1] )+    {+        if ( *alt == '.' || *ref==*alt ) { var->n = 0; var->type = VCF_REF; return; }+        if ( *alt == 'X' ) { var->n = 0; var->type = VCF_REF; return; }  // mpileup's X allele shouldn't be treated as variant+        if ( *alt == '*' ) { var->n = 0; var->type = VCF_REF; return; } +        var->n = 1; var->type = VCF_SNP; return;+    }+    if ( alt[0]=='<' )+    {+        if ( alt[1]=='X' && alt[2]=='>' ) { var->n = 0; var->type = VCF_REF; return; }  // mpileup's X allele shouldn't be treated as variant+        if ( alt[1]=='*' && alt[2]=='>' ) { var->n = 0; var->type = VCF_REF; return; }+        var->type = VCF_OTHER;+        return;+    }++    const char *r = ref, *a = alt;+    while (*r && *a && toupper(*r)==toupper(*a) ) { r++; a++; }     // unfortunately, matching REF,ALT case is not guaranteed++    if ( *a && !*r )+    {+        while ( *a ) a++;+        var->n = (a-alt)-(r-ref); var->type = VCF_INDEL; return;+    }+    else if ( *r && !*a )+    {+        while ( *r ) r++;+        var->n = (a-alt)-(r-ref); var->type = VCF_INDEL; return;+    }+    else if ( !*r && !*a )+    {+        var->n = 0; var->type = VCF_REF; return;+    }++    const char *re = r, *ae = a;+    while ( re[1] ) re++;+    while ( ae[1] ) ae++;+    while ( re>r && ae>a && toupper(*re)==toupper(*ae) ) { re--; ae--; }+    if ( ae==a )+    {+        if ( re==r ) { var->n = 1; var->type = VCF_SNP; return; }+        var->n = -(re-r);+        if ( toupper(*re)==toupper(*ae) ) { var->type = VCF_INDEL; return; }+        var->type = VCF_OTHER; return;+    }+    else if ( re==r )+    {+        var->n = ae-a;+        if ( toupper(*re)==toupper(*ae) ) { var->type = VCF_INDEL; return; }+        var->type = VCF_OTHER; return;+    }++    var->type = ( re-r == ae-a ) ? VCF_MNP : VCF_OTHER;+    var->n = ( re-r > ae-a ) ? -(re-r+1) : ae-a+1;++    // should do also complex events, SVs, etc...+}++static void bcf_set_variant_types(bcf1_t *b)+{+    if ( !(b->unpacked & BCF_UN_STR) ) bcf_unpack(b, BCF_UN_STR);+    bcf_dec_t *d = &b->d;+    if ( d->n_var < b->n_allele )+    {+        d->var = (variant_t *) realloc(d->var, sizeof(variant_t)*b->n_allele);+        d->n_var = b->n_allele;+    }+    int i;+    b->d.var_type = 0;+    for (i=1; i<b->n_allele; i++)+    {+        bcf_set_variant_type(d->allele[0],d->allele[i], &d->var[i]);+        b->d.var_type |= d->var[i].type;+        //fprintf(stderr,"[set_variant_type] %d   %s %s -> %d %d .. %d\n", b->pos+1,d->allele[0],d->allele[i],d->var[i].type,d->var[i].n, b->d.var_type);+    }+}++int bcf_get_variant_types(bcf1_t *rec)+{+    if ( rec->d.var_type==-1 ) bcf_set_variant_types(rec);+    return rec->d.var_type;+}+int bcf_get_variant_type(bcf1_t *rec, int ith_allele)+{+    if ( rec->d.var_type==-1 ) bcf_set_variant_types(rec);+    return rec->d.var[ith_allele].type;+}++int bcf_update_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type)+{+    // Is the field already present?+    int i, inf_id = bcf_hdr_id2int(hdr,BCF_DT_ID,key);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_INFO,inf_id) ) return -1;    // No such INFO field in the header+    if ( !(line->unpacked & BCF_UN_INFO) ) bcf_unpack(line, BCF_UN_INFO);++    for (i=0; i<line->n_info; i++)+        if ( inf_id==line->d.info[i].key ) break;+    bcf_info_t *inf = i==line->n_info ? NULL : &line->d.info[i];++    if ( !n || (type==BCF_HT_STR && !values) )+    {+        if ( n==0 && !strcmp("END",key) )+            line->rlen = line->n_allele ? strlen(line->d.allele[0]) : 0;+        if ( inf )+        {+            // Mark the tag for removal, free existing memory if necessary+            if ( inf->vptr_free )+            {+                free(inf->vptr - inf->vptr_off);+                inf->vptr_free = 0;+            }+            line->d.shared_dirty |= BCF1_DIRTY_INF;+            inf->vptr = NULL;+        }+        return 0;+    }++    // Encode the values and determine the size required to accommodate the values+    kstring_t str = {0,0,0};+    bcf_enc_int1(&str, inf_id);+    if ( type==BCF_HT_INT )+        bcf_enc_vint(&str, n, (int32_t*)values, -1);+    else if ( type==BCF_HT_REAL )+        bcf_enc_vfloat(&str, n, (float*)values);+    else if ( type==BCF_HT_FLAG || type==BCF_HT_STR )+    {+        if ( values==NULL )+            bcf_enc_size(&str, 0, BCF_BT_NULL);+        else+            bcf_enc_vchar(&str, strlen((char*)values), (char*)values);+    }+    else+    {+        fprintf(stderr, "[E::%s] the type %d not implemented yet\n", __func__, type);+        abort();+    }++    // Is the INFO tag already present+    if ( inf )+    {+        // Is it big enough to accommodate new block?+        if ( str.l <= inf->vptr_len + inf->vptr_off )+        {+            if ( str.l != inf->vptr_len + inf->vptr_off ) line->d.shared_dirty |= BCF1_DIRTY_INF;+            uint8_t *ptr = inf->vptr - inf->vptr_off;+            memcpy(ptr, str.s, str.l);+            free(str.s);+            int vptr_free = inf->vptr_free;+            bcf_unpack_info_core1(ptr, inf);+            inf->vptr_free = vptr_free;+        }+        else+        {+            assert( !inf->vptr_free );  // fix the caller or improve here: this has been modified before+            bcf_unpack_info_core1((uint8_t*)str.s, inf);+            inf->vptr_free = 1;+            line->d.shared_dirty |= BCF1_DIRTY_INF;+        }+    }+    else+    {+        // The tag is not present, create new one+        line->n_info++;+        hts_expand0(bcf_info_t, line->n_info, line->d.m_info , line->d.info);+        inf = &line->d.info[line->n_info-1];+        bcf_unpack_info_core1((uint8_t*)str.s, inf);+        inf->vptr_free = 1;+        line->d.shared_dirty |= BCF1_DIRTY_INF;+    }+    line->unpacked |= BCF_UN_INFO;++    if ( n==1 && !strcmp("END",key) ) line->rlen = ((int32_t*)values)[0] - line->pos;+    return 0;+}++int bcf_update_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const char **values, int n)+{+    if ( !n )+        return bcf_update_format(hdr,line,key,NULL,0,BCF_HT_STR);++    int i, max_len = 0;+    for (i=0; i<n; i++)+    {+        int len = strlen(values[i]);+        if ( len > max_len ) max_len = len;+    }+    char *out = (char*) malloc(max_len*n);+    if ( !out ) return -2;+    for (i=0; i<n; i++)+    {+        char *dst = out+i*max_len;+        const char *src = values[i];+        int j = 0;+        while ( src[j] ) { dst[j] = src[j]; j++; }+        for (; j<max_len; j++) dst[j] = 0;+    }+    int ret = bcf_update_format(hdr,line,key,out,max_len*n,BCF_HT_STR);+    free(out);+    return ret;+}++int bcf_update_format(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type)+{+    // Is the field already present?+    int i, fmt_id = bcf_hdr_id2int(hdr,BCF_DT_ID,key);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,fmt_id) )+    {+        if ( !n ) return 0;+        return -1;  // the key not present in the header+    }++    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);++    for (i=0; i<line->n_fmt; i++)+        if ( line->d.fmt[i].id==fmt_id ) break;+    bcf_fmt_t *fmt = i==line->n_fmt ? NULL : &line->d.fmt[i];++    if ( !n )+    {+        if ( fmt )+        {+            // Mark the tag for removal, free existing memory if necessary+            if ( fmt->p_free )+            {+                free(fmt->p - fmt->p_off);+                fmt->p_free = 0;+            }+            line->d.indiv_dirty = 1;+            fmt->p = NULL;+        }+        return 0;+    }++    line->n_sample = bcf_hdr_nsamples(hdr);+    int nps = n / line->n_sample;  // number of values per sample+    assert( nps && nps*line->n_sample==n );     // must be divisible by n_sample++    // Encode the values and determine the size required to accommodate the values+    kstring_t str = {0,0,0};+    bcf_enc_int1(&str, fmt_id);+    if ( type==BCF_HT_INT )+        bcf_enc_vint(&str, n, (int32_t*)values, nps);+    else if ( type==BCF_HT_REAL )+    {+        bcf_enc_size(&str, nps, BCF_BT_FLOAT);+        kputsn((char*)values, nps*line->n_sample*sizeof(float), &str);+    }+    else if ( type==BCF_HT_STR )+    {+        bcf_enc_size(&str, nps, BCF_BT_CHAR);+        kputsn((char*)values, nps*line->n_sample, &str);+    }+    else+    {+        fprintf(stderr, "[E::%s] the type %d not implemented yet\n", __func__, type);+        abort();+    }++    if ( !fmt )+    {+        // Not present, new format field+        line->n_fmt++;+        hts_expand0(bcf_fmt_t, line->n_fmt, line->d.m_fmt, line->d.fmt);++        // Special case: VCF specification requires that GT is always first+        if ( line->n_fmt > 1 && key[0]=='G' && key[1]=='T' && !key[2] )+        {+            for (i=line->n_fmt-1; i>0; i--)+                line->d.fmt[i] = line->d.fmt[i-1];+            fmt = &line->d.fmt[0];+        }+        else+            fmt = &line->d.fmt[line->n_fmt-1];+        bcf_unpack_fmt_core1((uint8_t*)str.s, line->n_sample, fmt);+        line->d.indiv_dirty = 1;+        fmt->p_free = 1;+    }+    else+    {+        // The tag is already present, check if it is big enough to accomodate the new block+        if ( str.l <= fmt->p_len + fmt->p_off )+        {+            // good, the block is big enough+            if ( str.l != fmt->p_len + fmt->p_off ) line->d.indiv_dirty = 1;+            uint8_t *ptr = fmt->p - fmt->p_off;+            memcpy(ptr, str.s, str.l);+            free(str.s);+            int p_free = fmt->p_free;+            bcf_unpack_fmt_core1(ptr, line->n_sample, fmt);+            fmt->p_free = p_free;+        }+        else+        {+            assert( !fmt->p_free );  // fix the caller or improve here: this has been modified before+            bcf_unpack_fmt_core1((uint8_t*)str.s, line->n_sample, fmt);+            fmt->p_free = 1;+            line->d.indiv_dirty = 1;+        }+    }+    line->unpacked |= BCF_UN_FMT;+    return 0;+}+++int bcf_update_filter(const bcf_hdr_t *hdr, bcf1_t *line, int *flt_ids, int n)+{+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);+    line->d.shared_dirty |= BCF1_DIRTY_FLT;+    line->d.n_flt = n;+    if ( !n ) return 0;+    hts_expand(int, line->d.n_flt, line->d.m_flt, line->d.flt);+    int i;+    for (i=0; i<n; i++)+        line->d.flt[i] = flt_ids[i];+    return 0;+}++int bcf_add_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id)+{+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);+    int i;+    for (i=0; i<line->d.n_flt; i++)+        if ( flt_id==line->d.flt[i] ) break;+    if ( i<line->d.n_flt ) return 0;    // this filter is already set+    line->d.shared_dirty |= BCF1_DIRTY_FLT;+    if ( flt_id==0 )    // set to PASS+        line->d.n_flt = 1;+    else if ( line->d.n_flt==1 && line->d.flt[0]==0 )+        line->d.n_flt = 1;+    else+        line->d.n_flt++;+    hts_expand(int, line->d.n_flt, line->d.m_flt, line->d.flt);+    line->d.flt[line->d.n_flt-1] = flt_id;+    return 1;+}+int bcf_remove_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id, int pass)+{+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);+    int i;+    for (i=0; i<line->d.n_flt; i++)+        if ( flt_id==line->d.flt[i] ) break;+    if ( i==line->d.n_flt ) return 0;   // the filter is not present+    line->d.shared_dirty |= BCF1_DIRTY_FLT;+    if ( i!=line->d.n_flt-1 ) memmove(line->d.flt+i,line->d.flt+i+1,(line->d.n_flt-i-1)*sizeof(*line->d.flt));+    line->d.n_flt--;+    if ( !line->d.n_flt && pass ) bcf_add_filter(hdr,line,0);+    return 0;+}++int bcf_has_filter(const bcf_hdr_t *hdr, bcf1_t *line, char *filter)+{+    if ( filter[0]=='.' && !filter[1] ) filter = "PASS";+    int id = bcf_hdr_id2int(hdr, BCF_DT_ID, filter);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FLT,id) ) return -1;  // not defined in the header++    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);+    if ( id==0 && !line->d.n_flt) return 1; // PASS++    int i;+    for (i=0; i<line->d.n_flt; i++)+        if ( line->d.flt[i]==id ) return 1;+    return 0;+}++static inline int _bcf1_sync_alleles(const bcf_hdr_t *hdr, bcf1_t *line, int nals)+{+    line->d.shared_dirty |= BCF1_DIRTY_ALS;++    line->n_allele = nals;+    hts_expand(char*, line->n_allele, line->d.m_allele, line->d.allele);++    char *als = line->d.als;+    int n = 0;+    while (n<nals)+    {+        line->d.allele[n] = als;+        while ( *als ) als++;+        als++;+        n++;+    }++    // Update REF length+    bcf_info_t *end_info = bcf_get_info(hdr,line,"END");+    line->rlen = end_info ? end_info->v1.i : strlen(line->d.allele[0]);++    return 0;+}+int bcf_update_alleles(const bcf_hdr_t *hdr, bcf1_t *line, const char **alleles, int nals)+{+    kstring_t tmp = {0,0,0};+    char *free_old = NULL;++    // If the supplied alleles are not pointers to line->d.als, the existing block can be reused.+    int i;+    for (i=0; i<nals; i++)+        if ( alleles[i]>=line->d.als && alleles[i]<line->d.als+line->d.m_als ) break;+    if ( i==nals )+    {+        // all alleles point elsewhere, reuse the existing block+        tmp.l = 0; tmp.s = line->d.als; tmp.m = line->d.m_als;+    }+    else+        free_old = line->d.als;++    for (i=0; i<nals; i++)+    {+        kputs(alleles[i], &tmp);+        kputc(0, &tmp);+    }+    line->d.als = tmp.s; line->d.m_als = tmp.m;+    free(free_old);+    return _bcf1_sync_alleles(hdr,line,nals);+}++int bcf_update_alleles_str(const bcf_hdr_t *hdr, bcf1_t *line, const char *alleles_string)+{+    kstring_t tmp;+    tmp.l = 0; tmp.s = line->d.als; tmp.m = line->d.m_als;+    kputs(alleles_string, &tmp);+    line->d.als = tmp.s; line->d.m_als = tmp.m;++    int nals = 1;+    char *t = line->d.als;+    while (*t)+    {+        if ( *t==',' ) { *t = 0; nals++; }+        t++;+    }+    return _bcf1_sync_alleles(hdr, line, nals);+}++int bcf_update_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id)+{+    kstring_t tmp;+    tmp.l = 0; tmp.s = line->d.id; tmp.m = line->d.m_id;+    if ( id )+        kputs(id, &tmp);+    else+        kputs(".", &tmp);+    line->d.id = tmp.s; line->d.m_id = tmp.m;+    line->d.shared_dirty |= BCF1_DIRTY_ID;+    return 0;+}++int bcf_add_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id)+{+    if ( !id ) return 0;++    kstring_t tmp;+    tmp.l = 0; tmp.s = line->d.id; tmp.m = line->d.m_id;++    int len = strlen(id);+    char *dst = line->d.id;+    while ( *dst && (dst=strstr(dst,id)) )+    {+        if ( dst[len]!=0 && dst[len]!=';' ) dst++;              // a prefix, not a match+        else if ( dst==line->d.id || dst[-1]==';' ) return 0;   // already present+        dst++;  // a suffix, not a match+    }+    if ( line->d.id && (line->d.id[0]!='.' || line->d.id[1]) )+    {+        tmp.l = strlen(line->d.id);+        kputc(';',&tmp);+    }+    kputs(id,&tmp);++    line->d.id = tmp.s; line->d.m_id = tmp.m;+    line->d.shared_dirty |= BCF1_DIRTY_ID;+    return 0;++}++bcf_fmt_t *bcf_get_fmt(const bcf_hdr_t *hdr, bcf1_t *line, const char *key)+{+    int id = bcf_hdr_id2int(hdr, BCF_DT_ID, key);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,id) ) return NULL;   // no such FMT field in the header+    return bcf_get_fmt_id(line, id);+}++bcf_info_t *bcf_get_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key)+{+    int id = bcf_hdr_id2int(hdr, BCF_DT_ID, key);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_INFO,id) ) return NULL;   // no such INFO field in the header+    return bcf_get_info_id(line, id);+}++bcf_fmt_t *bcf_get_fmt_id(bcf1_t *line, const int id) +{+    int i;+    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);+    for (i=0; i<line->n_fmt; i++)+    {+        if ( line->d.fmt[i].id==id ) return &line->d.fmt[i];+    }+    return NULL;+}++bcf_info_t *bcf_get_info_id(bcf1_t *line, const int id) +{+    int i;+    if ( !(line->unpacked & BCF_UN_INFO) ) bcf_unpack(line, BCF_UN_INFO);+    for (i=0; i<line->n_info; i++)+    {+        if ( line->d.info[i].key==id ) return &line->d.info[i];+    }+    return NULL;+}+++int bcf_get_info_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type)+{+    int i,j, tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, tag);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_INFO,tag_id) ) return -1;    // no such INFO field in the header+    if ( bcf_hdr_id2type(hdr,BCF_HL_INFO,tag_id)!=type ) return -2;     // expected different type++    if ( !(line->unpacked & BCF_UN_INFO) ) bcf_unpack(line, BCF_UN_INFO);++    for (i=0; i<line->n_info; i++)+        if ( line->d.info[i].key==tag_id ) break;+    if ( i==line->n_info ) return ( type==BCF_HT_FLAG ) ? 0 : -3;       // the tag is not present in this record+    if ( type==BCF_HT_FLAG ) return 1;++    bcf_info_t *info = &line->d.info[i];+    if ( !info->vptr ) return -3;           // the tag was marked for removal+    if ( type==BCF_HT_STR )+    {+        if ( *ndst < info->len+1 )+        {+            *ndst = info->len + 1;+            *dst  = realloc(*dst, *ndst);+        }+        memcpy(*dst,info->vptr,info->len);+        ((uint8_t*)*dst)[info->len] = 0;+        return info->len;+    }++    // Make sure the buffer is big enough+    int size1 = type==BCF_HT_INT ? sizeof(int32_t) : sizeof(float);+    if ( *ndst < info->len )+    {+        *ndst = info->len;+        *dst  = realloc(*dst, *ndst * size1);+    }++    if ( info->len == 1 )+    {+        if ( info->type==BCF_BT_FLOAT ) *((float*)*dst) = info->v1.f;+        else +        {+            #define BRANCH(type_t, missing) { \+                if ( info->v1.i==missing ) *((int32_t*)*dst) = bcf_int32_missing; \+                else *((int32_t*)*dst) = info->v1.i; \+            }+            switch (info->type)+            {+                case BCF_BT_INT8:  BRANCH(int8_t,  bcf_int8_missing ); break;+                case BCF_BT_INT16: BRANCH(int16_t, bcf_int16_missing); break;+                case BCF_BT_INT32: BRANCH(int32_t, bcf_int32_missing); break;+            }+            #undef BRANCH+        }+        return 1;+    }++    #define BRANCH(type_t, is_missing, is_vector_end, set_missing, out_type_t) { \+        out_type_t *tmp = (out_type_t *) *dst; \+        type_t *p = (type_t *) info->vptr; \+        for (j=0; j<info->len; j++) \+        { \+            if ( is_vector_end ) return j; \+            if ( is_missing ) set_missing; \+            else *tmp = p[j]; \+            tmp++; \+        } \+        return j; \+    }+    switch (info->type) {+        case BCF_BT_INT8:  BRANCH(int8_t,  p[j]==bcf_int8_missing,  p[j]==bcf_int8_vector_end,  *tmp=bcf_int32_missing, int32_t); break;+        case BCF_BT_INT16: BRANCH(int16_t, p[j]==bcf_int16_missing, p[j]==bcf_int16_vector_end, *tmp=bcf_int32_missing, int32_t); break;+        case BCF_BT_INT32: BRANCH(int32_t, p[j]==bcf_int32_missing, p[j]==bcf_int32_vector_end, *tmp=bcf_int32_missing, int32_t); break;+        case BCF_BT_FLOAT: BRANCH(float,   bcf_float_is_missing(p[j]), bcf_float_is_vector_end(p[j]), bcf_float_set_missing(*tmp), float); break;+        default: fprintf(stderr,"TODO: %s:%d .. info->type=%d\n", __FILE__,__LINE__, info->type); exit(1);+    }+    #undef BRANCH+    return -4;  // this can never happen+}++int bcf_get_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, char ***dst, int *ndst)+{+    int i,tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, tag);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,tag_id) ) return -1;    // no such FORMAT field in the header+    if ( bcf_hdr_id2type(hdr,BCF_HL_FMT,tag_id)!=BCF_HT_STR ) return -2;     // expected different type++    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);++    for (i=0; i<line->n_fmt; i++)+        if ( line->d.fmt[i].id==tag_id ) break;+    if ( i==line->n_fmt ) return -3;                               // the tag is not present in this record+    bcf_fmt_t *fmt = &line->d.fmt[i];+    if ( !fmt->p ) return -3;                                      // the tag was marked for removal++    int nsmpl = bcf_hdr_nsamples(hdr);+    if ( !*dst )+    {+        *dst = (char**) malloc(sizeof(char*)*nsmpl);+        if ( !*dst ) return -4;     // could not alloc+        (*dst)[0] = NULL;+    }+    int n = (fmt->n+1)*nsmpl;+    if ( *ndst < n )+    {+        (*dst)[0] = realloc((*dst)[0], n);+        if ( !(*dst)[0] ) return -4;    // could not alloc+        *ndst = n;+    }+    for (i=0; i<nsmpl; i++)+    {+        uint8_t *src = fmt->p + i*fmt->n;+        uint8_t *tmp = (uint8_t*)(*dst)[0] + i*(fmt->n+1);+        memcpy(tmp,src,fmt->n);+        tmp[fmt->n] = 0;+        (*dst)[i] = (char*) tmp;+    }+    return n;+}++int bcf_get_format_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type)+{+    int i,j, tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, tag);+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,tag_id) ) return -1;    // no such FORMAT field in the header+    if ( tag[0]=='G' && tag[1]=='T' && tag[2]==0 )+    {+        // Ugly: GT field is considered to be a string by the VCF header but BCF represents it as INT.+        if ( bcf_hdr_id2type(hdr,BCF_HL_FMT,tag_id)!=BCF_HT_STR ) return -2;+    }+    else if ( bcf_hdr_id2type(hdr,BCF_HL_FMT,tag_id)!=type ) return -2;     // expected different type++    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);++    for (i=0; i<line->n_fmt; i++)+        if ( line->d.fmt[i].id==tag_id ) break;+    if ( i==line->n_fmt ) return -3;                               // the tag is not present in this record+    bcf_fmt_t *fmt = &line->d.fmt[i];+    if ( !fmt->p ) return -3;                                      // the tag was marked for removal++    if ( type==BCF_HT_STR )+    {+        int n = fmt->n*bcf_hdr_nsamples(hdr);+        if ( *ndst < n )+        {+            *dst  = realloc(*dst, n);+            if ( !*dst ) return -4;     // could not alloc+            *ndst = n;+        }+        memcpy(*dst,fmt->p,n);+        return n;+    }++    // Make sure the buffer is big enough+    int nsmpl = bcf_hdr_nsamples(hdr);+    int size1 = type==BCF_HT_INT ? sizeof(int32_t) : sizeof(float);+    if ( *ndst < fmt->n*nsmpl )+    {+        *ndst = fmt->n*nsmpl;+        *dst  = realloc(*dst, *ndst*size1);+        if ( !dst ) return -4;     // could not alloc+    }++    #define BRANCH(type_t, is_missing, is_vector_end, set_missing, set_vector_end, out_type_t) { \+        out_type_t *tmp = (out_type_t *) *dst; \+        type_t *p = (type_t*) fmt->p; \+        for (i=0; i<nsmpl; i++) \+        { \+            for (j=0; j<fmt->n; j++) \+            { \+                if ( is_missing ) set_missing; \+                else if ( is_vector_end ) { set_vector_end; break; } \+                else *tmp = p[j]; \+                tmp++; \+            } \+            for (; j<fmt->n; j++) { set_vector_end; tmp++; } \+            p = (type_t *)((char *)p + fmt->size); \+        } \+    }+    switch (fmt->type) {+        case BCF_BT_INT8:  BRANCH(int8_t,  p[j]==bcf_int8_missing,  p[j]==bcf_int8_vector_end,  *tmp=bcf_int32_missing, *tmp=bcf_int32_vector_end, int32_t); break;+        case BCF_BT_INT16: BRANCH(int16_t, p[j]==bcf_int16_missing, p[j]==bcf_int16_vector_end, *tmp=bcf_int32_missing, *tmp=bcf_int32_vector_end, int32_t); break;+        case BCF_BT_INT32: BRANCH(int32_t, p[j]==bcf_int32_missing, p[j]==bcf_int32_vector_end, *tmp=bcf_int32_missing, *tmp=bcf_int32_vector_end, int32_t); break;+        case BCF_BT_FLOAT: BRANCH(float,   bcf_float_is_missing(p[j]), bcf_float_is_vector_end(p[j]), bcf_float_set_missing(*tmp), bcf_float_set_vector_end(*tmp), float); break;+        default: fprintf(stderr,"TODO: %s:%d .. fmt->type=%d\n", __FILE__,__LINE__, fmt->type); exit(1);+    }+    #undef BRANCH+    return nsmpl*fmt->n;+}+
+ htslib-1.3.2/vcf_sweep.c view
@@ -0,0 +1,183 @@+/*  vcf_sweep.c -- forward/reverse sweep API.++    Copyright (C) 2013 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include "htslib/vcf_sweep.h"+#include "htslib/bgzf.h"++#define SW_FWD 0+#define SW_BWD 1++struct _bcf_sweep_t+{+    htsFile *file;+    bcf_hdr_t *hdr;+    BGZF *fp;++    int direction;          // to tell if the direction has changed+    int block_size;         // the size of uncompressed data to hold in memory+    bcf1_t *rec;            // bcf buffer+    int nrec, mrec;         // number of used records; total size of the buffer+    int lrid, lpos, lnals, lals_len, mlals;   // to check uniqueness of a record+    char *lals;++    uint64_t *idx;          // uncompressed offsets of VCF/BCF records+    int iidx, nidx, midx;   // i: current offset; n: used; m: allocated+    int idx_done;           // the index is built during the first pass+};++BGZF *hts_get_bgzfp(htsFile *fp);+int hts_useek(htsFile *file, long uoffset, int where);+long hts_utell(htsFile *file);++static inline int sw_rec_equal(bcf_sweep_t *sw, bcf1_t *rec)+{+    if ( sw->lrid!=rec->rid ) return 0;+    if ( sw->lpos!=rec->pos ) return 0;+    if ( sw->lnals!=rec->n_allele ) return 0;++    char *t = rec->d.allele[sw->lnals-1];+    int len = t - rec->d.allele[0] + 1;+    while ( *t ) { t++; len++; }+    if ( sw->lals_len!=len ) return 0;+    if ( memcmp(sw->lals,rec->d.allele[0],len) ) return 0;+    return 1;+}++static void sw_rec_save(bcf_sweep_t *sw, bcf1_t *rec)+{+    sw->lrid  = rec->rid;+    sw->lpos  = rec->pos;+    sw->lnals = rec->n_allele;++    char *t = rec->d.allele[sw->lnals-1];+    int len = t - rec->d.allele[0] + 1;+    while ( *t ) { t++; len++; }+    sw->lals_len = len;+    hts_expand(char, len, sw->mlals, sw->lals);+    memcpy(sw->lals, rec->d.allele[0], len);+}++static void sw_fill_buffer(bcf_sweep_t *sw)+{+    if ( !sw->iidx ) return;+    sw->iidx--;++    int ret = hts_useek(sw->file, sw->idx[sw->iidx], 0);+    assert( ret==0 );++    sw->nrec = 0;+    bcf1_t *rec = &sw->rec[sw->nrec];+    while ( (ret=bcf_read1(sw->file, sw->hdr, rec))==0 )+    {+        bcf_unpack(rec, BCF_UN_STR);++        // if not in the last block, stop at the saved record+        if ( sw->iidx+1 < sw->nidx && sw_rec_equal(sw,rec) ) break;++        sw->nrec++;+        hts_expand0(bcf1_t, sw->nrec+1, sw->mrec, sw->rec);+        rec = &sw->rec[sw->nrec];+    }+    sw_rec_save(sw, &sw->rec[0]);+}++bcf_sweep_t *bcf_sweep_init(const char *fname)+{+    bcf_sweep_t *sw = (bcf_sweep_t*) calloc(1,sizeof(bcf_sweep_t));+    sw->file = hts_open(fname, "r");+    sw->fp   = hts_get_bgzfp(sw->file);+    bgzf_index_build_init(sw->fp);+    sw->hdr  = bcf_hdr_read(sw->file);+    sw->mrec = 1;+    sw->rec  = (bcf1_t*) calloc(sw->mrec,(sizeof(bcf1_t)));+    sw->block_size = 1024*1024*3;+    sw->direction = SW_FWD;+    return sw;+}++void bcf_sweep_destroy(bcf_sweep_t *sw)+{+    int i;+    for (i=0; i<sw->mrec; i++) bcf_empty1(&sw->rec[i]);+    free(sw->idx);+    free(sw->rec);+    free(sw->lals);+    bcf_hdr_destroy(sw->hdr);+    hts_close(sw->file);+    free(sw);+}++static void sw_seek(bcf_sweep_t *sw, int direction)+{+    sw->direction = direction;+    if ( direction==SW_FWD )+        hts_useek(sw->file, sw->idx[0], 0);+    else+    {+        sw->iidx = sw->nidx;+        sw->nrec = 0;+    }+}++bcf1_t *bcf_sweep_fwd(bcf_sweep_t *sw)+{+    if ( sw->direction==SW_BWD ) sw_seek(sw, SW_FWD);++    long pos = hts_utell(sw->file);++    bcf1_t *rec = &sw->rec[0];+    int ret = bcf_read1(sw->file, sw->hdr, rec);++    if ( ret!=0 )   // last record, get ready for sweeping backwards+    {+        sw->idx_done = 1;+        sw->fp->idx_build_otf = 0;+        sw_seek(sw, SW_BWD);+        return NULL;+    }++    if ( !sw->idx_done )+    {+        if ( !sw->nidx || pos - sw->idx[sw->nidx-1] > sw->block_size )+        {+            sw->nidx++;+            hts_expand(uint64_t, sw->nidx, sw->midx, sw->idx);+            sw->idx[sw->nidx-1] = pos;+        }+    }+    return rec;+}++bcf1_t *bcf_sweep_bwd(bcf_sweep_t *sw)+{+    if ( sw->direction==SW_FWD ) sw_seek(sw, SW_BWD);+    if ( !sw->nrec ) sw_fill_buffer(sw);+    if ( !sw->nrec ) return NULL;+    return &sw->rec[ --sw->nrec ];+}++bcf_hdr_t *bcf_sweep_hdr(bcf_sweep_t *sw) { return sw->hdr; }+
+ htslib-1.3.2/vcfutils.c view
@@ -0,0 +1,691 @@+/*  vcfutils.c -- allele-related utility functions.++    Copyright (C) 2012-2015 Genome Research Ltd.++    Author: Petr Danecek <pd3@sanger.ac.uk>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.  */++#include <config.h>++#include "htslib/vcfutils.h"+#include "htslib/kbitset.h"++int bcf_calc_ac(const bcf_hdr_t *header, bcf1_t *line, int *ac, int which)+{+    int i;+    for (i=0; i<line->n_allele; i++) ac[i]=0;++    // Use INFO/AC,AN field only when asked+    if ( which&BCF_UN_INFO )+    {+        bcf_unpack(line, BCF_UN_INFO);+        int an_id = bcf_hdr_id2int(header, BCF_DT_ID, "AN");+        int ac_id = bcf_hdr_id2int(header, BCF_DT_ID, "AC");+        int i, an=-1, ac_len=0, ac_type=0;+        uint8_t *ac_ptr=NULL;+        if ( an_id>=0 && ac_id>=0 )+        {+            for (i=0; i<line->n_info; i++)+            {+                bcf_info_t *z = &line->d.info[i];+                if ( z->key == an_id ) an = z->v1.i;+                else if ( z->key == ac_id ) { ac_ptr = z->vptr; ac_len = z->len; ac_type = z->type; }+            }+        }+        if ( an>=0 && ac_ptr )+        {+            int nac = 0;+            #define BRANCH_INT(type_t) {        \+                type_t *p = (type_t *) ac_ptr;  \+                for (i=0; i<ac_len; i++)        \+                {                               \+                    ac[i+1] = p[i];             \+                    nac += p[i];                \+                }                               \+            }+            switch (ac_type) {+                case BCF_BT_INT8:  BRANCH_INT(int8_t); break;+                case BCF_BT_INT16: BRANCH_INT(int16_t); break;+                case BCF_BT_INT32: BRANCH_INT(int32_t); break;+                default: fprintf(stderr, "[E::%s] todo: %d at %s:%d\n", __func__, ac_type, header->id[BCF_DT_CTG][line->rid].key, line->pos+1); exit(1); break;+            }+            #undef BRANCH_INT+            if ( an<nac )+            {+                fprintf(stderr,"[E::%s] Incorrect AN/AC counts at %s:%d\n", __func__,header->id[BCF_DT_CTG][line->rid].key, line->pos+1); +                exit(1); +            }+            ac[0] = an - nac;+            return 1;+        }+    }++    // Split genotype fields only when asked+    if ( which&BCF_UN_FMT )+    {+        int i, gt_id = bcf_hdr_id2int(header,BCF_DT_ID,"GT");+        if ( gt_id<0 ) return 0;+        bcf_unpack(line, BCF_UN_FMT);+        bcf_fmt_t *fmt_gt = NULL;+        for (i=0; i<(int)line->n_fmt; i++)+            if ( line->d.fmt[i].id==gt_id ) { fmt_gt = &line->d.fmt[i]; break; }+        if ( !fmt_gt ) return 0;+        #define BRANCH_INT(type_t,vector_end) { \+            for (i=0; i<line->n_sample; i++) \+            { \+                type_t *p = (type_t*) (fmt_gt->p + i*fmt_gt->size); \+                int ial; \+                for (ial=0; ial<fmt_gt->n; ial++) \+                { \+                    if ( p[ial]==vector_end ) break; /* smaller ploidy */ \+                    if ( bcf_gt_is_missing(p[ial]) ) continue; /* missing allele */ \+                    if ( p[ial]>>1 > line->n_allele ) \+                    { \+                        fprintf(stderr,"[E::%s] Incorrect allele (\"%d\") in %s at %s:%d\n", __func__,(p[ial]>>1)-1, header->samples[i],header->id[BCF_DT_CTG][line->rid].key, line->pos+1); \+                        exit(1); \+                    } \+                    ac[(p[ial]>>1)-1]++; \+                } \+            } \+        }+        switch (fmt_gt->type) {+            case BCF_BT_INT8:  BRANCH_INT(int8_t,  bcf_int8_vector_end); break;+            case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;+            case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;+            default: fprintf(stderr, "[E::%s] todo: %d at %s:%d\n", __func__, fmt_gt->type, header->id[BCF_DT_CTG][line->rid].key, line->pos+1); exit(1); break;+        }+        #undef BRANCH_INT+        return 1;+    }+    return 0;+}++int bcf_gt_type(bcf_fmt_t *fmt_ptr, int isample, int *_ial, int *_jal)+{+    int i, nals = 0, has_ref = 0, has_alt = 0, ial = 0, jal = 0;+    #define BRANCH_INT(type_t,vector_end) { \+        type_t *p = (type_t*) (fmt_ptr->p + isample*fmt_ptr->size); \+        for (i=0; i<fmt_ptr->n; i++) \+        { \+            if ( p[i] == vector_end ) break; /* smaller ploidy */ \+            if ( bcf_gt_is_missing(p[i]) ) return GT_UNKN; /* missing allele */ \+            int tmp = p[i]>>1; \+            if ( tmp>1 ) \+            { \+                if ( !ial ) { ial = tmp; has_alt = 1; } \+                else if ( tmp!=ial ) \+                { \+                    if ( tmp<ial ) \+                    { \+                        jal = ial; \+                        ial = tmp; \+                    } \+                    else \+                    { \+                        jal = tmp; \+                    } \+                    has_alt = 2; \+                } \+            } \+            else has_ref = 1; \+            nals++; \+        } \+    }+    switch (fmt_ptr->type) {+        case BCF_BT_INT8:  BRANCH_INT(int8_t,  bcf_int8_vector_end); break;+        case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;+        case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;+        default: fprintf(stderr, "[E::%s] todo: fmt_type %d\n", __func__, fmt_ptr->type); exit(1); break;+    }+    #undef BRANCH_INT++    if ( _ial ) *_ial = ial>0 ? ial-1 : ial;+    if ( _jal ) *_jal = jal>0 ? jal-1 : jal;+    if ( !nals ) return GT_UNKN;+    if ( nals==1 )+        return has_ref ? GT_HAPL_R : GT_HAPL_A;+    if ( !has_ref )+        return has_alt==1 ? GT_HOM_AA : GT_HET_AA;+    if ( !has_alt )+        return GT_HOM_RR;+    return GT_HET_RA;+}++int bcf_trim_alleles(const bcf_hdr_t *header, bcf1_t *line)+{+    int i;+    bcf_fmt_t *gt = bcf_get_fmt(header, line, "GT");+    if ( !gt ) return 0;++    int *ac = (int*) calloc(line->n_allele,sizeof(int));++    // check if all alleles are populated+    #define BRANCH(type_t,vector_end) { \+        for (i=0; i<line->n_sample; i++) \+        { \+            type_t *p = (type_t*) (gt->p + i*gt->size); \+            int ial; \+            for (ial=0; ial<gt->n; ial++) \+            { \+                if ( p[ial]==vector_end ) break; /* smaller ploidy */ \+                if ( bcf_gt_is_missing(p[ial]) ) continue; /* missing allele */ \+                if ( (p[ial]>>1)-1 >= line->n_allele ) { free(ac); return -1; } \+                ac[(p[ial]>>1)-1]++; \+            } \+        } \+    }+    switch (gt->type) {+        case BCF_BT_INT8:  BRANCH(int8_t,  bcf_int8_vector_end); break;+        case BCF_BT_INT16: BRANCH(int16_t, bcf_int16_vector_end); break;+        case BCF_BT_INT32: BRANCH(int32_t, bcf_int32_vector_end); break;+        default: fprintf(stderr, "[E::%s] todo: %d at %s:%d\n", __func__, gt->type, header->id[BCF_DT_CTG][line->rid].key, line->pos+1); exit(1); break;+    }+    #undef BRANCH++    int nrm = 0;+    kbitset_t *rm_set = kbs_init(line->n_allele);+    for (i=1; i<line->n_allele; i++)+    {+        if ( !ac[i] ) { kbs_insert(rm_set, i); nrm++; }+    }+    free(ac);++    if ( nrm ) bcf_remove_allele_set(header, line, rm_set);+    kbs_destroy(rm_set);+    return nrm;+}++void bcf_remove_alleles(const bcf_hdr_t *header, bcf1_t *line, int rm_mask)+{+    int i;+    kbitset_t *rm_set = kbs_init(line->n_allele);+    for (i=1; i<line->n_allele; i++)+        if ( rm_mask & 1<<i ) kbs_insert(rm_set, i);++    bcf_remove_allele_set(header, line, rm_set);+    kbs_destroy(rm_set);+}++void bcf_remove_allele_set(const bcf_hdr_t *header, bcf1_t *line, const struct kbitset_t *rm_set)+{+    int *map = (int*) calloc(line->n_allele, sizeof(int));++    // create map of indexes from old to new ALT numbering and modify ALT+    kstring_t str = {0,0,0};+    kputs(line->d.allele[0], &str);++    int nrm = 0, i,j;  // i: ori alleles, j: new alleles+    for (i=1, j=1; i<line->n_allele; i++)+    {+        if ( kbs_exists(rm_set, i) )+        {+            // remove this allele+            line->d.allele[i] = NULL;+            nrm++;+            continue;+        }+        kputc(',', &str);+        kputs(line->d.allele[i], &str);+        map[i] = j;+        j++;+    }+    if ( !nrm ) { free(map); free(str.s); return; }++    int nR_ori = line->n_allele;+    int nR_new = line->n_allele-nrm;+    assert(nR_new > 0); // should not be able to remove reference allele+    int nA_ori = nR_ori-1;+    int nA_new = nR_new-1;++    int nG_ori = nR_ori*(nR_ori + 1)/2;+    int nG_new = nR_new*(nR_new + 1)/2;++    bcf_update_alleles_str(header, line, str.s);++    // remove from Number=G, Number=R and Number=A INFO fields.+    uint8_t *dat = NULL;+    int mdat = 0, ndat = 0, mdat_bytes = 0, nret;+    for (i=0; i<line->n_info; i++)+    {+        bcf_info_t *info = &line->d.info[i];+        int vlen = bcf_hdr_id2length(header,BCF_HL_INFO,info->key);++        if ( vlen!=BCF_VL_A && vlen!=BCF_VL_G && vlen!=BCF_VL_R ) continue; // no need to change++        int type = bcf_hdr_id2type(header,BCF_HL_INFO,info->key);+        if ( type==BCF_HT_FLAG ) continue;+        int size = 1;+        if ( type==BCF_HT_REAL || type==BCF_HT_INT ) size = 4;++        mdat = mdat_bytes / size;+        nret = bcf_get_info_values(header, line, bcf_hdr_int2id(header,BCF_DT_ID,info->key), (void**)&dat, &mdat, type);+        mdat_bytes = mdat * size;+        if ( nret<0 )+        {+            fprintf(stderr,"[%s:%d %s] Could not access INFO/%s at %s:%d [%d]\n", __FILE__,__LINE__,__FUNCTION__,+                bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname(header,line), line->pos+1, nret);+            exit(1);+        }+        if ( type==BCF_HT_STR )+        {+            str.l = 0;+            char *ss = (char*) dat, *se = (char*) dat;+            if ( vlen==BCF_VL_A || vlen==BCF_VL_R )+            {+                int nexp, inc = 0;+                if ( vlen==BCF_VL_A )+                {+                    nexp = nA_ori;+                    inc  = 1;+                }+                else+                    nexp = nR_ori;+                for (j=0; j<nexp; j++)+                {+                    if ( !*se ) break;+                    while ( *se && *se!=',' ) se++;+                    if ( kbs_exists(rm_set, j+inc) )+                    {+                        if ( *se ) se++;+                        ss = se;+                        continue;+                    }+                    if ( str.l ) kputc(',',&str);+                    kputsn(ss,se-ss,&str);+                    if ( *se ) se++;+                    ss = se;+                }+                assert( j==nexp );+            }+            else    // Number=G, assuming diploid genotype+            {+                int k = 0, n = 0;+                for (j=0; j<nR_ori; j++)+                {+                    for (k=0; k<=j; k++)+                    {+                        if ( !*se ) break;+                        while ( *se && *se!=',' ) se++;+                        n++;+                        if ( kbs_exists(rm_set, j) || kbs_exists(rm_set, k) )+                        {+                            if ( *se ) se++;+                            ss = se;+                            continue;+                        }+                        if ( str.l ) kputc(',',&str);+                        kputsn(ss,se-ss,&str);+                        if ( *se ) se++;+                        ss = se;+                    }+                    if ( !*se ) break;+                }+                assert( n==nG_ori );+            }++            nret = bcf_update_info(header, line, bcf_hdr_int2id(header,BCF_DT_ID,info->key), (void*)str.s, str.l, type);+            if ( nret<0 )+            {+                fprintf(stderr,"[%s:%d %s] Could not update INFO/%s at %s:%d [%d]\n", __FILE__,__LINE__,__FUNCTION__,+                        bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname(header,line), line->pos+1, nret);+                exit(1);+            }+            continue;+        }++        if ( vlen==BCF_VL_A || vlen==BCF_VL_R )+        {+            int inc = 0, ntop;+            if ( vlen==BCF_VL_A )+            {+                assert( nret==nA_ori );+                ntop = nA_ori;+                ndat = nA_new;+                inc  = 1;+            }+            else+            {+                assert( nret==nR_ori );+                ntop = nR_ori;+                ndat = nR_new;+            }+            int k = 0;++            #define BRANCH(type_t,is_vector_end) \+            { \+                type_t *ptr = (type_t*) dat; \+                int size = sizeof(type_t); \+                for (j=0; j<ntop; j++) /* j:ori, k:new */ \+                { \+                    if ( is_vector_end ) { memcpy(dat+k*size, dat+j*size, size); break; } \+                    if ( kbs_exists(rm_set, j+inc) ) continue; \+                    if ( j!=k ) memcpy(dat+k*size, dat+j*size, size); \+                    k++; \+                } \+            }+            switch (type)+            {+                case BCF_HT_INT:  BRANCH(int32_t,ptr[j]==bcf_int32_vector_end); break;+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr[j])); break;+            }+            #undef BRANCH+        }+        else    // Number=G+        {+            assert( nret==nG_ori );+            int k, l_ori = -1, l_new = 0;+            ndat = nG_new;++            #define BRANCH(type_t,is_vector_end) \+            { \+                type_t *ptr = (type_t*) dat; \+                int size = sizeof(type_t); \+                for (j=0; j<nR_ori; j++) \+                { \+                    for (k=0; k<=j; k++) \+                    { \+                        l_ori++; \+                        if ( is_vector_end ) { memcpy(dat+l_new*size, dat+l_ori*size, size); break; } \+                        if ( kbs_exists(rm_set, j) || kbs_exists(rm_set, k) ) continue; \+                        if ( l_ori!=l_new ) memcpy(dat+l_new*size, dat+l_ori*size, size); \+                        l_new++; \+                    } \+                } \+            }+            switch (type)+            {+                case BCF_HT_INT:  BRANCH(int32_t,ptr[l_ori]==bcf_int32_vector_end); break;+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr[l_ori])); break;+            }+            #undef BRANCH+        }++        nret = bcf_update_info(header, line, bcf_hdr_int2id(header,BCF_DT_ID,info->key), (void*)dat, ndat, type);+        if ( nret<0 )+        {+            fprintf(stderr,"[%s:%d %s] Could not update INFO/%s at %s:%d [%d]\n", __FILE__,__LINE__,__FUNCTION__,+                    bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname(header,line), line->pos+1, nret);+            exit(1);+        }+    }++    // Update GT fields, the allele indexes might have changed+    for (i=1; i<line->n_allele; i++) if ( map[i]!=i ) break;+    if ( i<line->n_allele )+    {+        mdat = mdat_bytes / 4;  // sizeof(int32_t)+        nret = bcf_get_genotypes(header,line,(void**)&dat,&mdat);+        mdat_bytes = mdat * 4;+        if ( nret>0 )+        {+            nret /= line->n_sample;+            int32_t *ptr = (int32_t*) dat;+            for (i=0; i<line->n_sample; i++)+            {+                for (j=0; j<nret; j++)+                {+                    if ( bcf_gt_is_missing(ptr[j]) ) continue;+                    if ( ptr[j]==bcf_int32_vector_end ) break;+                    int al = bcf_gt_allele(ptr[j]);+                    assert( al<nR_ori && map[al]>=0 );+                    ptr[j] = (map[al]+1)<<1 | (ptr[j]&1);+                }+                ptr += nret;+            }+            bcf_update_genotypes(header, line, (void*)dat, nret*line->n_sample);+        }+    }++    // Remove from Number=G, Number=R and Number=A FORMAT fields.+    // Assuming haploid or diploid GTs+    for (i=0; i<line->n_fmt; i++)+    {+        bcf_fmt_t *fmt = &line->d.fmt[i];+        int vlen = bcf_hdr_id2length(header,BCF_HL_FMT,fmt->id);++        if ( vlen!=BCF_VL_A && vlen!=BCF_VL_G && vlen!=BCF_VL_R ) continue; // no need to change++        int type = bcf_hdr_id2type(header,BCF_HL_FMT,fmt->id);+        if ( type==BCF_HT_FLAG ) continue;++        int size = 1;+        if ( type==BCF_HT_REAL || type==BCF_HT_INT ) size = 4;++        mdat = mdat_bytes / size;+        nret = bcf_get_format_values(header, line, bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), (void**)&dat, &mdat, type);+        mdat_bytes = mdat * size;+        if ( nret<0 )+        {+            fprintf(stderr,"[%s:%d %s] Could not access FORMAT/%s at %s:%d [%d]\n", __FILE__,__LINE__,__FUNCTION__,+                    bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname(header,line), line->pos+1, nret);+            exit(1);+        }++        if ( type==BCF_HT_STR )+        {+            int size = nret/line->n_sample;     // number of bytes per sample+            str.l = 0;+            if ( vlen==BCF_VL_A || vlen==BCF_VL_R )+            {+                int nexp, inc = 0;+                if ( vlen==BCF_VL_A )+                {+                    nexp = nA_ori;+                    inc  = 1;+                }+                else+                    nexp = nR_ori;+                for (j=0; j<line->n_sample; j++)+                {+                    char *ss = ((char*)dat) + j*size, *se = ss + size, *ptr = ss;+                    int k_src = 0, k_dst = 0, l = str.l;+                    for (k_src=0; k_src<nexp; k_src++)+                    {+                        if ( ptr>=se || !*ptr) break;+                        while ( ptr<se && *ptr && *ptr!=',' ) ptr++;+                        if ( kbs_exists(rm_set, k_src+inc) )+                        {+                            ss = ++ptr;+                            continue;+                        }+                        if ( k_dst ) kputc(',',&str);+                        kputsn(ss,ptr-ss,&str);+                        ss = ++ptr;+                        k_dst++;+                    }+                    assert( k_src==nexp );+                    l = str.l - l;+                    for (; l<size; l++) kputc(0, &str);+                }+            }+            else    // Number=G, diploid or haploid+            {+                for (j=0; j<line->n_sample; j++)+                {+                    char *ss = ((char*)dat) + j*size, *se = ss + size, *ptr = ss;+                    int k_src = 0, k_dst = 0, l = str.l;+                    int nexp = 0; // diploid or haploid?+                    while ( ptr<se )+                    {+                        if ( !*ptr ) break;+                        if ( *ptr==',' ) nexp++;+                        ptr++;+                    }+                    if ( ptr!=ss ) nexp++;+                    assert( nexp==nG_ori || nexp==nR_ori );+                    ptr = ss;+                    if ( nexp==nG_ori ) // diploid+                    {+                        int ia, ib;+                        for (ia=0; ia<nR_ori; ia++)+                        {+                            for (ib=0; ib<=ia; ib++)+                            {+                                if ( ptr>=se || !*ptr ) break;+                                while ( ptr<se && *ptr && *ptr!=',' ) ptr++;+                                if ( kbs_exists(rm_set, ia) || kbs_exists(rm_set, ib) )+                                {+                                    ss = ++ptr;+                                    continue;+                                }+                                if ( k_dst ) kputc(',',&str);+                                kputsn(ss,ptr-ss,&str);+                                ss = ++ptr;+                                k_dst++;+                            }+                            if ( ptr>=se || !*ptr ) break;+                        }+                    }+                    else    // haploid+                    {+                        for (k_src=0; k_src<nR_ori; k_src++)+                        {+                            if ( ptr>=se || !*ptr ) break;+                            while ( ptr<se && *ptr && *ptr!=',' ) ptr++;+                            if ( kbs_exists(rm_set, k_src) )+                            {+                                ss = ++ptr;+                                continue;+                            }+                            if ( k_dst ) kputc(',',&str);+                            kputsn(ss,ptr-ss,&str);+                            ss = ++ptr;+                            k_dst++;+                        }+                        assert( k_src==nR_ori );+                        l = str.l - l;+                        for (; l<size; l++) kputc(0, &str);+                    }+                }+            }+            nret = bcf_update_format(header, line, bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), (void*)str.s, str.l, type);+            if ( nret<0 )+            {+                fprintf(stderr,"[%s:%d %s] Could not update FORMAT/%s at %s:%d [%d]\n", __FILE__,__LINE__,__FUNCTION__,+                        bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname(header,line), line->pos+1, nret);+                exit(1);+            }+            continue;+        }++        int nori = nret / line->n_sample;+        if ( vlen==BCF_VL_A || vlen==BCF_VL_R || (vlen==BCF_VL_G && nori==nR_ori) ) // Number=A, R or haploid Number=G+        {+            int inc = 0, nnew;+            if ( vlen==BCF_VL_A )+            {+                assert( nori==nA_ori );     // todo: will fail if all values are missing+                ndat = nA_new*line->n_sample;+                nnew = nA_new;+                inc  = 1;+            }+            else+            {+                assert( nori==nR_ori );     // todo: will fail if all values are missing+                ndat = nR_new*line->n_sample;+                nnew = nR_new;+            }++            #define BRANCH(type_t,is_vector_end) \+            { \+                for (j=0; j<line->n_sample; j++) \+                { \+                    type_t *ptr_src = ((type_t*)dat) + j*nori; \+                    type_t *ptr_dst = ((type_t*)dat) + j*nnew; \+                    int size = sizeof(type_t); \+                    int k_src, k_dst = 0; \+                    for (k_src=0; k_src<nori; k_src++) \+                    { \+                        if ( is_vector_end ) { memcpy(ptr_dst+k_dst, ptr_src+k_src, size); break; } \+                        if ( kbs_exists(rm_set, k_src+inc) ) continue; \+                        memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \+                        k_dst++; \+                    } \+                } \+            }+            switch (type)+            {+                case BCF_HT_INT:  BRANCH(int32_t,ptr_src[k_src]==bcf_int32_vector_end); break;+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr_src[k_src])); break;+            }+            #undef BRANCH+        }+        else    // Number=G, diploid or mixture of haploid+diploid+        {+            assert( nori==nG_ori );+            ndat = nG_new*line->n_sample;++            #define BRANCH(type_t,is_vector_end) \+            { \+                for (j=0; j<line->n_sample; j++) \+                { \+                    type_t *ptr_src = ((type_t*)dat) + j*nori; \+                    type_t *ptr_dst = ((type_t*)dat) + j*nG_new; \+                    int size = sizeof(type_t); \+                    int ia, ib, k_dst = 0, k_src; \+                    int nset = 0;   /* haploid or diploid? */ \+                    for (k_src=0; k_src<nG_ori; k_src++) { if ( is_vector_end ) break; nset++; } \+                    if ( nset==nR_ori ) /* haploid */ \+                    { \+                        for (k_src=0; k_src<nR_ori; k_src++) \+                        { \+                            if ( kbs_exists(rm_set, k_src) ) continue; \+                            memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \+                            k_dst++; \+                        } \+                        memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \+                    } \+                    else /* diploid */ \+                    { \+                        k_src = -1; \+                        for (ia=0; ia<nR_ori; ia++) \+                        { \+                            for (ib=0; ib<=ia; ib++) \+                            { \+                                k_src++; \+                                if ( is_vector_end ) { memcpy(ptr_dst+k_dst, ptr_src+k_src, size); ia = nR_ori; break; }  \+                                if ( kbs_exists(rm_set, ia) || kbs_exists(rm_set, ib) ) continue; \+                                memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \+                                k_dst++; \+                            } \+                        } \+                    } \+                } \+            }+            switch (type)+            {+                case BCF_HT_INT:  BRANCH(int32_t,ptr_src[k_src]==bcf_int32_vector_end); break;+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr_src[k_src])); break;+            }+            #undef BRANCH+        }+        nret = bcf_update_format(header, line, bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), (void*)dat, ndat, type);+        if ( nret<0 )+        {+            fprintf(stderr,"[%s:%d %s] Could not update FORMAT/%s at %s:%d [%d]\n", __FILE__,__LINE__,__FUNCTION__,+                    bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname(header,line), line->pos+1, nret);+            exit(1);+        }+    }+    free(dat);+    free(str.s);+    free(map);+}+
+ htslib-1.3.2/version.h view
@@ -0,0 +1,1 @@+#define HTS_VERSION "1.3.2"
+ src/Bio/HTS.c view
@@ -0,0 +1,167 @@++#include "htslib/sam.h"++int inline_c_Bio_HTS_0_0efc6161c1b66bc79e862910b9e93f1d32ddc02d(htsFile * fp_inline_c_0, bam1_t * b_inline_c_1) {+return ( bam_write1(fp_inline_c_0->fp.bgzf, b_inline_c_1) );+}+++int inline_c_Bio_HTS_1_0c54bc94ad2e32d428064e2a5c26a7036890b9a6(htsFile * fp_inline_c_0, bam_hdr_t * hdr_inline_c_1) {+return (+                        bam_hdr_write(fp_inline_c_0->fp.bgzf, hdr_inline_c_1) );+}+++htsFile * inline_c_Bio_HTS_2_07f3fede9c57363b6e41738becd5a00d64df9d5c(char * fn_27_inline_c_0) {+return ( hts_open(fn_27_inline_c_0, "r") );+}+++htsFile * inline_c_Bio_HTS_3_4af84655e4829571c79edc68791d221a719c60ab(char * fn_27_inline_c_0) {+return ( hts_open(fn_27_inline_c_0, "wb") );+}+++void inline_c_Bio_HTS_4_d667db7c61cb0de3c96f1d806701fe63606a1af0(htsFile * h_inline_c_0) {+ hts_close(h_inline_c_0) ;+}+++bam_hdr_t * inline_c_Bio_HTS_5_9a30278929e5b8570bc58dc24e83fd868ec2513f(htsFile * h_inline_c_0) {+return ( bam_hdr_read(h_inline_c_0->fp.bgzf) );+}+++char * inline_c_Bio_HTS_6_5a31ba58a253c55850867b8802710825075bca03(bam_hdr_t * hdr_inline_c_0) {+return ( hdr_inline_c_0->text );+}+++bam1_t * inline_c_Bio_HTS_7_25f5b2f49b2eac08644b77f5a1c27030b81e5dc3(int * r_inline_c_0, htsFile * h_inline_c_1) {++            bam1_t *b = bam_init1();+            *r_inline_c_0 = bam_read1(h_inline_c_1->fp.bgzf, b);+            return b;+        +}+++int32_t inline_c_Bio_HTS_8_8e12d9d09c24fcc6b41593e5bc03c97f756f6da1(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.tid );+}+++char * inline_c_Bio_HTS_9_31676430ec791ba4f73764dc21c992d6d39307fc(bam_hdr_t * h_inline_c_0, int32_t i_inline_c_1) {+return ( h_inline_c_0->target_name[i_inline_c_1] );+}+++int32_t inline_c_Bio_HTS_10_c89b42fc83db79c1cd226594b3a86cd92b81f38b(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.pos );+}+++int32_t inline_c_Bio_HTS_11_cc329ca55a7e13133af0b3dbb6abb3edc854b65e(bam1_t * b_inline_c_0) {+return ( bam_endpos(b_inline_c_0) );+}+++int32_t inline_c_Bio_HTS_12_6fb2dbf3d90d4141663e524757f874dcf05a03e4(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.l_qseq );+}+++int inline_c_Bio_HTS_13_44d7a287b2cc95c82e760427675196e1ba7261a1(bam1_t * b_inline_c_0) {+return (bam_is_rev(b_inline_c_0) );+}+++uint16_t inline_c_Bio_HTS_14_12453aaa173031b617539dcd57698955b11bd327(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.flag );+}+++uint8_t inline_c_Bio_HTS_15_49728fef988bc0954110504a26677faa43ce6ce9(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.qual );+}+++int32_t inline_c_Bio_HTS_16_6fb2dbf3d90d4141663e524757f874dcf05a03e4(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.l_qseq );+}+++void inline_c_Bio_HTS_17_ed1829c64b8823aa1711ba319219f175ed965ce4(bam1_t * b_inline_c_0, int32_t l_inline_c_1, char * str_inline_c_2) {++                        int32_t i;+                        uint8_t *s = bam_get_seq(b_inline_c_0);+                        for (i = 0; i < l_inline_c_1; ++i)+                            str_inline_c_2[i] = "=ACMGRSVTWYHKDBN"[bam_seqi(s, i)];+                    +}+++char * inline_c_Bio_HTS_18_7083ac4175bb445932400343a035586aa327afa8(bam1_t * b_inline_c_0) {+return (bam_get_qname(b_inline_c_0) );+}+++int32_t inline_c_Bio_HTS_19_6fb2dbf3d90d4141663e524757f874dcf05a03e4(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.l_qseq );+}+++int8_t inline_c_Bio_HTS_20_ff7eee4e1210259019f9cd0f9e269fdcf8ed1666(bam1_t * b_inline_c_0) {++                        uint8_t *s = bam_get_qual(b_inline_c_0);+                        return (s[0] == 0xff);+                     +}+++void inline_c_Bio_HTS_21_4b537c6c8c081040b4dd320c93ee949f58ac34df(bam1_t * b_inline_c_0, int32_t l_inline_c_1, char * str_inline_c_2) {++                                int32_t i;+                                uint8_t *s = bam_get_qual(b_inline_c_0);+                                for (i = 0; i < l_inline_c_1; ++i)+                                    str_inline_c_2[i] = s[i];+                            +}+++uint16_t inline_c_Bio_HTS_22_ba4509ac6cfa51e21b3315e1bc0c79664332c151(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.n_cigar );+}+++void inline_c_Bio_HTS_23_04238a6d67efdaba04e6fe62c62067daa422dedb(bam1_t * b_inline_c_0, uint16_t n_inline_c_1, int * num_inline_c_2, char * str_inline_c_3) {++                            uint16_t i;+                            uint32_t *cigar = bam_get_cigar(b_inline_c_0);+                            for (i = 0; i < n_inline_c_1; ++i) {+                                num_inline_c_2[i] = bam_cigar_oplen(cigar[i]);+                                str_inline_c_3[i] = bam_cigar_opchr(cigar[i]);+                            }+                         +}+++int32_t inline_c_Bio_HTS_24_a6dc5594b24fbf473396136e59bcee0228785fc4(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.mtid );+}+++char * inline_c_Bio_HTS_25_31676430ec791ba4f73764dc21c992d6d39307fc(bam_hdr_t * h_inline_c_0, int32_t i_inline_c_1) {+return ( h_inline_c_0->target_name[i_inline_c_1] );+}+++int32_t inline_c_Bio_HTS_26_54a6e402cf38aa9b617612b54be97e879e2b17b7(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.mpos );+}+++int32_t inline_c_Bio_HTS_27_14020e59f59859519591c231f45ae2ab8f85c395(bam1_t * b_inline_c_0) {+return ( b_inline_c_0->core.isize );+}+
+ src/Bio/HTS.hs view
@@ -0,0 +1,291 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes       #-}+{-# LANGUAGE TemplateHaskell   #-}++module Bio.HTS where++import           Conduit+import           Control.Monad+import           Control.Monad.State+import qualified Data.ByteString.Char8    as B+import           Data.Int+import           Data.Monoid              ((<>))+import           Data.Word+import           Foreign.C.String+import           Foreign.C.Types+import           Foreign.ForeignPtr+import           Foreign.Marshal.Array+import           Foreign.Ptr+import           Language.C.Inline        (baseCtx, bsCtx, context, include,+                                           withPtr)+import qualified Language.C.Inline.Unsafe as CU+import           System.IO+import           System.IO.Unsafe         (unsafePerformIO)++import           Bio.HTS.Types++context (baseCtx <> bsCtx <> htsCtx)++include "htslib/sam.h"++type HeaderState = ResourceT (StateT FileHeader IO)++runBam :: HeaderState a -> IO a+runBam x = evalStateT (runResourceT x) Empty++readBam :: FilePath -> Source HeaderState Bam+readBam fn = bracketP (openBamFile fn ReadMode) closeBamFile $ \h -> do+    hdr <- liftIO (readBamHeader h)+    lift $ put hdr+    source h+  where+    source x = do+        r <- liftIO $ bamRead1 x+        case r of+            Nothing -> return ()+            Just b -> yield b >> source x+{-# INLINE readBam #-}++writeBam :: FilePath -> Sink Bam HeaderState ()+writeBam fn = bracketP (openBamFile fn WriteMode) closeBamFile $ \(BamFileHandle fp) -> do+    maybeBam <- await+    case maybeBam of+        Nothing -> return ()+        Just b' -> do+            leftover b'+            header <- lift get+            case header of+                BamHeader hdr -> do+                    err <- liftIO $ [CU.exp| int {+                        bam_hdr_write($(htsFile* fp)->fp.bgzf, $(bam_hdr_t* hdr)) } |]+                    if err /= 0+                        then error "'bam_hdr_write' failed."+                        else sink fp+                _ -> error "No Bam header was found!"+  where+    sink fp = awaitForever $ \b' -> liftIO $ withForeignPtr b' $ \b ->+        [CU.exp| int { bam_write1($(htsFile* fp)->fp.bgzf, $(bam1_t* b)) } |]+{-# INLINE writeBam #-}++openBamFile :: FilePath -> IOMode -> IO BamFileHandle+openBamFile fn mode = do+    fn' <- newCString fn+    h <- case mode of+        ReadMode -> [CU.exp| htsFile* { hts_open($(char* fn'), "r") } |]+        WriteMode -> [CU.exp| htsFile* { hts_open($(char* fn'), "wb") } |]+        _ -> error ""+    return $ BamFileHandle h++closeBamFile :: BamFileHandle -> IO ()+closeBamFile (BamFileHandle h) = [CU.exp| void { hts_close($(htsFile* h)) } |]++readBamHeader :: BamFileHandle -> IO FileHeader+readBamHeader (BamFileHandle h) =+    BamHeader <$> [CU.exp| bam_hdr_t* { bam_hdr_read($(htsFile* h)->fp.bgzf) } |]++showBamHeader :: FileHeader -> B.ByteString+showBamHeader (BamHeader hdr) = unsafePerformIO $ join $ B.packCString <$>+    [CU.exp| char* { $(bam_hdr_t* hdr)->text } |]+showBamHeader _ = ""++data SortOrder = Unknown+               | Unsorted+               | Queryname+               | Coordinate++getSortOrder :: FileHeader -> SortOrder+getSortOrder header = case lookup "SO" fields of+    Just "unknown" -> Unknown+    Just "unsorted" -> Unsorted+    Just "queryname" -> Queryname+    Just "coordinate" -> Coordinate+    _ -> Unknown+  where+    fields = map (f . B.split ':') $ tail $ B.split '\t' $ head $ B.lines $+        showBamHeader header+    f [a,b] = (a,b)+    f _ = error "Auxiliary field parsing failed!"++bamRead1 :: BamFileHandle -> IO (Maybe Bam)+bamRead1 (BamFileHandle h) = do+    (r, b) <- withPtr $ \r -> [CU.block| bam1_t* {+            bam1_t *b = bam_init1();+            *$(int* r) = bam_read1($(htsFile* h)->fp.bgzf, b);+            return b;+        } |]+    if r < 0 then return Nothing else Just <$> newForeignPtr bamDestory b+{-# INLINE bamRead1 #-}++foreign import ccall unsafe "&bam_destroy1"+   bamDestory :: FunPtr (Ptr Bam' -> IO ())++-- | Return the chromosome id.+getChrId :: Bam -> Int32+getChrId = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { $(bam1_t* b)->core.tid } |]+{-# INLINE getChrId #-}++-- | Return the chromosome name given the bam file header.+getChr :: Ptr BamHdr -> Bam -> Maybe B.ByteString+getChr h b' | i < 0 = Nothing+            | otherwise = Just $ unsafePerformIO $ join $ B.packCString <$>+                [CU.exp| char* { $(bam_hdr_t* h)->target_name[$(int32_t i)] } |]+  where+    i = getChrId b'+{-# INLINE getChr #-}++-- | Return the 0-based starting location.+position :: Bam -> Int32+position = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { $(bam1_t* b)->core.pos } |]+{-# INLINE position #-}++-- | For a mapped read, this is just position + cigar2rlen.+-- For an unmapped read (either according to its flags or if it has no cigar+-- string), we return position + 1 by convention.+endPos :: Bam -> Int32+endPos = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { bam_endpos($(bam1_t* b)) } |]+{-# INLINE endPos #-}++-- | Return the query length (read length).+queryLen :: Bam -> Int32+queryLen = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { $(bam1_t* b)->core.l_qseq } |]+{-# INLINE queryLen #-}++-- | Whether the query is on the reverse strand.+isRev :: Bam -> Bool+isRev = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = do+        r <- [CU.exp| int {bam_is_rev($(bam1_t* b)) } |]+        return $ if r == 0 then False else True+{-# INLINE isRev #-}++-- | Return the flag.+flag :: Bam -> Word16+flag = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| uint16_t { $(bam1_t* b)->core.flag } |]+{-# INLINE flag #-}++-- | MAPping Quality. It equals −10 log10 Pr{mapping position is wrong}, rounded to the nearest integer. A value 255 indicates that the mapping quality is not available.+mapq :: Bam -> Word8+mapq = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| uint8_t { $(bam1_t* b)->core.qual } |]+{-# INLINE mapq #-}+++--int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };+-- | Return the DNA sequence.+getSeq :: Bam -> Maybe B.ByteString+getSeq = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = do+        l <- [CU.exp| int32_t { $(bam1_t* b)->core.l_qseq } |]+        if (l == 0)+            then return Nothing+            else allocaArray (fromIntegral l) $ \str -> do+                    [CU.block| void {+                        int32_t i;+                        uint8_t *s = bam_get_seq($(bam1_t* b));+                        for (i = 0; i < $(int32_t l); ++i)+                            $(char* str)[i] = "=ACMGRSVTWYHKDBN"[bam_seqi(s, i)];+                    } |]+                    Just <$> B.packCStringLen (str, fromIntegral l)+{-# INLINE getSeq #-}+++-- | Get the name of the query.+qName :: Bam -> B.ByteString+qName = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = join $ B.packCString <$>+        [CU.exp| char* {bam_get_qname($(bam1_t* b)) } |]+{-# INLINE qName #-}++quality :: Bam -> Maybe B.ByteString+quality = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = do+        l <- [CU.exp| int32_t { $(bam1_t* b)->core.l_qseq } |]+        if l == 0+            then return Nothing+            else do+                x <- [CU.block| int8_t {+                        uint8_t *s = bam_get_qual($(bam1_t* b));+                        return (s[0] == 0xff);+                     } |]+                if x /= 0+                    then return Nothing+                    else allocaArray (fromIntegral l) $ \str -> do+                            [CU.block| void {+                                int32_t i;+                                uint8_t *s = bam_get_qual($(bam1_t* b));+                                for (i = 0; i < $(int32_t l); ++i)+                                    $(char* str)[i] = s[i];+                            } |]+                            Just <$> B.packCStringLen (str, fromIntegral l)+{-# INLINE quality #-}++cigar :: Bam -> Maybe [(Int, Char)]+cigar = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = do+        n <- [CU.exp| uint16_t { $(bam1_t* b)->core.n_cigar } |]+        if (n == 0)+            then return Nothing+            else allocaArray (fromIntegral n) $ \num ->+                     allocaArray (fromIntegral n) $ \str -> do+                         [CU.block| void {+                            uint16_t i;+                            uint32_t *cigar = bam_get_cigar($(bam1_t* b));+                            for (i = 0; i < $(uint16_t n); ++i) {+                                $(int* num)[i] = bam_cigar_oplen(cigar[i]);+                                $(char* str)[i] = bam_cigar_opchr(cigar[i]);+                            }+                         } |]+                         num' <- peekArray (fromIntegral n) num+                         str' <- peekArray (fromIntegral n) str+                         return $ Just $+                            zip (map fromIntegral num') $ map castCCharToChar str'+{-# INLINE cigar #-}++mateChrId :: Bam -> Int32+mateChrId = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { $(bam1_t* b)->core.mtid } |]+{-# INLINE mateChrId #-}++mateChr :: Ptr BamHdr -> Bam -> Maybe B.ByteString+mateChr h b' | i < 0 = Nothing+             | otherwise = Just $ unsafePerformIO $ join $ B.packCString <$>+                [CU.exp| char* { $(bam_hdr_t* h)->target_name[$(int32_t i)] } |]+  where+    i = mateChrId b'+{-# INLINE mateChr #-}++-- | 0-based+matePos :: Bam -> Int32+matePos = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { $(bam1_t* b)->core.mpos } |]+{-# INLINE matePos #-}++tLen :: Bam -> Int32+tLen = unsafePerformIO . flip withForeignPtr fn+  where+    fn b = [CU.exp| int32_t { $(bam1_t* b)->core.isize } |]+{-# INLINE tLen #-}+++-- | Convert Bam record to Sam record.+bamToSam :: Ptr BamHdr -> Bam -> Sam+bamToSam h b = Sam (qName b) (flag b) (getChr h b) (position b) (mapq b)+    (cigar b) (mateChr h b) (matePos b) (tLen b) (getSeq b) (quality b)
+ src/Bio/HTS/Types.hs view
@@ -0,0 +1,77 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes       #-}+{-# LANGUAGE TemplateHaskell   #-}++module Bio.HTS.Types+    ( htsCtx+    , BamFileHandle(..)+    , FileHeader(..)+    , BamHdr+    , HTSFile+    , Bam+    , Bam'+    , Sam(..)+    , showSam+    ) where++import qualified Data.ByteString              as BS+import qualified Data.ByteString.Char8        as B+import           Data.ByteString.Lex.Integral+import           Data.Int+import qualified Data.Map                     as M+import           Data.Maybe                   (fromJust, fromMaybe)+import           Data.Word+import           Foreign.ForeignPtr+import           Foreign.Ptr+import qualified Language.C.Inline            as C+import qualified Language.C.Inline.Context    as C+import qualified Language.C.Types             as C+import qualified Language.Haskell.TH          as TH++type Bam = ForeignPtr Bam'+data Bam'++data FileHeader = Empty+                | BamHeader (Ptr BamHdr)+data BamHdr++data HTSFile+newtype BamFileHandle = BamFileHandle (Ptr HTSFile)++htsCtx :: C.Context+htsCtx = mempty+  { C.ctxTypesTable = htsTypesTable+  }++htsTypesTable :: M.Map C.TypeSpecifier TH.TypeQ+htsTypesTable = M.fromList+   [ (C.TypeName "htsFile", [t| HTSFile |])+   , (C.TypeName "bam_hdr_t", [t| BamHdr |])+   , (C.TypeName "bam1_t", [t| Bam' |])+   ]++-- | currently only contains 11 mandatory fields+data Sam = Sam+    { samQname :: !B.ByteString+    , samFlag  :: !Word16+    , samRname :: !(Maybe B.ByteString)+    , samPos   :: !Int32+    , samMapq  :: !Word8+    , samCigar :: !(Maybe [(Int, Char)])+    , samRnext :: !(Maybe B.ByteString)+    , samPnext :: !Int32+    , samTlen  :: !Int32+    , samSeq   :: !(Maybe B.ByteString)+    , samQual  :: !(Maybe B.ByteString)+    } deriving (Show)++showSam :: Sam -> B.ByteString+showSam s = B.intercalate "\t"+    [ samQname s, pack' $ samFlag s, fromMaybe "*" $ samRname s, pack' $ samPos s + 1+    , pack' $ samMapq s, fromMaybe "*" $ f <$> samCigar s, fromMaybe "*" $ samRnext s+    , pack' $ samPnext s + 1, pack' $ samTlen s, fromMaybe "*" $ samSeq s+    , fromMaybe "*" $ BS.map (+33) <$> samQual s ]+  where+    f = B.concat . concatMap (\(i, x) -> [pack' i, B.singleton x])+    pack' :: Integral a => a -> B.ByteString+    pack' = fromJust . packDecimal