HsHTSLib 1.9.0.1 → 1.9.1
raw patch · 7 files changed
+20/−2011 lines, 7 filesdep ~base
Dependency ranges changed: base
Files
- HsHTSLib.cabal +9/−8
- README.md +4/−0
- cbits/hs_htslib.c +1/−1
- htslib-1.9/config.h +6/−6
- htslib-1.9/hfile_gcs.c +0/−137
- htslib-1.9/hfile_libcurl.c +0/−1393
- htslib-1.9/hfile_s3.c +0/−466
HsHTSLib.cabal view
@@ -1,5 +1,5 @@ name: HsHTSLib-version: 1.9.0.1+version: 1.9.1 synopsis: Bindings to htslib. description: This package provides bindings to htslib, a library for processing high throughput DNA sequencing data.@@ -12,6 +12,7 @@ build-type: Simple cabal-version: >=1.10 extra-source-files:+ README.md ChangeLog.md htslib-1.9/*.h htslib-1.9/htslib/*.h@@ -29,14 +30,14 @@ Bio.HTS.Types build-depends:- base >=4.8 && <5.0+ base >= 4.10 && < 5.0 , bytestring , bytestring-lexing- , conduit >=1.3.0+ , conduit >= 1.3.0 hs-source-dirs: src- build-tools: c2hs >=0.25.0- extra-libraries: pthread z curl bz2 lzma crypto+ build-tools: c2hs >= 0.25.0+ extra-libraries: pthread z include-dirs: htslib-1.9 c-sources: cbits/hs_htslib.c@@ -47,10 +48,10 @@ htslib-1.9/errmod.c htslib-1.9/faidx.c htslib-1.9/hfile.c- htslib-1.9/hfile_gcs.c- htslib-1.9/hfile_libcurl.c+ --htslib-1.9/hfile_gcs.c+ --htslib-1.9/hfile_libcurl.c htslib-1.9/hfile_net.c- htslib-1.9/hfile_s3.c+ --htslib-1.9/hfile_s3.c htslib-1.9/hts.c -- htslib-1.9/htsfile.c htslib-1.9/hts_os.c
+ README.md view
@@ -0,0 +1,4 @@+htslib configuration+--------------------++bz2, lzma, libcurl are disabled as CRAM is not supported by this library.
cbits/hs_htslib.c view
@@ -15,7 +15,7 @@ } int bam_is_rev_(bam1_t *b) {- return b->core.flag&BAM_FREVERSE != 0;+ return (b->core.flag&BAM_FREVERSE) != 0; } void bam_get_seq_(bam1_t *b, char *res, uint32_t l) {
htslib-1.9/config.h view
@@ -12,13 +12,13 @@ accurate PACKAGE_VERSION as computed by the Makefile. */ /* Define if HTSlib should enable GCS support. */-#define ENABLE_GCS 1+/* #define ENABLE_GCS 1 /* Define if HTSlib should enable plugins. */ /* #undef ENABLE_PLUGINS */ /* Define if HTSlib should enable S3 support. */-#define ENABLE_S3 1+/* #define ENABLE_S3 1 /* Define if you have the Common Crypto library. */ /* #undef HAVE_COMMONCRYPTO */@@ -48,22 +48,22 @@ #define HAVE_INTTYPES_H 1 /* Define to 1 if you have the `bz2' library (-lbz2). */-#define HAVE_LIBBZ2 1+/* #define HAVE_LIBBZ2 1 /* Define if libcurl file access is enabled. */-#define HAVE_LIBCURL 1+/* #define HAVE_LIBCURL 1 /* Define if libdeflate is available. */ /* #undef HAVE_LIBDEFLATE */ /* Define to 1 if you have the `lzma' library (-llzma). */-#define HAVE_LIBLZMA 1+/* #define HAVE_LIBLZMA 1 /* Define to 1 if you have the `z' library (-lz). */ #define HAVE_LIBZ 1 /* Define to 1 if you have the <lzma.h> header file. */-#define HAVE_LZMA_H 1+/* #define HAVE_LZMA_H 1 /* Define to 1 if you have the <memory.h> header file. */ #define HAVE_MEMORY_H 1
− htslib-1.9/hfile_gcs.c
@@ -1,137 +0,0 @@-/* hfile_gcs.c -- Google Cloud Storage backend for low-level file streams.-- Copyright (C) 2016 Genome Research Ltd.-- Author: John Marshall <jm18@sanger.ac.uk>--Permission is hereby granted, free of charge, to any person obtaining a copy-of this software and associated documentation files (the "Software"), to deal-in the Software without restriction, including without limitation the rights-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-copies of the Software, and to permit persons to whom the Software is-furnished to do so, subject to the following conditions:--The above copyright notice and this permission notice shall be included in-all copies or substantial portions of the Software.--THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER-DEALINGS IN THE SOFTWARE. */--#include <config.h>--#include <stdarg.h>-#include <stdio.h>-#include <stdlib.h>-#include <string.h>-#include <errno.h>--#include "htslib/hts.h"-#include "htslib/kstring.h"-#include "hfile_internal.h"-#ifdef ENABLE_PLUGINS-#include "version.h"-#endif--static hFILE *-gcs_rewrite(const char *gsurl, const char *mode, int mode_has_colon,- va_list *argsp)-{- const char *bucket, *path, *access_token;- kstring_t mode_colon = { 0, 0, NULL };- kstring_t url = { 0, 0, NULL };- kstring_t auth_hdr = { 0, 0, NULL };- hFILE *fp = NULL;-- // GCS URL format is gs[+SCHEME]://BUCKET/PATH-- if (gsurl[2] == '+') {- bucket = strchr(gsurl, ':') + 1;- kputsn(&gsurl[3], bucket - &gsurl[3], &url);- }- else {- kputs("https:", &url);- bucket = &gsurl[3];- }- while (*bucket == '/') kputc(*bucket++, &url);-- path = bucket + strcspn(bucket, "/?#");-- kputsn(bucket, path - bucket, &url);- if (strchr(mode, 'r')) kputs(".storage-download", &url);- else if (strchr(mode, 'w')) kputs(".storage-upload", &url);- else kputs(".storage", &url);- kputs(".googleapis.com", &url);-- kputs(path, &url);-- if (hts_verbose >= 8)- fprintf(stderr, "[M::gcs_open] rewrote URL as %s\n", url.s);-- // TODO Find the access token in a more standard way- access_token = getenv("GCS_OAUTH_TOKEN");-- if (access_token) {- kputs("Authorization: Bearer ", &auth_hdr);- kputs(access_token, &auth_hdr);- }-- if (argsp || auth_hdr.l > 0 || mode_has_colon) {- if (! mode_has_colon) {- kputs(mode, &mode_colon);- kputc(':', &mode_colon);- mode = mode_colon.s;- }-- fp = hopen(url.s, mode, "va_list", argsp,- "httphdr", (auth_hdr.l > 0)? auth_hdr.s : NULL, NULL);- }- else- fp = hopen(url.s, mode);-- free(mode_colon.s);- free(url.s);- free(auth_hdr.s);- return fp;-}--static hFILE *gcs_open(const char *url, const char *mode)-{- return gcs_rewrite(url, mode, 0, NULL);-}--static hFILE *gcs_vopen(const char *url, const char *mode_colon, va_list args0)-{- // Need to use va_copy() as we can only take the address of an actual- // va_list object, not that of a parameter as its type may have decayed.- va_list args;- va_copy(args, args0);- hFILE *fp = gcs_rewrite(url, mode_colon, 1, &args);- va_end(args);- return fp;-}--int PLUGIN_GLOBAL(hfile_plugin_init,_gcs)(struct hFILE_plugin *self)-{- static const struct hFILE_scheme_handler handler =- { gcs_open, hfile_always_remote, "Google Cloud Storage",- 2000 + 50, gcs_vopen- };--#ifdef ENABLE_PLUGINS- // Embed version string for examination via strings(1) or what(1)- static const char id[] = "@(#)hfile_gcs plugin (htslib)\t" HTS_VERSION;- if (hts_verbose >= 9)- fprintf(stderr, "[M::hfile_gcs.init] version %s\n", strchr(id, '\t')+1);-#endif-- self->name = "Google Cloud Storage";- hfile_add_scheme_handler("gs", &handler);- hfile_add_scheme_handler("gs+http", &handler);- hfile_add_scheme_handler("gs+https", &handler);- return 0;-}
− htslib-1.9/hfile_libcurl.c
@@ -1,1393 +0,0 @@-/* hfile_libcurl.c -- libcurl backend for low-level file streams.-- Copyright (C) 2015-2017 Genome Research Ltd.-- Author: John Marshall <jm18@sanger.ac.uk>--Permission is hereby granted, free of charge, to any person obtaining a copy-of this software and associated documentation files (the "Software"), to deal-in the Software without restriction, including without limitation the rights-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-copies of the Software, and to permit persons to whom the Software is-furnished to do so, subject to the following conditions:--The above copyright notice and this permission notice shall be included in-all copies or substantial portions of the Software.--THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER-DEALINGS IN THE SOFTWARE. */--#include <config.h>--#include <stdarg.h>-#include <stdlib.h>-#include <string.h>-#include <errno.h>-#include <pthread.h>-#ifndef _WIN32-# include <sys/select.h>-#endif-#include <assert.h>--#include "hfile_internal.h"-#ifdef ENABLE_PLUGINS-#include "version.h"-#endif-#include "htslib/hts.h" // for hts_version() and hts_verbose-#include "htslib/kstring.h"-#include "htslib/khash.h"--#include <curl/curl.h>--// Number of seconds to take off auth_token expiry, to allow for clock skew-// and slow servers-#define AUTH_REFRESH_EARLY_SECS 60--// Minimum number of bytes to skip when seeking forward. Seeks less than-// this will just read the data and throw it away. The optimal value-// depends on how long it takes to make a new connection compared-// to how fast the data arrives.-#define MIN_SEEK_FORWARD 1000000--typedef struct {- char *path;- char *token;- time_t expiry;- int failed;- pthread_mutex_t lock;-} auth_token;--// For the authorization header cache-KHASH_MAP_INIT_STR(auth_map, auth_token *)--// Curl-compatible header linked list-typedef struct {- struct curl_slist *list;- unsigned int num;- unsigned int size;-} hdrlist;--typedef struct {- hdrlist fixed; // List of headers supplied at hopen()- hdrlist extra; // List of headers from callback- hts_httphdr_callback callback; // Callback to get more headers- void *callback_data; // Data to pass to callback- auth_token *auth; // Authentication token- int auth_hdr_num; // Location of auth_token in hdrlist extra- // If -1, Authorization header is in fixed- // -2, it came from the callback- // -3, "auth_token_enabled", "false"- // passed to hopen()-} http_headers;--typedef struct {- hFILE base;- CURL *easy;- CURLM *multi;- off_t file_size;- struct {- union { char *rd; const char *wr; } ptr;- size_t len;- } buffer;- CURLcode final_result; // easy result code for finished transfers- // Flags for communicating with libcurl callbacks:- unsigned paused : 1; // callback tells us that it has paused transfer- unsigned closing : 1; // informs callback that hclose() has been invoked- unsigned finished : 1; // wait_perform() tells us transfer is complete- unsigned perform_again : 1;- unsigned is_read : 1; // Opened in read mode- unsigned can_seek : 1; // Can (attempt to) seek on this handle- unsigned is_recursive:1; // Opened by hfile_libcurl itself- unsigned tried_seek : 1; // At least one seek has been attempted- int nrunning;- http_headers headers;- off_t delayed_seek; // Location to seek to before reading- off_t last_offset; // Location we're seeking from-} hFILE_libcurl;--static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence);-static int restart_from_position(hFILE_libcurl *fp, off_t pos);--static int http_status_errno(int status)-{- if (status >= 500)- switch (status) {- case 501: return ENOSYS;- case 503: return EBUSY;- case 504: return ETIMEDOUT;- default: return EIO;- }- else if (status >= 400)- switch (status) {- case 401: return EPERM;- case 403: return EACCES;- case 404: return ENOENT;- case 405: return EROFS;- case 407: return EPERM;- case 408: return ETIMEDOUT;- case 410: return ENOENT;- default: return EINVAL;- }- else return 0;-}--static int easy_errno(CURL *easy, CURLcode err)-{- long lval;-- switch (err) {- case CURLE_OK:- return 0;-- case CURLE_UNSUPPORTED_PROTOCOL:- case CURLE_URL_MALFORMAT:- return EINVAL;--#if LIBCURL_VERSION_NUM >= 0x071505- case CURLE_NOT_BUILT_IN:- return ENOSYS;-#endif-- case CURLE_COULDNT_RESOLVE_PROXY:- case CURLE_COULDNT_RESOLVE_HOST:- case CURLE_FTP_CANT_GET_HOST:- return EDESTADDRREQ; // Lookup failure-- case CURLE_COULDNT_CONNECT:- case CURLE_SEND_ERROR:- case CURLE_RECV_ERROR:- if (curl_easy_getinfo(easy, CURLINFO_OS_ERRNO, &lval) == CURLE_OK)- return lval;- else- return ECONNABORTED;-- case CURLE_REMOTE_ACCESS_DENIED:- case CURLE_LOGIN_DENIED:- case CURLE_TFTP_PERM:- return EACCES;-- case CURLE_PARTIAL_FILE:- return EPIPE;-- case CURLE_HTTP_RETURNED_ERROR:- if (curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &lval) == CURLE_OK)- return http_status_errno(lval);- else- return EIO;-- case CURLE_OUT_OF_MEMORY:- return ENOMEM;-- case CURLE_OPERATION_TIMEDOUT:- return ETIMEDOUT;-- case CURLE_RANGE_ERROR:- return ESPIPE;-- case CURLE_SSL_CONNECT_ERROR:- // TODO return SSL error buffer messages- return ECONNABORTED;-- case CURLE_FILE_COULDNT_READ_FILE:- case CURLE_TFTP_NOTFOUND:- return ENOENT;-- case CURLE_TOO_MANY_REDIRECTS:- return ELOOP;-- case CURLE_FILESIZE_EXCEEDED:- return EFBIG;-- case CURLE_REMOTE_DISK_FULL:- return ENOSPC;-- case CURLE_REMOTE_FILE_EXISTS:- return EEXIST;-- default:- return EIO;- }-}--static int multi_errno(CURLMcode errm)-{- switch (errm) {- case CURLM_CALL_MULTI_PERFORM:- case CURLM_OK:- return 0;-- case CURLM_BAD_HANDLE:- case CURLM_BAD_EASY_HANDLE:- case CURLM_BAD_SOCKET:- return EBADF;-- case CURLM_OUT_OF_MEMORY:- return ENOMEM;-- default:- return EIO;- }-}--static struct {- kstring_t useragent;- CURLSH *share;- char *auth_path;- khash_t(auth_map) *auth_map;- int allow_unencrypted_auth_header;- pthread_mutex_t auth_lock;- pthread_mutex_t share_lock;-} curl = { { 0, 0, NULL }, NULL, NULL, NULL, 0, PTHREAD_MUTEX_INITIALIZER,- PTHREAD_MUTEX_INITIALIZER };--static void share_lock(CURL *handle, curl_lock_data data,- curl_lock_access access, void *userptr) {- pthread_mutex_lock(&curl.share_lock);-}--static void share_unlock(CURL *handle, curl_lock_data data, void *userptr) {- pthread_mutex_unlock(&curl.share_lock);-}--static void free_auth(auth_token *tok) {- if (!tok) return;- if (pthread_mutex_destroy(&tok->lock)) abort();- free(tok->path);- free(tok->token);- free(tok);-}--static void libcurl_exit()-{- if (curl_share_cleanup(curl.share) == CURLSHE_OK)- curl.share = NULL;-- free(curl.useragent.s);- curl.useragent.l = curl.useragent.m = 0; curl.useragent.s = NULL;-- free(curl.auth_path);- curl.auth_path = NULL;-- if (curl.auth_map) {- khiter_t i;- for (i = kh_begin(curl.auth_map); i != kh_end(curl.auth_map); ++i) {- if (kh_exist(curl.auth_map, i)) {- free_auth(kh_value(curl.auth_map, i));- kh_key(curl.auth_map, i) = NULL;- kh_value(curl.auth_map, i) = NULL;- }- }- kh_destroy(auth_map, curl.auth_map);- curl.auth_map = NULL;- }-- curl_global_cleanup();-}--static int append_header(hdrlist *hdrs, const char *data, int dup) {- if (hdrs->num == hdrs->size) {- unsigned int new_sz = hdrs->size ? hdrs->size * 2 : 4, i;- struct curl_slist *new_list = realloc(hdrs->list,- new_sz * sizeof(*new_list));- if (!new_list) return -1;- hdrs->size = new_sz;- hdrs->list = new_list;- for (i = 1; i < hdrs->num; i++) hdrs->list[i-1].next = &hdrs->list[i];- }- // Annoyingly, libcurl doesn't declare the char * as const...- hdrs->list[hdrs->num].data = dup ? strdup(data) : (char *) data;- if (!hdrs->list[hdrs->num].data) return -1;- if (hdrs->num > 0) hdrs->list[hdrs->num - 1].next = &hdrs->list[hdrs->num];- hdrs->list[hdrs->num].next = NULL;- hdrs->num++;- return 0;-}--static void free_headers(hdrlist *hdrs, int completely) {- unsigned int i;- for (i = 0; i < hdrs->num; i++) {- free(hdrs->list[i].data);- hdrs->list[i].data = NULL;- hdrs->list[i].next = NULL;- }- hdrs->num = 0;- if (completely) {- free(hdrs->list);- hdrs->size = 0;- hdrs->list = NULL;- }-}--static struct curl_slist * get_header_list(hFILE_libcurl *fp) {- if (fp->headers.fixed.num > 0)- return &fp->headers.fixed.list[0];- if (fp->headers.extra.num > 0)- return &fp->headers.extra.list[0];- return 0;-}--static inline int is_authorization(const char *hdr) {- return (strncasecmp("authorization:", hdr, 14) == 0);-}--static int add_callback_headers(hFILE_libcurl *fp) {- char **hdrs = NULL, **hdr;-- if (!fp->headers.callback)- return 0;-- // Get the headers from the callback- if (fp->headers.callback(fp->headers.callback_data, &hdrs) != 0) {- return -1;- }-- if (!hdrs) // No change- return 0;-- // Remove any old callback headers- if (fp->headers.fixed.num > 0) {- // Unlink lists- fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL;- }- free_headers(&fp->headers.extra, 0);-- if (fp->headers.auth_hdr_num > 0 || fp->headers.auth_hdr_num == -2)- fp->headers.auth_hdr_num = 0; // Just removed it...-- // Convert to libcurl-suitable form- for (hdr = hdrs; *hdr; hdr++) {- if (append_header(&fp->headers.extra, *hdr, 0) < 0) {- goto cleanup;- }- if (is_authorization(*hdr) && !fp->headers.auth_hdr_num)- fp->headers.auth_hdr_num = -2;- }- for (hdr = hdrs; *hdr; hdr++) *hdr = NULL;-- if (fp->headers.fixed.num > 0 && fp->headers.extra.num > 0) {- // Relink lists- fp->headers.fixed.list[fp->headers.fixed.num - 1].next- = &fp->headers.extra.list[0];- }- return 0;-- cleanup:- while (hdr && *hdr) {- free(*hdr);- *hdr = NULL;- }- return -1;-}--/*- * Read an OAUTH2-style Bearer access token (see- * https://tools.ietf.org/html/rfc6750#section-4).- * Returns 'v' for valid; 'i' for invalid (token missing or wrong sort);- * '?' for a JSON parse error; 'm' if it runs out of memory.- */-static int read_auth_json(auth_token *tok, hFILE *auth_fp) {- hts_json_token *t = hts_json_alloc_token();- kstring_t str = {0, 0, NULL};- char *token = NULL, *type = NULL, *expiry = NULL;- int ret = 'i';-- if (!t) goto error;-- if ((ret = hts_json_fnext(auth_fp, t, &str)) != '{') goto error;- while (hts_json_fnext(auth_fp, t, &str) != '}') {- char *key;- if (hts_json_token_type(t) != 's') {- ret = '?';- goto error;- }- key = hts_json_token_str(t);- if (!key) goto error;- if (strcmp(key, "access_token") == 0) {- if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error;- token = ks_release(&str);- } else if (strcmp(key, "token_type") == 0) {- if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error;- type = ks_release(&str);- } else if (strcmp(key, "expires_in") == 0) {- if ((ret = hts_json_fnext(auth_fp, t, &str)) != 'n') goto error;- expiry = ks_release(&str);- } else if (hts_json_fskip_value(auth_fp, '\0') != 'v') {- ret = '?';- goto error;- }- }-- if (!token || (type && strcmp(type, "Bearer") != 0)) {- ret = 'i';- goto error;- }-- ret = 'm';- str.l = 0;- if (kputs("Authorization: Bearer ", &str) < 0) goto error;- if (kputs(token, &str) < 0) goto error;- free(tok->token);- tok->token = ks_release(&str);- if (expiry) {- long exp = strtol(expiry, NULL, 10);- if (exp < 0) exp = 0;- tok->expiry = time(NULL) + exp;- } else {- tok->expiry = 0;- }- ret = 'v';-- error:- free(token);- free(type);- free(expiry);- free(str.s);- hts_json_free_token(t);- return ret;-}--static int read_auth_plain(auth_token *tok, hFILE *auth_fp) {- kstring_t line = {0, 0, NULL};- kstring_t token = {0, 0, NULL};- const char *start, *end;-- if (kgetline(&line, (char * (*)(char *, int, void *)) hgets, auth_fp) < 0) goto error;- if (kputc('\0', &line) < 0) goto error;-- for (start = line.s; *start && isspace_c(*start); start++) {}- for (end = start; *end && !isspace_c(*end); end++) {}-- if (end > start) {- if (kputs("Authorization: Bearer ", &token) < 0) goto error;- if (kputsn(start, end - start, &token) < 0) goto error;- }-- free(tok->token);- tok->token = ks_release(&token);- tok->expiry = 0;- free(line.s);- return 0;-- error:- free(line.s);- free(token.s);- return -1;-}--static int renew_auth_token(auth_token *tok, int *changed) {- hFILE *auth_fp = NULL;- char buffer[16];- ssize_t len;-- *changed = 0;- if (tok->expiry == 0 || time(NULL) + AUTH_REFRESH_EARLY_SECS < tok->expiry)- return 0; // Still valid-- if (tok->failed)- return -1;-- *changed = 1;- auth_fp = hopen(tok->path, "rR");- if (!auth_fp) {- // Not worried about missing files; other errors are bad.- if (errno != ENOENT)- goto fail;-- tok->expiry = 0; // Prevent retry- free(tok->token); // Just in case it was set- return 0;- }-- len = hpeek(auth_fp, buffer, sizeof(buffer));- if (len < 0)- goto fail;-- if (memchr(buffer, '{', len) != NULL) {- if (read_auth_json(tok, auth_fp) != 'v')- goto fail;- } else {- if (read_auth_plain(tok, auth_fp) < 0)- goto fail;- }-- return hclose(auth_fp) < 0 ? -1 : 0;-- fail:- tok->failed = 1;- if (auth_fp) hclose_abruptly(auth_fp);- return -1;-}--static int add_auth_header(hFILE_libcurl *fp) {- int changed = 0;-- if (fp->headers.auth_hdr_num < 0)- return 0; // Have an Authorization header from open or header callback-- if (!fp->headers.auth)- return 0; // Nothing to add-- pthread_mutex_lock(&fp->headers.auth->lock);- if (renew_auth_token(fp->headers.auth, &changed) < 0)- goto unlock_fail;-- if (!changed && fp->headers.auth_hdr_num > 0) {- pthread_mutex_unlock(&fp->headers.auth->lock);- return 0;- }-- if (fp->headers.auth_hdr_num > 0) {- // Had a previous header, so swap in the new one- char *header = fp->headers.auth->token;- char *header_copy = header ? strdup(header) : NULL;- int idx = fp->headers.auth_hdr_num - 1;- if (header && !header_copy)- goto unlock_fail;-- if (header_copy) {- free(fp->headers.extra.list[idx].data);- fp->headers.extra.list[idx].data = header_copy;- } else {- unsigned int j;- // More complicated case - need to get rid of the old header- // and tidy up linked lists- free(fp->headers.extra.list[idx].data);- for (j = idx + 1; j < fp->headers.extra.num; j++) {- fp->headers.extra.list[j - 1] = fp->headers.extra.list[j];- fp->headers.extra.list[j - 1].next = &fp->headers.extra.list[j];- }- fp->headers.extra.num--;- if (fp->headers.extra.num > 0) {- fp->headers.extra.list[fp->headers.extra.num-1].next = NULL;- } else if (fp->headers.fixed.num > 0) {- fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL;- }- fp->headers.auth_hdr_num = 0;- }- } else if (fp->headers.auth->token) {- // Add new header and remember where it is- if (append_header(&fp->headers.extra,- fp->headers.auth->token, 1) < 0) {- goto unlock_fail;- }- fp->headers.auth_hdr_num = fp->headers.extra.num;- }-- pthread_mutex_unlock(&fp->headers.auth->lock);- return 0;-- unlock_fail:- pthread_mutex_unlock(&fp->headers.auth->lock);- return -1;-}--static int get_auth_token(hFILE_libcurl *fp, const char *url) {- const char *host = NULL, *p, *q;- kstring_t name = {0, 0, NULL};- size_t host_len = 0;- khiter_t idx;- auth_token *tok = NULL;-- // Nothing to do if:- // curl.auth_path has not been set- // fp was made by hfile_libcurl (e.g. auth_path is a http:// url)- // we already have an Authorization header- if (!curl.auth_path || fp->is_recursive || fp->headers.auth_hdr_num != 0)- return 0;-- // Insist on having a secure connection unless the user insists harder- if (!curl.allow_unencrypted_auth_header && strncmp(url, "https://", 8) != 0)- return 0;-- host = strstr(url, "://");- if (host) {- host += 3;- host_len = strcspn(host, "/");- }-- p = curl.auth_path;- while ((q = strstr(p, "%h")) != NULL) {- if (q - p > INT_MAX || host_len > INT_MAX) goto error;- if (kputsn_(p, q - p, &name) < 0) goto error;- if (kputsn_(host, host_len, &name) < 0) goto error;- p = q + 2;- }- if (kputs(p, &name) < 0) goto error;-- pthread_mutex_lock(&curl.auth_lock);- idx = kh_get(auth_map, curl.auth_map, name.s);- if (idx < kh_end(curl.auth_map)) {- tok = kh_value(curl.auth_map, idx);- } else {- tok = calloc(1, sizeof(*tok));- if (tok && pthread_mutex_init(&tok->lock, NULL) != 0) {- free(tok);- tok = NULL;- }- if (tok) {- int ret = -1;- tok->path = ks_release(&name);- tok->token = NULL;- tok->expiry = 1; // Force refresh- idx = kh_put(auth_map, curl.auth_map, tok->path, &ret);- if (ret < 0) {- free_auth(tok);- tok = NULL;- }- kh_value(curl.auth_map, idx) = tok;- }- }- pthread_mutex_unlock(&curl.auth_lock);-- fp->headers.auth = tok;- free(name.s);-- return add_auth_header(fp);-- error:- free(name.s);- return -1;-}--static void process_messages(hFILE_libcurl *fp)-{- CURLMsg *msg;- int remaining;-- while ((msg = curl_multi_info_read(fp->multi, &remaining)) != NULL) {- switch (msg->msg) {- case CURLMSG_DONE:- fp->finished = 1;- fp->final_result = msg->data.result;- break;-- default:- break;- }- }-}--static int wait_perform(hFILE_libcurl *fp)-{- fd_set rd, wr, ex;- int maxfd, nrunning;- long timeout;- CURLMcode errm;-- if (!fp->perform_again) {- FD_ZERO(&rd);- FD_ZERO(&wr);- FD_ZERO(&ex);- if (curl_multi_fdset(fp->multi, &rd, &wr, &ex, &maxfd) != CURLM_OK)- maxfd = -1, timeout = 1000;- else {- if (curl_multi_timeout(fp->multi, &timeout) != CURLM_OK)- timeout = 1000;- else if (timeout < 0) {- timeout = 10000; // as recommended by curl_multi_timeout(3)- }- }- if (maxfd < 0 && timeout > 100)- timeout = 100; // as recommended by curl_multi_fdset(3)-- if (timeout > 0) {- struct timeval tval;- tval.tv_sec = (timeout / 1000);- tval.tv_usec = (timeout % 1000) * 1000;-- if (select(maxfd + 1, &rd, &wr, &ex, &tval) < 0) return -1;- }- }-- errm = curl_multi_perform(fp->multi, &nrunning);- fp->perform_again = 0;- if (errm == CURLM_CALL_MULTI_PERFORM) fp->perform_again = 1;- else if (errm != CURLM_OK) { errno = multi_errno(errm); return -1; }-- if (nrunning < fp->nrunning) process_messages(fp);- return 0;-}---static size_t recv_callback(char *ptr, size_t size, size_t nmemb, void *fpv)-{- hFILE_libcurl *fp = (hFILE_libcurl *) fpv;- size_t n = size * nmemb;-- if (n > fp->buffer.len) { fp->paused = 1; return CURL_WRITEFUNC_PAUSE; }- else if (n == 0) return 0;-- memcpy(fp->buffer.ptr.rd, ptr, n);- fp->buffer.ptr.rd += n;- fp->buffer.len -= n;- return n;-}--static ssize_t libcurl_read(hFILE *fpv, void *bufferv, size_t nbytes)-{- hFILE_libcurl *fp = (hFILE_libcurl *) fpv;- char *buffer = (char *) bufferv;- off_t to_skip = -1;- ssize_t got = 0;- CURLcode err;-- if (fp->delayed_seek >= 0) {- assert(fp->base.offset == fp->delayed_seek- && fp->base.begin == fp->base.buffer- && fp->base.end == fp->base.buffer);-- if (fp->last_offset >= 0- && fp->delayed_seek > fp->last_offset- && fp->delayed_seek - fp->last_offset < MIN_SEEK_FORWARD) {- // If not seeking far, just read the data and throw it away. This- // is likely to be quicker than opening a new stream- to_skip = fp->delayed_seek - fp->last_offset;- } else {- if (restart_from_position(fp, fp->delayed_seek) < 0) {- return -1;- }- }- fp->delayed_seek = -1;- fp->last_offset = -1;- }-- do {- fp->buffer.ptr.rd = buffer;- fp->buffer.len = nbytes;- fp->paused = 0;- err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);- if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; }-- while (! fp->paused && ! fp->finished)- if (wait_perform(fp) < 0) return -1;-- got = fp->buffer.ptr.rd - buffer;-- if (to_skip >= 0) { // Skipping over a small seek- if (got < to_skip) { // Need to skip more data- to_skip -= got;- } else {- got -= to_skip;- if (got > 0) { // If enough was skipped, return the rest- memmove(buffer, buffer + to_skip, got);- to_skip = -1;- }- }- }- } while (to_skip >= 0 && ! fp->finished);- fp->buffer.ptr.rd = NULL;- fp->buffer.len = 0;-- if (fp->finished && fp->final_result != CURLE_OK) {- errno = easy_errno(fp->easy, fp->final_result);- return -1;- }-- return got;-}--static size_t send_callback(char *ptr, size_t size, size_t nmemb, void *fpv)-{- hFILE_libcurl *fp = (hFILE_libcurl *) fpv;- size_t n = size * nmemb;-- if (fp->buffer.len == 0) {- // Send buffer is empty; normally pause, or signal EOF if we're closing- if (fp->closing) return 0;- else { fp->paused = 1; return CURL_READFUNC_PAUSE; }- }-- if (n > fp->buffer.len) n = fp->buffer.len;- memcpy(ptr, fp->buffer.ptr.wr, n);- fp->buffer.ptr.wr += n;- fp->buffer.len -= n;- return n;-}--static ssize_t libcurl_write(hFILE *fpv, const void *bufferv, size_t nbytes)-{- hFILE_libcurl *fp = (hFILE_libcurl *) fpv;- const char *buffer = (const char *) bufferv;- CURLcode err;-- fp->buffer.ptr.wr = buffer;- fp->buffer.len = nbytes;- fp->paused = 0;- err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);- if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; }-- while (! fp->paused && ! fp->finished)- if (wait_perform(fp) < 0) return -1;-- nbytes = fp->buffer.ptr.wr - buffer;- fp->buffer.ptr.wr = NULL;- fp->buffer.len = 0;-- if (fp->finished && fp->final_result != CURLE_OK) {- errno = easy_errno(fp->easy, fp->final_result);- return -1;- }-- return nbytes;-}--static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence)-{- hFILE_libcurl *fp = (hFILE_libcurl *) fpv;- off_t origin, pos;-- if (!fp->is_read || !fp->can_seek) {- // Cowardly refuse to seek when writing or a previous seek failed.- errno = ESPIPE;- return -1;- }-- switch (whence) {- case SEEK_SET:- origin = 0;- break;- case SEEK_CUR:- errno = ENOSYS;- return -1;- case SEEK_END:- if (fp->file_size < 0) { errno = ESPIPE; return -1; }- origin = fp->file_size;- break;- default:- errno = EINVAL;- return -1;- }-- // Check 0 <= origin+offset < fp->file_size carefully, avoiding overflow- if ((offset < 0)? origin + offset < 0- : (fp->file_size >= 0 && offset > fp->file_size - origin)) {- errno = EINVAL;- return -1;- }-- pos = origin + offset;-- if (fp->tried_seek) {- /* Seeking has worked at least once, so now we can delay doing- the actual work until the next read. This avoids lots of pointless- http or ftp reconnections if the caller does lots of seeks- without any intervening reads. */- if (fp->delayed_seek < 0) {- fp->last_offset = fp->base.offset + (fp->base.end - fp->base.buffer);- }- fp->delayed_seek = pos;- return pos;- }-- if (restart_from_position(fp, pos) < 0) {- /* This value for errno may not be entirely true, but the caller may be- able to carry on with the existing handle. */- errno = ESPIPE;- return -1;- }-- fp->tried_seek = 1;- return pos;-}--static int restart_from_position(hFILE_libcurl *fp, off_t pos) {- hFILE_libcurl temp_fp;- CURLcode err;- CURLMcode errm;- int update_headers = 0;- int save_errno = 0;-- // TODO If we seem to be doing random access, use CURLOPT_RANGE to do- // limited reads (e.g. about a BAM block!) so seeking can reuse the- // existing connection more often.-- // Get new headers from the callback (if defined). This changes the- // headers in fp before it gets duplicated, but they should be have been- // sent by now.-- if (fp->headers.callback) {- if (add_callback_headers(fp) != 0)- return -1;- update_headers = 1;- }- if (fp->headers.auth_hdr_num > 0 && fp->headers.auth) {- if (add_auth_header(fp) != 0)- return -1;- update_headers = 1;- }- if (update_headers) {- struct curl_slist *list = get_header_list(fp);- if (list) {- err = curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list);- if (err != CURLE_OK) {- errno = easy_errno(fp->easy,err);- return -1;- }- }- }-- /*- Duplicate the easy handle, and use CURLOPT_RESUME_FROM_LARGE to open- a new request to the server, reading from the location that we want- to seek to. If the new request works and returns the correct data,- the original easy handle in *fp is closed and replaced with the new- one. If not, we close the new handle and leave *fp unchanged.- */-- memcpy(&temp_fp, fp, sizeof(temp_fp));- temp_fp.buffer.len = 0;- temp_fp.buffer.ptr.rd = NULL;- temp_fp.easy = curl_easy_duphandle(fp->easy);- if (!temp_fp.easy)- goto early_error;-- err = curl_easy_setopt(temp_fp.easy, CURLOPT_RESUME_FROM_LARGE,(curl_off_t)pos);- err |= curl_easy_setopt(temp_fp.easy, CURLOPT_PRIVATE, &temp_fp);- err |= curl_easy_setopt(temp_fp.easy, CURLOPT_WRITEDATA, &temp_fp);- if (err != CURLE_OK) {- save_errno = easy_errno(temp_fp.easy, err);- goto error;- }-- temp_fp.buffer.len = 0; // Ensures we only read the response headers- temp_fp.paused = temp_fp.finished = 0;-- // fp->multi and temp_fp.multi are the same.- errm = curl_multi_add_handle(fp->multi, temp_fp.easy);- if (errm != CURLM_OK) {- save_errno = multi_errno(errm);- goto error;- }- temp_fp.nrunning = ++fp->nrunning;-- err = curl_easy_pause(temp_fp.easy, CURLPAUSE_CONT);- if (err != CURLE_OK) {- save_errno = easy_errno(temp_fp.easy, err);- goto error_remove;- }-- while (! temp_fp.paused && ! temp_fp.finished)- if (wait_perform(&temp_fp) < 0) {- save_errno = errno;- goto error_remove;- }-- if (temp_fp.finished && temp_fp.final_result != CURLE_OK) {- save_errno = easy_errno(temp_fp.easy, temp_fp.final_result);- goto error_remove;- }-- // We've got a good response, close the original connection and- // replace it with the new one.-- errm = curl_multi_remove_handle(fp->multi, fp->easy);- if (errm != CURLM_OK) {- // Clean up as much as possible- curl_easy_reset(temp_fp.easy);- if (curl_multi_remove_handle(fp->multi, temp_fp.easy) == CURLM_OK) {- fp->nrunning--;- curl_easy_cleanup(temp_fp.easy);- }- save_errno = multi_errno(errm);- goto early_error;- }- fp->nrunning--;-- curl_easy_cleanup(fp->easy);- fp->easy = temp_fp.easy;- err = curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp);- err |= curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp);- if (err != CURLE_OK) {- save_errno = easy_errno(fp->easy, err);- curl_easy_reset(fp->easy);- errno = save_errno;- return -1;- }- fp->buffer.len = 0;- fp->paused = temp_fp.paused;- fp->finished = temp_fp.finished;- fp->perform_again = temp_fp.perform_again;- fp->final_result = temp_fp.final_result;-- return 0;-- error_remove:- curl_easy_reset(temp_fp.easy); // Ensure no pointers to on-stack temp_fp- errm = curl_multi_remove_handle(fp->multi, temp_fp.easy);- if (errm != CURLM_OK) {- errno = multi_errno(errm);- return -1;- }- fp->nrunning--;- error:- curl_easy_cleanup(temp_fp.easy);- early_error:- fp->can_seek = 0; // Don't try to seek again- if (save_errno)- errno = save_errno;- return -1;-}--static int libcurl_close(hFILE *fpv)-{- hFILE_libcurl *fp = (hFILE_libcurl *) fpv;- CURLcode err;- CURLMcode errm;- int save_errno = 0;-- // Before closing the file, unpause it and perform on it so that uploads- // have the opportunity to signal EOF to the server -- see send_callback().-- fp->buffer.len = 0;- fp->closing = 1;- fp->paused = 0;- err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);- if (err != CURLE_OK) save_errno = easy_errno(fp->easy, err);-- while (save_errno == 0 && ! fp->paused && ! fp->finished)- if (wait_perform(fp) < 0) save_errno = errno;-- if (fp->finished && fp->final_result != CURLE_OK)- save_errno = easy_errno(fp->easy, fp->final_result);-- errm = curl_multi_remove_handle(fp->multi, fp->easy);- if (errm != CURLM_OK && save_errno == 0) save_errno = multi_errno(errm);- fp->nrunning--;-- curl_easy_cleanup(fp->easy);- curl_multi_cleanup(fp->multi);-- if (fp->headers.callback) // Tell callback to free any data it needs to- fp->headers.callback(fp->headers.callback_data, NULL);- free_headers(&fp->headers.fixed, 1);- free_headers(&fp->headers.extra, 1);-- if (save_errno) { errno = save_errno; return -1; }- else return 0;-}--static const struct hFILE_backend libcurl_backend =-{- libcurl_read, libcurl_write, libcurl_seek, NULL, libcurl_close-};--static hFILE *-libcurl_open(const char *url, const char *modes, http_headers *headers)-{- hFILE_libcurl *fp;- struct curl_slist *list;- char mode;- const char *s;- CURLcode err;- CURLMcode errm;- int save, is_recursive;-- is_recursive = strchr(modes, 'R') != NULL;-- if ((s = strpbrk(modes, "rwa+")) != NULL) {- mode = *s;- if (strpbrk(&s[1], "rwa+")) mode = 'e';- }- else mode = '\0';-- if (mode != 'r' && mode != 'w') { errno = EINVAL; goto early_error; }-- fp = (hFILE_libcurl *) hfile_init(sizeof (hFILE_libcurl), modes, 0);- if (fp == NULL) goto early_error;-- if (headers) {- fp->headers = *headers;- } else {- memset(&fp->headers, 0, sizeof(fp->headers));- }-- fp->file_size = -1;- fp->buffer.ptr.rd = NULL;- fp->buffer.len = 0;- fp->final_result = (CURLcode) -1;- fp->paused = fp->closing = fp->finished = fp->perform_again = 0;- fp->can_seek = 1;- fp->tried_seek = 0;- fp->delayed_seek = fp->last_offset = -1;- fp->is_recursive = is_recursive;- fp->nrunning = 0;- fp->easy = NULL;-- fp->multi = curl_multi_init();- if (fp->multi == NULL) { errno = ENOMEM; goto error; }-- fp->easy = curl_easy_init();- if (fp->easy == NULL) { errno = ENOMEM; goto error; }-- // Make a route to the hFILE_libcurl* given just a CURL* easy handle- err = curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp);-- // Avoid many repeated CWD calls with FTP, instead requesting the filename- // by full path (as done in knet, but not strictly compliant with RFC1738).- err |= curl_easy_setopt(fp->easy, CURLOPT_FTP_FILEMETHOD, CURLFTPMETHOD_NOCWD);-- if (mode == 'r') {- err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEFUNCTION, recv_callback);- err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp);- fp->is_read = 1;- }- else {- err |= curl_easy_setopt(fp->easy, CURLOPT_READFUNCTION, send_callback);- err |= curl_easy_setopt(fp->easy, CURLOPT_READDATA, fp);- err |= curl_easy_setopt(fp->easy, CURLOPT_UPLOAD, 1L);- if (append_header(&fp->headers.fixed,- "Transfer-Encoding: chunked", 1) < 0)- goto error;- fp->is_read = 0;- }-- err |= curl_easy_setopt(fp->easy, CURLOPT_SHARE, curl.share);- err |= curl_easy_setopt(fp->easy, CURLOPT_URL, url);- {- char* env_curl_ca_bundle = getenv("CURL_CA_BUNDLE");- if (env_curl_ca_bundle) {- err |= curl_easy_setopt(fp->easy, CURLOPT_CAINFO, env_curl_ca_bundle);- }- }- err |= curl_easy_setopt(fp->easy, CURLOPT_USERAGENT, curl.useragent.s);- if (fp->headers.callback) {- if (add_callback_headers(fp) != 0) goto error;- }- if (get_auth_token(fp, url) < 0)- goto error;- if ((list = get_header_list(fp)) != NULL)- err |= curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list);- err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L);- if (hts_verbose <= 8)- err |= curl_easy_setopt(fp->easy, CURLOPT_FAILONERROR, 1L);- if (hts_verbose >= 8)- err |= curl_easy_setopt(fp->easy, CURLOPT_VERBOSE, 1L);-- if (err != 0) { errno = ENOSYS; goto error; }-- errm = curl_multi_add_handle(fp->multi, fp->easy);- if (errm != CURLM_OK) { errno = multi_errno(errm); goto error; }- fp->nrunning++;-- while (! fp->paused && ! fp->finished)- if (wait_perform(fp) < 0) goto error_remove;-- if (fp->finished && fp->final_result != CURLE_OK) {- errno = easy_errno(fp->easy, fp->final_result);- goto error_remove;- }-- if (mode == 'r') {- double dval;- if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD,- &dval) == CURLE_OK && dval >= 0.0)- fp->file_size = (off_t) (dval + 0.1);- }-- fp->base.backend = &libcurl_backend;- return &fp->base;--error_remove:- save = errno;- (void) curl_multi_remove_handle(fp->multi, fp->easy);- fp->nrunning--;- errno = save;--error:- save = errno;- if (fp->easy) curl_easy_cleanup(fp->easy);- if (fp->multi) curl_multi_cleanup(fp->multi);- free_headers(&fp->headers.extra, 1);- hfile_destroy((hFILE *) fp);- errno = save;- return NULL;--early_error:- return NULL;-}--static hFILE *hopen_libcurl(const char *url, const char *modes)-{- return libcurl_open(url, modes, NULL);-}--static int parse_va_list(http_headers *headers, va_list args)-{- const char *argtype;-- while ((argtype = va_arg(args, const char *)) != NULL)- if (strcmp(argtype, "httphdr:v") == 0) {- const char **hdr;- for (hdr = va_arg(args, const char **); *hdr; hdr++) {- if (append_header(&headers->fixed, *hdr, 1) < 0)- return -1;- if (is_authorization(*hdr))- headers->auth_hdr_num = -1;- }- }- else if (strcmp(argtype, "httphdr:l") == 0) {- const char *hdr;- while ((hdr = va_arg(args, const char *)) != NULL) {- if (append_header(&headers->fixed, hdr, 1) < 0)- return -1;- if (is_authorization(hdr))- headers->auth_hdr_num = -1;- }- }- else if (strcmp(argtype, "httphdr") == 0) {- const char *hdr = va_arg(args, const char *);- if (hdr) {- if (append_header(&headers->fixed, hdr, 1) < 0)- return -1;- if (is_authorization(hdr))- headers->auth_hdr_num = -1;- }- }- else if (strcmp(argtype, "httphdr_callback") == 0) {- headers->callback = va_arg(args, const hts_httphdr_callback);- }- else if (strcmp(argtype, "httphdr_callback_data") == 0) {- headers->callback_data = va_arg(args, void *);- }- else if (strcmp(argtype, "va_list") == 0) {- va_list *args2 = va_arg(args, va_list *);- if (args2) {- if (parse_va_list(headers, *args2) < 0) return -1;- }- }- else if (strcmp(argtype, "auth_token_enabled") == 0) {- const char *flag = va_arg(args, const char *);- if (strcmp(flag, "false") == 0)- headers->auth_hdr_num = -3;- }- else { errno = EINVAL; return -1; }-- return 0;-}--/*- HTTP headers to be added to the request can be passed in as extra- arguments to hopen(). The headers can be specified as follows:-- * Single header:- hopen(url, mode, "httphdr", "X-Hdr-1: text", NULL);-- * Multiple headers in the argument list:- hopen(url, mode, "httphdr:l", "X-Hdr-1: text", "X-Hdr-2: text", NULL, NULL);-- * Multiple headers in a char* array:- hopen(url, mode, "httphdr:v", hdrs, NULL);- where `hdrs` is a char **. The list ends with a NULL pointer.-- * A callback function- hopen(url, mode, "httphdr_callback", func,- "httphdr_callback_data", arg, NULL);- `func` has type- int (* hts_httphdr_callback) (void *cb_data, char ***hdrs);- `arg` is passed to the callback as a void *.-- The function is called at file open, and when attempting to seek (which- opens a new HTTP request). This allows, for example, access tokens- that may have gone stale to be regenerated. The function is also- called (with `hdrs` == NULL) on file close so that the callback can- free any memory that it needs to.-- The callback should return 0 on success, non-zero on failure. It should- return in *hdrs a list of strings containing the new headers (terminated- with a NULL pointer). These will replace any headers previously supplied- by the callback. If no changes are necessary, it can return NULL- in *hdrs, in which case the previous headers will be left unchanged.-- Ownership of the strings in the header list passes to hfile_libcurl,- so the callback should not attempt to use or free them itself. The memory- containing the array belongs to the callback and will not be freed by- hfile_libcurl.-- Headers supplied by the callback are appended after any specified- using the "httphdr", "httphdr:l" or "httphdr:v" methods. No attempt- is made to replace these headers (even if a key is repeated) so anything- that is expected to vary needs to come from the callback.- */--static hFILE *vhopen_libcurl(const char *url, const char *modes, va_list args)-{- hFILE *fp = NULL;- http_headers headers = { { NULL, 0, 0 }, { NULL, 0, 0 }, NULL, NULL };- if (parse_va_list(&headers, args) == 0) {- fp = libcurl_open(url, modes, &headers);- }-- if (!fp) {- free_headers(&headers.fixed, 1);- }- return fp;-}--int PLUGIN_GLOBAL(hfile_plugin_init,_libcurl)(struct hFILE_plugin *self)-{- static const struct hFILE_scheme_handler handler =- { hopen_libcurl, hfile_always_remote, "libcurl",- 2000 + 50,- vhopen_libcurl };--#ifdef ENABLE_PLUGINS- // Embed version string for examination via strings(1) or what(1)- static const char id[] = "@(#)hfile_libcurl plugin (htslib)\t" HTS_VERSION;- const char *version = strchr(id, '\t')+1;-#else- const char *version = hts_version();-#endif- const curl_version_info_data *info;- const char * const *protocol;- const char *auth;- CURLcode err;- CURLSHcode errsh;-- err = curl_global_init(CURL_GLOBAL_ALL);- if (err != CURLE_OK) { errno = easy_errno(NULL, err); return -1; }-- curl.share = curl_share_init();- if (curl.share == NULL) { curl_global_cleanup(); errno = EIO; return -1; }- errsh = curl_share_setopt(curl.share, CURLSHOPT_LOCKFUNC, share_lock);- errsh |= curl_share_setopt(curl.share, CURLSHOPT_UNLOCKFUNC, share_unlock);- errsh |= curl_share_setopt(curl.share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);- if (errsh != 0) {- curl_share_cleanup(curl.share);- curl_global_cleanup();- errno = EIO;- return -1;- }-- if ((auth = getenv("HTS_AUTH_LOCATION")) != NULL) {- curl.auth_path = strdup(auth);- curl.auth_map = kh_init(auth_map);- if (!curl.auth_path || !curl.auth_map) {- int save_errno = errno;- free(curl.auth_path);- kh_destroy(auth_map, curl.auth_map);- curl_share_cleanup(curl.share);- curl_global_cleanup();- errno = save_errno;- return -1;- }- }- if ((auth = getenv("HTS_ALLOW_UNENCRYPTED_AUTHORIZATION_HEADER")) != NULL- && strcmp(auth, "I understand the risks") == 0) {- curl.allow_unencrypted_auth_header = 1;- }-- info = curl_version_info(CURLVERSION_NOW);- ksprintf(&curl.useragent, "htslib/%s libcurl/%s", version, info->version);-- self->name = "libcurl";- self->destroy = libcurl_exit;-- for (protocol = info->protocols; *protocol; protocol++)- hfile_add_scheme_handler(*protocol, &handler);- return 0;-}
− htslib-1.9/hfile_s3.c
@@ -1,466 +0,0 @@-/* hfile_s3.c -- Amazon S3 backend for low-level file streams.-- Copyright (C) 2015-2017 Genome Research Ltd.-- Author: John Marshall <jm18@sanger.ac.uk>--Permission is hereby granted, free of charge, to any person obtaining a copy-of this software and associated documentation files (the "Software"), to deal-in the Software without restriction, including without limitation the rights-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-copies of the Software, and to permit persons to whom the Software is-furnished to do so, subject to the following conditions:--The above copyright notice and this permission notice shall be included in-all copies or substantial portions of the Software.--THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER-DEALINGS IN THE SOFTWARE. */--#include <config.h>--#include <stdarg.h>-#include <stdio.h>-#include <stdlib.h>-#include <string.h>-#include <time.h>--#include "hfile_internal.h"-#ifdef ENABLE_PLUGINS-#include "version.h"-#endif-#include "htslib/hts.h" // for hts_version() and hts_verbose-#include "htslib/kstring.h"--typedef struct {- kstring_t id;- kstring_t token;- kstring_t secret;- char *bucket;- kstring_t auth_hdr;- time_t auth_time;- char date[40];- char mode;- char *headers[3];-} s3_auth_data;--#define AUTH_LIFETIME 60--#if defined HAVE_COMMONCRYPTO--#include <CommonCrypto/CommonHMAC.h>--#define DIGEST_BUFSIZ CC_SHA1_DIGEST_LENGTH--static size_t-s3_sign(unsigned char *digest, kstring_t *key, kstring_t *message)-{- CCHmac(kCCHmacAlgSHA1, key->s, key->l, message->s, message->l, digest);- return CC_SHA1_DIGEST_LENGTH;-}--#elif defined HAVE_HMAC--#include <openssl/hmac.h>--#define DIGEST_BUFSIZ EVP_MAX_MD_SIZE--static size_t-s3_sign(unsigned char *digest, kstring_t *key, kstring_t *message)-{- unsigned int len;- HMAC(EVP_sha1(), key->s, key->l,- (unsigned char *) message->s, message->l, digest, &len);- return len;-}--#else-#error No HMAC() routine found by configure-#endif--static void-urldecode_kput(const char *s, int len, kstring_t *str)-{- char buf[3];- int i = 0;-- while (i < len)- if (s[i] == '%' && i+2 < len) {- buf[0] = s[i+1], buf[1] = s[i+2], buf[2] = '\0';- kputc(strtol(buf, NULL, 16), str);- i += 3;- }- else kputc(s[i++], str);-}--static void base64_kput(const unsigned char *data, size_t len, kstring_t *str)-{- static const char base64[] =- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";-- size_t i = 0;- unsigned x = 0;- int bits = 0, pad = 0;-- while (bits || i < len) {- if (bits < 6) {- x <<= 8, bits += 8;- if (i < len) x |= data[i++];- else pad++;- }-- bits -= 6;- kputc(base64[(x >> bits) & 63], str);- }-- str->l -= pad;- kputsn("==", pad, str);-}--static int is_dns_compliant(const char *s0, const char *slim)-{- int has_nondigit = 0, len = 0;- const char *s;-- for (s = s0; s < slim; len++, s++)- if (islower_c(*s))- has_nondigit = 1;- else if (*s == '-') {- has_nondigit = 1;- if (s == s0 || s+1 == slim) return 0;- }- else if (isdigit_c(*s))- ;- else if (*s == '.') {- if (s == s0 || ! isalnum_c(s[-1])) return 0;- if (s+1 == slim || ! isalnum_c(s[1])) return 0;- }- else return 0;-- return has_nondigit && len >= 3 && len <= 63;-}--static FILE *expand_tilde_open(const char *fname, const char *mode)-{- FILE *fp;-- if (strncmp(fname, "~/", 2) == 0) {- kstring_t full_fname = { 0, 0, NULL };- const char *home = getenv("HOME");- if (! home) return NULL;-- kputs(home, &full_fname);- kputs(&fname[1], &full_fname);-- fp = fopen(full_fname.s, mode);- free(full_fname.s);- }- else- fp = fopen(fname, mode);-- return fp;-}--static void parse_ini(const char *fname, const char *section, ...)-{- kstring_t line = { 0, 0, NULL };- int active = 1; // Start active, so global properties are accepted- char *s;-- FILE *fp = expand_tilde_open(fname, "r");- if (fp == NULL) return;-- while (line.l = 0, kgetline(&line, (kgets_func *) fgets, fp) >= 0)- if (line.s[0] == '[' && (s = strchr(line.s, ']')) != NULL) {- *s = '\0';- active = (strcmp(&line.s[1], section) == 0);- }- else if (active && (s = strpbrk(line.s, ":=")) != NULL) {- const char *key = line.s, *value = &s[1], *akey;- va_list args;-- while (isspace_c(*key)) key++;- while (s > key && isspace_c(s[-1])) s--;- *s = '\0';-- while (isspace_c(*value)) value++;- while (line.l > 0 && isspace_c(line.s[line.l-1]))- line.s[--line.l] = '\0';-- va_start(args, section);- while ((akey = va_arg(args, const char *)) != NULL) {- kstring_t *avar = va_arg(args, kstring_t *);- if (strcmp(key, akey) == 0) { kputs(value, avar); break; }- }- va_end(args);- }-- fclose(fp);- free(line.s);-}--static void parse_simple(const char *fname, kstring_t *id, kstring_t *secret)-{- kstring_t text = { 0, 0, NULL };- char *s;- size_t len;-- FILE *fp = expand_tilde_open(fname, "r");- if (fp == NULL) return;-- while (kgetline(&text, (kgets_func *) fgets, fp) >= 0)- kputc(' ', &text);- fclose(fp);-- s = text.s;- while (isspace_c(*s)) s++;- kputsn(s, len = strcspn(s, " \t"), id);-- s += len;- while (isspace_c(*s)) s++;- kputsn(s, strcspn(s, " \t"), secret);-- free(text.s);-}--static int copy_auth_headers(s3_auth_data *ad, char ***hdrs) {- char **hdr = &ad->headers[0];- *hdrs = hdr;- *hdr = strdup(ad->date);- if (!*hdr) return -1;- hdr++;- if (ad->auth_hdr.l) {- *hdr = strdup(ad->auth_hdr.s);- if (!*hdr) { free(ad->headers[0]); return -1; }- hdr++;- }- *hdr = NULL;- return 0;-}--static void free_auth_data(s3_auth_data *ad) {- free(ad->id.s);- free(ad->token.s);- free(ad->secret.s);- free(ad->bucket);- free(ad->auth_hdr.s);- free(ad);-}--static int auth_header_callback(void *ctx, char ***hdrs) {- s3_auth_data *ad = (s3_auth_data *) ctx;-- time_t now = time(NULL);-#ifdef HAVE_GMTIME_R- struct tm tm_buffer;- struct tm *tm = gmtime_r(&now, &tm_buffer);-#else- struct tm *tm = gmtime(&now);-#endif- kstring_t message = { 0, 0, NULL };- unsigned char digest[DIGEST_BUFSIZ];- size_t digest_len;-- if (!hdrs) { // Closing connection- free_auth_data(ad);- return 0;- }-- if (now - ad->auth_time < AUTH_LIFETIME) {- // Last auth string should still be valid- *hdrs = NULL;- return 0;- }-- strftime(ad->date, sizeof(ad->date), "Date: %a, %d %b %Y %H:%M:%S GMT", tm);- if (!ad->id.l || !ad->secret.l) {- ad->auth_time = now;- return copy_auth_headers(ad, hdrs);- }-- if (ksprintf(&message, "%s\n\n\n%s\n%s%s%s/%s",- ad->mode == 'r' ? "GET" : "PUT", ad->date + 6,- ad->token.l ? "x-amz-security-token:" : "",- ad->token.l ? ad->token.s : "",- ad->token.l ? "\n" : "",- ad->bucket) < 0) {- return -1;- }-- digest_len = s3_sign(digest, &ad->secret, &message);- ad->auth_hdr.l = 0;- if (ksprintf(&ad->auth_hdr, "Authorization: AWS %s:", ad->id.s) < 0)- goto fail;- base64_kput(digest, digest_len, &ad->auth_hdr);-- free(message.s);- ad->auth_time = now;- return copy_auth_headers(ad, hdrs);-- fail:- free(message.s);- return -1;-}--static hFILE * s3_rewrite(const char *s3url, const char *mode, va_list *argsp)-{- const char *bucket, *path;- char *header_list[4], **header = header_list;-- kstring_t url = { 0, 0, NULL };- kstring_t profile = { 0, 0, NULL };- kstring_t host_base = { 0, 0, NULL };- kstring_t token_hdr = { 0, 0, NULL };-- s3_auth_data *ad = calloc(1, sizeof(*ad));-- if (!ad)- return NULL;- ad->mode = strchr(mode, 'r') ? 'r' : 'w';-- // Our S3 URL format is s3[+SCHEME]://[ID[:SECRET[:TOKEN]]@]BUCKET/PATH-- if (s3url[2] == '+') {- bucket = strchr(s3url, ':') + 1;- kputsn(&s3url[3], bucket - &s3url[3], &url);- }- else {- kputs("https:", &url);- bucket = &s3url[3];- }- while (*bucket == '/') kputc(*bucket++, &url);-- path = bucket + strcspn(bucket, "/?#@");- if (*path == '@') {- const char *colon = strpbrk(bucket, ":@");- if (*colon != ':') {- urldecode_kput(bucket, colon - bucket, &profile);- }- else {- const char *colon2 = strpbrk(&colon[1], ":@");- urldecode_kput(bucket, colon - bucket, &ad->id);- urldecode_kput(&colon[1], colon2 - &colon[1], &ad->secret);- if (*colon2 == ':')- urldecode_kput(&colon2[1], path - &colon2[1], &ad->token);- }-- bucket = &path[1];- path = bucket + strcspn(bucket, "/?#");- }- else {- // If the URL has no ID[:SECRET]@, consider environment variables.- const char *v;- if ((v = getenv("AWS_ACCESS_KEY_ID")) != NULL) kputs(v, &ad->id);- if ((v = getenv("AWS_SECRET_ACCESS_KEY")) != NULL) kputs(v, &ad->secret);- if ((v = getenv("AWS_SESSION_TOKEN")) != NULL) kputs(v, &ad->token);-- if ((v = getenv("AWS_DEFAULT_PROFILE")) != NULL) kputs(v, &profile);- else if ((v = getenv("AWS_PROFILE")) != NULL) kputs(v, &profile);- else kputs("default", &profile);- }-- if (ad->id.l == 0) {- const char *v = getenv("AWS_SHARED_CREDENTIALS_FILE");- parse_ini(v? v : "~/.aws/credentials", profile.s,- "aws_access_key_id", &ad->id,- "aws_secret_access_key", &ad->secret,- "aws_session_token", &ad->token, NULL);- }- if (ad->id.l == 0)- parse_ini("~/.s3cfg", profile.s, "access_key", &ad->id,- "secret_key", &ad->secret, "access_token", &ad->token,- "host_base", &host_base, NULL);- if (ad->id.l == 0)- parse_simple("~/.awssecret", &ad->id, &ad->secret);-- if (host_base.l == 0)- kputs("s3.amazonaws.com", &host_base);- // Use virtual hosted-style access if possible, otherwise path-style.- if (is_dns_compliant(bucket, path)) {- kputsn(bucket, path - bucket, &url);- kputc('.', &url);- kputs(host_base.s, &url);- }- else {- kputs(host_base.s, &url);- kputc('/', &url);- kputsn(bucket, path - bucket, &url);- }- kputs(path, &url);-- if (ad->token.l > 0) {- kputs("X-Amz-Security-Token: ", &token_hdr);- kputs(ad->token.s, &token_hdr);- *header++ = token_hdr.s;- }-- ad->bucket = strdup(bucket);- if (!ad->bucket)- goto fail;-- *header = NULL;- hFILE *fp = hopen(url.s, mode, "va_list", argsp, "httphdr:v", header_list,- "httphdr_callback", auth_header_callback,- "httphdr_callback_data", ad, NULL);- if (!fp) goto fail;-- free(url.s);- free(profile.s);- free(host_base.s);- free(token_hdr.s);- return fp;-- fail:- free(url.s);- free(profile.s);- free(host_base.s);- free(token_hdr.s);- free_auth_data(ad);- return NULL;-}--static hFILE *s3_open(const char *url, const char *mode)-{- kstring_t mode_colon = { 0, 0, NULL };- kputs(mode, &mode_colon);- kputc(':', &mode_colon);- hFILE *fp = s3_rewrite(url, mode_colon.s, NULL);- free(mode_colon.s);- return fp;-}--static hFILE *s3_vopen(const char *url, const char *mode_colon, va_list args0)-{- // Need to use va_copy() as we can only take the address of an actual- // va_list object, not that of a parameter whose type may have decayed.- va_list args;- va_copy(args, args0);- hFILE *fp = s3_rewrite(url, mode_colon, &args);- va_end(args);- return fp;-}--int PLUGIN_GLOBAL(hfile_plugin_init,_s3)(struct hFILE_plugin *self)-{- static const struct hFILE_scheme_handler handler =- { s3_open, hfile_always_remote, "Amazon S3", 2000 + 50, s3_vopen- };--#ifdef ENABLE_PLUGINS- // Embed version string for examination via strings(1) or what(1)- static const char id[] = "@(#)hfile_s3 plugin (htslib)\t" HTS_VERSION;- if (hts_verbose >= 9)- fprintf(stderr, "[M::hfile_s3.init] version %s\n", strchr(id, '\t')+1);-#endif-- self->name = "Amazon S3";- hfile_add_scheme_handler("s3", &handler);- hfile_add_scheme_handler("s3+http", &handler);- hfile_add_scheme_handler("s3+https", &handler);- return 0;-}