diff --git a/HsHTSLib.cabal b/HsHTSLib.cabal --- a/HsHTSLib.cabal +++ b/HsHTSLib.cabal @@ -1,5 +1,5 @@ name: HsHTSLib -version: 1.9.0.1 +version: 1.9.1 synopsis: Bindings to htslib. description: This package provides bindings to htslib, a library for processing high throughput DNA sequencing data. @@ -12,6 +12,7 @@ build-type: Simple cabal-version: >=1.10 extra-source-files: + README.md ChangeLog.md htslib-1.9/*.h htslib-1.9/htslib/*.h @@ -29,14 +30,14 @@ Bio.HTS.Types build-depends: - base >=4.8 && <5.0 + base >= 4.10 && < 5.0 , bytestring , bytestring-lexing - , conduit >=1.3.0 + , conduit >= 1.3.0 hs-source-dirs: src - build-tools: c2hs >=0.25.0 - extra-libraries: pthread z curl bz2 lzma crypto + build-tools: c2hs >= 0.25.0 + extra-libraries: pthread z include-dirs: htslib-1.9 c-sources: cbits/hs_htslib.c @@ -47,10 +48,10 @@ htslib-1.9/errmod.c htslib-1.9/faidx.c htslib-1.9/hfile.c - htslib-1.9/hfile_gcs.c - htslib-1.9/hfile_libcurl.c + --htslib-1.9/hfile_gcs.c + --htslib-1.9/hfile_libcurl.c htslib-1.9/hfile_net.c - htslib-1.9/hfile_s3.c + --htslib-1.9/hfile_s3.c htslib-1.9/hts.c -- htslib-1.9/htsfile.c htslib-1.9/hts_os.c diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +htslib configuration +-------------------- + +bz2, lzma, libcurl are disabled as CRAM is not supported by this library. diff --git a/cbits/hs_htslib.c b/cbits/hs_htslib.c --- a/cbits/hs_htslib.c +++ b/cbits/hs_htslib.c @@ -15,7 +15,7 @@ } int bam_is_rev_(bam1_t *b) { - return b->core.flag&BAM_FREVERSE != 0; + return (b->core.flag&BAM_FREVERSE) != 0; } void bam_get_seq_(bam1_t *b, char *res, uint32_t l) { diff --git a/htslib-1.9/config.h b/htslib-1.9/config.h --- a/htslib-1.9/config.h +++ b/htslib-1.9/config.h @@ -12,13 +12,13 @@ accurate PACKAGE_VERSION as computed by the Makefile. */ /* Define if HTSlib should enable GCS support. */ -#define ENABLE_GCS 1 +/* #define ENABLE_GCS 1 /* Define if HTSlib should enable plugins. */ /* #undef ENABLE_PLUGINS */ /* Define if HTSlib should enable S3 support. */ -#define ENABLE_S3 1 +/* #define ENABLE_S3 1 /* Define if you have the Common Crypto library. */ /* #undef HAVE_COMMONCRYPTO */ @@ -48,22 +48,22 @@ #define HAVE_INTTYPES_H 1 /* Define to 1 if you have the `bz2' library (-lbz2). */ -#define HAVE_LIBBZ2 1 +/* #define HAVE_LIBBZ2 1 /* Define if libcurl file access is enabled. */ -#define HAVE_LIBCURL 1 +/* #define HAVE_LIBCURL 1 /* Define if libdeflate is available. */ /* #undef HAVE_LIBDEFLATE */ /* Define to 1 if you have the `lzma' library (-llzma). */ -#define HAVE_LIBLZMA 1 +/* #define HAVE_LIBLZMA 1 /* Define to 1 if you have the `z' library (-lz). */ #define HAVE_LIBZ 1 /* Define to 1 if you have the header file. */ -#define HAVE_LZMA_H 1 +/* #define HAVE_LZMA_H 1 /* Define to 1 if you have the header file. */ #define HAVE_MEMORY_H 1 diff --git a/htslib-1.9/hfile_gcs.c b/htslib-1.9/hfile_gcs.c deleted file mode 100644 --- a/htslib-1.9/hfile_gcs.c +++ /dev/null @@ -1,137 +0,0 @@ -/* hfile_gcs.c -- Google Cloud Storage backend for low-level file streams. - - Copyright (C) 2016 Genome Research Ltd. - - Author: John Marshall - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. */ - -#include - -#include -#include -#include -#include -#include - -#include "htslib/hts.h" -#include "htslib/kstring.h" -#include "hfile_internal.h" -#ifdef ENABLE_PLUGINS -#include "version.h" -#endif - -static hFILE * -gcs_rewrite(const char *gsurl, const char *mode, int mode_has_colon, - va_list *argsp) -{ - const char *bucket, *path, *access_token; - kstring_t mode_colon = { 0, 0, NULL }; - kstring_t url = { 0, 0, NULL }; - kstring_t auth_hdr = { 0, 0, NULL }; - hFILE *fp = NULL; - - // GCS URL format is gs[+SCHEME]://BUCKET/PATH - - if (gsurl[2] == '+') { - bucket = strchr(gsurl, ':') + 1; - kputsn(&gsurl[3], bucket - &gsurl[3], &url); - } - else { - kputs("https:", &url); - bucket = &gsurl[3]; - } - while (*bucket == '/') kputc(*bucket++, &url); - - path = bucket + strcspn(bucket, "/?#"); - - kputsn(bucket, path - bucket, &url); - if (strchr(mode, 'r')) kputs(".storage-download", &url); - else if (strchr(mode, 'w')) kputs(".storage-upload", &url); - else kputs(".storage", &url); - kputs(".googleapis.com", &url); - - kputs(path, &url); - - if (hts_verbose >= 8) - fprintf(stderr, "[M::gcs_open] rewrote URL as %s\n", url.s); - - // TODO Find the access token in a more standard way - access_token = getenv("GCS_OAUTH_TOKEN"); - - if (access_token) { - kputs("Authorization: Bearer ", &auth_hdr); - kputs(access_token, &auth_hdr); - } - - if (argsp || auth_hdr.l > 0 || mode_has_colon) { - if (! mode_has_colon) { - kputs(mode, &mode_colon); - kputc(':', &mode_colon); - mode = mode_colon.s; - } - - fp = hopen(url.s, mode, "va_list", argsp, - "httphdr", (auth_hdr.l > 0)? auth_hdr.s : NULL, NULL); - } - else - fp = hopen(url.s, mode); - - free(mode_colon.s); - free(url.s); - free(auth_hdr.s); - return fp; -} - -static hFILE *gcs_open(const char *url, const char *mode) -{ - return gcs_rewrite(url, mode, 0, NULL); -} - -static hFILE *gcs_vopen(const char *url, const char *mode_colon, va_list args0) -{ - // Need to use va_copy() as we can only take the address of an actual - // va_list object, not that of a parameter as its type may have decayed. - va_list args; - va_copy(args, args0); - hFILE *fp = gcs_rewrite(url, mode_colon, 1, &args); - va_end(args); - return fp; -} - -int PLUGIN_GLOBAL(hfile_plugin_init,_gcs)(struct hFILE_plugin *self) -{ - static const struct hFILE_scheme_handler handler = - { gcs_open, hfile_always_remote, "Google Cloud Storage", - 2000 + 50, gcs_vopen - }; - -#ifdef ENABLE_PLUGINS - // Embed version string for examination via strings(1) or what(1) - static const char id[] = "@(#)hfile_gcs plugin (htslib)\t" HTS_VERSION; - if (hts_verbose >= 9) - fprintf(stderr, "[M::hfile_gcs.init] version %s\n", strchr(id, '\t')+1); -#endif - - self->name = "Google Cloud Storage"; - hfile_add_scheme_handler("gs", &handler); - hfile_add_scheme_handler("gs+http", &handler); - hfile_add_scheme_handler("gs+https", &handler); - return 0; -} diff --git a/htslib-1.9/hfile_libcurl.c b/htslib-1.9/hfile_libcurl.c deleted file mode 100644 --- a/htslib-1.9/hfile_libcurl.c +++ /dev/null @@ -1,1393 +0,0 @@ -/* hfile_libcurl.c -- libcurl backend for low-level file streams. - - Copyright (C) 2015-2017 Genome Research Ltd. - - Author: John Marshall - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. */ - -#include - -#include -#include -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include - -#include "hfile_internal.h" -#ifdef ENABLE_PLUGINS -#include "version.h" -#endif -#include "htslib/hts.h" // for hts_version() and hts_verbose -#include "htslib/kstring.h" -#include "htslib/khash.h" - -#include - -// Number of seconds to take off auth_token expiry, to allow for clock skew -// and slow servers -#define AUTH_REFRESH_EARLY_SECS 60 - -// Minimum number of bytes to skip when seeking forward. Seeks less than -// this will just read the data and throw it away. The optimal value -// depends on how long it takes to make a new connection compared -// to how fast the data arrives. -#define MIN_SEEK_FORWARD 1000000 - -typedef struct { - char *path; - char *token; - time_t expiry; - int failed; - pthread_mutex_t lock; -} auth_token; - -// For the authorization header cache -KHASH_MAP_INIT_STR(auth_map, auth_token *) - -// Curl-compatible header linked list -typedef struct { - struct curl_slist *list; - unsigned int num; - unsigned int size; -} hdrlist; - -typedef struct { - hdrlist fixed; // List of headers supplied at hopen() - hdrlist extra; // List of headers from callback - hts_httphdr_callback callback; // Callback to get more headers - void *callback_data; // Data to pass to callback - auth_token *auth; // Authentication token - int auth_hdr_num; // Location of auth_token in hdrlist extra - // If -1, Authorization header is in fixed - // -2, it came from the callback - // -3, "auth_token_enabled", "false" - // passed to hopen() -} http_headers; - -typedef struct { - hFILE base; - CURL *easy; - CURLM *multi; - off_t file_size; - struct { - union { char *rd; const char *wr; } ptr; - size_t len; - } buffer; - CURLcode final_result; // easy result code for finished transfers - // Flags for communicating with libcurl callbacks: - unsigned paused : 1; // callback tells us that it has paused transfer - unsigned closing : 1; // informs callback that hclose() has been invoked - unsigned finished : 1; // wait_perform() tells us transfer is complete - unsigned perform_again : 1; - unsigned is_read : 1; // Opened in read mode - unsigned can_seek : 1; // Can (attempt to) seek on this handle - unsigned is_recursive:1; // Opened by hfile_libcurl itself - unsigned tried_seek : 1; // At least one seek has been attempted - int nrunning; - http_headers headers; - off_t delayed_seek; // Location to seek to before reading - off_t last_offset; // Location we're seeking from -} hFILE_libcurl; - -static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence); -static int restart_from_position(hFILE_libcurl *fp, off_t pos); - -static int http_status_errno(int status) -{ - if (status >= 500) - switch (status) { - case 501: return ENOSYS; - case 503: return EBUSY; - case 504: return ETIMEDOUT; - default: return EIO; - } - else if (status >= 400) - switch (status) { - case 401: return EPERM; - case 403: return EACCES; - case 404: return ENOENT; - case 405: return EROFS; - case 407: return EPERM; - case 408: return ETIMEDOUT; - case 410: return ENOENT; - default: return EINVAL; - } - else return 0; -} - -static int easy_errno(CURL *easy, CURLcode err) -{ - long lval; - - switch (err) { - case CURLE_OK: - return 0; - - case CURLE_UNSUPPORTED_PROTOCOL: - case CURLE_URL_MALFORMAT: - return EINVAL; - -#if LIBCURL_VERSION_NUM >= 0x071505 - case CURLE_NOT_BUILT_IN: - return ENOSYS; -#endif - - case CURLE_COULDNT_RESOLVE_PROXY: - case CURLE_COULDNT_RESOLVE_HOST: - case CURLE_FTP_CANT_GET_HOST: - return EDESTADDRREQ; // Lookup failure - - case CURLE_COULDNT_CONNECT: - case CURLE_SEND_ERROR: - case CURLE_RECV_ERROR: - if (curl_easy_getinfo(easy, CURLINFO_OS_ERRNO, &lval) == CURLE_OK) - return lval; - else - return ECONNABORTED; - - case CURLE_REMOTE_ACCESS_DENIED: - case CURLE_LOGIN_DENIED: - case CURLE_TFTP_PERM: - return EACCES; - - case CURLE_PARTIAL_FILE: - return EPIPE; - - case CURLE_HTTP_RETURNED_ERROR: - if (curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &lval) == CURLE_OK) - return http_status_errno(lval); - else - return EIO; - - case CURLE_OUT_OF_MEMORY: - return ENOMEM; - - case CURLE_OPERATION_TIMEDOUT: - return ETIMEDOUT; - - case CURLE_RANGE_ERROR: - return ESPIPE; - - case CURLE_SSL_CONNECT_ERROR: - // TODO return SSL error buffer messages - return ECONNABORTED; - - case CURLE_FILE_COULDNT_READ_FILE: - case CURLE_TFTP_NOTFOUND: - return ENOENT; - - case CURLE_TOO_MANY_REDIRECTS: - return ELOOP; - - case CURLE_FILESIZE_EXCEEDED: - return EFBIG; - - case CURLE_REMOTE_DISK_FULL: - return ENOSPC; - - case CURLE_REMOTE_FILE_EXISTS: - return EEXIST; - - default: - return EIO; - } -} - -static int multi_errno(CURLMcode errm) -{ - switch (errm) { - case CURLM_CALL_MULTI_PERFORM: - case CURLM_OK: - return 0; - - case CURLM_BAD_HANDLE: - case CURLM_BAD_EASY_HANDLE: - case CURLM_BAD_SOCKET: - return EBADF; - - case CURLM_OUT_OF_MEMORY: - return ENOMEM; - - default: - return EIO; - } -} - -static struct { - kstring_t useragent; - CURLSH *share; - char *auth_path; - khash_t(auth_map) *auth_map; - int allow_unencrypted_auth_header; - pthread_mutex_t auth_lock; - pthread_mutex_t share_lock; -} curl = { { 0, 0, NULL }, NULL, NULL, NULL, 0, PTHREAD_MUTEX_INITIALIZER, - PTHREAD_MUTEX_INITIALIZER }; - -static void share_lock(CURL *handle, curl_lock_data data, - curl_lock_access access, void *userptr) { - pthread_mutex_lock(&curl.share_lock); -} - -static void share_unlock(CURL *handle, curl_lock_data data, void *userptr) { - pthread_mutex_unlock(&curl.share_lock); -} - -static void free_auth(auth_token *tok) { - if (!tok) return; - if (pthread_mutex_destroy(&tok->lock)) abort(); - free(tok->path); - free(tok->token); - free(tok); -} - -static void libcurl_exit() -{ - if (curl_share_cleanup(curl.share) == CURLSHE_OK) - curl.share = NULL; - - free(curl.useragent.s); - curl.useragent.l = curl.useragent.m = 0; curl.useragent.s = NULL; - - free(curl.auth_path); - curl.auth_path = NULL; - - if (curl.auth_map) { - khiter_t i; - for (i = kh_begin(curl.auth_map); i != kh_end(curl.auth_map); ++i) { - if (kh_exist(curl.auth_map, i)) { - free_auth(kh_value(curl.auth_map, i)); - kh_key(curl.auth_map, i) = NULL; - kh_value(curl.auth_map, i) = NULL; - } - } - kh_destroy(auth_map, curl.auth_map); - curl.auth_map = NULL; - } - - curl_global_cleanup(); -} - -static int append_header(hdrlist *hdrs, const char *data, int dup) { - if (hdrs->num == hdrs->size) { - unsigned int new_sz = hdrs->size ? hdrs->size * 2 : 4, i; - struct curl_slist *new_list = realloc(hdrs->list, - new_sz * sizeof(*new_list)); - if (!new_list) return -1; - hdrs->size = new_sz; - hdrs->list = new_list; - for (i = 1; i < hdrs->num; i++) hdrs->list[i-1].next = &hdrs->list[i]; - } - // Annoyingly, libcurl doesn't declare the char * as const... - hdrs->list[hdrs->num].data = dup ? strdup(data) : (char *) data; - if (!hdrs->list[hdrs->num].data) return -1; - if (hdrs->num > 0) hdrs->list[hdrs->num - 1].next = &hdrs->list[hdrs->num]; - hdrs->list[hdrs->num].next = NULL; - hdrs->num++; - return 0; -} - -static void free_headers(hdrlist *hdrs, int completely) { - unsigned int i; - for (i = 0; i < hdrs->num; i++) { - free(hdrs->list[i].data); - hdrs->list[i].data = NULL; - hdrs->list[i].next = NULL; - } - hdrs->num = 0; - if (completely) { - free(hdrs->list); - hdrs->size = 0; - hdrs->list = NULL; - } -} - -static struct curl_slist * get_header_list(hFILE_libcurl *fp) { - if (fp->headers.fixed.num > 0) - return &fp->headers.fixed.list[0]; - if (fp->headers.extra.num > 0) - return &fp->headers.extra.list[0]; - return 0; -} - -static inline int is_authorization(const char *hdr) { - return (strncasecmp("authorization:", hdr, 14) == 0); -} - -static int add_callback_headers(hFILE_libcurl *fp) { - char **hdrs = NULL, **hdr; - - if (!fp->headers.callback) - return 0; - - // Get the headers from the callback - if (fp->headers.callback(fp->headers.callback_data, &hdrs) != 0) { - return -1; - } - - if (!hdrs) // No change - return 0; - - // Remove any old callback headers - if (fp->headers.fixed.num > 0) { - // Unlink lists - fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL; - } - free_headers(&fp->headers.extra, 0); - - if (fp->headers.auth_hdr_num > 0 || fp->headers.auth_hdr_num == -2) - fp->headers.auth_hdr_num = 0; // Just removed it... - - // Convert to libcurl-suitable form - for (hdr = hdrs; *hdr; hdr++) { - if (append_header(&fp->headers.extra, *hdr, 0) < 0) { - goto cleanup; - } - if (is_authorization(*hdr) && !fp->headers.auth_hdr_num) - fp->headers.auth_hdr_num = -2; - } - for (hdr = hdrs; *hdr; hdr++) *hdr = NULL; - - if (fp->headers.fixed.num > 0 && fp->headers.extra.num > 0) { - // Relink lists - fp->headers.fixed.list[fp->headers.fixed.num - 1].next - = &fp->headers.extra.list[0]; - } - return 0; - - cleanup: - while (hdr && *hdr) { - free(*hdr); - *hdr = NULL; - } - return -1; -} - -/* - * Read an OAUTH2-style Bearer access token (see - * https://tools.ietf.org/html/rfc6750#section-4). - * Returns 'v' for valid; 'i' for invalid (token missing or wrong sort); - * '?' for a JSON parse error; 'm' if it runs out of memory. - */ -static int read_auth_json(auth_token *tok, hFILE *auth_fp) { - hts_json_token *t = hts_json_alloc_token(); - kstring_t str = {0, 0, NULL}; - char *token = NULL, *type = NULL, *expiry = NULL; - int ret = 'i'; - - if (!t) goto error; - - if ((ret = hts_json_fnext(auth_fp, t, &str)) != '{') goto error; - while (hts_json_fnext(auth_fp, t, &str) != '}') { - char *key; - if (hts_json_token_type(t) != 's') { - ret = '?'; - goto error; - } - key = hts_json_token_str(t); - if (!key) goto error; - if (strcmp(key, "access_token") == 0) { - if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error; - token = ks_release(&str); - } else if (strcmp(key, "token_type") == 0) { - if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error; - type = ks_release(&str); - } else if (strcmp(key, "expires_in") == 0) { - if ((ret = hts_json_fnext(auth_fp, t, &str)) != 'n') goto error; - expiry = ks_release(&str); - } else if (hts_json_fskip_value(auth_fp, '\0') != 'v') { - ret = '?'; - goto error; - } - } - - if (!token || (type && strcmp(type, "Bearer") != 0)) { - ret = 'i'; - goto error; - } - - ret = 'm'; - str.l = 0; - if (kputs("Authorization: Bearer ", &str) < 0) goto error; - if (kputs(token, &str) < 0) goto error; - free(tok->token); - tok->token = ks_release(&str); - if (expiry) { - long exp = strtol(expiry, NULL, 10); - if (exp < 0) exp = 0; - tok->expiry = time(NULL) + exp; - } else { - tok->expiry = 0; - } - ret = 'v'; - - error: - free(token); - free(type); - free(expiry); - free(str.s); - hts_json_free_token(t); - return ret; -} - -static int read_auth_plain(auth_token *tok, hFILE *auth_fp) { - kstring_t line = {0, 0, NULL}; - kstring_t token = {0, 0, NULL}; - const char *start, *end; - - if (kgetline(&line, (char * (*)(char *, int, void *)) hgets, auth_fp) < 0) goto error; - if (kputc('\0', &line) < 0) goto error; - - for (start = line.s; *start && isspace_c(*start); start++) {} - for (end = start; *end && !isspace_c(*end); end++) {} - - if (end > start) { - if (kputs("Authorization: Bearer ", &token) < 0) goto error; - if (kputsn(start, end - start, &token) < 0) goto error; - } - - free(tok->token); - tok->token = ks_release(&token); - tok->expiry = 0; - free(line.s); - return 0; - - error: - free(line.s); - free(token.s); - return -1; -} - -static int renew_auth_token(auth_token *tok, int *changed) { - hFILE *auth_fp = NULL; - char buffer[16]; - ssize_t len; - - *changed = 0; - if (tok->expiry == 0 || time(NULL) + AUTH_REFRESH_EARLY_SECS < tok->expiry) - return 0; // Still valid - - if (tok->failed) - return -1; - - *changed = 1; - auth_fp = hopen(tok->path, "rR"); - if (!auth_fp) { - // Not worried about missing files; other errors are bad. - if (errno != ENOENT) - goto fail; - - tok->expiry = 0; // Prevent retry - free(tok->token); // Just in case it was set - return 0; - } - - len = hpeek(auth_fp, buffer, sizeof(buffer)); - if (len < 0) - goto fail; - - if (memchr(buffer, '{', len) != NULL) { - if (read_auth_json(tok, auth_fp) != 'v') - goto fail; - } else { - if (read_auth_plain(tok, auth_fp) < 0) - goto fail; - } - - return hclose(auth_fp) < 0 ? -1 : 0; - - fail: - tok->failed = 1; - if (auth_fp) hclose_abruptly(auth_fp); - return -1; -} - -static int add_auth_header(hFILE_libcurl *fp) { - int changed = 0; - - if (fp->headers.auth_hdr_num < 0) - return 0; // Have an Authorization header from open or header callback - - if (!fp->headers.auth) - return 0; // Nothing to add - - pthread_mutex_lock(&fp->headers.auth->lock); - if (renew_auth_token(fp->headers.auth, &changed) < 0) - goto unlock_fail; - - if (!changed && fp->headers.auth_hdr_num > 0) { - pthread_mutex_unlock(&fp->headers.auth->lock); - return 0; - } - - if (fp->headers.auth_hdr_num > 0) { - // Had a previous header, so swap in the new one - char *header = fp->headers.auth->token; - char *header_copy = header ? strdup(header) : NULL; - int idx = fp->headers.auth_hdr_num - 1; - if (header && !header_copy) - goto unlock_fail; - - if (header_copy) { - free(fp->headers.extra.list[idx].data); - fp->headers.extra.list[idx].data = header_copy; - } else { - unsigned int j; - // More complicated case - need to get rid of the old header - // and tidy up linked lists - free(fp->headers.extra.list[idx].data); - for (j = idx + 1; j < fp->headers.extra.num; j++) { - fp->headers.extra.list[j - 1] = fp->headers.extra.list[j]; - fp->headers.extra.list[j - 1].next = &fp->headers.extra.list[j]; - } - fp->headers.extra.num--; - if (fp->headers.extra.num > 0) { - fp->headers.extra.list[fp->headers.extra.num-1].next = NULL; - } else if (fp->headers.fixed.num > 0) { - fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL; - } - fp->headers.auth_hdr_num = 0; - } - } else if (fp->headers.auth->token) { - // Add new header and remember where it is - if (append_header(&fp->headers.extra, - fp->headers.auth->token, 1) < 0) { - goto unlock_fail; - } - fp->headers.auth_hdr_num = fp->headers.extra.num; - } - - pthread_mutex_unlock(&fp->headers.auth->lock); - return 0; - - unlock_fail: - pthread_mutex_unlock(&fp->headers.auth->lock); - return -1; -} - -static int get_auth_token(hFILE_libcurl *fp, const char *url) { - const char *host = NULL, *p, *q; - kstring_t name = {0, 0, NULL}; - size_t host_len = 0; - khiter_t idx; - auth_token *tok = NULL; - - // Nothing to do if: - // curl.auth_path has not been set - // fp was made by hfile_libcurl (e.g. auth_path is a http:// url) - // we already have an Authorization header - if (!curl.auth_path || fp->is_recursive || fp->headers.auth_hdr_num != 0) - return 0; - - // Insist on having a secure connection unless the user insists harder - if (!curl.allow_unencrypted_auth_header && strncmp(url, "https://", 8) != 0) - return 0; - - host = strstr(url, "://"); - if (host) { - host += 3; - host_len = strcspn(host, "/"); - } - - p = curl.auth_path; - while ((q = strstr(p, "%h")) != NULL) { - if (q - p > INT_MAX || host_len > INT_MAX) goto error; - if (kputsn_(p, q - p, &name) < 0) goto error; - if (kputsn_(host, host_len, &name) < 0) goto error; - p = q + 2; - } - if (kputs(p, &name) < 0) goto error; - - pthread_mutex_lock(&curl.auth_lock); - idx = kh_get(auth_map, curl.auth_map, name.s); - if (idx < kh_end(curl.auth_map)) { - tok = kh_value(curl.auth_map, idx); - } else { - tok = calloc(1, sizeof(*tok)); - if (tok && pthread_mutex_init(&tok->lock, NULL) != 0) { - free(tok); - tok = NULL; - } - if (tok) { - int ret = -1; - tok->path = ks_release(&name); - tok->token = NULL; - tok->expiry = 1; // Force refresh - idx = kh_put(auth_map, curl.auth_map, tok->path, &ret); - if (ret < 0) { - free_auth(tok); - tok = NULL; - } - kh_value(curl.auth_map, idx) = tok; - } - } - pthread_mutex_unlock(&curl.auth_lock); - - fp->headers.auth = tok; - free(name.s); - - return add_auth_header(fp); - - error: - free(name.s); - return -1; -} - -static void process_messages(hFILE_libcurl *fp) -{ - CURLMsg *msg; - int remaining; - - while ((msg = curl_multi_info_read(fp->multi, &remaining)) != NULL) { - switch (msg->msg) { - case CURLMSG_DONE: - fp->finished = 1; - fp->final_result = msg->data.result; - break; - - default: - break; - } - } -} - -static int wait_perform(hFILE_libcurl *fp) -{ - fd_set rd, wr, ex; - int maxfd, nrunning; - long timeout; - CURLMcode errm; - - if (!fp->perform_again) { - FD_ZERO(&rd); - FD_ZERO(&wr); - FD_ZERO(&ex); - if (curl_multi_fdset(fp->multi, &rd, &wr, &ex, &maxfd) != CURLM_OK) - maxfd = -1, timeout = 1000; - else { - if (curl_multi_timeout(fp->multi, &timeout) != CURLM_OK) - timeout = 1000; - else if (timeout < 0) { - timeout = 10000; // as recommended by curl_multi_timeout(3) - } - } - if (maxfd < 0 && timeout > 100) - timeout = 100; // as recommended by curl_multi_fdset(3) - - if (timeout > 0) { - struct timeval tval; - tval.tv_sec = (timeout / 1000); - tval.tv_usec = (timeout % 1000) * 1000; - - if (select(maxfd + 1, &rd, &wr, &ex, &tval) < 0) return -1; - } - } - - errm = curl_multi_perform(fp->multi, &nrunning); - fp->perform_again = 0; - if (errm == CURLM_CALL_MULTI_PERFORM) fp->perform_again = 1; - else if (errm != CURLM_OK) { errno = multi_errno(errm); return -1; } - - if (nrunning < fp->nrunning) process_messages(fp); - return 0; -} - - -static size_t recv_callback(char *ptr, size_t size, size_t nmemb, void *fpv) -{ - hFILE_libcurl *fp = (hFILE_libcurl *) fpv; - size_t n = size * nmemb; - - if (n > fp->buffer.len) { fp->paused = 1; return CURL_WRITEFUNC_PAUSE; } - else if (n == 0) return 0; - - memcpy(fp->buffer.ptr.rd, ptr, n); - fp->buffer.ptr.rd += n; - fp->buffer.len -= n; - return n; -} - -static ssize_t libcurl_read(hFILE *fpv, void *bufferv, size_t nbytes) -{ - hFILE_libcurl *fp = (hFILE_libcurl *) fpv; - char *buffer = (char *) bufferv; - off_t to_skip = -1; - ssize_t got = 0; - CURLcode err; - - if (fp->delayed_seek >= 0) { - assert(fp->base.offset == fp->delayed_seek - && fp->base.begin == fp->base.buffer - && fp->base.end == fp->base.buffer); - - if (fp->last_offset >= 0 - && fp->delayed_seek > fp->last_offset - && fp->delayed_seek - fp->last_offset < MIN_SEEK_FORWARD) { - // If not seeking far, just read the data and throw it away. This - // is likely to be quicker than opening a new stream - to_skip = fp->delayed_seek - fp->last_offset; - } else { - if (restart_from_position(fp, fp->delayed_seek) < 0) { - return -1; - } - } - fp->delayed_seek = -1; - fp->last_offset = -1; - } - - do { - fp->buffer.ptr.rd = buffer; - fp->buffer.len = nbytes; - fp->paused = 0; - err = curl_easy_pause(fp->easy, CURLPAUSE_CONT); - if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; } - - while (! fp->paused && ! fp->finished) - if (wait_perform(fp) < 0) return -1; - - got = fp->buffer.ptr.rd - buffer; - - if (to_skip >= 0) { // Skipping over a small seek - if (got < to_skip) { // Need to skip more data - to_skip -= got; - } else { - got -= to_skip; - if (got > 0) { // If enough was skipped, return the rest - memmove(buffer, buffer + to_skip, got); - to_skip = -1; - } - } - } - } while (to_skip >= 0 && ! fp->finished); - fp->buffer.ptr.rd = NULL; - fp->buffer.len = 0; - - if (fp->finished && fp->final_result != CURLE_OK) { - errno = easy_errno(fp->easy, fp->final_result); - return -1; - } - - return got; -} - -static size_t send_callback(char *ptr, size_t size, size_t nmemb, void *fpv) -{ - hFILE_libcurl *fp = (hFILE_libcurl *) fpv; - size_t n = size * nmemb; - - if (fp->buffer.len == 0) { - // Send buffer is empty; normally pause, or signal EOF if we're closing - if (fp->closing) return 0; - else { fp->paused = 1; return CURL_READFUNC_PAUSE; } - } - - if (n > fp->buffer.len) n = fp->buffer.len; - memcpy(ptr, fp->buffer.ptr.wr, n); - fp->buffer.ptr.wr += n; - fp->buffer.len -= n; - return n; -} - -static ssize_t libcurl_write(hFILE *fpv, const void *bufferv, size_t nbytes) -{ - hFILE_libcurl *fp = (hFILE_libcurl *) fpv; - const char *buffer = (const char *) bufferv; - CURLcode err; - - fp->buffer.ptr.wr = buffer; - fp->buffer.len = nbytes; - fp->paused = 0; - err = curl_easy_pause(fp->easy, CURLPAUSE_CONT); - if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; } - - while (! fp->paused && ! fp->finished) - if (wait_perform(fp) < 0) return -1; - - nbytes = fp->buffer.ptr.wr - buffer; - fp->buffer.ptr.wr = NULL; - fp->buffer.len = 0; - - if (fp->finished && fp->final_result != CURLE_OK) { - errno = easy_errno(fp->easy, fp->final_result); - return -1; - } - - return nbytes; -} - -static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence) -{ - hFILE_libcurl *fp = (hFILE_libcurl *) fpv; - off_t origin, pos; - - if (!fp->is_read || !fp->can_seek) { - // Cowardly refuse to seek when writing or a previous seek failed. - errno = ESPIPE; - return -1; - } - - switch (whence) { - case SEEK_SET: - origin = 0; - break; - case SEEK_CUR: - errno = ENOSYS; - return -1; - case SEEK_END: - if (fp->file_size < 0) { errno = ESPIPE; return -1; } - origin = fp->file_size; - break; - default: - errno = EINVAL; - return -1; - } - - // Check 0 <= origin+offset < fp->file_size carefully, avoiding overflow - if ((offset < 0)? origin + offset < 0 - : (fp->file_size >= 0 && offset > fp->file_size - origin)) { - errno = EINVAL; - return -1; - } - - pos = origin + offset; - - if (fp->tried_seek) { - /* Seeking has worked at least once, so now we can delay doing - the actual work until the next read. This avoids lots of pointless - http or ftp reconnections if the caller does lots of seeks - without any intervening reads. */ - if (fp->delayed_seek < 0) { - fp->last_offset = fp->base.offset + (fp->base.end - fp->base.buffer); - } - fp->delayed_seek = pos; - return pos; - } - - if (restart_from_position(fp, pos) < 0) { - /* This value for errno may not be entirely true, but the caller may be - able to carry on with the existing handle. */ - errno = ESPIPE; - return -1; - } - - fp->tried_seek = 1; - return pos; -} - -static int restart_from_position(hFILE_libcurl *fp, off_t pos) { - hFILE_libcurl temp_fp; - CURLcode err; - CURLMcode errm; - int update_headers = 0; - int save_errno = 0; - - // TODO If we seem to be doing random access, use CURLOPT_RANGE to do - // limited reads (e.g. about a BAM block!) so seeking can reuse the - // existing connection more often. - - // Get new headers from the callback (if defined). This changes the - // headers in fp before it gets duplicated, but they should be have been - // sent by now. - - if (fp->headers.callback) { - if (add_callback_headers(fp) != 0) - return -1; - update_headers = 1; - } - if (fp->headers.auth_hdr_num > 0 && fp->headers.auth) { - if (add_auth_header(fp) != 0) - return -1; - update_headers = 1; - } - if (update_headers) { - struct curl_slist *list = get_header_list(fp); - if (list) { - err = curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list); - if (err != CURLE_OK) { - errno = easy_errno(fp->easy,err); - return -1; - } - } - } - - /* - Duplicate the easy handle, and use CURLOPT_RESUME_FROM_LARGE to open - a new request to the server, reading from the location that we want - to seek to. If the new request works and returns the correct data, - the original easy handle in *fp is closed and replaced with the new - one. If not, we close the new handle and leave *fp unchanged. - */ - - memcpy(&temp_fp, fp, sizeof(temp_fp)); - temp_fp.buffer.len = 0; - temp_fp.buffer.ptr.rd = NULL; - temp_fp.easy = curl_easy_duphandle(fp->easy); - if (!temp_fp.easy) - goto early_error; - - err = curl_easy_setopt(temp_fp.easy, CURLOPT_RESUME_FROM_LARGE,(curl_off_t)pos); - err |= curl_easy_setopt(temp_fp.easy, CURLOPT_PRIVATE, &temp_fp); - err |= curl_easy_setopt(temp_fp.easy, CURLOPT_WRITEDATA, &temp_fp); - if (err != CURLE_OK) { - save_errno = easy_errno(temp_fp.easy, err); - goto error; - } - - temp_fp.buffer.len = 0; // Ensures we only read the response headers - temp_fp.paused = temp_fp.finished = 0; - - // fp->multi and temp_fp.multi are the same. - errm = curl_multi_add_handle(fp->multi, temp_fp.easy); - if (errm != CURLM_OK) { - save_errno = multi_errno(errm); - goto error; - } - temp_fp.nrunning = ++fp->nrunning; - - err = curl_easy_pause(temp_fp.easy, CURLPAUSE_CONT); - if (err != CURLE_OK) { - save_errno = easy_errno(temp_fp.easy, err); - goto error_remove; - } - - while (! temp_fp.paused && ! temp_fp.finished) - if (wait_perform(&temp_fp) < 0) { - save_errno = errno; - goto error_remove; - } - - if (temp_fp.finished && temp_fp.final_result != CURLE_OK) { - save_errno = easy_errno(temp_fp.easy, temp_fp.final_result); - goto error_remove; - } - - // We've got a good response, close the original connection and - // replace it with the new one. - - errm = curl_multi_remove_handle(fp->multi, fp->easy); - if (errm != CURLM_OK) { - // Clean up as much as possible - curl_easy_reset(temp_fp.easy); - if (curl_multi_remove_handle(fp->multi, temp_fp.easy) == CURLM_OK) { - fp->nrunning--; - curl_easy_cleanup(temp_fp.easy); - } - save_errno = multi_errno(errm); - goto early_error; - } - fp->nrunning--; - - curl_easy_cleanup(fp->easy); - fp->easy = temp_fp.easy; - err = curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp); - err |= curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp); - if (err != CURLE_OK) { - save_errno = easy_errno(fp->easy, err); - curl_easy_reset(fp->easy); - errno = save_errno; - return -1; - } - fp->buffer.len = 0; - fp->paused = temp_fp.paused; - fp->finished = temp_fp.finished; - fp->perform_again = temp_fp.perform_again; - fp->final_result = temp_fp.final_result; - - return 0; - - error_remove: - curl_easy_reset(temp_fp.easy); // Ensure no pointers to on-stack temp_fp - errm = curl_multi_remove_handle(fp->multi, temp_fp.easy); - if (errm != CURLM_OK) { - errno = multi_errno(errm); - return -1; - } - fp->nrunning--; - error: - curl_easy_cleanup(temp_fp.easy); - early_error: - fp->can_seek = 0; // Don't try to seek again - if (save_errno) - errno = save_errno; - return -1; -} - -static int libcurl_close(hFILE *fpv) -{ - hFILE_libcurl *fp = (hFILE_libcurl *) fpv; - CURLcode err; - CURLMcode errm; - int save_errno = 0; - - // Before closing the file, unpause it and perform on it so that uploads - // have the opportunity to signal EOF to the server -- see send_callback(). - - fp->buffer.len = 0; - fp->closing = 1; - fp->paused = 0; - err = curl_easy_pause(fp->easy, CURLPAUSE_CONT); - if (err != CURLE_OK) save_errno = easy_errno(fp->easy, err); - - while (save_errno == 0 && ! fp->paused && ! fp->finished) - if (wait_perform(fp) < 0) save_errno = errno; - - if (fp->finished && fp->final_result != CURLE_OK) - save_errno = easy_errno(fp->easy, fp->final_result); - - errm = curl_multi_remove_handle(fp->multi, fp->easy); - if (errm != CURLM_OK && save_errno == 0) save_errno = multi_errno(errm); - fp->nrunning--; - - curl_easy_cleanup(fp->easy); - curl_multi_cleanup(fp->multi); - - if (fp->headers.callback) // Tell callback to free any data it needs to - fp->headers.callback(fp->headers.callback_data, NULL); - free_headers(&fp->headers.fixed, 1); - free_headers(&fp->headers.extra, 1); - - if (save_errno) { errno = save_errno; return -1; } - else return 0; -} - -static const struct hFILE_backend libcurl_backend = -{ - libcurl_read, libcurl_write, libcurl_seek, NULL, libcurl_close -}; - -static hFILE * -libcurl_open(const char *url, const char *modes, http_headers *headers) -{ - hFILE_libcurl *fp; - struct curl_slist *list; - char mode; - const char *s; - CURLcode err; - CURLMcode errm; - int save, is_recursive; - - is_recursive = strchr(modes, 'R') != NULL; - - if ((s = strpbrk(modes, "rwa+")) != NULL) { - mode = *s; - if (strpbrk(&s[1], "rwa+")) mode = 'e'; - } - else mode = '\0'; - - if (mode != 'r' && mode != 'w') { errno = EINVAL; goto early_error; } - - fp = (hFILE_libcurl *) hfile_init(sizeof (hFILE_libcurl), modes, 0); - if (fp == NULL) goto early_error; - - if (headers) { - fp->headers = *headers; - } else { - memset(&fp->headers, 0, sizeof(fp->headers)); - } - - fp->file_size = -1; - fp->buffer.ptr.rd = NULL; - fp->buffer.len = 0; - fp->final_result = (CURLcode) -1; - fp->paused = fp->closing = fp->finished = fp->perform_again = 0; - fp->can_seek = 1; - fp->tried_seek = 0; - fp->delayed_seek = fp->last_offset = -1; - fp->is_recursive = is_recursive; - fp->nrunning = 0; - fp->easy = NULL; - - fp->multi = curl_multi_init(); - if (fp->multi == NULL) { errno = ENOMEM; goto error; } - - fp->easy = curl_easy_init(); - if (fp->easy == NULL) { errno = ENOMEM; goto error; } - - // Make a route to the hFILE_libcurl* given just a CURL* easy handle - err = curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp); - - // Avoid many repeated CWD calls with FTP, instead requesting the filename - // by full path (as done in knet, but not strictly compliant with RFC1738). - err |= curl_easy_setopt(fp->easy, CURLOPT_FTP_FILEMETHOD, CURLFTPMETHOD_NOCWD); - - if (mode == 'r') { - err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEFUNCTION, recv_callback); - err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp); - fp->is_read = 1; - } - else { - err |= curl_easy_setopt(fp->easy, CURLOPT_READFUNCTION, send_callback); - err |= curl_easy_setopt(fp->easy, CURLOPT_READDATA, fp); - err |= curl_easy_setopt(fp->easy, CURLOPT_UPLOAD, 1L); - if (append_header(&fp->headers.fixed, - "Transfer-Encoding: chunked", 1) < 0) - goto error; - fp->is_read = 0; - } - - err |= curl_easy_setopt(fp->easy, CURLOPT_SHARE, curl.share); - err |= curl_easy_setopt(fp->easy, CURLOPT_URL, url); - { - char* env_curl_ca_bundle = getenv("CURL_CA_BUNDLE"); - if (env_curl_ca_bundle) { - err |= curl_easy_setopt(fp->easy, CURLOPT_CAINFO, env_curl_ca_bundle); - } - } - err |= curl_easy_setopt(fp->easy, CURLOPT_USERAGENT, curl.useragent.s); - if (fp->headers.callback) { - if (add_callback_headers(fp) != 0) goto error; - } - if (get_auth_token(fp, url) < 0) - goto error; - if ((list = get_header_list(fp)) != NULL) - err |= curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list); - err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L); - if (hts_verbose <= 8) - err |= curl_easy_setopt(fp->easy, CURLOPT_FAILONERROR, 1L); - if (hts_verbose >= 8) - err |= curl_easy_setopt(fp->easy, CURLOPT_VERBOSE, 1L); - - if (err != 0) { errno = ENOSYS; goto error; } - - errm = curl_multi_add_handle(fp->multi, fp->easy); - if (errm != CURLM_OK) { errno = multi_errno(errm); goto error; } - fp->nrunning++; - - while (! fp->paused && ! fp->finished) - if (wait_perform(fp) < 0) goto error_remove; - - if (fp->finished && fp->final_result != CURLE_OK) { - errno = easy_errno(fp->easy, fp->final_result); - goto error_remove; - } - - if (mode == 'r') { - double dval; - if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD, - &dval) == CURLE_OK && dval >= 0.0) - fp->file_size = (off_t) (dval + 0.1); - } - - fp->base.backend = &libcurl_backend; - return &fp->base; - -error_remove: - save = errno; - (void) curl_multi_remove_handle(fp->multi, fp->easy); - fp->nrunning--; - errno = save; - -error: - save = errno; - if (fp->easy) curl_easy_cleanup(fp->easy); - if (fp->multi) curl_multi_cleanup(fp->multi); - free_headers(&fp->headers.extra, 1); - hfile_destroy((hFILE *) fp); - errno = save; - return NULL; - -early_error: - return NULL; -} - -static hFILE *hopen_libcurl(const char *url, const char *modes) -{ - return libcurl_open(url, modes, NULL); -} - -static int parse_va_list(http_headers *headers, va_list args) -{ - const char *argtype; - - while ((argtype = va_arg(args, const char *)) != NULL) - if (strcmp(argtype, "httphdr:v") == 0) { - const char **hdr; - for (hdr = va_arg(args, const char **); *hdr; hdr++) { - if (append_header(&headers->fixed, *hdr, 1) < 0) - return -1; - if (is_authorization(*hdr)) - headers->auth_hdr_num = -1; - } - } - else if (strcmp(argtype, "httphdr:l") == 0) { - const char *hdr; - while ((hdr = va_arg(args, const char *)) != NULL) { - if (append_header(&headers->fixed, hdr, 1) < 0) - return -1; - if (is_authorization(hdr)) - headers->auth_hdr_num = -1; - } - } - else if (strcmp(argtype, "httphdr") == 0) { - const char *hdr = va_arg(args, const char *); - if (hdr) { - if (append_header(&headers->fixed, hdr, 1) < 0) - return -1; - if (is_authorization(hdr)) - headers->auth_hdr_num = -1; - } - } - else if (strcmp(argtype, "httphdr_callback") == 0) { - headers->callback = va_arg(args, const hts_httphdr_callback); - } - else if (strcmp(argtype, "httphdr_callback_data") == 0) { - headers->callback_data = va_arg(args, void *); - } - else if (strcmp(argtype, "va_list") == 0) { - va_list *args2 = va_arg(args, va_list *); - if (args2) { - if (parse_va_list(headers, *args2) < 0) return -1; - } - } - else if (strcmp(argtype, "auth_token_enabled") == 0) { - const char *flag = va_arg(args, const char *); - if (strcmp(flag, "false") == 0) - headers->auth_hdr_num = -3; - } - else { errno = EINVAL; return -1; } - - return 0; -} - -/* - HTTP headers to be added to the request can be passed in as extra - arguments to hopen(). The headers can be specified as follows: - - * Single header: - hopen(url, mode, "httphdr", "X-Hdr-1: text", NULL); - - * Multiple headers in the argument list: - hopen(url, mode, "httphdr:l", "X-Hdr-1: text", "X-Hdr-2: text", NULL, NULL); - - * Multiple headers in a char* array: - hopen(url, mode, "httphdr:v", hdrs, NULL); - where `hdrs` is a char **. The list ends with a NULL pointer. - - * A callback function - hopen(url, mode, "httphdr_callback", func, - "httphdr_callback_data", arg, NULL); - `func` has type - int (* hts_httphdr_callback) (void *cb_data, char ***hdrs); - `arg` is passed to the callback as a void *. - - The function is called at file open, and when attempting to seek (which - opens a new HTTP request). This allows, for example, access tokens - that may have gone stale to be regenerated. The function is also - called (with `hdrs` == NULL) on file close so that the callback can - free any memory that it needs to. - - The callback should return 0 on success, non-zero on failure. It should - return in *hdrs a list of strings containing the new headers (terminated - with a NULL pointer). These will replace any headers previously supplied - by the callback. If no changes are necessary, it can return NULL - in *hdrs, in which case the previous headers will be left unchanged. - - Ownership of the strings in the header list passes to hfile_libcurl, - so the callback should not attempt to use or free them itself. The memory - containing the array belongs to the callback and will not be freed by - hfile_libcurl. - - Headers supplied by the callback are appended after any specified - using the "httphdr", "httphdr:l" or "httphdr:v" methods. No attempt - is made to replace these headers (even if a key is repeated) so anything - that is expected to vary needs to come from the callback. - */ - -static hFILE *vhopen_libcurl(const char *url, const char *modes, va_list args) -{ - hFILE *fp = NULL; - http_headers headers = { { NULL, 0, 0 }, { NULL, 0, 0 }, NULL, NULL }; - if (parse_va_list(&headers, args) == 0) { - fp = libcurl_open(url, modes, &headers); - } - - if (!fp) { - free_headers(&headers.fixed, 1); - } - return fp; -} - -int PLUGIN_GLOBAL(hfile_plugin_init,_libcurl)(struct hFILE_plugin *self) -{ - static const struct hFILE_scheme_handler handler = - { hopen_libcurl, hfile_always_remote, "libcurl", - 2000 + 50, - vhopen_libcurl }; - -#ifdef ENABLE_PLUGINS - // Embed version string for examination via strings(1) or what(1) - static const char id[] = "@(#)hfile_libcurl plugin (htslib)\t" HTS_VERSION; - const char *version = strchr(id, '\t')+1; -#else - const char *version = hts_version(); -#endif - const curl_version_info_data *info; - const char * const *protocol; - const char *auth; - CURLcode err; - CURLSHcode errsh; - - err = curl_global_init(CURL_GLOBAL_ALL); - if (err != CURLE_OK) { errno = easy_errno(NULL, err); return -1; } - - curl.share = curl_share_init(); - if (curl.share == NULL) { curl_global_cleanup(); errno = EIO; return -1; } - errsh = curl_share_setopt(curl.share, CURLSHOPT_LOCKFUNC, share_lock); - errsh |= curl_share_setopt(curl.share, CURLSHOPT_UNLOCKFUNC, share_unlock); - errsh |= curl_share_setopt(curl.share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); - if (errsh != 0) { - curl_share_cleanup(curl.share); - curl_global_cleanup(); - errno = EIO; - return -1; - } - - if ((auth = getenv("HTS_AUTH_LOCATION")) != NULL) { - curl.auth_path = strdup(auth); - curl.auth_map = kh_init(auth_map); - if (!curl.auth_path || !curl.auth_map) { - int save_errno = errno; - free(curl.auth_path); - kh_destroy(auth_map, curl.auth_map); - curl_share_cleanup(curl.share); - curl_global_cleanup(); - errno = save_errno; - return -1; - } - } - if ((auth = getenv("HTS_ALLOW_UNENCRYPTED_AUTHORIZATION_HEADER")) != NULL - && strcmp(auth, "I understand the risks") == 0) { - curl.allow_unencrypted_auth_header = 1; - } - - info = curl_version_info(CURLVERSION_NOW); - ksprintf(&curl.useragent, "htslib/%s libcurl/%s", version, info->version); - - self->name = "libcurl"; - self->destroy = libcurl_exit; - - for (protocol = info->protocols; *protocol; protocol++) - hfile_add_scheme_handler(*protocol, &handler); - return 0; -} diff --git a/htslib-1.9/hfile_s3.c b/htslib-1.9/hfile_s3.c deleted file mode 100644 --- a/htslib-1.9/hfile_s3.c +++ /dev/null @@ -1,466 +0,0 @@ -/* hfile_s3.c -- Amazon S3 backend for low-level file streams. - - Copyright (C) 2015-2017 Genome Research Ltd. - - Author: John Marshall - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. */ - -#include - -#include -#include -#include -#include -#include - -#include "hfile_internal.h" -#ifdef ENABLE_PLUGINS -#include "version.h" -#endif -#include "htslib/hts.h" // for hts_version() and hts_verbose -#include "htslib/kstring.h" - -typedef struct { - kstring_t id; - kstring_t token; - kstring_t secret; - char *bucket; - kstring_t auth_hdr; - time_t auth_time; - char date[40]; - char mode; - char *headers[3]; -} s3_auth_data; - -#define AUTH_LIFETIME 60 - -#if defined HAVE_COMMONCRYPTO - -#include - -#define DIGEST_BUFSIZ CC_SHA1_DIGEST_LENGTH - -static size_t -s3_sign(unsigned char *digest, kstring_t *key, kstring_t *message) -{ - CCHmac(kCCHmacAlgSHA1, key->s, key->l, message->s, message->l, digest); - return CC_SHA1_DIGEST_LENGTH; -} - -#elif defined HAVE_HMAC - -#include - -#define DIGEST_BUFSIZ EVP_MAX_MD_SIZE - -static size_t -s3_sign(unsigned char *digest, kstring_t *key, kstring_t *message) -{ - unsigned int len; - HMAC(EVP_sha1(), key->s, key->l, - (unsigned char *) message->s, message->l, digest, &len); - return len; -} - -#else -#error No HMAC() routine found by configure -#endif - -static void -urldecode_kput(const char *s, int len, kstring_t *str) -{ - char buf[3]; - int i = 0; - - while (i < len) - if (s[i] == '%' && i+2 < len) { - buf[0] = s[i+1], buf[1] = s[i+2], buf[2] = '\0'; - kputc(strtol(buf, NULL, 16), str); - i += 3; - } - else kputc(s[i++], str); -} - -static void base64_kput(const unsigned char *data, size_t len, kstring_t *str) -{ - static const char base64[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - size_t i = 0; - unsigned x = 0; - int bits = 0, pad = 0; - - while (bits || i < len) { - if (bits < 6) { - x <<= 8, bits += 8; - if (i < len) x |= data[i++]; - else pad++; - } - - bits -= 6; - kputc(base64[(x >> bits) & 63], str); - } - - str->l -= pad; - kputsn("==", pad, str); -} - -static int is_dns_compliant(const char *s0, const char *slim) -{ - int has_nondigit = 0, len = 0; - const char *s; - - for (s = s0; s < slim; len++, s++) - if (islower_c(*s)) - has_nondigit = 1; - else if (*s == '-') { - has_nondigit = 1; - if (s == s0 || s+1 == slim) return 0; - } - else if (isdigit_c(*s)) - ; - else if (*s == '.') { - if (s == s0 || ! isalnum_c(s[-1])) return 0; - if (s+1 == slim || ! isalnum_c(s[1])) return 0; - } - else return 0; - - return has_nondigit && len >= 3 && len <= 63; -} - -static FILE *expand_tilde_open(const char *fname, const char *mode) -{ - FILE *fp; - - if (strncmp(fname, "~/", 2) == 0) { - kstring_t full_fname = { 0, 0, NULL }; - const char *home = getenv("HOME"); - if (! home) return NULL; - - kputs(home, &full_fname); - kputs(&fname[1], &full_fname); - - fp = fopen(full_fname.s, mode); - free(full_fname.s); - } - else - fp = fopen(fname, mode); - - return fp; -} - -static void parse_ini(const char *fname, const char *section, ...) -{ - kstring_t line = { 0, 0, NULL }; - int active = 1; // Start active, so global properties are accepted - char *s; - - FILE *fp = expand_tilde_open(fname, "r"); - if (fp == NULL) return; - - while (line.l = 0, kgetline(&line, (kgets_func *) fgets, fp) >= 0) - if (line.s[0] == '[' && (s = strchr(line.s, ']')) != NULL) { - *s = '\0'; - active = (strcmp(&line.s[1], section) == 0); - } - else if (active && (s = strpbrk(line.s, ":=")) != NULL) { - const char *key = line.s, *value = &s[1], *akey; - va_list args; - - while (isspace_c(*key)) key++; - while (s > key && isspace_c(s[-1])) s--; - *s = '\0'; - - while (isspace_c(*value)) value++; - while (line.l > 0 && isspace_c(line.s[line.l-1])) - line.s[--line.l] = '\0'; - - va_start(args, section); - while ((akey = va_arg(args, const char *)) != NULL) { - kstring_t *avar = va_arg(args, kstring_t *); - if (strcmp(key, akey) == 0) { kputs(value, avar); break; } - } - va_end(args); - } - - fclose(fp); - free(line.s); -} - -static void parse_simple(const char *fname, kstring_t *id, kstring_t *secret) -{ - kstring_t text = { 0, 0, NULL }; - char *s; - size_t len; - - FILE *fp = expand_tilde_open(fname, "r"); - if (fp == NULL) return; - - while (kgetline(&text, (kgets_func *) fgets, fp) >= 0) - kputc(' ', &text); - fclose(fp); - - s = text.s; - while (isspace_c(*s)) s++; - kputsn(s, len = strcspn(s, " \t"), id); - - s += len; - while (isspace_c(*s)) s++; - kputsn(s, strcspn(s, " \t"), secret); - - free(text.s); -} - -static int copy_auth_headers(s3_auth_data *ad, char ***hdrs) { - char **hdr = &ad->headers[0]; - *hdrs = hdr; - *hdr = strdup(ad->date); - if (!*hdr) return -1; - hdr++; - if (ad->auth_hdr.l) { - *hdr = strdup(ad->auth_hdr.s); - if (!*hdr) { free(ad->headers[0]); return -1; } - hdr++; - } - *hdr = NULL; - return 0; -} - -static void free_auth_data(s3_auth_data *ad) { - free(ad->id.s); - free(ad->token.s); - free(ad->secret.s); - free(ad->bucket); - free(ad->auth_hdr.s); - free(ad); -} - -static int auth_header_callback(void *ctx, char ***hdrs) { - s3_auth_data *ad = (s3_auth_data *) ctx; - - time_t now = time(NULL); -#ifdef HAVE_GMTIME_R - struct tm tm_buffer; - struct tm *tm = gmtime_r(&now, &tm_buffer); -#else - struct tm *tm = gmtime(&now); -#endif - kstring_t message = { 0, 0, NULL }; - unsigned char digest[DIGEST_BUFSIZ]; - size_t digest_len; - - if (!hdrs) { // Closing connection - free_auth_data(ad); - return 0; - } - - if (now - ad->auth_time < AUTH_LIFETIME) { - // Last auth string should still be valid - *hdrs = NULL; - return 0; - } - - strftime(ad->date, sizeof(ad->date), "Date: %a, %d %b %Y %H:%M:%S GMT", tm); - if (!ad->id.l || !ad->secret.l) { - ad->auth_time = now; - return copy_auth_headers(ad, hdrs); - } - - if (ksprintf(&message, "%s\n\n\n%s\n%s%s%s/%s", - ad->mode == 'r' ? "GET" : "PUT", ad->date + 6, - ad->token.l ? "x-amz-security-token:" : "", - ad->token.l ? ad->token.s : "", - ad->token.l ? "\n" : "", - ad->bucket) < 0) { - return -1; - } - - digest_len = s3_sign(digest, &ad->secret, &message); - ad->auth_hdr.l = 0; - if (ksprintf(&ad->auth_hdr, "Authorization: AWS %s:", ad->id.s) < 0) - goto fail; - base64_kput(digest, digest_len, &ad->auth_hdr); - - free(message.s); - ad->auth_time = now; - return copy_auth_headers(ad, hdrs); - - fail: - free(message.s); - return -1; -} - -static hFILE * s3_rewrite(const char *s3url, const char *mode, va_list *argsp) -{ - const char *bucket, *path; - char *header_list[4], **header = header_list; - - kstring_t url = { 0, 0, NULL }; - kstring_t profile = { 0, 0, NULL }; - kstring_t host_base = { 0, 0, NULL }; - kstring_t token_hdr = { 0, 0, NULL }; - - s3_auth_data *ad = calloc(1, sizeof(*ad)); - - if (!ad) - return NULL; - ad->mode = strchr(mode, 'r') ? 'r' : 'w'; - - // Our S3 URL format is s3[+SCHEME]://[ID[:SECRET[:TOKEN]]@]BUCKET/PATH - - if (s3url[2] == '+') { - bucket = strchr(s3url, ':') + 1; - kputsn(&s3url[3], bucket - &s3url[3], &url); - } - else { - kputs("https:", &url); - bucket = &s3url[3]; - } - while (*bucket == '/') kputc(*bucket++, &url); - - path = bucket + strcspn(bucket, "/?#@"); - if (*path == '@') { - const char *colon = strpbrk(bucket, ":@"); - if (*colon != ':') { - urldecode_kput(bucket, colon - bucket, &profile); - } - else { - const char *colon2 = strpbrk(&colon[1], ":@"); - urldecode_kput(bucket, colon - bucket, &ad->id); - urldecode_kput(&colon[1], colon2 - &colon[1], &ad->secret); - if (*colon2 == ':') - urldecode_kput(&colon2[1], path - &colon2[1], &ad->token); - } - - bucket = &path[1]; - path = bucket + strcspn(bucket, "/?#"); - } - else { - // If the URL has no ID[:SECRET]@, consider environment variables. - const char *v; - if ((v = getenv("AWS_ACCESS_KEY_ID")) != NULL) kputs(v, &ad->id); - if ((v = getenv("AWS_SECRET_ACCESS_KEY")) != NULL) kputs(v, &ad->secret); - if ((v = getenv("AWS_SESSION_TOKEN")) != NULL) kputs(v, &ad->token); - - if ((v = getenv("AWS_DEFAULT_PROFILE")) != NULL) kputs(v, &profile); - else if ((v = getenv("AWS_PROFILE")) != NULL) kputs(v, &profile); - else kputs("default", &profile); - } - - if (ad->id.l == 0) { - const char *v = getenv("AWS_SHARED_CREDENTIALS_FILE"); - parse_ini(v? v : "~/.aws/credentials", profile.s, - "aws_access_key_id", &ad->id, - "aws_secret_access_key", &ad->secret, - "aws_session_token", &ad->token, NULL); - } - if (ad->id.l == 0) - parse_ini("~/.s3cfg", profile.s, "access_key", &ad->id, - "secret_key", &ad->secret, "access_token", &ad->token, - "host_base", &host_base, NULL); - if (ad->id.l == 0) - parse_simple("~/.awssecret", &ad->id, &ad->secret); - - if (host_base.l == 0) - kputs("s3.amazonaws.com", &host_base); - // Use virtual hosted-style access if possible, otherwise path-style. - if (is_dns_compliant(bucket, path)) { - kputsn(bucket, path - bucket, &url); - kputc('.', &url); - kputs(host_base.s, &url); - } - else { - kputs(host_base.s, &url); - kputc('/', &url); - kputsn(bucket, path - bucket, &url); - } - kputs(path, &url); - - if (ad->token.l > 0) { - kputs("X-Amz-Security-Token: ", &token_hdr); - kputs(ad->token.s, &token_hdr); - *header++ = token_hdr.s; - } - - ad->bucket = strdup(bucket); - if (!ad->bucket) - goto fail; - - *header = NULL; - hFILE *fp = hopen(url.s, mode, "va_list", argsp, "httphdr:v", header_list, - "httphdr_callback", auth_header_callback, - "httphdr_callback_data", ad, NULL); - if (!fp) goto fail; - - free(url.s); - free(profile.s); - free(host_base.s); - free(token_hdr.s); - return fp; - - fail: - free(url.s); - free(profile.s); - free(host_base.s); - free(token_hdr.s); - free_auth_data(ad); - return NULL; -} - -static hFILE *s3_open(const char *url, const char *mode) -{ - kstring_t mode_colon = { 0, 0, NULL }; - kputs(mode, &mode_colon); - kputc(':', &mode_colon); - hFILE *fp = s3_rewrite(url, mode_colon.s, NULL); - free(mode_colon.s); - return fp; -} - -static hFILE *s3_vopen(const char *url, const char *mode_colon, va_list args0) -{ - // Need to use va_copy() as we can only take the address of an actual - // va_list object, not that of a parameter whose type may have decayed. - va_list args; - va_copy(args, args0); - hFILE *fp = s3_rewrite(url, mode_colon, &args); - va_end(args); - return fp; -} - -int PLUGIN_GLOBAL(hfile_plugin_init,_s3)(struct hFILE_plugin *self) -{ - static const struct hFILE_scheme_handler handler = - { s3_open, hfile_always_remote, "Amazon S3", 2000 + 50, s3_vopen - }; - -#ifdef ENABLE_PLUGINS - // Embed version string for examination via strings(1) or what(1) - static const char id[] = "@(#)hfile_s3 plugin (htslib)\t" HTS_VERSION; - if (hts_verbose >= 9) - fprintf(stderr, "[M::hfile_s3.init] version %s\n", strchr(id, '\t')+1); -#endif - - self->name = "Amazon S3"; - hfile_add_scheme_handler("s3", &handler); - hfile_add_scheme_handler("s3+http", &handler); - hfile_add_scheme_handler("s3+https", &handler); - return 0; -}