download (empty) → 0.1
raw patch · 15 files changed
+4333/−0 lines, 15 filesdep +basedep +bytestringdep +tagsoupsetup-changed
Dependencies added: base, bytestring, tagsoup
Files
- LICENSE +26/−0
- Network/Download.hsc +296/−0
- Setup.lhs +3/−0
- cbits/common.c +717/−0
- cbits/common.h +115/−0
- cbits/download.c +360/−0
- cbits/download.h +141/−0
- cbits/file.c +92/−0
- cbits/ftp.c +1175/−0
- cbits/ftperr.h +45/−0
- cbits/hs_download_utils.c +9/−0
- cbits/hs_download_utils.h +4/−0
- cbits/http.c +1226/−0
- cbits/httperr.h +44/−0
- download.cabal +80/−0
+ LICENSE view
@@ -0,0 +1,26 @@+Copyright (c) 2008 Don Stewart++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions+are met:++1. Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright+ notice, this list of conditions and the following disclaimer in the+ documentation and/or other materials provided with the distribution.++THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE+POSSIBILITY OF SUCH DAMAGE.
+ Network/Download.hsc view
@@ -0,0 +1,296 @@+--------------------------------------------------------------------+-- |+-- Module : Network.Download+-- Copyright : (c) Don Stewart+-- License : BSD3+--+-- Maintainer: Don Stewart <dons@galois.com>+-- Stability : provisional+-- Portability: posix+--+-- A binding to libdownload, a efficient, high‐level library for+-- retrieving files using Uniform Resource Locators (URLs). This+-- provides simple, uniform access to file, ftp and http resources.+-- Content may be retrieved as a string, bytestring or as parsed tags.+--+-- Example:+--+-- > s <- openURI "http://haskell.org"+--+--------------------------------------------------------------------++module Network.Download (+ openURI+ , openURIString+ , parseURI+ ) where++import Foreign+import Foreign.C.Types+import Foreign.C.String++import qualified Foreign.Concurrent as C+import Control.Monad+import Control.Monad.Instances+import Data.Word+import System.IO+import System.IO.Unsafe+import Control.Exception++import qualified Data.ByteString as S+import qualified Data.ByteString.Unsafe as S+import qualified Data.ByteString.Internal as S++import qualified Data.ByteString.Char8 as Char8+import Text.HTML.TagSoup++#include "download.h"++------------------------------------------------------------------------+-- Some API ideas:++-- | Download content specified by url (in RFC1738 form), using either+-- ftp, http or file protocols, returning the content as a strict+-- 'ByteString'.+--+-- If the url is malformed, a 'Left' value is returned.+-- Similarly, if an error occurs, 'Left' is returned, with a+-- protocol-specific error string.+--+-- If the file protocol is used, documents will be retrieved from the+-- local filesystem. If the ftp scheme is used, the FTP protocol+-- (RFC959) is used. If no user name or passoword are provided,+-- anonymous login, with user name "anonymous" and password "anonymous@<hostname>".+-- will be attempted.+--+-- If the http method is used, HTTP/1.1 will be used.+--+-- Examples:+--+-- > openURI "http://haskell.org"+--+openURI :: String -> IO (Either String S.ByteString)+openURI s = case parseURL s of+ Nothing -> return $ Left $ "Malformed url: "++ s+ Just url -> do+ e <- getFile url []+ case e of+ Left err -> return $ Left $ "Failed to connect: " ++ err+ Right s -> return $ Right s++-- | Like openURI, but returns the result as a 'String'+--+openURIString :: String -> IO (Either String String)+openURIString s = (fmap Char8.unpack) `fmap` openURI s++-- | Download the content via openURI, but return it as a list of parsed+-- tags using the tagsoup html parser.+--+parseURI :: String -> IO (Either String [Tag])+parseURI s = (fmap parseTags) `fmap` openURIString s++------------------------------------------------------------------------+-- Internal:++-- A data type, tracked by the garbage collector, that handles+-- libdownload's url structures.+--+data URL = URL {-# UNPACK #-} !(ForeignPtr URL_)+ deriving (Eq, Ord, Show)++data URL_ = URL_++-- | Takes a URL in the form of a String and splits it into its+-- components function according to the Common Internet Scheme Syntax+-- detailed in RFC1738.+--+parseURL :: String -> Maybe URL+parseURL s = unsafePerformIO $ withCString s $ \cstr -> do+ p <- c_parseURL cstr+ if p == nullPtr+ then return Nothing+ else do+ fp <- C.newForeignPtr p (c_freeURL p) -- The pointer returned by downloadMakeURL()+ -- or downloadParseURL() should be freed+ -- using downloadFreeURL()+ return . Just $ URL fp++-- struct url *downloadParseURL(const char *);+foreign import ccall unsafe "downloadParseURL"+ c_parseURL :: CString -> IO (Ptr URL_)++-- void downloadFreeURL(struct url *);+foreign import ccall unsafe "downloadFreeURL"+ c_freeURL :: Ptr URL_ -> IO ()++------------------------------------------------------------------------++-- The flags argument is a string of characters which specify+-- transfer options. The meaning of the individual flags is+-- scheme‐dependent+--+data Flag+ = Passive -- ^ a passive (rather than active) connection will be attempted.+ | Low -- ^ data sockets will be allocated in the low (or+ -- default) port range instead of the high port range+ | Direct -- ^ use a direct connection even if a proxy server is defined+ deriving (Eq,Show)++encodeFlag :: Flag -> Char+encodeFlag Passive = 'p'+encodeFlag Low = 'l'+encodeFlag Direct = 'd'++------------------------------------------------------------------------+++-- The flags argument is a string of characters which specify+-- transferoptions. The meaning of the individual flags is+-- scheme‐dependent, and is detailed in the appropriate section below.+--+getFile :: URL -> [Flag] -> IO (Either String S.ByteString)+getFile (URL fp) flags = do+ withForeignPtr fp $ \c_url ->+ withCString (map encodeFlag flags) $ \c_flags ->+ bracket+ (do sp <- c_get c_url c_flags+ dl <- lastErrorCode+ if dl /= dlerr_none || sp == nullPtr+ then return Nothing+ else return $ Just sp)++-- ll other functions return a stream pointer which may be used to+-- access the requested document, or NULL if an error occurred.+ --+ (maybe (return ()) c_fclose)++ $ \sp -> case sp of+ Nothing -> Left `fmap` lastError+ Just stream -> load stream++ -- but if we can stat it, we can be more efficient.+ where+ load :: Ptr CFile -> IO (Either String S.ByteString)+ load stream = do+ let start = 1024+ p <- mallocBytes start+ i <- c_fread p 1 (fromIntegral start) stream++ -- I hate C error handling+ dl_err <- lastErrorCode+ st_err <- c_ferror stream+ if dl_err /= dlerr_none || st_err /= 0+ then do free p+ Left `fmap` lastError+ else+ if i < fromIntegral start+ then do p' <- reallocBytes p (fromIntegral i)+ fp <- newForeignPtr finalizerFree p'+ return (Right $! S.fromForeignPtr fp 0 (fromIntegral i))++ else go stream p start++ -- duplicated too much code+ go stream p n = do+ let n' = 2 * n+ p' <- reallocBytes p n'+ i <- c_fread (p' `plusPtr` n) 1 (fromIntegral n) stream++ dl_err <- lastErrorCode+ st_err <- c_ferror stream+ if dl_err /= dlerr_none || st_err /= 0+ then do free p+ Left `fmap` lastError+ else+ if i < fromIntegral n+ then do let i' = n + fromIntegral i+ p'' <- reallocBytes p' i'+ fp <- newForeignPtr finalizerFree p''+ return (Right $! S.fromForeignPtr fp 0 (fromIntegral i'))+ else go stream p' n'++------------------------------------------------------------------------++-- size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream);+--+foreign import ccall unsafe "fread"+ c_fread :: Ptr Word8 -> CSize -> CSize -> Ptr CFile -> IO CSize++-- int fclose(FILE *fp);+--+foreign import ccall unsafe "fclose"+ c_fclose :: Ptr CFile -> IO () -- ignoring CInt++-- int ferror(FILE *stream);+--+foreign import ccall unsafe "ferror"+ c_ferror :: Ptr CFile -> IO CInt++------------------------------------------------------------------------++foreign import ccall unsafe "downloadGet"+ c_get :: Ptr URL_ -> CString -> IO (Ptr CFile)++foreign import ccall unsafe "hs_download_utils.h hs_download_last_error"+ c_lastErrorString :: IO CString++lastError :: IO String+lastError = peekCString =<< c_lastErrorString++-- Wrap this safely...+foreign import ccall unsafe "hs_download_utils.h hs_download_last_error_code"+ lastErrorCode :: IO DLError++------------------------------------------------------------------------++version :: String+version = #const_str USER_AGENT_STRING++------------------------------------------------------------------------++type URLLen = Int++#{enum URLLen,+ , url_schemelen = URL_SCHEMELEN+ , url_userlen = URL_USERLEN+ , url_pwdlen = URL_PWDLEN+ }++------------------------------------------------------------------------++newtype Scheme = Scheme { unScheme :: String }+ deriving (Eq, Show)++scheme_ftp, scheme_http, scheme_https, scheme_file :: Scheme+scheme_ftp = Scheme #const_str SCHEME_FTP+scheme_http = Scheme #const_str SCHEME_HTTP+scheme_https = Scheme #const_str SCHEME_HTTPS+scheme_file = Scheme #const_str SCHEME_FILE++------------------------------------------------------------------------++newtype DLError = DLError { unDLError :: CInt }+ deriving (Eq, Show)++#{enum DLError, DLError+ , dlerr_none = DLERR_NONE+ , dlerr_abort = DLERR_ABORT+ , dlerr_auth = DLERR_AUTH+ , dlerr_down = DLERR_DOWN+ , dlerr_exists = DLERR_EXISTS+ , dlerr_full = DLERR_FULL+ , dlerr_info = DLERR_INFO+ , dlerr_memory = DLERR_MEMORY+ , dlerr_moved = DLERR_MOVED+ , dlerr_network = DLERR_NETWORK+ , dlerr_ok = DLERR_OK+ , dlerr_proto = DLERR_PROTO+ , dlerr_resolv = DLERR_RESOLV+ , dlerr_server = DLERR_SERVER+ , dlerr_temp = DLERR_TEMP+ , dlerr_timeout = DLERR_TIMEOUT+ , dlerr_unavail = DLERR_UNAVAIL+ , dlerr_unknown = DLERR_UNKNOWN+ , dlerr_url = DLERR_URL+ , dlerr_verbose = DLERR_VERBOSE+ }
+ Setup.lhs view
@@ -0,0 +1,3 @@+#!/usr/bin/env runhaskell+> import Distribution.Simple+> main = defaultMain
+ cbits/common.c view
@@ -0,0 +1,717 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++#include <sys/cdefs.h>++#include <sys/param.h>+#include <sys/socket.h>+#include <sys/time.h>+#include <sys/uio.h>+#include <netinet/in.h>+#include <signal.h>++#include <errno.h>+#include <netdb.h>+#include <pwd.h>+#include <stdarg.h>+#include <stdlib.h>+#include <stdio.h>+#include <string.h>+#include <unistd.h>++#include "download.h"+#include "common.h"+++/*** Local data **************************************************************/++/*+ * Error messages for resolver errors+ */+static struct downloaderr _netdb_errlist[] = {+#ifdef EAI_NODATA+ { EAI_NODATA, DLERR_RESOLV, "Host not found" },+#endif+ { EAI_AGAIN, DLERR_TEMP, "Transient resolver failure" },+ { EAI_FAIL, DLERR_RESOLV, "Non-recoverable resolver failure" },+ { EAI_NONAME, DLERR_RESOLV, "No address record" },+ { -1, DLERR_UNKNOWN, "Unknown resolver error" }+};++/* End-of-Line */+static const char ENDL[2] = "\r\n";+++/*** Error-reporting functions ***********************************************/++/*+ * Map error code to string+ */+static struct downloaderr *+_download_finderr(struct downloaderr *p, int e)+{+ while (p->num != -1 && p->num != e)+ p++;+ return (p);+}++/*+ * Set error code+ */+void+_download_seterr(struct downloaderr *p, int e)+{+ p = _download_finderr(p, e);+ downloadLastErrCode = p->cat;+ snprintf(downloadLastErrString, MAXERRSTRING, "%s", p->string);+ DBG(fprintf(stderr, "ERR %d: %s\n", downloadLastErrCode, downloadLastErrString));+}++/*+ * Unset error code+ */+void+_download_unseterr(void)+{+ downloadLastErrCode = DLERR_NONE;+ memset(downloadLastErrString, 0, MAXERRSTRING);+}++/*+ * Set error code according to errno+ */+void+_download_syserr(void)+{+ switch (errno) {+ case 0:+ downloadLastErrCode = DLERR_OK;+ break;+ case EPERM:+ case EACCES:+ case EROFS:+ downloadLastErrCode = DLERR_AUTH;+ break;+ case ENOENT:+ case EISDIR: /* XXX */+ downloadLastErrCode = DLERR_UNAVAIL;+ break;+ case ENOMEM:+ downloadLastErrCode = DLERR_MEMORY;+ break;+ case EBUSY:+ case EAGAIN:+ downloadLastErrCode = DLERR_TEMP;+ break;+ case EEXIST:+ downloadLastErrCode = DLERR_EXISTS;+ break;+ case ENOSPC:+ downloadLastErrCode = DLERR_FULL;+ break;+ case EADDRINUSE:+ case EADDRNOTAVAIL:+ case ENETDOWN:+ case ENETUNREACH:+ case ENETRESET:+ case EHOSTUNREACH:+ downloadLastErrCode = DLERR_NETWORK;+ break;+ case ECONNABORTED:+ case ECONNRESET:+ downloadLastErrCode = DLERR_ABORT;+ break;+ case ETIMEDOUT:+ downloadLastErrCode = DLERR_TIMEOUT;+ break;+ case ECONNREFUSED:+ case EHOSTDOWN:+ downloadLastErrCode = DLERR_DOWN;+ break;+ default:+ downloadLastErrCode = DLERR_UNKNOWN;+ }+ snprintf(downloadLastErrString, MAXERRSTRING, "%s", strerror(errno));+}+++/*+ * Emit status message+ */+void+_download_info(const char *fmt, ...)+{+ va_list ap;++ va_start(ap, fmt);+ vfprintf(stderr, fmt, ap);+ va_end(ap);+ fputc('\n', stderr);+}+++/*** Network-related utility functions ***************************************/++/*+ * Return the default port for a scheme+ */+int+_download_default_port(const char *scheme)+{+ struct servent *se;++ if ((se = getservbyname(scheme, "tcp")) != NULL)+ return (ntohs(se->s_port));+ if (strcasecmp(scheme, SCHEME_FTP) == 0)+ return (FTP_DEFAULT_PORT);+ if (strcasecmp(scheme, SCHEME_HTTP) == 0)+ return (HTTP_DEFAULT_PORT);+ return (0);+}++/*+ * Return the default proxy port for a scheme+ */+int+_download_default_proxy_port(const char *scheme)+{+ if (strcasecmp(scheme, SCHEME_FTP) == 0)+ return (FTP_DEFAULT_PROXY_PORT);+ if (strcasecmp(scheme, SCHEME_HTTP) == 0)+ return (HTTP_DEFAULT_PROXY_PORT);+ return (0);+}+++/*+ * Create a connection for an existing descriptor.+ */+conn_t *+_download_reopen(int sd)+{+ conn_t *conn;++ /* allocate and fill connection structure */+ if ((conn = calloc(1, sizeof(*conn))) == NULL)+ return (NULL);+ conn->sd = sd;+ ++conn->ref;+ return (conn);+}+++/*+ * Bump a connection's reference count.+ */+conn_t *+_download_ref(conn_t *conn)+{+ ++conn->ref;+ return (conn);+}+++/*+ * Bind a socket to a specific local address+ */+int+_download_bind(int sd, int af, const char *addr)+{+ struct addrinfo hints, *res, *res0;+ int err;++ memset(&hints, 0, sizeof(hints));+ hints.ai_family = af;+ hints.ai_socktype = SOCK_STREAM;+ hints.ai_protocol = 0;+ if ((err = getaddrinfo(addr, NULL, &hints, &res0)) != 0)+ return (-1);+ for (res = res0; res; res = res->ai_next)+ if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)+ return (0);+ return (-1);+}+++/*+ * Establish a TCP connection to the specified port on the specified host.+ */+conn_t *+_download_connect(const char *host, int port, int af, int verbose)+{+ conn_t *conn;+ char pbuf[10];+ const char *bindaddr;+ struct addrinfo hints, *res, *res0;+ int sd, err;++ DBG(fprintf(stderr, "---> %s:%d\n", host, port));++ if (verbose)+ _download_info("looking up %s", host);++ /* look up host name and set up socket address structure */+ snprintf(pbuf, sizeof(pbuf), "%d", port);+ memset(&hints, 0, sizeof(hints));+ hints.ai_family = af;+ hints.ai_socktype = SOCK_STREAM;+ hints.ai_protocol = 0;+ if ((err = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {+ _netdb_seterr(err);+ return (NULL);+ }+ bindaddr = getenv("DOWNLOAD_BIND_ADDRESS");++ if (verbose)+ _download_info("connecting to %s:%d", host, port);++ /* try to connect */+ for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {+ if ((sd = socket(res->ai_family, res->ai_socktype,+ res->ai_protocol)) == -1)+ continue;+ if (bindaddr != NULL && *bindaddr != '\0' &&+ _download_bind(sd, res->ai_family, bindaddr) != 0) {+ _download_info("failed to bind to '%s'", bindaddr);+ close(sd);+ continue;+ }+ if (connect(sd, res->ai_addr, res->ai_addrlen) == 0)+ break;+ close(sd);+ }+ freeaddrinfo(res0);+ if (sd == -1) {+ _download_syserr();+ return (NULL);+ }++ if ((conn = _download_reopen(sd)) == NULL) {+ _download_syserr();+ close(sd);+ }+ return (conn);+}+++/*+ * Enable SSL on a connection.+ */+int+_download_ssl(conn_t *conn, int verbose)+{++#ifdef WITH_SSL+ /* Init the SSL library and context */+ if (!SSL_library_init()){+ fprintf(stderr, "SSL library init failed\n");+ return (-1);+ }++ SSL_load_error_strings();++ conn->ssl_meth = SSLv23_client_method();+ conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);+ SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);++ conn->ssl = SSL_new(conn->ssl_ctx);+ if (conn->ssl == NULL){+ fprintf(stderr, "SSL context creation failed\n");+ return (-1);+ }+ SSL_set_fd(conn->ssl, conn->sd);+ if (SSL_connect(conn->ssl) == -1){+ ERR_print_errors_fp(stderr);+ return (-1);+ }++ if (verbose) {+ X509_NAME *name;+ char *str;++ fprintf(stderr, "SSL connection established using %s\n",+ SSL_get_cipher(conn->ssl));+ conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);+ name = X509_get_subject_name(conn->ssl_cert);+ str = X509_NAME_oneline(name, 0, 0);+ printf("Certificate subject: %s\n", str);+ free(str);+ name = X509_get_issuer_name(conn->ssl_cert);+ str = X509_NAME_oneline(name, 0, 0);+ printf("Certificate issuer: %s\n", str);+ free(str);+ }++ return (0);+#else+ (void)conn;+ (void)verbose;+ DBG(fprintf(stderr, "SSL support disabled\n"));+ return (-1);+#endif+}++/*+ * Read a character from a connection w/ timeout+ */+ssize_t+_download_read(conn_t *conn, char *buf, size_t len)+{+ struct timeval wait;+ fd_set readfds;+ ssize_t rlen, total;+ int r;++ if (downloadTimeout) {+ FD_ZERO(&readfds);+ }++ total = 0;+ while (len > 0) {+ while (downloadTimeout && !FD_ISSET(conn->sd, &readfds)) {+ FD_SET(conn->sd, &readfds);+ wait.tv_sec = downloadTimeout / 1000;+ wait.tv_usec = (downloadTimeout % 1000) * 1000;+ errno = 0;+ r = select(conn->sd + 1, &readfds, NULL, NULL, &wait);+ if (r == -1) {+ if (errno == EINTR && downloadRestartCalls)+ continue;+ _download_syserr();+ return (-1);+ } else if (r == 0) {+ errno = ETIMEDOUT;+ _download_syserr();+ return (-1);+ }+ }+#ifdef WITH_SSL+ if (conn->ssl != NULL)+ rlen = SSL_read(conn->ssl, buf, len);+ else+#endif+ rlen = read(conn->sd, buf, len);+ if (rlen == 0)+ break;+ if (rlen < 0) {+ if (errno == EINTR && downloadRestartCalls)+ continue;+ return (-1);+ }+ len -= rlen;+ buf += rlen;+ total += rlen;+ }+ return (total);+}+++/*+ * Read a line of text from a connection w/ timeout+ */+#define MIN_BUF_SIZE 1024++int+_download_getln(conn_t *conn)+{+ char *tmp;+ size_t tmpsize;+ ssize_t len;+ char c;++ if (conn->buf == NULL) {+ if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {+ errno = ENOMEM;+ return (-1);+ }+ conn->bufsize = MIN_BUF_SIZE;+ }++ conn->buf[0] = '\0';+ conn->buflen = 0;++ do {+ len = _download_read(conn, &c, 1);+ if (len == -1)+ return (-1);+ if (len == 0)+ break;+ conn->buf[conn->buflen++] = c;+ if (conn->buflen == conn->bufsize) {+ tmp = conn->buf;+ tmpsize = conn->bufsize * 2 + 1;+ if ((tmp = realloc(tmp, tmpsize)) == NULL) {+ errno = ENOMEM;+ return (-1);+ }+ conn->buf = tmp;+ conn->bufsize = tmpsize;+ }+ } while (c != '\n');++ conn->buf[conn->buflen] = '\0';+ DBG(fprintf(stderr, "<<< %s", conn->buf));+ return (0);+}+++/*+ * Write to a connection w/ timeout+ */+ssize_t+_download_write(conn_t *conn, const char *buf, size_t len)+{+ struct iovec iov;++ iov.iov_base = __DECONST(char *, buf);+ iov.iov_len = len;+ return _download_writev(conn, &iov, 1);+}++/*+ * Write a vector to a connection w/ timeout+ * Note: can modify the iovec.+ */+ssize_t+_download_writev(conn_t *conn, struct iovec *iov, int iovcnt)+{+ struct timeval wait;+ fd_set writefds;+ ssize_t wlen, total;+ int r;++ if (downloadTimeout) {+ FD_ZERO(&writefds);+ /*+ gettimeofday(&timeout, NULL);+ timeout.tv_sec += downloadTimeout;+ */+ }++ total = 0;+ while (iovcnt > 0) {+ while (downloadTimeout && !FD_ISSET(conn->sd, &writefds)) {+ FD_SET(conn->sd, &writefds);+ /*+ gettimeofday(&now, NULL);+ wait.tv_sec = timeout.tv_sec - now.tv_sec;+ wait.tv_usec = timeout.tv_usec - now.tv_usec;+ if (wait.tv_usec < 0) {+ wait.tv_usec += 1000000;+ wait.tv_sec--;+ }+ if (wait.tv_sec < 0) {+ errno = ETIMEDOUT;+ _download_syserr();+ return (-1);+ }+ */+ wait.tv_sec = downloadTimeout / 1000;+ wait.tv_usec = (downloadTimeout % 1000) * 1000;+ errno = 0;+ r = select(conn->sd + 1, NULL, &writefds, NULL, &wait);+ if (r == -1) {+ if (errno == EINTR && downloadRestartCalls)+ continue;+ return (-1);+ } else if (r == 0) {+ errno = ETIMEDOUT;+ _download_syserr();+ return (-1);+ }+ }+ errno = 0;++ /* Protect against SIGPIPE in the case of a timeout */+ signal(SIGPIPE, SIG_IGN);+#ifdef WITH_SSL+ if (conn->ssl != NULL)+ wlen = SSL_write(conn->ssl,+ iov->iov_base, iov->iov_len);+ else+#endif+ wlen = writev(conn->sd, iov, iovcnt);+ signal(SIGPIPE, SIG_DFL);++ if (wlen == 0) {+ /* we consider a short write a failure */+ errno = EPIPE;+ _download_syserr();+ return (-1);+ }+ if (wlen < 0) {+ if (errno == EINTR && downloadRestartCalls)+ continue;+ return (-1);+ }+ total += wlen;+ while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {+ wlen -= iov->iov_len;+ iov++;+ iovcnt--;+ }+ if (iovcnt > 0) {+ iov->iov_len -= wlen;+ iov->iov_base = __DECONST(char *, iov->iov_base) + wlen;+ }+ }+ return (total);+}+++/*+ * Write a line of text to a connection w/ timeout+ */+int+_download_putln(conn_t *conn, const char *str, size_t len)+{+ struct iovec iov[2];+ int ret;++ DBG(fprintf(stderr, ">>> %s\n", str));+ iov[0].iov_base = __DECONST(char *, str);+ iov[0].iov_len = len;+ iov[1].iov_base = __DECONST(char *, ENDL);+ iov[1].iov_len = sizeof(ENDL);+ if (len == 0)+ ret = _download_writev(conn, &iov[1], 1);+ else+ ret = _download_writev(conn, iov, 2);+ if (ret == -1)+ return (-1);+ return (0);+}+++/*+ * Close connection+ */+int+_download_close(conn_t *conn)+{+ int ret;++ if (--conn->ref > 0)+ return (0);+ ret = close(conn->sd);+ free(conn->buf);+ free(conn);+ return (ret);+}++/*** Authentication-related utility functions ********************************/++static const char *+_download_read_word(FILE *f)+{+ static char word[1024];++ if (fscanf(f, " %1024s ", word) != 1)+ return (NULL);+ return (word);+}++/*+ * Get authentication data for a URL from .netrc+ */+int+_download_netrc_auth(struct url *url)+{+ char fn[PATH_MAX];+ const char *word;+ char *p;+ FILE *f;++ if ((p = getenv("NETRC")) != NULL) {+ if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {+ _download_info("$NETRC specifies a file name "+ "longer than PATH_MAX");+ return (-1);+ }+ } else {+ if ((p = getenv("HOME")) != NULL) {+ struct passwd *pwd;++ if ((pwd = getpwuid(getuid())) == NULL ||+ (p = pwd->pw_dir) == NULL)+ return (-1);+ }+ if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))+ return (-1);+ }++ if ((f = fopen(fn, "r")) == NULL)+ return (-1);+ while ((word = _download_read_word(f)) != NULL) {+ if (strcmp(word, "default") == 0) {+ DBG(_download_info("Using default .netrc settings"));+ break;+ }+ if (strcmp(word, "machine") == 0 &&+ (word = _download_read_word(f)) != NULL &&+ strcasecmp(word, url->host) == 0) {+ DBG(_download_info("Using .netrc settings for %s", word));+ break;+ }+ }+ if (word == NULL)+ goto ferr;+ while ((word = _download_read_word(f)) != NULL) {+ if (strcmp(word, "login") == 0) {+ if ((word = _download_read_word(f)) == NULL)+ goto ferr;+ if (snprintf(url->user, sizeof(url->user),+ "%s", word) > (int)sizeof(url->user)) {+ _download_info("login name in .netrc is too long");+ url->user[0] = '\0';+ }+ } else if (strcmp(word, "password") == 0) {+ if ((word = _download_read_word(f)) == NULL)+ goto ferr;+ if (snprintf(url->pwd, sizeof(url->pwd),+ "%s", word) > (int)sizeof(url->pwd)) {+ _download_info("password in .netrc is too long");+ url->pwd[0] = '\0';+ }+ } else if (strcmp(word, "account") == 0) {+ if ((word = _download_read_word(f)) == NULL)+ goto ferr;+ /* XXX not supported! */+ } else {+ break;+ }+ }+ fclose(f);+ return (0);+ferr:+ fclose(f);+ return (-1);+}
+ cbits/common.h view
@@ -0,0 +1,115 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ *+ */++#ifndef COMMON_H_INCLUDED+#define COMMON_H_INCLUDED++/* BSD seems to use this alot - we don't have it */+#define __DECONST(type, v) (type)((uintptr_t)(void *)(v))++#define FTP_DEFAULT_PORT 21+#define HTTP_DEFAULT_PORT 80+#define FTP_DEFAULT_PROXY_PORT 21+#define HTTP_DEFAULT_PROXY_PORT 3128++#ifdef WITH_SSL+#include <openssl/crypto.h>+#include <openssl/x509.h>+#include <openssl/pem.h>+#include <openssl/ssl.h>+#include <openssl/err.h>+#endif++/* Connection */+struct connection {+ int sd; /* socket descriptor */+ char *buf; /* buffer */+ size_t bufsize; /* buffer size */+ size_t buflen; /* length of buffer contents */+ int err; /* last protocol reply code */+ fd_set readfds;+#ifdef WITH_SSL+ SSL *ssl; /* SSL handle */+ SSL_CTX *ssl_ctx; /* SSL context */+ X509 *ssl_cert; /* server certificate */+ SSL_METHOD *ssl_meth; /* SSL method */+#endif+ int ref; /* reference count */+};+typedef struct connection conn_t;++/* Structure used for error message lists */+struct downloaderr {+ const int num;+ const int cat;+ const char *string;+};++/* for _download_writev */+struct iovec;++void _download_seterr(struct downloaderr *, int);+void _download_unseterr(void);+void _download_syserr(void);+void _download_info(const char *, ...);+int _download_default_port(const char *);+int _download_default_proxy_port(const char *);+int _download_bind(int, int, const char *);+conn_t *_download_connect(const char *, int, int, int);+conn_t *_download_reopen(int);+conn_t *_download_ref(conn_t *);+int _download_ssl(conn_t *, int);+ssize_t _download_read(conn_t *, char *, size_t);+int _download_getln(conn_t *);+ssize_t _download_write(conn_t *, const char *, size_t);+ssize_t _download_writev(conn_t *, struct iovec *, int);+int _download_putln(conn_t *, const char *, size_t);+int _download_close(conn_t *);+int _download_netrc_auth(struct url *url);++#define _ftp_seterr(n) _download_seterr(_ftp_errlist, n)+#define _http_seterr(n) _download_seterr(_http_errlist, n)+#define _netdb_seterr(n) _download_seterr(_netdb_errlist, n)+#define _url_seterr(n) _download_seterr(_url_errlist, n)++#ifdef DEBUG+#define DBG(x) do { if (downloadDebug) { x; } } while (0)+#else+#define DBG(x) do { } while (0)+#endif++/*+ * Check whether a particular flag is set+ */+#define CHECK_FLAG(x) (flags && strchr(flags, (x)))++#endif /*COMMON_H_INCLUDED*/
+ cbits/download.c view
@@ -0,0 +1,360 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++#include <sys/cdefs.h>++#include <sys/param.h>+#include <sys/errno.h>++#include <ctype.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "download.h"+#include "common.h"++auth_t downloadAuthMethod;+int downloadLastErrCode;+char downloadLastErrString[MAXERRSTRING];+int downloadTimeout;+int downloadRestartCalls = 1;+int downloadDebug;+++/*** Local data **************************************************************/++/*+ * Error messages for parser errors+ */+#define URL_MALFORMED 1+#define URL_BAD_SCHEME 2+#define URL_BAD_PORT 3+static struct downloaderr _url_errlist[] = {+ { URL_MALFORMED, DLERR_URL, "Malformed URL" },+ { URL_BAD_SCHEME, DLERR_URL, "Invalid URL scheme" },+ { URL_BAD_PORT, DLERR_URL, "Invalid server port" },+ { -1, DLERR_UNKNOWN, "Unknown parser error" }+};+++/*** Public API **************************************************************/++/*+ * Select the appropriate protocol for the URL scheme, and return a+ * read-only stream connected to the document referenced by the URL.+ * Also fill out the struct url_stat.+ */+FILE *+downloadXGet(struct url *URL, struct url_stat *us, const char *flags)+{+ int direct;++ direct = CHECK_FLAG('d');+ if (us != NULL) {+ us->size = -1;+ us->atime = us->mtime = 0;+ }+ if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)+ return (downloadXGetFile(URL, us, flags));+ else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)+ return (downloadXGetFTP(URL, us, flags));+ else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)+ return (downloadXGetHTTP(URL, us, flags));+ else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)+ return (downloadXGetHTTP(URL, us, flags));+ _url_seterr(URL_BAD_SCHEME);+ return (NULL);+}++/*+ * Select the appropriate protocol for the URL scheme, and return a+ * read-only stream connected to the document referenced by the URL.+ */+FILE *+downloadGet(struct url *URL, const char *flags)+{+ return (downloadXGet(URL, NULL, flags));+}++/*+ * Select the appropriate protocol for the URL scheme, and return the+ * size of the document referenced by the URL if it exists.+ */+int+downloadStat(struct url *URL, struct url_stat *us, const char *flags)+{+ int direct;++ direct = CHECK_FLAG('d');+ if (us != NULL) {+ us->size = -1;+ us->atime = us->mtime = 0;+ }+ if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)+ return (downloadStatFile(URL, us, flags));+ else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)+ return (downloadStatFTP(URL, us, flags));+ else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)+ return (downloadStatHTTP(URL, us, flags));+ else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)+ return (downloadStatHTTP(URL, us, flags));+ _url_seterr(URL_BAD_SCHEME);+ return (-1);+}++/*+ * Attempt to parse the given URL; if successful, call downloadXGet().+ */+FILE *+downloadXGetURL(const char *URL, struct url_stat *us, const char *flags)+{+ struct url *u;+ FILE *f;++ if ((u = downloadParseURL(URL)) == NULL)+ return (NULL);++ f = downloadXGet(u, us, flags);++ downloadFreeURL(u);+ return (f);+}++/*+ * Attempt to parse the given URL; if successful, call downloadGet().+ */+FILE *+downloadGetURL(const char *URL, const char *flags)+{+ return (downloadXGetURL(URL, NULL, flags));+}++/*+ * Attempt to parse the given URL; if successful, call downloadStat().+ */+int+downloadStatURL(const char *URL, struct url_stat *us, const char *flags)+{+ struct url *u;+ int s;++ if ((u = downloadParseURL(URL)) == NULL)+ return (-1);++ s = downloadStat(u, us, flags);++ downloadFreeURL(u);+ return (s);+}++/*+ * Make a URL+ */+struct url *+downloadMakeURL(const char *scheme, const char *host, int port, const char *doc,+ const char *user, const char *pwd)+{+ struct url *u;++ if (!scheme || (!host && !doc)) {+ _url_seterr(URL_MALFORMED);+ return (NULL);+ }++ if (port < 0 || port > 65535) {+ _url_seterr(URL_BAD_PORT);+ return (NULL);+ }++ /* allocate struct url */+ if ((u = calloc(1, sizeof(*u))) == NULL) {+ _download_syserr();+ return (NULL);+ }++ if ((u->doc = strdup(doc ? doc : "/")) == NULL) {+ _download_syserr();+ free(u);+ return (NULL);+ }++#define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)+ seturl(scheme);+ seturl(host);+ seturl(user);+ seturl(pwd);+#undef seturl+ u->port = port;++ return (u);+}++/*+ * Split an URL into components. URL syntax is:+ * [method:/][/[user[:pwd]@]host[:port]/][document]+ * This almost, but not quite, RFC1738 URL syntax.+ */+struct url *+downloadParseURL(const char *URL)+{+ char *doc;+ const char *p, *q;+ struct url *u;+ int i;++ /* allocate struct url */+ if ((u = calloc(1, sizeof(*u))) == NULL) {+ _download_syserr();+ return (NULL);+ }++ /* scheme name */+ if ((p = strstr(URL, ":/"))) {+ snprintf(u->scheme, URL_SCHEMELEN+1,+ "%.*s", (int)(p - URL), URL);+ URL = ++p;+ /*+ * Only one slash: no host, leave slash as part of document+ * Two slashes: host follows, strip slashes+ */+ if (URL[1] == '/')+ URL = (p += 2);+ } else {+ p = URL;+ }+ if (!*URL || *URL == '/' || *URL == '.' ||+ (u->scheme[0] == '\0' &&+ strchr(URL, '/') == NULL && strchr(URL, ':') == NULL))+ goto nohost;++ p = strpbrk(URL, "/@");+ if (p && *p == '@') {+ /* username */+ for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++)+ if (i < URL_USERLEN)+ u->user[i++] = *q;++ /* password */+ if (*q == ':')+ for (q++, i = 0; (*q != ':') && (*q != '@'); q++)+ if (i < URL_PWDLEN)+ u->pwd[i++] = *q;++ p++;+ } else {+ p = URL;+ }++ /* hostname */+#ifdef INET6+ if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&+ (*++q == '\0' || *q == '/' || *q == ':')) {+ if ((i = q - p - 2) > MAXHOSTNAMELEN)+ i = MAXHOSTNAMELEN;+ strncpy(u->host, ++p, i);+ p = q;+ } else+#endif+ for (i = 0; *p && (*p != '/') && (*p != ':'); p++)+ if (i < MAXHOSTNAMELEN)+ u->host[i++] = *p;++ /* port */+ if (*p == ':') {+ for (q = ++p; *q && (*q != '/'); q++)+ if (isdigit(*q))+ u->port = u->port * 10 + (*q - '0');+ else {+ /* invalid port */+ _url_seterr(URL_BAD_PORT);+ goto ouch;+ }+ p = q;+ }++nohost:+ /* document */+ if (!*p)+ p = "/";++ if (strcasecmp(u->scheme, SCHEME_HTTP) == 0 ||+ strcasecmp(u->scheme, SCHEME_HTTPS) == 0) {+ const char hexnums[] = "0123456789abcdef";++ /* percent-escape whitespace. */+ if ((doc = malloc(strlen(p) * 3 + 1)) == NULL) {+ _download_syserr();+ goto ouch;+ }+ u->doc = doc;+ while (*p != '\0') {+ if (!isspace(*p)) {+ *doc++ = *p++;+ } else {+ *doc++ = '%';+ *doc++ = hexnums[((unsigned int)*p) >> 4];+ *doc++ = hexnums[((unsigned int)*p) & 0xf];+ p++;+ }+ }+ *doc = '\0';+ } else if ((u->doc = strdup(p)) == NULL) {+ _download_syserr();+ goto ouch;+ }++ DBG(fprintf(stderr,+ "scheme: [%s]\n"+ "user: [%s]\n"+ "password: [%s]\n"+ "host: [%s]\n"+ "port: [%d]\n"+ "document: [%s]\n",+ u->scheme, u->user, u->pwd,+ u->host, u->port, u->doc));++ return (u);++ouch:+ free(u);+ return (NULL);+}++/*+ * Free a URL+ */+void+downloadFreeURL(struct url *u)+{+ free(u->doc);+ free(u);+}
+ cbits/download.h view
@@ -0,0 +1,141 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ *+ */++#ifndef DOWNLOAD_H_INCLUDED+#define DOWNLOAD_H_INCLUDED++#include <stdio.h>+#include <sys/param.h> /* MAXHOSTNAMELEN */++#define USER_AGENT_STRING "libdownload/1.0"++#define URL_SCHEMELEN 16+#define URL_USERLEN 256+#define URL_PWDLEN 256++struct url {+ char scheme[URL_SCHEMELEN+1];+ char user[URL_USERLEN+1];+ char pwd[URL_PWDLEN+1];+ char host[MAXHOSTNAMELEN+1];+ int port;+ char *doc;+ off_t offset;+ size_t length;+};++struct url_stat {+ off_t size;+ time_t atime;+ time_t mtime;+};++/* Recognized schemes */+#define SCHEME_FTP "ftp"+#define SCHEME_HTTP "http"+#define SCHEME_HTTPS "https"+#define SCHEME_FILE "file"++/* Error codes */+#define DLERR_NONE 0+#define DLERR_ABORT 1+#define DLERR_AUTH 2+#define DLERR_DOWN 3+#define DLERR_EXISTS 4+#define DLERR_FULL 5+#define DLERR_INFO 6+#define DLERR_MEMORY 7+#define DLERR_MOVED 8+#define DLERR_NETWORK 9+#define DLERR_OK 10+#define DLERR_PROTO 11+#define DLERR_RESOLV 12+#define DLERR_SERVER 13+#define DLERR_TEMP 14+#define DLERR_TIMEOUT 15+#define DLERR_UNAVAIL 16+#define DLERR_UNKNOWN 17+#define DLERR_URL 18+#define DLERR_VERBOSE 19++__BEGIN_DECLS++/* FILE-specific functions */+FILE *downloadXGetFile(struct url *, struct url_stat *, const char *);+FILE *downloadGetFile(struct url *, const char *);+int downloadStatFile(struct url *, struct url_stat *, const char *);++/* HTTP-specific functions */+FILE *downloadXGetHTTP(struct url *, struct url_stat *, const char *);+FILE *downloadGetHTTP(struct url *, const char *);+int downloadStatHTTP(struct url *, struct url_stat *, const char *);++/* FTP-specific functions */+FILE *downloadXGetFTP(struct url *, struct url_stat *, const char *);+FILE *downloadGetFTP(struct url *, const char *);+int downloadStatFTP(struct url *, struct url_stat *, const char *);++/* Generic functions */+FILE *downloadXGetURL(const char *, struct url_stat *, const char *);+FILE *downloadGetURL(const char *, const char *);+int downloadStatURL(const char *, struct url_stat *, const char *);+FILE *downloadXGet(struct url *, struct url_stat *, const char *);+FILE *downloadGet(struct url *, const char *);+int downloadStat(struct url *, struct url_stat *, const char *);++/* URL parsing */+struct url *downloadMakeURL(const char *, const char *, int, const char *,+ const char *, const char *);+struct url *downloadParseURL(const char *);+void downloadFreeURL(struct url *);++__END_DECLS++/* Authentication */+typedef int (*auth_t)(struct url *);+extern auth_t downloadAuthMethod;++/* Last error code */+extern int downloadLastErrCode;+#define MAXERRSTRING 256+extern char downloadLastErrString[MAXERRSTRING];++/* I/O timeout */+extern int downloadTimeout;++/* Restart interrupted syscalls */+extern int downloadRestartCalls;++/* Extra verbosity */+extern int downloadDebug;++#endif /*DOWNLOAD_H_INCLUDED*/
+ cbits/file.c view
@@ -0,0 +1,92 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++#include <sys/cdefs.h>++#include <sys/param.h>+#include <sys/stat.h>++#include <dirent.h>+#include <stdio.h>+#include <string.h>++#include "download.h"+#include "common.h"++FILE *+downloadXGetFile(struct url *u, struct url_stat *us, const char *flags)+{+ FILE *f;++ if (us && downloadStatFile(u, us, flags) == -1)+ return (NULL);++ f = fopen(u->doc, "r");++ if (f == NULL)+ _download_syserr();++ if (u->offset && fseeko(f, u->offset, SEEK_SET) == -1) {+ fclose(f);+ _download_syserr();+ }++ return (f);+}++FILE *+downloadGetFile(struct url *u, const char *flags)+{+ return (downloadXGetFile(u, NULL, flags));+}++static int+_download_stat_file(const char *fn, struct url_stat *us)+{+ struct stat sb;++ us->size = -1;+ us->atime = us->mtime = 0;+ if (stat(fn, &sb) == -1) {+ _download_syserr();+ return (-1);+ }+ us->size = sb.st_size;+ us->atime = sb.st_atime;+ us->mtime = sb.st_mtime;+ return (0);+}++int+downloadStatFile(struct url *u, struct url_stat *us, const char *flags)+{+ return (_download_stat_file(u->doc, us));+}
+ cbits/ftp.c view
@@ -0,0 +1,1175 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++#include <sys/cdefs.h>++/*+ * Portions of this code were taken from or based on ftpio.c:+ *+ * ----------------------------------------------------------------------------+ * "THE BEER-WARE LICENSE" (Revision 42):+ * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you+ * can do whatever you want with this stuff. If we meet some day, and you think+ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp+ * ----------------------------------------------------------------------------+ *+ * Major Changelog:+ *+ * Dag-Erling Coïdan Smørgrav+ * 9 Jun 1998+ *+ * Incorporated into libfetch+ *+ * Jordan K. Hubbard+ * 17 Jan 1996+ *+ * Turned inside out. Now returns xfers as new file ids, not as a special+ * `state' of FTP_t+ *+ * $ftpioId: ftpio.c,v 1.30 1998/04/11 07:28:53 phk Exp $+ *+ */++#include <sys/param.h>+#include <sys/socket.h>+#include <netinet/in.h>++#include <ctype.h>+#include <libio.h>+#include <err.h>+#include <errno.h>+#include <fcntl.h>+#include <netdb.h>+#include <stdarg.h>+#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <time.h>+#include <unistd.h>++#include "download.h"+#include "common.h"+#include "ftperr.h"++#define FTP_ANONYMOUS_USER "anonymous"++#define FTP_CONNECTION_ALREADY_OPEN 125+#define FTP_OPEN_DATA_CONNECTION 150+#define FTP_OK 200+#define FTP_FILE_STATUS 213+#define FTP_SERVICE_READY 220+#define FTP_TRANSFER_COMPLETE 226+#define FTP_PASSIVE_MODE 227+#define FTP_LPASSIVE_MODE 228+#define FTP_EPASSIVE_MODE 229+#define FTP_LOGGED_IN 230+#define FTP_FILE_ACTION_OK 250+#define FTP_DIRECTORY_CREATED 257 /* multiple meanings */+#define FTP_FILE_CREATED 257 /* multiple meanings */+#define FTP_WORKING_DIRECTORY 257 /* multiple meanings */+#define FTP_NEED_PASSWORD 331+#define FTP_NEED_ACCOUNT 332+#define FTP_FILE_OK 350+#define FTP_SYNTAX_ERROR 500+#define FTP_PROTOCOL_ERROR 999++static struct url cached_host;+static conn_t *cached_connection;++#define isftpreply(foo) (isdigit(foo[0]) && isdigit(foo[1]) \+ && isdigit(foo[2]) \+ && (foo[3] == ' ' || foo[3] == '\0'))+#define isftpinfo(foo) (isdigit(foo[0]) && isdigit(foo[1]) \+ && isdigit(foo[2]) && foo[3] == '-')++/* for easy redirection */+FILE *_http_request(struct url *, const char *, struct url_stat *,+ struct url *, const char *);++/*+ * Translate IPv4 mapped IPv6 address to IPv4 address+ */+static void+unmappedaddr(struct sockaddr_in6 *sin6)+{+ struct sockaddr_in *sin4;+ u_int32_t addr;+ int port;++ if (sin6->sin6_family != AF_INET6 ||+ !IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr))+ return;+ sin4 = (struct sockaddr_in *)sin6;+ addr = *(u_int32_t *)&sin6->sin6_addr.s6_addr[12];+ port = sin6->sin6_port;+ memset(sin4, 0, sizeof(struct sockaddr_in));+ sin4->sin_addr.s_addr = addr;+ sin4->sin_port = port;+ sin4->sin_family = AF_INET;+ //sin6->sin_len = sizeof(struct sockaddr_in);+}++/*+ * Get server response+ */+static int+_ftp_chkerr(conn_t *conn)+{+ if (_download_getln(conn) == -1) {+ _download_syserr();+ return (-1);+ }+ if (isftpinfo(conn->buf)) {+ while (conn->buflen && !isftpreply(conn->buf)) {+ if (_download_getln(conn) == -1) {+ _download_syserr();+ return (-1);+ }+ }+ }++ while (conn->buflen && isspace(conn->buf[conn->buflen - 1]))+ conn->buflen--;+ conn->buf[conn->buflen] = '\0';++ if (!isftpreply(conn->buf)) {+ _ftp_seterr(FTP_PROTOCOL_ERROR);+ return (-1);+ }++ conn->err = (conn->buf[0] - '0') * 100+ + (conn->buf[1] - '0') * 10+ + (conn->buf[2] - '0');++ return (conn->err);+}++/*+ * Send a command and check reply+ */+static int+_ftp_cmd(conn_t *conn, const char *fmt, ...)+{+ va_list ap;+ size_t len;+ char *msg;+ int r;++ va_start(ap, fmt);+ len = vasprintf(&msg, fmt, ap);+ va_end(ap);++ if (msg == NULL) {+ errno = ENOMEM;+ _download_syserr();+ return (-1);+ }++ r = _download_putln(conn, msg, len);+ free(msg);++ if (r == -1) {+ _download_syserr();+ return (-1);+ }++ return (_ftp_chkerr(conn));+}++/*+ * Return a pointer to the filename part of a path+ */+static const char *+_ftp_filename(const char *file, int *len, int *type)+{+ const char *s;++ if ((s = strrchr(file, '/')) == NULL)+ s = file;+ else+ s = s + 1;+ *len = strlen(s);+ if (*len > 7 && strncmp(s + *len - 7, ";type=", 6) == 0) {+ *type = s[*len - 1];+ *len -= 7;+ } else {+ *type = '\0';+ }+ return (s);+}++/*+ * Get current working directory from the reply to a CWD, PWD or CDUP+ * command.+ */+static int+_ftp_pwd(conn_t *conn, char *pwd, size_t pwdlen)+{+ char *src, *dst, *end;+ int q;++ if (conn->err != FTP_WORKING_DIRECTORY &&+ conn->err != FTP_FILE_ACTION_OK)+ return (FTP_PROTOCOL_ERROR);+ end = conn->buf + conn->buflen;+ src = conn->buf + 4;+ if (src >= end || *src++ != '"')+ return (FTP_PROTOCOL_ERROR);+ for (q = 0, dst = pwd; src < end && pwdlen--; ++src) {+ if (!q && *src == '"')+ q = 1;+ else if (q && *src != '"')+ break;+ else if (q)+ *dst++ = '"', q = 0;+ else+ *dst++ = *src;+ }+ if (!pwdlen)+ return (FTP_PROTOCOL_ERROR);+ *dst = '\0';+ return (FTP_OK);+}++/*+ * Change working directory to the directory that contains the specified+ * file.+ */+static int+_ftp_cwd(conn_t *conn, const char *file)+{+ const char *beg, *end;+ char pwd[PATH_MAX];+ int e, i, len;++ if ((end = strrchr(file, '/')) == NULL)+ return (0);+ if ((e = _ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY ||+ (e = _ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {+ _ftp_seterr(e);+ return (-1);+ }+ for (;;) {+ len = strlen(pwd);+ /* look for a common prefix */+ for (i = 0; i <= len && i <= end - file; ++i)+ if (pwd[i] != file[i])+ break;+ if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/'))+ break;+ if ((e = _ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK ||+ (e = _ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY ||+ (e = _ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {+ _ftp_seterr(e);+ return (-1);+ }+ }+ for (beg = file + i; beg < end; beg = file + i + 1) {+ while (*beg == '/')+ ++beg, ++i;+ for (++i; file + i < end && file[i] != '/'; ++i)+ /* nothing */;+ if(beg < end)+ e = _ftp_cmd(conn, "CWD %.*s", file + i - beg, beg);+ if (e != FTP_FILE_ACTION_OK) {+ _ftp_seterr(e);+ return (-1);+ }+ }+ return (0);+}++/*+ * Set transfer mode and data type+ */+static int+_ftp_mode_type(conn_t *conn, int mode, int type)+{+ int e;++ switch (mode) {+ case 0:+ case 's':+ mode = 'S';+ case 'S':+ break;+ default:+ return (FTP_PROTOCOL_ERROR);+ }+ if ((e = _ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) {+ if (mode == 'S') {+ /*+ * Stream mode is supposed to be the default - so+ * much so that some servers not only do not+ * support any other mode, but do not support the+ * MODE command at all.+ *+ * If "MODE S" fails, it is unlikely that we+ * previously succeeded in setting a different+ * mode. Therefore, we simply hope that the+ * server is already in the correct mode, and+ * silently ignore the failure.+ */+ } else {+ return (e);+ }+ }++ switch (type) {+ case 0:+ case 'i':+ type = 'I';+ case 'I':+ break;+ case 'a':+ type = 'A';+ case 'A':+ break;+ case 'd':+ type = 'D';+ case 'D':+ /* can't handle yet */+ default:+ return (FTP_PROTOCOL_ERROR);+ }+ if ((e = _ftp_cmd(conn, "TYPE %c", type)) != FTP_OK)+ return (e);++ return (FTP_OK);+}++/*+ * Request and parse file stats+ */+static int+_ftp_stat(conn_t *conn, const char *file, struct url_stat *us)+{+ char *ln;+ const char *filename;+ int filenamelen, type;+ struct tm tm;+ time_t t;+ int e;++ us->size = -1;+ us->atime = us->mtime = 0;++ filename = _ftp_filename(file, &filenamelen, &type);++ if ((e = _ftp_mode_type(conn, 0, type)) != FTP_OK) {+ _ftp_seterr(e);+ return (-1);+ }++ e = _ftp_cmd(conn, "SIZE %.*s", filenamelen, filename);+ if (e != FTP_FILE_STATUS) {+ _ftp_seterr(e);+ return (-1);+ }+ for (ln = conn->buf + 4; *ln && isspace(*ln); ln++)+ /* nothing */ ;+ for (us->size = 0; *ln && isdigit(*ln); ln++)+ us->size = us->size * 10 + *ln - '0';+ if (*ln && !isspace(*ln)) {+ _ftp_seterr(FTP_PROTOCOL_ERROR);+ us->size = -1;+ return (-1);+ }+ if (us->size == 0)+ us->size = -1;+ DBG(fprintf(stderr, "size: [%lld]\n", (long long)us->size));++ e = _ftp_cmd(conn, "MDTM %.*s", filenamelen, filename);+ if (e != FTP_FILE_STATUS) {+ _ftp_seterr(e);+ return (-1);+ }+ for (ln = conn->buf + 4; *ln && isspace(*ln); ln++)+ /* nothing */ ;+ switch (strspn(ln, "0123456789")) {+ case 14:+ break;+ case 15:+ ln++;+ ln[0] = '2';+ ln[1] = '0';+ break;+ default:+ _ftp_seterr(FTP_PROTOCOL_ERROR);+ return (-1);+ }+ if (sscanf(ln, "%04d%02d%02d%02d%02d%02d",+ &tm.tm_year, &tm.tm_mon, &tm.tm_mday,+ &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) {+ _ftp_seterr(FTP_PROTOCOL_ERROR);+ return (-1);+ }+ tm.tm_mon--;+ tm.tm_year -= 1900;+ tm.tm_isdst = -1;+ t = timegm(&tm);+ if (t == (time_t)-1)+ t = time(NULL);+ us->mtime = t;+ us->atime = t;+ DBG(fprintf(stderr,+ "last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",+ tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,+ tm.tm_hour, tm.tm_min, tm.tm_sec));+ return (0);+}++/*+ * I/O functions for FTP+ */+struct ftpio {+ conn_t *cconn; /* Control connection */+ conn_t *dconn; /* Data connection */+ int dir; /* Direction */+ int eof; /* EOF reached */+ int err; /* Error code */+};++static int _ftp_readfn(void *, char *, int);+static int _ftp_writefn(void *, const char *, int);+static fpos_t _ftp_seekfn(void *, fpos_t, int);+static int _ftp_closefn(void *);++static int+_ftp_readfn(void *v, char *buf, int len)+{+ struct ftpio *io;+ int r;++ io = (struct ftpio *)v;+ if (io == NULL) {+ errno = EBADF;+ return (-1);+ }+ if (io->cconn == NULL || io->dconn == NULL || io->dir == O_WRONLY) {+ errno = EBADF;+ return (-1);+ }+ if (io->err) {+ errno = io->err;+ return (-1);+ }+ if (io->eof)+ return (0);+ r = _download_read(io->dconn, buf, len);+ if (r > 0)+ return (r);+ if (r == 0) {+ io->eof = 1;+ return (0);+ }+ if (errno != EINTR)+ io->err = errno;+ return (-1);+}++static int+_ftp_writefn(void *v, const char *buf, int len)+{+ struct ftpio *io;+ int w;++ io = (struct ftpio *)v;+ if (io == NULL) {+ errno = EBADF;+ return (-1);+ }+ if (io->cconn == NULL || io->dconn == NULL || io->dir == O_RDONLY) {+ errno = EBADF;+ return (-1);+ }+ if (io->err) {+ errno = io->err;+ return (-1);+ }+ w = _download_write(io->dconn, buf, len);+ if (w >= 0)+ return (w);+ if (errno != EINTR)+ io->err = errno;+ return (-1);+}++static fpos_t+_ftp_seekfn(void *v, fpos_t pos, int whence)+{+ fpos_t invalid = {0};+ struct ftpio *io;++ io = (struct ftpio *)v;+ if (io == NULL) {+ errno = EBADF;+ return invalid;+ }+ errno = ESPIPE;+ return invalid;+}++static int+_ftp_closefn(void *v)+{+ struct ftpio *io;+ int r;++ io = (struct ftpio *)v;+ if (io == NULL) {+ errno = EBADF;+ return (-1);+ }+ if (io->dir == -1)+ return (0);+ if (io->cconn == NULL || io->dconn == NULL) {+ errno = EBADF;+ return (-1);+ }+ _download_close(io->dconn);+ io->dir = -1;+ io->dconn = NULL;+ DBG(fprintf(stderr, "Waiting for final status\n"));+ r = _ftp_chkerr(io->cconn);+ if (io->cconn == cached_connection && io->cconn->ref == 1)+ cached_connection = NULL;+ _download_close(io->cconn);+ free(io);+ return (r == FTP_TRANSFER_COMPLETE) ? 0 : -1;+}++static FILE *+_ftp_setup(conn_t *cconn, conn_t *dconn, int mode)+{+ struct ftpio *io;+ FILE *f;++ if (cconn == NULL || dconn == NULL)+ return (NULL);+ if ((io = malloc(sizeof(*io))) == NULL)+ return (NULL);+ io->cconn = cconn;+ io->dconn = dconn;+ io->dir = mode;+ io->eof = io->err = 0;+ cookie_io_functions_t cookies;+ cookies.read = (cookie_read_function_t *)_ftp_readfn;+ cookies.write = (cookie_write_function_t *)_ftp_writefn;+ cookies.seek = (cookie_seek_function_t *)_ftp_seekfn;+ cookies.close = (cookie_close_function_t *)_ftp_closefn;+ f = fopencookie(io, "rw", cookies);+ if (f == NULL)+ free(io);+ return (f);+}++/*+ * Transfer file+ */+static FILE *+_ftp_transfer(conn_t *conn, const char *oper, const char *file,+ int mode, off_t offset, const char *flags)+{+ struct sockaddr_storage sa;+ struct sockaddr_in6 *sin6;+ struct sockaddr_in *sin4;+ const char *bindaddr;+ const char *filename;+ int filenamelen, type;+ int low, pasv, verbose;+ int e, sd = -1;+ socklen_t l;+ char *s;+ FILE *df;++ /* check flags */+ low = CHECK_FLAG('l');+ pasv = CHECK_FLAG('p');+ verbose = CHECK_FLAG('v');++ /* passive mode */+ if (!pasv)+ pasv = ((s = getenv("FTP_PASSIVE_MODE")) != NULL &&+ strncasecmp(s, "no", 2) != 0);++ /* isolate filename */+ filename = _ftp_filename(file, &filenamelen, &type);++ /* set transfer mode and data type */+ if ((e = _ftp_mode_type(conn, 0, type)) != FTP_OK)+ goto ouch;++ /* find our own address, bind, and listen */+ l = sizeof(sa);+ if (getsockname(conn->sd, (struct sockaddr *)&sa, &l) == -1)+ goto sysouch;+ if (sa.ss_family == AF_INET6)+ unmappedaddr((struct sockaddr_in6 *)&sa);++ /* open data socket */+ if ((sd = socket(sa.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) {+ _download_syserr();+ return (NULL);+ }++ if (pasv) {+ u_char addr[64];+ char *ln, *p;+ unsigned int i;+ int port;++ /* send PASV command */+ if (verbose)+ _download_info("setting passive mode");+ switch (sa.ss_family) {+ case AF_INET:+ if ((e = _ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE)+ goto ouch;+ break;+ case AF_INET6:+ if ((e = _ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) {+ if (e == -1)+ goto ouch;+ if ((e = _ftp_cmd(conn, "LPSV")) !=+ FTP_LPASSIVE_MODE)+ goto ouch;+ }+ break;+ default:+ e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */+ goto ouch;+ }++ /*+ * Find address and port number. The reply to the PASV command+ * is IMHO the one and only weak point in the FTP protocol.+ */+ ln = conn->buf;+ switch (e) {+ case FTP_PASSIVE_MODE:+ case FTP_LPASSIVE_MODE:+ for (p = ln + 3; *p && !isdigit(*p); p++)+ /* nothing */ ;+ if (!*p) {+ e = FTP_PROTOCOL_ERROR;+ goto ouch;+ }+ l = (e == FTP_PASSIVE_MODE ? 6 : 21);+ for (i = 0; *p && i < l; i++, p++)+ addr[i] = strtol(p, &p, 10);+ if (i < l) {+ e = FTP_PROTOCOL_ERROR;+ goto ouch;+ }+ break;+ case FTP_EPASSIVE_MODE:+ for (p = ln + 3; *p && *p != '('; p++)+ /* nothing */ ;+ if (!*p) {+ e = FTP_PROTOCOL_ERROR;+ goto ouch;+ }+ ++p;+ if (sscanf(p, "%c%c%c%d%c", &addr[0], &addr[1], &addr[2],+ &port, &addr[3]) != 5 ||+ addr[0] != addr[1] ||+ addr[0] != addr[2] || addr[0] != addr[3]) {+ e = FTP_PROTOCOL_ERROR;+ goto ouch;+ }+ break;+ }++ /* seek to required offset */+ if (offset)+ if (_ftp_cmd(conn, "REST %lu", (u_long)offset) != FTP_FILE_OK)+ goto sysouch;++ /* construct sockaddr for data socket */+ l = sizeof(sa);+ if (getpeername(conn->sd, (struct sockaddr *)&sa, &l) == -1)+ goto sysouch;+ if (sa.ss_family == AF_INET6)+ unmappedaddr((struct sockaddr_in6 *)&sa);+ switch (sa.ss_family) {+ case AF_INET6:+ sin6 = (struct sockaddr_in6 *)&sa;+ if (e == FTP_EPASSIVE_MODE)+ sin6->sin6_port = htons(port);+ else {+ bcopy(addr + 2, (char *)&sin6->sin6_addr, 16);+ bcopy(addr + 19, (char *)&sin6->sin6_port, 2);+ }+ break;+ case AF_INET:+ sin4 = (struct sockaddr_in *)&sa;+ if (e == FTP_EPASSIVE_MODE)+ sin4->sin_port = htons(port);+ else {+ bcopy(addr, (char *)&sin4->sin_addr, 4);+ bcopy(addr + 4, (char *)&sin4->sin_port, 2);+ }+ break;+ default:+ e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */+ break;+ }++ /* connect to data port */+ if (verbose)+ _download_info("opening data connection");+ bindaddr = getenv("DOWNLOAD_BIND_ADDRESS");+ if (bindaddr != NULL && *bindaddr != '\0' &&+ _download_bind(sd, sa.ss_family, bindaddr) != 0)+ goto sysouch;+ if (connect(sd, (struct sockaddr *)&sa, sizeof(struct sockaddr_storage)) == -1)+ goto sysouch;++ /* make the server initiate the transfer */+ if (verbose)+ _download_info("initiating transfer");+ e = _ftp_cmd(conn, "%s %.*s", oper, filenamelen, filename);+ if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)+ goto ouch;++ } else {+ u_int32_t a;+ u_short p;+ int d;+#ifdef IPV6_PORTRANGE+ int arg;+#endif+ char *ap;+ char hname[INET6_ADDRSTRLEN];++ switch (sa.ss_family) {+ case AF_INET6:+ ((struct sockaddr_in6 *)&sa)->sin6_port = 0;+#ifdef IPV6_PORTRANGE+ arg = low ? IPV6_PORTRANGE_DEFAULT : IPV6_PORTRANGE_HIGH;+ if (setsockopt(sd, IPPROTO_IPV6, IPV6_PORTRANGE,+ (char *)&arg, sizeof(arg)) == -1)+ goto sysouch;+#endif+ break;+ case AF_INET:+ ((struct sockaddr_in *)&sa)->sin_port = 0;+#ifdef IP_PORTRANGE+ arg = low ? IP_PORTRANGE_DEFAULT : IP_PORTRANGE_HIGH;+ if (setsockopt(sd, IPPROTO_IP, IP_PORTRANGE,+ (char *)&arg, sizeof(arg)) == -1)+ goto sysouch;+#endif+ break;+ }+ if (verbose)+ _download_info("binding data socket");+ if (bind(sd, (struct sockaddr *)&sa, sizeof(struct sockaddr_storage)) == -1)+ goto sysouch;+ if (listen(sd, 1) == -1)+ goto sysouch;++ /* find what port we're on and tell the server */+ if (getsockname(sd, (struct sockaddr *)&sa, &l) == -1)+ goto sysouch;+ switch (sa.ss_family) {+ case AF_INET:+ sin4 = (struct sockaddr_in *)&sa;+ a = ntohl(sin4->sin_addr.s_addr);+ p = ntohs(sin4->sin_port);+ e = _ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d",+ (a >> 24) & 0xff, (a >> 16) & 0xff,+ (a >> 8) & 0xff, a & 0xff,+ (p >> 8) & 0xff, p & 0xff);+ break;+ case AF_INET6:+#define UC(b) (((int)b)&0xff)+ e = -1;+ sin6 = (struct sockaddr_in6 *)&sa;+ sin6->sin6_scope_id = 0;+ if (getnameinfo((struct sockaddr *)&sa,sizeof(struct sockaddr_storage),+ hname, sizeof(hname),+ NULL, 0, NI_NUMERICHOST) == 0) {+ e = _ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname,+ htons(sin6->sin6_port));+ if (e == -1)+ goto ouch;+ }+ if (e != FTP_OK) {+ ap = (char *)&sin6->sin6_addr;+ e = _ftp_cmd(conn,+ "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",+ 6, 16,+ UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]),+ UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]),+ UC(ap[8]), UC(ap[9]), UC(ap[10]), UC(ap[11]),+ UC(ap[12]), UC(ap[13]), UC(ap[14]), UC(ap[15]),+ 2,+ (ntohs(sin6->sin6_port) >> 8) & 0xff,+ ntohs(sin6->sin6_port) & 0xff);+ }+ break;+ default:+ e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */+ goto ouch;+ }+ if (e != FTP_OK)+ goto ouch;++ /* seek to required offset */+ if (offset)+ if (_ftp_cmd(conn, "REST %ju", (uintmax_t)offset) != FTP_FILE_OK)+ goto sysouch;++ /* make the server initiate the transfer */+ if (verbose)+ _download_info("initiating transfer");+ e = _ftp_cmd(conn, "%s %.*s", oper, filenamelen, filename);+ if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)+ goto ouch;++ /* accept the incoming connection and go to town */+ if ((d = accept(sd, NULL, NULL)) == -1)+ goto sysouch;+ close(sd);+ sd = d;+ }++ if ((df = _ftp_setup(conn, _download_reopen(sd), mode)) == NULL)+ goto sysouch;+ return (df);++sysouch:+ _download_syserr();+ if (sd >= 0)+ close(sd);+ return (NULL);++ouch:+ if (e != -1)+ _ftp_seterr(e);+ if (sd >= 0)+ close(sd);+ return (NULL);+}++/*+ * Authenticate+ */+static int+_ftp_authenticate(conn_t *conn, struct url *url, struct url *purl)+{+ const char *user, *pwd, *logname;+ char pbuf[MAXHOSTNAMELEN + MAXPATHLEN + 1];+ int e, len;++ /* XXX FTP_AUTH, and maybe .netrc */++ /* send user name and password */+ if (url->user[0] == '\0')+ _download_netrc_auth(url);+ user = url->user;+ if (*user == '\0')+ user = getenv("FTP_LOGIN");+ if (user == NULL || *user == '\0')+ user = FTP_ANONYMOUS_USER;+ if (purl && url->port == _download_default_port(url->scheme))+ e = _ftp_cmd(conn, "USER %s@%s", user, url->host);+ else if (purl)+ e = _ftp_cmd(conn, "USER %s@%s@%d", user, url->host, url->port);+ else+ e = _ftp_cmd(conn, "USER %s", user);++ /* did the server request a password? */+ if (e == FTP_NEED_PASSWORD) {+ pwd = url->pwd;+ if (*pwd == '\0')+ pwd = getenv("FTP_PASSWORD");+ if (pwd == NULL || *pwd == '\0') {+ if ((logname = getlogin()) == 0)+ logname = FTP_ANONYMOUS_USER;+ if ((len = snprintf(pbuf, MAXPATHLEN + 1, "%s@", logname)) < 0)+ len = 0;+ else if (len > MAXPATHLEN)+ len = MAXPATHLEN;+ gethostname(pbuf + len, sizeof(pbuf) - len);+ pwd = pbuf;+ }+ e = _ftp_cmd(conn, "PASS %s", pwd);+ }++ return (e);+}++/*+ * Log on to FTP server+ */+static conn_t *+_ftp_connect(struct url *url, struct url *purl, const char *flags)+{+ conn_t *conn;+ int e, direct, verbose;+#ifdef INET6+ int af = AF_UNSPEC;+#else+ int af = AF_INET;+#endif++ direct = CHECK_FLAG('d');+ verbose = CHECK_FLAG('v');+ if (CHECK_FLAG('4'))+ af = AF_INET;+ else if (CHECK_FLAG('6'))+ af = AF_INET6;++ if (direct)+ purl = NULL;++ /* check for proxy */+ if (purl) {+ /* XXX proxy authentication! */+ conn = _download_connect(purl->host, purl->port, af, verbose);+ } else {+ /* no proxy, go straight to target */+ conn = _download_connect(url->host, url->port, af, verbose);+ purl = NULL;+ }++ /* check connection */+ if (conn == NULL)+ /* _download_connect() has already set an error code */+ return (NULL);++ /* expect welcome message */+ if ((e = _ftp_chkerr(conn)) != FTP_SERVICE_READY)+ goto fouch;++ /* authenticate */+ if ((e = _ftp_authenticate(conn, url, purl)) != FTP_LOGGED_IN)+ goto fouch;++ /* done */+ return (conn);++fouch:+ if (e != -1)+ _ftp_seterr(e);+ _download_close(conn);+ return (NULL);+}++/*+ * Disconnect from server+ */+static void+_ftp_disconnect(conn_t *conn)+{+ (void)_ftp_cmd(conn, "QUIT");+ _download_unseterr(); /* we don't care about errors on disconnect */+ if (conn == cached_connection && conn->ref == 1)+ cached_connection = NULL;+ _download_close(conn);+}++/*+ * Check if we're already connected+ */+static int+_ftp_isconnected(struct url *url)+{+ return (cached_connection+ && (strcmp(url->host, cached_host.host) == 0)+ && (strcmp(url->user, cached_host.user) == 0)+ && (strcmp(url->pwd, cached_host.pwd) == 0)+ && (url->port == cached_host.port));+}++/*+ * Check the cache, reconnect if no luck+ */+static conn_t *+_ftp_cached_connect(struct url *url, struct url *purl, const char *flags)+{+ conn_t *conn;+ int e;++ /* set default port */+ if (!url->port)+ url->port = _download_default_port(url->scheme);++ /* try to use previously cached connection */+ if (_ftp_isconnected(url)) {+ e = _ftp_cmd(cached_connection, "NOOP");+ if (e == FTP_OK || e == FTP_SYNTAX_ERROR)+ return (_download_ref(cached_connection));+ }++ /* connect to server */+ if ((conn = _ftp_connect(url, purl, flags)) == NULL)+ return (NULL);+ if (cached_connection)+ _ftp_disconnect(cached_connection);+ cached_connection = _download_ref(conn);+ memcpy(&cached_host, url, sizeof(*url));+ return (conn);+}++/*+ * Check the proxy settings+ */+static struct url *+_ftp_get_proxy(const char *flags)+{+ struct url *purl;+ char *p;++ if (flags != NULL && strchr(flags, 'd') != NULL)+ return (NULL);+ if (((p = getenv("FTP_PROXY")) || (p = getenv("ftp_proxy")) ||+ (p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) &&+ *p && (purl = downloadParseURL(p)) != NULL) {+ if (!*purl->scheme) {+ if (getenv("FTP_PROXY") || getenv("ftp_proxy"))+ strcpy(purl->scheme, SCHEME_FTP);+ else+ strcpy(purl->scheme, SCHEME_HTTP);+ }+ if (!purl->port)+ purl->port = _download_default_proxy_port(purl->scheme);+ if (strcasecmp(purl->scheme, SCHEME_FTP) == 0 ||+ strcasecmp(purl->scheme, SCHEME_HTTP) == 0)+ return (purl);+ downloadFreeURL(purl);+ }+ return (NULL);+}++/*+ * Process an FTP request+ */+FILE *+_ftp_request(struct url *url, const char *op, struct url_stat *us,+ struct url *purl, const char *flags)+{+ conn_t *conn;+ int oflag;++ char *no_proxy, *no_proxy_item;+ size_t b, s;++ if ((no_proxy = getenv("NO_PROXY")) != NULL) {+ b = strlen(url->host);+ while ((no_proxy_item = strsep(&no_proxy, ",")) != NULL) {+ s = strlen(no_proxy_item);+ if (b > s && strcasecmp(url->host + b - s,+ no_proxy_item) == 0) {+ /* disable proxy */+ downloadFreeURL(purl);+ purl = NULL;+ break;+ }+ }+ }++ /* check if we should use HTTP instead */+ if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) {+ if (strcmp(op, "STAT") == 0)+ return (_http_request(url, "HEAD", us, purl, flags));+ else if (strcmp(op, "RETR") == 0)+ return (_http_request(url, "GET", us, purl, flags));+ /*+ * Our HTTP code doesn't support PUT requests yet, so try+ * a direct connection.+ */+ }++ /* connect to server */+ conn = _ftp_cached_connect(url, purl, flags);+ if (purl)+ downloadFreeURL(purl);+ if (conn == NULL)+ return (NULL);++ /* change directory */+ if (_ftp_cwd(conn, url->doc) == -1)+ return (NULL);++ /* stat file */+ if (us && _ftp_stat(conn, url->doc, us) == -1+ && downloadLastErrCode != DLERR_PROTO+ && downloadLastErrCode != DLERR_UNAVAIL)+ return (NULL);++ /* just a stat */+ if (strcmp(op, "STAT") == 0)+ return (FILE *)1; /* bogus return value */+ if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0)+ oflag = O_WRONLY;+ else+ oflag = O_RDONLY;++ /* initiate the transfer */+ return (_ftp_transfer(conn, op, url->doc, oflag, url->offset, flags));+}++/*+ * Get and stat file+ */+FILE *+downloadXGetFTP(struct url *url, struct url_stat *us, const char *flags)+{+ return (_ftp_request(url, "RETR", us, _ftp_get_proxy(flags), flags));+}++/*+ * Get file+ */+FILE *+downloadGetFTP(struct url *url, const char *flags)+{+ return (downloadXGetFTP(url, NULL, flags));+}++/*+ * Get file stats+ */+int+downloadStatFTP(struct url *url, struct url_stat *us, const char *flags)+{+ FILE *f;++ f = _ftp_request(url, "STAT", us, _ftp_get_proxy(flags), flags);+ if (f == NULL)+ return (-1);+ return (0);+}
+ cbits/ftperr.h view
@@ -0,0 +1,45 @@+static struct downloaderr _ftp_errlist[] = {+ { 110, DLERR_OK, "Restart marker reply" },+ { 120, DLERR_TEMP, "Service ready in a few minutes" },+ { 125, DLERR_OK, "Data connection already open; transfer starting" },+ { 150, DLERR_OK, "File status okay; about to open data connection" },+ { 200, DLERR_OK, "Command okay" },+ { 202, DLERR_PROTO, "Command not implemented, superfluous at this site" },+ { 211, DLERR_INFO, "System status, or system help reply" },+ { 212, DLERR_INFO, "Directory status" },+ { 213, DLERR_INFO, "File status" },+ { 214, DLERR_INFO, "Help message" },+ { 215, DLERR_INFO, "Set system type" },+ { 220, DLERR_OK, "Service ready for new user" },+ { 221, DLERR_OK, "Service closing control connection" },+ { 225, DLERR_OK, "Data connection open; no transfer in progress" },+ { 226, DLERR_OK, "Requested file action successful" },+ { 227, DLERR_OK, "Entering Passive Mode" },+ { 229, DLERR_OK, "Entering Extended Passive Mode" },+ { 230, DLERR_OK, "User logged in, proceed" },+ { 250, DLERR_OK, "Requested file action okay, completed" },+ { 257, DLERR_OK, "File/directory created" },+ { 331, DLERR_AUTH, "User name okay, need password" },+ { 332, DLERR_AUTH, "Need account for login" },+ { 350, DLERR_OK, "Requested file action pending further information" },+ { 421, DLERR_DOWN, "Service not available, closing control connection" },+ { 425, DLERR_NETWORK, "Can't open data connection" },+ { 426, DLERR_ABORT, "Connection closed; transfer aborted" },+ { 450, DLERR_UNAVAIL, "File unavailable (e.g., file busy)" },+ { 451, DLERR_SERVER, "Requested action aborted: local error in processing" },+ { 452, DLERR_FULL, "Insufficient storage space in system" },+ { 500, DLERR_PROTO, "Syntax error, command unrecognized" },+ { 501, DLERR_PROTO, "Syntax error in parameters or arguments" },+ { 502, DLERR_PROTO, "Command not implemented" },+ { 503, DLERR_PROTO, "Bad sequence of commands" },+ { 504, DLERR_PROTO, "Command not implemented for that parameter" },+ { 530, DLERR_AUTH, "Not logged in" },+ { 532, DLERR_AUTH, "Need account for storing files" },+ { 535, DLERR_PROTO, "Bug in MediaHawk Video Kernel FTP server" },+ { 550, DLERR_UNAVAIL, "File unavailable (e.g., file not found, no access)" },+ { 551, DLERR_PROTO, "Requested action aborted. Page type unknown" },+ { 552, DLERR_FULL, "Exceeded storage allocation" },+ { 553, DLERR_EXISTS, "File name not allowed" },+ { 999, DLERR_PROTO, "Protocol error" },+ { -1, DLERR_UNKNOWN, "Unknown FTP error" }+};
+ cbits/hs_download_utils.c view
@@ -0,0 +1,9 @@+#include "hs_download_utils.h"++char * hs_download_last_error () {+ return downloadLastErrString;+}++int hs_download_last_error_code () {+ return downloadLastErrCode;+}
+ cbits/hs_download_utils.h view
@@ -0,0 +1,4 @@+#include "download.h"++char * hs_download_last_error ();+int hs_download_last_error_code ();
+ cbits/http.c view
@@ -0,0 +1,1226 @@+/*-+ * Copyright (c) 2006 Aaron Griffin <aaronmgriffin@gmail.com>+ *+ * Based on original libfetch code from:+ * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer+ * in this position and unchanged.+ * 2. Redistributions in binary form must reproduce the above copyright+ * notice, this list of conditions and the following disclaimer in the+ * documentation and/or other materials provided with the distribution.+ * 3. The name of the author may not be used to endorse or promote products+ * derived from this software without specific prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ */++#include <sys/cdefs.h>++/*+ * The following copyright applies to the base64 code:+ *+ *-+ * Copyright 1997 Massachusetts Institute of Technology+ *+ * Permission to use, copy, modify, and distribute this software and+ * its documentation for any purpose and without fee is hereby+ * granted, provided that both the above copyright notice and this+ * permission notice appear in all copies, that both the above+ * copyright notice and this permission notice appear in all+ * supporting documentation, and that the name of M.I.T. not be used+ * in advertising or publicity pertaining to distribution of the+ * software without specific, written prior permission. M.I.T. makes+ * no representations about the suitability of this software for any+ * purpose. It is provided "as is" without express or implied+ * warranty.+ *+ * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS+ * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT+ * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF+ * SUCH DAMAGE.+ */++#include <sys/param.h>+#include <sys/socket.h>++#include <ctype.h>+#include <libio.h>+#include <err.h>+#include <errno.h>+#include <locale.h>+#include <netdb.h>+#include <stdarg.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <time.h>+#include <unistd.h>++#include <netinet/in.h>+#include <netinet/tcp.h>++#include "download.h"+#include "common.h"+#include "httperr.h"++/* Maximum number of redirects to follow */+#define MAX_REDIRECT 5++/* Symbolic names for reply codes we care about */+#define HTTP_OK 200+#define HTTP_PARTIAL 206+#define HTTP_MOVED_PERM 301+#define HTTP_MOVED_TEMP 302+#define HTTP_SEE_OTHER 303+#define HTTP_NEED_AUTH 401+#define HTTP_NEED_PROXY_AUTH 407+#define HTTP_BAD_RANGE 416+#define HTTP_PROTOCOL_ERROR 999++#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \+ || (xyz) == HTTP_MOVED_TEMP \+ || (xyz) == HTTP_SEE_OTHER)++#define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599)++/* for easy redirection */+FILE *_ftp_request(struct url *, const char *, struct url_stat *,+ struct url *, const char *);++/*****************************************************************************+ * I/O functions for decoding chunked streams+ */++struct httpio+{+ conn_t *conn; /* connection */+ int chunked; /* chunked mode */+ char *buf; /* chunk buffer */+ size_t bufsize; /* size of chunk buffer */+ ssize_t buflen; /* amount of data currently in buffer */+ int bufpos; /* current read offset in buffer */+ int eof; /* end-of-file flag */+ int error; /* error flag */+ size_t chunksize; /* remaining size of current chunk */+#ifdef DEBUG+ size_t total;+#endif+};++/*+ * Get next chunk header+ */+static int+_http_new_chunk(struct httpio *io)+{+ char hexdigits[23] = "0123456789abcdefABCDEF\0";+ char *p = NULL;++ if (_download_getln(io->conn) == -1)+ return (-1);++ if (io->conn->buflen < 2 || !strchr(hexdigits, *io->conn->buf))+ return (-1);++ for (p = io->conn->buf; *p && !isspace(*p); ++p) {+ if (*p == ';')+ break;+ if (!strchr(hexdigits, *p))+ return (-1);+ if (isdigit(*p)) {+ io->chunksize = io->chunksize * 16 ++ *p - '0';+ } else {+ io->chunksize = io->chunksize * 16 ++ 10 + tolower(*p) - 'a';+ }+ }++#ifdef DEBUG+ if (downloadDebug) {+ io->total += io->chunksize;+ if (io->chunksize == 0)+ fprintf(stderr, "%s(): end of last chunk\n", __func__);+ else+ fprintf(stderr, "%s(): new chunk: %lu (%lu)\n",+ __func__, (unsigned long)io->chunksize,+ (unsigned long)io->total);+ }+#endif++ return (io->chunksize);+}++/*+ * Grow the input buffer to at least len bytes+ */+static inline int+_http_growbuf(struct httpio *io, size_t len)+{+ char *tmp;++ if (io->bufsize >= len)+ return (0);++ if ((tmp = realloc(io->buf, len)) == NULL)+ return (-1);+ io->buf = tmp;+ io->bufsize = len;+ return (0);+}++/*+ * Fill the input buffer, do chunk decoding on the fly+ */+static int+_http_fillbuf(struct httpio *io, size_t len)+{+ if (io->error)+ return (-1);+ if (io->eof)+ return (0);++ if (io->chunked == 0) {+ if (_http_growbuf(io, len) == -1)+ return (-1);+ if ((io->buflen = _download_read(io->conn, io->buf, len)) == -1) {+ io->error = 1;+ return (-1);+ }+ io->bufpos = 0;+ return (io->buflen);+ }++ if (io->chunksize == 0) {+ switch (_http_new_chunk(io)) {+ case -1:+ io->error = 1;+ return (-1);+ case 0:+ io->eof = 1;+ return (0);+ }+ }++ if (len > io->chunksize)+ len = io->chunksize;+ if (_http_growbuf(io, len) == -1)+ return (-1);+ if ((io->buflen = _download_read(io->conn, io->buf, len)) == -1) {+ io->error = 1;+ return (-1);+ }+ io->chunksize -= io->buflen;++ if (io->chunksize == 0) {+ char endl[2];++ if (_download_read(io->conn, endl, 2) != 2 ||+ endl[0] != '\r' || endl[1] != '\n')+ return (-1);+ }++ io->bufpos = 0;++ return (io->buflen);+}++/*+ * Read function+ */+static int+_http_readfn(void *v, char *buf, int len)+{+ struct httpio *io = (struct httpio *)v;+ int l, pos;++ if (io->error)+ return (-1);+ if (io->eof)+ return (0);++ for (pos = 0; len > 0; pos += l, len -= l) {+ /* empty buffer */+ if (!io->buf || io->bufpos == io->buflen)+ if (_http_fillbuf(io, len) < 1)+ break;+ l = io->buflen - io->bufpos;+ if (len < l)+ l = len;+ bcopy(io->buf + io->bufpos, buf + pos, l);+ io->bufpos += l;+ }++ if (!pos && io->error)+ return (-1);+ return (pos);+}++/*+ * Write function+ */+static int+_http_writefn(void *v, const char *buf, int len)+{+ struct httpio *io = (struct httpio *)v;++ return (_download_write(io->conn, buf, len));+}++/*+ * Close function+ */+static int+_http_closefn(void *v)+{+ struct httpio *io = (struct httpio *)v;+ int r;++ r = _download_close(io->conn);+ if (io->buf)+ free(io->buf);+ free(io);+ return (r);+}++/*+ * Wrap a file descriptor up+ */+static FILE *+_http_funopen(conn_t *conn, int chunked)+{+ struct httpio *io;+ FILE *f;++ if ((io = calloc(1, sizeof(*io))) == NULL) {+ _download_syserr();+ return (NULL);+ }+ io->conn = conn;+ io->chunked = chunked;+ cookie_io_functions_t cookie;+ cookie.read = (cookie_read_function_t *)_http_readfn;+ cookie.write = (cookie_write_function_t *)_http_writefn;+ cookie.seek = (cookie_seek_function_t *)NULL;+ cookie.close = (cookie_close_function_t *)_http_closefn;+ f = fopencookie(io, "rw", cookie);+ if (f == NULL) {+ _download_syserr();+ free(io);+ return (NULL);+ }+ return (f);+}+++/*****************************************************************************+ * Helper functions for talking to the server and parsing its replies+ */++/* Header types */+typedef enum {+ hdr_syserror = -2,+ hdr_error = -1,+ hdr_end = 0,+ hdr_unknown = 1,+ hdr_content_length,+ hdr_content_range,+ hdr_last_modified,+ hdr_location,+ hdr_transfer_encoding,+ hdr_www_authenticate+} hdr_t;++/* Names of interesting headers */+static struct {+ hdr_t num;+ const char *name;+} hdr_names[] = {+ { hdr_content_length, "Content-Length" },+ { hdr_content_range, "Content-Range" },+ { hdr_last_modified, "Last-Modified" },+ { hdr_location, "Location" },+ { hdr_transfer_encoding, "Transfer-Encoding" },+ { hdr_www_authenticate, "WWW-Authenticate" },+ { hdr_unknown, NULL },+};++/*+ * Send a formatted line; optionally echo to terminal+ */+static int+_http_cmd(conn_t *conn, const char *fmt, ...)+{+ va_list ap;+ size_t len;+ char *msg;+ int r;++ va_start(ap, fmt);+ len = vasprintf(&msg, fmt, ap);+ va_end(ap);++ if (msg == NULL) {+ errno = ENOMEM;+ _download_syserr();+ return (-1);+ }++ r = _download_putln(conn, msg, len);+ free(msg);++ if (r == -1) {+ _download_syserr();+ return (-1);+ }++ return (0);+}++/*+ * Get and parse status line+ */+static int+_http_get_reply(conn_t *conn)+{+ char *p;++ if (_download_getln(conn) == -1)+ return (-1);+ /*+ * A valid status line looks like "HTTP/m.n xyz reason" where m+ * and n are the major and minor protocol version numbers and xyz+ * is the reply code.+ * Unfortunately, there are servers out there (NCSA 1.5.1, to name+ * just one) that do not send a version number, so we can't rely+ * on finding one, but if we do, insist on it being 1.0 or 1.1.+ * We don't care about the reason phrase.+ */+ if (strncmp(conn->buf, "HTTP", 4) != 0)+ return (HTTP_PROTOCOL_ERROR);+ p = conn->buf + 4;+ if (*p == '/') {+ if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1'))+ return (HTTP_PROTOCOL_ERROR);+ p += 4;+ }+ if (*p != ' ' || !isdigit(p[1]) || !isdigit(p[2]) || !isdigit(p[3]))+ return (HTTP_PROTOCOL_ERROR);++ conn->err = (p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0');+ return (conn->err);+}++/*+ * Check a header; if the type matches the given string, return a pointer+ * to the beginning of the value.+ */+static const char *+_http_match(const char *str, const char *hdr)+{+ while (*str && *hdr && tolower(*str++) == tolower(*hdr++))+ /* nothing */;+ if (*str || *hdr != ':')+ return (NULL);+ while (*hdr && isspace(*++hdr))+ /* nothing */;+ return (hdr);+}++/*+ * Get the next header and return the appropriate symbolic code.+ */+static hdr_t+_http_next_header(conn_t *conn, const char **p)+{+ int i;++ if (_download_getln(conn) == -1)+ return (hdr_syserror);+ while (conn->buflen && isspace(conn->buf[conn->buflen - 1]))+ conn->buflen--;+ conn->buf[conn->buflen] = '\0';+ if (conn->buflen == 0)+ return (hdr_end);+ /*+ * We could check for malformed headers but we don't really care.+ * A valid header starts with a token immediately followed by a+ * colon; a token is any sequence of non-control, non-whitespace+ * characters except "()<>@,;:\\\"{}".+ */+ for (i = 0; hdr_names[i].num != hdr_unknown; i++)+ if ((*p = _http_match(hdr_names[i].name, conn->buf)) != NULL)+ return (hdr_names[i].num);+ return (hdr_unknown);+}++/*+ * Parse a last-modified header+ */+static int+_http_parse_mtime(const char *p, time_t *mtime)+{+ char locale[64], *r;+ struct tm tm;++ strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale));+ setlocale(LC_TIME, "C");+ r = (char *)strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);+ /* XXX should add support for date-2 and date-3 */+ setlocale(LC_TIME, locale);+ if (r == NULL)+ return (-1);+ DBG(fprintf(stderr, "last modified: [%04d-%02d-%02d "+ "%02d:%02d:%02d]\n",+ tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,+ tm.tm_hour, tm.tm_min, tm.tm_sec));+ *mtime = timegm(&tm);+ return (0);+}++/*+ * Parse a content-length header+ */+static int+_http_parse_length(const char *p, off_t *length)+{+ off_t len;++ for (len = 0; *p && isdigit(*p); ++p)+ len = len * 10 + (*p - '0');+ if (*p)+ return (-1);+ DBG(fprintf(stderr, "content length: [%lld]\n",+ (long long)len));+ *length = len;+ return (0);+}++/*+ * Parse a content-range header+ */+static int+_http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size)+{+ off_t first, last, len;++ if (strncasecmp(p, "bytes ", 6) != 0)+ return (-1);+ p += 6;+ if (*p == '*') {+ first = last = -1;+ ++p;+ } else {+ for (first = 0; *p && isdigit(*p); ++p)+ first = first * 10 + *p - '0';+ if (*p != '-')+ return (-1);+ for (last = 0, ++p; *p && isdigit(*p); ++p)+ last = last * 10 + *p - '0';+ }+ if (first > last || *p != '/')+ return (-1);+ for (len = 0, ++p; *p && isdigit(*p); ++p)+ len = len * 10 + *p - '0';+ if (*p || len < last - first + 1)+ return (-1);+ if (first == -1) {+ DBG(fprintf(stderr, "content range: [*/%lld]\n",+ (long long)len));+ *length = 0;+ } else {+ DBG(fprintf(stderr, "content range: [%lld-%lld/%lld]\n",+ (long long)first, (long long)last, (long long)len));+ *length = last - first + 1;+ }+ *offset = first;+ *size = len;+ return (0);+}+++/*****************************************************************************+ * Helper functions for authorization+ */++/*+ * Base64 encoding+ */+static char *+_http_base64(const char *src)+{+ static const char base64[] =+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ "abcdefghijklmnopqrstuvwxyz"+ "0123456789+/";+ char *str, *dst;+ size_t l;+ int t, r;++ l = strlen(src);+ if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL)+ return (NULL);+ dst = str;+ r = 0;++ while (l >= 3) {+ t = (src[0] << 16) | (src[1] << 8) | src[2];+ dst[0] = base64[(t >> 18) & 0x3f];+ dst[1] = base64[(t >> 12) & 0x3f];+ dst[2] = base64[(t >> 6) & 0x3f];+ dst[3] = base64[(t >> 0) & 0x3f];+ src += 3; l -= 3;+ dst += 4; r += 4;+ }++ switch (l) {+ case 2:+ t = (src[0] << 16) | (src[1] << 8);+ dst[0] = base64[(t >> 18) & 0x3f];+ dst[1] = base64[(t >> 12) & 0x3f];+ dst[2] = base64[(t >> 6) & 0x3f];+ dst[3] = '=';+ dst += 4;+ r += 4;+ break;+ case 1:+ t = src[0] << 16;+ dst[0] = base64[(t >> 18) & 0x3f];+ dst[1] = base64[(t >> 12) & 0x3f];+ dst[2] = dst[3] = '=';+ dst += 4;+ r += 4;+ break;+ case 0:+ break;+ }++ *dst = 0;+ return (str);+}++/*+ * Encode username and password+ */+static int+_http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)+{+ char *upw, *auth;+ int r;++ DBG(fprintf(stderr, "usr: [%s]\n", usr));+ DBG(fprintf(stderr, "pwd: [%s]\n", pwd));+ if (asprintf(&upw, "%s:%s", usr, pwd) == -1)+ return (-1);+ auth = _http_base64(upw);+ free(upw);+ if (auth == NULL)+ return (-1);+ r = _http_cmd(conn, "%s: Basic %s", hdr, auth);+ free(auth);+ return (r);+}++/*+ * Send an authorization header+ */+static int+_http_authorize(conn_t *conn, const char *hdr, const char *p)+{+ /* basic authorization */+ if (strncasecmp(p, "basic:", 6) == 0) {+ char *user, *pwd, *str;+ int r;++ /* skip realm */+ for (p += 6; *p && *p != ':'; ++p)+ /* nothing */ ;+ if (!*p || strchr(++p, ':') == NULL)+ return (-1);+ if ((str = strdup(p)) == NULL)+ return (-1); /* XXX */+ user = str;+ pwd = strchr(str, ':');+ *pwd++ = '\0';+ r = _http_basic_auth(conn, hdr, user, pwd);+ free(str);+ return (r);+ }+ return (-1);+}+++/*****************************************************************************+ * Helper functions for connecting to a server or proxy+ */++/*+ * Connect to the correct HTTP server or proxy.+ */+static conn_t *+_http_connect(struct url *URL, struct url *purl, const char *flags)+{+ conn_t *conn;+ int verbose;+ int af;++#ifdef INET6+ af = AF_UNSPEC;+#else+ af = AF_INET;+#endif++ verbose = CHECK_FLAG('v');+ if (CHECK_FLAG('4'))+ af = AF_INET;+#ifdef INET6+ else if (CHECK_FLAG('6'))+ af = AF_INET6;+#endif++ if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) {+ URL = purl;+ } else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) {+ /* can't talk http to an ftp server */+ /* XXX should set an error code */+ return (NULL);+ }++ if ((conn = _download_connect(URL->host, URL->port, af, verbose)) == NULL)+ /* _download_connect() has already set an error code */+ return (NULL);+ if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&+ _download_ssl(conn, verbose) == -1) {+ _download_close(conn);+ /* grrr */+ _download_syserr();+ return (NULL);+ }++#ifdef TCP_NOPUSH+ int val = 1;+ setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));+#endif++ return (conn);+}++static struct url *+_http_get_proxy(const char *flags)+{+ struct url *purl;+ char *p;++ if (flags != NULL && strchr(flags, 'd') != NULL)+ return (NULL);+ if (((p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) &&+ *p && (purl = downloadParseURL(p))) {+ if (!*purl->scheme)+ strcpy(purl->scheme, SCHEME_HTTP);+ if (!purl->port)+ purl->port = _download_default_proxy_port(purl->scheme);+ if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0)+ return (purl);+ downloadFreeURL(purl);+ }+ return (NULL);+}++static void+_http_print_html(FILE *out, FILE *in)+{+ size_t len;+ unsigned int sz = 1024;+ char *line, *p, *q;+ int comment, tag;++ line = (char *)calloc(1024, sizeof(char));+ comment = tag = 0;+ while ((len = getline(&line, &sz, in)) > 0) {+ while (len && isspace(line[len - 1]))+ --len;+ for (p = q = line; q < line + len; ++q) {+ if (comment && *q == '-') {+ if (q + 2 < line + len &&+ strcmp(q, "-->") == 0) {+ tag = comment = 0;+ q += 2;+ }+ } else if (tag && !comment && *q == '>') {+ p = q + 1;+ tag = 0;+ } else if (!tag && *q == '<') {+ if (q > p)+ fwrite(p, q - p, 1, out);+ tag = 1;+ if (q + 3 < line + len &&+ strcmp(q, "<!--") == 0) {+ comment = 1;+ q += 3;+ }+ }+ }+ if (!tag && q > p)+ fwrite(p, q - p, 1, out);+ fputc('\n', out);+ }+}+++/*****************************************************************************+ * Core+ */++/*+ * Send a request and process the reply+ *+ * XXX This function is way too long, the do..while loop should be split+ * XXX off into a separate function.+ */+FILE *+_http_request(struct url *URL, const char *op, struct url_stat *us,+ struct url *purl, const char *flags)+{+ conn_t *conn;+ struct url *url, *new;+ int chunked, direct, need_auth, noredirect, verbose;+ int e, i, n, val;+ off_t offset, clength, length, size;+ time_t mtime;+ const char *p;+ FILE *f;+ hdr_t h;+ char hbuf[MAXHOSTNAMELEN + 7], *host;++ char *no_proxy, *no_proxy_item;+ size_t b, s;++ if ((no_proxy = getenv("NO_PROXY")) != NULL) {+ b = strlen(URL->host);+ while ((no_proxy_item = strsep(&no_proxy, ",")) != NULL) {+ s = strlen(no_proxy_item);+ if (b > s && strcasecmp(URL->host + b - s,+ no_proxy_item) == 0) {+ /* disable proxy */+ downloadFreeURL(purl);+ purl = NULL;+ break;+ }+ }+ }++ direct = CHECK_FLAG('d');+ noredirect = CHECK_FLAG('A');+ verbose = CHECK_FLAG('v');++ if (direct && purl) {+ downloadFreeURL(purl);+ purl = NULL;+ }++ /* try the provided URL first */+ url = URL;++ /* if the A flag is set, we only get one try */+ n = noredirect ? 1 : MAX_REDIRECT;+ i = 0;++ e = HTTP_PROTOCOL_ERROR;+ need_auth = 0;+ do {+ new = NULL;+ chunked = 0;+ offset = 0;+ clength = -1;+ length = -1;+ size = -1;+ mtime = 0;++ /* check port */+ if (!url->port)+ url->port = _download_default_port(url->scheme);++ /* were we redirected to an FTP URL? */+ if (purl == NULL && strcmp(url->scheme, SCHEME_FTP) == 0) {+ if (strcmp(op, "GET") == 0)+ return (_ftp_request(url, "RETR", us, purl, flags));+ else if (strcmp(op, "HEAD") == 0)+ return (_ftp_request(url, "STAT", us, purl, flags));+ }++ /* connect to server or proxy */+ if ((conn = _http_connect(url, purl, flags)) == NULL)+ goto ouch;++ host = url->host;+#ifdef INET6+ if (strchr(url->host, ':')) {+ snprintf(hbuf, sizeof(hbuf), "[%s]", url->host);+ host = hbuf;+ }+#endif+ if (url->port != _download_default_port(url->scheme)) {+ if (host != hbuf) {+ strcpy(hbuf, host);+ host = hbuf;+ }+ snprintf(hbuf + strlen(hbuf),+ sizeof(hbuf) - strlen(hbuf), ":%d", url->port);+ }++ /* send request */+ if (verbose)+ _download_info("requesting %s://%s%s",+ url->scheme, host, url->doc);+ if (purl) {+ _http_cmd(conn, "%s %s://%s%s HTTP/1.1",+ op, url->scheme, host, url->doc);+ } else {+ _http_cmd(conn, "%s %s HTTP/1.1",+ op, url->doc);+ }++ /* virtual host */+ _http_cmd(conn, "Host: %s", host);++ /* proxy authorization */+ if (purl) {+ if (*purl->user || *purl->pwd)+ _http_basic_auth(conn, "Proxy-Authorization",+ purl->user, purl->pwd);+ else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0')+ _http_authorize(conn, "Proxy-Authorization", p);+ }++ /* server authorization */+ if (need_auth || *url->user || *url->pwd) {+ if (*url->user || *url->pwd)+ _http_basic_auth(conn, "Authorization", url->user, url->pwd);+ else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0')+ _http_authorize(conn, "Authorization", p);+ else if (downloadAuthMethod && downloadAuthMethod(url) == 0) {+ _http_basic_auth(conn, "Authorization", url->user, url->pwd);+ } else {+ _http_seterr(HTTP_NEED_AUTH);+ goto ouch;+ }+ }++ /* other headers */+ if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') {+ if (strcasecmp(p, "auto") == 0)+ _http_cmd(conn, "Referer: %s://%s%s",+ url->scheme, host, url->doc);+ else+ _http_cmd(conn, "Referer: %s", p);+ }+ if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')+ _http_cmd(conn, "User-Agent: %s", p);+ else+ _http_cmd(conn, "User-Agent: " USER_AGENT_STRING);+ if (url->offset > 0)+ _http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);+ _http_cmd(conn, "Connection: close");+ _http_cmd(conn, "");++ /*+ * Force the queued request to be dispatched. Normally, one+ * would do this with shutdown(2) but squid proxies can be+ * configured to disallow such half-closed connections. To+ * be compatible with such configurations, fiddle with socket+ * options to force the pending data to be written.+ */+#ifdef TCP_NOPUSH+ val = 0;+ setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,+ sizeof(val));+#endif+ val = 1;+ setsockopt(conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,+ sizeof(val));++ /* get reply */+ switch (_http_get_reply(conn)) {+ case HTTP_OK:+ case HTTP_PARTIAL:+ /* fine */+ break;+ case HTTP_MOVED_PERM:+ case HTTP_MOVED_TEMP:+ case HTTP_SEE_OTHER:+ /*+ * Not so fine, but we still have to read the+ * headers to get the new location.+ */+ break;+ case HTTP_NEED_AUTH:+ if (need_auth) {+ /*+ * We already sent out authorization code,+ * so there's nothing more we can do.+ */+ _http_seterr(conn->err);+ goto ouch;+ }+ /* try again, but send the password this time */+ if (verbose)+ _download_info("server requires authorization");+ break;+ case HTTP_NEED_PROXY_AUTH:+ /*+ * If we're talking to a proxy, we already sent+ * our proxy authorization code, so there's+ * nothing more we can do.+ */+ _http_seterr(conn->err);+ goto ouch;+ case HTTP_BAD_RANGE:+ /*+ * This can happen if we ask for 0 bytes because+ * we already have the whole file. Consider this+ * a success for now, and check sizes later.+ */+ break;+ case HTTP_PROTOCOL_ERROR:+ /* fall through */+ case -1:+ _download_syserr();+ goto ouch;+ default:+ _http_seterr(conn->err);+ goto ouch;+ /* fall through so we can get the full error message */+ }++ /* get headers */+ do {+ switch ((h = _http_next_header(conn, &p))) {+ case hdr_syserror:+ _download_syserr();+ goto ouch;+ case hdr_error:+ _http_seterr(HTTP_PROTOCOL_ERROR);+ goto ouch;+ case hdr_content_length:+ _http_parse_length(p, &clength);+ break;+ case hdr_content_range:+ _http_parse_range(p, &offset, &length, &size);+ break;+ case hdr_last_modified:+ _http_parse_mtime(p, &mtime);+ break;+ case hdr_location:+ if (!HTTP_REDIRECT(conn->err))+ break;+ if (new)+ free(new);+ if (verbose)+ _download_info("%d redirect to %s", conn->err, p);+ if (*p == '/')+ /* absolute path */+ new = downloadMakeURL(url->scheme, url->host, url->port, p,+ url->user, url->pwd);+ else+ new = downloadParseURL(p);+ if (new == NULL) {+ /* XXX should set an error code */+ DBG(fprintf(stderr, "failed to parse new URL\n"));+ goto ouch;+ }+ if (!*new->user && !*new->pwd) {+ strcpy(new->user, url->user);+ strcpy(new->pwd, url->pwd);+ }+ new->offset = url->offset;+ new->length = url->length;+ break;+ case hdr_transfer_encoding:+ /* XXX weak test*/+ chunked = (strcasecmp(p, "chunked") == 0);+ break;+ case hdr_www_authenticate:+ if (conn->err != HTTP_NEED_AUTH)+ break;+ /* if we were smarter, we'd check the method and realm */+ break;+ case hdr_end:+ /* fall through */+ case hdr_unknown:+ /* ignore */+ break;+ }+ } while (h > hdr_end);++ /* we need to provide authentication */+ if (conn->err == HTTP_NEED_AUTH) {+ e = conn->err;+ need_auth = 1;+ _download_close(conn);+ conn = NULL;+ continue;+ }++ /* requested range not satisfiable */+ if (conn->err == HTTP_BAD_RANGE) {+ if (url->offset == size && url->length == 0) {+ /* asked for 0 bytes; fake it */+ offset = url->offset;+ conn->err = HTTP_OK;+ break;+ } else {+ _http_seterr(conn->err);+ goto ouch;+ }+ }++ /* we have a hit or an error */+ if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))+ break;++ /* all other cases: we got a redirect */+ e = conn->err;+ need_auth = 0;+ _download_close(conn);+ conn = NULL;+ if (!new) {+ DBG(fprintf(stderr, "redirect with no new location\n"));+ break;+ }+ if (url != URL)+ downloadFreeURL(url);+ url = new;+ } while (++i < n);++ /* we failed, or ran out of retries */+ if (conn == NULL) {+ _http_seterr(e);+ goto ouch;+ }++ DBG(fprintf(stderr, "offset %lld, length %lld,"+ " size %lld, clength %lld\n",+ (long long)offset, (long long)length,+ (long long)size, (long long)clength));++ /* check for inconsistencies */+ if (clength != -1 && length != -1 && clength != length) {+ _http_seterr(HTTP_PROTOCOL_ERROR);+ goto ouch;+ }+ if (clength == -1)+ clength = length;+ if (clength != -1)+ length = offset + clength;+ if (length != -1 && size != -1 && length != size) {+ _http_seterr(HTTP_PROTOCOL_ERROR);+ goto ouch;+ }+ if (size == -1)+ size = length;++ /* fill in stats */+ if (us) {+ us->size = size;+ us->atime = us->mtime = mtime;+ }++ /* too far? */+ if (URL->offset > 0 && offset > URL->offset) {+ _http_seterr(HTTP_PROTOCOL_ERROR);+ goto ouch;+ }++ /* report back real offset and size */+ URL->offset = offset;+ URL->length = clength;++ /* wrap it up in a FILE */+ if ((f = _http_funopen(conn, chunked)) == NULL) {+ _download_syserr();+ goto ouch;+ }++ if (url != URL)+ downloadFreeURL(url);+ if (purl)+ downloadFreeURL(purl);++ if (HTTP_ERROR(conn->err)) {+ _http_print_html(stderr, f);+ fclose(f);+ f = NULL;+ }++ return (f);++ouch:+ if (url != URL)+ downloadFreeURL(url);+ if (purl)+ downloadFreeURL(purl);+ if (conn != NULL)+ _download_close(conn);+ return (NULL);+}+++/*****************************************************************************+ * Entry points+ */++/*+ * Retrieve and stat a file by HTTP+ */+FILE *+downloadXGetHTTP(struct url *URL, struct url_stat *us, const char *flags)+{+ return (_http_request(URL, "GET", us, _http_get_proxy(flags), flags));+}++/*+ * Retrieve a file by HTTP+ */+FILE *+downloadGetHTTP(struct url *URL, const char *flags)+{+ return (downloadXGetHTTP(URL, NULL, flags));+}++/*+ * Get an HTTP document's metadata+ */+int+downloadStatHTTP(struct url *URL, struct url_stat *us, const char *flags)+{+ FILE *f;++ f = _http_request(URL, "HEAD", us, _http_get_proxy(flags), flags);+ if (f == NULL)+ return (-1);+ fclose(f);+ return (0);+}
+ cbits/httperr.h view
@@ -0,0 +1,44 @@+static struct downloaderr _http_errlist[] = {+ { 100, DLERR_OK, "Continue" },+ { 101, DLERR_OK, "Switching Protocols" },+ { 200, DLERR_OK, "OK" },+ { 201, DLERR_OK, "Created" },+ { 202, DLERR_OK, "Accepted" },+ { 203, DLERR_INFO, "Non-Authoritative Information" },+ { 204, DLERR_OK, "No Content" },+ { 205, DLERR_OK, "Reset Content" },+ { 206, DLERR_OK, "Partial Content" },+ { 300, DLERR_MOVED, "Multiple Choices" },+ { 301, DLERR_MOVED, "Moved Permanently" },+ { 302, DLERR_MOVED, "Moved Temporarily" },+ { 303, DLERR_MOVED, "See Other" },+ { 304, DLERR_OK, "Not Modified" },+ { 305, DLERR_INFO, "Use Proxy" },+ { 307, DLERR_MOVED, "Temporary Redirect" },+ { 400, DLERR_PROTO, "Bad Request" },+ { 401, DLERR_AUTH, "Unauthorized" },+ { 402, DLERR_AUTH, "Payment Required" },+ { 403, DLERR_AUTH, "Forbidden" },+ { 404, DLERR_UNAVAIL, "Not Found" },+ { 405, DLERR_PROTO, "Method Not Allowed" },+ { 406, DLERR_PROTO, "Not Acceptable" },+ { 407, DLERR_AUTH, "Proxy Authentication Required" },+ { 408, DLERR_TIMEOUT, "Request Time-out" },+ { 409, DLERR_EXISTS, "Conflict" },+ { 410, DLERR_UNAVAIL, "Gone" },+ { 411, DLERR_PROTO, "Length Required" },+ { 412, DLERR_SERVER, "Precondition Failed" },+ { 413, DLERR_PROTO, "Request Entity Too Large" },+ { 414, DLERR_PROTO, "Request-URI Too Large" },+ { 415, DLERR_PROTO, "Unsupported Media Type" },+ { 416, DLERR_UNAVAIL, "Requested Range Not Satisfiable" },+ { 417, DLERR_SERVER, "Expectation Failed" },+ { 500, DLERR_SERVER, "Internal Server Error" },+ { 501, DLERR_PROTO, "Not Implemented" },+ { 502, DLERR_SERVER, "Bad Gateway" },+ { 503, DLERR_TEMP, "Service Unavailable" },+ { 504, DLERR_TIMEOUT, "Gateway Time-out" },+ { 505, DLERR_PROTO, "HTTP Version not supported" },+ { 999, DLERR_PROTO, "Protocol error" },+ { -1, DLERR_UNKNOWN, "Unknown HTTP error" }+};
+ download.cabal view
@@ -0,0 +1,80 @@+name: download+version: 0.1+homepage: http://code.haskell.org/~dons/code/download+synopsis: High-level file download based on URLs+description: High-level File download based on URLs+ .+ Download content as bytestrings, strings or parsed+ tags, via ftp, http or file protocols.+category: Network+license: BSD3+license-file: LICENSE+copyright: (c) 2008, Don Stewart <dons@galois.com>+author: Don Stewart+maintainer: Don Stewart <dons@galois.com>+cabal-version: >= 1.2.0+build-type: Simple+tested-with: GHC ==6.8.2++-- flag external+-- description: Build with an external libdownload+-- default: False++-- flag small_base+-- description: Build with new smaller base library++library+ exposed-modules: Network.Download+ extensions: CPP,+ ForeignFunctionInterface,+ GeneralizedNewtypeDeriving+ ghc-options: -Wall -fvia-C+ build-depends: base, bytestring, tagsoup++ ------------------------------------------------------------------------+ -- Building libdownload+ --+ -- We can build against either an external libdownload, or an internal one.++-- if flag(external)+-- extra-libraries: download+-- c-sources: cbits/hs_download_utils.c+-- include-dirs: cbits+-- includes: hs_download_utils.h+-- install-includes: hs_download_utils.h+-- else++ cc-options: -O2+ -pipe+ -DINET6+ -D_GNU_SOURCE+ -D_FILE_OFFSET_BITS=64+ -Wall+ -Wstrict-prototypes+ -Wsign-compare+ -Wchar-subscripts+ -Wpointer-arith+ -Wcast-align+ -Wsign-compare+ -UDEBUG++ c-sources: cbits/download.c+ cbits/common.c+ cbits/ftp.c+ cbits/http.c+ cbits/file.c+ cbits/hs_download_utils.c++ include-dirs: cbits+ includes: common.h+ download.h+ ftperr.h+ httperr.h+ hs_download_utils.h++ install-includes: common.h+ download.h+ ftperr.h+ httperr.h+ hs_download_utils.h+