quandl-api (empty) → 0.1.0.0
raw patch · 5 files changed
+387/−0 lines, 5 filesdep +aesondep +basedep +blaze-buildersetup-changed
Dependencies added: aeson, base, blaze-builder, bytestring, http-conduit, http-types, old-locale, syb, text, time, unordered-containers
Files
- Data/Quandl.hs +267/−0
- LICENSE +27/−0
- README.md +49/−0
- Setup.hs +2/−0
- quandl-api.cabal +42/−0
+ Data/Quandl.hs view
@@ -0,0 +1,267 @@+{-# LANGUAGE OverloadedStrings, DeriveDataTypeable, TupleSections, FlexibleInstances #-}++-- |+-- Module: Data.Quandl+-- Copyright: (c) 2014 Peter van den Brand+-- License: BSD3+-- Maintainer: Peter van den Brand <peter@vdbrand.nl>+-- Stability: experimental+-- Portability: portable+--+-- This library provides an easy way to download data from Quandl.com.+-- See <http://www.quandl.com/help/api> for detailed information on the Quandl API.+--+-- For basic usage, see the 'getTable' function. This function is all you need to download tables.+--+-- For more advanced usage, see the 'getTableWith' function. This function allows you+-- to use query a subset of the data, query multiple tables (multisets),+-- apply frequency conversions, and apply transformations supported by Quandl.++module Data.Quandl (++ -- * Download a whole table at once+ getTable,++ -- * Download using API parameters+ defaultOptions,+ getTableWith,++ -- * Datatypes+ Options(..),+ Frequency(..),+ Transformation(..),+ Dataset(..),+ Metadata(..),++ -- * Low-level functions+ downloadJSON,+ createUrl+ ) where++import qualified Data.ByteString.Lazy as L+import qualified Data.ByteString.Char8 as BC+import qualified Data.Text as T+import qualified Data.HashMap.Strict as H++import Data.Char (toLower, toUpper)+import Data.Time (UTCTime, Day, readTime)+import Data.Monoid ((<>))+import Data.Generics (Data, Typeable)+import Data.List (intercalate)+import Data.Aeson (decode', Value(..), (.:), (.:?), FromJSON(..))+import Data.Aeson.Types (Parser)+import Control.Monad (mzero)+import Control.Applicative ((<$>), (<*>), pure)+import System.Locale (defaultTimeLocale)++import Network.HTTP.Conduit (simpleHttp)+import Network.HTTP.Types.URI (encodePathSegments, renderQueryBuilder)+import Blaze.ByteString.Builder (toByteString, fromByteString)++-- | API parameters supported by Quandl, for use with the 'getTableWith' function.+-- See <http://www.quandl.com/help/api> for detailed information.+data Options = Options {+ opAuthToken :: Maybe String, -- ^ The Quandl Auth Token ('Nothing' means no token supplied, limited to 50 requests per day)+ opSortAscending :: Bool, -- ^ Sort results ascending or descending+ opNumRows :: Maybe Int, -- ^ Limits the number of returned rows ('Nothing' means no limit)+ opStartDate :: Maybe Day, -- ^ Start date of returned results (inclusive, 'Nothing' means no restriction on start date)+ opEndDate :: Maybe Day, -- ^ End date of returned results (inclusive, 'Nothing' means no restriction on end date)+ opFrequency :: Maybe Frequency, -- ^ Desired frequency of returned results ('Nothing' means frequency of original dataset)+ opTransformation :: Maybe Transformation, -- ^ Desired transformation of returned results ('Nothing' means no transformation)+ opMetadataOnly :: Bool -- ^ Only return metadata, do not return any data (only works for single table queries)+ } deriving (Eq, Ord, Show, Data, Typeable)++-- | Desired frequency of returned results.+-- See <http://www.quandl.com/help/api> for detailed information.+data Frequency+ = Daily+ | Weekly + | Monthly + | Quarterly + | Annual + deriving (Eq, Ord, Show, Data, Typeable)++-- | Desired transformation of returned results.+-- See <http://www.quandl.com/help/api> for detailed information.+data Transformation+ = Diff + | RDiff + | Cumul + | Normalize + deriving (Eq, Ord, Show, Data, Typeable)++-- | Metadata of a table. +-- Only returned by Quandl when downloading from a single table.+data Metadata = Metadata {+ meId :: Int, -- ^ Table ID+ meSourceCode :: String, -- ^ Source code (can be used as parameter to 'getTable' and 'getTableWith')+ meTableCode :: String, -- ^ Table code (can be used as parameter to 'getTable' and 'getTableWith')+ meSourceName :: T.Text, -- ^ Human-readable name of the source+ meTableName :: T.Text, -- ^ Human-readable name of the table+ meUrlName :: T.Text, -- ^ Urlized name of the table as used on Quandl.com+ meDescription :: T.Text, -- ^ Description of the table+ meSourceUrl :: T.Text, -- ^ URL of the original data source+ meUpdatedAt :: UTCTime, -- ^ Timestamp of latest update+ mePrivate :: Bool -- ^ Private or public table+ } deriving (Eq, Ord, Show, Data, Typeable)++-- | Results from a Quandl API call.+data Dataset = Dataset {+ daTable :: Maybe Metadata, -- ^ Metadata of the table ('Nothing' if fields from multiple tables are downloaded)+ daColumnNames :: [T.Text], -- ^ The column names of the table+ daData :: [[T.Text]], -- ^ The contents of the table+ daFromDate :: Day, -- ^ The starting date of the returned data (inclusive)+ daToDate :: Day, -- ^ The ending date of the returned data (inclusive)+ daFrequency :: T.Text -- ^ The frequency of the returned data (daily, monthly, etc)+ } deriving (Eq, Show, Data, Typeable)++-------------------------------------------------------------------------------+-- JSON Parsers++asRows :: Maybe [[Value]] -> [[T.Text]]+asRows Nothing = [[]]+asRows (Just x) = map (map convert) x+ where+ convert (Object o) = T.pack $ show o+ convert (Array a) = T.pack $ show a+ convert (String s) = s+ convert (Number n) = T.pack $ show n+ convert (Bool b) = T.pack $ show b+ convert (Null) = T.empty++asUTCTime :: String -> UTCTime+asUTCTime = readTime defaultTimeLocale "%FT%T%QZ"++asDay :: String -> Day+asDay = readTime defaultTimeLocale "%F"++parseMetadata :: Value -> Parser (Maybe Metadata)+parseMetadata o@(Object obj) = case H.lookup "id" obj of+ Nothing -> pure Nothing+ Just _ -> parseJSON o+parseMetadata _ = pure Nothing++instance FromJSON Metadata where+ parseJSON (Object v) = Metadata <$>+ v .: "id" <*>+ v .: "source_code" <*>+ v .: "code" <*>+ v .: "source_name" <*>+ v .: "name" <*>+ v .: "urlize_name" <*>+ v .: "description" <*>+ v .: "display_url" <*>+ (asUTCTime <$> v .: "updated_at") <*>+ v .: "private"+ parseJSON _ = mzero++instance FromJSON Dataset where+ parseJSON o@(Object v) = Dataset <$>+ parseMetadata o <*>+ v .: "column_names" <*>+ (asRows <$> v .:? "data") <*>+ (asDay <$> v .: "from_date") <*>+ (asDay <$> v .: "to_date") <*>+ v .: "frequency"+ parseJSON _ = mzero++-------------------------------------------------------------------------------+-- Public functions++-- | Default options to use for Quandl API calls.+-- The default options do not use an auth token+-- and will return all rows in descending order.+defaultOptions :: Options+defaultOptions = Options {+ opAuthToken = Nothing,+ opSortAscending = False,+ opNumRows = Nothing,+ opStartDate = Nothing,+ opEndDate = Nothing,+ opFrequency = Nothing,+ opTransformation = Nothing,+ opMetadataOnly = False+}++-- | Download all rows and columns from a single table.+-- To get all data points for the dataset FRED/GDP:+--+-- > getTable "FRED" "GDP" Nothing+--+-- Registered users should include their auth_token, like this:+--+-- > getTable "FRED" "GDP" (Just "dsahFHUiewjjd")+getTable :: String -- ^ Quandl code for the source+ -> String -- ^ Quandl code for the table+ -> Maybe String -- ^ Auth code+ -> IO (Maybe Dataset) -- ^ Dataset returned by Quandl, or 'Nothing' if parsing failed+getTable source table auth = decode' <$> downloadJSON (defaultOptions { opAuthToken = auth }) [(source, table, Nothing)]++-- | Download data from Quandl using the full API.+-- This function supports all data manipulation options, plus downloading from multiple datasets (multisets).+--+-- For example, here is the annual percentage return for AAPL stock over the previous decade, in ascending date order:+--+-- > import Data.Quandl+-- > import Data.Time (fromGregorian)+-- > getTableWith (defaultOptions {opSortAscending = True, +-- > opStartDate = Just (fromGregorian 2000 1 1), +-- > opEndDate = Just (fromGregorian 2010 1 1), +-- > opFrequency = Just Annual, +-- > opTransformation = Just RDiff})+-- > [("WIKI", "AAPL", Just 4)] -- Just 4 means we only want the 4'th column (Close price)+--+-- You can pull data from multiple datasets (or from multiple columns in a single dataset) using this function as well.+-- In the example below, we combine US GDP from FRED\/GDP, crude oil spot prices from DOE\/RWTC, and Apple closing prices from WIKI\/AAPL. +-- We are going to convert all of them to annual percentage changes, and look only at data for the last 10 years.+--+-- > import Data.Quandl+-- > getTableWith (defaultOptions {opNumRows = Just 10,+-- > opFrequency = Just Annual,+-- > opTransformation = Just RDiff})+-- > [("FRED", "GDP", Just 1), ("DOE", "RWTC", Just 1), ("WIKI", "AAPL", Just 4)]+--+-- Please note that Quandl does not return the table metadata when pulling data from multiple datasets.+-- In that case the 'daTable' field of the returned 'Dataset' will be 'Nothing'.+getTableWith :: Options -- ^ API parameters+ -> [(String, String, Maybe Int)] -- ^ List of (source, table, column) items to retrieve. 'Nothing' retrieves all columns.+ -> IO (Maybe Dataset) -- ^ Dataset returned by Quandl, or 'Nothing' if parsing failed+getTableWith options items = decode' <$> downloadJSON options items++-- | Returns unparsed JSON from a Quandl API call.+downloadJSON :: Options -> [(String, String, Maybe Int)] -> IO L.ByteString+downloadJSON options items = simpleHttp $ createUrl options items++-- | Construct a URL given the various API parameter options.+createUrl :: Options -> [(String, String, Maybe Int)] -> String+createUrl options items =+ let pathSegs = case items of+ [(source, table, _)] -> ["api", "v1", "datasets", T.pack (map toUpper source), T.pack (map toUpper table ++ ".json")]+ _ -> ["api", "v1", "multisets.json"]+ column = case items of+ [(_, _, c)] -> maybeParam ("column", fmap show . const c)+ _ -> param ("columns", createColumns items)+ query = column +++ concatMap maybeParam [+ ("auth_token", opAuthToken),+ ("rows", fmap show . opNumRows),+ ("trim_start", fmap show . opStartDate),+ ("trim_end", fmap show . opEndDate),+ ("collapse", fmap (map toLower . show) . opFrequency),+ ("transformation", fmap (map toLower . show) . opTransformation)] +++ concatMap param [+ ("sort_order", if opSortAscending options then "asc" else "desc"),+ ("exclude_data", if opMetadataOnly options then "true" else "false")+ ]+ maybeParam (k, fn) = maybe [] (param . (k,)) $ fn options+ param (k, v) = [(k, Just $ BC.pack v)]+ in BC.unpack $ toByteString $+ fromByteString "http://www.quandl.com" <>+ encodePathSegments pathSegs <>+ renderQueryBuilder True query++createColumns :: [(String, String, Maybe Int)] -> String+createColumns items =+ let toItem (source, table, column) = map toUpper source ++ "." ++ map toUpper table ++ (case column of { Nothing -> ""; Just c -> "." ++ show c })+ in intercalate "," $ map toItem items+
+ LICENSE view
@@ -0,0 +1,27 @@+Copyright (c) 2014, Peter van den Brand+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++* Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++* Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++* Neither the name of the {organization} nor the names of its+ contributors may be used to endorse or promote products derived from+ this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,49 @@+# quandl-api++This library provides an easy way to download data from Quandl.com in Haskell.++## Installation++* Install the [Haskell Platform](http://www.haskell.org/platform/)+* `cabal install quandl-api`++## Basic Usage++The `getTable` function is all you need to download tables.+To get all data points for the dataset FRED/GDP:++ getTable "FRED" "GDP" Nothing++Registered users should include their auth_token, like this:++ getTable "FRED" "GDP" (Just "dsahFHUiewjjd")++## Advanced Usage++The `getTableWith` function allows you to use query a subset of the data,+query multiple tables (multisets), apply frequency conversions,+and apply transformations supported by Quandl.+For example, here is the annual percentage return for AAPL stock+over the previous decade, in ascending date order:++ import Data.Quandl+ import Data.Time (fromGregorian)+ getTableWith (defaultOptions {opSortAscending = True, + opStartDate = Just (fromGregorian 2000 1 1), + opEndDate = Just (fromGregorian 2010 1 1), + opFrequency = Just Annual, + opTransformation = Just RDiff})+ [("WIKI", "AAPL", Just 4)] -- Just 4 means we only want the 4'th column (Close price)++You can pull data from multiple datasets (or from multiple columns in a single dataset) using this function as well.+In the example below, we combine US GDP from FRED/GDP, crude oil spot prices from DOE/RWTC, and Apple closing prices from WIKI/AAPL. +We are going to convert all of them to annual percentage changes, and look only at data for the last 10 years:++ import Data.Quandl+ getTableWith (defaultOptions {opNumRows = Just 10,+ opFrequency = Just Annual,+ opTransformation = Just RDiff})+ [("FRED", "GDP", Just 1), ("DOE", "RWTC", Just 1), ("WIKI", "AAPL", Just 4)]++See http://www.quandl.com/help/api for detailed information on the Quandl API.+
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ quandl-api.cabal view
@@ -0,0 +1,42 @@+name: quandl-api+version: 0.1.0.0+copyright: (c) 2014 Peter van den Brand+homepage: https://github.com/pvdbrand/quandl-api+bug-reports: https://github.com/pvdbrand/quandl-api/issues+license: BSD3+license-file: LICENSE+author: Peter van den Brand <peter@vdbrand.nl>+maintainer: Peter van den Brand <peter@vdbrand.nl>+category: Data, API, Web+build-type: Simple+cabal-version: >=1.8+stability: experimental+synopsis: Quandl.com API library+description: This library provides an easy way to download data from Quandl.com.++extra-source-files:+ README.md++source-repository head+ type: git+ location: git://github.com/pvdbrand/quandl-api.git++library+ exposed-modules:+ Data.Quandl++ build-depends:+ base == 4.*,+ http-types,+ http-conduit,+ blaze-builder,+ unordered-containers,+ old-locale,+ bytestring,+ text,+ time,+ syb,+ aeson++ ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2+