aws-kinesis-client 0.1.0.1 → 0.1.0.2
raw patch · 5 files changed
+109/−189 lines, 5 filesdep +hoist-errordep −aesondep −hourglassdep −timePVP ok
version bump matches the API change (PVP)
Dependencies added: hoist-error
Dependencies removed: aeson, hourglass, time
API changes (from Hackage documentation)
Files
- CHANGELOG.md +10/−0
- aws-kinesis-client.cabal +7/−18
- cli/CLI.hs +39/−44
- cli/CLI/Options.hs +53/−48
- cli/CLI/Record.hs +0/−79
CHANGELOG.md view
@@ -1,3 +1,13 @@+### v0.1.0.2++- Support specifying AWS keys as options to the CLI (either directly or from a+ file). You must specify one of these options.++- Remove filtering & date range capabilities from the CLI; remove `--raw` option,+ which will now be the only behavior.++- Fix CLI to print unescaped strings.+ ### v0.1.0.1 - Add some throttling to the consumer loop to avoid rate limiting
aws-kinesis-client.cabal view
@@ -1,5 +1,5 @@ name: aws-kinesis-client-version: 0.1.0.1+version: 0.1.0.2 synopsis: A producer & consumer client library for AWS Kinesis -- description: license: Apache-2.0@@ -48,41 +48,30 @@ transformers >=0.3.0.0 hs-source-dirs: src default-language: Haskell2010+ ghc-options: -Wall executable kinesis-cli default-language: Haskell2010 hs-source-dirs: cli main-is: CLI.hs- other-modules: CLI.Options,- CLI.Record+ other-modules: CLI.Options build-depends: base >=4.7 && <4.8, base-unicode-symbols, aws >=0.10.5,+ aws-general >=0.1.1, aws-kinesis >=0.1.2, aws-kinesis-client, conduit >=1.2.3.1,- data-carousel >=0.1.0.0, either >=4.3.2.1,- errors >=1.4.7,+ hoist-error >=0.1.0.2, http-conduit >=2.1.5, kan-extensions >=4.2,- hourglass >=0.2.6, lens >=4.7,- lens-action >=0.1.0.1,- lifted-async >=0.3.0, lifted-base >=0.2.3.3, monad-control >=0.3.3.0, mtl >=2.2.1,- random >=1.1,- resourcet >=1.1.3.3,- stm >=2.4.4,- stm-chans >=3.0.0.2,- stm-conduit >=2.5.3,- stm-queue-extras >=0.2.0.0, text >=1.2.0.3, transformers >=0.3.0.0,- aeson >=0.8,- aws-general >=0.1.1, bytestring >=0.10.4,- optparse-applicative >=0.11,- time >=1.4.2+ optparse-applicative >=0.11+ ghc-options: -Wall
cli/CLI.hs view
@@ -17,13 +17,16 @@ -- under the License. {-# LANGUAGE ConstraintKinds #-}+{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE FlexibleContexts #-}-{-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-}+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE TemplateHaskell #-}+{-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE UnicodeSyntax #-} -- |@@ -34,41 +37,45 @@ -- Stability: experimental -- -module Main where+module Main+( main+) where -import Aws.Aws+import Aws import Aws.General import Aws.Kinesis hiding (Record) import Aws.Kinesis.Client.Common import Aws.Kinesis.Client.Consumer import CLI.Options-import CLI.Record -import Control.Applicative-import Control.Monad+import Control.Exception+import Control.Lens import Control.Monad.Trans import Control.Monad.Trans.Control import Control.Monad.Trans.Either-import Control.Monad.Trans.Resource import Control.Monad.Codensity import Control.Monad.Reader.Class import Control.Monad.Trans.Reader (ReaderT(..)) import Control.Monad.Error.Class+import Control.Monad.Error.Hoist -import Data.Aeson import qualified Data.ByteString.Char8 as B8-import qualified Data.ByteString.Lazy as BL import Data.Conduit-import Data.Monoid import qualified Data.Conduit.List as CL-import qualified Data.Text.Lens as T+import Data.Typeable import Options.Applicative import qualified Network.HTTP.Conduit as HC import Prelude.Unicode import Control.Monad.Unicode +data CLIError+ = MissingCredentials+ deriving (Typeable, Show)++instance Exception CLIError+ type MonadCLI m = ( MonadReader CLIOptions m , MonadIO m@@ -76,33 +83,25 @@ , MonadError ConsumerError m ) -identityConduit- ∷ Monad m- ⇒ Conduit a m a-identityConduit = CL.map id- limitConduit ∷ MonadCLI m ⇒ Conduit a m a limitConduit =- lift (asks clioLimit) ≫=+ lift (view clioLimit) ≫= CL.isolate -filterConduit+fetchCredentials ∷ MonadCLI m- ⇒ Conduit Record m Record-filterConduit = do- CLIOptions{..} ← lift ask- CL.filter (\Record{..} → maybe True (rTimestamp ≥) clioStartDate)- =$ takeTill (\Record{..} → maybe False (rTimestamp >) clioEndDate)--takeTill- ∷ Monad m- ⇒ (i → Bool)- → Conduit i m i-takeTill f = loop- where- loop = await ≫= maybe (return ()) (\x → unless (f x) $ yield x ≫ loop)+ ⇒ m Credentials+fetchCredentials = do+ view clioAccessKeys ≫= \case+ Left aks →+ makeCredentials+ (aks ^. akAccessKeyId)+ (aks ^. akSecretAccessKey)+ Right path →+ loadCredentialsFromFile path credentialsDefaultKey+ <!?> KinesisError (toException MissingCredentials) app ∷ MonadCLI m@@ -110,28 +109,24 @@ app = do CLIOptions{..} ← ask manager ← managedHttpManager- awsConfiguration ← liftIO baseConfiguration+ credentials ← lift fetchCredentials consumer ← managedKinesisConsumer $ ConsumerKit { _ckKinesisKit = KinesisKit { _kkManager = manager- , _kkConfiguration = awsConfiguration+ , _kkConfiguration = Configuration+ { timeInfo = Timestamp+ , credentials = credentials+ , logger = defaultLog Warning+ } , _kkKinesisConfiguration = KinesisConfiguration UsWest2 }- , _ckStreamName = clioStreamName+ , _ckStreamName = _clioStreamName , _ckBatchSize = 100- , _ckIteratorType = clioIteratorType+ , _ckIteratorType = _clioIteratorType } lift $ consumerSource consumer $$- case clioRaw of- False →- CL.mapMaybe (decode ∘ BL.fromChunks ∘ (:[]) ∘ recordData)- =$ filterConduit- =$ limitConduit- =$ CL.mapM_ (liftIO ∘ print)- True →- limitConduit- =$ CL.mapM_ (liftIO ∘ B8.putStrLn ∘ recordData)+ limitConduit =$ CL.mapM_ (liftIO ∘ B8.putStrLn ∘ recordData) return () main ∷ IO ()
cli/CLI/Options.hs view
@@ -18,6 +18,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE UnicodeSyntax #-} -- |@@ -30,32 +31,43 @@ module CLI.Options ( CLIOptions(..)+, clioStreamName+, clioLimit+, clioIteratorType+, clioAccessKeys+, AccessKeys(..)+, akAccessKeyId+, akSecretAccessKey , optionsParser , parserInfo ) where import Aws.Kinesis+import qualified Data.ByteString.Char8 as B8 import Control.Applicative.Unicode import Control.Lens-import Control.Monad.Unicode import Data.Monoid.Unicode-import qualified Data.Text as T import qualified Data.Text.Lens as T-import Data.Hourglass import Options.Applicative import Prelude.Unicode +data AccessKeys+ = AccessKeys+ { _akAccessKeyId ∷ !B8.ByteString+ , _akSecretAccessKey ∷ !B8.ByteString+ } deriving Show+ data CLIOptions = CLIOptions- { clioStreamName ∷ StreamName- , clioLimit ∷ Int- , clioIteratorType ∷ ShardIteratorType- , clioStartDate ∷ Maybe DateTime- , clioEndDate ∷ Maybe DateTime- -- , clioTimeDuration ∷ Maybe Seconds- , clioRaw ∷ Bool+ { _clioStreamName ∷ !StreamName+ , _clioLimit ∷ !Int+ , _clioIteratorType ∷ !ShardIteratorType+ , _clioAccessKeys ∷ !(Either AccessKeys FilePath) } deriving Show +makeLenses ''AccessKeys+makeLenses ''CLIOptions+ eitherTextReader ∷ ( T.IsText i , T.IsText e@@ -66,6 +78,33 @@ eitherReader $ (_Left %~ view T.unpacked) ∘ f ∘ view T.packed +accessKeyIdParser ∷ Parser B8.ByteString+accessKeyIdParser =+ fmap B8.pack ∘ strOption $+ long "access-key-id"+ ⊕ metavar "ID"+ ⊕ help "Your AWS access key id"++secretAccessKeyParser ∷ Parser B8.ByteString+secretAccessKeyParser =+ fmap B8.pack ∘ strOption $+ long "secret-access-key"+ ⊕ metavar "SK"+ ⊕ help "Your AWS secret access key"++accessKeysParser ∷ Parser AccessKeys+accessKeysParser =+ pure AccessKeys+ ⊛ accessKeyIdParser+ ⊛ secretAccessKeyParser++accessKeysPathParser ∷ Parser FilePath+accessKeysPathParser =+ strOption $+ long "access-keys-path"+ ⊕ metavar "PATH"+ ⊕ help "The path to a file containing your access keys. To be formatted \"default ID SECRET\""+ streamNameParser ∷ Parser StreamName streamNameParser = option (eitherTextReader streamName) $@@ -92,52 +131,18 @@ ⊕ value TrimHorizon ⊕ showDefault -readDateTime- ∷ String- → ReadM DateTime-readDateTime =- maybe (readerError "Invalid DateTime") return- ∘ timeParse ISO8601_DateAndTime--startDateParser ∷ Parser DateTime-startDateParser =- option (str ≫= readDateTime) $- long "start-date"- ⊕ short 's'- ⊕ metavar "SD"- ⊕ help "Start Date (ISO 8601)"--endDateParser ∷ Parser DateTime-endDateParser =- option (str ≫= readDateTime) $- long "end-date"- ⊕ short 'e'- ⊕ metavar "ED"- ⊕ help "End Date (ISO 8601)"--timeDurationParser ∷ Parser Seconds-timeDurationParser =- option auto $- long "end-date"- ⊕ short 'e'- ⊕ metavar "ED"- ⊕ help "Time window from start (in seconds)"- optionsParser ∷ Parser CLIOptions optionsParser =- CLIOptions- <$> streamNameParser+ pure CLIOptions+ ⊛ streamNameParser ⊛ limitParser ⊛ iteratorTypeParser- ⊛ optional startDateParser- ⊛ optional endDateParser- -- ⊛ optional timeDurationParser- ⊛ switch (long "raw" ⊕ help "Treat records as raw text")+ ⊛ (Left <$> accessKeysParser <|> Right <$> accessKeysPathParser) parserInfo ∷ ParserInfo CLIOptions parserInfo = info (helper ⊛ optionsParser) $ fullDesc- ⊕ progDesc "Fetch `L` records from a Kinesis stream `SN`. Put your AWS keys in ~/.aws-keys"+ ⊕ progDesc "Fetch `L` records from a Kinesis stream `SN`." ⊕ header "The Kinesis Consumer CLI"
− cli/CLI/Record.hs
@@ -1,79 +0,0 @@--- Copyright (c) 2013-2014 PivotCloud, Inc.------ Aws.Kinesis.Client.Consumer------ Please feel free to contact us at licensing@pivotmail.com with any--- contributions, additions, or other feedback; we would love to hear from--- you.------ Licensed under the Apache License, Version 2.0 (the "License"); you may--- not use this file except in compliance with the License. You may obtain a--- copy of the License at http://www.apache.org/licenses/LICENSE-2.0------ Unless required by applicable law or agreed to in writing, software--- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT--- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the--- License for the specific language governing permissions and limitations--- under the License.--{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE UnicodeSyntax #-}---- |--- Module: CLI.Record--- Copyright: Copyright © 2013-2014 PivotCloud, Inc.--- License: Apache-2.0--- Maintainer: Jon Sterling <jsterling@alephcloud.com>--- Stability: experimental-----module CLI.Record where--import Data.Aeson-import Data.Aeson.Types-import Data.Time-import qualified Data.Text as T-import qualified Data.ByteString as BS-import Control.Applicative-import Control.Applicative.Unicode-import Data.Hourglass-import Prelude.Unicode--data Record- = Record- { rTimestamp ∷ DateTime- , rMessage ∷ T.Text- } deriving (Show, Eq)--instance FromJSON DateTime where- parseJSON =- withText "DateTime" $- maybe (fail "Invalid DateTime") return- ∘ timeParse fmt- ∘ T.unpack- where- fmt =- [ Format_Year4- , hyphen- , Format_Month2- , hyphen- , Format_Day2- , Format_Text 'T'- , Format_Hour- , colon- , Format_Minute- , colon- , Format_Second- ]- hyphen = Format_Text '-'- colon = Format_Text ':'--instance FromJSON Record where- parseJSON =- withObject "Record" $ \xs →- Record <$> xs .: "time" ⊛ xs .: "message"--instance Ord Record where- compare r1 r2 = compare (rTimestamp r1) (rTimestamp r2)-