diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +### v0.1.0.2 + +- Support specifying AWS keys as options to the CLI (either directly or from a + file). You must specify one of these options. + +- Remove filtering & date range capabilities from the CLI; remove `--raw` option, + which will now be the only behavior. + +- Fix CLI to print unescaped strings. + ### v0.1.0.1 - Add some throttling to the consumer loop to avoid rate limiting diff --git a/aws-kinesis-client.cabal b/aws-kinesis-client.cabal --- a/aws-kinesis-client.cabal +++ b/aws-kinesis-client.cabal @@ -1,5 +1,5 @@ name: aws-kinesis-client -version: 0.1.0.1 +version: 0.1.0.2 synopsis: A producer & consumer client library for AWS Kinesis -- description: license: Apache-2.0 @@ -48,41 +48,30 @@ transformers >=0.3.0.0 hs-source-dirs: src default-language: Haskell2010 + ghc-options: -Wall executable kinesis-cli default-language: Haskell2010 hs-source-dirs: cli main-is: CLI.hs - other-modules: CLI.Options, - CLI.Record + other-modules: CLI.Options build-depends: base >=4.7 && <4.8, base-unicode-symbols, aws >=0.10.5, + aws-general >=0.1.1, aws-kinesis >=0.1.2, aws-kinesis-client, conduit >=1.2.3.1, - data-carousel >=0.1.0.0, either >=4.3.2.1, - errors >=1.4.7, + hoist-error >=0.1.0.2, http-conduit >=2.1.5, kan-extensions >=4.2, - hourglass >=0.2.6, lens >=4.7, - lens-action >=0.1.0.1, - lifted-async >=0.3.0, lifted-base >=0.2.3.3, monad-control >=0.3.3.0, mtl >=2.2.1, - random >=1.1, - resourcet >=1.1.3.3, - stm >=2.4.4, - stm-chans >=3.0.0.2, - stm-conduit >=2.5.3, - stm-queue-extras >=0.2.0.0, text >=1.2.0.3, transformers >=0.3.0.0, - aeson >=0.8, - aws-general >=0.1.1, bytestring >=0.10.4, - optparse-applicative >=0.11, - time >=1.4.2 + optparse-applicative >=0.11 + ghc-options: -Wall diff --git a/cli/CLI.hs b/cli/CLI.hs --- a/cli/CLI.hs +++ b/cli/CLI.hs @@ -17,13 +17,16 @@ -- under the License. {-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE UnicodeSyntax #-} -- | @@ -34,41 +37,45 @@ -- Stability: experimental -- -module Main where +module Main +( main +) where -import Aws.Aws +import Aws import Aws.General import Aws.Kinesis hiding (Record) import Aws.Kinesis.Client.Common import Aws.Kinesis.Client.Consumer import CLI.Options -import CLI.Record -import Control.Applicative -import Control.Monad +import Control.Exception +import Control.Lens import Control.Monad.Trans import Control.Monad.Trans.Control import Control.Monad.Trans.Either -import Control.Monad.Trans.Resource import Control.Monad.Codensity import Control.Monad.Reader.Class import Control.Monad.Trans.Reader (ReaderT(..)) import Control.Monad.Error.Class +import Control.Monad.Error.Hoist -import Data.Aeson import qualified Data.ByteString.Char8 as B8 -import qualified Data.ByteString.Lazy as BL import Data.Conduit -import Data.Monoid import qualified Data.Conduit.List as CL -import qualified Data.Text.Lens as T +import Data.Typeable import Options.Applicative import qualified Network.HTTP.Conduit as HC import Prelude.Unicode import Control.Monad.Unicode +data CLIError + = MissingCredentials + deriving (Typeable, Show) + +instance Exception CLIError + type MonadCLI m = ( MonadReader CLIOptions m , MonadIO m @@ -76,33 +83,25 @@ , MonadError ConsumerError m ) -identityConduit - ∷ Monad m - ⇒ Conduit a m a -identityConduit = CL.map id - limitConduit ∷ MonadCLI m ⇒ Conduit a m a limitConduit = - lift (asks clioLimit) ≫= + lift (view clioLimit) ≫= CL.isolate -filterConduit +fetchCredentials ∷ MonadCLI m - ⇒ Conduit Record m Record -filterConduit = do - CLIOptions{..} ← lift ask - CL.filter (\Record{..} → maybe True (rTimestamp ≥) clioStartDate) - =$ takeTill (\Record{..} → maybe False (rTimestamp >) clioEndDate) - -takeTill - ∷ Monad m - ⇒ (i → Bool) - → Conduit i m i -takeTill f = loop - where - loop = await ≫= maybe (return ()) (\x → unless (f x) $ yield x ≫ loop) + ⇒ m Credentials +fetchCredentials = do + view clioAccessKeys ≫= \case + Left aks → + makeCredentials + (aks ^. akAccessKeyId) + (aks ^. akSecretAccessKey) + Right path → + loadCredentialsFromFile path credentialsDefaultKey + KinesisError (toException MissingCredentials) app ∷ MonadCLI m @@ -110,28 +109,24 @@ app = do CLIOptions{..} ← ask manager ← managedHttpManager - awsConfiguration ← liftIO baseConfiguration + credentials ← lift fetchCredentials consumer ← managedKinesisConsumer $ ConsumerKit { _ckKinesisKit = KinesisKit { _kkManager = manager - , _kkConfiguration = awsConfiguration + , _kkConfiguration = Configuration + { timeInfo = Timestamp + , credentials = credentials + , logger = defaultLog Warning + } , _kkKinesisConfiguration = KinesisConfiguration UsWest2 } - , _ckStreamName = clioStreamName + , _ckStreamName = _clioStreamName , _ckBatchSize = 100 - , _ckIteratorType = clioIteratorType + , _ckIteratorType = _clioIteratorType } lift $ consumerSource consumer $$ - case clioRaw of - False → - CL.mapMaybe (decode ∘ BL.fromChunks ∘ (:[]) ∘ recordData) - =$ filterConduit - =$ limitConduit - =$ CL.mapM_ (liftIO ∘ print) - True → - limitConduit - =$ CL.mapM_ (liftIO ∘ B8.putStrLn ∘ recordData) + limitConduit =$ CL.mapM_ (liftIO ∘ B8.putStrLn ∘ recordData) return () main ∷ IO () diff --git a/cli/CLI/Options.hs b/cli/CLI/Options.hs --- a/cli/CLI/Options.hs +++ b/cli/CLI/Options.hs @@ -18,6 +18,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE UnicodeSyntax #-} -- | @@ -30,32 +31,43 @@ module CLI.Options ( CLIOptions(..) +, clioStreamName +, clioLimit +, clioIteratorType +, clioAccessKeys +, AccessKeys(..) +, akAccessKeyId +, akSecretAccessKey , optionsParser , parserInfo ) where import Aws.Kinesis +import qualified Data.ByteString.Char8 as B8 import Control.Applicative.Unicode import Control.Lens -import Control.Monad.Unicode import Data.Monoid.Unicode -import qualified Data.Text as T import qualified Data.Text.Lens as T -import Data.Hourglass import Options.Applicative import Prelude.Unicode +data AccessKeys + = AccessKeys + { _akAccessKeyId ∷ !B8.ByteString + , _akSecretAccessKey ∷ !B8.ByteString + } deriving Show + data CLIOptions = CLIOptions - { clioStreamName ∷ StreamName - , clioLimit ∷ Int - , clioIteratorType ∷ ShardIteratorType - , clioStartDate ∷ Maybe DateTime - , clioEndDate ∷ Maybe DateTime - -- , clioTimeDuration ∷ Maybe Seconds - , clioRaw ∷ Bool + { _clioStreamName ∷ !StreamName + , _clioLimit ∷ !Int + , _clioIteratorType ∷ !ShardIteratorType + , _clioAccessKeys ∷ !(Either AccessKeys FilePath) } deriving Show +makeLenses ''AccessKeys +makeLenses ''CLIOptions + eitherTextReader ∷ ( T.IsText i , T.IsText e @@ -66,6 +78,33 @@ eitherReader $ (_Left %~ view T.unpacked) ∘ f ∘ view T.packed +accessKeyIdParser ∷ Parser B8.ByteString +accessKeyIdParser = + fmap B8.pack ∘ strOption $ + long "access-key-id" + ⊕ metavar "ID" + ⊕ help "Your AWS access key id" + +secretAccessKeyParser ∷ Parser B8.ByteString +secretAccessKeyParser = + fmap B8.pack ∘ strOption $ + long "secret-access-key" + ⊕ metavar "SK" + ⊕ help "Your AWS secret access key" + +accessKeysParser ∷ Parser AccessKeys +accessKeysParser = + pure AccessKeys + ⊛ accessKeyIdParser + ⊛ secretAccessKeyParser + +accessKeysPathParser ∷ Parser FilePath +accessKeysPathParser = + strOption $ + long "access-keys-path" + ⊕ metavar "PATH" + ⊕ help "The path to a file containing your access keys. To be formatted \"default ID SECRET\"" + streamNameParser ∷ Parser StreamName streamNameParser = option (eitherTextReader streamName) $ @@ -92,52 +131,18 @@ ⊕ value TrimHorizon ⊕ showDefault -readDateTime - ∷ String - → ReadM DateTime -readDateTime = - maybe (readerError "Invalid DateTime") return - ∘ timeParse ISO8601_DateAndTime - -startDateParser ∷ Parser DateTime -startDateParser = - option (str ≫= readDateTime) $ - long "start-date" - ⊕ short 's' - ⊕ metavar "SD" - ⊕ help "Start Date (ISO 8601)" - -endDateParser ∷ Parser DateTime -endDateParser = - option (str ≫= readDateTime) $ - long "end-date" - ⊕ short 'e' - ⊕ metavar "ED" - ⊕ help "End Date (ISO 8601)" - -timeDurationParser ∷ Parser Seconds -timeDurationParser = - option auto $ - long "end-date" - ⊕ short 'e' - ⊕ metavar "ED" - ⊕ help "Time window from start (in seconds)" - optionsParser ∷ Parser CLIOptions optionsParser = - CLIOptions - <$> streamNameParser + pure CLIOptions + ⊛ streamNameParser ⊛ limitParser ⊛ iteratorTypeParser - ⊛ optional startDateParser - ⊛ optional endDateParser - -- ⊛ optional timeDurationParser - ⊛ switch (long "raw" ⊕ help "Treat records as raw text") + ⊛ (Left <$> accessKeysParser <|> Right <$> accessKeysPathParser) parserInfo ∷ ParserInfo CLIOptions parserInfo = info (helper ⊛ optionsParser) $ fullDesc - ⊕ progDesc "Fetch `L` records from a Kinesis stream `SN`. Put your AWS keys in ~/.aws-keys" + ⊕ progDesc "Fetch `L` records from a Kinesis stream `SN`." ⊕ header "The Kinesis Consumer CLI" diff --git a/cli/CLI/Record.hs b/cli/CLI/Record.hs deleted file mode 100644 --- a/cli/CLI/Record.hs +++ /dev/null @@ -1,79 +0,0 @@ --- Copyright (c) 2013-2014 PivotCloud, Inc. --- --- Aws.Kinesis.Client.Consumer --- --- Please feel free to contact us at licensing@pivotmail.com with any --- contributions, additions, or other feedback; we would love to hear from --- you. --- --- Licensed under the Apache License, Version 2.0 (the "License"); you may --- not use this file except in compliance with the License. You may obtain a --- copy of the License at http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT --- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the --- License for the specific language governing permissions and limitations --- under the License. - -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RecordWildCards #-} -{-# LANGUAGE UnicodeSyntax #-} - --- | --- Module: CLI.Record --- Copyright: Copyright © 2013-2014 PivotCloud, Inc. --- License: Apache-2.0 --- Maintainer: Jon Sterling --- Stability: experimental --- - -module CLI.Record where - -import Data.Aeson -import Data.Aeson.Types -import Data.Time -import qualified Data.Text as T -import qualified Data.ByteString as BS -import Control.Applicative -import Control.Applicative.Unicode -import Data.Hourglass -import Prelude.Unicode - -data Record - = Record - { rTimestamp ∷ DateTime - , rMessage ∷ T.Text - } deriving (Show, Eq) - -instance FromJSON DateTime where - parseJSON = - withText "DateTime" $ - maybe (fail "Invalid DateTime") return - ∘ timeParse fmt - ∘ T.unpack - where - fmt = - [ Format_Year4 - , hyphen - , Format_Month2 - , hyphen - , Format_Day2 - , Format_Text 'T' - , Format_Hour - , colon - , Format_Minute - , colon - , Format_Second - ] - hyphen = Format_Text '-' - colon = Format_Text ':' - -instance FromJSON Record where - parseJSON = - withObject "Record" $ \xs → - Record <$> xs .: "time" ⊛ xs .: "message" - -instance Ord Record where - compare r1 r2 = compare (rTimestamp r1) (rTimestamp r2) -