wolf 0.3.34 → 0.3.35
raw patch · 8 files changed
+97/−25 lines, 8 filesdep +amazonka-coredep +amazonka-s3dep +conduit-combinators
Dependencies added: amazonka-core, amazonka-s3, conduit-combinators, filemanip
Files
- Shakefile.hs +3/−2
- src/Network/AWS/Wolf/Act.hs +10/−15
- src/Network/AWS/Wolf/Ctx.hs +1/−1
- src/Network/AWS/Wolf/File.hs +19/−2
- src/Network/AWS/Wolf/Prelude.hs +1/−1
- src/Network/AWS/Wolf/S3.hs +54/−0
- src/Network/AWS/Wolf/SWF.hs +3/−3
- wolf.cabal +6/−1
Shakefile.hs view
@@ -37,8 +37,9 @@ -- | sanity --- fake "." pats "sanity" $ const $- need [ "build-error", "lint", "weed" ]+ fake "." pats "sanity" $ const $ do+ need [ "build-error" ]+ need [ "lint", "weed" ] -- | Default things to run. --
src/Network/AWS/Wolf/Act.hs view
@@ -14,38 +14,33 @@ import Network.AWS.Wolf.Ctx import Network.AWS.Wolf.File import Network.AWS.Wolf.Prelude+import Network.AWS.Wolf.S3 import Network.AWS.Wolf.SWF import Network.AWS.Wolf.Types import System.Directory import System.Exit import System.Process --- | S3 copy call.----cp :: MonadIO m => FilePath -> FilePath -> m ()-cp f t = liftIO $ callProcess "aws" [ "s3", "cp", "--quiet", "--recursive", f, t ]---- | Key to download and upload objects from.----key :: MonadAmazonStore c m => m FilePath-key = do- b <- view cBucket <$> view ccConf- p <- view ascPrefix- pure $ "s3:/" -/- textToString b -/- textToString p- -- | Download artifacts to the store input directory. -- download :: MonadAmazonStore c m => FilePath -> m () download dir = do traceInfo "download" [ "dir" .= dir ]- flip cp dir =<< key+ ks <- listArtifacts+ forM_ ks $ \k -> do+ traceInfo "download" [ "dir" .= dir, "key" .= k ]+ touchDirectory (dir </> k)+ getArtifact (dir </> k) k -- | Upload artifacts from the store output directory. -- upload :: MonadAmazonStore c m => FilePath -> m () upload dir = do traceInfo "upload" [ "dir" .= dir ]- cp dir =<< key+ ks <- findRelativeFiles dir+ forM_ ks $ \k -> do+ traceInfo "upload" [ "dir" .= dir, "key" .= k ]+ putArtifact (dir </> k) k -- | callCommand wrapper that maybe returns an exception. --
src/Network/AWS/Wolf/Ctx.hs view
@@ -43,7 +43,7 @@ -- topSomeExceptionCatch :: MonadStatsCtx c m => SomeException -> m a topSomeExceptionCatch ex = do- statsIncrement "exception" [ "reason" =. textFromString (displayException ex) ]+ statsIncrement "wolf.exception" [ "reason" =. textFromString (displayException ex) ] throwIO ex -- | Run bottom TransT.
src/Network/AWS/Wolf/File.hs view
@@ -4,7 +4,9 @@ -- | Files, directories, and encoding / decoding file functions. -- module Network.AWS.Wolf.File- ( dataDirectory+ ( findRelativeFiles+ , touchDirectory+ , dataDirectory , storeDirectory , inputDirectory , outputDirectory@@ -20,9 +22,24 @@ import qualified Data.ByteString.Lazy as LBS import Data.Time import Data.Yaml hiding (encode)-import Network.AWS.Wolf.Prelude+import Network.AWS.Wolf.Prelude hiding (find) import System.Directory+import System.FilePath+import System.FilePath.Find import System.IO hiding (readFile, writeFile)++-- | Recursively find all files under a directory.+--+findRelativeFiles :: MonadIO m => FilePath -> m [FilePath]+findRelativeFiles dir = do+ files <- liftIO $ find always (fileType ==? RegularFile) dir+ pure $ makeRelative dir <$> files++-- | Create parent directory of file if missing.+--+touchDirectory :: MonadIO m => FilePath -> m ()+touchDirectory =+ liftIO . createDirectoryIfMissing True . takeDirectory -- | Determine path to data directory and create it. --
src/Network/AWS/Wolf/Prelude.hs view
@@ -10,7 +10,7 @@ ) where import Control.Concurrent.Async.Lifted-import Preamble as Exports+import Preamble as Exports hiding (stripPrefix) -- | Run a list of actions concurrently. --
+ src/Network/AWS/Wolf/S3.hs view
@@ -0,0 +1,54 @@+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE NoImplicitPrelude #-}++-- | S3 Calls.+--+module Network.AWS.Wolf.S3+ ( listArtifacts+ , getArtifact+ , putArtifact+ ) where++import Control.Monad.Trans.AWS+import Data.Conduit+import Data.Conduit.Combinators+import Network.AWS.Data+import Network.AWS.Data.Body+import Network.AWS.S3 hiding (cBucket)+import Network.AWS.Wolf.Ctx+import Network.AWS.Wolf.Prelude hiding (concatMap)+import Network.AWS.Wolf.Types+import System.FilePath++-- | List keys in bucket with prefix.+--+listArtifacts :: MonadAmazonStore c m => m [FilePath]+listArtifacts = do+ b <- view cBucket <$> view ccConf+ p <- view ascPrefix+ runResourceT $ runAmazonCtx $+ paginate (set loPrefix (pure p) $ listObjects (BucketName b))+ =$= concatMap ((makeRelative (textToString p) . textToString . toText . view oKey <$>) . view lorsContents)+ $$ sinkList++-- | Get object in bucket with key.+--+getArtifact :: MonadAmazonStore c m => FilePath -> FilePath -> m ()+getArtifact file key = do+ b <- view cBucket <$> view ccConf+ p <- view ascPrefix+ runResourceT $ runAmazonCtx $ do+ gors <- send $ getObject (BucketName b) (ObjectKey (p -/- textFromString key))+ sinkBody (gors ^. gorsBody) (sinkFileBS file)++-- | Put object in bucket with key.+--+putArtifact :: MonadAmazonStore c m => FilePath -> FilePath -> m ()+putArtifact file key = do+ b <- view cBucket <$> view ccConf+ p <- view ascPrefix+ runResourceT $ runAmazonCtx $ do+ (sha, len) <- sourceFileBS file $$ getZipSink $ (,) <$> ZipSink sinkSHA256 <*> ZipSink lengthE+ void $ send $ putObject (BucketName b) (ObjectKey (p -/- textFromString key)) $+ Hashed $ HashedStream sha len $ sourceFileBS file+
src/Network/AWS/Wolf/SWF.hs view
@@ -18,7 +18,7 @@ import Control.Monad.Trans.AWS import Data.Conduit-import Data.Conduit.List hiding (concatMap, map)+import Data.Conduit.Combinators hiding (concatMap) import Network.AWS.SWF import Network.AWS.Wolf.Ctx import Network.AWS.Wolf.Prelude@@ -45,9 +45,9 @@ d <- view cDomain <$> view ccConf tl <- taskList <$> view awcQueue runResourceT $ runAmazonCtx $ do- pfdtrs <- paginate (pollForDecisionTask d tl) $$ consume+ pfdtrs <- paginate (pollForDecisionTask d tl) $$ sinkList pure- ( join $ headMay $ map (view pfdtrsTaskToken) pfdtrs+ ( join $ headMay $ view pfdtrsTaskToken <$> pfdtrs , reverse $ concatMap (view pfdtrsEvents) pfdtrs )
wolf.cabal view
@@ -1,5 +1,5 @@ name: wolf-version: 0.3.34+version: 0.3.35 cabal-version: >=1.22 build-type: Simple license: MIT@@ -23,11 +23,15 @@ build-depends: aeson >=0.11.3.0, amazonka >=1.4.3,+ amazonka-core >=1.4.3,+ amazonka-s3 >=1.4.3, amazonka-swf >=1.4.3, base >=4.8 && <5, bytestring >=0.10.6.0, conduit >=1.2.10,+ conduit-combinators >=1.0.8.3, directory >=1.2.2.0,+ filemanip >=0.3.6.3, filepath >=1.4.0.0, http-types >=0.9.1, lifted-async >=0.9.1.1,@@ -46,6 +50,7 @@ Network.AWS.Wolf.Decide Network.AWS.Wolf.File Network.AWS.Wolf.Prelude+ Network.AWS.Wolf.S3 Network.AWS.Wolf.SWF Network.AWS.Wolf.Types Network.AWS.Wolf.Types.Ctx