summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormarkfine <>2017-11-28 00:53:00 (GMT)
committerhdiff <hdiff@hdiff.luite.com>2017-11-28 00:53:00 (GMT)
commit95eaa72c835ac338e1d682578dcd4ba8f3c6000b (patch)
tree4c402e1cdccd088da4433ad0823be87250bf727d
parent6c5b211897922969fbc9d407398ca86525773446 (diff)
version 0.3.370.3.37
-rw-r--r--src/Network/AWS/Wolf/Act.hs25
-rw-r--r--src/Network/AWS/Wolf/File.hs21
-rw-r--r--src/Network/AWS/Wolf/Prelude.hs2
-rw-r--r--src/Network/AWS/Wolf/S3.hs54
-rw-r--r--src/Network/AWS/Wolf/SWF.hs4
-rw-r--r--wolf.cabal7
6 files changed, 21 insertions, 92 deletions
diff --git a/src/Network/AWS/Wolf/Act.hs b/src/Network/AWS/Wolf/Act.hs
index c41255a..5ba3652 100644
--- a/src/Network/AWS/Wolf/Act.hs
+++ b/src/Network/AWS/Wolf/Act.hs
@@ -14,33 +14,38 @@ import Data.Time
import Network.AWS.Wolf.Ctx
import Network.AWS.Wolf.File
import Network.AWS.Wolf.Prelude
-import Network.AWS.Wolf.S3
import Network.AWS.Wolf.SWF
import Network.AWS.Wolf.Types
import System.Directory
import System.Exit
import System.Process
+-- | S3 copy call.
+--
+cp :: MonadIO m => FilePath -> FilePath -> m ()
+cp f t = liftIO $ callProcess "aws" [ "s3", "cp", "--quiet", "--recursive", f, t ]
+
+-- | Key to download and upload objects from.
+--
+key :: MonadAmazonStore c m => m FilePath
+key = do
+ b <- view cBucket <$> view ccConf
+ p <- view ascPrefix
+ pure $ "s3:/" -/- textToString b -/- textToString p
+
-- | Download artifacts to the store input directory.
--
download :: MonadAmazonStore c m => FilePath -> m ()
download dir = do
traceInfo "download" [ "dir" .= dir ]
- ks <- listArtifacts
- forM_ ks $ \k -> do
- traceInfo "download" [ "dir" .= dir, "key" .= k ]
- touchDirectory (dir </> k)
- getArtifact (dir </> k) k
+ flip cp dir =<< key
-- | Upload artifacts from the store output directory.
--
upload :: MonadAmazonStore c m => FilePath -> m ()
upload dir = do
traceInfo "upload" [ "dir" .= dir ]
- ks <- findRelativeFiles dir
- forM_ ks $ \k -> do
- traceInfo "upload" [ "dir" .= dir, "key" .= k ]
- putArtifact (dir </> k) k
+ cp dir =<< key
-- | callCommand wrapper that maybe returns an exception.
--
diff --git a/src/Network/AWS/Wolf/File.hs b/src/Network/AWS/Wolf/File.hs
index 2d6f9cf..5396662 100644
--- a/src/Network/AWS/Wolf/File.hs
+++ b/src/Network/AWS/Wolf/File.hs
@@ -4,9 +4,7 @@
-- | Files, directories, and encoding / decoding file functions.
--
module Network.AWS.Wolf.File
- ( findRelativeFiles
- , touchDirectory
- , dataDirectory
+ ( dataDirectory
, storeDirectory
, inputDirectory
, outputDirectory
@@ -22,25 +20,10 @@ import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as LBS
import Data.Time
import Data.Yaml hiding (encode)
-import Network.AWS.Wolf.Prelude hiding (find)
+import Network.AWS.Wolf.Prelude
import System.Directory
-import System.FilePath
-import System.FilePath.Find
import System.IO hiding (readFile, writeFile)
--- | Recursively find all files under a directory.
---
-findRelativeFiles :: MonadIO m => FilePath -> m [FilePath]
-findRelativeFiles dir = do
- files <- liftIO $ find always (fileType ==? RegularFile) dir
- pure $ makeRelative dir <$> files
-
--- | Create parent directory of file if missing.
---
-touchDirectory :: MonadIO m => FilePath -> m ()
-touchDirectory =
- liftIO . createDirectoryIfMissing True . takeDirectory
-
-- | Determine path to data directory and create it.
--
dataDirectory :: MonadIO m => FilePath -> m FilePath
diff --git a/src/Network/AWS/Wolf/Prelude.hs b/src/Network/AWS/Wolf/Prelude.hs
index db81bde..b1df6a1 100644
--- a/src/Network/AWS/Wolf/Prelude.hs
+++ b/src/Network/AWS/Wolf/Prelude.hs
@@ -10,7 +10,7 @@ module Network.AWS.Wolf.Prelude
) where
import Control.Concurrent.Async.Lifted
-import Preamble as Exports hiding (stripPrefix)
+import Preamble as Exports
-- | Run a list of actions concurrently.
--
diff --git a/src/Network/AWS/Wolf/S3.hs b/src/Network/AWS/Wolf/S3.hs
deleted file mode 100644
index 2bab00c..0000000
--- a/src/Network/AWS/Wolf/S3.hs
+++ /dev/null
@@ -1,54 +0,0 @@
-{-# LANGUAGE FlexibleContexts #-}
-{-# LANGUAGE NoImplicitPrelude #-}
-
--- | S3 Calls.
---
-module Network.AWS.Wolf.S3
- ( listArtifacts
- , getArtifact
- , putArtifact
- ) where
-
-import Control.Monad.Trans.AWS
-import Data.Conduit
-import Data.Conduit.Combinators
-import Network.AWS.Data
-import Network.AWS.Data.Body
-import Network.AWS.S3 hiding (cBucket)
-import Network.AWS.Wolf.Ctx
-import Network.AWS.Wolf.Prelude hiding (concatMap)
-import Network.AWS.Wolf.Types
-import System.FilePath
-
--- | List keys in bucket with prefix.
---
-listArtifacts :: MonadAmazonStore c m => m [FilePath]
-listArtifacts = do
- b <- view cBucket <$> view ccConf
- p <- view ascPrefix
- runResourceT $ runAmazonCtx $
- paginate (set loPrefix (pure p) $ listObjects (BucketName b))
- =$= concatMap ((makeRelative (textToString p) . textToString . toText . view oKey <$>) . view lorsContents)
- $$ sinkList
-
--- | Get object in bucket with key.
---
-getArtifact :: MonadAmazonStore c m => FilePath -> FilePath -> m ()
-getArtifact file key = do
- b <- view cBucket <$> view ccConf
- p <- view ascPrefix
- runResourceT $ runAmazonCtx $ do
- gors <- send $ getObject (BucketName b) (ObjectKey (p -/- textFromString key))
- sinkBody (gors ^. gorsBody) (sinkFileBS file)
-
--- | Put object in bucket with key.
---
-putArtifact :: MonadAmazonStore c m => FilePath -> FilePath -> m ()
-putArtifact file key = do
- b <- view cBucket <$> view ccConf
- p <- view ascPrefix
- runResourceT $ runAmazonCtx $ do
- (sha, len) <- sourceFileBS file $$ getZipSink $ (,) <$> ZipSink sinkSHA256 <*> ZipSink lengthE
- void $ send $ putObject (BucketName b) (ObjectKey (p -/- textFromString key)) $
- Hashed $ HashedStream sha len $ sourceFileBS file
-
diff --git a/src/Network/AWS/Wolf/SWF.hs b/src/Network/AWS/Wolf/SWF.hs
index 9b16122..2050a29 100644
--- a/src/Network/AWS/Wolf/SWF.hs
+++ b/src/Network/AWS/Wolf/SWF.hs
@@ -18,7 +18,7 @@ module Network.AWS.Wolf.SWF
import Control.Monad.Trans.AWS
import Data.Conduit
-import Data.Conduit.Combinators hiding (concatMap)
+import Data.Conduit.List hiding (concatMap)
import Network.AWS.SWF
import Network.AWS.Wolf.Ctx
import Network.AWS.Wolf.Prelude
@@ -45,7 +45,7 @@ pollDecision = do
d <- view cDomain <$> view ccConf
tl <- taskList <$> view awcQueue
runResourceT $ runAmazonCtx $ do
- pfdtrs <- paginate (pollForDecisionTask d tl) $$ sinkList
+ pfdtrs <- paginate (pollForDecisionTask d tl) $$ consume
pure
( join $ headMay $ view pfdtrsTaskToken <$> pfdtrs
, reverse $ concatMap (view pfdtrsEvents) pfdtrs
diff --git a/wolf.cabal b/wolf.cabal
index c7c35a3..1bd8c5e 100644
--- a/wolf.cabal
+++ b/wolf.cabal
@@ -1,5 +1,5 @@
name: wolf
-version: 0.3.36
+version: 0.3.37
cabal-version: >=1.22
build-type: Simple
license: MIT
@@ -23,15 +23,11 @@ library
build-depends:
aeson >=0.11.3.0,
amazonka >=1.4.3,
- amazonka-core >=1.4.3,
- amazonka-s3 >=1.4.3,
amazonka-swf >=1.4.3,
base >=4.8 && <5,
bytestring >=0.10.6.0,
conduit >=1.2.10,
- conduit-combinators >=1.0.8.3,
directory >=1.2.2.0,
- filemanip >=0.3.6.3,
filepath >=1.4.0.0,
http-types >=0.9.1,
lifted-async >=0.9.1.1,
@@ -50,7 +46,6 @@ library
Network.AWS.Wolf.Decide
Network.AWS.Wolf.File
Network.AWS.Wolf.Prelude
- Network.AWS.Wolf.S3
Network.AWS.Wolf.SWF
Network.AWS.Wolf.Types
Network.AWS.Wolf.Types.Ctx