packages feed

amazonka-s3-streaming 0.2.0.5 → 1.0.0.0

raw patch · 3 files changed

+174/−216 lines, 3 filesdep +asyncdep −deepseqdep −foundationdep −lifted-asyncdep ~amazonkadep ~amazonka-coredep ~amazonka-s3PVP ok

version bump matches the API change (PVP)

Dependencies added: async

Dependencies removed: deepseq, foundation, lifted-async, mmap, resourcet, semigroups, transformers

Dependency ranges changed: amazonka, amazonka-core, amazonka-s3, bytestring, conduit, exceptions

API changes (from Hackage documentation)

+ Network.AWS.S3.StreamingUpload: type ChunkSize = Int
+ Network.AWS.S3.StreamingUpload: type NumThreads = Int
- Network.AWS.S3.StreamingUpload: abortAllUploads :: (MonadAWS m) => BucketName -> m ()
+ Network.AWS.S3.StreamingUpload: abortAllUploads :: MonadAWS m => BucketName -> m ()
- Network.AWS.S3.StreamingUpload: concurrentUpload :: (AWSConstraint r m, MonadAWS m, MonadBaseControl IO m) => Maybe ChunkSize -> Maybe NumThreads -> UploadLocation -> CreateMultipartUpload -> m CompleteMultipartUploadResponse
+ Network.AWS.S3.StreamingUpload: concurrentUpload :: MonadAWS m => Maybe ChunkSize -> Maybe NumThreads -> UploadLocation -> CreateMultipartUpload -> m CompleteMultipartUploadResponse
- Network.AWS.S3.StreamingUpload: streamUpload :: (MonadResource m, AWSConstraint r m, MonadAWS m) => Maybe ChunkSize -> CreateMultipartUpload -> Sink ByteString m CompleteMultipartUploadResponse
+ Network.AWS.S3.StreamingUpload: streamUpload :: (MonadUnliftIO m, MonadAWS m) => Maybe ChunkSize -> CreateMultipartUpload -> ConduitT ByteString Void m (Either (AbortMultipartUploadResponse, SomeException) CompleteMultipartUploadResponse)

Files

Main.hs view
@@ -1,23 +1,23 @@-{-# LANGUAGE CPP               #-}+{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} module Main where  -import           Data.Conduit                         (($$))-import           Data.Conduit.Binary                  (sourceHandle)-import           Data.Text                            (pack)+import Data.Conduit        ( runConduit, (.|) )+import Data.Conduit.Binary ( sourceHandle )+import Data.Text           ( pack ) -import           Network.AWS-import           Network.AWS.Data.Text                (fromText)-import           Network.AWS.S3.CreateMultipartUpload-import           Network.AWS.S3.StreamingUpload+import Network.AWS+import Network.AWS.Data.Text                ( fromText )+import Network.AWS.S3.CreateMultipartUpload+import Network.AWS.S3.StreamingUpload -import           System.Environment-import           System.IO                            (BufferMode(BlockBuffering),-                                                       hSetBuffering, stdin)+import Control.Monad.IO.Class ( liftIO )+import System.Environment+import System.IO              ( BufferMode(BlockBuffering), hSetBuffering, stdin )  #if !MIN_VERSION_base(4,8,0)-import           Control.Applicative                  (pure, (<$>), (<*>))+import Control.Applicative ( pure, (<$>), (<*>) ) #endif  main :: IO ()@@ -31,17 +31,14 @@                  <*> fromText (pack key)       of         Right (creds,_reg,buck,ky) -> do-#if !MIN_VERSION_amazonka(1,4,4)-          env <- newEnv _reg creds-#else           env <- newEnv creds-#endif           hSetBuffering stdin (BlockBuffering Nothing)           res <- runResourceT . runAWS env $ case file of-                  -- Stream data from stdin-                  "-" -> sourceHandle stdin $$ streamUpload Nothing (createMultipartUpload buck ky)-                  -- Read from a file-                  _   -> concurrentUpload Nothing Nothing (FP file) $ createMultipartUpload buck ky+                  "-" -> runConduit (sourceHandle stdin .| streamUpload Nothing (createMultipartUpload buck ky))+                          >>= liftIO . either print print+                  _   -> concurrentUpload Nothing Nothing (FP file) (createMultipartUpload buck ky)+                          >>= liftIO . print+           print res         Left err -> print err >> usage     ("abort":region:profile:credfile:bucket:_) ->@@ -50,11 +47,7 @@                 <*> fromText (pack bucket)       of         Right (creds,_reg,buck) -> do-#if !MIN_VERSION_amazonka(1,4,4)-          env <- newEnv _reg creds-#else           env <- newEnv creds-#endif           res <- runResourceT . runAWS env . abortAllUploads $ buck           print res         Left err -> print err >> usage
amazonka-s3-streaming.cabal view
@@ -1,5 +1,5 @@ name:                amazonka-s3-streaming-version:             0.2.0.5+version:             1.0.0.0 synopsis:            Provides conduits to upload data to S3 using the Multipart API description:         Provides a conduit based streaming interface and a concurrent interface to                      uploading data to S3 using the Multipart API. Also provides method to upload@@ -9,50 +9,35 @@ license-file:        LICENSE author:              Alex Mason maintainer:          Alex.Mason@data61.csiro.au-copyright:           Copyright (c) 2016 Commonwealth Scientific and Industrial Research Organisation (CSIRO)+copyright:           Alex Mason, Copyright (c) 2016 Commonwealth Scientific and Industrial Research Organisation (CSIRO) category:            Network, AWS, Cloud, Distributed Computing build-type:          Simple extra-source-files:  README.md, Changelog.md cabal-version:       >=1.10-tested-with:         GHC == 7.8.4, GHC == 7.10.2, GHC == 8.0.2, GHC == 8.2.2, GHC == 8.4.*+tested-with:         GHC == 8.0.* || == 8.2.2 || == 8.4.* || == 8.6.*  library   hs-source-dirs:      src   exposed-modules:     Network.AWS.S3.StreamingUpload   default-language:    Haskell2010   build-depends:       base >= 4.6 && < 5-                       , amazonka         >= 1.3        && < 1.6-                       , amazonka-core    >= 1.3        && < 1.6-                       , amazonka-s3      >= 1.3        && < 1.6-                       , resourcet        >= 1.1.7.4    && < 1.2-                       , conduit          >= 1.2.6.6    && < 1.3+                       , amazonka         >= 1.6        && < 1.7+                       , amazonka-core    >= 1.6        && < 1.7+                       , amazonka-s3      >= 1.6        && < 1.7+                       , conduit          >= 1.3        && < 1.4                        , bytestring       >= 0.10.8.0   && < 0.11                        , mmorph           >= 1.0.6      && < 1.2                        , lens             >= 4.13       && < 5.0                        , mtl              >= 2.2.1      && < 2.3-                       , exceptions       >= 0.8.2.1    && < 0.9+                       , exceptions       >= 0.8.2.1    && < 0.11                        , dlist            >= 0.8        && < 0.9-                       , lifted-async     >= 0.9        && < 0.10-                       , mmap             >= 0.5        && < 0.6-                       , deepseq          >= 1.2        && < 1.5+                       , async            >= 2          && < 2.3                        , http-client      >= 0.4        && < 0.6 -  if impl(ghc < 7.11)-    build-depends:-      semigroups >= 0.12-      , transformers >=0.4-  if impl(ghc == 7.8.*)-    build-depends:-      foundation == 0.0.15-  -  -- -- ini 0.3.4 fails to build because it doesn't have <*-  -- if impl(ghc < 7.9)-  --   build-depends:-  --     ini >= 0.3.5-  -Flag s3upload-exe+flag s3upload-exe   Description: Whether to build the s3upload executable for uploading files using this library.-  Default: False+  default: False+  manual: True  source-repository head   type:     git@@ -62,7 +47,10 @@   main-is: Main.hs   ghc-options: -threaded -rtsopts -with-rtsopts=-N   default-language: Haskell2010-  buildable: False+  if flag(s3upload-exe)+     buildable: True+  else+     buildable: False   build-depends: base                  , amazonka                  , amazonka-core@@ -70,7 +58,5 @@                  , amazonka-s3-streaming                  , conduit-extra                  , conduit-                 , bytestring                  , text-  if flag(s3upload-exe)-    buildable: True+
src/Network/AWS/S3/StreamingUpload.hs view
@@ -1,80 +1,66 @@-{-# LANGUAGE BangPatterns          #-}-{-# LANGUAGE CPP                   #-}-{-# LANGUAGE ConstraintKinds       #-}-{-# LANGUAGE FlexibleContexts      #-}-{-# LANGUAGE MultiParamTypeClasses #-}-{-# LANGUAGE OverloadedStrings     #-}-{-# LANGUAGE ParallelListComp      #-}-{-# LANGUAGE PatternGuards         #-}-{-# LANGUAGE ScopedTypeVariables   #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE ParallelListComp #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TupleSections #-}  module Network.AWS.S3.StreamingUpload-(-    streamUpload-    , UploadLocation(..)-    , concurrentUpload-    , abortAllUploads-    , module Network.AWS.S3.CreateMultipartUpload-    , module Network.AWS.S3.CompleteMultipartUpload-    , minimumChunkSize-) where+  ( streamUpload+  , ChunkSize+  , minimumChunkSize+  , NumThreads+  , concurrentUpload+  , UploadLocation(..)+  , abortAllUploads+  , module Network.AWS.S3.CreateMultipartUpload+  , module Network.AWS.S3.CompleteMultipartUpload+  ) where -import           Network.AWS                            (HasEnv(..),-                                                         LogLevel(..), MonadAWS,-                                                         getFileSize,-                                                         hashedBody, send,-                                                         toBody)+import Network.AWS+       ( AWS, HasEnv(..), LogLevel(..), MonadAWS, getFileSize, hashedBody, hashedFileRange,+       liftAWS, runAWS, runResourceT, send, toBody ) -import           Control.Monad.Trans.AWS                (AWSConstraint)-import           Network.AWS.Data.Crypto                (Digest, SHA256,-                                                         hashFinalize, hashInit,-                                                         hashUpdate)+import Network.AWS.Data.Crypto ( Digest, SHA256, hashFinalize, hashInit, hashUpdate ) -import           Network.AWS.S3.AbortMultipartUpload-import           Network.AWS.S3.CompleteMultipartUpload-import           Network.AWS.S3.CreateMultipartUpload-import           Network.AWS.S3.ListMultipartUploads-import           Network.AWS.S3.Types                   (BucketName, cmuParts, completedMultipartUpload,-                                                         completedPart, muKey,-                                                         muUploadId)-import           Network.AWS.S3.UploadPart+import Network.AWS.S3.AbortMultipartUpload+import Network.AWS.S3.CompleteMultipartUpload+import Network.AWS.S3.CreateMultipartUpload+import Network.AWS.S3.ListMultipartUploads+import Network.AWS.S3.Types+       ( BucketName, cmuParts, completedMultipartUpload, completedPart, muKey, muUploadId )+import Network.AWS.S3.UploadPart -import           Control.Applicative-import           Control.Category                       ((>>>))-import           Control.Monad                          (forM_, when, (>=>))-import           Control.Monad.IO.Class                 (MonadIO, liftIO)-import           Control.Monad.Morph                    (lift)-import           Control.Monad.Reader.Class             (local)-import           Control.Monad.Trans.Resource           (MonadBaseControl,-                                                         MonadResource)+import Control.Applicative+import Control.Category           ( (>>>) )+import Control.Monad              ( forM_, when, (>=>) )+import Control.Monad.IO.Class     ( MonadIO, liftIO )+import Control.Monad.Morph        ( lift )+import Control.Monad.Reader.Class ( local ) -import           Data.Conduit                           (Sink, await)-import           Data.Conduit.List                      (sourceList)+import Conduit           ( MonadUnliftIO(..) )+import Data.Conduit      ( ConduitT, Void, await, catchC )+import Data.Conduit.List ( sourceList ) -import           Data.ByteString                        (ByteString)-import qualified Data.ByteString                        as BS-import           Data.ByteString.Builder                (stringUtf8)-import           System.IO.MMap                         (mmapFileByteString)+import           Data.ByteString         ( ByteString )+import qualified Data.ByteString         as BS+import           Data.ByteString.Builder ( stringUtf8 ) -import           Control.DeepSeq                        (rnf)-import qualified Data.DList                             as D-import           Data.List                              (unfoldr)-import           Data.List.NonEmpty                     (nonEmpty)+import qualified Data.DList         as D+import           Data.List          ( unfoldr )+import           Data.List.NonEmpty ( nonEmpty ) -import           Control.Lens                           (set, view)-import           Control.Lens.Operators-import           Control.Monad.Catch                    (onException)+import Control.Lens           ( set, view )+import Control.Lens.Operators -import           Text.Printf                            (printf)+import Text.Printf ( printf ) -import           Control.Concurrent                     (newQSem, signalQSem,-                                                         waitQSem)-import           Control.Concurrent.Async.Lifted        (forConcurrently)-import           System.Mem                             (performGC)+import Control.Concurrent       ( newQSem, signalQSem, waitQSem )+import Control.Concurrent.Async ( forConcurrently )+import Control.Exception.Base   ( SomeException, bracket_ )+import Control.Monad.Catch      ( onException )+import System.Mem               ( performGC ) -import           Network.HTTP.Client                    (defaultManagerSettings,-                                                         managerConnCount,-                                                         newManager)+import Network.HTTP.Client ( defaultManagerSettings, managerConnCount, newManager )  type ChunkSize = Int type NumThreads = Int@@ -86,11 +72,11 @@  {- | Given a 'CreateMultipartUpload', creates a 'Sink' which will sequentially-upload the data streamed in in chunks of at least 'chunkSize' and return-the 'CompleteMultipartUploadResponse'.+upload the data streamed in in chunks of at least 'minimumChunkSize' and return either+the 'CompleteMultipartUploadResponse', or if an exception is thrown,+`AbortMultipartUploadResponse` and the exception as `SomeException`. If aborting+the upload also fails then the exception caused by the call to abort will be thrown. -If uploading of any parts fails, an attempt is made to abort the Multipart-upload, but it is likely that if an upload fails this abort may also fail. 'Network.AWS.S3.ListMultipartUploads' can be used to list any pending uploads - it is important to abort multipart uploads because you will be charged for storage of the parts until it is completed or aborted.@@ -98,51 +84,45 @@  May throw 'Network.AWS.Error' -}-streamUpload :: (MonadResource m, AWSConstraint r m, MonadAWS m)+streamUpload :: (MonadUnliftIO m, MonadAWS m)              => Maybe ChunkSize -- ^ Optional chunk size              -> CreateMultipartUpload -- ^ Upload location-             -> Sink ByteString m CompleteMultipartUploadResponse-streamUpload mcs cmu = do-  logger <- lift $ view envLogger+             -> ConduitT ByteString Void m (Either (AbortMultipartUploadResponse, SomeException) CompleteMultipartUploadResponse)+streamUpload mChunkSize multiPartUploadDesc = do+  logger <- lift $ liftAWS $ view envLogger   let logStr :: MonadIO m => String -> m ()       logStr = liftIO . logger Debug . stringUtf8-      chunkSize = maybe minimumChunkSize (max minimumChunkSize) mcs+      chunkSize = maybe minimumChunkSize (max minimumChunkSize) mChunkSize -  cmur <- lift $ send cmu-  when (cmur ^. cmursResponseStatus /= 200) $+  multiPartUpload <- lift $ send multiPartUploadDesc+  when (multiPartUpload ^. cmursResponseStatus /= 200) $     fail "Failed to create upload"    logStr "\n**** Created upload\n" -  let Just upId = cmur ^. cmursUploadId-      bucket    = cmu  ^. cmuBucket-      key       = cmu  ^. cmuKey-      -- go :: Text -> Builder -> Int -> Int -> Sink ByteString m ()-      go !bss !bufsize !ctx !partnum !completed = Data.Conduit.await >>= \mbs -> case mbs of-        Just bs | l <- BS.length bs-                , bufsize + l <= chunkSize ->-                    go (D.snoc bss bs) (bufsize + l) (hashUpdate ctx bs) partnum completed--                | otherwise -> do-                    rs <- lift $ partUploader partnum (bufsize + BS.length bs)-                                              (hashFinalize $ hashUpdate ctx bs)-                                              (D.snoc bss bs)+  let Just upId = multiPartUpload ^. cmursUploadId+      bucket    = multiPartUploadDesc  ^. cmuBucket+      key       = multiPartUploadDesc  ^. cmuKey+      -- go :: DList ByteString -> Int -> Context SHA256 -> Int -> DList (Maybe CompletedPart) -> Sink ByteString m ()+      go !bss !bufsize !ctx !partnum !completed = await >>= \case+        Just bs+            | l <- BS.length bs, bufsize + l <= chunkSize+                -> go (D.snoc bss bs) (bufsize + l) (hashUpdate ctx bs) partnum completed+            | otherwise -> do+                rs <- lift $ performUpload partnum (bufsize + BS.length bs)+                                            (hashFinalize $ hashUpdate ctx bs)+                                            (D.snoc bss bs) -                    logStr $ printf "\n**** Uploaded part %d size %d\n" partnum bufsize-                    let part = completedPart partnum <$> (rs ^. uprsETag)-#if MIN_VERSION_amazonka_s3(1,4,1)-                        !_ = rnf part-#endif-                    liftIO performGC-                    go empty 0 hashInit (partnum+1) . D.snoc completed $! part+                logStr $ printf "\n**** Uploaded part %d size %d\n" partnum bufsize+                let !part = completedPart partnum <$> (rs ^. uprsETag)+                liftIO performGC+                go empty 0 hashInit (partnum+1) . D.snoc completed $! part          Nothing -> lift $ do             prts <- if bufsize > 0                 then do-                    rs <- partUploader partnum bufsize (hashFinalize ctx) bss-+                    rs <- performUpload partnum bufsize (hashFinalize ctx) bss                     logStr $ printf "\n**** Uploaded (final) part %d size %d\n" partnum bufsize-                     let allParts = D.toList $ D.snoc completed $ completedPart partnum <$> (rs ^. uprsETag)                     pure $ nonEmpty =<< sequence allParts                 else do@@ -153,8 +133,8 @@                     & cMultipartUpload ?~ set cmuParts prts completedMultipartUpload  -      partUploader :: MonadAWS m => Int -> Int -> Digest SHA256 -> D.DList ByteString -> m UploadPartResponse-      partUploader pnum size digest =+      performUpload :: MonadAWS m => Int -> Int -> Digest SHA256 -> D.DList ByteString -> m UploadPartResponse+      performUpload pnum size digest =         D.toList         >>> sourceList         >>> hashedBody digest (fromIntegral size)@@ -168,8 +148,9 @@         when (upr ^. uprsResponseStatus /= 200) $ fail "Failed to upload piece"         return upr -  go D.empty 0 hashInit 1 D.empty `onException` lift (send (abortMultipartUpload bucket key upId))-      -- Whatever happens, we abort the upload and rethrow+  (Right <$> go D.empty 0 hashInit 1 D.empty) `catchC` \(except :: SomeException) ->+    Left . (,except) <$> lift (send (abortMultipartUpload bucket key upId))+      -- Whatever happens, we abort the upload and return the exception   -- | Specifies whether to upload a file or 'ByteString@@ -177,10 +158,6 @@     = FP FilePath -- ^ A file to be uploaded     | BS ByteString -- ^ A strict 'ByteString' - --  IO (Int -> IO (Maybe ByteString)) (Either (IO ()) (IO ()))-        -- part number as input, may be called many times until Nothing is returned,-        -- and a function to close either this part or all parts- {-| Allows a file or 'ByteString' to be uploaded concurrently, using the async library.  The chunk size may optionally be specified, but will be at least@@ -196,78 +173,80 @@ multipart upload on any error, but this may also fail if, for example, the network connection has been broken. See `abortAllUploads` for a crude cleanup method. -}-concurrentUpload :: (AWSConstraint r m, MonadAWS m, MonadBaseControl IO m)+concurrentUpload :: (MonadAWS m)                  => Maybe ChunkSize -- ^ Optional chunk size                  -> Maybe NumThreads -- ^ Optional number of threads to upload with                  -> UploadLocation -- ^ Whether to upload a file on disk or a `ByteString` that's already in memory.                  -> CreateMultipartUpload -- ^ Description of where to upload.                  -> m CompleteMultipartUploadResponse-concurrentUpload mcs mnt ud cmu = do-    cmur <- send cmu-    when (cmur ^. cmursResponseStatus /= 200) $-        fail "Failed to create upload"-    logger <- view envLogger-    let logStr :: MonadIO m => String -> m ()-        logStr = liftIO . logger Info . stringUtf8-    let Just upId = cmur ^. cmursUploadId-        bucket    = cmu  ^. cmuBucket-        key       = cmu  ^. cmuKey+concurrentUpload mChunkSize mNumThreads uploadLoc multiPartUploadDesc = do+  env <- liftAWS $ view environment+  cmur <- send multiPartUploadDesc+  when (cmur ^. cmursResponseStatus /= 200) $+      fail "Failed to create upload" -        calcChunkSize :: Int -> Int-        calcChunkSize len =-            let chunkSize' = maybe minimumChunkSize (max minimumChunkSize) mcs-            in if len `div` chunkSize' >= 10000 then len `div` 9999 else chunkSize'+  let logStr :: MonadIO m => String -> m ()+      logStr    = liftIO . (env ^. envLogger) Info . stringUtf8+      bucket    = multiPartUploadDesc  ^. cmuBucket+      key       = multiPartUploadDesc  ^. cmuKey+      Just upId = cmur ^. cmursUploadId -    let mConnCount = managerConnCount defaultManagerSettings-        nThreads = maybe mConnCount (max 1) mnt-        exec run = if maybe False (> mConnCount) mnt-                then do-                    mgr' <- liftIO $ newManager  defaultManagerSettings{managerConnCount = nThreads}-                    local (envManager .~ mgr') run-                else run-    exec $ flip onException (send (abortMultipartUpload bucket key upId)) $ do-        sem <- liftIO $ newQSem nThreads-        umrs <- case ud of-            BS bs ->-                let chunkSize = calcChunkSize $ BS.length bs-                in forConcurrently (zip [1..] $ chunksOf chunkSize bs) $ \(partnum, b) -> do-                    liftIO $ waitQSem sem-                    logStr $ "Starting part: " ++ show partnum-                    umr <- send . uploadPart bucket key partnum upId . toBody $ b-                    logStr $ "Finished part: " ++ show partnum-                    liftIO $ signalQSem sem-                    pure $ completedPart partnum <$> (umr ^. uprsETag)+      calculateChunkSize :: Int -> Int+      calculateChunkSize len =+          let chunkSize' = maybe minimumChunkSize (max minimumChunkSize) mChunkSize+          in if len `div` chunkSize' >= 10000 then len `div` 9999 else chunkSize' -            FP fp -> do-                fsize <- liftIO $ getFileSize fp-                let chunkSize = calcChunkSize $ fromIntegral fsize-                    (count,lst) = divMod (fromIntegral fsize) chunkSize-                    params = [(partnum, chunkSize*offset, size)-                            | partnum <- [1..]-                            | offset  <- [0..count]-                            | size    <- (chunkSize <$ [0..count-1]) ++ [lst]-                            ]+      mConnCount = managerConnCount defaultManagerSettings+      nThreads   = maybe mConnCount (max 1) mNumThreads -                forConcurrently params $ \(partnum,off,size) -> do-                    liftIO $ waitQSem sem-                    b <- liftIO $ mmapFileByteString fp (Just (fromIntegral off,size))-                    umr <- send . uploadPart bucket key partnum upId . toBody $ b-                    liftIO $ signalQSem sem-                    pure $ completedPart partnum <$> (umr ^. uprsETag)+      exec :: MonadAWS m => AWS a -> m a+      exec act = if maybe False (> mConnCount) mNumThreads+              then do+                  mgr' <- liftIO $ newManager  defaultManagerSettings{managerConnCount = nThreads}+                  liftAWS $ local (envManager .~ mgr') act+              else liftAWS act+  exec $ flip onException (send (abortMultipartUpload bucket key upId)) $ do+      sem <- liftIO $ newQSem nThreads+      uploadResponses <- case uploadLoc of+          BS bytes ->+            let chunkSize = calculateChunkSize $ BS.length bytes+            in liftIO $ forConcurrently (zip [1..] $ chunksOf chunkSize bytes) $ \(partnum, chunk) ->+                bracket_ (waitQSem sem) (signalQSem sem) $ do+                  logStr $ "Starting part: " ++ show partnum+                  umr <- runResourceT $ runAWS env $ send . uploadPart bucket key partnum upId . toBody $ chunk+                  logStr $ "Finished part: " ++ show partnum+                  pure $ completedPart partnum <$> (umr ^. uprsETag) -        let prts = nonEmpty =<< sequence umrs-        send $ completeMultipartUpload bucket key upId-                & cMultipartUpload ?~ set cmuParts prts completedMultipartUpload+          FP filePath -> do+            fsize <- liftIO $ getFileSize filePath+            let chunkSize = calculateChunkSize $ fromIntegral fsize+                (count,lst) = fromIntegral fsize `divMod` chunkSize+                params = [(partnum, chunkSize*offset, size)+                          | partnum <- [1..]+                          | offset  <- [0..count]+                          | size    <- (chunkSize <$ [0..count-1]) ++ [lst]+                          ] +            liftIO $ forConcurrently params $ \(partnum,off,size) ->+              bracket_ (waitQSem sem) (signalQSem sem) $ do+                logStr $ "Starting file part: " ++ show partnum+                chunkStream <- hashedFileRange filePath (fromIntegral off) (fromIntegral size)+                uploadResp <- runResourceT $ runAWS env $+                  send . uploadPart bucket key partnum upId . toBody $ chunkStream+                logStr $ "Finished file part: " ++ show partnum+                pure $ completedPart partnum <$> (uploadResp ^. uprsETag)++      let parts = nonEmpty =<< sequence uploadResponses+      send $ completeMultipartUpload bucket key upId+              & cMultipartUpload ?~ set cmuParts parts completedMultipartUpload+ -- | Aborts all uploads in a given bucket - useful for cleaning up. abortAllUploads :: (MonadAWS m) => BucketName -> m () abortAllUploads bucket = do   rs <- send (listMultipartUploads bucket)   forM_ (rs ^. lmursUploads) $ \mu -> do     let mki = (,) <$> mu ^. muKey <*> mu ^. muUploadId-    case mki of-      Nothing        -> pure ()-      Just (key,uid) -> send (abortMultipartUpload bucket key uid) >> pure ()+    forM_ mki $ \(key,uid) -> send (abortMultipartUpload bucket key uid)