zip-stream 0.1.0.1 → 0.1.1

raw patch · 7 files changed

+86/−31 lines, 7 filesdep ~basedep ~conduitdep ~conduit-extraPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: base, conduit, conduit-extra

API changes (from Hackage documentation)

Files

• Codec/Archive/Zip/Conduit/Internal.hs +20/−8
• Codec/Archive/Zip/Conduit/Types.hs +8/−4
• Codec/Archive/Zip/Conduit/UnZip.hs +15/−4
• Codec/Archive/Zip/Conduit/Zip.hs +23/−8
• cmd/unzip.hs +2/−1
• cmd/zip.hs +16/−4
• zip-stream.cabal +2/−2

@@ -1,5 +1,6 @@+{-# LANGUAGE CPP #-} module Codec.Archive.Zip.Conduit.Internal-  ( zipVersion+  ( osVersion, zipVersion   , zipError   , idConduit   , sizeCRC@@ -15,18 +16,29 @@ import qualified Data.Conduit as C import qualified Data.Conduit.Internal as CI import           Data.Digest.CRC32 (crc32Update)-import           Data.Word (Word16, Word32, Word64)+import           Data.Word (Word8, Word32, Word64)  import           Codec.Archive.Zip.Conduit.Types --- |The version of this zip program, really just rough indicator of compatibility-zipVersion :: Word16+#if MIN_VERSION_conduit(1,3,0)+#define ConduitM ConduitT+#define PRE13(x)+#else+#define PRE13(x) x+#endif++-- | The version of this zip program, really just rough indicator of compatibility+zipVersion :: Word8 zipVersion = 48 +-- | The OS this implementation tries to be compatible to+osVersion :: Word8+osVersion = 0 -- DOS+ zipError :: MonadThrow m => String -> m a zipError = throwM . ZipError -idConduit :: Monad m => C.Conduit a m a+idConduit :: Monad m => C.ConduitM a a m () idConduit = C.awaitForever C.yield  passthroughFold :: Monad m => (a -> b -> a) -> a -> C.ConduitM b b m a@@ -42,16 +54,16 @@ sizeC :: Monad m => C.ConduitM BS.ByteString BS.ByteString m Word64 sizeC = passthroughFold (\l b -> l + fromIntegral (BS.length b)) 0 -- fst <$> sizeCRC -outputSize :: Monad m => C.Conduit i m BS.ByteString -> C.ConduitM i BS.ByteString m Word64+outputSize :: Monad m => C.ConduitM i BS.ByteString m () -> C.ConduitM i BS.ByteString m Word64 outputSize = (C..| sizeC) -inputSize :: Monad m => C.Conduit BS.ByteString m o -> C.ConduitM BS.ByteString o m Word64+inputSize :: Monad m => C.ConduitM BS.ByteString o m () -> C.ConduitM BS.ByteString o m Word64 -- inputSize = fuseUpstream sizeC -- won't work because we need to deal with leftovers properly inputSize (CI.ConduitM src) = CI.ConduitM $ \rest -> let   go n (CI.Done ()) = rest n   go n (CI.PipeM m) = CI.PipeM $ go n <$> m   go n (CI.Leftover p b) = CI.Leftover (go (n - fromIntegral (BS.length b)) p) b-  go n (CI.HaveOutput p f o) = CI.HaveOutput (go n p) f o+  go n (CI.HaveOutput p PRE13(f) o) = CI.HaveOutput (go n p) PRE13(f) o   go n (CI.NeedInput p q) = CI.NeedInput (\b -> go (n + fromIntegral (BS.length b)) (p b)) (go n . q)   in go 0 (src CI.Done) 

@@ -5,6 +5,7 @@ import qualified Data.ByteString.Lazy as BSL import qualified Data.Conduit as C import           Data.Conduit.Binary (sourceLbs)+import           Data.Semigroup (Semigroup(..)) import           Data.String (IsString(..)) import           Data.Time.LocalTime (LocalTime) import           Data.Typeable (Typeable)@@ -36,15 +37,18 @@ -- |The data contents for a 'ZipEntry'. For empty entries (e.g., directories), use 'mempty'. data ZipData m   = ZipDataByteString BSL.ByteString -- ^A known ByteString, which will be fully evaluated (not streamed)-  | ZipDataSource (C.Source m ByteString) -- ^A byte stream producer, streamed (and compressed) directly into the zip+  | ZipDataSource (C.ConduitM () ByteString m ()) -- ^A byte stream producer, streamed (and compressed) directly into the zip +instance Monad m => Semigroup (ZipData m) where+  ZipDataByteString a <> ZipDataByteString b = ZipDataByteString $ mappend a b+  a <> b = ZipDataSource $ mappend (sourceZipData a) (sourceZipData b)+ instance Monad m => Monoid (ZipData m) where   mempty = ZipDataByteString BSL.empty-  mappend (ZipDataByteString a) (ZipDataByteString b) = ZipDataByteString $ mappend a b-  mappend a b = ZipDataSource $ mappend (sourceZipData a) (sourceZipData b)+  mappend = (<>)  -- |Normalize any 'ZipData' to a simple source-sourceZipData :: Monad m => ZipData m -> C.Source m ByteString+sourceZipData :: Monad m => ZipData m -> C.ConduitM () ByteString m () sourceZipData (ZipDataByteString b) = sourceLbs b sourceZipData (ZipDataSource s) = s 

@@ -1,4 +1,5 @@ -- |Stream the extraction of a zip file, e.g., as it's being downloaded.+{-# LANGUAGE CPP #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE RankNTypes #-} module Codec.Archive.Zip.Conduit.UnZip@@ -9,7 +10,9 @@  import           Control.Applicative ((<|>), empty) import           Control.Monad (when, unless, guard)+#if !MIN_VERSION_conduit(1,3,0) import           Control.Monad.Base (MonadBase)+#endif import           Control.Monad.Catch (MonadThrow) import           Control.Monad.Primitive (PrimMonad) import qualified Data.Binary.Get as G@@ -28,7 +31,7 @@  data Header m   = FileHeader-    { fileDecompress :: C.Conduit BS.ByteString m BS.ByteString+    { fileDecompress :: C.ConduitM BS.ByteString BS.ByteString m ()     , fileEntry :: !ZipEntry     , fileCRC :: !Word32     , fileCSize :: !Word64@@ -53,7 +56,7 @@   } -} -pass :: (MonadThrow m, Integral n) => n -> C.Conduit BS.ByteString m BS.ByteString+pass :: (MonadThrow m, Integral n) => n -> C.ConduitM BS.ByteString BS.ByteString m () pass 0 = return () pass n = C.await >>= maybe   (zipError $ "EOF in file data, expecting " ++ show ni ++ " more bytes")@@ -94,7 +97,14 @@ -- It does not (ironically) support uncompressed zip files that have been created as streams, where file sizes are not known beforehand. -- Since it does not use the offset information at the end of the file, it assumes all entries are packed sequentially, which is usually the case. -- Any errors are thrown in the underlying monad (as 'ZipError's or 'Data.Conduit.Serialization.Binary.ParseError').-unZipStream :: (MonadBase b m, PrimMonad b, MonadThrow m) => C.ConduitM BS.ByteString (Either ZipEntry BS.ByteString) m ZipInfo+unZipStream ::+  ( MonadThrow m+#if MIN_VERSION_conduit(1,3,0)+  , PrimMonad m+#else+  , MonadBase b m, PrimMonad b+#endif+  ) => C.ConduitM BS.ByteString (Either ZipEntry BS.ByteString) m ZipInfo unZipStream = next where   next = do -- local header, or start central directory     h <- sinkGet $ do@@ -151,7 +161,8 @@   centralBody 0x06054b50 = EndOfCentralDirectory <$> endDirectory   centralBody sig = fail $ "Unknown header signature: " ++ show sig   fileHeader = do-    ver <- G.getWord16le+    ver <- G.getWord8+    _os <- G.getWord8 -- OS Version (could require 0 = DOS, but we ignore ext attrs altogether)     when (ver > zipVersion) $ fail $ "Unsupported version: " ++ show ver     gpf <- G.getWord16le     -- when (gpf .&. complement (bit 1 .|. bit 2 .|. bit 3) /= 0) $ fail $ "Unsupported flags: " ++ show gpf

@@ -1,4 +1,5 @@ -- |Stream the creation of a zip file, e.g., as it's being uploaded.+{-# LANGUAGE CPP #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ViewPatterns #-} module Codec.Archive.Zip.Conduit.Zip@@ -14,7 +15,9 @@ import qualified Codec.Compression.Zlib.Raw as Z import           Control.Arrow ((&&&), (+++), left) import           Control.Monad (when)+#if !MIN_VERSION_conduit(1,3,0) import           Control.Monad.Base (MonadBase)+#endif import           Control.Monad.Catch (MonadThrow) import           Control.Monad.Primitive (PrimMonad) import           Control.Monad.State.Strict (StateT, get)@@ -63,7 +66,7 @@ zipFileData :: MonadResource m => FilePath -> ZipData m zipFileData = ZipDataSource . CB.sourceFile -zipData :: Monad m => ZipData m -> Either (C.Source m BS.ByteString) BSL.ByteString+zipData :: Monad m => ZipData m -> Either (C.ConduitM () BS.ByteString m ()) BSL.ByteString zipData (ZipDataByteString b) = Right b zipData (ZipDataSource s) = Left s @@ -77,7 +80,7 @@   , fromIntegral (year - 1980) `shiftL` 9 .|. fromIntegral month `shiftL` 5 .|. fromIntegral day   ) -countOutput :: Monad m => C.Conduit i m BS.ByteString -> C.Conduit i (StateT Word64 m) BS.ByteString+countOutput :: Monad m => C.ConduitM i BS.ByteString m () -> C.ConduitM i BS.ByteString (StateT Word64 m) () countOutput c = stateC $ \s -> (,) () . (s +) <$> outputSize c  output :: MonadThrow m => P.Put -> C.ConduitM i BS.ByteString (StateT Word64 m) ()@@ -92,7 +95,14 @@ -- -- Depending on options, the resulting zip file should be compatible with most unzipping applications. -- Any errors are thrown in the underlying monad (as 'ZipError's).-zipStream :: (MonadBase b m, PrimMonad b, MonadThrow m) => ZipOptions -> C.ConduitM (ZipEntry, ZipData m) BS.ByteString m Word64+zipStream :: +  ( MonadThrow m+#if MIN_VERSION_conduit(1,3,0)+  , PrimMonad m+#else+  , MonadBase b m, PrimMonad b+#endif+  ) => ZipOptions -> C.ConduitM (ZipEntry, ZipData m) BS.ByteString m Word64 zipStream ZipOptions{..} = execStateC 0 $ do   (cnt, cdir) <- next 0 (return ())   cdoff <- get@@ -129,7 +139,8 @@     off <- get     output $ do       P.putWord32le 0x04034b50-      P.putWord16le $ if z64 then 45 else 20+      P.putWord8 $ if z64 then 45 else 20+      P.putWord8 osVersion       common       P.putWord32le $ fromMaybe 0 mcrc       P.putWord32le $ if z64 then maxBound32 else maybe 0 fromIntegral csiz@@ -165,8 +176,10 @@           l64 = z64 ?* 16 + o64 ?* 8           a64 = z64 || o64       P.putWord32le 0x02014b50-      P.putWord16le zipVersion-      P.putWord16le $ if a64 then 45 else 20+      P.putWord8 zipVersion+      P.putWord8 osVersion+      P.putWord8 $ if a64 then 45 else 20+      P.putWord8 osVersion       common       P.putWord32le crc       P.putWord32le $ if z64 then maxBound32 else fromIntegral csz@@ -192,8 +205,10 @@     when z64 $ output $ do       P.putWord32le 0x06064b50 -- zip64 end       P.putWord64le 44 -- length of this record-      P.putWord16le zipVersion-      P.putWord16le 45+      P.putWord8 zipVersion+      P.putWord8 osVersion+      P.putWord8 45+      P.putWord8 osVersion       P.putWord32le 0 -- disk       P.putWord32le 0 -- central disk       P.putWord64le cnt

@@ -7,6 +7,7 @@ import qualified Data.Conduit as C import qualified Data.Conduit.Binary as CB import           Data.Time.LocalTime (localTimeToUTC, utc)+import           Data.Void (Void) import           System.Directory (createDirectoryIfMissing #if MIN_VERSION_directory(1,2,3)   , setModificationTime@@ -19,7 +20,7 @@  import           Codec.Archive.Zip.Conduit.UnZip -extract :: C.Sink (Either ZipEntry BS.ByteString) IO ()+extract :: C.ConduitM (Either ZipEntry BS.ByteString) Void IO () extract = C.awaitForever start where   start (Left ZipEntry{..}) = do     liftIO $ BSC.putStrLn zipEntryName

@@ -10,8 +10,13 @@ import qualified System.Console.GetOpt as Opt import           System.Directory (doesDirectoryExist, getModificationTime #if MIN_VERSION_directory(1,2,6)-  , isSymbolicLink, listDirectory+#if MIN_VERSION_directory(1,3,0)+  , pathIsSymbolicLink #else+  , isSymbolicLink+#endif+  , listDirectory+#else   , getDirectoryContents #endif   )@@ -34,7 +39,7 @@     "set zip comment"   ] -generate :: (MonadIO m, MonadResource m) => [FilePath] -> C.Source m (ZipEntry, ZipData m)+generate :: (MonadIO m, MonadResource m) => [FilePath] -> C.ConduitM () (ZipEntry, ZipData m) m () generate (p:paths) = do   t <- liftIO $ getModificationTime p   let e = ZipEntry@@ -45,10 +50,17 @@   isd <- liftIO $ doesDirectoryExist p   if isd     then do+      dl <- liftIO $ #if MIN_VERSION_directory(1,2,6)-      dl <- liftIO $ filterM (fmap not . isSymbolicLink) . map (p </>) =<< listDirectory p+        filterM (fmap not .+#if MIN_VERSION_directory(1,3,0)+          pathIsSymbolicLink #else-      dl <- liftIO $ filter (`notElem` [".",".."]) . map (p </>) <$> getDirectoryContents p+          isSymbolicLink+#endif+          ) . map (p </>) =<< listDirectory p+#else+        filter (`notElem` [".",".."]) . map (p </>) <$> getDirectoryContents p #endif       C.yield (e{ zipEntryName = zipEntryName e `BSC.snoc` '/', zipEntrySize = Just 0 }, mempty)       generate $ dl ++ paths

@@ -1,5 +1,5 @@ name:                zip-stream-version:             0.1.0.1+version:             0.1.1 synopsis:            ZIP archive streaming using conduits description:         Process (extract and create) zip files as streams (e.g., over the network), accessing contained files without having to write the zip file to disk (unlike zip-conduit). license:             BSD3@@ -25,7 +25,7 @@   default-language:    Haskell2010   ghc-options: -Wall   build-depends:       -    base >= 4.8 && < 5,+    base >= 4.9 && < 5,     binary >= 0.7.2,     binary-conduit,     bytestring,