sak 0.1.2.5 → 0.1.2.6
raw patch · 12 files changed
+289/−110 lines, 12 files
Files
- CHANGELOG.md +4/−0
- README.md +32/−0
- man/sak.1 +3/−1
- sak.cabal +10/−1
- src/Compression.cpphs +4/−105
- src/Compression/Level.cpphs +108/−0
- src/Compression/Type.cpphs +19/−0
- src/Detect.hs +18/−0
- src/Info.cpphs +45/−0
- src/Main.hs +22/−3
- src/Version.cpphs +4/−0
- src/Version/Foreign.hs +20/−0
CHANGELOG.md view
@@ -1,5 +1,9 @@ # sak +## 0.1.2.6++ * Default to `-with-snappy` for sake of Mac+ ## 0.1.2.5 * Add lzop support
README.md view
@@ -2,6 +2,21 @@ sak is a command-line tool for compression. +## Installation++### Binary Releases++You can find binaries for many platforms+[here](https://www.permanent.org/p/archive/0236-0000/0236-0070/447897).++### Source++With [cabal-install](https://www.haskell.org/cabal/):++```+cabal install sak+```+ ## Use To convert compression format:@@ -29,3 +44,20 @@ ### Manpages `man/sak.1` contains manpages for `sak`++## Comparison++### Features++`sak` has some features not present in most decompressors (such as `lzip` or+`zstd`)++ - `transcode` - converts from one compressed format to another (streaming)+ - `recompress` - loads a file into memory and compresses it at+ a higher level+ - `verify` - simply verifies that a file is valid++### Performance++Performance should be on the same order of magnitude as the various+C command-line tools (`gzip`, `lzop`, `zstd`, &c.).
man/sak.1 view
@@ -1,4 +1,4 @@-.\" Automatically generated by Pandoc 2.9.2.1+.\" Automatically generated by Pandoc 2.10 .\" .TH "sak (1)" "" "" "" "" .hy@@ -31,6 +31,8 @@ \f[B]matrix\f[R] - Generate multiple compressed files in various formats .PP \f[B]recompress\f[R] - Recompress a file in-memory.+.PP+\f[B]info\f[R] - Guess a file format (for debugging) .SH OPTIONS .TP \f[B]-h\f[R] \f[B]--help\f[R]
sak.cabal view
@@ -1,11 +1,12 @@ cabal-version: 1.18 name: sak-version: 0.1.2.5+version: 0.1.2.6 license: BSD3 license-file: LICENSE copyright: Copyright: (c) 2020 Vanessa McHale maintainer: vamchale@gmail.com author: Vanessa McHale+bug-reports: https://hub.darcs.net/vmchale/sak/issues synopsis: Compression command-line tool description: sak is a command-line tool that detects and handles various compression formats@@ -25,8 +26,11 @@ description: Build with support for brotli encoding/decoding (disable this to statically link) + default: False+ flag with-snappy description: Build with support for snappy encoding/decoding+ default: False executable sak main-is: Main.hs@@ -34,7 +38,12 @@ hs-source-dirs: src other-modules: Version+ Version.Foreign Compression+ Compression.Level+ Compression.Type+ Info+ Detect Paths_sak autogen-modules: Paths_sak
src/Compression.cpphs view
@@ -1,5 +1,4 @@-module Compression ( Compression (Lzma, Lz4, Lzip, BZip)- , CompressionLevel (..)+module Compression ( Compression (..) , detectCompression , toCompressor , toDecompressor@@ -26,33 +25,13 @@ import qualified Codec.Compression.Zstd.Lazy as Zstd import qualified Codec.Lz4 as Lz4 import qualified Codec.Lzip as Lzip+import Compression.Level+import Compression.Type import qualified Data.ByteString.Lazy as BSL import Data.List (isSuffixOf) import System.FilePath (dropExtension, (-<.>)) import System.IO (IOMode (ReadMode), hFileSize, withFile) -data CompressionLevel = Best- | Fastest- | Default- | Custom !Int--data Compression = Lzma- | Lzip- | BZip- | GZip- | Zstd-#ifdef BROTLI- | Brotli-#endif-#ifdef SNAPPY- | Snappy-#endif- | Lzo- | Lz4- | Z- | None- deriving (Enum)- ext :: Compression -> String ext Lzma = ".xz" ext Lzip = ".lz"@@ -89,6 +68,7 @@ detectCompression :: FilePath -> Compression detectCompression fp | ".xz" `isSuffixOf` fp = Lzma+ | ".lzma" `isSuffixOf` fp = Lzma | ".txz" `isSuffixOf` fp = Lzma | ".lz" `isSuffixOf` fp = Lzip | ".tlz" `isSuffixOf` fp = Lzip@@ -137,87 +117,6 @@ check :: Compression -> BSL.ByteString -> IO () check = (forceBSL .) . toDecompressor--levelGuard :: (Int, Int) -> Int -> Int-levelGuard (min', max') lvl | lvl < min' || lvl > max' =- error ("Invalid compression level. Compression must be between " ++ show min' ++ " and " ++ show max')- | otherwise = lvl--toFileCompressor :: Compression -> CompressionLevel -> FilePath -> IO BSL.ByteString-toFileCompressor Lzip lvl = Lzip.compressFileLevel (toEnum $ toInt Lzip lvl)-toFileCompressor x lvl = fmap (toCompressor x lvl Nothing) . BSL.readFile--gzipCompression :: Int -> GZip.CompressParams-gzipCompression lvl = GZip.defaultCompressParams { GZip.compressLevel = GZip.compressionLevel lvl }--zlibCompression :: Int -> Zlib.CompressParams-zlibCompression lvl = Zlib.defaultCompressParams { Zlib.compressLevel = Zlib.compressionLevel lvl }--lzmaCompression :: Int -> Lzma.CompressParams-lzmaCompression lvl = Lzma.defaultCompressParams { Lzma.compressLevel = toEnum lvl }--#ifdef BROTLI-brotliCompression :: Int -> Br.CompressParams-brotliCompression lvl = Br.defaultCompressParams { Br.compressLevel = toEnum lvl }-#endif--toInt :: Compression -> CompressionLevel -> Int-#ifdef SNAPPY-toInt Snappy _ = undefined-#endif-toInt Lzo _ = undefined-#ifdef BROTLI-toInt Brotli Fastest = fromEnum (minBound :: Br.CompressionLevel)-toInt Brotli (Custom i) = levelGuard (0,11) i-toInt Brotli _ = fromEnum (maxBound :: Br.CompressionLevel)-#endif-toInt Zstd Best = Zstd.maxCLevel-toInt Zstd Fastest = 1-toInt Zstd (Custom i) = levelGuard (1, Zstd.maxCLevel) i-toInt Zstd Default = 3-toInt Lzip Best = fromEnum (maxBound :: Lzip.CompressionLevel)-toInt Lzip Fastest = fromEnum (minBound :: Lzip.CompressionLevel)-toInt Lzip (Custom i) = levelGuard (0, 9) i-toInt Lzip Default = 6-toInt Lzma Best = 9-toInt Lzma Fastest = 0-toInt Lzma (Custom i) = levelGuard (0, 9) i-toInt Lzma Default = 6-toInt BZip Best = 9-toInt BZip Fastest = 1-toInt BZip (Custom i) = i-toInt BZip Default = 7-toInt GZip Best = 9-toInt GZip Fastest = 0-toInt GZip (Custom i) = i-toInt GZip Default = 6-toInt Z Best = 9-toInt Z Fastest = 0-toInt Z (Custom i) = i-toInt Z Default = 6-toInt Lz4 Best = Lz4.lZ4HCClevelMax-toInt Lz4 Fastest = 0-toInt Lz4 (Custom i) = levelGuard (0, Lz4.lZ4HCClevelMax) i-toInt Lz4 Default = 0 -- 1?-toInt None _ = error "Internal error."--toCompressor :: Compression -> CompressionLevel -> Maybe Int -> BSL.ByteString -> BSL.ByteString-toCompressor Lzma lvl _ = Lzma.compressWith (lzmaCompression $ toInt Lzma lvl)-toCompressor Lzip lvl (Just sz) = flip (Lzip.compressWithSz (toEnum $ toInt Lzip lvl)) sz-toCompressor Lzip _ _ = error "Internal error."-toCompressor BZip lvl _ = BZip.compressWith (fromIntegral $ toInt BZip lvl) 30-toCompressor GZip lvl _ = GZip.compressWith (gzipCompression $ toInt GZip lvl)-toCompressor Z lvl _ = Zlib.compressWith (zlibCompression $ toInt Z lvl)-toCompressor Zstd lvl _ = Zstd.compress (toInt Zstd lvl)-toCompressor Lz4 lvl _ = Lz4.compressSz (toInt Lz4 lvl)-#ifdef BROTLI-toCompressor Brotli lvl _ = Br.compressWith (brotliCompression $ toInt Brotli lvl)-#endif-#ifdef SNAPPY-toCompressor Snappy _ _ = Snappy.compress-#endif-toCompressor Lzo _ _ = Lzo.compressFile-toCompressor None _ _ = id fileSize :: FilePath -> IO Integer fileSize fp = withFile fp ReadMode hFileSize
+ src/Compression/Level.cpphs view
@@ -0,0 +1,108 @@+module Compression.Level ( CompressionLevel (..)+ , toCompressor+ , toFileCompressor+ ) where++#ifdef BROTLI+import qualified Codec.Compression.Brotli as Br+#endif+import qualified Codec.Compression.BZip as BZip+import qualified Codec.Compression.GZip as GZip+import qualified Codec.Compression.Lzma as Lzma+import qualified Codec.Compression.Lzo as Lzo+#ifdef SNAPPY+import qualified Codec.Compression.Snappy.BSL as Snappy+#endif+import qualified Codec.Compression.Zlib as Zlib+import qualified Codec.Compression.Zstd.Lazy as Zstd+import qualified Codec.Lz4 as Lz4+import qualified Codec.Lzip as Lzip+import Compression.Type+import qualified Data.ByteString.Lazy as BSL++data CompressionLevel = Best+ | Fastest+ | Default+ | Custom !Int++levelGuard :: (Int, Int) -> Int -> Int+levelGuard (min', max') lvl | lvl < min' || lvl > max' =+ error ("Invalid compression level. Compression must be between " ++ show min' ++ " and " ++ show max')+ | otherwise = lvl++gzipCompression :: Int -> GZip.CompressParams+gzipCompression lvl = GZip.defaultCompressParams { GZip.compressLevel = GZip.compressionLevel lvl }++zlibCompression :: Int -> Zlib.CompressParams+zlibCompression lvl = Zlib.defaultCompressParams { Zlib.compressLevel = Zlib.compressionLevel lvl }++lzmaCompression :: Int -> Lzma.CompressParams+lzmaCompression lvl = Lzma.defaultCompressParams { Lzma.compressLevel = toEnum lvl }++#ifdef BROTLI+brotliCompression :: Int -> Br.CompressParams+brotliCompression lvl = Br.defaultCompressParams { Br.compressLevel = toEnum lvl }+#endif++toInt :: Compression -> CompressionLevel -> Int+#ifdef SNAPPY+toInt Snappy _ = undefined+#endif+toInt Lzo _ = undefined+#ifdef BROTLI+toInt Brotli Fastest = fromEnum (minBound :: Br.CompressionLevel)+toInt Brotli (Custom i) = levelGuard (0,11) i+toInt Brotli _ = fromEnum (maxBound :: Br.CompressionLevel)+#endif+toInt Zstd Best = Zstd.maxCLevel+toInt Zstd Fastest = 1+toInt Zstd (Custom i) = levelGuard (1, Zstd.maxCLevel) i+toInt Zstd Default = 3+toInt Lzip Best = fromEnum (maxBound :: Lzip.CompressionLevel)+toInt Lzip Fastest = fromEnum (minBound :: Lzip.CompressionLevel)+toInt Lzip (Custom i) = levelGuard (0, 9) i+toInt Lzip Default = 6+toInt Lzma Best = 9+toInt Lzma Fastest = 0+toInt Lzma (Custom i) = levelGuard (0, 9) i+toInt Lzma Default = 6+toInt BZip Best = 9+toInt BZip Fastest = 1+toInt BZip (Custom i) = i+toInt BZip Default = 7+toInt GZip Best = 9+toInt GZip Fastest = 0+toInt GZip (Custom i) = i+toInt GZip Default = 6+toInt Z Best = 9+toInt Z Fastest = 0+toInt Z (Custom i) = i+toInt Z Default = 6+toInt Lz4 Best = Lz4.lZ4HCClevelMax+toInt Lz4 Fastest = 0+toInt Lz4 (Custom i) = levelGuard (0, Lz4.lZ4HCClevelMax) i+toInt Lz4 Default = 0 -- 1?+toInt None _ = error "Internal error."++toFileCompressor :: Compression -> CompressionLevel -> FilePath -> IO BSL.ByteString+toFileCompressor Lzip lvl = Lzip.compressFileLevel (toEnum $ toInt Lzip lvl)+toFileCompressor x lvl = fmap (toCompressor x lvl Nothing) . BSL.readFile++toCompressor :: Compression -> CompressionLevel -> Maybe Int -> BSL.ByteString -> BSL.ByteString+toCompressor Lzma lvl _ = Lzma.compressWith (lzmaCompression $ toInt Lzma lvl)+toCompressor Lzip lvl (Just sz) = flip (Lzip.compressWithSz (toEnum $ toInt Lzip lvl)) sz+toCompressor Lzip _ _ = error "Internal error."+toCompressor BZip lvl _ = BZip.compressWith (fromIntegral $ toInt BZip lvl) 30+toCompressor GZip lvl _ = GZip.compressWith (gzipCompression $ toInt GZip lvl)+toCompressor Z lvl _ = Zlib.compressWith (zlibCompression $ toInt Z lvl)+toCompressor Zstd lvl _ = Zstd.compress (toInt Zstd lvl)+toCompressor Lz4 lvl _ = Lz4.compressSz (toInt Lz4 lvl)+#ifdef BROTLI+toCompressor Brotli lvl _ = Br.compressWith (brotliCompression $ toInt Brotli lvl)+#endif+#ifdef SNAPPY+toCompressor Snappy _ _ = Snappy.compress+#endif+toCompressor Lzo _ _ = Lzo.compressFile+toCompressor None _ _ = id+
+ src/Compression/Type.cpphs view
@@ -0,0 +1,19 @@+module Compression.Type ( Compression (..)+ ) where++data Compression = Lzma+ | Lzip+ | BZip+ | GZip+ | Zstd+#ifdef BROTLI+ | Brotli+#endif+#ifdef SNAPPY+ | Snappy+#endif+ | Lzo+ | Lz4+ | Z+ | None+ deriving (Show, Enum)
+ src/Detect.hs view
@@ -0,0 +1,18 @@+module Detect ( detectFileCompression+ ) where++import Compression+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BSL+import Info++firstChunk :: FilePath -> IO BS.ByteString+firstChunk = fmap (go . BSL.toChunks) . BSL.readFile+ where go [] = error "Empty file; could not detect compression"+ go (x:_) = x++detectFileCompression :: FilePath -> IO Compression+detectFileCompression fp =+ case detectCompression fp of+ None -> identify <$> firstChunk fp+ x -> pure x
+ src/Info.cpphs view
@@ -0,0 +1,45 @@+module Info ( identify+ ) where++import Compression+import Data.ByteString as BS++identify :: BS.ByteString -> Compression+identify bs | lzipMagicBytes `isPrefixOf` bs = Lzip+ | lzopMagicBytes `isPrefixOf` bs = Lzo+#ifdef SNAPPY+ | snappyStreamIdentifier `isPrefixOf` bs = Snappy+#endif+ | lz4Magic `isPrefixOf` bs = Lz4+ | gzipId `isPrefixOf` bs = GZip+ | xzMagic `isPrefixOf` bs = Lzma+ | zstdMagic `isPrefixOf` bs = Zstd+ | bzMagic `isPrefixOf` bs = BZip+ | otherwise = None++-- lrzipMagic :: BS.ByteString+-- lrzipMagic = BS.pack [0x4c, 0x52, 0x5a, 0x49]++zstdMagic :: BS.ByteString+zstdMagic = BS.pack [0x28, 0xb5, 0x2f, 0xfd]++lzipMagicBytes :: BS.ByteString+lzipMagicBytes = BS.pack [0x4c, 0x5a, 0x49, 0x50]++lzopMagicBytes :: BS.ByteString+lzopMagicBytes = BS.pack [0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a]++snappyStreamIdentifier :: BS.ByteString+snappyStreamIdentifier = BS.pack [0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]++gzipId :: BS.ByteString+gzipId = BS.pack [0x1f, 0x8b]++bzMagic :: BS.ByteString+bzMagic = BS.pack [0x42, 0x5a]++lz4Magic :: BS.ByteString+lz4Magic = BS.pack [0x04, 0x22, 0x4d, 0x18]++xzMagic :: BS.ByteString+xzMagic = BS.pack [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]
src/Main.hs view
@@ -3,9 +3,12 @@ module Main ( main ) where import Compression+import Compression.Level import Control.Concurrent.ParallelIO.Global (parallel_, stopGlobalPool)+import Control.Monad (forM_) import qualified Data.ByteString.Lazy as BSL import Data.Semigroup ((<>))+import Detect import Options.Applicative import System.Directory (getSymbolicLinkTarget, pathIsSymbolicLink) import Version (allVersionsString)@@ -23,6 +26,7 @@ | Transcode !FilePath !FilePath !CompressionLevel | Verify !FilePath | Matrix !FilePath !CompressionLevel+ | Info ![FilePath] forceBSL :: BSL.ByteString -> IO () forceBSL = (`seq` mempty) . last . BSL.toChunks@@ -31,7 +35,7 @@ -> FilePath -> IO () recompressFile lvl inp = do- let enc = detectCompression inp+ enc <- detectFileCompression inp pre <- case enc of BZip -> do { contents <- BSL.readFile inp ; toDecompressor enc contents <$ forceBSL contents@@ -51,6 +55,13 @@ decompressFile inp o = BSL.writeFile o =<< f inp where f = toFileDecompressor (detectCompression inp) +decompressDetectFile :: FilePath -- ^ Compressed file+ -> FilePath -- ^ Output+ -> IO ()+decompressDetectFile inp o = do+ f <- toFileDecompressor <$> (detectFileCompression inp)+ BSL.writeFile o =<< (f inp)+ compressFile :: CompressionLevel -> FilePath -- ^ Input file -> FilePath -- ^ Compressed output@@ -68,10 +79,10 @@ run :: Command -> IO () run (Recompress i lvl) = recompressFile lvl =<< reifyPath i run (Decompress i Nothing) = flip decompressFile (uncompressedExt i) =<< reifyPath i-run (Decompress i (Just o)) = flip decompressFile o =<< reifyPath i+run (Decompress i (Just o)) = flip decompressDetectFile o =<< reifyPath i run (Compress i o lvl) = flip (compressFile lvl) o =<< reifyPath i run (Transcode i o lvl) = do- let cO = detectCompression o+ cO <- detectFileCompression o guessSz <- case cO of Lzip -> Just . (8*) . fromIntegral <$> fileSize i _ -> pure Nothing@@ -80,6 +91,10 @@ run (Matrix inp lvl) = parallel_ (compressMatrix lvl inp <$> [Lzma .. Lz4]) *> stopGlobalPool+run (Info fps) =+ forM_ fps $ \fp ->+ putStr (fp ++ ": ") *>+ (print =<< detectFileCompression fp) fileCompletions :: HasCompleter f => Mod f a fileCompletions = completer (bashCompleter "file -o plusdirs")@@ -87,6 +102,9 @@ inpFile :: Parser FilePath inpFile = fileHelp "Input file" +infoFile :: Parser Command+infoFile = Info <$> some (fileHelp "Archive")+ verify :: Parser Command verify = Verify <$> inpFile @@ -161,6 +179,7 @@ <> command "verify" (info verify (progDesc "Check the integrity of a compressed file")) <> command "matrix" (info matrix (progDesc "Compress a file to all available formats")) <> command "recompress" (info recompress (progDesc "Recompress a file (for instance, to compress it at a higher level)"))+ <> command "info" (info infoFile (progDesc "Guess file compression")) ) versionMod :: Parser (a -> a)
src/Version.cpphs view
@@ -6,6 +6,7 @@ import Codec.Lzip import qualified Data.Version as V import qualified Paths_sak as P+import Version.Foreign allVersionsString :: String allVersionsString =@@ -13,8 +14,11 @@ ++ "lzlib-hs: " ++ VERSION_lzlib ++ "\n" ++ "lzlib: " ++ lZVersion ++ "\n" ++ "lzlib API: " ++ show (lZApiVersion :: Int) ++ "\n"+ ++ "zlib: " ++ zlib ++ "\n" ++ "zlib-hs: " ++ VERSION_zlib ++ "\n"+ ++ "lzma: " ++ lzma ++ "\n" ++ "lzma-hs: " ++ VERSION_lzma ++ "\n"+ ++ "zstd: " ++ zstd ++ "\n" ++ "zstd-hs: " ++ VERSION_zstd ++ "\n" ++ "bz2-hs: " ++ VERSION_bz2 ++ "\n" ++ "bz2: " ++ bZ2BzlibVersion ++ "\n"
+ src/Version/Foreign.hs view
@@ -0,0 +1,20 @@+module Version.Foreign ( zstd+ , zlib+ , lzma+ ) where++import Foreign.C.String (CString, peekCString)+import System.IO.Unsafe (unsafeDupablePerformIO)++foreign import ccall unsafe "ZSTD_versionString" zstdVersionString :: CString+foreign import ccall unsafe zlibVersion :: CString+foreign import ccall unsafe lzma_version_string :: CString++zstd :: String+zstd = unsafeDupablePerformIO (peekCString zstdVersionString)++zlib :: String+zlib = unsafeDupablePerformIO (peekCString zlibVersion)++lzma :: String+lzma = unsafeDupablePerformIO (peekCString lzma_version_string)