diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # sak +## 0.1.2.6 + + * Default to `-with-snappy` for sake of Mac + ## 0.1.2.5 * Add lzop support diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -2,6 +2,21 @@ sak is a command-line tool for compression. +## Installation + +### Binary Releases + +You can find binaries for many platforms +[here](https://www.permanent.org/p/archive/0236-0000/0236-0070/447897). + +### Source + +With [cabal-install](https://www.haskell.org/cabal/): + +``` +cabal install sak +``` + ## Use To convert compression format: @@ -29,3 +44,20 @@ ### Manpages `man/sak.1` contains manpages for `sak` + +## Comparison + +### Features + +`sak` has some features not present in most decompressors (such as `lzip` or +`zstd`) + + - `transcode` - converts from one compressed format to another (streaming) + - `recompress` - loads a file into memory and compresses it at + a higher level + - `verify` - simply verifies that a file is valid + +### Performance + +Performance should be on the same order of magnitude as the various +C command-line tools (`gzip`, `lzop`, `zstd`, &c.). diff --git a/man/sak.1 b/man/sak.1 --- a/man/sak.1 +++ b/man/sak.1 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pandoc 2.9.2.1 +.\" Automatically generated by Pandoc 2.10 .\" .TH "sak (1)" "" "" "" "" .hy @@ -31,6 +31,8 @@ \f[B]matrix\f[R] - Generate multiple compressed files in various formats .PP \f[B]recompress\f[R] - Recompress a file in-memory. +.PP +\f[B]info\f[R] - Guess a file format (for debugging) .SH OPTIONS .TP \f[B]-h\f[R] \f[B]--help\f[R] diff --git a/sak.cabal b/sak.cabal --- a/sak.cabal +++ b/sak.cabal @@ -1,11 +1,12 @@ cabal-version: 1.18 name: sak -version: 0.1.2.5 +version: 0.1.2.6 license: BSD3 license-file: LICENSE copyright: Copyright: (c) 2020 Vanessa McHale maintainer: vamchale@gmail.com author: Vanessa McHale +bug-reports: https://hub.darcs.net/vmchale/sak/issues synopsis: Compression command-line tool description: sak is a command-line tool that detects and handles various compression formats @@ -25,8 +26,11 @@ description: Build with support for brotli encoding/decoding (disable this to statically link) + default: False + flag with-snappy description: Build with support for snappy encoding/decoding + default: False executable sak main-is: Main.hs @@ -34,7 +38,12 @@ hs-source-dirs: src other-modules: Version + Version.Foreign Compression + Compression.Level + Compression.Type + Info + Detect Paths_sak autogen-modules: Paths_sak diff --git a/src/Compression.cpphs b/src/Compression.cpphs --- a/src/Compression.cpphs +++ b/src/Compression.cpphs @@ -1,5 +1,4 @@ -module Compression ( Compression (Lzma, Lz4, Lzip, BZip) - , CompressionLevel (..) +module Compression ( Compression (..) , detectCompression , toCompressor , toDecompressor @@ -26,33 +25,13 @@ import qualified Codec.Compression.Zstd.Lazy as Zstd import qualified Codec.Lz4 as Lz4 import qualified Codec.Lzip as Lzip +import Compression.Level +import Compression.Type import qualified Data.ByteString.Lazy as BSL import Data.List (isSuffixOf) import System.FilePath (dropExtension, (-<.>)) import System.IO (IOMode (ReadMode), hFileSize, withFile) -data CompressionLevel = Best - | Fastest - | Default - | Custom !Int - -data Compression = Lzma - | Lzip - | BZip - | GZip - | Zstd -#ifdef BROTLI - | Brotli -#endif -#ifdef SNAPPY - | Snappy -#endif - | Lzo - | Lz4 - | Z - | None - deriving (Enum) - ext :: Compression -> String ext Lzma = ".xz" ext Lzip = ".lz" @@ -89,6 +68,7 @@ detectCompression :: FilePath -> Compression detectCompression fp | ".xz" `isSuffixOf` fp = Lzma + | ".lzma" `isSuffixOf` fp = Lzma | ".txz" `isSuffixOf` fp = Lzma | ".lz" `isSuffixOf` fp = Lzip | ".tlz" `isSuffixOf` fp = Lzip @@ -137,87 +117,6 @@ check :: Compression -> BSL.ByteString -> IO () check = (forceBSL .) . toDecompressor - -levelGuard :: (Int, Int) -> Int -> Int -levelGuard (min', max') lvl | lvl < min' || lvl > max' = - error ("Invalid compression level. Compression must be between " ++ show min' ++ " and " ++ show max') - | otherwise = lvl - -toFileCompressor :: Compression -> CompressionLevel -> FilePath -> IO BSL.ByteString -toFileCompressor Lzip lvl = Lzip.compressFileLevel (toEnum $ toInt Lzip lvl) -toFileCompressor x lvl = fmap (toCompressor x lvl Nothing) . BSL.readFile - -gzipCompression :: Int -> GZip.CompressParams -gzipCompression lvl = GZip.defaultCompressParams { GZip.compressLevel = GZip.compressionLevel lvl } - -zlibCompression :: Int -> Zlib.CompressParams -zlibCompression lvl = Zlib.defaultCompressParams { Zlib.compressLevel = Zlib.compressionLevel lvl } - -lzmaCompression :: Int -> Lzma.CompressParams -lzmaCompression lvl = Lzma.defaultCompressParams { Lzma.compressLevel = toEnum lvl } - -#ifdef BROTLI -brotliCompression :: Int -> Br.CompressParams -brotliCompression lvl = Br.defaultCompressParams { Br.compressLevel = toEnum lvl } -#endif - -toInt :: Compression -> CompressionLevel -> Int -#ifdef SNAPPY -toInt Snappy _ = undefined -#endif -toInt Lzo _ = undefined -#ifdef BROTLI -toInt Brotli Fastest = fromEnum (minBound :: Br.CompressionLevel) -toInt Brotli (Custom i) = levelGuard (0,11) i -toInt Brotli _ = fromEnum (maxBound :: Br.CompressionLevel) -#endif -toInt Zstd Best = Zstd.maxCLevel -toInt Zstd Fastest = 1 -toInt Zstd (Custom i) = levelGuard (1, Zstd.maxCLevel) i -toInt Zstd Default = 3 -toInt Lzip Best = fromEnum (maxBound :: Lzip.CompressionLevel) -toInt Lzip Fastest = fromEnum (minBound :: Lzip.CompressionLevel) -toInt Lzip (Custom i) = levelGuard (0, 9) i -toInt Lzip Default = 6 -toInt Lzma Best = 9 -toInt Lzma Fastest = 0 -toInt Lzma (Custom i) = levelGuard (0, 9) i -toInt Lzma Default = 6 -toInt BZip Best = 9 -toInt BZip Fastest = 1 -toInt BZip (Custom i) = i -toInt BZip Default = 7 -toInt GZip Best = 9 -toInt GZip Fastest = 0 -toInt GZip (Custom i) = i -toInt GZip Default = 6 -toInt Z Best = 9 -toInt Z Fastest = 0 -toInt Z (Custom i) = i -toInt Z Default = 6 -toInt Lz4 Best = Lz4.lZ4HCClevelMax -toInt Lz4 Fastest = 0 -toInt Lz4 (Custom i) = levelGuard (0, Lz4.lZ4HCClevelMax) i -toInt Lz4 Default = 0 -- 1? -toInt None _ = error "Internal error." - -toCompressor :: Compression -> CompressionLevel -> Maybe Int -> BSL.ByteString -> BSL.ByteString -toCompressor Lzma lvl _ = Lzma.compressWith (lzmaCompression $ toInt Lzma lvl) -toCompressor Lzip lvl (Just sz) = flip (Lzip.compressWithSz (toEnum $ toInt Lzip lvl)) sz -toCompressor Lzip _ _ = error "Internal error." -toCompressor BZip lvl _ = BZip.compressWith (fromIntegral $ toInt BZip lvl) 30 -toCompressor GZip lvl _ = GZip.compressWith (gzipCompression $ toInt GZip lvl) -toCompressor Z lvl _ = Zlib.compressWith (zlibCompression $ toInt Z lvl) -toCompressor Zstd lvl _ = Zstd.compress (toInt Zstd lvl) -toCompressor Lz4 lvl _ = Lz4.compressSz (toInt Lz4 lvl) -#ifdef BROTLI -toCompressor Brotli lvl _ = Br.compressWith (brotliCompression $ toInt Brotli lvl) -#endif -#ifdef SNAPPY -toCompressor Snappy _ _ = Snappy.compress -#endif -toCompressor Lzo _ _ = Lzo.compressFile -toCompressor None _ _ = id fileSize :: FilePath -> IO Integer fileSize fp = withFile fp ReadMode hFileSize diff --git a/src/Compression/Level.cpphs b/src/Compression/Level.cpphs new file mode 100644 --- /dev/null +++ b/src/Compression/Level.cpphs @@ -0,0 +1,108 @@ +module Compression.Level ( CompressionLevel (..) + , toCompressor + , toFileCompressor + ) where + +#ifdef BROTLI +import qualified Codec.Compression.Brotli as Br +#endif +import qualified Codec.Compression.BZip as BZip +import qualified Codec.Compression.GZip as GZip +import qualified Codec.Compression.Lzma as Lzma +import qualified Codec.Compression.Lzo as Lzo +#ifdef SNAPPY +import qualified Codec.Compression.Snappy.BSL as Snappy +#endif +import qualified Codec.Compression.Zlib as Zlib +import qualified Codec.Compression.Zstd.Lazy as Zstd +import qualified Codec.Lz4 as Lz4 +import qualified Codec.Lzip as Lzip +import Compression.Type +import qualified Data.ByteString.Lazy as BSL + +data CompressionLevel = Best + | Fastest + | Default + | Custom !Int + +levelGuard :: (Int, Int) -> Int -> Int +levelGuard (min', max') lvl | lvl < min' || lvl > max' = + error ("Invalid compression level. Compression must be between " ++ show min' ++ " and " ++ show max') + | otherwise = lvl + +gzipCompression :: Int -> GZip.CompressParams +gzipCompression lvl = GZip.defaultCompressParams { GZip.compressLevel = GZip.compressionLevel lvl } + +zlibCompression :: Int -> Zlib.CompressParams +zlibCompression lvl = Zlib.defaultCompressParams { Zlib.compressLevel = Zlib.compressionLevel lvl } + +lzmaCompression :: Int -> Lzma.CompressParams +lzmaCompression lvl = Lzma.defaultCompressParams { Lzma.compressLevel = toEnum lvl } + +#ifdef BROTLI +brotliCompression :: Int -> Br.CompressParams +brotliCompression lvl = Br.defaultCompressParams { Br.compressLevel = toEnum lvl } +#endif + +toInt :: Compression -> CompressionLevel -> Int +#ifdef SNAPPY +toInt Snappy _ = undefined +#endif +toInt Lzo _ = undefined +#ifdef BROTLI +toInt Brotli Fastest = fromEnum (minBound :: Br.CompressionLevel) +toInt Brotli (Custom i) = levelGuard (0,11) i +toInt Brotli _ = fromEnum (maxBound :: Br.CompressionLevel) +#endif +toInt Zstd Best = Zstd.maxCLevel +toInt Zstd Fastest = 1 +toInt Zstd (Custom i) = levelGuard (1, Zstd.maxCLevel) i +toInt Zstd Default = 3 +toInt Lzip Best = fromEnum (maxBound :: Lzip.CompressionLevel) +toInt Lzip Fastest = fromEnum (minBound :: Lzip.CompressionLevel) +toInt Lzip (Custom i) = levelGuard (0, 9) i +toInt Lzip Default = 6 +toInt Lzma Best = 9 +toInt Lzma Fastest = 0 +toInt Lzma (Custom i) = levelGuard (0, 9) i +toInt Lzma Default = 6 +toInt BZip Best = 9 +toInt BZip Fastest = 1 +toInt BZip (Custom i) = i +toInt BZip Default = 7 +toInt GZip Best = 9 +toInt GZip Fastest = 0 +toInt GZip (Custom i) = i +toInt GZip Default = 6 +toInt Z Best = 9 +toInt Z Fastest = 0 +toInt Z (Custom i) = i +toInt Z Default = 6 +toInt Lz4 Best = Lz4.lZ4HCClevelMax +toInt Lz4 Fastest = 0 +toInt Lz4 (Custom i) = levelGuard (0, Lz4.lZ4HCClevelMax) i +toInt Lz4 Default = 0 -- 1? +toInt None _ = error "Internal error." + +toFileCompressor :: Compression -> CompressionLevel -> FilePath -> IO BSL.ByteString +toFileCompressor Lzip lvl = Lzip.compressFileLevel (toEnum $ toInt Lzip lvl) +toFileCompressor x lvl = fmap (toCompressor x lvl Nothing) . BSL.readFile + +toCompressor :: Compression -> CompressionLevel -> Maybe Int -> BSL.ByteString -> BSL.ByteString +toCompressor Lzma lvl _ = Lzma.compressWith (lzmaCompression $ toInt Lzma lvl) +toCompressor Lzip lvl (Just sz) = flip (Lzip.compressWithSz (toEnum $ toInt Lzip lvl)) sz +toCompressor Lzip _ _ = error "Internal error." +toCompressor BZip lvl _ = BZip.compressWith (fromIntegral $ toInt BZip lvl) 30 +toCompressor GZip lvl _ = GZip.compressWith (gzipCompression $ toInt GZip lvl) +toCompressor Z lvl _ = Zlib.compressWith (zlibCompression $ toInt Z lvl) +toCompressor Zstd lvl _ = Zstd.compress (toInt Zstd lvl) +toCompressor Lz4 lvl _ = Lz4.compressSz (toInt Lz4 lvl) +#ifdef BROTLI +toCompressor Brotli lvl _ = Br.compressWith (brotliCompression $ toInt Brotli lvl) +#endif +#ifdef SNAPPY +toCompressor Snappy _ _ = Snappy.compress +#endif +toCompressor Lzo _ _ = Lzo.compressFile +toCompressor None _ _ = id + diff --git a/src/Compression/Type.cpphs b/src/Compression/Type.cpphs new file mode 100644 --- /dev/null +++ b/src/Compression/Type.cpphs @@ -0,0 +1,19 @@ +module Compression.Type ( Compression (..) + ) where + +data Compression = Lzma + | Lzip + | BZip + | GZip + | Zstd +#ifdef BROTLI + | Brotli +#endif +#ifdef SNAPPY + | Snappy +#endif + | Lzo + | Lz4 + | Z + | None + deriving (Show, Enum) diff --git a/src/Detect.hs b/src/Detect.hs new file mode 100644 --- /dev/null +++ b/src/Detect.hs @@ -0,0 +1,18 @@ +module Detect ( detectFileCompression + ) where + +import Compression +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BSL +import Info + +firstChunk :: FilePath -> IO BS.ByteString +firstChunk = fmap (go . BSL.toChunks) . BSL.readFile + where go [] = error "Empty file; could not detect compression" + go (x:_) = x + +detectFileCompression :: FilePath -> IO Compression +detectFileCompression fp = + case detectCompression fp of + None -> identify <$> firstChunk fp + x -> pure x diff --git a/src/Info.cpphs b/src/Info.cpphs new file mode 100644 --- /dev/null +++ b/src/Info.cpphs @@ -0,0 +1,45 @@ +module Info ( identify + ) where + +import Compression +import Data.ByteString as BS + +identify :: BS.ByteString -> Compression +identify bs | lzipMagicBytes `isPrefixOf` bs = Lzip + | lzopMagicBytes `isPrefixOf` bs = Lzo +#ifdef SNAPPY + | snappyStreamIdentifier `isPrefixOf` bs = Snappy +#endif + | lz4Magic `isPrefixOf` bs = Lz4 + | gzipId `isPrefixOf` bs = GZip + | xzMagic `isPrefixOf` bs = Lzma + | zstdMagic `isPrefixOf` bs = Zstd + | bzMagic `isPrefixOf` bs = BZip + | otherwise = None + +-- lrzipMagic :: BS.ByteString +-- lrzipMagic = BS.pack [0x4c, 0x52, 0x5a, 0x49] + +zstdMagic :: BS.ByteString +zstdMagic = BS.pack [0x28, 0xb5, 0x2f, 0xfd] + +lzipMagicBytes :: BS.ByteString +lzipMagicBytes = BS.pack [0x4c, 0x5a, 0x49, 0x50] + +lzopMagicBytes :: BS.ByteString +lzopMagicBytes = BS.pack [0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a] + +snappyStreamIdentifier :: BS.ByteString +snappyStreamIdentifier = BS.pack [0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59] + +gzipId :: BS.ByteString +gzipId = BS.pack [0x1f, 0x8b] + +bzMagic :: BS.ByteString +bzMagic = BS.pack [0x42, 0x5a] + +lz4Magic :: BS.ByteString +lz4Magic = BS.pack [0x04, 0x22, 0x4d, 0x18] + +xzMagic :: BS.ByteString +xzMagic = BS.pack [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00] diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -3,9 +3,12 @@ module Main ( main ) where import Compression +import Compression.Level import Control.Concurrent.ParallelIO.Global (parallel_, stopGlobalPool) +import Control.Monad (forM_) import qualified Data.ByteString.Lazy as BSL import Data.Semigroup ((<>)) +import Detect import Options.Applicative import System.Directory (getSymbolicLinkTarget, pathIsSymbolicLink) import Version (allVersionsString) @@ -23,6 +26,7 @@ | Transcode !FilePath !FilePath !CompressionLevel | Verify !FilePath | Matrix !FilePath !CompressionLevel + | Info ![FilePath] forceBSL :: BSL.ByteString -> IO () forceBSL = (`seq` mempty) . last . BSL.toChunks @@ -31,7 +35,7 @@ -> FilePath -> IO () recompressFile lvl inp = do - let enc = detectCompression inp + enc <- detectFileCompression inp pre <- case enc of BZip -> do { contents <- BSL.readFile inp ; toDecompressor enc contents <$ forceBSL contents @@ -51,6 +55,13 @@ decompressFile inp o = BSL.writeFile o =<< f inp where f = toFileDecompressor (detectCompression inp) +decompressDetectFile :: FilePath -- ^ Compressed file + -> FilePath -- ^ Output + -> IO () +decompressDetectFile inp o = do + f <- toFileDecompressor <$> (detectFileCompression inp) + BSL.writeFile o =<< (f inp) + compressFile :: CompressionLevel -> FilePath -- ^ Input file -> FilePath -- ^ Compressed output @@ -68,10 +79,10 @@ run :: Command -> IO () run (Recompress i lvl) = recompressFile lvl =<< reifyPath i run (Decompress i Nothing) = flip decompressFile (uncompressedExt i) =<< reifyPath i -run (Decompress i (Just o)) = flip decompressFile o =<< reifyPath i +run (Decompress i (Just o)) = flip decompressDetectFile o =<< reifyPath i run (Compress i o lvl) = flip (compressFile lvl) o =<< reifyPath i run (Transcode i o lvl) = do - let cO = detectCompression o + cO <- detectFileCompression o guessSz <- case cO of Lzip -> Just . (8*) . fromIntegral <$> fileSize i _ -> pure Nothing @@ -80,6 +91,10 @@ run (Matrix inp lvl) = parallel_ (compressMatrix lvl inp <$> [Lzma .. Lz4]) *> stopGlobalPool +run (Info fps) = + forM_ fps $ \fp -> + putStr (fp ++ ": ") *> + (print =<< detectFileCompression fp) fileCompletions :: HasCompleter f => Mod f a fileCompletions = completer (bashCompleter "file -o plusdirs") @@ -87,6 +102,9 @@ inpFile :: Parser FilePath inpFile = fileHelp "Input file" +infoFile :: Parser Command +infoFile = Info <$> some (fileHelp "Archive") + verify :: Parser Command verify = Verify <$> inpFile @@ -161,6 +179,7 @@ <> command "verify" (info verify (progDesc "Check the integrity of a compressed file")) <> command "matrix" (info matrix (progDesc "Compress a file to all available formats")) <> command "recompress" (info recompress (progDesc "Recompress a file (for instance, to compress it at a higher level)")) + <> command "info" (info infoFile (progDesc "Guess file compression")) ) versionMod :: Parser (a -> a) diff --git a/src/Version.cpphs b/src/Version.cpphs --- a/src/Version.cpphs +++ b/src/Version.cpphs @@ -6,6 +6,7 @@ import Codec.Lzip import qualified Data.Version as V import qualified Paths_sak as P +import Version.Foreign allVersionsString :: String allVersionsString = @@ -13,8 +14,11 @@ ++ "lzlib-hs: " ++ VERSION_lzlib ++ "\n" ++ "lzlib: " ++ lZVersion ++ "\n" ++ "lzlib API: " ++ show (lZApiVersion :: Int) ++ "\n" + ++ "zlib: " ++ zlib ++ "\n" ++ "zlib-hs: " ++ VERSION_zlib ++ "\n" + ++ "lzma: " ++ lzma ++ "\n" ++ "lzma-hs: " ++ VERSION_lzma ++ "\n" + ++ "zstd: " ++ zstd ++ "\n" ++ "zstd-hs: " ++ VERSION_zstd ++ "\n" ++ "bz2-hs: " ++ VERSION_bz2 ++ "\n" ++ "bz2: " ++ bZ2BzlibVersion ++ "\n" diff --git a/src/Version/Foreign.hs b/src/Version/Foreign.hs new file mode 100644 --- /dev/null +++ b/src/Version/Foreign.hs @@ -0,0 +1,20 @@ +module Version.Foreign ( zstd + , zlib + , lzma + ) where + +import Foreign.C.String (CString, peekCString) +import System.IO.Unsafe (unsafeDupablePerformIO) + +foreign import ccall unsafe "ZSTD_versionString" zstdVersionString :: CString +foreign import ccall unsafe zlibVersion :: CString +foreign import ccall unsafe lzma_version_string :: CString + +zstd :: String +zstd = unsafeDupablePerformIO (peekCString zstdVersionString) + +zlib :: String +zlib = unsafeDupablePerformIO (peekCString zlibVersion) + +lzma :: String +lzma = unsafeDupablePerformIO (peekCString lzma_version_string)