diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # sak +## 0.1.2.3 + + * Add `recompress` subcommand + +## 0.1.2.2 + + * Add `with-brotli` cabal package flag to enable/disable brotli support + ## 0.1.2.1 * Add brotli format support (`.br`) diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -19,3 +19,7 @@ ``` You can put this in your `~/.bashrc` or `~/.bash_profile` as needed. + +### Manpages + +`man/sak.1` contains manpages for `sak` diff --git a/man/sak.1 b/man/sak.1 --- a/man/sak.1 +++ b/man/sak.1 @@ -14,7 +14,9 @@ .PP sak verify file.gz .PP -sak matrix release-binary \[en]best +sak matrix release-binary --best +.PP +sak recompress binary.tar.zst --best .SH SUBCOMMANDS .PP \f[B]compress\f[R] - Compress a file @@ -27,6 +29,8 @@ \f[B]verify\f[R] - Check the integrity of a compressed file .PP \f[B]matrix\f[R] - Generate multiple compressed files in various formats +.PP +\f[B]recompress\f[R] - Recompress a file in-memory. .SH OPTIONS .TP \f[B]-h\f[R] \f[B]--help\f[R] diff --git a/sak.cabal b/sak.cabal --- a/sak.cabal +++ b/sak.cabal @@ -1,6 +1,6 @@ cabal-version: 1.18 name: sak -version: 0.1.2.1 +version: 0.1.2.3 license: BSD3 license-file: LICENSE copyright: Copyright: (c) 2020 Vanessa McHale @@ -21,6 +21,10 @@ type: darcs location: https://hub.darcs.net/vmchale/sak +flag with-brotli + description: + Build with support for brotli encoding/decoding (disable this to statically link) + executable sak main-is: Main.hs build-tool-depends: cpphs:cpphs -any @@ -45,8 +49,13 @@ filepath -any, bytestring -any, directory >=1.3.1.0, - parallel-io -any, - brotli -any + parallel-io -any + + if flag(with-brotli) + build-depends: brotli -any + + if flag(with-brotli) + cpp-options: -DBROTLI if impl(ghc >=8.0) ghc-options: diff --git a/src/Compression.cpphs b/src/Compression.cpphs new file mode 100644 --- /dev/null +++ b/src/Compression.cpphs @@ -0,0 +1,203 @@ +module Compression ( Compression (Lzma, Lz4, Lzip, BZip) + , CompressionLevel (..) + , detectCompression + , toCompressor + , toDecompressor + , toFileCompressor + , toFileDecompressor + , uncompressedExt + , check + , ext + -- * Utilities + , fileSize + ) where + +#ifdef BROTLI +import qualified Codec.Compression.Brotli as Br +#endif +import qualified Codec.Compression.BZip as BZip +import qualified Codec.Compression.GZip as GZip +import qualified Codec.Compression.Lzma as Lzma +import qualified Codec.Compression.Zlib as Zlib +import qualified Codec.Compression.Zstd.Lazy as Zstd +import qualified Codec.Lz4 as Lz4 +import qualified Codec.Lzip as Lzip +import qualified Data.ByteString.Lazy as BSL +import Data.List (isSuffixOf) +import System.FilePath (dropExtension, (-<.>)) +import System.IO (IOMode (ReadMode), hFileSize, withFile) + +data CompressionLevel = Best + | Fastest + | Default + | Custom !Int + +data Compression = Lzma + | Lzip + | BZip + | GZip + | Zstd +#ifdef BROTLI + | Brotli +#endif + | Lz4 + | Z + | None + deriving (Enum) + +ext :: Compression -> String +ext Lzma = ".xz" +ext Lzip = ".lz" +ext BZip = ".bz2" +ext GZip = ".gz" +ext Zstd = ".zst" +ext Lz4 = ".lz4" +ext Z = ".Z" +#ifdef BROTLI +ext Brotli = ".br" +#endif +ext None = "" + +uncompressedExt :: FilePath -> FilePath +uncompressedExt fp | ".tlz" `isSuffixOf` fp = fp -<.> ".tar" + | ".txz" `isSuffixOf` fp = fp -<.> ".tar" + | ".tlz" `isSuffixOf` fp = fp -<.> ".tar" + | ".tgz" `isSuffixOf` fp = fp -<.> ".tar" + | ".cpgz" `isSuffixOf` fp = fp -<.> ".cpio" + | ".cpbz2" `isSuffixOf` fp = fp -<.> ".cpio" + | ".xcfgz" `isSuffixOf` fp = fp -<.> ".xcf" + | ".xcfbz2" `isSuffixOf` fp = fp -<.> ".xcf" + | ".tbz2" `isSuffixOf` fp = fp -<.> "tar" + | ".tbz" `isSuffixOf` fp = fp -<.> "tar" + | ".tbr" `isSuffixOf` fp = fp -<.> "tar" + | otherwise = dropExtension fp + +detectCompression :: FilePath -> Compression +detectCompression fp + | ".xz" `isSuffixOf` fp = Lzma + | ".txz" `isSuffixOf` fp = Lzma + | ".lz" `isSuffixOf` fp = Lzip + | ".tlz" `isSuffixOf` fp = Lzip + | ".tbz2" `isSuffixOf` fp = BZip + | ".tbz" `isSuffixOf` fp = BZip + | ".bz2" `isSuffixOf` fp = BZip + | ".gz" `isSuffixOf` fp = GZip + | ".tgz" `isSuffixOf` fp = GZip + | ".Z" `isSuffixOf` fp = Z + | ".zst" `isSuffixOf` fp = Zstd + | ".lz4" `isSuffixOf` fp = Lz4 + | ".xcfgz" `isSuffixOf` fp = GZip + | ".xcfbz2" `isSuffixOf` fp = BZip + | ".cpgz" `isSuffixOf` fp = GZip + | ".cpbz2" `isSuffixOf` fp = BZip +#ifdef BROTLI + | ".br" `isSuffixOf` fp = Brotli +#endif + | otherwise = None -- error "Suffix not supported or invalid" + +toFileDecompressor :: Compression -> FilePath -> IO BSL.ByteString +toFileDecompressor Lz4 = lz4DecompressFile +toFileDecompressor x = fmap (toDecompressor x) . BSL.readFile + +toDecompressor :: Compression -> BSL.ByteString -> BSL.ByteString +toDecompressor Lzma = Lzma.decompress +toDecompressor Lzip = Lzip.decompress +toDecompressor BZip = BZip.decompress +toDecompressor GZip = GZip.decompress +toDecompressor Z = Zlib.decompress +toDecompressor Zstd = Zstd.decompress +toDecompressor Lz4 = Lz4.decompress +#ifdef BROTLI +toDecompressor Brotli = Br.decompress +#endif +toDecompressor None = id + +check :: Compression -> BSL.ByteString -> IO () +check = (forceBSL .) . toDecompressor + +levelGuard :: (Int, Int) -> Int -> Int +levelGuard (min', max') lvl | lvl < min' || lvl > max' = + error ("Invalid compression level. Compression must be between " ++ show min' ++ " and " ++ show max') + | otherwise = lvl + +toFileCompressor :: Compression -> CompressionLevel -> FilePath -> IO BSL.ByteString +toFileCompressor Lzip lvl = Lzip.compressFileLevel (toEnum $ toInt Lzip lvl) +toFileCompressor x lvl = fmap (toCompressor x lvl Nothing) . BSL.readFile + +gzipCompression :: Int -> GZip.CompressParams +gzipCompression lvl = GZip.defaultCompressParams { GZip.compressLevel = GZip.compressionLevel lvl } + +zlibCompression :: Int -> Zlib.CompressParams +zlibCompression lvl = Zlib.defaultCompressParams { Zlib.compressLevel = Zlib.compressionLevel lvl } + +lzmaCompression :: Int -> Lzma.CompressParams +lzmaCompression lvl = Lzma.defaultCompressParams { Lzma.compressLevel = toEnum lvl } + +#ifdef BROTLI +brotliCompression :: Int -> Br.CompressParams +brotliCompression lvl = Br.defaultCompressParams { Br.compressLevel = toEnum lvl } +#endif + +toInt :: Compression -> CompressionLevel -> Int +#ifdef BROTLI +toInt Brotli Fastest = fromEnum (minBound :: Br.CompressionLevel) +toInt Brotli (Custom i) = levelGuard (0,11) i +toInt Brotli _ = fromEnum (maxBound :: Br.CompressionLevel) +#endif +toInt Zstd Best = Zstd.maxCLevel +toInt Zstd Fastest = 1 +toInt Zstd (Custom i) = levelGuard (1, Zstd.maxCLevel) i +toInt Zstd Default = 3 +toInt Lzip Best = fromEnum (maxBound :: Lzip.CompressionLevel) +toInt Lzip Fastest = fromEnum (minBound :: Lzip.CompressionLevel) +toInt Lzip (Custom i) = levelGuard (0, 9) i +toInt Lzip Default = 6 +toInt Lzma Best = 9 +toInt Lzma Fastest = 0 +toInt Lzma (Custom i) = levelGuard (0, 9) i +toInt Lzma Default = 6 +toInt BZip Best = 9 +toInt BZip Fastest = 1 +toInt BZip (Custom i) = i +toInt BZip Default = 7 +toInt GZip Best = 9 +toInt GZip Fastest = 0 +toInt GZip (Custom i) = i +toInt GZip Default = 6 +toInt Z Best = 9 +toInt Z Fastest = 0 +toInt Z (Custom i) = i +toInt Z Default = 6 +toInt Lz4 Best = Lz4.lZ4HCClevelMax +toInt Lz4 Fastest = 0 +toInt Lz4 (Custom i) = levelGuard (0, Lz4.lZ4HCClevelMax) i +toInt Lz4 Default = 0 -- 1? +toInt None _ = error "Internal error." + +toCompressor :: Compression -> CompressionLevel -> Maybe Int -> BSL.ByteString -> BSL.ByteString +toCompressor Lzma lvl _ = Lzma.compressWith (lzmaCompression $ toInt Lzma lvl) +toCompressor Lzip lvl (Just sz) = flip (Lzip.compressWithSz (toEnum $ toInt Lzip lvl)) sz +toCompressor Lzip _ _ = error "Internal error." +toCompressor BZip lvl _ = BZip.compressWith (fromIntegral $ toInt BZip lvl) 30 +toCompressor GZip lvl _ = GZip.compressWith (gzipCompression $ toInt GZip lvl) +toCompressor Z lvl _ = Zlib.compressWith (zlibCompression $ toInt Z lvl) +toCompressor Zstd lvl _ = Zstd.compress (toInt Zstd lvl) +toCompressor Lz4 lvl _ = Lz4.compressSz (toInt Lz4 lvl) +#ifdef BROTLI +toCompressor Brotli lvl _ = Br.compressWith (brotliCompression $ toInt Brotli lvl) +#endif +toCompressor None _ _ = id + +fileSize :: FilePath -> IO Integer +fileSize fp = withFile fp ReadMode hFileSize + +lz4DecompressFile :: FilePath -> IO BSL.ByteString +lz4DecompressFile fp = do + fSz <- fileSize fp + let f = if fSz <= 32 * 1024 + then Lz4.decompressBufSz (32 * 1024) + else Lz4.decompressBufSz (128 * 1024) + f <$> BSL.readFile fp + +forceBSL :: BSL.ByteString -> IO () +forceBSL bsl = last (BSL.toChunks bsl) `seq` mempty diff --git a/src/Compression.hs b/src/Compression.hs deleted file mode 100644 --- a/src/Compression.hs +++ /dev/null @@ -1,186 +0,0 @@ -module Compression ( Compression (Lzma,Lz4,Lzip) - , CompressionLevel (..) - , detectCompression - , toCompressor - , toFileCompressor - , toFileDecompressor - , uncompressedExt - , check - , ext - -- * Utilities - , fileSize - ) where - -import qualified Codec.Compression.Brotli as Br -import qualified Codec.Compression.BZip as BZip -import qualified Codec.Compression.GZip as GZip -import qualified Codec.Compression.Lzma as Lzma -import qualified Codec.Compression.Zlib as Zlib -import qualified Codec.Compression.Zstd.Lazy as Zstd -import qualified Codec.Lz4 as Lz4 -import qualified Codec.Lzip as Lzip -import qualified Data.ByteString.Lazy as BSL -import Data.List (isSuffixOf) -import System.FilePath (dropExtension, (-<.>)) -import System.IO (IOMode (ReadMode), hFileSize, withFile) - -data CompressionLevel = Best - | Fastest - | Default - | Custom !Int - -data Compression = Lzma - | Lzip - | BZip - | GZip - | Zstd - | Brotli - | Lz4 - | Z - | None - deriving (Enum) - -ext :: Compression -> String -ext Lzma = ".xz" -ext Lzip = ".lz" -ext BZip = ".bz2" -ext GZip = ".gz" -ext Zstd = ".zst" -ext Lz4 = ".lz4" -ext Z = ".Z" -ext Brotli = ".br" -ext None = "" - -uncompressedExt :: FilePath -> FilePath -uncompressedExt fp | ".tlz" `isSuffixOf` fp = fp -<.> ".tar" - | ".txz" `isSuffixOf` fp = fp -<.> ".tar" - | ".tlz" `isSuffixOf` fp = fp -<.> ".tar" - | ".tgz" `isSuffixOf` fp = fp -<.> ".tar" - | ".cpgz" `isSuffixOf` fp = fp -<.> ".cpio" - | ".cpbz2" `isSuffixOf` fp = fp -<.> ".cpio" - | ".xcfgz" `isSuffixOf` fp = fp -<.> ".xcf" - | ".xcfbz2" `isSuffixOf` fp = fp -<.> ".xcf" - | ".tbz2" `isSuffixOf` fp = fp -<.> "tar" - | ".tbz" `isSuffixOf` fp = fp -<.> "tar" - | ".tbr" `isSuffixOf` fp = fp -<.> "tar" - | otherwise = dropExtension fp - -detectCompression :: FilePath -> Compression -detectCompression fp - | ".xz" `isSuffixOf` fp = Lzma - | ".txz" `isSuffixOf` fp = Lzma - | ".lz" `isSuffixOf` fp = Lzip - | ".tlz" `isSuffixOf` fp = Lzip - | ".tbz2" `isSuffixOf` fp = BZip - | ".tbz" `isSuffixOf` fp = BZip - | ".bz2" `isSuffixOf` fp = BZip - | ".gz" `isSuffixOf` fp = GZip - | ".tgz" `isSuffixOf` fp = GZip - | ".Z" `isSuffixOf` fp = Z - | ".zst" `isSuffixOf` fp = Zstd - | ".lz4" `isSuffixOf` fp = Lz4 - | ".xcfgz" `isSuffixOf` fp = GZip - | ".xcfbz2" `isSuffixOf` fp = BZip - | ".cpgz" `isSuffixOf` fp = GZip - | ".cpbz2" `isSuffixOf` fp = BZip - | ".br" `isSuffixOf` fp = Brotli - | otherwise = None -- error "Suffix not supported or invalid" - -toFileDecompressor :: Compression -> FilePath -> IO BSL.ByteString -toFileDecompressor Lz4 = lz4DecompressFile -toFileDecompressor x = fmap (toDecompressor x) . BSL.readFile - -toDecompressor :: Compression -> BSL.ByteString -> BSL.ByteString -toDecompressor Lzma = Lzma.decompress -toDecompressor Lzip = Lzip.decompress -toDecompressor BZip = BZip.decompress -toDecompressor GZip = GZip.decompress -toDecompressor Z = Zlib.decompress -toDecompressor Zstd = Zstd.decompress -toDecompressor Lz4 = Lz4.decompress -toDecompressor Brotli = Br.decompress -toDecompressor None = id - -check :: Compression -> BSL.ByteString -> IO () -check = (forceBSL .) . toDecompressor - -levelGuard :: (Int, Int) -> Int -> Int -levelGuard (min', max') lvl | lvl < min' || lvl > max' = - error ("Invalid compression level. Compression must be between " ++ show min' ++ " and " ++ show max') - | otherwise = lvl - -toFileCompressor :: Compression -> CompressionLevel -> FilePath -> IO BSL.ByteString -toFileCompressor Lzip lvl = Lzip.compressFileLevel (toEnum $ toInt Lzip lvl) -toFileCompressor x lvl = fmap (toCompressor x lvl Nothing) . BSL.readFile - -gzipCompression :: Int -> GZip.CompressParams -gzipCompression lvl = GZip.defaultCompressParams { GZip.compressLevel = GZip.compressionLevel lvl } - -zlibCompression :: Int -> Zlib.CompressParams -zlibCompression lvl = Zlib.defaultCompressParams { Zlib.compressLevel = Zlib.compressionLevel lvl } - -lzmaCompression :: Int -> Lzma.CompressParams -lzmaCompression lvl = Lzma.defaultCompressParams { Lzma.compressLevel = toEnum lvl } - -brotliCompression :: Int -> Br.CompressParams -brotliCompression lvl = Br.defaultCompressParams { Br.compressLevel = toEnum lvl } - -toInt :: Compression -> CompressionLevel -> Int -toInt Brotli Fastest = fromEnum (minBound :: Br.CompressionLevel) -toInt Brotli (Custom i) = levelGuard (0,11) i -toInt Brotli _ = fromEnum (maxBound :: Br.CompressionLevel) -toInt Zstd Best = Zstd.maxCLevel -toInt Zstd Fastest = 1 -toInt Zstd (Custom i) = levelGuard (1, Zstd.maxCLevel) i -toInt Zstd Default = 3 -toInt Lzip Best = fromEnum (maxBound :: Lzip.CompressionLevel) -toInt Lzip Fastest = fromEnum (minBound :: Lzip.CompressionLevel) -toInt Lzip (Custom i) = levelGuard (0, 9) i -toInt Lzip Default = 6 -toInt Lzma Best = 9 -toInt Lzma Fastest = 0 -toInt Lzma (Custom i) = levelGuard (0, 9) i -toInt Lzma Default = 6 -toInt BZip Best = 9 -toInt BZip Fastest = 1 -toInt BZip (Custom i) = i -toInt BZip Default = 7 -toInt GZip Best = 9 -toInt GZip Fastest = 0 -toInt GZip (Custom i) = i -toInt GZip Default = 6 -toInt Z Best = 9 -toInt Z Fastest = 0 -toInt Z (Custom i) = i -toInt Z Default = 6 -toInt Lz4 Best = Lz4.lZ4HCClevelMax -toInt Lz4 Fastest = 0 -toInt Lz4 (Custom i) = levelGuard (0, Lz4.lZ4HCClevelMax) i -toInt Lz4 Default = 0 -- 1? -toInt None _ = error "Internal error." - -toCompressor :: Compression -> CompressionLevel -> Maybe Int -> BSL.ByteString -> BSL.ByteString -toCompressor Lzma lvl _ = Lzma.compressWith (lzmaCompression $ toInt Lzma lvl) -toCompressor Lzip lvl (Just sz) = flip (Lzip.compressWithSz (toEnum $ toInt Lzip lvl)) sz -toCompressor Lzip _ _ = error "Internal error." -toCompressor BZip lvl _ = BZip.compressWith (fromIntegral $ toInt BZip lvl) 30 -toCompressor GZip lvl _ = GZip.compressWith (gzipCompression $ toInt GZip lvl) -toCompressor Z lvl _ = Zlib.compressWith (zlibCompression $ toInt Z lvl) -toCompressor Zstd lvl _ = Zstd.compress (toInt Zstd lvl) -toCompressor Lz4 lvl _ = Lz4.compressSz (toInt Lz4 lvl) -toCompressor Brotli lvl _ = Br.compressWith (brotliCompression $ toInt Brotli lvl) -toCompressor None _ _ = id - -fileSize :: FilePath -> IO Integer -fileSize fp = withFile fp ReadMode hFileSize - -lz4DecompressFile :: FilePath -> IO BSL.ByteString -lz4DecompressFile fp = do - fSz <- fileSize fp - let f = if fSz <= 32 * 1024 - then Lz4.decompressBufSz (32 * 1024) - else Lz4.decompressBufSz (128 * 1024) - f <$> BSL.readFile fp - -forceBSL :: BSL.ByteString -> IO () -forceBSL bsl = last (BSL.toChunks bsl) `seq` mempty diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -19,10 +19,32 @@ data Command = Decompress !FilePath !(Maybe FilePath) | Compress !FilePath !FilePath !CompressionLevel + | Recompress !FilePath !CompressionLevel | Transcode !FilePath !FilePath !CompressionLevel | Verify !FilePath | Matrix !FilePath !CompressionLevel +forceBSL :: BSL.ByteString -> IO () +forceBSL = (`seq` mempty) . last . BSL.toChunks + +recompressFile :: CompressionLevel + -> FilePath + -> IO () +recompressFile lvl inp = do + let enc = detectCompression inp + pre <- case enc of + BZip -> do { contents <- BSL.readFile inp + ; toDecompressor enc contents <$ forceBSL contents + } + _ -> do { res <- toFileDecompressor enc inp + ; forceBSL res + ; pure res + } + let guessSz = case enc of + Lzip -> Just $ 8 * fromIntegral (BSL.length pre) + _ -> Nothing + BSL.writeFile inp $ toCompressor enc lvl guessSz pre + decompressFile :: FilePath -- ^ Compressed file -> FilePath -- ^ Output -> IO () @@ -44,6 +66,7 @@ BSL.writeFile (inp ++ ext c) =<< toFileCompressor c lvl inp run :: Command -> IO () +run (Recompress i lvl) = recompressFile lvl =<< reifyPath i run (Decompress i Nothing) = flip decompressFile (uncompressedExt i) =<< reifyPath i run (Decompress i (Just o)) = flip decompressFile o =<< reifyPath i run (Compress i o lvl) = flip (compressFile lvl) o =<< reifyPath i @@ -112,6 +135,11 @@ <$> fileHelp "File to compress" <*> compressionLevel +recompress :: Parser Command +recompress = Recompress + <$> fileHelp "File to recompress" + <*> compressionLevel + compress :: Parser Command compress = Compress <$> fileHelp "File to compress" @@ -132,6 +160,7 @@ <> command "transcode" (info transcode (progDesc "Convert a file's compression")) <> command "verify" (info verify (progDesc "Check the integrity of a compressed file")) <> command "matrix" (info matrix (progDesc "Compress a file to all available formats")) + <> command "recompress" (info recompress (progDesc "Recompress a file (for instance, to compress it at a higher level)")) ) versionMod :: Parser (a -> a) diff --git a/src/Version.cpphs b/src/Version.cpphs --- a/src/Version.cpphs +++ b/src/Version.cpphs @@ -19,3 +19,6 @@ ++ "bz2: " ++ bZ2BzlibVersion ++ "\n" ++ "lz4: " ++ lZ4VersionString ++ "\n" ++ "lz4-hs: " ++ VERSION_lz4_hs +#ifdef BROTLI + ++ "\nbrotli-hs: " ++ VERSION_brotli +#endif