diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# 0.1.0.2 + + * Add support for brotli (`.br`) compressed archives with `with-brotli` flag + * Add `sanitize` subcommand diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +# hstar + +`hstar` is a command-line tool like [tar](https://www.gnu.org/software/tar/), +but written in Haskell and taking advantage of +[backpack](http://blog.ezyang.com/category/haskell/backpack/). + +It can be built with +a [libarchive](http://hackage.haskell.org/package/libarchive) backend wrapping +the C library, or the [tar](http://hackage.haskell.org/package/tar) library +written in Haskell. + +## Use + +To generate bash shell completions: + +``` +hstar --bash-completion-script hstar +``` + +## Performance + +Benchmarks run on Linux. + +`hstar` built with the `libarchive` backend. + +| Program | Compression | Command | Time | +| ------- | ----------- | ------- | ---: | +| bsdtar | lz4 | `bsdtar -xf ghc-8.8.2-x86_64-deb9-linux.tar.lz4` | 2.411 s | +| hstar | lz4 | `hstar unpack ghc-8.8.2-x86_64-deb9-linux.tar.lz4` | 2.529 s | +| GNU tar | lz4 | `lz4 -cd ghc-8.8.2-x86_64-deb9-linux.tar.lz4 \| tar -xf -` | 2.539 s | +| Schily tar | lz4 | `lz4 -cd ghc-8.8.2-x86_64-deb9-linux.tar.lz4 \| star -xf -` | 1.572 s | +| bsdtar | gzip | `bsdtar -xf ghc-8.8.2-x86_64-deb9-linux.tar.gz` | 4.625 s | +| hstar | gzip | `hstar unpack ghc-8.8.2-x86_64-deb9-linux.tar.gz` | 6.378 s | +| GNU tar | gzip | `tar xf ghc-8.8.2-x86_64-deb9-linux.tar.gz` | 8.335 s | +| Schily tar | gzip | `star xf ghc-8.8.2-x86_64-deb9-linux.tar.gz` | 8.379 s | +| bsdtar | lzip | `bsdtar -xf ghc-8.8.2-x86_64-deb9-linux.tar.lz` | 12.49 s | +| hstar | lzip | `hstar unpack ghc-8.8.2-x86_64-deb9-linux.tar.lz` | 16.12 s | +| GNU tar | lzip | `tar xf ghc-8.8.2-x86_64-deb9-linux.tar.lz` | 16.71 s | +| Schily tar | lzip | `star xf ghc-8.8.2-x86_64-deb9-linux.tar.lz` | 15.81 s | +| bsdtar | lzma | `bsdtar -xf ghc-8.8.2-x86_64-deb9-linux.tar.xz` | 14.95 s | +| hstar | lzma | `hstar unpack ghc-8.8.2-x86_64-deb9-linux.tar.xz` | 15.37 s | +| GNU tar | lzma | `tar xf ghc-8.8.2-x86_64-deb9-linux.tar.xz` | 14.32 s | +| Schily tar | lzma | `star xf ghc-8.8.2-x86_64-deb9-linux.tar.xz` | 14.37 s | +| bsdtar | bzip2 | `bsdtar -xf ghc-8.8.2-x86_64-deb9-linux.tar.bz2` | 37.85 s | +| hstar | bzip2 | `hstar unpack ghc-8.8.2-x86_64-deb9-linux.tar.bz2` | 38.99 s | +| GNU tar | bzip2 | `tar xf ghc-8.8.2-x86_64-deb9-linux.tar.bz2` | 37.90 s | +| Schily tar | bzip2 | `star xf ghc-8.8.2-x86_64-deb9-linux.tar.bz2` | 38.27 s | +| bsdtar | zstd | `bsdtar -xf ghc-8.8.2-x86_64-deb9-linux.tar.zst` | 2.232 s | +| hstar | zstd | `hstar unpack ghc-8.8.2-x86_64-deb9-linux.tar.zst` | 3.053 s | +| GNU tar | zstd | `tar xf ghc-8.8.2-x86_64-deb9-linux.tar.zst` | 2.644 s | +| Schily tar | zstd | `star xf ghc-8.8.2-x86_64-deb9-linux.tar.zst` | 2.237 s | diff --git a/hstar.cabal b/hstar.cabal --- a/hstar.cabal +++ b/hstar.cabal @@ -1,17 +1,20 @@ -cabal-version: 2.0 -name: hstar -version: 0.1.0.1 -license: BSD3 -license-file: LICENSE -copyright: Copyright: (c) 2019-2020 Vanessa McHale -maintainer: vamchale@gmail.com -author: Vanessa McHale -synopsis: Haskell version of tar CLI utility +cabal-version: 2.0 +name: hstar +version: 0.1.0.2 +license: BSD3 +license-file: LICENSE +copyright: Copyright: (c) 2019-2020 Vanessa McHale +maintainer: vamchale@gmail.com +author: Vanessa McHale +synopsis: Haskell version of tar CLI utility description: Haskell implementation of the tar utility, demonstrating backpack -category: Codec, Tar, Archive -build-type: Simple +category: Codec, Tar, Archive +build-type: Simple +extra-source-files: + CHANGELOG.md + README.md source-repository head type: git @@ -27,6 +30,13 @@ description: Use Haskell backend instead of libarchive default: False +flag tar-bytestring + description: Use tar-bytestring + default: False + +flag with-brotli + description: Enable brotli compression support + executable hstar main-is: Main.hs build-tool-depends: cpphs:cpphs -any @@ -54,19 +64,31 @@ lz4-hs >=0.1.1.0, lzlib >=1.0.1.0 + if flag(with-brotli) + build-depends: brotli -any + + if flag(with-brotli) + cpp-options: -DBROTLI + if impl(ghc >=8.10) ghc-options: -Wunused-packages if flag(pure) build-depends: archive-tar -any - else + if flag(tar-bytestring) + build-depends: archive-tar-bytestring -any + + if (!flag(pure) && !flag(tar-bytestring)) build-depends: archive-libarchive -any if flag(pure) mixins: archive-tar (Archive.Tar as Archive) - else + if flag(tar-bytestring) + mixins: archive-tar-bytestring (Archive.Tar as Archive) + + if (!flag(pure) && !flag(tar-bytestring)) mixins: archive-libarchive (Archive.FFI as Archive) if flag(development) diff --git a/src/Compression.cpphs b/src/Compression.cpphs new file mode 100644 --- /dev/null +++ b/src/Compression.cpphs @@ -0,0 +1,76 @@ +module Compression ( compressionByFileExt + , decompressor + , compressor + ) where + +#ifdef BROTLI +import qualified Codec.Compression.Brotli as Brotli +#endif +import qualified Codec.Compression.BZip as BZip +import qualified Codec.Compression.GZip as GZip +import qualified Codec.Compression.Lzma as Lzma +import qualified Codec.Compression.Zlib as Zlib +import qualified Codec.Compression.Zstd.Lazy as Zstd +import qualified Codec.Lz4 as Lz4 +import Codec.Lzip as Lzip +import qualified Data.ByteString.Lazy as BSL +import Data.List (isSuffixOf) + +data Compressor = Lzma + | Lz + | Bz2 + | GZip + | Zstd + | Deflate + | Lz4 +#ifdef BROTLI + | Brotli +#endif + | None + +compressionByFileExt :: FilePath -> Compressor +compressionByFileExt fp | ".tgz" `isSuffixOf` fp = GZip + | ".tar.bz2" `isSuffixOf` fp = Bz2 + | ".tar.bz" `isSuffixOf` fp = Bz2 + | ".tbz2" `isSuffixOf` fp = Bz2 + | ".tbz" `isSuffixOf` fp = Bz2 + | ".tar.gz" `isSuffixOf` fp = GZip + | ".tar.xz" `isSuffixOf` fp = Lzma + | ".txz" `isSuffixOf` fp = Lzma + | ".tar.lz" `isSuffixOf` fp = Lz + | ".tlz" `isSuffixOf` fp = Lz + | ".tar.zst" `isSuffixOf` fp = Zstd + | ".tar.Z" `isSuffixOf` fp = Deflate + | ".tar.lz4" `isSuffixOf` fp = Lz4 +#ifdef BROTLI + | ".tbr" `isSuffixOf` fp = Brotli + | ".tar.br" `isSuffixOf` fp = Brotli +#endif + | ".tar" `isSuffixOf` fp = None + | otherwise = error "Suffix not supported or invalid." + +decompressor :: Compressor -> (BSL.ByteString -> BSL.ByteString) +decompressor Lzma = Lzma.decompress +decompressor Bz2 = BZip.decompress +decompressor GZip = GZip.decompress +decompressor Lz = Lzip.decompress +decompressor Zstd = Zstd.decompress +decompressor Deflate = Zlib.decompress +decompressor Lz4 = Lz4.decompress +#ifdef BROTLI +decompressor Brotli = Brotli.decompress +#endif +decompressor None = id + +compressor :: Compressor -> (BSL.ByteString -> BSL.ByteString) +compressor Lzma = Lzma.compress +compressor Bz2 = BZip.compress +compressor GZip = GZip.compress +compressor Lz = Lzip.compress +compressor Zstd = Zstd.compress 3 +compressor Deflate = Zlib.compress +compressor Lz4 = Lz4.compress +#ifdef BROTLI +compressor Brotli = Brotli.compress +#endif +compressor None = id diff --git a/src/Compression.hs b/src/Compression.hs deleted file mode 100644 --- a/src/Compression.hs +++ /dev/null @@ -1,60 +0,0 @@ -module Compression ( compressionByFileExt - , decompressor - , compressor - ) where - -import qualified Codec.Compression.BZip as BZip -import qualified Codec.Compression.GZip as GZip -import qualified Codec.Compression.Lzma as Lzma -import qualified Codec.Compression.Zlib as Zlib -import qualified Codec.Compression.Zstd.Lazy as Zstd -import qualified Codec.Lz4 as Lz4 -import Codec.Lzip as Lzip -import qualified Data.ByteString.Lazy as BSL -import Data.List (isSuffixOf) - -data Compressor = Lzma - | Lz - | Bz2 - | GZip - | Zstd - | Deflate - | Lz4 - | None - -compressionByFileExt :: FilePath -> Compressor -compressionByFileExt fp | ".tgz" `isSuffixOf` fp = GZip - | ".tar.bz2" `isSuffixOf` fp = Bz2 - | ".tar.bz" `isSuffixOf` fp = Bz2 - | ".tbz2" `isSuffixOf` fp = Bz2 - | ".tbz" `isSuffixOf` fp = Bz2 - | ".tar.gz" `isSuffixOf` fp = GZip - | ".tar.xz" `isSuffixOf` fp = Lzma - | ".txz" `isSuffixOf` fp = Lzma - | ".tar.lz" `isSuffixOf` fp = Lz - | ".tlz" `isSuffixOf` fp = Lz - | ".tar.zst" `isSuffixOf` fp = Zstd - | ".tar.Z" `isSuffixOf` fp = Deflate - | ".tar.lz4" `isSuffixOf` fp = Lz4 - | ".tar" `isSuffixOf` fp = None - | otherwise = error "Suffix not supported or invalid." - -decompressor :: Compressor -> (BSL.ByteString -> BSL.ByteString) -decompressor Lzma = Lzma.decompress -decompressor Bz2 = BZip.decompress -decompressor GZip = GZip.decompress -decompressor Lz = Lzip.decompress -decompressor Zstd = Zstd.decompress -decompressor Deflate = Zlib.decompress -decompressor Lz4 = Lz4.decompress -decompressor None = id - -compressor :: Compressor -> (BSL.ByteString -> BSL.ByteString) -compressor Lzma = Lzma.compress -compressor Bz2 = BZip.compress -compressor GZip = GZip.compress -compressor Lz = Lzip.compress -compressor Zstd = Zstd.compress 3 -compressor Deflate = Zlib.compress -compressor Lz4 = Lz4.compress -compressor None = id diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -1,8 +1,11 @@ module Main ( main ) where +import Archive import Archive.Compression import Compression -import Data.Maybe (fromMaybe) +import Control.Exception (throw) +import qualified Data.ByteString.Lazy as BSL +import Data.Maybe (fromMaybe) import Options.Applicative import Version @@ -11,8 +14,22 @@ | Pack ![FilePath] !FilePath | Unpack !FilePath !(Maybe FilePath) | PackSrc !FilePath !FilePath + | Sanitize !FilePath +forceBSL :: BSL.ByteString -> IO () +forceBSL = (`seq` mempty) . last . BSL.toChunks + +sanitize :: FilePath -> IO () +sanitize fp = do + let enc = compressionByFileExt fp + contents <- decompressor enc <$> BSL.readFile fp + forceBSL contents + let es = either throw id $ readArchiveBytes contents + paxContents = writeArchiveBytes es + BSL.writeFile fp (compressor enc paxContents) + run :: Command -> IO () +run (Sanitize src) = sanitize src run (Unpack src dest) = let dec = decompressor (compressionByFileExt src) in unpackFileToDirAndDecompress dec src (fromMaybe "." dest) @@ -26,6 +43,13 @@ let comp = compressor (compressionByFileExt tar) in packSrcDirAndCompress comp dir' tar +sanitizeP :: Parser Command +sanitizeP = Sanitize + <$> argument str + (metavar "SRC" + <> fileCompletions + <> help "Archive to pax-ify") + unpack :: Parser Command unpack = Unpack <$> argument str @@ -81,6 +105,7 @@ <> command "pack-dir" (info packDir (progDesc "Pack a directory's contents into an archive")) <> command "pack" (info pack (progDesc "Pack an archive from a list of files")) <> command "pack-src" (info packSrc (progDesc "Pack up a source directory as a bundle, ignoring version control and artifact directories")) + <> command "sanitize" (info sanitizeP (progDesc "Sanitize a tar archive so it is pax-compatible")) ) versionMod :: Parser (a -> a) diff --git a/src/Version.cpphs b/src/Version.cpphs --- a/src/Version.cpphs +++ b/src/Version.cpphs @@ -21,5 +21,8 @@ ++ "lz4-hs: " ++ VERSION_lz4_hs ++ "\n" ++ "lz4: " ++ lZ4VersionString ++ "\n" ++ "zstd-hs: " ++ VERSION_zstd ++ "\n" +#ifdef BROTLI + ++ "brotli-hs: " ++ VERSION_brotli ++ "\n" +#endif ++ "bz2-hs: " ++ VERSION_bz2 ++ "\n" ++ "bz2: " ++ bZ2BzlibVersion