diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# [0.1.2.0](https://github.com/haskell-nix/hnix-store/compare/nar-0.1.1.0...nar-0.1.2.0) 2026-08-10 + +* Switch from `cryptonite` to `crypton` +* Preserve non-UTF-8 filename and symlink target bytes when streaming NARs [#320](https://github.com/haskell-nix/hnix-store/pull/320) + # [0.1.1.0](https://github.com/haskell-nix/hnix-store/compare/nar-0.1.0.0...nar-0.1.1.0) 2024-10-09 * Fix ordering of case-hacked paths on macOS [#286](https://github.com/haskell-nix/hnix-store/pull/286) diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -4,5 +4,8 @@ For a description of the NAR format, see [`Eelco's thesis`](https://nixos.org/~eelco/pubs/phd-thesis.pdf). +The [NAR streaming benchmark](./benchmarks/README.md) provides a reproducible +Hyperfine comparison between the working tree and a baseline Git revision. + [System.Nix.Nar]: ./src/System/Nix/Nar.hs [System.Nix.Nar.Effects]: ./src/System/Nix/Nar/Effects.hs diff --git a/benchmarks/NarStream.hs b/benchmarks/NarStream.hs new file mode 100644 --- /dev/null +++ b/benchmarks/NarStream.hs @@ -0,0 +1,66 @@ +{-# LANGUAGE ImportQualifiedPost #-} + +module Main (main) where + +import Control.Monad (forM_, replicateM_, unless) +import Data.ByteString qualified as Bytes +import Data.IORef (modifyIORef', newIORef, readIORef) +import System.Directory qualified as Directory +import System.Environment (getArgs) +import System.Exit (die) +import System.FilePath (()) +import Text.Printf (printf) +import Text.Read (readMaybe) + +import System.Nix.Nar.Streamer (dumpPath) + +main :: IO () +main = do + args <- getArgs + case args of + ["prepare", fixturePath, fileCountText] -> do + fileCount <- readPositive "file count" fileCountText + prepareFixture fixturePath fileCount + ["run", fixturePath, iterationsText] -> do + iterations <- readPositive "iteration count" iterationsText + runBenchmark fixturePath iterations + [] -> putStrLn usage + _ -> die usage + +usage :: String +usage = unlines + [ "Usage: nar-stream prepare FIXTURE_PATH FILE_COUNT" + , " nar-stream run FIXTURE_PATH ITERATIONS" + ] + +readPositive :: String -> String -> IO Int +readPositive label input = + case readMaybe input of + Just value | value > 0 -> pure value + _ -> die $ label <> " must be a positive integer" + +prepareFixture :: FilePath -> Int -> IO () +prepareFixture fixturePath fileCount = do + exists <- Directory.doesPathExist fixturePath + if exists + then die $ "fixture path already exists: " <> fixturePath + else Directory.createDirectory fixturePath + + forM_ [1 .. fileCount] $ \index -> + Bytes.writeFile (fixturePath fileName index) Bytes.empty + where + fileName :: Int -> FilePath + fileName = printf "file-%08d" + +runBenchmark :: FilePath -> Int -> IO () +runBenchmark fixturePath iterations = do + exists <- Directory.doesDirectoryExist fixturePath + unless exists $ die $ "fixture directory does not exist: " <> fixturePath + + byteCount <- newIORef (0 :: Int) + replicateM_ iterations $ + dumpPath fixturePath $ \chunk -> + modifyIORef' byteCount (+ Bytes.length chunk) + + streamedBytes <- readIORef byteCount + unless (streamedBytes > 0) $ die "NAR streamer produced no output" diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,57 @@ +# NAR streaming wall-time benchmark + +This benchmark compares the working tree's NAR streamer with a baseline Git +revision. Both binaries archive the same pre-created wide directory, so fixture +creation, compilation, and cleanup are excluded from Hyperfine's measurements. + +## 1. Prepare + +From the repository root: + +```console +nix-shell hnix-store-nar/benchmarks/shell.nix --run \ + 'hnix-store-nar/benchmarks/prepare-hyperfine.sh' +``` + +The default baseline is `HEAD`. This is appropriate while the fix is still an +uncommitted working-tree change. After committing the fix, select its parent or +another known revision explicitly: + +```console +HNIX_NAR_BASELINE_REF=HEAD^ \ + nix-shell hnix-store-nar/benchmarks/shell.nix --run \ + 'hnix-store-nar/benchmarks/prepare-hyperfine.sh' +``` + +Preparation creates ignored state under `benchmark-results/nar-stream-state`: +the two prebuilt runners, a shared 5,000-file fixture, build metadata, +and the working-tree patch. It does not run Hyperfine. + +## 2. Measure when the machine is idle + +```console +nix-shell hnix-store-nar/benchmarks/shell.nix --run \ + 'hnix-store-nar/benchmarks/run-hyperfine.sh' +``` + +The runner refuses to start when one-minute load divided by the CPU count is +above `0.5`. It pins both binaries to the same CPU, performs five warmups and 30 +measurements, then repeats the comparison in reverse order to expose drift. +JSON and Markdown reports are written below the prepared state's `results/` +directory. + +Useful overrides: + +```console +HNIX_NAR_FILES=10000 # preparation only +HNIX_NAR_ITERATIONS=10 # dumps per measured process +HNIX_NAR_WARMUPS=5 +HNIX_NAR_RUNS=30 +HNIX_NAR_CPU=14 +HNIX_NAR_MAX_LOAD_PER_CPU=0.5 +HNIX_NAR_STATE_DIR=/some/path +``` + +This is a warm-cache benchmark of NAR traversal and encoding. Do not clear the +machine's page cache between runs; doing so needs elevated privileges, disrupts +other work, and measures a different workload. diff --git a/benchmarks/prepare-hyperfine.sh b/benchmarks/prepare-hyperfine.sh new file mode 100644 --- /dev/null +++ b/benchmarks/prepare-hyperfine.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +benchmark_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(git -C "$benchmark_dir" rev-parse --show-toplevel) +state_dir=${HNIX_NAR_STATE_DIR:-"$repo_root/benchmark-results/nar-stream-state"} +baseline_ref=${HNIX_NAR_BASELINE_REF:-HEAD} +file_count=${HNIX_NAR_FILES:-5000} + +if [[ -e "$state_dir" ]]; then + echo "Benchmark state already exists: $state_dir" >&2 + echo "Set HNIX_NAR_STATE_DIR to a new path or remove the old state first." >&2 + exit 1 +fi + +scratch_dir=$(mktemp -d "${TMPDIR:-/tmp}/hnix-nar-prepare.XXXXXX") +baseline_tree="$scratch_dir/baseline" +baseline_added=false + +cleanup() { + if [[ "$baseline_added" == true ]]; then + git -C "$repo_root" worktree remove --force "$baseline_tree" >/dev/null 2>&1 || true + fi + rm -rf -- "$scratch_dir" +} +trap cleanup EXIT + +mkdir -p "$state_dir/bin" + +build_runner() { + local source_tree=$1 + local label=$2 + local build_dir="$scratch_dir/dist-$label" + local object_dir="$scratch_dir/objects-$label" + + mkdir -p "$object_dir" + ( + cd "$source_tree" + cabal build hnix-store-nar \ + --builddir="$build_dir" \ + --disable-tests \ + --disable-benchmarks \ + -j1 + cabal exec --builddir="$build_dir" -- \ + ghc -O2 -threaded -rtsopts "-with-rtsopts=-N1" \ + -odir "$object_dir" \ + -hidir "$object_dir" \ + -package bytestring \ + -package directory \ + -package filepath \ + -package hnix-store-nar \ + "$benchmark_dir/NarStream.hs" \ + -o "$state_dir/bin/$label" + ) +} + +echo "Building fixed benchmark runner..." +build_runner "$repo_root" fixed + +echo "Creating baseline worktree at $baseline_ref..." +git -C "$repo_root" worktree add --detach "$baseline_tree" "$baseline_ref" +baseline_added=true + +echo "Building baseline benchmark runner..." +build_runner "$baseline_tree" baseline + +echo "Creating the shared $file_count-file fixture..." +"$state_dir/bin/fixed" prepare "$state_dir/fixture" "$file_count" + +baseline_commit=$(git -C "$baseline_tree" rev-parse HEAD) +fixed_commit=$(git -C "$repo_root" rev-parse HEAD) +{ + printf 'baseline_ref=%s\n' "$baseline_ref" + printf 'baseline_commit=%s\n' "$baseline_commit" + printf 'fixed_commit=%s\n' "$fixed_commit" + printf 'file_count=%s\n' "$file_count" + printf 'ghc=%s\n' "$(ghc --numeric-version)" + printf 'cabal=%s\n' "$(cabal --numeric-version)" +} > "$state_dir/metadata.txt" +git -C "$repo_root" diff --binary > "$state_dir/fixed.patch" + +echo +echo "Benchmark state is ready: $state_dir" +echo "When the machine is idle, run:" +echo " nix-shell $benchmark_dir/shell.nix --run '$benchmark_dir/run-hyperfine.sh'" diff --git a/benchmarks/run-hyperfine.sh b/benchmarks/run-hyperfine.sh new file mode 100644 --- /dev/null +++ b/benchmarks/run-hyperfine.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +benchmark_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(git -C "$benchmark_dir" rev-parse --show-toplevel) +state_dir=${HNIX_NAR_STATE_DIR:-"$repo_root/benchmark-results/nar-stream-state"} +iterations=${HNIX_NAR_ITERATIONS:-7} +warmups=${HNIX_NAR_WARMUPS:-5} +runs=${HNIX_NAR_RUNS:-30} +max_load_per_cpu=${HNIX_NAR_MAX_LOAD_PER_CPU:-0.5} +cpu_count=$(nproc) +default_cpu=$((cpu_count > 1 ? cpu_count - 2 : 0)) +benchmark_cpu=${HNIX_NAR_CPU:-$default_cpu} + +baseline_bin="$state_dir/bin/baseline" +fixed_bin="$state_dir/bin/fixed" +fixture="$state_dir/fixture" + +for required_path in "$baseline_bin" "$fixed_bin" "$fixture" "$state_dir/metadata.txt"; do + if [[ ! -e "$required_path" ]]; then + echo "Missing benchmark state: $required_path" >&2 + echo "Run prepare-hyperfine.sh first." >&2 + exit 1 + fi +done + +check_load() { + local load_one + load_one=$(awk '{ print $1 }' /proc/loadavg) + if ! awk \ + -v current_load="$load_one" \ + -v cpus="$cpu_count" \ + -v maximum="$max_load_per_cpu" \ + 'BEGIN { exit !((current_load / cpus) <= maximum) }' + then + echo "Refusing to benchmark: load is $load_one across $cpu_count CPUs." >&2 + echo "Wait for load/CPU <= $max_load_per_cpu, or override HNIX_NAR_MAX_LOAD_PER_CPU." >&2 + exit 75 + fi +} + +if ! taskset -c "$benchmark_cpu" true; then + echo "CPU $benchmark_cpu is not available to taskset." >&2 + exit 1 +fi + +check_load + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +results_dir="$state_dir/results/$timestamp" +mkdir -p "$results_dir" + +baseline_command="taskset -c $benchmark_cpu $baseline_bin run $fixture $iterations +RTS -N1 -A64m -RTS" +fixed_command="taskset -c $benchmark_cpu $fixed_bin run $fixture $iterations +RTS -N1 -A64m -RTS" + +common_options=( + --warmup "$warmups" + --runs "$runs" + --shell=none +) + +echo "Benchmarking baseline first on CPU $benchmark_cpu..." +hyperfine \ + "${common_options[@]}" \ + --export-json "$results_dir/baseline-first.json" \ + --export-markdown "$results_dir/baseline-first.md" \ + --command-name baseline "$baseline_command" \ + --command-name fixed "$fixed_command" + +check_load + +echo "Benchmarking fixed first on CPU $benchmark_cpu..." +hyperfine \ + "${common_options[@]}" \ + --export-json "$results_dir/fixed-first.json" \ + --export-markdown "$results_dir/fixed-first.md" \ + --command-name fixed "$fixed_command" \ + --command-name baseline "$baseline_command" + +cp "$state_dir/metadata.txt" "$results_dir/metadata.txt" + +echo +echo "Benchmark results: $results_dir" diff --git a/benchmarks/shell.nix b/benchmarks/shell.nix new file mode 100644 --- /dev/null +++ b/benchmarks/shell.nix @@ -0,0 +1,13 @@ +{ pkgs ? (import ../../default.nix {}).pkgs }: + +let + narShell = (import ../../shell.nix { inherit pkgs; }).hnix-store-nar; +in +pkgs.mkShell { + inputsFrom = [ narShell ]; + packages = [ + pkgs.cabal-install + pkgs.hyperfine + pkgs.util-linux + ]; +} diff --git a/hnix-store-nar.cabal b/hnix-store-nar.cabal --- a/hnix-store-nar.cabal +++ b/hnix-store-nar.cabal @@ -1,6 +1,6 @@ cabal-version: 2.2 name: hnix-store-nar -version: 0.1.1.0 +version: 0.1.2.0 synopsis: NAR file format description: Packing and unpacking for NAR file format used by Nix. @@ -16,6 +16,10 @@ CHANGELOG.md extra-source-files: README.md + , benchmarks/README.md + , benchmarks/prepare-hyperfine.sh + , benchmarks/run-hyperfine.sh + , benchmarks/shell.nix , tests/fixtures/case-conflict.nar flag bounded_memory @@ -40,6 +44,7 @@ , FlexibleContexts , FlexibleInstances , GADTs + , ImportQualifiedPost , StandaloneDeriving , ScopedTypeVariables , StandaloneDeriving @@ -68,7 +73,7 @@ , System.Nix.Nar.Options build-depends: base >=4.12 && <5 - , algebraic-graphs >= 0.5 && < 0.8 + , algebraic-graphs >= 0.5 && < 0.9 , bytestring , case-insensitive , cereal @@ -98,7 +103,7 @@ tasty-discover:tasty-discover build-depends: base - , cryptonite + , crypton , hnix-store-nar , base64-bytestring , cereal @@ -110,8 +115,21 @@ , process , temporary , tasty - , tasty-hspec + , tasty-hspec >= 1.1 , tasty-hunit , tasty-quickcheck , text , unix + +benchmark nar-stream + import: commons + type: exitcode-stdio-1.0 + main-is: NarStream.hs + hs-source-dirs: benchmarks + ghc-options: -O2 -threaded -rtsopts "-with-rtsopts -N1" + build-depends: + base + , bytestring + , directory + , filepath + , hnix-store-nar diff --git a/src/System/Nix/Nar.hs b/src/System/Nix/Nar.hs --- a/src/System/Nix/Nar.hs +++ b/src/System/Nix/Nar.hs @@ -33,14 +33,14 @@ , Nar.NarSource ) where -import qualified Control.Concurrent as Concurrent -import qualified Data.ByteString as BS -import qualified System.IO as IO +import Control.Concurrent qualified as Concurrent +import Data.ByteString qualified as BS +import System.IO qualified as IO -import qualified System.Nix.Nar.Effects as Nar -import qualified System.Nix.Nar.Options as Nar -import qualified System.Nix.Nar.Parser as Nar -import qualified System.Nix.Nar.Streamer as Nar +import System.Nix.Nar.Effects qualified as Nar +import System.Nix.Nar.Options qualified as Nar +import System.Nix.Nar.Parser qualified as Nar +import System.Nix.Nar.Streamer qualified as Nar -- For a description of the NAR format, see Eelco's thesis -- https://nixos.org/~eelco/pubs/phd-thesis.pdf diff --git a/src/System/Nix/Nar/Effects.hs b/src/System/Nix/Nar/Effects.hs --- a/src/System/Nix/Nar/Effects.hs +++ b/src/System/Nix/Nar/Effects.hs @@ -16,10 +16,10 @@ import Data.Kind (Type) import System.IO (Handle, IOMode(WriteMode)) -import qualified Control.Monad -import qualified Data.ByteString -import qualified Data.ByteString.Lazy as Bytes.Lazy -import qualified System.Directory as Directory +import Control.Monad qualified +import Data.ByteString qualified +import Data.ByteString.Lazy qualified as Bytes.Lazy +import System.Directory qualified as Directory import System.Posix.Files ( createSymbolicLink , fileMode , fileSize @@ -37,8 +37,8 @@ , setFileMode , unionFileModes ) -import qualified System.IO as IO -import qualified Control.Exception.Lifted as Exception.Lifted +import System.IO qualified as IO +import Control.Exception.Lifted qualified as Exception.Lifted data IsExecutable = NonExecutable | Executable deriving (Eq, Show) diff --git a/src/System/Nix/Nar/Options.hs b/src/System/Nix/Nar/Options.hs --- a/src/System/Nix/Nar/Options.hs +++ b/src/System/Nix/Nar/Options.hs @@ -6,7 +6,7 @@ ) where import Data.Text (Text) -import qualified System.Info +import System.Info qualified -- | Options for configuring how NAR files are encoded and decoded. data NarOptions = NarOptions { diff --git a/src/System/Nix/Nar/Parser.hs b/src/System/Nix/Nar/Parser.hs --- a/src/System/Nix/Nar/Parser.hs +++ b/src/System/Nix/Nar/Parser.hs @@ -11,42 +11,42 @@ ) where -import qualified Algebra.Graph as Graph -import qualified Algebra.Graph.ToGraph as Graph -import qualified Control.Concurrent as Concurrent -import qualified Control.Exception.Lifted as Exception.Lifted +import Algebra.Graph qualified as Graph +import Algebra.Graph.ToGraph qualified as Graph +import Control.Concurrent qualified as Concurrent +import Control.Exception.Lifted qualified as Exception.Lifted import Control.Monad ( forM , when , forM_ ) -import qualified Control.Monad.Except as Except -import qualified Control.Monad.Fail as Fail -import qualified Control.Monad.IO.Class as IO -import qualified Control.Monad.Reader as Reader -import qualified Control.Monad.State as State -import qualified Control.Monad.Trans as Trans -import qualified Control.Monad.Trans.Control as Base +import Control.Monad.Except qualified as Except +import Control.Monad.Fail qualified as Fail +import Control.Monad.IO.Class qualified as IO +import Control.Monad.Reader qualified as Reader +import Control.Monad.State qualified as State +import Control.Monad.Trans qualified as Trans +import Control.Monad.Trans.Control qualified as Base import Data.ByteString (ByteString) -import qualified Data.ByteString as Bytes +import Data.ByteString qualified as Bytes import Data.Bool ( bool ) -import qualified Data.Either as Either +import Data.Either qualified as Either import Data.Int ( Int64 ) -import qualified Data.IORef as IORef -import qualified Data.CaseInsensitive as CI -import qualified Data.HashMap.Strict as HashMap -import qualified Data.List as List -import qualified Data.Map as Map +import Data.IORef qualified as IORef +import Data.CaseInsensitive qualified as CI +import Data.HashMap.Strict qualified as HashMap +import Data.List qualified as List +import Data.Map qualified as Map import Data.Maybe ( catMaybes ) -import qualified Data.Serialize as Serialize +import Data.Serialize qualified as Serialize import Data.Text ( Text ) -import qualified Data.Text as Text -import qualified Data.Text.Encoding as Text -import qualified System.Directory as Directory +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Text +import System.Directory qualified as Directory import System.FilePath as FilePath -import qualified System.IO as IO +import System.IO qualified as IO -import qualified System.Nix.Nar.Effects as Nar -import qualified System.Nix.Nar.Options as Nar +import System.Nix.Nar.Effects qualified as Nar +import System.Nix.Nar.Options qualified as Nar -- | NarParser is a monad for parsing a Nar file as a byte stream -- and reconstructing the file system objects inside diff --git a/src/System/Nix/Nar/Streamer.hs b/src/System/Nix/Nar/Streamer.hs --- a/src/System/Nix/Nar/Streamer.hs +++ b/src/System/Nix/Nar/Streamer.hs @@ -12,23 +12,26 @@ import Data.ByteString (ByteString) import Data.Int (Int64) -import qualified Data.Map.Strict as Map +import Data.Map.Strict qualified as Map -import Control.Monad ( forM_ +import Control.Monad ( forM + , forM_ , when ) -import qualified Control.Monad.IO.Class as IO -import qualified Data.ByteString as Bytes -import qualified Data.ByteString.Lazy as Bytes.Lazy -import qualified Data.Foldable -import qualified Data.List -import qualified Data.Serialize as Serial -import qualified Data.Text as T (pack, unpack) -import qualified Data.Text.Encoding as TE (encodeUtf8) +import Control.Monad.IO.Class qualified as IO +import Data.ByteString qualified as Bytes +import Data.ByteString.Lazy qualified as Bytes.Lazy +import Data.Foldable qualified +import Data.List qualified +import Data.Serialize qualified as Serial +import Data.Text qualified as T (unpack) +import GHC.Foreign qualified as Foreign +import GHC.IO.Encoding qualified as Encoding import System.FilePath (()) +import System.IO (TextEncoding) -import qualified System.Nix.Nar.Effects as Nar -import qualified System.Nix.Nar.Options as Nar +import System.Nix.Nar.Effects qualified as Nar +import System.Nix.Nar.Options qualified as Nar -- | NarSource -- The source to provide nar to the handler `(ByteString -> m ())`. @@ -78,38 +81,41 @@ -> FilePath -> NarSource m streamNarIOWithOptions opts effs basePath yield = do + fileSystemEncoding <- IO.liftIO Encoding.getFileSystemEncoding yield $ str "nix-archive-1" - parens $ go basePath + parens $ go fileSystemEncoding basePath where - go :: FilePath -> m () - go path = do + go :: TextEncoding -> FilePath -> m () + go fileSystemEncoding path = do isSymLink <- IO.liftIO $ Nar.narIsSymLink effs path if isSymLink then do target <- IO.liftIO $ Nar.narReadLink effs path + targetBytes <- IO.liftIO $ filePathToBS fileSystemEncoding target yield $ - strs ["type", "symlink", "target", filePathToBS target] + strs ["type", "symlink", "target", targetBytes] else do isDir <- IO.liftIO $ Nar.narIsDir effs path if isDir then do fs <- IO.liftIO (Nar.narListDir effs path) + names <- IO.liftIO $ forM fs $ \f -> do + let name = + if Nar.optUseCaseHack opts + then undoCaseHack f + else f + nameBytes <- filePathToBS fileSystemEncoding name + pure (nameBytes, name, f) let entries = - foldr (\f acc -> - let - name = - if Nar.optUseCaseHack opts - then undoCaseHack f - else f - in - case Map.insertLookupWithKey (\_ n _ -> n) name f acc of + foldr (\(nameBytes, name, original) acc -> + case Map.insertLookupWithKey (\_ n _ -> n) nameBytes (name, original) acc of (Nothing, newMap) -> newMap - (Just conflict, _) -> error $ "File name collision between " ++ (path name) ++ " and " ++ (path conflict) - ) Map.empty fs + (Just (conflict, _), _) -> error $ "File name collision between " ++ (path name) ++ " and " ++ (path conflict) + ) Map.empty names yield $ strs ["type", "directory"] - forM_ (Map.toAscList entries) $ \(unhacked, original) -> do + forM_ (Map.toAscList entries) $ \(nameBytes, (_, original)) -> do yield $ str "entry" parens $ do - yield $ strs ["name", filePathToBS unhacked, "node"] - parens $ go (path original) + yield $ strs ["name", nameBytes, "node"] + parens $ go fileSystemEncoding (path original) else do isExec <- IO.liftIO $ Nar.narIsExec effs path yield $ strs ["type", "regular"] @@ -151,8 +157,9 @@ strs :: [ByteString] -> ByteString strs xs = Bytes.concat $ str <$> xs -filePathToBS :: FilePath -> ByteString -filePathToBS = TE.encodeUtf8 . T.pack +filePathToBS :: TextEncoding -> FilePath -> IO ByteString +filePathToBS encoding filePath = + Foreign.withCStringLen encoding filePath Bytes.packCStringLen undoCaseHack :: FilePath -> FilePath undoCaseHack f = diff --git a/tests/NarFormat.hs b/tests/NarFormat.hs --- a/tests/NarFormat.hs +++ b/tests/NarFormat.hs @@ -4,7 +4,7 @@ module NarFormat where import Control.Applicative (many, optional, (<|>)) -import qualified Control.Concurrent as Concurrent +import Control.Concurrent qualified as Concurrent import Control.Exception (SomeException, try) import Control.Monad (replicateM, void, forM_, when) import Crypto.Hash (hash, Digest, SHA256) @@ -15,35 +15,38 @@ import Data.Serialize (Putter, putInt64le, putByteString, runPut) import Data.Bool (bool) -import qualified Data.ByteString as BS -import qualified Data.ByteString.Base64 as B64 -import qualified Data.ByteString.Char8 as BSC -import qualified Data.ByteString.Lazy as BSL -import qualified Data.ByteString.Lazy.Char8 as BSLC +import Data.ByteString qualified as BS +import Data.ByteString.Base64 qualified as B64 +import Data.ByteString.Char8 qualified as BSC +import Data.ByteString.Lazy qualified as BSL +import Data.ByteString.Lazy.Char8 qualified as BSLC import Data.Int ( Int64 ) -import qualified Data.Map as Map +import Data.Map qualified as Map import Data.Maybe (fromMaybe) -import qualified Data.Text as T -import qualified Data.Text.Encoding as E +import Data.Text qualified as T +import Data.Text.Encoding qualified as E import GHC.Generics ( Generic ) import System.Directory ( doesDirectoryExist , doesPathExist , removeDirectoryRecursive , removeFile ) -import qualified System.Directory as Directory +import System.Directory qualified as Directory import System.Environment (getEnv) import System.FilePath ((<.>), ()) -import qualified System.IO as IO -import qualified System.IO.Temp as Temp -import qualified System.Posix.Files as Unix -import qualified System.Posix.Process as Unix -import qualified System.Process as P +import System.Info qualified as Info +import System.IO qualified as IO +import System.IO.Temp qualified as Temp +import System.Posix.Files qualified as Unix +import System.Posix.Files.ByteString qualified as UnixFilesBS +import System.Posix.IO.ByteString qualified as UnixBS +import System.Posix.Process qualified as Unix +import System.Process qualified as P import Test.Tasty as T import Test.Hspec -import qualified Test.Tasty.HUnit as HU +import Test.Tasty.HUnit qualified as HU import Test.Tasty.QuickCheck -import qualified Text.Printf as Printf +import Text.Printf qualified as Printf import Text.Read (readMaybe) import System.Nix.Nar.Streamer (IsExecutable(Executable, NonExecutable)) @@ -143,6 +146,52 @@ unit_nixStoreDirectory' :: HU.Assertion unit_nixStoreDirectory' = filesystemNixStore "directory'" (Nar sampleDirectory') + +unit_nixStoreNonUtf8FilePaths :: HU.Assertion +unit_nixStoreNonUtf8FilePaths + -- APFS only permits valid UTF-8 filenames, so the fixture cannot be + -- constructed on macOS. Linux still exercises the filesystem integration. + | Info.os == "darwin" = pure () + | otherwise = Temp.withSystemTempDirectory "hnix-store-non-utf8" $ \baseDir -> do + let rawBaseDir = BSC.pack baseDir + rawFileNames = + [ "NetLock_Arany_=Class_Gold=_F" <> BS.pack [0xf5] + <> "tan" <> BS.pack [0xfa, 0x73, 0xed, 0x74, 0x76, 0xe1] + <> "ny.crt" + -- These two names sort in the opposite order after the invalid byte + -- is decoded to a surrogate, so they also check raw-byte ordering. + , "sort-" <> BS.pack [0x80] + , "sort-" <> BS.pack [0xc2, 0x80] + ] + rawLinkTarget = "broken-target-" <> BS.pack [0xff] + rawLinkName = "link" + + forM_ rawFileNames $ \rawFileName -> do + fd <- UnixBS.createFile + (rawBaseDir <> "/" <> rawFileName) + Unix.ownerReadMode + UnixBS.closeFd fd + UnixFilesBS.createSymbolicLink + rawLinkTarget + (rawBaseDir <> "/" <> rawLinkName) + + hnixNar <- Temp.withSystemTempFile "hnix-store-non-utf8.nar" $ \narPath h -> do + buildNarIO narEffectsIO baseDir h + IO.hClose h + BS.readFile narPath + + forM_ (rawLinkTarget : rawFileNames) $ \rawPath -> + rawPath `BS.isInfixOf` hnixNar `shouldBe` True + + ver <- try (P.readProcess "nix-store" ["--version"] "") + case ver of + Left (_ :: SomeException) -> print ("No nix-store on system" :: String) + Right _ -> do + nixStoreNar <- BSL.toStrict <$> getNixStoreDump baseDir + HU.assertEqual + "non-UTF-8 paths serialize the same between hnix-store and nix-store" + nixStoreNar + hnixNar -- | Test that the executable permissions are handled correctly in app bundles on macOS. -- In this case, access() returns false for a file under this specific path, even when the executable bit is set.