hnix-store-nar 0.1.1.0 → 0.1.2.0

raw patch · 14 files changed

+477/−91 lines, 14 filesdep +cryptondep −cryptonitedep ~basedep ~tasty-hspecPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: crypton

Dependencies removed: cryptonite

Dependency ranges changed: base, tasty-hspec

API changes (from Hackage documentation)

Files

• CHANGELOG.md +5/−0
• README.md +3/−0
• benchmarks/NarStream.hs +66/−0
• benchmarks/README.md +57/−0
• benchmarks/prepare-hyperfine.sh +85/−0
• benchmarks/run-hyperfine.sh +83/−0
• benchmarks/shell.nix +13/−0
• hnix-store-nar.cabal +22/−4
• src/System/Nix/Nar.hs +7/−7
• src/System/Nix/Nar/Effects.hs +6/−6
• src/System/Nix/Nar/Options.hs +1/−1
• src/System/Nix/Nar/Parser.hs +25/−25
• src/System/Nix/Nar/Streamer.hs +38/−31
• tests/NarFormat.hs +66/−17

@@ -1,3 +1,8 @@+# [0.1.2.0](https://github.com/haskell-nix/hnix-store/compare/nar-0.1.1.0...nar-0.1.2.0)  2026-08-10++* Switch from `cryptonite` to `crypton`+* Preserve non-UTF-8 filename and symlink target bytes when streaming NARs [#320](https://github.com/haskell-nix/hnix-store/pull/320)+ # [0.1.1.0](https://github.com/haskell-nix/hnix-store/compare/nar-0.1.0.0...nar-0.1.1.0)  2024-10-09  * Fix ordering of case-hacked paths on macOS [#286](https://github.com/haskell-nix/hnix-store/pull/286)

@@ -4,5 +4,8 @@  For a description of the NAR format, see [`Eelco's thesis`](https://nixos.org/~eelco/pubs/phd-thesis.pdf). +The [NAR streaming benchmark](./benchmarks/README.md) provides a reproducible+Hyperfine comparison between the working tree and a baseline Git revision.+ [System.Nix.Nar]: ./src/System/Nix/Nar.hs [System.Nix.Nar.Effects]: ./src/System/Nix/Nar/Effects.hs

@@ -0,0 +1,66 @@+{-# LANGUAGE ImportQualifiedPost #-}++module Main (main) where++import Control.Monad (forM_, replicateM_, unless)+import Data.ByteString qualified as Bytes+import Data.IORef (modifyIORef', newIORef, readIORef)+import System.Directory qualified as Directory+import System.Environment (getArgs)+import System.Exit (die)+import System.FilePath ((</>))+import Text.Printf (printf)+import Text.Read (readMaybe)++import System.Nix.Nar.Streamer (dumpPath)++main :: IO ()+main = do+  args <- getArgs+  case args of+    ["prepare", fixturePath, fileCountText] -> do+      fileCount <- readPositive "file count" fileCountText+      prepareFixture fixturePath fileCount+    ["run", fixturePath, iterationsText] -> do+      iterations <- readPositive "iteration count" iterationsText+      runBenchmark fixturePath iterations+    [] -> putStrLn usage+    _ -> die usage++usage :: String+usage = unlines+  [ "Usage: nar-stream prepare FIXTURE_PATH FILE_COUNT"+  , "       nar-stream run FIXTURE_PATH ITERATIONS"+  ]++readPositive :: String -> String -> IO Int+readPositive label input =+  case readMaybe input of+    Just value | value > 0 -> pure value+    _ -> die $ label <> " must be a positive integer"++prepareFixture :: FilePath -> Int -> IO ()+prepareFixture fixturePath fileCount = do+  exists <- Directory.doesPathExist fixturePath+  if exists+    then die $ "fixture path already exists: " <> fixturePath+    else Directory.createDirectory fixturePath++  forM_ [1 .. fileCount] $ \index ->+    Bytes.writeFile (fixturePath </> fileName index) Bytes.empty+ where+  fileName :: Int -> FilePath+  fileName = printf "file-%08d"++runBenchmark :: FilePath -> Int -> IO ()+runBenchmark fixturePath iterations = do+  exists <- Directory.doesDirectoryExist fixturePath+  unless exists $ die $ "fixture directory does not exist: " <> fixturePath++  byteCount <- newIORef (0 :: Int)+  replicateM_ iterations $+    dumpPath fixturePath $ \chunk ->+      modifyIORef' byteCount (+ Bytes.length chunk)++  streamedBytes <- readIORef byteCount+  unless (streamedBytes > 0) $ die "NAR streamer produced no output"

@@ -0,0 +1,57 @@+# NAR streaming wall-time benchmark++This benchmark compares the working tree's NAR streamer with a baseline Git+revision. Both binaries archive the same pre-created wide directory, so fixture+creation, compilation, and cleanup are excluded from Hyperfine's measurements.++## 1. Prepare++From the repository root:++```console+nix-shell hnix-store-nar/benchmarks/shell.nix --run \+  'hnix-store-nar/benchmarks/prepare-hyperfine.sh'+```++The default baseline is `HEAD`. This is appropriate while the fix is still an+uncommitted working-tree change. After committing the fix, select its parent or+another known revision explicitly:++```console+HNIX_NAR_BASELINE_REF=HEAD^ \+  nix-shell hnix-store-nar/benchmarks/shell.nix --run \+    'hnix-store-nar/benchmarks/prepare-hyperfine.sh'+```++Preparation creates ignored state under `benchmark-results/nar-stream-state`:+the two prebuilt runners, a shared 5,000-file fixture, build metadata,+and the working-tree patch. It does not run Hyperfine.++## 2. Measure when the machine is idle++```console+nix-shell hnix-store-nar/benchmarks/shell.nix --run \+  'hnix-store-nar/benchmarks/run-hyperfine.sh'+```++The runner refuses to start when one-minute load divided by the CPU count is+above `0.5`. It pins both binaries to the same CPU, performs five warmups and 30+measurements, then repeats the comparison in reverse order to expose drift.+JSON and Markdown reports are written below the prepared state's `results/`+directory.++Useful overrides:++```console+HNIX_NAR_FILES=10000             # preparation only+HNIX_NAR_ITERATIONS=10           # dumps per measured process+HNIX_NAR_WARMUPS=5+HNIX_NAR_RUNS=30+HNIX_NAR_CPU=14+HNIX_NAR_MAX_LOAD_PER_CPU=0.5+HNIX_NAR_STATE_DIR=/some/path+```++This is a warm-cache benchmark of NAR traversal and encoding. Do not clear the+machine's page cache between runs; doing so needs elevated privileges, disrupts+other work, and measures a different workload.

@@ -0,0 +1,85 @@+#!/usr/bin/env bash+set -euo pipefail++benchmark_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)+repo_root=$(git -C "$benchmark_dir" rev-parse --show-toplevel)+state_dir=${HNIX_NAR_STATE_DIR:-"$repo_root/benchmark-results/nar-stream-state"}+baseline_ref=${HNIX_NAR_BASELINE_REF:-HEAD}+file_count=${HNIX_NAR_FILES:-5000}++if [[ -e "$state_dir" ]]; then+  echo "Benchmark state already exists: $state_dir" >&2+  echo "Set HNIX_NAR_STATE_DIR to a new path or remove the old state first." >&2+  exit 1+fi++scratch_dir=$(mktemp -d "${TMPDIR:-/tmp}/hnix-nar-prepare.XXXXXX")+baseline_tree="$scratch_dir/baseline"+baseline_added=false++cleanup() {+  if [[ "$baseline_added" == true ]]; then+    git -C "$repo_root" worktree remove --force "$baseline_tree" >/dev/null 2>&1 || true+  fi+  rm -rf -- "$scratch_dir"+}+trap cleanup EXIT++mkdir -p "$state_dir/bin"++build_runner() {+  local source_tree=$1+  local label=$2+  local build_dir="$scratch_dir/dist-$label"+  local object_dir="$scratch_dir/objects-$label"++  mkdir -p "$object_dir"+  (+    cd "$source_tree"+    cabal build hnix-store-nar \+      --builddir="$build_dir" \+      --disable-tests \+      --disable-benchmarks \+      -j1+    cabal exec --builddir="$build_dir" -- \+      ghc -O2 -threaded -rtsopts "-with-rtsopts=-N1" \+        -odir "$object_dir" \+        -hidir "$object_dir" \+        -package bytestring \+        -package directory \+        -package filepath \+        -package hnix-store-nar \+        "$benchmark_dir/NarStream.hs" \+        -o "$state_dir/bin/$label"+  )+}++echo "Building fixed benchmark runner..."+build_runner "$repo_root" fixed++echo "Creating baseline worktree at $baseline_ref..."+git -C "$repo_root" worktree add --detach "$baseline_tree" "$baseline_ref"+baseline_added=true++echo "Building baseline benchmark runner..."+build_runner "$baseline_tree" baseline++echo "Creating the shared $file_count-file fixture..."+"$state_dir/bin/fixed" prepare "$state_dir/fixture" "$file_count"++baseline_commit=$(git -C "$baseline_tree" rev-parse HEAD)+fixed_commit=$(git -C "$repo_root" rev-parse HEAD)+{+  printf 'baseline_ref=%s\n' "$baseline_ref"+  printf 'baseline_commit=%s\n' "$baseline_commit"+  printf 'fixed_commit=%s\n' "$fixed_commit"+  printf 'file_count=%s\n' "$file_count"+  printf 'ghc=%s\n' "$(ghc --numeric-version)"+  printf 'cabal=%s\n' "$(cabal --numeric-version)"+} > "$state_dir/metadata.txt"+git -C "$repo_root" diff --binary > "$state_dir/fixed.patch"++echo+echo "Benchmark state is ready: $state_dir"+echo "When the machine is idle, run:"+echo "  nix-shell $benchmark_dir/shell.nix --run '$benchmark_dir/run-hyperfine.sh'"

@@ -0,0 +1,83 @@+#!/usr/bin/env bash+set -euo pipefail++benchmark_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)+repo_root=$(git -C "$benchmark_dir" rev-parse --show-toplevel)+state_dir=${HNIX_NAR_STATE_DIR:-"$repo_root/benchmark-results/nar-stream-state"}+iterations=${HNIX_NAR_ITERATIONS:-7}+warmups=${HNIX_NAR_WARMUPS:-5}+runs=${HNIX_NAR_RUNS:-30}+max_load_per_cpu=${HNIX_NAR_MAX_LOAD_PER_CPU:-0.5}+cpu_count=$(nproc)+default_cpu=$((cpu_count > 1 ? cpu_count - 2 : 0))+benchmark_cpu=${HNIX_NAR_CPU:-$default_cpu}++baseline_bin="$state_dir/bin/baseline"+fixed_bin="$state_dir/bin/fixed"+fixture="$state_dir/fixture"++for required_path in "$baseline_bin" "$fixed_bin" "$fixture" "$state_dir/metadata.txt"; do+  if [[ ! -e "$required_path" ]]; then+    echo "Missing benchmark state: $required_path" >&2+    echo "Run prepare-hyperfine.sh first." >&2+    exit 1+  fi+done++check_load() {+  local load_one+  load_one=$(awk '{ print $1 }' /proc/loadavg)+  if ! awk \+    -v current_load="$load_one" \+    -v cpus="$cpu_count" \+    -v maximum="$max_load_per_cpu" \+    'BEGIN { exit !((current_load / cpus) <= maximum) }'+  then+    echo "Refusing to benchmark: load is $load_one across $cpu_count CPUs." >&2+    echo "Wait for load/CPU <= $max_load_per_cpu, or override HNIX_NAR_MAX_LOAD_PER_CPU." >&2+    exit 75+  fi+}++if ! taskset -c "$benchmark_cpu" true; then+  echo "CPU $benchmark_cpu is not available to taskset." >&2+  exit 1+fi++check_load++timestamp=$(date -u +%Y%m%dT%H%M%SZ)+results_dir="$state_dir/results/$timestamp"+mkdir -p "$results_dir"++baseline_command="taskset -c $benchmark_cpu $baseline_bin run $fixture $iterations +RTS -N1 -A64m -RTS"+fixed_command="taskset -c $benchmark_cpu $fixed_bin run $fixture $iterations +RTS -N1 -A64m -RTS"++common_options=(+  --warmup "$warmups"+  --runs "$runs"+  --shell=none+)++echo "Benchmarking baseline first on CPU $benchmark_cpu..."+hyperfine \+  "${common_options[@]}" \+  --export-json "$results_dir/baseline-first.json" \+  --export-markdown "$results_dir/baseline-first.md" \+  --command-name baseline "$baseline_command" \+  --command-name fixed "$fixed_command"++check_load++echo "Benchmarking fixed first on CPU $benchmark_cpu..."+hyperfine \+  "${common_options[@]}" \+  --export-json "$results_dir/fixed-first.json" \+  --export-markdown "$results_dir/fixed-first.md" \+  --command-name fixed "$fixed_command" \+  --command-name baseline "$baseline_command"++cp "$state_dir/metadata.txt" "$results_dir/metadata.txt"++echo+echo "Benchmark results: $results_dir"

@@ -0,0 +1,13 @@+{ pkgs ? (import ../../default.nix {}).pkgs }:++let+  narShell = (import ../../shell.nix { inherit pkgs; }).hnix-store-nar;+in+pkgs.mkShell {+  inputsFrom = [ narShell ];+  packages = [+    pkgs.cabal-install+    pkgs.hyperfine+    pkgs.util-linux+  ];+}

@@ -1,6 +1,6 @@ cabal-version:       2.2 name:                hnix-store-nar-version:             0.1.1.0+version:             0.1.2.0 synopsis:            NAR file format description:   Packing and unpacking for NAR file format used by Nix.@@ -16,6 +16,10 @@     CHANGELOG.md extra-source-files:     README.md+  , benchmarks/README.md+  , benchmarks/prepare-hyperfine.sh+  , benchmarks/run-hyperfine.sh+  , benchmarks/shell.nix   , tests/fixtures/case-conflict.nar  flag bounded_memory@@ -40,6 +44,7 @@     , FlexibleContexts     , FlexibleInstances     , GADTs+    , ImportQualifiedPost     , StandaloneDeriving     , ScopedTypeVariables     , StandaloneDeriving@@ -68,7 +73,7 @@     , System.Nix.Nar.Options   build-depends:       base >=4.12 && <5-    , algebraic-graphs >= 0.5 && < 0.8+    , algebraic-graphs >= 0.5 && < 0.9     , bytestring     , case-insensitive     , cereal@@ -98,7 +103,7 @@     tasty-discover:tasty-discover   build-depends:       base-    , cryptonite+    , crypton     , hnix-store-nar     , base64-bytestring     , cereal@@ -110,8 +115,21 @@     , process     , temporary     , tasty-    , tasty-hspec+    , tasty-hspec >= 1.1     , tasty-hunit     , tasty-quickcheck     , text     , unix++benchmark nar-stream+  import: commons+  type: exitcode-stdio-1.0+  main-is: NarStream.hs+  hs-source-dirs: benchmarks+  ghc-options: -O2 -threaded -rtsopts "-with-rtsopts -N1"+  build-depends:+      base+    , bytestring+    , directory+    , filepath+    , hnix-store-nar

@@ -33,14 +33,14 @@   , Nar.NarSource   ) where -import qualified Control.Concurrent                as Concurrent-import qualified Data.ByteString                   as BS-import qualified System.IO                         as IO+import Control.Concurrent qualified                as Concurrent+import Data.ByteString qualified                   as BS+import System.IO qualified                         as IO -import qualified System.Nix.Nar.Effects   as Nar-import qualified System.Nix.Nar.Options   as Nar-import qualified System.Nix.Nar.Parser    as Nar-import qualified System.Nix.Nar.Streamer  as Nar+import System.Nix.Nar.Effects qualified   as Nar+import System.Nix.Nar.Options qualified   as Nar+import System.Nix.Nar.Parser qualified    as Nar+import System.Nix.Nar.Streamer qualified  as Nar  -- For a description of the NAR format, see Eelco's thesis -- https://nixos.org/~eelco/pubs/phd-thesis.pdf

@@ -16,10 +16,10 @@ import Data.Kind (Type) import System.IO (Handle, IOMode(WriteMode)) -import qualified Control.Monad-import qualified Data.ByteString-import qualified Data.ByteString.Lazy        as Bytes.Lazy-import qualified System.Directory            as Directory+import Control.Monad qualified+import Data.ByteString qualified+import Data.ByteString.Lazy qualified        as Bytes.Lazy+import System.Directory qualified            as Directory import           System.Posix.Files          ( createSymbolicLink                                              , fileMode                                              , fileSize@@ -37,8 +37,8 @@                                              , setFileMode                                              , unionFileModes                                              )-import qualified System.IO                   as IO-import qualified Control.Exception.Lifted    as Exception.Lifted+import System.IO qualified                   as IO+import Control.Exception.Lifted qualified    as Exception.Lifted  data IsExecutable = NonExecutable | Executable   deriving (Eq, Show)

@@ -6,7 +6,7 @@   ) where  import Data.Text (Text)-import qualified System.Info+import System.Info qualified  -- | Options for configuring how NAR files are encoded and decoded. data NarOptions = NarOptions {

@@ -11,42 +11,42 @@   ) where  -import qualified Algebra.Graph                   as Graph-import qualified Algebra.Graph.ToGraph           as Graph-import qualified Control.Concurrent              as Concurrent-import qualified Control.Exception.Lifted        as Exception.Lifted+import Algebra.Graph qualified                   as Graph+import Algebra.Graph.ToGraph qualified           as Graph+import Control.Concurrent qualified              as Concurrent+import Control.Exception.Lifted qualified        as Exception.Lifted import           Control.Monad                    ( forM                                                   , when                                                   , forM_                                                   )-import qualified Control.Monad.Except            as Except-import qualified Control.Monad.Fail              as Fail-import qualified Control.Monad.IO.Class          as IO-import qualified Control.Monad.Reader            as Reader-import qualified Control.Monad.State             as State-import qualified Control.Monad.Trans             as Trans-import qualified Control.Monad.Trans.Control     as Base+import Control.Monad.Except qualified            as Except+import Control.Monad.Fail qualified              as Fail+import Control.Monad.IO.Class qualified          as IO+import Control.Monad.Reader qualified            as Reader+import Control.Monad.State qualified             as State+import Control.Monad.Trans qualified             as Trans+import Control.Monad.Trans.Control qualified     as Base import           Data.ByteString                  (ByteString)-import qualified Data.ByteString                 as Bytes+import Data.ByteString qualified                 as Bytes import           Data.Bool                        ( bool )-import qualified Data.Either                     as Either+import Data.Either qualified                     as Either import           Data.Int                         ( Int64 )-import qualified Data.IORef                      as IORef-import qualified Data.CaseInsensitive            as CI-import qualified Data.HashMap.Strict             as HashMap-import qualified Data.List                       as List-import qualified Data.Map                        as Map+import Data.IORef qualified                      as IORef+import Data.CaseInsensitive qualified            as CI+import Data.HashMap.Strict qualified             as HashMap+import Data.List qualified                       as List+import Data.Map qualified                        as Map import           Data.Maybe                       ( catMaybes )-import qualified Data.Serialize                  as Serialize+import Data.Serialize qualified                  as Serialize import           Data.Text                        ( Text )-import qualified Data.Text                       as Text-import qualified Data.Text.Encoding              as Text-import qualified System.Directory                as Directory+import Data.Text qualified                       as Text+import Data.Text.Encoding qualified              as Text+import System.Directory qualified                as Directory import           System.FilePath                 as FilePath-import qualified System.IO                       as IO+import System.IO qualified                       as IO -import qualified System.Nix.Nar.Effects as Nar-import qualified System.Nix.Nar.Options as Nar+import System.Nix.Nar.Effects qualified as Nar+import System.Nix.Nar.Options qualified as Nar  -- | NarParser is a monad for parsing a Nar file as a byte stream --   and reconstructing the file system objects inside

@@ -12,23 +12,26 @@  import Data.ByteString (ByteString) import Data.Int (Int64)-import qualified Data.Map.Strict                 as Map+import Data.Map.Strict qualified                 as Map -import           Control.Monad                    ( forM_+import           Control.Monad                    ( forM+                                                  , forM_                                                   , when                                                   )-import qualified Control.Monad.IO.Class          as IO-import qualified Data.ByteString                 as Bytes-import qualified Data.ByteString.Lazy            as Bytes.Lazy-import qualified Data.Foldable-import qualified Data.List-import qualified Data.Serialize                  as Serial-import qualified Data.Text                       as T (pack, unpack)-import qualified Data.Text.Encoding              as TE (encodeUtf8)+import Control.Monad.IO.Class qualified          as IO+import Data.ByteString qualified                 as Bytes+import Data.ByteString.Lazy qualified            as Bytes.Lazy+import Data.Foldable qualified+import Data.List qualified+import Data.Serialize qualified                  as Serial+import Data.Text qualified                       as T (unpack)+import GHC.Foreign qualified                     as Foreign+import GHC.IO.Encoding qualified                 as Encoding import           System.FilePath                 ((</>))+import           System.IO                       (TextEncoding) -import qualified System.Nix.Nar.Effects as Nar-import qualified System.Nix.Nar.Options as Nar+import System.Nix.Nar.Effects qualified as Nar+import System.Nix.Nar.Options qualified as Nar  -- | NarSource -- The source to provide nar to the handler `(ByteString -> m ())`.@@ -78,38 +81,41 @@   -> FilePath   -> NarSource m streamNarIOWithOptions opts effs basePath yield = do+  fileSystemEncoding <- IO.liftIO Encoding.getFileSystemEncoding   yield $ str "nix-archive-1"-  parens $ go basePath+  parens $ go fileSystemEncoding basePath  where-  go :: FilePath -> m ()-  go path = do+  go :: TextEncoding -> FilePath -> m ()+  go fileSystemEncoding path = do     isSymLink <- IO.liftIO $ Nar.narIsSymLink effs path     if isSymLink then do       target <- IO.liftIO $ Nar.narReadLink effs path+      targetBytes <- IO.liftIO $ filePathToBS fileSystemEncoding target       yield $-        strs ["type", "symlink", "target", filePathToBS target]+        strs ["type", "symlink", "target", targetBytes]       else do         isDir <- IO.liftIO $ Nar.narIsDir effs path         if isDir then do           fs <- IO.liftIO (Nar.narListDir effs path)+          names <- IO.liftIO $ forM fs $ \f -> do+            let name =+                  if Nar.optUseCaseHack opts+                  then undoCaseHack f+                  else f+            nameBytes <- filePathToBS fileSystemEncoding name+            pure (nameBytes, name, f)           let entries =-                foldr (\f acc ->-                  let-                    name =-                      if Nar.optUseCaseHack opts-                      then undoCaseHack f-                      else f-                  in-                  case Map.insertLookupWithKey (\_ n _ -> n) name f acc of+                foldr (\(nameBytes, name, original) acc ->+                  case Map.insertLookupWithKey (\_ n _ -> n) nameBytes (name, original) acc of                     (Nothing, newMap) -> newMap-                    (Just conflict, _) -> error $ "File name collision between " ++ (path </> name) ++ " and " ++ (path </> conflict)-                ) Map.empty fs+                    (Just (conflict, _), _) -> error $ "File name collision between " ++ (path </> name) ++ " and " ++ (path </> conflict)+                ) Map.empty names           yield $ strs ["type", "directory"]-          forM_ (Map.toAscList entries) $ \(unhacked, original) -> do+          forM_ (Map.toAscList entries) $ \(nameBytes, (_, original)) -> do             yield $ str "entry"             parens $ do-              yield $ strs ["name", filePathToBS unhacked, "node"]-              parens $ go (path </> original)+              yield $ strs ["name", nameBytes, "node"]+              parens $ go fileSystemEncoding (path </> original)         else do           isExec <- IO.liftIO $ Nar.narIsExec effs path           yield $ strs ["type", "regular"]@@ -151,8 +157,9 @@ strs :: [ByteString] -> ByteString strs xs = Bytes.concat $ str <$> xs -filePathToBS :: FilePath -> ByteString-filePathToBS = TE.encodeUtf8 . T.pack+filePathToBS :: TextEncoding -> FilePath -> IO ByteString+filePathToBS encoding filePath =+  Foreign.withCStringLen encoding filePath Bytes.packCStringLen  undoCaseHack :: FilePath -> FilePath undoCaseHack f =

@@ -4,7 +4,7 @@ module NarFormat where  import           Control.Applicative              (many, optional, (<|>))-import qualified Control.Concurrent               as Concurrent+import Control.Concurrent qualified               as Concurrent import           Control.Exception                (SomeException, try) import           Control.Monad                    (replicateM, void, forM_, when) import           Crypto.Hash                      (hash, Digest, SHA256)@@ -15,35 +15,38 @@ import           Data.Serialize                   (Putter, putInt64le,                                                    putByteString, runPut) import           Data.Bool                        (bool)-import qualified Data.ByteString                  as BS-import qualified Data.ByteString.Base64           as B64-import qualified Data.ByteString.Char8            as BSC-import qualified Data.ByteString.Lazy             as BSL-import qualified Data.ByteString.Lazy.Char8       as BSLC+import Data.ByteString qualified                  as BS+import Data.ByteString.Base64 qualified           as B64+import Data.ByteString.Char8 qualified            as BSC+import Data.ByteString.Lazy qualified             as BSL+import Data.ByteString.Lazy.Char8 qualified       as BSLC import           Data.Int                         ( Int64 )-import qualified Data.Map                         as Map+import Data.Map qualified                         as Map import           Data.Maybe                       (fromMaybe)-import qualified Data.Text                        as T-import qualified Data.Text.Encoding               as E+import Data.Text qualified                        as T+import Data.Text.Encoding qualified               as E import           GHC.Generics                     ( Generic ) import           System.Directory                 ( doesDirectoryExist                                                   , doesPathExist                                                   , removeDirectoryRecursive                                                   , removeFile                                                   )-import qualified System.Directory                 as Directory+import System.Directory qualified                 as Directory import           System.Environment               (getEnv) import           System.FilePath                  ((<.>), (</>))-import qualified System.IO                        as IO-import qualified System.IO.Temp                   as Temp-import qualified System.Posix.Files               as Unix-import qualified System.Posix.Process             as Unix-import qualified System.Process                   as P+import System.Info qualified                      as Info+import System.IO qualified                        as IO+import System.IO.Temp qualified                   as Temp+import System.Posix.Files qualified               as Unix+import System.Posix.Files.ByteString qualified    as UnixFilesBS+import System.Posix.IO.ByteString qualified       as UnixBS+import System.Posix.Process qualified             as Unix+import System.Process qualified                   as P import           Test.Tasty                       as T import           Test.Hspec-import qualified Test.Tasty.HUnit                 as HU+import Test.Tasty.HUnit qualified                 as HU import           Test.Tasty.QuickCheck-import qualified Text.Printf                      as Printf+import Text.Printf qualified                      as Printf import           Text.Read                        (readMaybe)  import System.Nix.Nar.Streamer (IsExecutable(Executable, NonExecutable))@@ -143,6 +146,52 @@  unit_nixStoreDirectory' :: HU.Assertion unit_nixStoreDirectory' = filesystemNixStore "directory'" (Nar sampleDirectory')++unit_nixStoreNonUtf8FilePaths :: HU.Assertion+unit_nixStoreNonUtf8FilePaths+  -- APFS only permits valid UTF-8 filenames, so the fixture cannot be+  -- constructed on macOS. Linux still exercises the filesystem integration.+  | Info.os == "darwin" = pure ()+  | otherwise = Temp.withSystemTempDirectory "hnix-store-non-utf8" $ \baseDir -> do+    let rawBaseDir = BSC.pack baseDir+        rawFileNames =+          [ "NetLock_Arany_=Class_Gold=_F" <> BS.pack [0xf5]+              <> "tan" <> BS.pack [0xfa, 0x73, 0xed, 0x74, 0x76, 0xe1]+              <> "ny.crt"+          -- These two names sort in the opposite order after the invalid byte+          -- is decoded to a surrogate, so they also check raw-byte ordering.+          , "sort-" <> BS.pack [0x80]+          , "sort-" <> BS.pack [0xc2, 0x80]+          ]+        rawLinkTarget = "broken-target-" <> BS.pack [0xff]+        rawLinkName = "link"++    forM_ rawFileNames $ \rawFileName -> do+      fd <- UnixBS.createFile+        (rawBaseDir <> "/" <> rawFileName)+        Unix.ownerReadMode+      UnixBS.closeFd fd+    UnixFilesBS.createSymbolicLink+      rawLinkTarget+      (rawBaseDir <> "/" <> rawLinkName)++    hnixNar <- Temp.withSystemTempFile "hnix-store-non-utf8.nar" $ \narPath h -> do+      buildNarIO narEffectsIO baseDir h+      IO.hClose h+      BS.readFile narPath++    forM_ (rawLinkTarget : rawFileNames) $ \rawPath ->+      rawPath `BS.isInfixOf` hnixNar `shouldBe` True++    ver <- try (P.readProcess "nix-store" ["--version"] "")+    case ver of+      Left (_ :: SomeException) -> print ("No nix-store on system" :: String)+      Right _ -> do+        nixStoreNar <- BSL.toStrict <$> getNixStoreDump baseDir+        HU.assertEqual+          "non-UTF-8 paths serialize the same between hnix-store and nix-store"+          nixStoreNar+          hnixNar  -- | Test that the executable permissions are handled correctly in app bundles on macOS. --   In this case, access() returns false for a file under this specific path, even when the executable bit is set.