packages feed

nova-cache 0.12.0.1 → 0.13.0.0

raw patch · 13 files changed

+5430/−2984 lines, 13 filesdep +networkdep +stmdep ~directoryPVP ok

version bump matches the API change (PVP)

Dependencies added: network, stm

Dependency ranges changed: directory

API changes (from Hackage documentation)

- NovaCache.Store: FileStore :: FilePath -> Text -> Int -> FileStore
- NovaCache.Store: [fsPriority] :: FileStore -> Int
- NovaCache.Store: [fsRoot] :: FileStore -> FilePath
- NovaCache.Store: [fsStoreDir] :: FileStore -> Text
- NovaCache.Store: fileStoreAt :: FilePath -> FileStore
- NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.FileStore
+ NovaCache.NAR: NarWalkError :: WalkAccess -> FilePath -> IOException -> NarWalkError
+ NovaCache.NAR: WalkListDirectory :: WalkAccess
+ NovaCache.NAR: WalkReadFile :: WalkAccess
+ NovaCache.NAR: WalkReadLink :: WalkAccess
+ NovaCache.NAR: WalkStatus :: WalkAccess
+ NovaCache.NAR: [nweAccess] :: NarWalkError -> WalkAccess
+ NovaCache.NAR: [nweCause] :: NarWalkError -> IOException
+ NovaCache.NAR: [nwePath] :: NarWalkError -> FilePath
+ NovaCache.NAR: data NarWalkError
+ NovaCache.NAR: data WalkAccess
+ NovaCache.NAR: instance GHC.Internal.Classes.Eq NovaCache.NAR.NarWalkError
+ NovaCache.NAR: instance GHC.Internal.Classes.Eq NovaCache.NAR.WalkAccess
+ NovaCache.NAR: instance GHC.Internal.Exception.Type.Exception NovaCache.NAR.NarWalkError
+ NovaCache.NAR: instance GHC.Internal.Show.Show NovaCache.NAR.NarWalkError
+ NovaCache.NAR: instance GHC.Internal.Show.Show NovaCache.NAR.WalkAccess
+ NovaCache.Reclaim: Reclaimed :: StorePlan -> [(NarFile, NarRemoveResult)] -> [(StagedUpload, UploadRemoveResult)] -> Reclaimed
+ NovaCache.Reclaim: StorePlan :: ReclaimPlan -> UploadPlan -> StorePlan
+ NovaCache.Reclaim: UploadPlan :: [StagedUpload] -> [StagedUpload] -> UploadPlan
+ NovaCache.Reclaim: [rdNars] :: Reclaimed -> [(NarFile, NarRemoveResult)]
+ NovaCache.Reclaim: [rdPlan] :: Reclaimed -> StorePlan
+ NovaCache.Reclaim: [rdUploads] :: Reclaimed -> [(StagedUpload, UploadRemoveResult)]
+ NovaCache.Reclaim: [scanStagedUploads] :: StoreScan -> [StagedUpload]
+ NovaCache.Reclaim: [spNars] :: StorePlan -> ReclaimPlan
+ NovaCache.Reclaim: [spUploads] :: StorePlan -> UploadPlan
+ NovaCache.Reclaim: [upEligible] :: UploadPlan -> [StagedUpload]
+ NovaCache.Reclaim: [upTooYoung] :: UploadPlan -> [StagedUpload]
+ NovaCache.Reclaim: data Reclaimed
+ NovaCache.Reclaim: data StorePlan
+ NovaCache.Reclaim: data UploadPlan
+ NovaCache.Reclaim: instance GHC.Internal.Classes.Eq NovaCache.Reclaim.Reclaimed
+ NovaCache.Reclaim: instance GHC.Internal.Classes.Eq NovaCache.Reclaim.StorePlan
+ NovaCache.Reclaim: instance GHC.Internal.Classes.Eq NovaCache.Reclaim.UploadPlan
+ NovaCache.Reclaim: instance GHC.Internal.Show.Show NovaCache.Reclaim.Reclaimed
+ NovaCache.Reclaim: instance GHC.Internal.Show.Show NovaCache.Reclaim.StorePlan
+ NovaCache.Reclaim: instance GHC.Internal.Show.Show NovaCache.Reclaim.UploadPlan
+ NovaCache.Reclaim: listReclaimable :: FileStore -> NominalDiffTime -> IO UTCTime -> IO (Either ReclaimBlocked StorePlan)
+ NovaCache.Reclaim: planStore :: NominalDiffTime -> StoreScan -> Either ReclaimBlocked StorePlan
+ NovaCache.Reclaim: reclaim :: FileStore -> NominalDiffTime -> IO UTCTime -> IO (Either ReclaimBlocked Reclaimed)
+ NovaCache.Reclaim: selectAbandoned :: NominalDiffTime -> UTCTime -> [StagedUpload] -> UploadPlan
+ NovaCache.Server: NarInfoBadKey :: NarInfoWrite
+ NovaCache.Server: NarInfoNarUnreadable :: IOException -> NarInfoWrite
+ NovaCache.Server: NarInfoRefused :: NarRefusal -> NarInfoWrite
+ NovaCache.Server: NarInfoSignFailed :: String -> NarInfoWrite
+ NovaCache.Server: NarInfoWritten :: NarInfoWrite
+ NovaCache.Server: data NarInfoWrite
+ NovaCache.Server: htmlRootResponse :: ByteString -> IO Response
+ NovaCache.Server: instance GHC.Internal.Classes.Eq NovaCache.Server.NarInfoWrite
+ NovaCache.Server: instance GHC.Internal.Show.Show NovaCache.Server.NarInfoWrite
+ NovaCache.Server: maxUploadTotalsBodySize :: Int
+ NovaCache.Server: parseMinimumAge :: Query -> Either Text NominalDiffTime
+ NovaCache.Server: parseUploadTotals :: ByteString -> Either Text UploadTotals
+ NovaCache.Server: renderUploadTotals :: UploadTotals -> ByteString
+ NovaCache.Server: storeNarInfo :: FileStore -> (NarInfo -> IO (Either String ByteString)) -> Text -> NarInfo -> NarClaim -> IO NarInfoWrite
+ NovaCache.Store: CompletionBadPath :: CompletionResult
+ NovaCache.Store: CompletionPartsMismatch :: Int -> Int -> CompletionResult
+ NovaCache.Store: CompletionSizeMismatch :: Integer -> Integer -> CompletionResult
+ NovaCache.Store: CompletionTooLarge :: Integer -> CompletionResult
+ NovaCache.Store: CompletionUploadUnknown :: CompletionResult
+ NovaCache.Store: NarInfoAbsent :: NarInfoRemoveResult
+ NovaCache.Store: NarInfoRemoveBadPath :: NarInfoRemoveResult
+ NovaCache.Store: NarInfoRemoveFailed :: IOException -> NarInfoRemoveResult
+ NovaCache.Store: NarInfoRemoved :: NarInfoRemoveResult
+ NovaCache.Store: PartBadPath :: PartWriteResult
+ NovaCache.Store: PartOutOfOrder :: Int -> PartWriteResult
+ NovaCache.Store: PartTooLarge :: PartWriteResult
+ NovaCache.Store: PartUploadUnknown :: PartWriteResult
+ NovaCache.Store: PartWritten :: PartWriteResult
+ NovaCache.Store: StagedUpload :: Text -> UploadId -> FilePath -> Integer -> UTCTime -> StagedUpload
+ NovaCache.Store: UploadCompleted :: CompletionResult
+ NovaCache.Store: UploadCompletedStagingKept :: IOException -> CompletionResult
+ NovaCache.Store: UploadRemoveBadPath :: UploadRemoveResult
+ NovaCache.Store: UploadRemoveFailed :: IOException -> UploadRemoveResult
+ NovaCache.Store: UploadRemoved :: UploadRemoveResult
+ NovaCache.Store: UploadTotals :: PartNumber -> Integer -> UploadTotals
+ NovaCache.Store: [suFileName] :: StagedUpload -> Text
+ NovaCache.Store: [suModified] :: StagedUpload -> UTCTime
+ NovaCache.Store: [suPath] :: StagedUpload -> FilePath
+ NovaCache.Store: [suSize] :: StagedUpload -> Integer
+ NovaCache.Store: [suUploadId] :: StagedUpload -> UploadId
+ NovaCache.Store: [utFileSize] :: UploadTotals -> Integer
+ NovaCache.Store: [utParts] :: UploadTotals -> PartNumber
+ NovaCache.Store: beginUpload :: FileStore -> Text -> IO (Maybe UploadId)
+ NovaCache.Store: completeUpload :: FileStore -> Text -> UploadId -> Int -> UploadTotals -> IO CompletionResult
+ NovaCache.Store: data CompletionResult
+ NovaCache.Store: data NarInfoRemoveResult
+ NovaCache.Store: data PartNumber
+ NovaCache.Store: data PartWriteResult
+ NovaCache.Store: data StagedUpload
+ NovaCache.Store: data UploadId
+ NovaCache.Store: data UploadRemoveResult
+ NovaCache.Store: data UploadTotals
+ NovaCache.Store: fsPriority :: FileStore -> Int
+ NovaCache.Store: fsRoot :: FileStore -> FilePath
+ NovaCache.Store: fsStoreDir :: FileStore -> Text
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.CompletionResult
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.NarInfoRemoveResult
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.PartNumber
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.PartWriteResult
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.StagedUpload
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.UploadId
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.UploadRemoveResult
+ NovaCache.Store: instance GHC.Internal.Classes.Eq NovaCache.Store.UploadTotals
+ NovaCache.Store: instance GHC.Internal.Classes.Ord NovaCache.Store.PartNumber
+ NovaCache.Store: instance GHC.Internal.Classes.Ord NovaCache.Store.UploadId
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.CompletionResult
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.NarInfoRemoveResult
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.PartNumber
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.PartWriteResult
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.StagedUpload
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.UploadId
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.UploadRemoveResult
+ NovaCache.Store: instance GHC.Internal.Show.Show NovaCache.Store.UploadTotals
+ NovaCache.Store: listStagedUploads :: FileStore -> IO [StagedUpload]
+ NovaCache.Store: maxUploadParts :: Int
+ NovaCache.Store: parsePartNumber :: Text -> Maybe PartNumber
+ NovaCache.Store: parseUploadId :: Text -> Maybe UploadId
+ NovaCache.Store: removeNarInfo :: FileStore -> Text -> IO NarInfoRemoveResult
+ NovaCache.Store: removeUpload :: FileStore -> Text -> UploadId -> IO UploadRemoveResult
+ NovaCache.Store: renderPartNumber :: PartNumber -> Text
+ NovaCache.Store: renderUploadId :: UploadId -> Text
+ NovaCache.Store: withRemoval :: FileStore -> IO a -> IO a
+ NovaCache.Store: withoutRemoval :: FileStore -> IO a -> IO a
+ NovaCache.Store: writeUploadPart :: FileStore -> Text -> UploadId -> PartNumber -> Int -> IO ByteString -> IO PartWriteResult
- NovaCache.Reclaim: StoreScan :: [(FilePath, NarInfoTarget)] -> [NarFile] -> UTCTime -> StoreScan
+ NovaCache.Reclaim: StoreScan :: [(FilePath, NarInfoTarget)] -> [NarFile] -> [StagedUpload] -> UTCTime -> StoreScan
- NovaCache.Reclaim: scanStore :: FileStore -> IO StoreScan
+ NovaCache.Reclaim: scanStore :: FileStore -> IO UTCTime -> IO StoreScan

Files

CHANGELOG.md view
@@ -1,5 +1,17 @@ # Changelog +## 0.13.0.0 - 2026-11-16++- **The server removes entries and NARs itself: `DELETE /<hash>.narinfo` removes an entry, `POST /reclaim` removes the NARs no narinfo names and the abandoned multipart uploads, and `GET /reclaim` lists what it would remove.** No request removed anything, and the one removal path, `reclaim-nars --delete`, ran on the host against the store directory and shared no lock with the server, so it raced the server twice: a narinfo that arrived after its listing, for a NAR past the minimum age, lost that NAR, as upstream's `nix copy` can send one after skipping a NAR the cache holds, and a NAR uploaded again after its listing was removed despite its new modification time. The README asked for no push in progress during a run. The server now orders its removals against its uploads through a gate each `FileStore` carries: a narinfo upload holds it, alongside other uploads, from the check of its NAR through its write, each rename into `nar/` holds it for the rename, and a reclaim or a narinfo removal holds it alone, the reclaim from its scan through its last removal. So a narinfo is never stored naming a NAR a reclaim removes, and a NAR that arrives during a reclaim lands after it. A removal waiting for the gate lets nothing new take it, so a stream of uploads cannot hold it off; uploads wait while a reclaim reads the store, and no request body is read under the gate. A run's rules are unchanged: a minimum age of two weeks by modification time (`?min-age-hours=N` sets another, and any other query is refused with 400), a narinfo that cannot be resolved or a store with NARs and no narinfo blocks the run with 409, and a removal that fails makes the answer a 500. All three requests need the write key, which can already replace any entry. `DELETE` leaves the NAR to the reclaim, which removes it once nothing names it, as attic's garbage collection reaps a NAR once no object refers to it. A narinfo uploaded for a path the cache holds replaces the entry, which lets a push store a smaller encoding of the same NAR, and the NAR it displaces is reclaimed the same way. Upstream ships no garbage collection for binary caches, so there is no server behavior to match, only its client's: `BinaryCacheStore::narFromPath` turns a missing NAR into `SubstituteGone`, which a substitution logs as a warning before it tries the next substituter (2.35.6), so a client holding a narinfo whose NAR was removed sees a miss, not a failure. One server serves a store directory.+- **`nova-cache-server reclaim-nars` exits nonzero, naming the requests that replace it.** The server removes NARs itself. The subcommand's name is still recognized so that a script running it stops: the server ignores arguments it does not know, and would otherwise start a second server over the store.+- **`FileStore`'s constructor is no longer exported: `newFileStore` makes one, with the gate that orders the store's additions against its removals.** `fsRoot`, `fsStoreDir` and `fsPriority` remain fields, and a copy made by a record update shares the original's gate. `withoutRemoval` and `withRemoval` in `NovaCache.Store` hold it; `writeNar`, `writeNarStreaming` and `completeUpload` take it for their rename, and `removeNarInfo` and `NarInfoRemoveResult` are new. `fileStoreAt` goes, since its one caller was `reclaim-nars`, and `FileStore` loses its `Show` instance. In `NovaCache.Reclaim`, `scanStore` takes the clock it reads after its listings and lists the staged uploads into `StoreScan`'s new `scanStagedUploads`; `planStore`, `StorePlan`, `listReclaimable`, `reclaim` and `Reclaimed` are new. `NovaCache.Server` gains `storeNarInfo`, which checks, signs and writes a narinfo within one hold of the gate, `NarInfoWrite`, and `parseMinimumAge`. `stm` joins the dependencies, for the gate's `TVar`. These changes break the library's API, which is why this is a major version.+- **A NAR can be uploaded in parts, past a proxy's limit on request bodies: `POST /nar/<file>/uploads` begins an upload, `PUT /nar/<file>/uploads/<id>/<n>` sends part `n`, and `POST /nar/<file>/uploads/<id>` joins the parts into `nar/<file>`.** A NAR went up in one `PUT /nar/<file>`, and a proxy in front of the server can refuse a body over its own limit, as Cloudflare does over 100 MB on its Free and Pro plans, so a larger NAR could not reach a cache behind one. Upstream's `HttpBinaryCacheStore` also sends each file in one `PUT` (2.35.6), and only its S3 store uploads in parts, through S3's own API, so there is no upstream HTTP protocol to match. The requests follow S3's multipart upload in plain paths: the server issues the upload's id, parts are numbered from 1, a part sent again replaces the one held, and the completion declares the total size, which S3 makes optional (`x-amz-mp-object-size`) and this server requires. Each request is authenticated as the other writes are. Parts are staged under `uploads/`, outside `nar/`, each streamed to disk in constant memory and renamed into place whole, and they go in order: a part that would leave a gap is refused with 409. The completion's body declares the part count and the total size, as `Parts:` and `FileSize:` lines in narinfo's form, and is refused with 413 when the declared size is over the cap, or with 409 unless the upload holds that many parts and they join to that size; only a join that matches is renamed into `nar/`, so an upload in progress or abandoned never appears there, and the narinfo check verifies the joined file's `FileSize` and `FileHash` as before. An upload's parts together are held to the 4 GB of a single `PUT`, checked as each part arrives and again at completion, since two parts written at once can each pass the first check; an upload holds at most 10,000 parts, S3's limit, and a name too long to stage (its directory's name is the NAR's with the 32-digit id before it) is refused with 400. `POST /reclaim` also removes an upload nothing has been written to for its minimum age: no narinfo can name a staged upload's parts, so removing them never leaves a narinfo without its NAR. `newFileStore` creates `uploads/`. The library gains the upload's store operations in `NovaCache.Store` (`beginUpload`, `writeUploadPart`, `completeUpload`, `listStagedUploads`, `removeUpload`) with the types they take and return, `UploadId` and `PartNumber`'s parse and render functions, and `maxUploadParts`; `selectAbandoned` and `UploadPlan` in `NovaCache.Reclaim`; and the completion body's `parseUploadTotals`, `renderUploadTotals` and `maxUploadTotalsBodySize` in `NovaCache.Server`.+- **`nova-cache-server` serves no landing page of its own: `GET /` answers the library's line of plain text, or, given `--root-page FILE`, that file as HTML.** The executable served cache.novavero.ai's branded page to every operator who ran it, with that deployment's name, styling and links, where the library has kept the root page per deployment through `scRootResponse` all along. A deployment's page is now a file it ships beside its unit, as upstream's cache.nixos.org serves a static page at its root; the public cache's is `deploy/index.html`, styled by novavero.ai's stylesheets. The server reads the file once at startup and refuses to start if it cannot. The executable's page also showed the signing key's public half and a count of stored paths; the plain-text line shows neither, and the server prints the public key in its startup log. The library gains `htmlRootResponse` in `NovaCache.Server`. Nix clients never request `/`, so substitution is unchanged.+- **The server writes its log to stdout line by line.** Under systemd its stdout is a pipe to the journal, where GHC buffered it by block, so the startup lines reached the journal only when the first request's log line flushed it, or, with `LOG_REQUESTS=0`, when a block filled.+- **When `serialiseFromPath` or `withNarSource` fails to read an entry's status, a symlink's target, a directory's listing or a regular file, it throws `NarWalkError`, naming the access, the entry and the `IOException` beneath.** A failure in any of these four accesses escaped as GHC's bare `IOException`, so a caller could tell which access failed on which entry only by parsing GHC's text. Upstream's `PosixSourceAccessor` at 2.24.9 words each access its own way (`opening file '/d/tree/sub/locked': Permission denied`, `reading directory /d/tree/sub: Permission denied`), and a caller reporting failures as upstream does needs the access as a value. `WalkAccess` says which of the four failed (a file's open, size and reads are one access), `nwePath` is the entry's on-disk path, and `nweCause` is the `IOException` itself. `withNarSource` throws a file's read failure from the pull that reads it. Only an `IOException` is wrapped: an asynchronous exception passes through as itself, and whatever an `ExecBitResolver` throws arrives as thrown. A caller that catches `IOException` around a walk no longer catches these failures, which is why this is a major version.+- **`serialiseFromPath` and `withNarSource` refuse a FIFO, a socket or a device with upstream's error at 2.24.9, `file '<path>' has an unsupported type`.** The walk took every entry that is not a directory or a symlink for a regular file, through `doesFileExist`, which is true for anything that is not a directory, so the refusal written for these types never ran. `serialiseFromPath` then read the entry with `ByteString.readFile`, which reads anything that is not a regular file to its end of input: a FIFO with no writer, or `/dev/null`, archived as an empty regular file, and a device with no end, such as `/dev/zero`, was read without bound. `withNarSource` failed at the pull for the entry's contents, with the `IOException` `hFileSize` raises for a file that is not regular, after earlier pulls had returned part of the archive, and a socket failed both entry points as a file read. Each entry's type now comes from one `lstat`, classified by its file mode as upstream's `SourceAccessor::dumpPath` classifies it, and any type but a regular file, a directory or a symlink is refused before it is read; `withNarSource` refuses before its continuation runs. The refusal is an `IOException`, as the walk's other refusals of what it finds are, not a `NarWalkError`: no access failed, and no `IOException` lies beneath it. An entry that vanishes mid-walk fails the access that finds it gone, as a `NarWalkError`, unless the first read to find it gone is `defaultExecBitResolver`'s own status read, which throws its bare `IOException` as any resolver's failure arrives. Windows has no file mode, and its classification is unchanged.+- **A failed `withNarSource` pull stays failed: every later pull rethrows the same exception, as the xz, zstd and bzip2 sources' pulls do.** A pull that found a file shorter than the size it had read marked the source drained before it threw, so the next pull returned the empty chunk that marks a clean end, and a consumer that caught the failure and pulled again took a truncated archive for a complete one. Any exception raised while a pull runs now latches, asynchronous ones included, since a pull interrupted after its read consumed bytes it never returned leaves the archive as torn as a failed one. An asynchronous exception that arrives as a pull returns, after the source has moved past the chunk it was returning, is not latched, so a consumer must not pull again after an asynchronous exception. A pull after any other failure, such as a file that could not be opened, rethrows it as well, where it used to carry on from where the failure left the source. The file a failed pull held open is closed as the failure latches, not when the continuation exits.+ ## 0.12.0.1 - 2026-11-05  - **The README in the package describes 0.12: its version bounds, the server's required `CACHE_API_KEY`, and which codecs are pure.** The README in the 0.12.0.0 tarball, the one Hackage shows, gave bounds that exclude 0.12 itself and a server command that exits at once, since the server refuses to start without `CACHE_API_KEY` unless given `--allow-open-writes`. It also called the WAI server optional, where `NovaCache.Server` is always part of the library and only the executable sits behind `-f server`, and counted one-shot xz decompression and zstd compression among the `IO` operations, where both are pure. The test command now passes `--enable-tests`, as CI's does.@@ -130,9 +142,8 @@   protocol endpoint documentation. The cache protocol routes are unchanged;   the page is briefly cacheable (`max-age=300, must-revalidate`) since its   stats change as paths are added.-- **Correctness and hardening sweep** (a post-0.4.0.0 audit; note: shipped in-  this release without individual entries at tag time - recorded here for the-  honest history):+- **Correctness and hardening** (shipped in this release without individual+  entries at tag time, and recorded here after it):   - `Base32.decode` rejects non-canonical encodings (overflow bits must be zero).   - NAR deserialisation is strict on untrusted input: trailing bytes rejected,     regular nodes require a `contents` token, directory entry names must be
README.md view
@@ -21,13 +21,13 @@ ## Installation  ```cabal-build-depends: nova-cache >= 0.12 && < 0.13+build-depends: nova-cache >= 0.13 && < 0.14 ```  The codecs are separate sublibraries, named alongside the main library:  ```cabal-build-depends: nova-cache:{nova-cache, xz, zstandard} >= 0.12 && < 0.13+build-depends: nova-cache:{nova-cache, xz, zstandard} >= 0.13 && < 0.14 ```  libbz2 and libzstd are bundled. liblzma comes from the `xz` package, which@@ -90,11 +90,16 @@ ```  Without `CACHE_API_KEY` the server refuses to start. `--allow-open-writes` runs-it with unauthenticated writes, for local testing only.+it with unauthenticated writes and removals, for local testing only. +`GET /` answers a line of plain text. `--root-page FILE` serves `FILE` there+as HTML instead, such as a page about the cache; the server reads it once at+startup and refuses to start if it cannot.+ The protocol lives in the `NovaCache.Server` library module as a WAI `Application`, so the cache can be embedded in another server with its own-root page. The bundled executable is one such embedding.+root page. The bundled executable is one such embedding. One server serves a+store directory: it orders its own removals against its own uploads.  ### Configuration @@ -103,7 +108,7 @@ | `PORT` | Listen port (default: 5000; also `--port`) | | `HOST` | Bind host (default: all interfaces; also `--host`) | | `NIX_CACHE_DIR` | Store directory (default: `./nix-cache`; also `--store`) |-| `CACHE_API_KEY` | Bearer token required for `PUT`. The server refuses to start without it unless `--allow-open-writes` is passed. |+| `CACHE_API_KEY` | Bearer token for the requests marked authenticated below. The server refuses to start without it unless `--allow-open-writes` is passed. | | `SIGNING_KEY_FILE` | Ed25519 secret key file for server-side narinfo signing | | `LOG_REQUESTS` | Set to `0` to disable request logging | @@ -111,52 +116,118 @@  | Method | Path | Description | | --- | --- | --- |-| `GET` | `/` | Landing page: live stats and the cache public key |+| `GET` | `/` | A line of plain text, or the `--root-page` file | | `GET` | `/nix-cache-info` | Cache metadata | | `GET` | `/narinfo-hashes` | All cached narinfo hashes, newline-delimited (authenticated) | | `GET` | `/<hash>.narinfo` | Fetch a narinfo | | `GET` | `/nar/<file>` | Fetch a NAR (streamed from disk) | | `PUT` | `/<hash>.narinfo` | Upload a narinfo (authenticated, validated; refused unless the NAR its `URL` names is stored with its `FileSize` and `FileHash`) | | `PUT` | `/nar/<file>` | Upload a NAR (authenticated, streamed to disk) |+| `DELETE` | `/<hash>.narinfo` | Remove a narinfo; its NAR stays until a reclaim (authenticated) |+| `GET` | `/reclaim` | List what `POST /reclaim` would remove (authenticated) |+| `POST` | `/reclaim` | Remove the NARs no narinfo names and abandoned uploads (authenticated) |+| `POST` | `/nar/<file>/uploads` | Begin a multipart upload of a NAR (authenticated) |+| `PUT` | `/nar/<file>/uploads/<id>/<n>` | Upload part `n` of it (authenticated, streamed to disk) |+| `POST` | `/nar/<file>/uploads/<id>` | Complete it: join the parts into `nar/<file>` (authenticated) |  `HEAD` is answered wherever `GET` is. -### Reclaiming unreferenced NARs+### Uploading a NAR in parts -A client uploads each NAR before its narinfo, so a NAR whose narinfo was-refused, or never sent because the push died, stays under `nar/` with nothing-referencing it. No endpoint deletes anything. The `reclaim-nars` subcommand-does, run on the host against the store directory:+A proxy in front of the server can refuse a request body over its own limit,+as Cloudflare does over 100 MB on its Free and Pro plans. A NAR over such a+limit is uploaded in parts, one request each: +1. `POST /nar/<file>/uploads` answers `201` with the upload's id, followed by+   a newline, as the body, and `Location: uploads/<id>`.+2. `PUT /nar/<file>/uploads/<id>/<n>` sends part `n` as the body. Parts are+   numbered from 1 and go in order: part `n` is accepted once parts 1 to+   `n - 1` are held. A part sent again replaces the one held, so a part whose+   response was lost can be sent again.+3. `POST /nar/<file>/uploads/<id>` completes the upload. Its body declares+   how many parts were sent and how many bytes they hold together:++   ```+   Parts: 3+   FileSize: 107200936+   ```++   The server joins the parts into `nar/<file>` only if it holds that many+   parts and they join to that size. Nothing of an upload is under `nar/`+   before then. The narinfo follows as for a NAR sent in one request.++Each part is as large as the client chooses. The parts of an upload together+are held to the 4 GB of a single `PUT /nar/<file>`, and an upload to 10,000+parts.++| Status | Meaning |+| --- | --- |+| `400` | The file name, the part number or the completion body is not valid |+| `401` | The write key is missing or wrong |+| `404` | No such upload of that file: never begun, completed, or reclaimed |+| `409` | The part would leave a gap, or the server holds other parts or another size than the completion declares |+| `413` | The parts would hold more than 4 GB, the completion declares a `FileSize` over 4 GB, or its body is over 1 KB |++A part whose response is lost can be sent again. To a client that sends its+parts in order, a `404` or `409` for a part means the server no longer holds+the parts sent before it, such as an upload a reclaim removed, so it begins a new+upload. Completion joins the whole NAR before it answers, so its+response can be lost, or a retry can meet the first attempt still joining:+whenever the completion is answered with anything but `200`, or not at all, the+client uploads the narinfo, whose check of the NAR's `FileSize` and `FileHash`+decides whether it is stored, and begins a new upload if the narinfo is refused+for want of its NAR.++### Removing entries and reclaiming NARs++`DELETE /<hash>.narinfo` removes an entry. Its NAR stays under `nar/`, still+served, until a reclaim removes it. A narinfo uploaded for a path the cache+holds replaces the entry, as for a smaller encoding of the same NAR, and the+NAR the old narinfo named likewise stays until a reclaim.++A reclaim removes the NARs no narinfo names, and the multipart uploads nothing+has been written to, once each is old enough. `GET /reclaim` lists what+`POST /reclaim` removes. Both answer with a report in plain text.+ ```bash-nova-cache-server reclaim-nars --store /var/lib/nix-cache           # list-nova-cache-server reclaim-nars --store /var/lib/nix-cache --delete  # remove+auth="Authorization: Bearer $CACHE_API_KEY"+curl -fsS -X DELETE -H "$auth" https://cache.example/<hash>.narinfo  # remove an entry+curl -fsS -H "$auth" https://cache.example/reclaim                   # list+curl -fsS -X POST -H "$auth" https://cache.example/reclaim           # remove ``` -- A NAR is eligible only once it is at least `--min-age-hours` old (default-  336, two weeks), measured from its modification time. A NAR is unreferenced-  until its push sends the narinfo, and a push can run for hours.+- A NAR or an upload is eligible once it is at least `min-age-hours` old+  (default 336, two weeks), measured from its modification time, as in+  `POST /reclaim?min-age-hours=24`. A NAR is unreferenced until its push sends+  the narinfo, and a push can run for hours. Any other query is refused with+  `400`. - If any narinfo cannot be read or parsed, or its `URL` is not `nar/<file>`,-  the run prints those narinfos, lists and deletes nothing, and exits nonzero:-  the NAR such a narinfo names is unknown, so no NAR can be called unreferenced.-- If the store holds NARs and no narinfo at all, the run lists and deletes-  nothing and exits nonzero: a store that lost its narinfos looks the same as-  one whose every NAR is unreferenced.-- A run is not atomic with the server, so run it with no push in progress. A-  narinfo that arrives during a run for a NAR past the minimum age loses that-  NAR, which leaves a signed narinfo with nothing behind it.-- A deletion that fails is reported for that file, and the exit status is-  nonzero.-- The store root comes from `--store`, then `NIX_CACHE_DIR`, then-  `./nix-cache`, as for the server. A missing `narinfo/` or `nar/` directory is-  an error, never an empty store.+  the reclaim answers `409` naming those narinfos and removes nothing: the NAR+  such a narinfo names is unknown, so no NAR can be called unreferenced.+- If the store holds NARs and no narinfo at all, the reclaim answers `409` and+  removes nothing: a store that lost its narinfos looks the same as one whose+  every NAR is unreferenced. Removing every entry leaves a store like that;+  stop the server to remove its NARs.+- A reclaim can run while pushes are in progress. A narinfo is never stored+  naming a NAR the reclaim removes, and a NAR uploaded during a reclaim is+  kept. A narinfo that arrives for a NAR the reclaim removed is refused, as+  when a client skipped uploading a NAR the cache held, which upstream's+  `nix copy` does; the client's next attempt uploads the NAR.+- Uploads wait while a reclaim or a `DELETE` runs, and a reclaim reads every+  narinfo, so on a store with many entries a reclaim holds pushes up for as+  long as that read takes.+- A Nix client holding a narinfo that names a removed NAR treats it as a+  missing substitute and tries its next substituter.+- A removal that fails is reported for that file, and the reclaim answers+  `500`.  ### Public cache -A public instance runs at `cache.novavero.ai`. It serves the closure of-nova-nix's Windows `hello` recipe, which nova-nix's CI publishes on every push-to its main branch. To use it from Nix, add it to `/etc/nix/nix.conf`. On a-multi-user install, Nix ignores substituters set by users it does not trust.+A public instance runs at `cache.novavero.ai`. It serves the sources+nova-nix's Windows packages are built from and the closure of its Windows+`hello` recipe, which nova-nix's CI publishes on every push to its main+branch. To use it from Nix, add it to `/etc/nix/nix.conf`. On a multi-user+install, Nix ignores substituters set by users it does not trust.  ``` extra-substituters = https://cache.novavero.ai
− exe/LandingPage.hs
@@ -1,114 +0,0 @@--- | The cache.novavero.ai landing page.------ Deployment branding lives here, in the executable: the--- "NovaCache.Server" library is brand-free, taking whatever root--- response its embedder supplies, so other operators running their own--- cache never ship this page.-module LandingPage (landingResponse) where--import qualified Data.ByteString.Lazy as BL-import Data.Text (Text)-import qualified Data.Text as T-import qualified Data.Text.Encoding as TE-import qualified Network.HTTP.Types as HTTP-import Network.Wai (Response, responseLBS)-import NovaCache.Store (CacheInfo (..), FileStore, getCacheInfo)---- | Build the @GET \/@ response: the branded page around live values--- (store-path count, store dir, signing status, public key).------ The count comes from the injected action, not a store scan here: this--- route is unauthenticated, so its per-request work must stay bounded ---- the caller supplies a TTL-cached counter ('NovaCache.Server.newTTLCache').-landingResponse :: IO Int -> FileStore -> Bool -> Maybe Text -> IO Response-landingResponse countPaths store signingEnabled pubKey = do-  pathCount <- countPaths-  let body = TE.encodeUtf8 (landingHtml (getCacheInfo store) signingEnabled pubKey pathCount)-  pure (responseLBS HTTP.status200 htmlHeaders (BL.fromStrict body))---- | The landing page markup.  Static apart from four live values; styled--- to match novavero.ai.  Nothing user-supplied is interpolated - the key--- line is operator configuration - so no escaping is needed.-landingHtml :: CacheInfo -> Bool -> Maybe Text -> Int -> Text-landingHtml info signingEnabled pubKey pathCount =-  T.unlines-    [ "<!DOCTYPE html>",-      "<html lang=\"en\">",-      "<head>",-      "<meta charset=\"UTF-8\" />",-      "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\" />",-      "<title>cache.novavero.ai - Nix binary cache</title>",-      "<meta name=\"description\" content=\"The Novavero Nix binary cache, serving store paths for nova-nix - the Windows-native Nix.\" />",-      "<link rel=\"icon\" type=\"image/svg+xml\" href=\"data:image/svg+xml," <> novaveroLogoSvgEscaped <> "\" />",-      "<style>",-      "* { margin: 0; padding: 0; box-sizing: border-box; }",-      "body { background: #0a0b0e; color: #d6d9de; -webkit-font-smoothing: antialiased; font-family: 'Inter', system-ui, sans-serif; line-height: 1.75; }",-      "body::before { content: ''; position: fixed; inset: 0 0 auto 0; height: 320px; pointer-events: none; background: radial-gradient(ellipse 70% 100% at 50% -20%, rgba(52,211,153,0.07), transparent 70%); }",-      ".container { max-width: 720px; margin: 0 auto; padding: 80px 24px; position: relative; }",-      "a { color: #34d399; text-decoration: none; }",-      "a:hover { text-decoration: underline; }",-      ".brand { display: flex; align-items: center; gap: 14px; margin-bottom: 0.75rem; }",-      ".brand svg { width: 44px; height: 44px; border-radius: 10px; }",-      "h1 { color: #fff; font-size: 1.6rem; font-family: ui-monospace, Consolas, monospace; }",-      ".tagline { color: #9ca3af; margin-bottom: 2.5rem; }",-      ".stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 12px; margin-bottom: 2.5rem; }",-      ".stat { padding: 16px; border: 1px solid #232733; border-radius: 12px; text-align: center; }",-      ".stat .value { color: #fff; font-size: 1.4rem; font-weight: 600; }",-      ".stat .label { color: #6b7280; font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.05em; }",-      "h2 { color: #fff; font-size: 1rem; margin: 2rem 0 0.5rem; }",-      "p { font-size: 0.9rem; margin-bottom: 0.75rem; }",-      "pre { background: #0d0f14; border: 1px solid #232733; border-radius: 8px; padding: 14px; white-space: pre-wrap; word-break: break-all; margin: 0.75rem 0; }",-      "code { font-family: ui-monospace, Consolas, monospace; font-size: 0.85rem; color: #e5e7eb; }",-      ".footer { margin-top: 3rem; padding-top: 1.5rem; border-top: 1px solid #232733; font-size: 0.85rem; color: #6b7280; }",-      "</style>",-      "</head>",-      "<body>",-      "<div class=\"container\">",-      "<div class=\"brand\">" <> novaveroLogoSvg <> "<h1>cache.novavero.ai</h1></div>",-      "<p class=\"tagline\">Nix binary cache - serving store paths for <a href=\"https://github.com/Novavero-AI/nova-nix\">nova-nix</a>, the Windows-native Nix.</p>",-      "<div class=\"stats\">",-      "<div class=\"stat\"><div class=\"value\">" <> T.pack (show pathCount) <> "</div><div class=\"label\">store paths</div></div>",-      "<div class=\"stat\"><div class=\"value\">" <> (if signingEnabled then "ed25519" else "off") <> "</div><div class=\"label\">signing</div></div>",-      "<div class=\"stat\"><div class=\"value\">" <> T.pack (show (ciPriority info)) <> "</div><div class=\"label\">priority</div></div>",-      "</div>",-      "<h2>Use it</h2>",-      "<pre><code>substituters = https://cache.novavero.ai" <> trustAnchorLine <> "</code></pre>",-      "<p>Store dir: <code>" <> ciStoreDir info <> "</code> &middot; protocol endpoints: <code>/nix-cache-info</code>, <code>/&lt;hash&gt;.narinfo</code>, <code>/nar/&lt;file&gt;</code></p>",-      "<h2>What this is</h2>",-      "<p>The binary cache behind the Novavero Nix toolchain. Powered by <a href=\"https://github.com/Novavero-AI/nova-cache\">nova-cache</a>, a Haskell implementation of the Nix binary cache protocol. Read about the first package built by Nix natively on Windows on <a href=\"https://novavero.ai/blog/first-native-windows-nix-build.html\">the blog</a>.</p>",-      "<div class=\"footer\"><a href=\"https://novavero.ai\">Novavero AI</a> &middot; Waterloo, Canada</div>",-      "</div>",-      "</body>",-      "</html>"-    ]-  where-    trustAnchorLine = maybe "" ("\ntrusted-public-keys = " <>) pubKey---- | The Novavero mark (the novavero.ai favicon), inlined so the page stays--- a single self-contained response with no external assets.-novaveroLogoSvg :: Text-novaveroLogoSvg =-  "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 64 64\" role=\"img\" aria-label=\"Novavero\">"-    <> "<g fill=\"#0a0f1a\"><rect width=\"64\" height=\"64\" rx=\"14\" ry=\"14\"/></g>"-    <> "<g fill=\"#ffffff\"><path d=\"M12 54L19 54L23 10L16 10Z\"/><path d=\"M16 10L23 10L48 54L41 54Z\"/><path d=\"M41 54L48 54L52 10L45 10Z\"/></g>"-    <> "</svg>"---- | The same mark, URL-escaped for a data-URI favicon link.-novaveroLogoSvgEscaped :: Text-novaveroLogoSvgEscaped =-  T.concatMap escapeForDataUri novaveroLogoSvg-  where-    escapeForDataUri c = case c of-      '<' -> "%3C"-      '>' -> "%3E"-      '"' -> "%22"-      '#' -> "%23"-      other -> T.singleton other---- | Content-Type and caching headers for the landing page.  Stats change as--- paths are added, so it stays briefly cacheable but revalidates.-htmlHeaders :: HTTP.ResponseHeaders-htmlHeaders =-  [ (HTTP.hContentType, "text/html; charset=utf-8"),-    (HTTP.hCacheControl, "public, max-age=300, must-revalidate")-  ]
exe/Main.hs view
@@ -1,23 +1,23 @@ module Main (main) where  import Control.Applicative ((<|>))+import Control.Exception (IOException, try) import Data.Bifunctor (first) import qualified Data.ByteString as BS-import Data.Maybe (fromMaybe, isJust)+import Data.Maybe (fromMaybe) import Data.String (fromString) import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.Encoding as TE-import LandingPage (landingResponse)+import Network.Wai (Response) import qualified Network.Wai.Handler.Warp as Warp import Network.Wai.Middleware.RequestLogger (logStdout)-import NovaCache.Server (ServerConfig (..), cacheApp, newTTLCache, onExceptionResponse)+import NovaCache.Server (ServerConfig (..), cacheApp, defaultRootResponse, htmlRootResponse, onExceptionResponse) import NovaCache.Signing (SecretKey, normalizeKeyText, parseSecretKey, renderPublicKey, toPublicKey)-import NovaCache.Store (listNarInfoHashes, newFileStore)-import ReclaimNars (reclaimCommand, reclaimMain)+import NovaCache.Store (newFileStore) import System.Environment (getArgs, lookupEnv) import System.Exit (exitFailure)-import System.IO (hPutStrLn, stderr)+import System.IO (BufferMode (LineBuffering), hPutStrLn, hSetBuffering, stderr, stdout) import Text.Read (readMaybe)  -- ---------------------------------------------------------------------------@@ -39,12 +39,9 @@ defaultStoreRoot :: FilePath defaultStoreRoot = "./nix-cache" --- | How long the landing page's store-path count may serve stale, in--- seconds.  The count is a display stat; a minute of staleness is--- invisible, and the bound keeps the unauthenticated root route at--- constant cost regardless of request rate or store size.-pathCountTTLSeconds :: Double-pathCountTTLSeconds = 60+-- | The flag naming the HTML file @GET \/@ serves.+rootPageFlag :: String+rootPageFlag = "--root-page"  -- | Environment variable for the server port. portEnvVar :: String@@ -70,6 +67,14 @@ requestLogEnvVar :: String requestLogEnvVar = "LOG_REQUESTS" +-- | A subcommand this executable does not have: the server reclaims+-- NARs itself, through @GET \/reclaim@ and @POST \/reclaim@.  Named so+-- that running it stops with a pointer to those requests rather than+-- reaching 'serve', which ignores arguments it does not know and would+-- start a second server over the store.+reclaimNarsCommand :: String+reclaimNarsCommand = "reclaim-nars"+ -- --------------------------------------------------------------------------- -- Main -- ---------------------------------------------------------------------------@@ -78,15 +83,25 @@ main = do   args <- getArgs   case args of-    command : rest-      | command == reclaimCommand -> do-          storeEnv <- lookupEnv storeEnvVar-          reclaimMain (fromMaybe defaultStoreRoot storeEnv) rest+    command : _+      | command == reclaimNarsCommand -> do+          hPutStrLn stderr $+            "FATAL: "+              ++ reclaimNarsCommand+              ++ " is not a subcommand: the server removes unreferenced NARs itself."+              ++ " With the write key, GET /reclaim lists them and POST /reclaim removes them;"+              ++ " see the README."+          exitFailure     _ -> serve args  -- | Run the cache server. serve :: [String] -> IO () serve args = do+  -- Under systemd, stdout is a pipe to the journal, which GHC buffers by+  -- block.  The request logger flushes after each line it writes, so+  -- without this the startup lines would wait for the first request, or,+  -- with request logging off, for a block to fill.+  hSetBuffering stdout LineBuffering   portEnv <- lookupEnv portEnvVar   hostEnv <- lookupEnv hostEnvVar   storeEnv <- lookupEnv storeEnvVar@@ -105,21 +120,20 @@       bindHost = fromMaybe defaultBindHost (argValue "--host" <|> hostEnv)       storeRoot = fromMaybe (fromMaybe defaultStoreRoot storeEnv) (argValue "--store")       allowOpenWrites = "--allow-open-writes" `elem` args+      rootPage = argValue rootPageFlag    store <- newFileStore storeRoot   apiKey <- loadApiKey apiKeyEnv   sigKey <- loadSigningKey sigKeyPath   pubKey <- derivePublicKeyLine sigKey-  -- The landing page's store-path count rescans at most once per TTL;-  -- the unauthenticated root route must not pay a full store scan per hit.-  countPaths <- newTTLCache pathCountTTLSeconds (length <$> listNarInfoHashes store)+  rootResponse <- loadRootResponse rootPage    let cfg =         ServerConfig           { scStore = store,             scApiKey = apiKey,             scSigningKey = sigKey,-            scRootResponse = landingResponse countPaths store (isJust sigKey) pubKey+            scRootResponse = rootResponse           }    let logRequests = logRequestsEnv /= Just "0"@@ -131,6 +145,7 @@   mapM_ (\k -> putStrLn ("public key: " ++ T.unpack k)) pubKey   putStrLn ("write auth: " ++ maybe "disabled (open writes!)" (const "enabled") apiKey)   putStrLn ("request logging: " ++ if logRequests then "enabled" else "disabled (LOG_REQUESTS=0)")+  putStrLn ("root page: " ++ fromMaybe "none, plain text" rootPage)    case apiKey of     Nothing@@ -152,6 +167,23 @@   Warp.runSettings settings (requestLogger (cacheApp cfg))  -- ---------------------------------------------------------------------------+-- Root response+-- ---------------------------------------------------------------------------++-- | The response for @GET \/@: the HTML file 'rootPageFlag' names, read+-- once at startup, or the library's line of plain text without one.  A+-- file that cannot be read stops the server at startup.+loadRootResponse :: Maybe FilePath -> IO (IO Response)+loadRootResponse Nothing = pure defaultRootResponse+loadRootResponse (Just path) = do+  readResult <- try (BS.readFile path) :: IO (Either IOException BS.ByteString)+  case readResult of+    Right page -> pure (htmlRootResponse page)+    Left err -> do+      hPutStrLn stderr ("FATAL: cannot read the root page " ++ path ++ ": " ++ show err)+      exitFailure++-- --------------------------------------------------------------------------- -- Credential loading -- --------------------------------------------------------------------------- @@ -186,9 +218,9 @@       exitFailure     Right sk -> pure (Just sk) --- | Derive the rendered public key line from the signing key, if any.--- The landing page shows this line, so the published trust anchor always--- matches the key in use; on a derivation failure the page omits it.+-- | Derive the rendered public key line from the signing key, if any,+-- for the startup log, so the key an operator publishes can be checked+-- against the key in use; on a derivation failure the log omits it. derivePublicKeyLine :: Maybe SecretKey -> IO (Maybe Text) derivePublicKeyLine Nothing = pure Nothing derivePublicKeyLine (Just sk) = case toPublicKey sk of
− exe/ReclaimNars.hs
@@ -1,185 +0,0 @@--- | The @reclaim-nars@ subcommand: list, and with @--delete@ remove,--- the NARs in a store directory that no narinfo references.------ A host tool run against the store directory rather than a route,--- since nothing over HTTP should be able to delete.  The decision is--- "NovaCache.Reclaim"'s; this module parses arguments, prints, and--- deletes.-module ReclaimNars (reclaimCommand, reclaimMain) where--import Control.Exception (IOException, displayException, try)-import Control.Monad (when)-import Data.Foldable (for_, traverse_)-import Data.Time.Clock (NominalDiffTime)-import NovaCache.Reclaim (ReclaimBlocked (..), ReclaimPlan (..), Unresolved (..), defaultMinimumAge, planReclaim, scanStore)-import NovaCache.Store (FileStore, NarFile (..), NarRemoveResult (..), fileStoreAt, removeNar)-import System.Exit (exitFailure)-import System.IO (BufferMode (LineBuffering), hPutStrLn, hSetBuffering, stderr, stdout)-import System.IO.Error (ioeGetErrorString)-import Text.Read (readMaybe)---- ------------------------------------------------------------------------------ Arguments--- ------------------------------------------------------------------------------- | The subcommand's name, the first argument to the executable.-reclaimCommand :: String-reclaimCommand = "reclaim-nars"--storeFlag, minAgeFlag, deleteFlag :: String-storeFlag = "--store"-minAgeFlag = "--min-age-hours"-deleteFlag = "--delete"--usage :: String-usage =-  "usage: nova-cache-server "-    ++ reclaimCommand-    ++ " ["-    ++ storeFlag-    ++ " DIR] ["-    ++ minAgeFlag-    ++ " N] ["-    ++ deleteFlag-    ++ "]"--secondsPerHour :: NominalDiffTime-secondsPerHour = 3600---- | Whether a run removes what it finds.  Listing is the default.-data Mode = ListOnly | Delete--data Options = Options-  { optStoreRoot :: !FilePath,-    optMinimumAge :: !NominalDiffTime,-    optMode :: !Mode-  }---- | Parse the subcommand's arguments over the store root the--- environment chose.  Unlike the server's flags, an unknown or--- malformed argument is an error: a typo here must not run with a--- default the operator did not choose.-parseOptions :: FilePath -> [String] -> Either String Options-parseOptions storeRoot =-  go Options {optStoreRoot = storeRoot, optMinimumAge = defaultMinimumAge, optMode = ListOnly}-  where-    go opts [] = Right opts-    go opts (flag : value : rest)-      | flag == storeFlag = go opts {optStoreRoot = value} rest-      | flag == minAgeFlag = case readMaybe value of-          Just hours-            | hours >= (0 :: Integer) ->-                go opts {optMinimumAge = fromInteger hours * secondsPerHour} rest-          _ -> Left (minAgeFlag ++ " takes a whole number of hours, not " ++ show value)-    go opts (flag : rest)-      | flag == deleteFlag = go opts {optMode = Delete} rest-      | flag == storeFlag || flag == minAgeFlag = Left (flag ++ " needs a value")-      | otherwise = Left ("unknown argument: " ++ flag)---- ------------------------------------------------------------------------------ Command--- ------------------------------------------------------------------------------- | Run the subcommand.  Exits nonzero when the arguments are wrong,--- the store cannot be listed, the run is blocked, or a deletion fails.-reclaimMain :: FilePath -> [String] -> IO ()-reclaimMain storeRoot args = case parseOptions storeRoot args of-  Left err -> do-    hPutStrLn stderr err-    hPutStrLn stderr usage-    exitFailure-  Right opts -> do-    -- Piped stdout is block buffered by default, which would print the-    -- report out of order with the FAILED and FATAL lines on stderr.-    hSetBuffering stdout LineBuffering-    let root = optStoreRoot opts-        store = fileStoreAt root-    putStrLn ("store root: " ++ root)-    putStrLn ("minimum age: " ++ counted (wholeHours (optMinimumAge opts)) "hour")-    scanned <- try (scanStore store)-    case scanned of-      Left err ->-        fatal ("cannot scan the store: " ++ displayException (err :: IOException))-      Right scan -> case planReclaim (optMinimumAge opts) scan of-        Left blocked -> refuse blocked-        Right plan -> case optMode opts of-          ListOnly -> listPlan plan-          Delete -> deletePlan store plan---- | List what a deleting run would remove.-listPlan :: ReclaimPlan -> IO ()-listPlan plan = do-  traverse_ (putStrLn . ("unreferenced: " ++) . describe) (rpEligible plan)-  putStrLn-    ( "eligible: "-        ++ counted (length (rpEligible plan)) "NAR"-        ++ ", "-        ++ counted (sum (map nfSize (rpEligible plan))) "byte"-        ++ " (pass "-        ++ deleteFlag-        ++ " to remove)"-    )-  reportTooYoung plan---- | Remove every eligible NAR, reporting each, and exit nonzero if any--- removal failed.-deletePlan :: FileStore -> ReclaimPlan -> IO ()-deletePlan store plan = do-  outcomes <- traverse removeOne (rpEligible plan)-  let removed = [nar | (nar, NarRemoved) <- outcomes]-      failures = length outcomes - length removed-  putStrLn-    ( "reclaimed: "-        ++ counted (length removed) "NAR"-        ++ ", "-        ++ counted (sum (map nfSize removed)) "byte"-    )-  reportTooYoung plan-  when (failures > 0) $-    fatal ("failed to delete " ++ counted failures "NAR")-  where-    removeOne nar = do-      outcome <- removeNar store (nfName nar)-      case outcome of-        NarRemoved -> putStrLn ("deleted: " ++ describe nar)-        NarRemoveFailed err -> failed nar (ioeGetErrorString err)-        NarRemoveBadPath -> failed nar "not a NAR filename"-      pure (nar, outcome)-    failed nar reason = hPutStrLn stderr ("FAILED: " ++ nfPath nar ++ ": " ++ reason)---- | Report what blocks the run and exit nonzero.-refuse :: ReclaimBlocked -> IO ()-refuse (UnresolvedNarInfos unresolved) = do-  for_ unresolved $ \u ->-    hPutStrLn stderr ("UNRESOLVED: " ++ unresolvedPath u ++ ": " ++ unresolvedReason u)-  fatal-    ( counted (length unresolved) "unresolved narinfo"-        ++ ": nothing was listed or deleted, since the NAR an unresolved narinfo names"-        ++ " would look unreferenced.  Fix or remove each one and run again."-    )-refuse NoNarInfos =-  fatal-    ( "the store holds NARs and no narinfo: nothing was listed or deleted, since a store"-        ++ " that lost its narinfos looks the same as one whose every NAR is unreferenced."-        ++ "  Restore the narinfos, or remove the NARs by hand if none is referenced."-    )--reportTooYoung :: ReclaimPlan -> IO ()-reportTooYoung plan =-  putStrLn ("kept, younger than the minimum age: " ++ counted (length (rpTooYoung plan)) "NAR")--describe :: NarFile -> String-describe nar = nfPath nar ++ " (" ++ counted (nfSize nar) "byte" ++ ")"--fatal :: String -> IO ()-fatal msg = do-  hPutStrLn stderr ("FATAL: " ++ msg)-  exitFailure--wholeHours :: NominalDiffTime -> Integer-wholeHours age = round (age / secondsPerHour)---- | A count with its noun: @1 NAR@, @2 NARs@.-counted :: (Show a, Eq a, Num a) => a -> String -> String-counted 1 noun = "1 " ++ noun-counted n noun = show n ++ " " ++ noun ++ "s"
nova-cache.cabal view
@@ -4,7 +4,7 @@ -- 3.0 an in-package component name shadowed its external namesake -- (the hazard the zstandard library's name dodges, documented there). name:               nova-cache-version:            0.12.0.1+version:            0.13.0.0 synopsis:           Nix binary cache protocol: NAR, narinfo, signing and a WAI server description:   The Nix binary cache protocol in Haskell: nix-base32, NAR serialization@@ -62,6 +62,7 @@     , filepath            >= 1.4.100 && < 1.6     , http-types          >= 0.12 && < 0.13     , ram                 >= 0.20 && < 1+    , stm                 >= 2.5 && < 2.6     , text                >= 2.0 && < 2.2     , time                >= 1.9 && < 1.17     , vector              >= 0.12 && < 0.14@@ -169,9 +170,6 @@     buildable: False    main-is:          Main.hs-  other-modules:-    LandingPage-    ReclaimNars   hs-source-dirs:   exe   default-language:  Haskell2010   default-extensions:@@ -182,9 +180,7 @@       base                >= 4.22 && < 5     , bytestring          >= 0.11 && < 0.13     , nova-cache-    , http-types          >= 0.12 && < 0.13     , text                >= 2.0 && < 2.2-    , time                >= 1.9 && < 1.17     , wai                 >= 3.2 && < 3.3     , wai-extra           >= 3.1 && < 3.2     , warp                >= 3.3 && < 3.5@@ -204,7 +200,7 @@     , bytestring          >= 0.11 && < 0.13     , containers          >= 0.6 && < 0.9     , crypton             >= 1.1 && < 2-    , directory           >= 1.3 && < 1.4+    , directory           >= 1.3.1.0 && < 1.4     , http-types          >= 0.12 && < 0.13     , nova-cache     -- 2.12 dropped InsufficientCoverage, leaving the four Result@@ -215,6 +211,14 @@     , time                >= 1.9 && < 1.17     , wai                 >= 3.2 && < 3.3     , wai-extra           >= 3.1 && < 3.2++  -- The tests that make a FIFO or a socket, or cut short a file a+  -- stream holds open, run on POSIX alone: Windows has no file mode for+  -- the walk to refuse them by, and no truncate(2).+  if !os(windows)+    build-depends:+        network           >= 3.0 && < 3.3+      , unix              >= 2.8 && < 2.9  test-suite nova-cache-xz-test   type:             exitcode-stdio-1.0
src/NovaCache/Bzip2.hs view
@@ -275,12 +275,12 @@         | otherwise -> decodeStep decoder      -- Input after a clean stream end.  Upstream C++ Nix decompresses-    -- bzip2 through libarchive (its own BzipDecompressionSink is gone),-    -- and libarchive decodes the payload and ignores trailing bytes that-    -- do not begin another stream: a single stray NUL or newline after-    -- the last stream substitutes fine under `nix copy` and used to fail-    -- here.  Bytes that DO begin a stream header are decoded as a-    -- concatenated stream, and a truncated one still fails, since+    -- bzip2 through libarchive (it has no BzipDecompressionSink of its+    -- own), and libarchive decodes the payload and ignores trailing bytes+    -- that do not begin another stream: a single stray NUL or newline+    -- after the last stream substitutes fine under `nix copy`, so it is+    -- ignored here too.  Bytes that DO begin a stream header are decoded+    -- as a concatenated stream, and a truncated one still fails, since     -- libarchive refuses those too and silently truncating a real stream     -- is the failure mode worth keeping.  One measured divergence     -- remains: a trailer that is a well-formed header carrying no block
src/NovaCache/NAR.hs view
@@ -32,6 +32,8 @@     serialiseFromPathOpts,     withNarSource,     withNarSourceOpts,+    NarWalkError (..),+    WalkAccess (..),     SerialiseOptions (..),     defaultSerialiseOptions,     ExecBitResolver,@@ -42,7 +44,7 @@   ) where -import Control.Exception (bracketOnError, finally)+import Control.Exception (Exception (displayException), IOException, SomeException, bracketOnError, catch, finally, mask, throwIO, try) import Data.ByteString (ByteString) import qualified Data.ByteString as BS import qualified Data.ByteString.Builder as B@@ -73,25 +75,21 @@     tokTarget,     tokType,   )-import System.Directory.OsPath-  ( doesDirectoryExist,-    doesFileExist,-    getSymbolicLinkTarget,-    listDirectory,-    pathIsSymbolicLink,-  )+import System.Directory.OsPath (getSymbolicLinkTarget, listDirectory)+import System.IO.Error (annotateIOError) import qualified System.Info import System.OsPath (OsPath, decodeFS, encodeFS, (</>)) import qualified System.OsPath as OP #ifdef mingw32_HOST_OS import qualified Data.Text as T import qualified Data.Text.Encoding as TE-import System.Directory.OsPath (executable, getPermissions)+import System.Directory.OsPath (doesDirectoryExist, executable, getPermissions, pathIsSymbolicLink) import System.IO (Handle, IOMode (ReadMode), hClose, hFileSize, openBinaryFile) #else import qualified Data.ByteString.Char8 as BS8 import System.IO (Handle, IOMode (ReadMode), hClose, hFileSize, latin1, openBinaryFile) import qualified System.Posix.Files as Posix+import qualified System.Posix.Files.ByteString as PosixBytes #endif  -- ---------------------------------------------------------------------------@@ -254,6 +252,75 @@ narHash = Hash.hashBytes . serialise  -- ---------------------------------------------------------------------------+-- Walk failures+-- ---------------------------------------------------------------------------++-- | A filesystem access the walk makes on an entry.+data WalkAccess+  = -- | Reading the entry's own status, without following a symlink+    -- (@lstat@ on POSIX), for its type.+    WalkStatus+  | -- | Reading a symlink's target (@readlink@ on POSIX).+    WalkReadLink+  | -- | Listing a directory's entries.+    WalkListDirectory+  | -- | Opening a regular file and reading its size and contents.+    WalkReadFile+  deriving (Eq, Show)++-- | A filesystem access that failed while a tree was walked for+-- serialisation: which access, on which entry, and the 'IOException'+-- it raised, so a caller can tell an unreadable file from an+-- unlistable directory without parsing GHC's text.+--+-- Refusals the walk raises itself rather than an access raising them+-- (an entry of a type a NAR cannot hold, a case-hack name collision, a+-- file that shrinks while it streams) stay 'IOException's.  The+-- 'ExecBitResolver' runs outside these accesses, since a caller's+-- resolver may throw anything, so what a resolver throws arrives as+-- thrown; that includes the status read 'defaultExecBitResolver'+-- makes.+data NarWalkError = NarWalkError+  { -- | The access that failed.+    nweAccess :: !WalkAccess,+    -- | The entry it failed on: the root as given, extended by the+    -- on-disk name of each entry below it (under 'CaseHackEnabled',+    -- the suffixed spelling the walk was reading).  The same+    -- 'decodeFS' rendering an 'ExecBitResolver' receives, so+    -- 'encodeFS' recovers its exact bytes.+    nwePath :: !FilePath,+    -- | What the access raised.+    nweCause :: !IOException+  }+  deriving (Eq, Show)++-- | Rendered as GHC renders an 'IOException', with the walk's access+-- standing in for the location: @\/d\/tree\/sub: listing directory:+-- permission denied (Permission denied)@.+instance Exception NarWalkError where+  displayException (NarWalkError access path cause) =+    displayException (annotateIOError cause (accessLabel access) Nothing (Just path))++-- | The location each access takes in the rendering of a t'NarWalkError'.+accessLabel :: WalkAccess -> String+accessLabel access = case access of+  WalkStatus -> "reading status"+  WalkReadLink -> "reading symlink target"+  WalkListDirectory -> "listing directory"+  WalkReadFile -> "reading file"++-- | Run one access of the walk on an entry, raising the 'IOException'+-- it throws as a t'NarWalkError'.  Nothing else is caught, so an+-- asynchronous exception passes through as itself.+accessing :: WalkAccess -> OsPath -> IO a -> IO a+accessing access path action = action `catch` failed+  where+    failed :: IOException -> IO b+    failed cause = do+      shownPath <- decodeFS path+      throwIO (NarWalkError access shownPath cause)++-- --------------------------------------------------------------------------- -- Filesystem to NarEntry (IO boundary) -- --------------------------------------------------------------------------- @@ -312,7 +379,7 @@ -- @access(2)@ - what the CALLING PROCESS may do - and so diverges for -- root (any execute bit reads as executable), ACLs, and files the -- caller does not own; the flag lands in NAR bytes, so that--- divergence moves a hash (#65).+-- divergence moves a hash. defaultExecBitResolver :: ExecBitResolver defaultExecBitResolver path = do   status <- Posix.getSymbolicLinkStatus path@@ -342,8 +409,11 @@ -- 'defaultCaseHack'. -- -- This is the module's IO boundary: the platform-native walk--- classifies each path as symlink, directory, or regular file and--- delegates to pure constructors.+-- classifies each path as regular file, directory, or symlink and+-- delegates to pure constructors.  Any other type (a FIFO, a socket, a+-- device) is refused with the 'IOException' @file \'\<path\>\' has an+-- unsupported type@, as upstream's dump refuses it.  A failure of an+-- access 'WalkAccess' names throws t'NarWalkError'. serialiseFromPath :: FilePath -> IO NarEntry serialiseFromPath = serialiseFromPathOpts defaultSerialiseOptions @@ -362,15 +432,61 @@ -- NAR. walkPath :: SerialiseOptions -> OsPath -> IO NarEntry walkPath opts path = do+  kind <- entryType path+  case kind of+    RegularEntry -> buildRegularFile (soExecBit opts) path+    DirectoryEntry -> buildDirectory opts path+    SymlinkEntry -> NarSymlink <$> (symlinkTargetBytes =<< accessing WalkReadLink path (getSymbolicLinkTarget path))+    UnsupportedEntry -> unsupportedType path++-- | An entry's type as the walk reads it: one of the three a NAR holds,+-- or any other (a FIFO, a socket, a device).+data EntryType+  = RegularEntry+  | DirectoryEntry+  | SymlinkEntry+  | UnsupportedEntry++-- | Read an entry's type without following a symlink.  On POSIX this+-- is one @lstat@, classified by its file mode as upstream's dump+-- classifies it: @S_ISREG@, @S_ISDIR@, @S_ISLNK@, and any other type+-- unsupported.+entryType :: OsPath -> IO EntryType+#ifdef mingw32_HOST_OS+-- Windows has no file mode.  The directory package reads any reparse+-- point as a link and anything that is neither a link nor a directory+-- as a file, so no entry reads as unsupported here; a reparse point+-- that is not a symlink or a junction fails its target read instead.+entryType path = accessing WalkStatus path $ do   isSym <- pathIsSymbolicLink path   if isSym-    then NarSymlink <$> (symlinkTargetBytes =<< getSymbolicLinkTarget path)+    then pure SymlinkEntry     else do       isDir <- doesDirectoryExist path-      if isDir-        then buildDirectory opts path-        else buildRegularFile (soExecBit opts) path+      pure (if isDir then DirectoryEntry else RegularEntry)+#else+-- The status read takes the path's raw bytes ('osPathBytes' is exact+-- on POSIX), as the directory package's own reads do, so no text+-- decoding stands between the walk's path and the lstat.+entryType path = do+  pathBytes <- osPathBytes path+  typeOf <$> accessing WalkStatus path (PosixBytes.getSymbolicLinkStatus pathBytes)+  where+    typeOf status+      | Posix.isRegularFile status = RegularEntry+      | Posix.isDirectory status = DirectoryEntry+      | Posix.isSymbolicLink status = SymlinkEntry+      | otherwise = UnsupportedEntry+#endif +-- | Refuse an entry of a type a NAR cannot hold, in upstream's words.+-- Archiving it as any type a NAR holds would change the archive and+-- its hash.+unsupportedType :: OsPath -> IO a+unsupportedType path = do+  shownPath <- decodeFS path+  fail ("file '" ++ shownPath ++ "' has an unsupported type")+ -- | Build a directory entry by recursively walking children. buildDirectory :: SerialiseOptions -> OsPath -> IO NarEntry buildDirectory opts path = do@@ -390,7 +506,7 @@ -- accepts. resolvedDirEntries :: CaseHack -> OsPath -> IO [(ByteString, OsPath)] resolvedDirEntries mode path = do-  names <- sort <$> listDirectory path+  names <- sort <$> accessing WalkListDirectory path (listDirectory path)   named <- traverse withNameBytes names   case unhackedDirNames mode named of     Left (collidedA, collidedB) -> do@@ -436,23 +552,9 @@ -- | Build a regular file entry, checking the executable bit. buildRegularFile :: ExecBitResolver -> OsPath -> IO NarEntry buildRegularFile resolver path = do-  isFile <- doesFileExist path-  if isFile-    then do-      contents <- readFileBytes path-      isExec <- resolveExecBit resolver path-      pure (NarRegular isExec contents)-    else specialFileFailure path---- | The shared refusal for a path that is not a symlink, directory, or--- regular file: a special file (FIFO, socket, device) or a path that--- vanished mid-walk.  Fail loudly rather than fabricating an empty--- regular (which would silently change the NAR and its hash) ---- matching Nix, which aborts on unsupported types.-specialFileFailure :: OsPath -> IO a-specialFileFailure path = do-  shownPath <- decodeFS path-  fail ("serialiseFromPath: not a regular file (special or vanished): " ++ shownPath)+  contents <- readFileBytes path+  isExec <- resolveExecBit resolver path+  pure (NarRegular isExec contents)  -- | Run the resolver on a platform-native path.  The resolver contract -- is 'FilePath', so the path bridges through 'decodeFS' - the same@@ -467,7 +569,9 @@ -- reopened path names the same file even when the name has no text -- decoding. readFileBytes :: OsPath -> IO ByteString-readFileBytes path = BS.readFile =<< decodeFS path+readFileBytes path = do+  shownPath <- decodeFS path+  accessing WalkReadFile path (BS.readFile shownPath)  -- | The NAR name for one platform-native path component: on POSIX the -- raw bytes the filesystem reports, on Windows the UTF-8 encoding of@@ -476,7 +580,7 @@ -- path.  The one refusal is a Windows name holding an unpaired -- surrogate: it has no UTF-8 form and upstream defines no byte -- spelling for it, so failing loudly beats inventing a name (the same--- policy 'buildRegularFile' applies to special files).+-- policy 'unsupportedType' applies to a FIFO, a socket or a device). #ifdef mingw32_HOST_OS osPathBytes :: OsPath -> IO ByteString osPathBytes path = case OP.decodeUtf path of@@ -536,10 +640,16 @@ -- documented boundary as the streaming write in "NovaCache.Store". data SourceState   = SourceSegments ![NarSegment]-  | -- | Mid-file: the open handle, its decoded path for error text,-    -- the bytes still owed, the padding after them, and the remaining-    -- segments.-    SourceFile !Handle !FilePath !Word64 !Int ![NarSegment]+  | -- | Mid-file: the open handle, its path for error text, the bytes+    -- still owed, the padding after them, and the remaining segments.+    SourceFile !Handle !OsPath !Word64 !Int ![NarSegment]+  | -- | An exception was raised while a pull ran, and every later pull+    -- rethrows it, so a consumer that catches the failure and pulls+    -- again never reads the empty chunk that marks a clean end.  Held at+    -- 'SomeException', as the codec sources hold theirs: a pull+    -- interrupted after its read consumed bytes it never returned+    -- leaves the archive as torn as one that failed.+    SourceFailed !SomeException   | SourceDrained  -- | Serialise a filesystem tree as a pull source of NAR chunks,@@ -558,8 +668,16 @@ -- order, so the pulled bytes equal @'serialise' \<tree\>@ exactly.  A -- file's size is read when its streaming starts and exactly that many -- bytes are emitted, as upstream's dump does; a file that shrinks--- mid-stream fails loudly rather than emitting a torn archive.  Any--- file handle still open when the continuation exits is closed.+-- mid-stream fails loudly rather than emitting a torn archive.  A+-- failure of an access 'WalkAccess' names throws t'NarWalkError':+-- before the continuation runs for the structure, from the pull for a+-- file's contents.  An exception raised while a pull runs is rethrown+-- by every later pull, so none of them reads as a clean end.  An+-- asynchronous exception can also arrive as a pull returns, after the+-- source has moved past the chunk it was returning, and is not+-- latched, so a consumer must not pull again after an asynchronous+-- exception.  Any file handle still open when the continuation exits+-- is closed. withNarSource :: CaseHack -> FilePath -> (IO ByteString -> IO a) -> IO a withNarSource mode = withNarSourceOpts defaultSerialiseOptions {soCaseHack = mode} @@ -569,56 +687,82 @@   rootPath <- encodeFS root   segments <- planSegments opts rootPath   stateRef <- newIORef (SourceSegments segments)-  consume (pullChunk stateRef) `finally` closeCurrent stateRef-  where-    closeCurrent stateRef = do-      state <- readIORef stateRef-      case state of-        SourceFile handle _ _ _ _ -> hClose handle-        _ -> pure ()+  consume (pullChunk stateRef) `finally` (closeHeldFile =<< readIORef stateRef) --- | Produce the next chunk of the planned archive.+-- | Close the file a source state holds open, if it holds one.+closeHeldFile :: SourceState -> IO ()+closeHeldFile state = case state of+  SourceFile handle _ _ _ _ -> hClose handle+  _ -> pure ()++-- | Produce the next chunk of the planned archive, latching any+-- exception raised while the pull runs as 'SourceFailed'. pullChunk :: IORef SourceState -> IO ByteString-pullChunk stateRef = advance =<< readIORef stateRef+pullChunk stateRef = dispatch =<< readIORef stateRef   where-    advance (SourceSegments []) = do+    dispatch (SourceSegments segments) = latching (fromSegments segments)+    dispatch (SourceFile handle path owed padLen rest) = latching (inFile handle path owed padLen rest)+    dispatch (SourceFailed failure) = throwIO failure+    dispatch SourceDrained = pure BS.empty++    -- Only the pull itself runs unmasked, so no asynchronous exception+    -- lands between its failure and the latch, or between recording+    -- the failure and closing the file the state held, which once the+    -- state no longer names it only the garbage collector would close.+    -- One delivered as the mask ends after a pull succeeded is not+    -- latched: the state has already moved past the chunk it loses.+    latching pull = mask $ \restore -> either latch pure =<< tryPull (restore pull)++    latch failure = do+      held <- readIORef stateRef+      writeIORef stateRef (SourceFailed failure)+      closeHeldFile held+      throwIO failure++    fromSegments [] = do       writeIORef stateRef SourceDrained       pure BS.empty-    advance (SourceSegments (SegmentBytes bytes : rest)) = do+    fromSegments (SegmentBytes bytes : rest) = do       writeIORef stateRef (SourceSegments rest)       pure bytes-    advance (SourceSegments (SegmentFile path : rest)) = do+    fromSegments (SegmentFile path : rest) = do       shownPath <- decodeFS path-      -- hFileSize can throw after a successful open (the path swapped-      -- for a FIFO between plan and pull); until the state ref records-      -- the handle, closeCurrent cannot see it, so ownership transfers-      -- under bracketOnError - the same discipline as the shrink path-      -- below.-      bracketOnError (openBinaryFile shownPath ReadMode) hClose $ \handle -> do-        size <- hFileSize handle-        let owed = fromIntegral size :: Word64-        writeIORef stateRef (SourceFile handle shownPath owed (narPadOf owed) rest)-        pure (BL.toStrict (B.toLazyByteString (B.word64LE owed)))-    advance (SourceFile handle _ 0 padLen rest) = do+      -- hFileSize throws after a successful open when the entry is no+      -- longer a regular file (replaced by a FIFO or a device after the+      -- plan read its type); until the state ref records the handle,+      -- the latch and the continuation's exit cannot see it, so+      -- ownership transfers under bracketOnError.+      accessing WalkReadFile path $+        bracketOnError (openBinaryFile shownPath ReadMode) hClose $ \handle -> do+          size <- hFileSize handle+          let owed = fromIntegral size :: Word64+          writeIORef stateRef (SourceFile handle path owed (narPadOf owed) rest)+          pure (BL.toStrict (B.toLazyByteString (B.word64LE owed)))++    inFile handle _ 0 padLen rest = do       hClose handle       writeIORef stateRef (SourceSegments rest)       if padLen == 0-        then pullChunk stateRef+        then fromSegments rest         else pure (BS.replicate padLen 0)-    advance (SourceFile handle shownPath owed padLen rest) = do-      chunk <- BS.hGet handle (fromIntegral (min owed (fromIntegral narSourceChunkBytes)))+    inFile handle path owed padLen rest = do+      chunk <- accessing WalkReadFile path (BS.hGet handle (fromIntegral (min owed (fromIntegral narSourceChunkBytes))))       if BS.null chunk         then do-          hClose handle-          writeIORef stateRef SourceDrained+          shownPath <- decodeFS path           ioError (userError ("withNarSource: " ++ shownPath ++ " shrank while streaming"))         else do           writeIORef             stateRef-            (SourceFile handle shownPath (owed - fromIntegral (BS.length chunk)) padLen rest)+            (SourceFile handle path (owed - fromIntegral (BS.length chunk)) padLen rest)           pure chunk-    advance SourceDrained = pure BS.empty +-- | 'try' at 'SomeException', monomorphic so the catch-all needs no+-- annotation at the call site.  The caller latches whatever this+-- catches and rethrows it, asynchronous exceptions included.+tryPull :: IO ByteString -> IO (Either SomeException ByteString)+tryPull = try+ -- | Plan the archive: every structural byte rendered, file contents -- deferred as 'SegmentFile's.  Holds structure only - O(entries), -- never contents.@@ -650,16 +794,14 @@ -- | Plan one node, mirroring 'walkPath'. planNode :: SerialiseOptions -> OsPath -> IO [PlanPiece] planNode opts path = do-  isSym <- pathIsSymbolicLink path-  if isSym-    then do-      target <- symlinkTargetBytes =<< getSymbolicLinkTarget path+  kind <- entryType path+  case kind of+    RegularEntry -> planRegular (soExecBit opts) path+    DirectoryEntry -> planDirectory opts path+    SymlinkEntry -> do+      target <- symlinkTargetBytes =<< accessing WalkReadLink path (getSymbolicLinkTarget path)       pure [PieceBytes (buildNode (NarSymlink target))]-    else do-      isDir <- doesDirectoryExist path-      if isDir-        then planDirectory opts path-        else planRegular (soExecBit opts) path+    UnsupportedEntry -> unsupportedType path  -- | Plan a directory.  Children are ordered by their NAR-name bytes - -- the same order 'buildNode' emits - not by on-disk order, which can@@ -693,19 +835,15 @@ -- with the contents wire string (length, bytes, padding) deferred. planRegular :: ExecBitResolver -> OsPath -> IO [PlanPiece] planRegular resolver path = do-  isFile <- doesFileExist path-  if isFile-    then do-      isExec <- resolveExecBit resolver path-      pure-        [ PieceBytes-            ( narStr tokLParen-                <> narStr tokType-                <> narStr tokRegular-                <> execFlag isExec-                <> narStr tokContents-            ),-          PieceFile path,-          PieceBytes (narStr tokRParen)-        ]-    else specialFileFailure path+  isExec <- resolveExecBit resolver path+  pure+    [ PieceBytes+        ( narStr tokLParen+            <> narStr tokType+            <> narStr tokRegular+            <> execFlag isExec+            <> narStr tokContents+        ),+      PieceFile path,+      PieceBytes (narStr tokRParen)+    ]
src/NovaCache/Reclaim.hs view
@@ -1,21 +1,38 @@--- | Finding the NARs a file store holds that no narinfo references.+-- | Reclaiming what a file store holds and nothing needs: the NARs no+-- narinfo names, and the multipart uploads begun there and abandoned. -- -- A client uploads a path's NAR before its narinfo (upstream's -- @BinaryCacheStore::addToStoreCommon@ does, and so does nova-nix's -- push), so a NAR whose narinfo is refused, or never sent because the--- push died, stays under @nar\/@ with nothing pointing at it.  Which--- NARs those are is decided purely ('planReclaim' over a 't:StoreScan');--- 'scanStore' is the IO edge that reads the store, and deletion is--- 'NovaCache.Store.removeNar'.+-- push died, stays under @nar\/@ with nothing pointing at it, as does a+-- NAR whose narinfo was removed or replaced by one naming another file.+-- Which NARs those are is decided purely ('planStore' over a+-- 't:StoreScan'); 'scanStore' is the IO edge that reads the store, and+-- 'reclaim' removes what the plan selects. ----- Three rules keep a run from deleting a NAR the cache still needs.  A+-- 'reclaim' holds the store's gate alone ('NovaCache.Store.withRemoval')+-- from its scan through its last removal, and a narinfo upload holds it+-- from the check of its NAR through its write, as does each rename into+-- @nar\/@ for the rename.  So a narinfo stored before a run keeps its+-- NAR, one sent during a run is checked after it, and a NAR that arrives+-- during a run lands after it.  Those uploads wait while the run reads+-- the store, which takes longer the more narinfos it holds.  A client that skipped uploading a NAR+-- because the cache held it, as upstream's @nix copy@ does, can still+-- find it removed when its narinfo arrives: the narinfo is refused and+-- stores nothing, and the client's next attempt uploads the NAR.+--+-- Three rules keep a run from removing a NAR the cache still needs.  A -- NAR younger than the minimum age is never eligible, because a NAR is -- unreferenced legitimately until its push sends the narinfo.  A narinfo -- that cannot be read, parsed, or resolved to a file under @nar\/@ -- blocks the whole run: the NAR it names is unknown, so no NAR can be -- shown to be unreferenced.  And a store holding NARs and no narinfo at--- all blocks it too ('NoNarInfos').  None of them makes a run atomic--- with the server: 'scanStore' says what remains.+-- all blocks it too ('NoNarInfos').+--+-- A multipart upload begun and never completed leaves its parts under+-- @uploads\/@ ("NovaCache.Store").  No narinfo can reference a staged+-- upload, so the minimum age alone decides ('selectAbandoned'), counted+-- from the upload's last activity. module NovaCache.Reclaim   ( -- * Narinfo references     NarInfoTarget (..),@@ -30,10 +47,19 @@     selectUnreferenced,     ReclaimBlocked (..),     planReclaim,+    UploadPlan (..),+    selectAbandoned,+    StorePlan (..),+    planStore,      -- * Scanning a store     StoreScan (..),     scanStore,++    -- * Reclaiming+    listReclaimable,+    Reclaimed (..),+    reclaim,   ) where @@ -48,9 +74,23 @@ import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.Encoding as TE-import Data.Time.Clock (NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime, nominalDay)+import Data.Time.Clock (NominalDiffTime, UTCTime, diffUTCTime, nominalDay) import NovaCache.NarInfo (NarInfo (..), parseNarInfo)-import NovaCache.Store (FileStore, NarFile (..), listNarFiles, listNarInfoFiles, sanitizePath)+import NovaCache.Store+  ( FileStore,+    NarFile (..),+    NarRemoveResult,+    StagedUpload (..),+    UploadRemoveResult,+    listNarFiles,+    listNarInfoFiles,+    listStagedUploads,+    removeNar,+    removeUpload,+    sanitizePath,+    withRemoval,+    withoutRemoval,+  ) import System.IO.Error (ioeGetErrorString)  -- ---------------------------------------------------------------------------@@ -119,12 +159,11 @@ -- -- A NAR stays unreferenced for the rest of the push that uploaded it, -- and nothing bounds that: nova-nix uploads every NAR of a closure--- before the first narinfo, with no response timeout.  Deleting one--- early leaves a narinfo naming a missing NAR, which no later push--- repairs (a push skips paths whose narinfo exists), while waiting--- costs only disk.  git guards the same race, between gc and a writer--- that has stored an object but not yet referenced it, with the same--- two weeks (@gc.pruneExpire@).+-- before the first narinfo, with no response timeout.  Removing one+-- early has its narinfo refused when it comes, which fails that push or+-- costs it the upload again, while waiting costs only disk.  git guards+-- the same race, between gc and a writer that has stored an object but+-- not yet referenced it, with the same two weeks (@gc.pruneExpire@). defaultMinimumAge :: NominalDiffTime defaultMinimumAge = 14 * nominalDay @@ -144,10 +183,15 @@ selectUnreferenced minimumAge now references nars =   ReclaimPlan {rpEligible = old, rpTooYoung = young}   where-    (old, young) = partition oldEnough (filter unreferenced nars)+    (old, young) = partitionByAge nfModified minimumAge now (filter unreferenced nars)     unreferenced nar = Set.notMember (nfName nar) references-    oldEnough nar = diffUTCTime now (nfModified nar) >= minimumAge +-- | Split by age at the given time, counted from a modification time:+-- those at least the minimum age old, then the younger.+partitionByAge :: (a -> UTCTime) -> NominalDiffTime -> UTCTime -> [a] -> ([a], [a])+partitionByAge modified minimumAge now =+  partition (\item -> diffUTCTime now (modified item) >= minimumAge)+ -- | Why a run cannot be planned. data ReclaimBlocked   = -- | These narinfos name NARs that are unknown.@@ -160,8 +204,8 @@     NoNarInfos   deriving (Eq, Show) --- | Plan a run from a scan: the unreferenced NARs by age, or why the--- run is blocked.+-- | Plan a run's NARs from a scan: the unreferenced ones by age, or why+-- the run is blocked. planReclaim :: NominalDiffTime -> StoreScan -> Either ReclaimBlocked ReclaimPlan planReclaim minimumAge scan   | null (scanNarInfos scan) && not (null (scanNars scan)) = Left NoNarInfos@@ -169,40 +213,86 @@       references <- first UnresolvedNarInfos (collectReferences (scanNarInfos scan))       pure (selectUnreferenced minimumAge (scanTime scan) references (scanNars scan)) +-- | The staged uploads of a store, split by the minimum age.+data UploadPlan = UploadPlan+  { -- | Idle for at least the minimum age: abandoned.+    upEligible :: ![StagedUpload],+    -- | Active within the minimum age: a push may yet complete them.+    upTooYoung :: ![StagedUpload]+  }+  deriving (Eq, Show)++-- | Split staged uploads by the time since their last activity, by the+-- same minimum age as unreferenced NARs.  A staged upload holds only+-- parts, which nothing serves and no narinfo names, so removing one+-- never leaves a narinfo without its NAR: a push that resumes an upload+-- removed under it is refused and begins a new one.  The age is there+-- for that push, which is the only thing the removal can hurt.+selectAbandoned :: NominalDiffTime -> UTCTime -> [StagedUpload] -> UploadPlan+selectAbandoned minimumAge now uploads =+  UploadPlan {upEligible = old, upTooYoung = young}+  where+    (old, young) = partitionByAge suModified minimumAge now uploads++-- | What a run removes and keeps: a store's NARs and its staged uploads.+data StorePlan = StorePlan+  { spNars :: !ReclaimPlan,+    spUploads :: !UploadPlan+  }+  deriving (Eq, Show)++-- | Plan a run from a scan: its NARs as 'planReclaim' splits them and its+-- staged uploads as 'selectAbandoned' does, or why the run is blocked.+-- A blocked run removes no upload either.+planStore :: NominalDiffTime -> StoreScan -> Either ReclaimBlocked StorePlan+planStore minimumAge scan = do+  nars <- planReclaim minimumAge scan+  pure+    StorePlan+      { spNars = nars,+        spUploads = selectAbandoned minimumAge (scanTime scan) (scanStagedUploads scan)+      }+ -- --------------------------------------------------------------------------- -- Scanning a store -- --------------------------------------------------------------------------- --- | What a scan read: each narinfo's target, each NAR, and the time the--- NARs were listed.+-- | What a scan read: each narinfo's target, each NAR, each staged+-- upload, and the time once they were listed. data StoreScan = StoreScan   { scanNarInfos :: ![(FilePath, NarInfoTarget)],     scanNars :: ![NarFile],+    scanStagedUploads :: ![StagedUpload],     scanTime :: !UTCTime   }   deriving (Eq, Show) --- | Read a store for 'planReclaim'.  A directory that cannot be listed,--- or a NAR that cannot be stat'd, fails the scan with its--- 'IOException'; an unreadable narinfo is recorded as 'Unresolvable'.+-- | Read a store for 'planStore', with the time from @clock@, read after+-- every listing so nothing a listing saw is in its future.  A directory+-- that cannot be listed, or a NAR that cannot be stat'd, fails the scan+-- with its 'IOException'; an unreadable narinfo is recorded as+-- 'Unresolvable'.  The scan takes no gate: its caller holds one, as+-- 'listReclaimable' and 'reclaim' do, so nothing is removed while it+-- reads. -- -- The narinfos are read before the NARs are listed, the order git's--- prune uses (reachability first, then each object's age): a NAR--- uploaded again while the narinfos are read shows its new--- modification time.  A run is still not atomic with the server, so it--- belongs on a cache with no push in progress.  A narinfo that arrives--- after the narinfo listing for a NAR older than the minimum age loses--- that NAR: a push slower than the minimum age does this, and so does a--- client (upstream @nix copy@) that skips re-uploading a NAR already--- present.  A NAR uploaded again after the NARs are listed goes too,--- since removal does not look at the file again.  Only a lock shared--- with the server would close these.-scanStore :: FileStore -> IO StoreScan-scanStore store = do+-- prune uses (reachability first, then each object's age): in a scan+-- that runs alongside uploads, as 'listReclaimable' runs one, a NAR+-- uploaded again while the narinfos are read shows its new modification+-- time.+scanStore :: FileStore -> IO UTCTime -> IO StoreScan+scanStore store clock = do   narInfos <- traverse readTarget =<< listNarInfoFiles store   nars <- listNarFiles store-  now <- getCurrentTime-  pure StoreScan {scanNarInfos = narInfos, scanNars = nars, scanTime = now}+  uploads <- listStagedUploads store+  now <- clock+  pure+    StoreScan+      { scanNarInfos = narInfos,+        scanNars = nars,+        scanStagedUploads = uploads,+        scanTime = now+      }   where     -- Forced per file, so each narinfo's bytes are garbage once its     -- target is known instead of held until the scan is planned.@@ -210,3 +300,37 @@       contents <- try (BS.readFile path)       let !target = narInfoTarget contents       pure (path, target)++-- ---------------------------------------------------------------------------+-- Reclaiming+-- ---------------------------------------------------------------------------++-- | What 'reclaim' would remove now, and what it would keep for its age,+-- removing nothing.  The scan holds the gate shared, so uploads go on+-- while it reads and no removal does.+listReclaimable :: FileStore -> NominalDiffTime -> IO UTCTime -> IO (Either ReclaimBlocked StorePlan)+listReclaimable store minimumAge clock =+  withoutRemoval store (planStore minimumAge <$> scanStore store clock)++-- | What a run removed: its plan, and the outcome of each removal.+data Reclaimed = Reclaimed+  { rdPlan :: !StorePlan,+    rdNars :: ![(NarFile, NarRemoveResult)],+    rdUploads :: ![(StagedUpload, UploadRemoveResult)]+  }+  deriving (Eq, Show)++-- | Scan the store, plan, and remove every eligible NAR and abandoned+-- upload, holding the gate alone throughout, or say why the run is+-- blocked and remove nothing.  A scan that fails throws its+-- 'IOException' before anything is removed; a removal that fails is an+-- outcome in t'Reclaimed'.+reclaim :: FileStore -> NominalDiffTime -> IO UTCTime -> IO (Either ReclaimBlocked Reclaimed)+reclaim store minimumAge clock = withRemoval store $ do+  planned <- planStore minimumAge <$> scanStore store clock+  traverse removePlanned planned+  where+    removePlanned plan = do+      nars <- traverse (\nar -> (,) nar <$> removeNar store (nfName nar)) (rpEligible (spNars plan))+      uploads <- traverse (\upload -> (,) upload <$> removeUpload store (suFileName upload) (suUploadId upload)) (upEligible (spUploads plan))+      pure Reclaimed {rdPlan = plan, rdNars = nars, rdUploads = uploads}
src/NovaCache/Server.hs view
@@ -10,20 +10,39 @@ -- Protocol surface: -- -- @--- GET  \/nix-cache-info      cache metadata--- GET  \/\<hash\>.narinfo      narinfo by store-path hash--- GET  \/nar\/\<file\>          NAR payload (streamed from disk)--- GET  \/narinfo-hashes      stored-hash listing (authenticated; push-tool plumbing)--- PUT  \/\<hash\>.narinfo      validated, checked against its stored NAR, signed, stored (authenticated)--- PUT  \/nar\/\<file\>          streamed to disk under a size cap (authenticated)+-- GET    \/nix-cache-info      cache metadata+-- GET    \/\<hash\>.narinfo      narinfo by store-path hash+-- GET    \/nar\/\<file\>          NAR payload (streamed from disk)+-- GET    \/narinfo-hashes      stored-hash listing (authenticated; push-tool plumbing)+-- PUT    \/\<hash\>.narinfo      validated, checked against its stored NAR, signed, stored (authenticated)+-- PUT    \/nar\/\<file\>          streamed to disk under a size cap (authenticated)+-- DELETE \/\<hash\>.narinfo      narinfo removed, its NAR left for a reclaim (authenticated)+-- GET    \/reclaim             what POST \/reclaim would remove (authenticated)+-- POST   \/reclaim             NARs no narinfo names, and abandoned uploads, removed (authenticated)+--+-- POST \/nar\/\<file\>\/uploads              begin a multipart upload (authenticated)+-- PUT  \/nar\/\<file\>\/uploads\/\<id\>\/\<n\>     part n, streamed to disk under the same cap (authenticated)+-- POST \/nar\/\<file\>\/uploads\/\<id\>         join the parts into nar\/\<file\> (authenticated) -- @ -- -- HEAD is answered wherever GET is: routing treats the two identically -- and Warp elides the body while keeping the status and headers.+--+-- A multipart upload carries a NAR past a proxy that refuses large+-- request bodies.  Its parts are staged outside @nar\/@ (see+-- "NovaCache.Store"), and the NAR appears there only when the request+-- that completes the upload joins them.+--+-- Removal runs in the server, ordered by the store's gate+-- ("NovaCache.Store"): a narinfo upload holds it from the check of its+-- NAR through its write, and a reclaim ("NovaCache.Reclaim") or a+-- narinfo removal holds it alone, so a narinfo is never stored naming a+-- NAR a reclaim removes.  One server serves a store directory. module NovaCache.Server   ( -- * Configuration     ServerConfig (..),     defaultRootResponse,+    htmlRootResponse,     newTTLCache,      -- * Application@@ -33,7 +52,15 @@     -- * Body limits     maxNarInfoBodySize,     maxNarBodySize,+    maxUploadTotalsBodySize, +    -- * Multipart upload+    parseUploadTotals,+    renderUploadTotals,++    -- * Reclaim+    parseMinimumAge,+     -- * Handler pieces (exported for tests)     requireAuth,     readBodyLimited,@@ -44,6 +71,8 @@     renderCacheInfo,      -- * Stored NAR check+    NarInfoWrite (..),+    storeNarInfo,     NarClaim (..),     StoredNar (..),     NarRefusal (..),@@ -61,10 +90,14 @@ import qualified Data.ByteString as BS import qualified Data.ByteString.Char8 as BS8 import qualified Data.ByteString.Lazy as BL+import Data.Either (lefts)+import Data.Foldable (toList) import Data.IORef (newIORef, readIORef, writeIORef) import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.Encoding as TE+import qualified Data.Text.Read as TR+import Data.Time.Clock (NominalDiffTime, getCurrentTime) import GHC.Clock (getMonotonicTime) import qualified Network.HTTP.Types as HTTP import Network.Wai@@ -75,6 +108,7 @@     ResponseReceived,     getRequestBodyChunk,     pathInfo,+    queryString,     requestBodyLength,     requestHeaders,     requestMethod,@@ -83,22 +117,55 @@   ) import NovaCache.Hash (NixHash, formatNixHash, hashFinalize, hashInit, hashUpdate, parseNixHash) import NovaCache.NarInfo (NarInfo (..), parseNarInfo, renderNarInfo)+import NovaCache.Reclaim+  ( ReclaimBlocked (..),+    ReclaimPlan (..),+    Reclaimed (..),+    StorePlan (..),+    Unresolved (..),+    UploadPlan (..),+    defaultMinimumAge,+    listReclaimable,+    reclaim,+  ) import NovaCache.Signing (SecretKey, sign) import NovaCache.Store   ( CacheInfo (..),+    CompletionResult (..),     FileStore,+    NarFile (..),+    NarInfoRemoveResult (..),+    NarRemoveResult (..),     NarWriteResult (..),+    PartWriteResult (..),+    StagedUpload (..),+    UploadId,+    UploadRemoveResult (..),+    UploadTotals (..),+    beginUpload,+    completeUpload,     getCacheInfo,     listNarInfoHashes,+    maxUploadParts,     narFilePath,+    parsePartNumber,+    parseUploadId,     readNarInfo,+    removeNarInfo,+    renderPartNumber,+    renderUploadId,     sanitizePath,+    withRemoval,+    withoutRemoval,     writeNarInfo,     writeNarStreaming,+    writeUploadPart,   ) import NovaCache.StorePath (defaultStoreDir, parseStorePath, storePathHashString) import NovaCache.Validate (validateNarInfo)+import System.FilePath (takeFileName) import System.IO (IOMode (ReadMode), hPutStrLn, stderr, withBinaryFile)+import System.IO.Error (ioeGetErrorString)  -- --------------------------------------------------------------------------- -- Constants@@ -114,6 +181,11 @@ maxNarBodySize :: Int maxNarBodySize = 4 * 1024 * 1024 * 1024 -- 4 GB +-- | Maximum body of the request that completes a multipart upload: two+-- short lines (see 'renderUploadTotals').+maxUploadTotalsBodySize :: Int+maxUploadTotalsBodySize = 1024+ -- | Suffix of narinfo request paths (@\/\<hash\>.narinfo@). narInfoSuffix :: Text narInfoSuffix = ".narinfo"@@ -122,6 +194,23 @@ narRouteSegment :: Text narRouteSegment = "nar" +-- | Path segment under @\/nar\/\<file\>@ that holds the file's multipart+-- uploads.+uploadsRouteSegment :: Text+uploadsRouteSegment = "uploads"++-- | Path of the reclaim: @GET@ lists, @POST@ removes.+reclaimRouteSegment :: Text+reclaimRouteSegment = "reclaim"++-- | The query parameter that sets a reclaim's minimum age, in whole+-- hours.+minAgeHoursParam :: Text+minAgeHoursParam = "min-age-hours"++secondsPerHour :: NominalDiffTime+secondsPerHour = 3600+ -- | Bytes read per step while hashing a stored NAR. storedNarReadBytes :: Int storedNarReadBytes = 128 * 1024@@ -152,6 +241,11 @@ defaultRootResponse =   pure (responseLBS HTTP.status200 textHeaders "nova-cache: a Nix binary cache\n") +-- | Root response serving a deployment's own HTML page, such as a page+-- about the cache, as the embedder read it.+htmlRootResponse :: ByteString -> IO Response+htmlRootResponse page = pure (responseLBS HTTP.status200 htmlHeaders (BL.fromStrict page))+ -- | Memoize an action's result for a time-to-live, in seconds -- (monotonic clock, so wall-time jumps cannot starve the refresh). --@@ -197,7 +291,15 @@     requireAuth cfg req respond $ do       hashes <- listNarInfoHashes (scStore cfg)       let body = TE.encodeUtf8 (T.unlines hashes)-      respond (responseLBS HTTP.status200 hashListHeaders (BL.fromStrict body))+      respond (responseLBS HTTP.status200 uncachedHeaders (BL.fromStrict body))+  -- GET /reclaim, POST /reclaim - store maintenance, read across the+  -- whole store, so authenticated like the hash listing+  ("GET", [segment])+    | segment == reclaimRouteSegment ->+        requireAuth cfg req respond (reclaimRequest cfg req respond ListReclaimable)+  ("POST", [segment])+    | segment == reclaimRouteSegment ->+        requireAuth cfg req respond (reclaimRequest cfg req respond RemoveReclaimable)   -- GET /<hash>.narinfo   ("GET", [hashNarinfo])     | Just hashKey <- T.stripSuffix narInfoSuffix hashNarinfo -> do@@ -226,6 +328,14 @@   ("PUT", segments)     | Just fileName <- narRoute segments ->         requireAuth cfg req respond (putNar cfg req respond fileName)+  -- DELETE /<hash>.narinfo (auth required)+  ("DELETE", [hashNarinfo])+    | Just hashKey <- T.stripSuffix narInfoSuffix hashNarinfo ->+        requireAuth cfg req respond (deleteNarInfo cfg req respond hashKey)+  -- /nar/<file>/uploads... - multipart upload (auth required)+  (method, segments)+    | Just route <- uploadRoute segments ->+        uploadRequest cfg req respond method route   -- Fallback   _ ->     respond notFound@@ -244,7 +354,11 @@ narRoute _ = Nothing  -- | Validate a narinfo upload, check it against the NAR it names, then--- sign and store it.+-- sign and store it.  A narinfo for a path the cache holds replaces it,+-- and the NAR the old one named stays until a reclaim finds nothing+-- naming it.  A stored NAR that cannot be read is the server's own+-- failure, a 500 like a signing failure, with the exception logged and+-- kept out of the response. putNarInfo :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> Text -> ByteString -> IO ResponseReceived putNarInfo cfg req respond hashKey body = case decodeAndValidate body of   Left err -> do@@ -254,42 +368,378 @@     | not (narInfoHashMatches hashKey ni) -> do         logWarn req "HASHMISMATCH"         respond (badRequest "narinfo StorePath hash does not match request")-    | otherwise -> requireStoredNar (scStore cfg) req respond ni $ do-        signedResult <- signNarInfo (scSigningKey cfg) ni-        case signedResult of-          Left err -> do-            logWarn req ("SIGNFAIL: " <> err)-            respond (responseLBS HTTP.status500 textHeaders "signing failed")-          Right signed -> do-            ok <- writeNarInfo (scStore cfg) hashKey signed-            if ok-              then respond (responseLBS HTTP.status200 textHeaders "ok")-              else do-                logWarn req "BADPATH"-                respond (badRequest "invalid path")+    | otherwise -> case narClaim ni of+        Left refusal -> refuseNar req respond refusal+        Right claim -> do+          written <- storeNarInfo (scStore cfg) (signNarInfo (scSigningKey cfg)) hashKey ni claim+          case written of+            NarInfoWritten -> respond okResponse+            NarInfoRefused refusal -> refuseNar req respond refusal+            NarInfoNarUnreadable err -> do+              logWarn req ("NARREAD: " <> show err)+              respond (responseLBS HTTP.status500 textHeaders (textBody ("NAR " <> narPath (ncFileName claim) <> " could not be read")))+            NarInfoSignFailed err -> do+              logWarn req ("SIGNFAIL: " <> err)+              respond (responseLBS HTTP.status500 textHeaders "signing failed")+            NarInfoBadKey -> badPath req respond +-- | Remove a narinfo, holding the gate alone as every removal does.  A+-- hash the store could not hold is refused before the gate, so it never+-- holds up an upload.+deleteNarInfo :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> Text -> IO ResponseReceived+deleteNarInfo cfg req respond hashKey = case sanitizePath hashKey of+  Nothing -> badPath req respond+  Just _ -> do+    result <- withRemoval (scStore cfg) (removeNarInfo (scStore cfg) hashKey)+    removed result+  where+    removed result = case result of+      NarInfoRemoved -> respond okResponse+      NarInfoAbsent -> respond notFound+      NarInfoRemoveBadPath -> badPath req respond+      NarInfoRemoveFailed err -> do+        logWarn req ("NARINFOREMOVE: " <> show err)+        respond (responseLBS HTTP.status500 textHeaders "narinfo could not be removed")+ -- | Stream a NAR upload into the store.  A declared @Content-Length@ over -- the cap is refused before reading anything; chunked or lying transfers -- are cut off by the store's running size check, so memory and storage -- stay bounded either way. putNar :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> Text -> IO ResponseReceived-putNar cfg req respond fileName = case requestBodyLength req of-  KnownLength len-    | len > fromIntegral maxNarBodySize -> overLimit-  _ -> do-    result <- writeNarStreaming (scStore cfg) fileName maxNarBodySize (getRequestBodyChunk req)+putNar cfg req respond fileName = refuseDeclaredOver maxNarBodySize req respond $ do+  result <- writeNarStreaming (scStore cfg) fileName maxNarBodySize (getRequestBodyChunk req)+  case result of+    NarWriteOk -> respond okResponse+    NarWriteTooLarge -> overLimit req respond+    NarWriteBadPath -> badPath req respond++-- ---------------------------------------------------------------------------+-- Multipart upload+-- ---------------------------------------------------------------------------++-- | A request path under @\/nar\/\<file\>\/uploads@.+data UploadRoute+  = -- | @\/nar\/\<file\>\/uploads@, where an upload of the file begins.+    UploadsOf !Text+  | -- | @\/nar\/\<file\>\/uploads\/\<id\>@, one upload. (file, id)+    UploadOf !Text !Text+  | -- | @\/nar\/\<file\>\/uploads\/\<id\>\/\<n\>@, one part of it.+    -- (file, id, part number)+    PartOf !Text !Text !Text++-- | The upload request a path names.  A NAR's name holds no @\/@, so no+-- such path names a file under @nar\/@.+uploadRoute :: [Text] -> Maybe UploadRoute+uploadRoute (segment : fileName : uploads : rest)+  | segment == narRouteSegment && uploads == uploadsRouteSegment = case rest of+      [] -> Just (UploadsOf fileName)+      [uploadId] -> Just (UploadOf fileName uploadId)+      [uploadId, part] -> Just (PartOf fileName uploadId part)+      _ -> Nothing+uploadRoute _ = Nothing++-- | Answer a request under @\/nar\/\<file\>\/uploads@, each one+-- authenticated as the other writes are.+uploadRequest :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> HTTP.Method -> UploadRoute -> IO ResponseReceived+uploadRequest cfg req respond method route = case (method, route) of+  ("POST", UploadsOf fileName) ->+    authed (beginNarUpload cfg req respond fileName)+  ("PUT", PartOf fileName rawId rawPart) ->+    authed (withUpload req respond fileName rawId (putNarPart cfg req respond fileName rawPart))+  ("POST", UploadOf fileName rawId) ->+    authed (withUpload req respond fileName rawId (completeNarUpload cfg req respond fileName))+  _ -> respond notFound+  where+    authed = requireAuth cfg req respond++-- | Run an action on the upload a request names: 400 when the NAR's name+-- is not one the store admits, 404 when the id is not one the server+-- issues.+withUpload :: Request -> (Response -> IO ResponseReceived) -> Text -> Text -> (UploadId -> IO ResponseReceived) -> IO ResponseReceived+withUpload req respond fileName rawId action = case (sanitizePath fileName, parseUploadId rawId) of+  (Nothing, _) -> badPath req respond+  (Just _, Nothing) -> unknownUpload req respond fileName rawId+  (Just _, Just uid) -> action uid++-- | Begin a multipart upload: 201, with the upload's id as the body and+-- its path in @Location@.  The path is relative to the request's, so it+-- stays right when the cache is served under a path prefix.+beginNarUpload :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> Text -> IO ResponseReceived+beginNarUpload cfg req respond fileName = do+  begun <- beginUpload (scStore cfg) fileName+  case begun of+    Nothing -> badPath req respond+    Just uid ->+      let location = TE.encodeUtf8 (uploadsRouteSegment <> "/" <> renderUploadId uid)+       in respond+            ( responseLBS+                HTTP.status201+                ((HTTP.hLocation, location) : textHeaders)+                (textBody (renderUploadId uid <> "\n"))+            )++-- | Stream one part of an upload to disk.  A declared @Content-Length@+-- over the NAR cap is refused before anything is read, as for a whole+-- NAR; otherwise the store's running check holds the part, with the+-- parts the upload holds, to the same cap.+putNarPart :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> Text -> Text -> UploadId -> IO ResponseReceived+putNarPart cfg req respond fileName rawPart uid = case parsePartNumber rawPart of+  Nothing -> do+    logWarn req "BADPART"+    respond (badRequest ("part number must be a whole number from 1 to " <> showText maxUploadParts))+  Just part -> refuseDeclaredOver maxNarBodySize req respond $ do+    result <- writeUploadPart (scStore cfg) fileName uid part maxNarBodySize (getRequestBodyChunk req)     case result of-      NarWriteOk -> respond (responseLBS HTTP.status200 textHeaders "ok")-      NarWriteTooLarge -> overLimit-      NarWriteBadPath -> do-        logWarn req "BADPATH"-        respond (badRequest "invalid path")+      PartWritten -> respond okResponse+      PartTooLarge -> overLimit req respond+      PartOutOfOrder held ->+        refuseConflict+          req+          respond+          ( "upload of "+              <> narPath fileName+              <> " holds "+              <> counted held "part"+              <> ", so the next is part "+              <> showText (held + 1)+              <> ", not part "+              <> renderPartNumber part+          )+      PartUploadUnknown -> unknownUpload req respond fileName (renderUploadId uid)+      PartBadPath -> badPath req respond++-- | Complete an upload from the totals its body declares.+completeNarUpload :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> Text -> UploadId -> IO ResponseReceived+completeNarUpload cfg req respond fileName uid =+  withLimitedBody maxUploadTotalsBodySize req respond $ \body -> case parseUploadTotals body of+    Left err -> do+      logWarn req ("BADTOTALS: " <> T.unpack err)+      respond (badRequest err)+    Right totals -> do+      result <- completeUpload (scStore cfg) fileName uid maxNarBodySize totals+      case result of+        UploadCompleted -> respond okResponse+        UploadCompletedStagingKept err -> do+          logWarn req ("STAGINGKEPT: " <> show err)+          respond okResponse+        CompletionPartsMismatch declared held ->+          refuseConflict req respond ("upload of " <> narPath fileName <> " holds " <> counted held "part" <> ", Parts is " <> showText declared)+        CompletionSizeMismatch declared joined ->+          refuseConflict req respond ("upload of " <> narPath fileName <> " joins to " <> showText joined <> " bytes, FileSize is " <> showText declared)+        CompletionTooLarge declared -> do+          logWarn req "OVERLIMIT"+          respond+            ( responseLBS+                HTTP.status413+                textHeaders+                (textBody ("FileSize " <> showText declared <> " is over the " <> showText maxNarBodySize <> " bytes a NAR may hold"))+            )+        CompletionUploadUnknown -> unknownUpload req respond fileName (renderUploadId uid)+        CompletionBadPath -> badPath req respond++-- | A count with its noun: @1 part@, @2 parts@.+counted :: (Show a, Eq a, Num a) => a -> Text -> Text+counted 1 noun = "1 " <> noun+counted n noun = showText n <> " " <> noun <> "s"++-- | Keys of the body that completes an upload.+partsKey, fileSizeKey :: Text+partsKey = "Parts"+fileSizeKey = "FileSize"++-- | Separates a key from its value, as in a narinfo.+fieldSeparator :: Text+fieldSeparator = ": "++-- | Digits in a whole number a request declares, a size or an age in+-- hours: at most a uint64's 20, as for a narinfo's FileSize.  The length+-- is bounded before the parse, so an overlong number costs nothing to+-- refuse; a size past 'maxNarBodySize' is refused when the upload is+-- completed.+maxWholeNumberDigits :: Int+maxWholeNumberDigits = 20++-- | The body that completes an upload: how many parts it holds and how+-- many bytes they join to, as @Parts@ and @FileSize@ lines in narinfo's+-- @Key: value@ form.+renderUploadTotals :: UploadTotals -> ByteString+renderUploadTotals totals =+  TE.encodeUtf8 $+    T.unlines+      [ partsKey <> fieldSeparator <> renderPartNumber (utParts totals),+        fileSizeKey <> fieldSeparator <> showText (utFileSize totals)+      ]++-- | Read the body that completes an upload: a @Parts@ line and a+-- @FileSize@ line, in either order, each once, each value in canonical+-- decimal.  Any other line is refused rather than skipped, so a field a+-- client means to have checked is never silently ignored.+parseUploadTotals :: ByteString -> Either Text UploadTotals+parseUploadTotals body = do+  text <- first (const "completion body is not valid UTF-8") (TE.decodeUtf8' body)+  fields <- traverse splitField (T.lines text)+  parts <- field partsKey fields >>= orRefuse ("Parts must be a whole number from 1 to " <> showText maxUploadParts) . parsePartNumber+  size <- field fileSizeKey fields >>= orRefuse "FileSize must be a whole number of bytes" . parseWholeNumber+  pure UploadTotals {utParts = parts, utFileSize = size}   where-    overLimit = do-      logWarn req "OVERLIMIT"-      respond (responseLBS HTTP.status413 textHeaders "request body too large")+    splitField line = case T.stripPrefix fieldSeparator <$> T.breakOn fieldSeparator line of+      (key, Just value)+        | key `elem` [partsKey, fileSizeKey] -> Right (key, value)+        | otherwise -> Left ("completion body has an unknown field " <> key)+      (_, Nothing) -> Left ("completion body line is not Key: value: " <> line)+    field key fields = case [value | (named, value) <- fields, named == key] of+      [value] -> Right value+      [] -> Left ("completion body has no " <> key <> " line")+      _ -> Left ("completion body has more than one " <> key <> " line")+    orRefuse msg = maybe (Left msg) Right +-- | A whole number in canonical decimal: digits only, with no sign and+-- no leading zero.+parseWholeNumber :: Text -> Maybe Integer+parseWholeNumber txt+  | T.length txt > maxWholeNumberDigits = Nothing+  | otherwise = case TR.decimal txt of+      Right (n, _) | showText n == txt -> Just n+      _ -> Nothing+ -- ---------------------------------------------------------------------------+-- Reclaim+-- ---------------------------------------------------------------------------++-- | Whether a reclaim request lists or removes.+data ReclaimMode = ListReclaimable | RemoveReclaimable++-- | Answer @GET \/reclaim@, which lists what @POST \/reclaim@ would+-- remove, or @POST \/reclaim@, which removes it, at the minimum age the+-- query sets ('parseMinimumAge'), with a report in plain text.  A+-- blocked run is a 409 naming what blocks it, and a removal that fails+-- makes the report a 500.  A store that cannot be scanned is the+-- server's own failure, logged and kept out of the response.+reclaimRequest :: ServerConfig -> Request -> (Response -> IO ResponseReceived) -> ReclaimMode -> IO ResponseReceived+reclaimRequest cfg req respond mode = case parseMinimumAge (queryString req) of+  Left err -> do+    logWarn req ("BADQUERY: " <> T.unpack err)+    respond (badRequest err)+  Right minimumAge -> do+    result <- try (run minimumAge)+    case result of+      Left err -> do+        logWarn req ("RECLAIMSCAN: " <> show (err :: IOException))+        respond (responseLBS HTTP.status500 textHeaders "the store could not be scanned")+      Right (Left blocked) -> do+        logWarn req "RECLAIMBLOCKED"+        respond (responseLBS HTTP.status409 uncachedHeaders (textLines (renderBlocked blocked)))+      Right (Right (status, report)) ->+        respond (responseLBS status uncachedHeaders (textLines (("minimum age: " <> counted (wholeHours minimumAge) "hour") : report)))+  where+    store = scStore cfg+    run minimumAge = case mode of+      ListReclaimable -> fmap renderPlan <$> listReclaimable store minimumAge getCurrentTime+      RemoveReclaimable -> fmap renderReclaimed <$> reclaim store minimumAge getCurrentTime++-- | A reclaim's minimum age from its request's query: 'defaultMinimumAge'+-- with no query, or @min-age-hours@ given once, as a whole number of+-- hours in canonical decimal.  Any other parameter is refused rather+-- than ignored, so a misspelled one never runs a removal at an age the+-- operator did not choose.+parseMinimumAge :: HTTP.Query -> Either Text NominalDiffTime+parseMinimumAge query = case query of+  [] -> Right defaultMinimumAge+  [(key, Just value)]+    | key == TE.encodeUtf8 minAgeHoursParam,+      Right digits <- TE.decodeUtf8' value,+      Just hours <- parseWholeNumber digits ->+        Right (fromInteger hours * secondsPerHour)+  _ -> Left ("the query takes one parameter, " <> minAgeHoursParam <> ", a whole number of hours")++-- | Whole hours in an age 'parseMinimumAge' read.+wholeHours :: NominalDiffTime -> Integer+wholeHours age = floor (age / secondsPerHour)++-- | What a listing finds: each NAR and upload a removal would take, then+-- the totals.+renderPlan :: StorePlan -> (HTTP.Status, [Text])+renderPlan plan =+  ( HTTP.status200,+    map (("unreferenced: " <>) . describeNar) nars+      ++ map (("abandoned: " <>) . describeUpload) uploads+      ++ ["eligible: " <> reclaimTotals nars uploads, keptLine plan]+  )+  where+    nars = rpEligible (spNars plan)+    uploads = upEligible (spUploads plan)++-- | What a removal did, each NAR and upload in turn, then the totals:+-- 500 when any removal failed.+renderReclaimed :: Reclaimed -> (HTTP.Status, [Text])+renderReclaimed done =+  ( if null failures then HTTP.status200 else HTTP.status500,+    map (either id id) outcomes+      ++ ["reclaimed: " <> reclaimTotals removedNars removedUploads, keptLine (rdPlan done)]+  )+  where+    outcomes = map narOutcome (rdNars done) ++ map uploadOutcome (rdUploads done)+    failures = lefts outcomes+    removedNars = [nar | (nar, NarRemoved) <- rdNars done]+    removedUploads = [upload | (upload, UploadRemoved) <- rdUploads done]+    narOutcome (nar, outcome) = case outcome of+      NarRemoved -> Right ("removed: " <> describeNar nar)+      NarRemoveFailed err -> Left (failed (narPath (nfName nar)) (T.pack (ioeGetErrorString err)))+      NarRemoveBadPath -> Left (failed (narPath (nfName nar)) "not a name the store holds")+    uploadOutcome (upload, outcome) = case outcome of+      UploadRemoved -> Right ("removed: " <> describeUpload upload)+      UploadRemoveFailed err -> Left (failed (uploadPath upload) (T.pack (ioeGetErrorString err)))+      UploadRemoveBadPath -> Left (failed (uploadPath upload) "not a name the store holds")+    failed path reason = "failed: " <> path <> ": " <> reason++-- | What blocks a reclaim, in a 409's body.+renderBlocked :: ReclaimBlocked -> [Text]+renderBlocked (UnresolvedNarInfos unresolved) =+  map unresolvedLine (toList unresolved)+    ++ [ counted (length unresolved) "unresolved narinfo"+           <> ": a reclaim removes nothing while one is unresolved, since the NAR it names"+           <> " would look unreferenced.  Replace or delete each and try again."+       ]+  where+    unresolvedLine u =+      "unresolved: " <> T.pack (takeFileName (unresolvedPath u)) <> narInfoSuffix <> ": " <> T.pack (unresolvedReason u)+renderBlocked NoNarInfos =+  [ "the store holds NARs and no narinfo: a reclaim removes nothing, since a store that lost"+      <> " its narinfos looks the same as one whose every NAR is unreferenced.  Restore the"+      <> " narinfos; if none was referenced, stop the server and remove the NARs."+  ]++-- | How many NARs and abandoned uploads, and how many bytes each hold.+reclaimTotals :: [NarFile] -> [StagedUpload] -> Text+reclaimTotals nars uploads =+  counted (length nars) "NAR"+    <> ", "+    <> counted (sum (map nfSize nars)) "byte"+    <> "; "+    <> counted (length uploads) "abandoned upload"+    <> ", "+    <> counted (sum (map suSize uploads)) "byte"++-- | What a run keeps for its age.+keptLine :: StorePlan -> Text+keptLine plan =+  "kept, younger than the minimum age: "+    <> counted (length (rpTooYoung (spNars plan))) "NAR"+    <> ", "+    <> counted (length (upTooYoung (spUploads plan))) "upload"++describeNar :: NarFile -> Text+describeNar nar = narPath (nfName nar) <> " (" <> counted (nfSize nar) "byte" <> ")"++describeUpload :: StagedUpload -> Text+describeUpload upload = uploadPath upload <> " (" <> counted (suSize upload) "byte" <> ")"++-- | The request path, relative to the cache root, that names an upload.+uploadPath :: StagedUpload -> Text+uploadPath upload =+  narPath (suFileName upload) <> "/" <> uploadsRouteSegment <> "/" <> renderUploadId (suUploadId upload)++-- --------------------------------------------------------------------------- -- Validation pipeline -- --------------------------------------------------------------------------- @@ -407,34 +857,49 @@     "NAR " <> narPath name <> " is " <> showText stored <> " bytes, narinfo FileSize is " <> showText declared   NarHashMismatch name declared stored ->     "NAR " <> narPath name <> " hashes to " <> formatNixHash stored <> ", narinfo FileHash is " <> formatNixHash declared-  where-    showText = T.pack . show  -- | The request path, relative to the cache root, that serves a NAR. narPath :: Text -> Text narPath name = narRouteSegment <> "/" <> name --- | Gate a narinfo write on the NAR it names (see 'narClaim' and--- 'judgeStoredNar').  A refusal is a 400 naming the problem.  A stored--- NAR that cannot be read is the server's own failure, a 500 like a--- signing failure, with the exception logged and kept out of the--- response.-requireStoredNar :: FileStore -> Request -> (Response -> IO ResponseReceived) -> NarInfo -> IO ResponseReceived -> IO ResponseReceived-requireStoredNar store req respond ni action = case narClaim ni of-  Left refusal -> refuse refusal-  Right claim -> do-    stored <- readStoredNar store (ncFileName claim)-    case stored of-      Left err -> do-        logWarn req ("NARREAD: " <> show err)-        respond (responseLBS HTTP.status500 textHeaders (textBody ("NAR " <> narPath (ncFileName claim) <> " could not be read")))-      Right found -> either refuse (const action) (judgeStoredNar claim found)-  where-    refuse refusal = do-      let msg = renderNarRefusal refusal-      logWarn req ("BADNAR: " <> T.unpack msg)-      respond (badRequest msg)+-- | What a narinfo upload came to once its NAR was checked.+data NarInfoWrite+  = NarInfoWritten+  | NarInfoRefused !NarRefusal+  | NarInfoNarUnreadable !IOException+  | NarInfoSignFailed !String+  | -- | The hash key failed 'sanitizePath'.+    NarInfoBadKey+  deriving (Eq, Show) +-- | Store a narinfo over the NAR it claims: check the stored NAR against+-- the claim ('judgeStoredNar'), sign the narinfo with @signer@ (as+-- 'signNarInfo' does), and write it under the hash key, all within one+-- hold of the store's gate ('withoutRemoval').  So no reclaim removes+-- the NAR between the check and the write: one that starts in between+-- waits for the write, and then finds the narinfo naming the NAR.+storeNarInfo :: FileStore -> (NarInfo -> IO (Either String ByteString)) -> Text -> NarInfo -> NarClaim -> IO NarInfoWrite+storeNarInfo store signer hashKey ni claim = withoutRemoval store $ do+  stored <- readStoredNar store (ncFileName claim)+  case judgeStoredNar claim <$> stored of+    Left err -> pure (NarInfoNarUnreadable err)+    Right (Left refusal) -> pure (NarInfoRefused refusal)+    Right (Right ()) -> do+      signedResult <- signer ni+      case signedResult of+        Left err -> pure (NarInfoSignFailed err)+        Right signed -> do+          ok <- writeNarInfo store hashKey signed+          pure (if ok then NarInfoWritten else NarInfoBadKey)++-- | Refuse a narinfo over the NAR it names (see 'narClaim' and+-- 'judgeStoredNar'): a 400 naming the problem.+refuseNar :: Request -> (Response -> IO ResponseReceived) -> NarRefusal -> IO ResponseReceived+refuseNar req respond refusal = do+  let msg = renderNarRefusal refusal+  logWarn req ("BADNAR: " <> T.unpack msg)+  respond (badRequest msg)+ -- | Read a stored NAR through once in constant memory, for its size and -- SHA-256; 'Nothing' when no file is stored under the name.  The name -- resolves through 'narFilePath', as @GET \/nar\/\<file\>@ does.@@ -482,11 +947,7 @@ withLimitedBody :: Int -> Request -> (Response -> IO ResponseReceived) -> (ByteString -> IO ResponseReceived) -> IO ResponseReceived withLimitedBody limit req respond action = do   bodyResult <- readBodyLimited limit req-  case bodyResult of-    Nothing -> do-      logWarn req "OVERLIMIT"-      respond (responseLBS HTTP.status413 textHeaders "request body too large")-    Just body -> action body+  maybe (overLimit req respond) action bodyResult  -- --------------------------------------------------------------------------- -- Auth@@ -575,10 +1036,57 @@ badRequest :: Text -> Response badRequest msg = responseLBS HTTP.status400 textHeaders (textBody msg) +-- | 200, the answer to every write that succeeds.+okResponse :: Response+okResponse = responseLBS HTTP.status200 textHeaders "ok"++-- | Refuse a request naming a file the store cannot hold.+badPath :: Request -> (Response -> IO ResponseReceived) -> IO ResponseReceived+badPath req respond = do+  logWarn req "BADPATH"+  respond (badRequest "invalid path")++-- | Refuse a request body over its cap.+overLimit :: Request -> (Response -> IO ResponseReceived) -> IO ResponseReceived+overLimit req respond = do+  logWarn req "OVERLIMIT"+  respond (responseLBS HTTP.status413 textHeaders "request body too large")++-- | Refuse a body whose declared @Content-Length@ is over the cap before+-- reading any of it, or run the action.+refuseDeclaredOver :: Int -> Request -> (Response -> IO ResponseReceived) -> IO ResponseReceived -> IO ResponseReceived+refuseDeclaredOver limit req respond action = case requestBodyLength req of+  KnownLength len+    | len > fromIntegral limit -> overLimit req respond+  _ -> action++-- | Refuse a request naming no staged upload of the file: one never+-- begun, already completed, or reclaimed.+unknownUpload :: Request -> (Response -> IO ResponseReceived) -> Text -> Text -> IO ResponseReceived+unknownUpload req respond fileName rawId = do+  logWarn req "NOUPLOAD"+  respond (responseLBS HTTP.status404 textHeaders (textBody ("no upload " <> rawId <> " of " <> narPath fileName)))++-- | Refuse a well-formed request that disagrees with the upload as+-- stored, with 409: the client can resolve it (send the missing part,+-- declare what it sent) and try again.+refuseConflict :: Request -> (Response -> IO ResponseReceived) -> Text -> IO ResponseReceived+refuseConflict req respond msg = do+  logWarn req ("CONFLICT: " <> T.unpack msg)+  respond (responseLBS HTTP.status409 textHeaders (textBody msg))++-- | A value shown as text.+showText :: (Show a) => a -> Text+showText = T.pack . show+ -- | A text response body. textBody :: Text -> BL.ByteString textBody = BL.fromStrict . TE.encodeUtf8 +-- | A text response body of lines.+textLines :: [Text] -> BL.ByteString+textLines = textBody . T.unlines+ -- | Map any uncaught handler exception to a generic 500, so internal error -- detail (filesystem paths, exception text) is never leaked to clients. onExceptionResponse :: e -> Response@@ -588,10 +1096,14 @@ textHeaders :: HTTP.ResponseHeaders textHeaders = [(HTTP.hContentType, "text/plain")] --- | Headers for the authenticated hash listing: push-tool plumbing that--- changes with every upload, so intermediaries must never cache it.-hashListHeaders :: HTTP.ResponseHeaders-hashListHeaders =+htmlHeaders :: HTTP.ResponseHeaders+htmlHeaders = [(HTTP.hContentType, "text/html; charset=utf-8")]++-- | Headers for the authenticated answers that read across the store+-- (the hash listing, a reclaim's report): they change with every upload,+-- so intermediaries must never cache them.+uncachedHeaders :: HTTP.ResponseHeaders+uncachedHeaders =   [ (HTTP.hContentType, "text/plain"),     (HTTP.hCacheControl, "no-store")   ]
src/NovaCache/Store.hs view
@@ -3,15 +3,42 @@ -- Stores narinfo and NAR files in a directory tree: -- -- @--- \<root\>\/narinfo\/\<hash\>   -- narinfo files by store path hash--- \<root\>\/nar\/\<filename\>   -- compressed NAR files+-- \<root\>\/narinfo\/\<hash\>              -- narinfo files by store path hash+-- \<root\>\/nar\/\<filename\>              -- compressed NAR files+-- \<root\>\/uploads\/\<id\>-\<filename\>\/\<n\>  -- parts of a multipart upload in progress -- @+--+-- A multipart upload is completed by a rename from its directory under+-- @uploads\/@ into @nar\/@, so the two share a filesystem, as they do+-- under one store root unless one of them is mounted apart.+--+-- One process serves a store directory, through one t'FileStore', whose+-- gate orders what the store gains against what is removed from it.  A+-- removal runs within 'withRemoval', alone: a narinfo's removal, and a+-- reclaim from its scan through its last removal.  Whatever must not+-- overlap a removal runs within 'withoutRemoval', alongside others: each+-- rename into @nar\/@ ('writeNar', 'writeNarStreaming' and+-- 'completeUpload' take the gate for the rename), and a narinfo upload+-- from the check of its NAR through its write.  So a removal never takes a+-- NAR between that check and that write, and never takes a NAR renamed+-- into place after it listed the store.  Parts of an upload are staged+-- outside the gate: no narinfo can name them. module NovaCache.Store-  ( FileStore (..),+  ( FileStore,+    fsRoot,+    fsStoreDir,+    fsPriority,     newFileStore,-    fileStoreAt,++    -- * Additions and removals+    withoutRemoval,+    withRemoval,++    -- * Narinfos and NARs     readNarInfo,     writeNarInfo,+    NarInfoRemoveResult (..),+    removeNarInfo,     readNar,     writeNar,     NarWriteResult (..),@@ -23,46 +50,83 @@     listNarInfoFiles,     NarFile (..),     listNarFiles,++    -- * Multipart uploads+    UploadId,+    parseUploadId,+    renderUploadId,+    PartNumber,+    maxUploadParts,+    parsePartNumber,+    renderPartNumber,+    beginUpload,+    PartWriteResult (..),+    writeUploadPart,+    UploadTotals (..),+    CompletionResult (..),+    completeUpload,+    StagedUpload (..),+    listStagedUploads,+    UploadRemoveResult (..),+    removeUpload,++    -- * Metadata and names     CacheInfo (..),     getCacheInfo,     sanitizePath,   ) where -import Control.Exception (IOException, SomeException, catch, onException, try)+import Control.Concurrent.STM (TVar, atomically, check, modifyTVar', newTVarIO, readTVar, writeTVar)+import Control.Exception (IOException, bracket_, catch, onException, try, tryJust)+import Control.Monad (foldM, guard, join)+import Crypto.Random (getRandomBytes) import Data.ByteString (ByteString) import qualified Data.ByteString as BS+import qualified Data.ByteString.Builder as Builder+import qualified Data.ByteString.Lazy as BL import Data.Char (isAsciiLower, isAsciiUpper, isDigit) import Data.List (isPrefixOf)-import Data.Maybe (catMaybes)+import Data.Maybe (catMaybes, fromMaybe)+import qualified Data.Set as Set import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.Encoding as TE+import qualified Data.Text.Read as TR import Data.Time.Clock (UTCTime) import NovaCache.SafeName (hasTrailingDotOrSpace, isReservedDeviceName) import System.Directory-  ( createDirectoryIfMissing,+  ( createDirectory,+    createDirectoryIfMissing,+    doesDirectoryExist,     doesFileExist,     getFileSize,     getModificationTime,     listDirectory,+    removeDirectoryRecursive,     removeFile,     renameFile,   ) import System.FilePath (takeDirectory, (</>))-import System.IO (hClose, openBinaryTempFile)+import System.IO (Handle, IOMode (ReadMode), hClose, openBinaryTempFile, withBinaryFile)+import System.IO.Error (isDoesNotExistError)  -- --------------------------------------------------------------------------- -- Types -- --------------------------------------------------------------------------- --- | Filesystem-backed cache store configuration.+-- | A filesystem-backed cache store: its root directory, what+-- @nix-cache-info@ reports, and the gate that orders its additions+-- against its removals.  'newFileStore' makes one. data FileStore = FileStore-  { fsRoot :: !FilePath,+  { -- | The directory that holds @narinfo\/@, @nar\/@ and @uploads\/@.+    fsRoot :: !FilePath,+    -- | The Nix store directory @nix-cache-info@ reports.     fsStoreDir :: !Text,-    fsPriority :: !Int+    -- | The priority @nix-cache-info@ reports; Nix prefers a lower one.+    fsPriority :: !Int,+    fsGate :: !(TVar GateState)   }-  deriving (Show)  -- --------------------------------------------------------------------------- -- Constants@@ -86,36 +150,111 @@ defaultStoreDir :: Text defaultStoreDir = "/nix/store" +-- | Subdirectory for multipart uploads in progress.+uploadsSubdir :: String+uploadsSubdir = "uploads"+ -- | Prefix for temporary files used during atomic writes. tempFilePrefix :: String tempFilePrefix = ".nova.tmp" +-- | Bytes read per step while joining an upload's parts.+joinReadBytes :: Int+joinReadBytes = 128 * 1024++-- | The longest file name ext4, APFS and NTFS admit: 255 bytes,+-- characters or UTF-16 units by filesystem, which agree for the ASCII+-- names a store holds.+maxFileNameLength :: Int+maxFileNameLength = 255+ -- --------------------------------------------------------------------------- -- Initialization -- --------------------------------------------------------------------------- --- | Create a 't:FileStore' rooted at the given directory.+-- | Create a 't:FileStore' rooted at the given directory, with a gate of+-- its own. ----- Ensures the @narinfo@ and @nar@ subdirectories exist.+-- Ensures the @narinfo@, @nar@ and @uploads@ subdirectories exist. newFileStore :: FilePath -> IO FileStore newFileStore root = do   createDirectoryIfMissing True (root </> narinfoSubdir)   createDirectoryIfMissing True (root </> narSubdir)-  pure (fileStoreAt root)+  createDirectoryIfMissing True (root </> uploadsSubdir)+  gate <- newTVarIO GateState {gsHolders = 0, gsRemoval = NoRemoval}+  pure+    FileStore+      { fsRoot = root,+        fsStoreDir = defaultStoreDir,+        fsPriority = defaultPriority,+        fsGate = gate+      } --- | A 't:FileStore' over a directory tree as it stands, creating--- nothing.  For a host tool that reads the store and acts on what it--- finds: a mistyped root, or a store missing its @narinfo@ directory,--- must fail the listing, not be scanned as an empty store whose every--- NAR looks unreferenced.-fileStoreAt :: FilePath -> FileStore-fileStoreAt root =-  FileStore-    { fsRoot = root,-      fsStoreDir = defaultStoreDir,-      fsPriority = defaultPriority-    }+-- ---------------------------------------------------------------------------+-- Additions and removals+-- --------------------------------------------------------------------------- +-- | Who holds a store's gate.+data GateState = GateState+  { -- | Actions running under 'withoutRemoval'.+    gsHolders :: !Int,+    gsRemoval :: !RemovalState+  }++-- | Where a removal stands.  A removal waiting for the holders to finish+-- admits no new holder, so a steady run of uploads cannot hold it off.+data RemovalState+  = NoRemoval+  | RemovalWaiting+  | RemovalRunning++-- | Run an action during which nothing is removed from the store.  Such+-- actions run together; one that starts while a removal runs or waits to+-- run waits for it to finish.+--+-- The gate is not re-entrant: the action must not call 'withoutRemoval'+-- or 'withRemoval', or an operation that takes the gate itself, since a+-- removal that begins waiting in between would wait for the action+-- while the action waits for it.+withoutRemoval :: FileStore -> IO a -> IO a+withoutRemoval fs = bracket_ (atomically enter) (atomically leave)+  where+    gate = fsGate fs+    enter = do+      state <- readTVar gate+      check (noRemoval (gsRemoval state))+      writeTVar gate $! state {gsHolders = gsHolders state + 1}+    leave = modifyTVar' gate (\state -> state {gsHolders = gsHolders state - 1})++-- | Run a removal: alone, once every action under 'withoutRemoval' that+-- began before it has finished.  Removals run one at a time.  A removal+-- interrupted while it waits withdraws, so the actions it was holding+-- off go ahead.  Not re-entrant, as 'withoutRemoval' is not.+withRemoval :: FileStore -> IO a -> IO a+withRemoval fs = bracket_ claim (atomically release)+  where+    gate = fsGate fs+    claim = do+      atomically $ do+        state <- readTVar gate+        check (noRemoval (gsRemoval state))+        writeTVar gate $! state {gsRemoval = RemovalWaiting}+      atomically begin `onException` atomically release+    begin = do+      state <- readTVar gate+      check (gsHolders state == 0)+      writeTVar gate $! state {gsRemoval = RemovalRunning}+    release = modifyTVar' gate (\state -> state {gsRemoval = NoRemoval})++noRemoval :: RemovalState -> Bool+noRemoval NoRemoval = True+noRemoval RemovalWaiting = False+noRemoval RemovalRunning = False++-- | Rename a finished file into @nar\/@, where a removal may be listing.+renameIntoNars :: FileStore -> FilePath -> FilePath -> IO ()+renameIntoNars fs from to = withoutRemoval fs (renameFile from to)+ -- --------------------------------------------------------------------------- -- NarInfo operations -- ---------------------------------------------------------------------------@@ -136,8 +275,37 @@ writeNarInfo :: FileStore -> Text -> ByteString -> IO Bool writeNarInfo fs hashKey body = case sanitizePath hashKey of   Nothing -> pure False-  Just safe -> atomicWriteFile (fsRoot fs </> narinfoSubdir </> safe) body >> pure True+  Just safe -> do+    let target = fsRoot fs </> narinfoSubdir </> safe+    atomicWriteFile (takeDirectory target) body (`renameFile` target)+    pure True +-- | Outcome of removing a narinfo.+data NarInfoRemoveResult+  = -- | The narinfo is gone.+    NarInfoRemoved+  | -- | No narinfo is stored under the hash.+    NarInfoAbsent+  | -- | The filesystem refused; the narinfo is still stored.+    NarInfoRemoveFailed !IOException+  | -- | The hash failed 'sanitizePath'; nothing was touched.+    NarInfoRemoveBadPath+  deriving (Eq, Show)++-- | Remove a narinfo by its store path hash, leaving the NAR it names+-- for a reclaim ("NovaCache.Reclaim") to remove once nothing names it.+-- A removal: call it within 'withRemoval'.+removeNarInfo :: FileStore -> Text -> IO NarInfoRemoveResult+removeNarInfo fs hashKey = case sanitizePath hashKey of+  Nothing -> pure NarInfoRemoveBadPath+  Just safe -> do+    removed <- tryIO (removeFile (fsRoot fs </> narinfoSubdir </> safe))+    pure $ case removed of+      Right () -> NarInfoRemoved+      Left err+        | isDoesNotExistError err -> NarInfoAbsent+        | otherwise -> NarInfoRemoveFailed err+ -- --------------------------------------------------------------------------- -- NAR operations -- ---------------------------------------------------------------------------@@ -153,12 +321,15 @@ -- | Write a NAR file by its filename. -- -- Uses atomic write-to-temp-then-rename so concurrent readers never--- see partial content.+-- see partial content, and takes the gate for the rename. -- Returns 'False' for path components containing traversal sequences. writeNar :: FileStore -> Text -> ByteString -> IO Bool writeNar fs fileName body = case sanitizePath fileName of   Nothing -> pure False-  Just safe -> atomicWriteFile (fsRoot fs </> narSubdir </> safe) body >> pure True+  Just safe -> do+    let target = fsRoot fs </> narSubdir </> safe+    atomicWriteFile (takeDirectory target) body (\tmpPath -> renameIntoNars fs tmpPath target)+    pure True  -- | Outcome of a streaming NAR write. data NarWriteResult@@ -175,28 +346,17 @@ -- input), enforcing a total-size cap as bytes arrive so the body is never -- held in memory.  Same atomic temp-then-rename discipline as 'writeNar': -- readers see the old file or the complete new one, never a partial write.+-- Only the rename takes the gate, so a removal never waits on a client+-- sending its body. writeNarStreaming :: FileStore -> Text -> Int -> IO ByteString -> IO NarWriteResult writeNarStreaming fs fileName limit nextChunk = case sanitizePath fileName of   Nothing -> pure NarWriteBadPath   Just safe -> do     let target = fsRoot fs </> narSubdir </> safe-    (tmpPath, handle) <- openBinaryTempFile (takeDirectory target) tempFilePrefix-    let cleanup = do-          ignoringExceptions (hClose handle)-          ignoringExceptions (removeFile tmpPath)-        consume !total = do-          chunk <- nextChunk-          if BS.null chunk-            then do-              hClose handle-              renameFile tmpPath target-              pure NarWriteOk-            else-              let grown = total + BS.length chunk-               in if grown > limit-                    then cleanup >> pure NarWriteTooLarge-                    else BS.hPut handle chunk >> consume grown-    consume 0 `onException` cleanup+    streamed <- streamIntoPlace (takeDirectory target) limit nextChunk (\tmpPath -> renameIntoNars fs tmpPath target)+    pure $ case streamed of+      StreamWritten -> NarWriteOk+      StreamOverLimit -> NarWriteTooLarge  -- | The on-disk path of a stored NAR, if present.  Lets a server hand the -- file to its transport layer (e.g. WAI's @responseFile@, which streams@@ -220,9 +380,8 @@     NarRemoveBadPath   deriving (Eq, Show) --- | Remove a stored NAR by its filename.  Nothing in the HTTP protocol--- calls this: deletion is for a tool the host operator runs against--- the store directory.+-- | Remove a stored NAR by its filename.  A removal: call it within+-- 'withRemoval', after finding that nothing names the NAR. removeNar :: FileStore -> Text -> IO NarRemoveResult removeNar fs fileName = case sanitizePath fileName of   Nothing -> pure NarRemoveBadPath@@ -293,6 +452,354 @@           else pure Nothing  -- ---------------------------------------------------------------------------+-- Multipart uploads+-- ---------------------------------------------------------------------------++-- | Names one multipart upload: 16 random bytes, as 32 lowercase hex+-- digits.  The constructor is not exported, so every value is+-- one 'beginUpload' drew or 'parseUploadId' accepted, and its text is+-- safe in a file name.+newtype UploadId = UploadId Text+  deriving (Eq, Ord, Show)++-- | Random bytes in an upload id.  At 128 bits, ids drawn independently+-- do not collide in practice, so two pushes uploading the same NAR at+-- once stage their parts apart, with no record kept of the ids issued.+uploadIdBytes :: Int+uploadIdBytes = 16++-- | Hex digits in an upload id.+uploadIdDigits :: Int+uploadIdDigits = 2 * uploadIdBytes++-- | An upload id as a request names it: exactly 32 lowercase hex+-- digits.+parseUploadId :: Text -> Maybe UploadId+parseUploadId txt+  | T.length txt == uploadIdDigits && T.all isLowerHexDigit txt = Just (UploadId txt)+  | otherwise = Nothing+  where+    isLowerHexDigit c = isDigit c || (c >= 'a' && c <= 'f')++-- | An upload id as the server issues it and a request names it.+renderUploadId :: UploadId -> Text+renderUploadId (UploadId txt) = txt++-- | Draw a fresh upload id from the system's random source.+newUploadId :: IO UploadId+newUploadId = do+  bytes <- getRandomBytes uploadIdBytes+  pure (UploadId (TE.decodeLatin1 (BL.toStrict (Builder.toLazyByteString (Builder.byteStringHex bytes)))))++-- | A part's place in its upload, from 1 to 'maxUploadParts'.  The+-- constructor is not exported: every value is one 'parsePartNumber'+-- accepted.+newtype PartNumber = PartNumber Int+  deriving (Eq, Ord, Show)++-- | The most parts one upload holds, S3's limit.  Each part is a file in+-- the upload's staging directory, so this bounds the files one upload+-- creates.+maxUploadParts :: Int+maxUploadParts = 10000++-- | A part number in canonical decimal, from 1 to 'maxUploadParts':+-- digits only, with no sign and no leading zero, so each part has one+-- spelling in a request path and in its staging directory.  The length+-- is bounded before the parse, so an overlong number cannot wrap 'Int'+-- into range.+parsePartNumber :: Text -> Maybe PartNumber+parsePartNumber txt+  | T.length txt > T.length (renderInt maxUploadParts) = Nothing+  | otherwise = case TR.decimal txt of+      Right (n, _)+        | renderInt n == txt && n >= 1 && n <= maxUploadParts -> Just (PartNumber n)+      _ -> Nothing++-- | A part number in the canonical decimal 'parsePartNumber' reads.+renderPartNumber :: PartNumber -> Text+renderPartNumber (PartNumber n) = renderInt n++renderInt :: Int -> Text+renderInt = T.pack . show++-- | Joins an upload's id to its NAR's name in the name of its staging+-- directory.+stagingSeparator :: Char+stagingSeparator = '-'++-- | The staging directory of an upload of a NAR, by the name+-- 'sanitizePath' admitted: @uploads\/\<id\>-\<name\>@.  The NAR's name is+-- part of the directory's, so an upload is bound to the NAR it was+-- begun for, and a request naming another NAR finds no upload.+stagingDir :: FileStore -> FilePath -> UploadId -> FilePath+stagingDir fs safe (UploadId uid) =+  fsRoot fs </> uploadsSubdir </> (T.unpack uid ++ stagingSeparator : safe)++-- | The upload a directory under @uploads\/@ stages, by the directory's+-- name: its id and its NAR's name, or 'Nothing' for a name 'stagingDir'+-- never makes.+parseStagingName :: FilePath -> Maybe (UploadId, Text)+parseStagingName entry = case splitAt uploadIdDigits entry of+  (idDigits, separator : name)+    | separator == stagingSeparator,+      Just uid <- parseUploadId (T.pack idDigits),+      Just _ <- sanitizePath (T.pack name) ->+        Just (uid, T.pack name)+  _ -> Nothing++-- | A part's file in its upload's staging directory, named by its number.+partPath :: FilePath -> Int -> FilePath+partPath dir n = dir </> T.unpack (renderInt n)++-- | The parts an upload holds, in order: the files named 1, 2, and so on,+-- up to the first number missing.  Temp files are dotfiles, never part+-- numbers.  'Nothing' when the upload's directory does not exist.+listStagedParts :: FilePath -> IO (Maybe [(Int, FilePath)])+listStagedParts dir = fmap inOrder <$> ifPresent (listDirectory dir)+  where+    inOrder entries =+      let held = Set.fromList [n | Just (PartNumber n) <- map (parsePartNumber . T.pack) entries]+       in [(n, partPath dir n) | n <- takeWhile (`Set.member` held) [1 ..]]++-- | Begin a multipart upload of a NAR: an empty staging directory under+-- @uploads\/@, outside @nar\/@, so nothing an upload holds is served+-- before 'completeUpload' joins it.  'Nothing' when the name fails+-- 'sanitizePath', or is too long to stage: the directory's name is the+-- NAR's with the id before it.  'createDirectory' fails on a directory+-- that exists, so a drawn id never joins another upload.+beginUpload :: FileStore -> Text -> IO (Maybe UploadId)+beginUpload fs fileName = traverse begin (sanitizePath fileName >>= stageable)+  where+    stageable safe = safe <$ guard (uploadIdDigits + 1 + length safe <= maxFileNameLength)+    begin safe = do+      uid <- newUploadId+      createDirectory (stagingDir fs safe uid)+      pure uid++-- | Outcome of writing one part of an upload.+data PartWriteResult+  = -- | Written whole, in place of any earlier part of the same number.+    PartWritten+  | -- | The name failed 'sanitizePath'; nothing was written.+    PartBadPath+  | -- | No upload of that id is staged for that name: never begun,+    -- completed, or removed.+    PartUploadUnknown+  | -- | The part would leave a gap: the upload holds this many parts, so+    -- the next is one more.  Nothing was written.+    PartOutOfOrder !Int+  | -- | The upload's parts would hold more than the cap; the partial part+    -- was deleted and the parts held are unchanged.+    PartTooLarge+  deriving (Eq, Show)++-- | Stream one part of an upload into its staging directory, as+-- 'writeNarStreaming' streams a whole NAR: in constant memory, to a temp+-- file renamed into place only once complete, so a part is held whole+-- or not at all.  Parts arrive in order: part @n@ is accepted when the+-- upload holds parts 1 to @n - 1@, and a part it already holds is+-- replaced, so a client that lost a response can send the part again.+-- The cap bounds the part with the parts the upload holds when it+-- begins; parts written at once can pass it together, and+-- 'completeUpload' holds the joined NAR to it.+writeUploadPart :: FileStore -> Text -> UploadId -> PartNumber -> Int -> IO ByteString -> IO PartWriteResult+writeUploadPart fs fileName uid part@(PartNumber n) limit nextChunk = case sanitizePath fileName of+  Nothing -> pure PartBadPath+  Just safe -> do+    let dir = stagingDir fs safe uid+    staged <- listStagedParts dir+    maybe (pure PartUploadUnknown) (writePart dir) staged+  where+    writePart dir parts+      | n > length parts + 1 = pure (PartOutOfOrder (length parts))+      | otherwise = do+          others <- sum <$> traverse getFileSize [path | (number, path) <- parts, number /= n]+          let allowance = toInteger limit - others+              target = dir </> T.unpack (renderPartNumber part)+          streamed <-+            if allowance < 0+              then pure StreamOverLimit+              else streamIntoPlace dir (fromInteger allowance) nextChunk (`renameFile` target)+          pure $ case streamed of+            StreamWritten -> PartWritten+            StreamOverLimit -> PartTooLarge++-- | What a client declares when it completes an upload: how many parts+-- it sent, which is the number of its last part, and how many bytes+-- they hold together.+data UploadTotals = UploadTotals+  { utParts :: !PartNumber,+    utFileSize :: !Integer+  }+  deriving (Eq, Show)++-- | Outcome of completing an upload.+data CompletionResult+  = -- | The parts are joined into @nar\/\<file\>@ and the upload is gone.+    UploadCompleted+  | -- | The parts are joined into @nar\/\<file\>@, but the upload's+    -- directory could not be removed, so 'listStagedUploads' still+    -- lists it.+    UploadCompletedStagingKept !IOException+  | -- | The name failed 'sanitizePath'; nothing changed.+    CompletionBadPath+  | -- | No upload of that id is staged for that name; nothing changed.+    CompletionUploadUnknown+  | -- | The upload holds another number of parts than declared; nothing+    -- changed. (declared, held)+    CompletionPartsMismatch !Int !Int+  | -- | The parts join to another size than declared; nothing changed+    -- under @nar\/@ and the parts are kept. (declared, joined)+    CompletionSizeMismatch !Integer !Integer+  | -- | The declared size is over the cap; nothing changed and the parts+    -- are kept. (declared)+    CompletionTooLarge !Integer+  deriving (Eq, Show)++-- | Complete an upload: check that it holds the declared number of+-- parts and that the declared size is within the cap, join the parts in+-- order into a temp file in its directory, check the bytes joined+-- against the declared size, and only then rename the file to+-- @nar\/\<file\>@.  A NAR appears there whole or not at all, and a+-- missing part or a short one fails here rather than at the narinfo.+-- Each part write holds the parts to the cap as it finds them, which two+-- writes at once can each pass, so the cap is checked again here: the+-- joined size must equal the declared one, so no NAR over the cap is+-- renamed into place.  The join reads each part once, in constant+-- memory, and only the rename takes the gate.+completeUpload :: FileStore -> Text -> UploadId -> Int -> UploadTotals -> IO CompletionResult+completeUpload fs fileName uid limit (UploadTotals (PartNumber declaredParts) declaredSize) =+  case sanitizePath fileName of+    Nothing -> pure CompletionBadPath+    Just safe -> do+      let dir = stagingDir fs safe uid+      staged <- listStagedParts dir+      maybe (pure CompletionUploadUnknown) (complete safe dir . map snd) staged+  where+    complete safe dir parts+      | declaredSize > toInteger limit = pure (CompletionTooLarge declaredSize)+      | length parts /= declaredParts = pure (CompletionPartsMismatch declaredParts (length parts))+      | otherwise = do+          joined <- joinThen dir parts (placeJoined safe)+          case joined of+            UploadCompleted ->+              either UploadCompletedStagingKept (const UploadCompleted)+                <$> tryIO (removeDirectoryRecursive dir)+            refused -> pure refused+    placeJoined safe joinedPath size+      | size /= declaredSize = CompletionSizeMismatch declaredSize size <$ removeFile joinedPath+      | otherwise = UploadCompleted <$ renameIntoNars fs joinedPath (fsRoot fs </> narSubdir </> safe)++-- | Concatenate files, in order, into a new temp file in a directory, in+-- constant memory, then hand the closed file and the bytes it holds to+-- @finish@, which moves it into place or removes it.  If the join or+-- @finish@ fails, the temp file is removed, so a failed rename leaves no+-- copy behind.+joinThen :: FilePath -> [FilePath] -> (FilePath -> Integer -> IO a) -> IO a+joinThen dir paths finish = do+  (tmpPath, handle) <- openBinaryTempFile dir tempFilePrefix+  ( do+      size <- foldM (appendFileTo handle) 0 paths+      hClose handle+      finish tmpPath size+    )+    `onException` discardTemp tmpPath handle++-- | Append a file's bytes to a handle in constant memory, adding their+-- count to a running total.+appendFileTo :: Handle -> Integer -> FilePath -> IO Integer+appendFileTo out total path = withBinaryFile path ReadMode (copy total)+  where+    copy !count input = do+      chunk <- BS.hGetSome input joinReadBytes+      if BS.null chunk+        then pure count+        else BS.hPut out chunk >> copy (count + toInteger (BS.length chunk)) input++-- | A multipart upload as staged: begun, and neither completed nor+-- removed.+data StagedUpload = StagedUpload+  { -- | The NAR it was begun for, as @POST \/nar\/\<file\>\/uploads@ named it.+    suFileName :: !Text,+    suUploadId :: !UploadId,+    -- | Its staging directory.+    suPath :: !FilePath,+    -- | Bytes its files hold: its parts, and any part or join in progress.+    suSize :: !Integer,+    -- | Its last activity: the latest modification time of its directory+    -- and of the files in it.+    suModified :: !UTCTime+  }+  deriving (Eq, Show)++-- | Every staged upload: the directories under @uploads\/@ named+-- @\<id\>-\<file\>@ for an upload id and a NAR's name.  A store with no+-- @uploads\/@ directory, as a server older than multipart uploads leaves+-- one, holds none.  An upload in progress renames its temp files and a+-- completion removes its directory, so a file or directory that+-- vanishes while it is read is activity, skipped rather than failing+-- the listing.+listStagedUploads :: FileStore -> IO [StagedUpload]+listStagedUploads fs = do+  entries <- fromMaybe [] <$> ifPresent (listDirectory dir)+  catMaybes <$> traverse describe entries+  where+    dir = fsRoot fs </> uploadsSubdir+    describe entry = case parseStagingName entry of+      Nothing -> pure Nothing+      Just (uid, name) -> do+        let path = dir </> entry+        isDir <- doesDirectoryExist path+        if isDir+          then fmap (staged uid name path) <$> measureStaging path+          else pure Nothing+    staged uid name path (size, modified) =+      StagedUpload+        { suFileName = name,+          suUploadId = uid,+          suPath = path,+          suSize = size,+          suModified = modified+        }++-- | The bytes an upload's directory holds and its last activity, or+-- 'Nothing' when the directory is gone.  The directory's own time is+-- read after its files, so a part renamed into place while they are+-- read still counts as activity.+measureStaging :: FilePath -> IO (Maybe (Integer, UTCTime))+measureStaging path = do+  listed <- ifPresent (listDirectory path)+  join <$> traverse measure listed+  where+    measure names = do+      files <- catMaybes <$> traverse (ifPresent . sizeAndTime . (path </>)) names+      dirTime <- ifPresent (getModificationTime path)+      pure (fmap (\time -> (sum (map fst files), foldl' max time (map snd files))) dirTime)+    sizeAndTime file = (,) <$> getFileSize file <*> getModificationTime file++-- | Outcome of removing a staged upload.+data UploadRemoveResult+  = -- | The upload and every part it held are gone.+    UploadRemoved+  | -- | The filesystem refused; what remains is still listed.+    UploadRemoveFailed !IOException+  | -- | The name failed 'sanitizePath'; nothing was touched.+    UploadRemoveBadPath+  deriving (Eq, Show)++-- | Remove a staged upload and every part it holds.  A reclaim calls it+-- within 'withRemoval', but parts are written outside the gate, so when+-- a part is written to the upload at the same moment, either the part's+-- write or the removal fails.+removeUpload :: FileStore -> Text -> UploadId -> IO UploadRemoveResult+removeUpload fs fileName uid = case sanitizePath fileName of+  Nothing -> pure UploadRemoveBadPath+  Just safe ->+    either UploadRemoveFailed (const UploadRemoved)+      <$> tryIO (removeDirectoryRecursive (stagingDir fs safe uid))++-- --------------------------------------------------------------------------- -- Cache metadata -- --------------------------------------------------------------------------- @@ -354,24 +861,59 @@  -- | Write a file atomically via write-to-temp-then-rename. ----- Writes to a temporary file in the same directory, then renames to--- the target path. On POSIX, @rename@ is atomic - readers see either--- the old content or the new content, never a partial write.--- Cleans up the temporary file on failure.-atomicWriteFile :: FilePath -> ByteString -> IO ()-atomicWriteFile target content = do-  let dir = takeDirectory target+-- Writes to a new temp file in a directory, then hands the closed file+-- to @commit@, which renames it to its target in the same directory. On+-- POSIX, @rename@ is atomic - readers see either the old content or the+-- new content, never a partial write.  Cleans up the temporary file on+-- failure.+atomicWriteFile :: FilePath -> ByteString -> (FilePath -> IO ()) -> IO ()+atomicWriteFile dir content commit = do   (tmpPath, h) <- openBinaryTempFile dir tempFilePrefix-  let cleanup = do-        ignoringExceptions (hClose h)-        ignoringExceptions (removeFile tmpPath)   ( do       BS.hPut h content       hClose h-      renameFile tmpPath target+      commit tmpPath     )-    `onException` cleanup+    `onException` discardTemp tmpPath h +-- | Whether a stream fit under its cap.+data Streamed = StreamWritten | StreamOverLimit++-- | Stream chunks (an empty chunk ends the input) into a new temp file in+-- a directory under a size cap, never holding the body in memory, then+-- hand the closed file to @commit@, which renames it into place.  Past+-- the cap, or on an exception, the temp file is removed and nothing is+-- committed.+streamIntoPlace :: FilePath -> Int -> IO ByteString -> (FilePath -> IO ()) -> IO Streamed+streamIntoPlace dir limit nextChunk commit = do+  (tmpPath, handle) <- openBinaryTempFile dir tempFilePrefix+  let consume !total = do+        chunk <- nextChunk+        if BS.null chunk+          then do+            hClose handle+            commit tmpPath+            pure StreamWritten+          else+            let grown = total + BS.length chunk+             in if grown > limit+                  then discardTemp tmpPath handle >> pure StreamOverLimit+                  else BS.hPut handle chunk >> consume grown+  consume 0 `onException` discardTemp tmpPath handle++-- | Close and remove a temp file a write abandons, past its cap or while+-- its own failure propagates.  Errors here are dropped, so they never+-- replace that failure.+discardTemp :: FilePath -> Handle -> IO ()+discardTemp tmpPath handle = do+  ignoringExceptions (hClose handle)+  ignoringExceptions (removeFile tmpPath)++-- | Run a filesystem read, or 'Nothing' when what it reads does not+-- exist; any other failure propagates.+ifPresent :: IO a -> IO (Maybe a)+ifPresent action = either (const Nothing) Just <$> tryJust (guard . isDoesNotExistError) action+ -- | A dotfile: every temp file the atomic writes create (the -- template's leading dot survives 'openBinaryTempFile'), and never a -- key 'sanitizePath' admits.@@ -383,11 +925,12 @@ tryIO :: IO a -> IO (Either IOException a) tryIO = try --- | Swallow all exceptions from a cleanup action.+-- | Drop the 'IOException' a cleanup action raises; anything else, an+-- interrupt among them, propagates. ignoringExceptions :: IO () -> IO () ignoringExceptions action = action `catch` silenceException   where-    silenceException :: SomeException -> IO ()+    silenceException :: IOException -> IO ()     silenceException _ = pure ()  -- | Read a file if it exists, returning 'Nothing' otherwise.
src/NovaCache/Xz.hs view
@@ -161,9 +161,8 @@ -- throws, or a consumer that exits early, holds the decoder state -- (bounded by 'xzMaxDecoderMemoryBytes') until a GC runs the -- finalizer.  The binding keeps @endLzmaStream@ in a hidden module,--- so nothing here can end a live stream early.  Once xz exports it--- (#46), this becomes a real bracket that ends the stream on every--- exit.+-- so nothing here can end a live stream early.  Once xz exports it,+-- this becomes a real bracket that ends the stream on every exit. withXzSource :: XzLimits -> IO ByteString -> (IO ByteString -> IO a) -> IO a withXzSource limits compressedSource consume = do   start <- Lzma.decompressIO (decompressParams limits)
test/Main.hs view
@@ -1,2348 +1,3659 @@-{-# LANGUAGE LambdaCase #-}--module Main (main) where--import Control.Exception (IOException, SomeException, try)-import qualified Crypto.PubKey.Ed25519 as Ed25519-import Data.Bits (shiftR, (.&.))-import Data.ByteArray (convert)-import Data.ByteString (ByteString)-import qualified Data.ByteString as BS-import qualified Data.ByteString.Base64 as B64-import qualified Data.ByteString.Builder as Builder-import qualified Data.ByteString.Lazy as BL-import Data.Either (isLeft, isRight)-import Data.Foldable (toList)-import Data.IORef (atomicModifyIORef', newIORef, readIORef)-import Data.List (isSuffixOf, sort)-import Data.Maybe (isJust)-import Data.Set (Set)-import qualified Data.Set as Set-import Data.Text (Text)-import qualified Data.Text as T-import qualified Data.Text.Encoding as TE-import Data.Time.Calendar (fromGregorian)-import Data.Time.Clock (NominalDiffTime, UTCTime (..), addUTCTime, diffUTCTime, getCurrentTime, nominalDay)-import Data.Word (Word64, Word8)-import qualified Network.HTTP.Types as HTTP-import Network.Wai (RequestBodyLength (..), defaultRequest, pathInfo, requestBodyLength, requestHeaders, requestMethod)-import qualified Network.Wai.Test as WT-import qualified NovaCache.Base32 as Base32-import qualified NovaCache.Hash as Hash-import qualified NovaCache.NAR as NAR-import qualified NovaCache.NAR.Stream as Stream-import qualified NovaCache.NarInfo as NarInfo-import qualified NovaCache.Reclaim as Reclaim-import qualified NovaCache.Server as Server-import qualified NovaCache.Signing as Signing-import qualified NovaCache.Store as Store-import qualified NovaCache.StorePath as StorePath-import qualified NovaCache.Validate as Validate-import System.Directory (createDirectory, createFileLink, doesDirectoryExist, emptyPermissions, getPermissions, getTemporaryDirectory, listDirectory, readable, removeDirectoryRecursive, removeFile, setModificationTime, setOwnerExecutable, setPermissions)-import System.Exit (exitFailure, exitSuccess)-import System.IO (hFlush, stdout)-import qualified System.Info-import Test.QuickCheck (Args (chatty), Gen, Result (Failure, GaveUp, NoExpectedFailure, Success, output), Testable, arbitrary, choose, elements, forAll, frequency, listOf, listOf1, oneof, property, quickCheckWithResult, stdArgs, sublistOf, suchThat, vectorOf, (===))---- ------------------------------------------------------------------------------ Test harness (hand-rolled, no framework)--- ------------------------------------------------------------------------------- | Run a named test, short-circuit on first failure.-test :: String -> IO Bool -> IO Bool-test name action = do-  putStr ("  " ++ name ++ "... ")-  hFlush stdout-  result <- action-  if result-    then do-      putStrLn "OK"-      pure True-    else do-      putStrLn "FAILED"-      pure False---- | Assert equality.-assertEqual :: (Eq a, Show a) => String -> a -> a -> IO Bool-assertEqual label expected actual-  | expected == actual = pure True-  | otherwise = do-      putStrLn ""-      putStrLn ("    " ++ label)-      putStrLn ("    expected: " ++ show expected)-      putStrLn ("    actual:   " ++ show actual)-      pure False---- | Assert a Right value matches.-assertRight :: (Eq a, Show a) => String -> a -> Either String a -> IO Bool-assertRight label expected (Right actual) = assertEqual label expected actual-assertRight label _ (Left err) = do-  putStrLn ""-  putStrLn ("    " ++ label)-  putStrLn ("    expected Right, got Left: " ++ err)-  pure False---- | Assert a Left (error case).-assertLeft :: (Show a) => String -> Either String a -> IO Bool-assertLeft _ (Left _) = pure True-assertLeft label (Right val) = do-  putStrLn ""-  putStrLn ("    " ++ label)-  putStrLn ("    expected Left, got Right: " ++ show val)-  pure False---- | Assert a Bool is True.-assertTrue :: String -> Bool -> IO Bool-assertTrue _ True = pure True-assertTrue label False = do-  putStrLn ""-  putStrLn ("    " ++ label ++ ": expected True")-  pure False---- | Assert a Bool is False.-assertFalse :: String -> Bool -> IO Bool-assertFalse _ False = pure True-assertFalse label True = do-  putStrLn ""-  putStrLn ("    " ++ label ++ ": expected False")-  pure False---- | Run a group of tests, stopping at first failure.-runGroup :: String -> [IO Bool] -> IO Bool-runGroup name tests = do-  putStrLn (name ++ ":")-  go tests-  where-    go [] = pure True-    go (t : ts) = do-      ok <- t-      if ok then go ts else pure False---- | Run all test groups.-runAll :: [IO Bool] -> IO ()-runAll groups = do-  results <- sequence groups-  let passed = length (filter id results)-      total = length results-  putStrLn ""-  if and results-    then do-      putStrLn ("All " ++ show total ++ " groups passed.")-      exitSuccess-    else do-      putStrLn (show passed ++ "/" ++ show total ++ " groups passed.")-      exitFailure--main :: IO ()-main = do-  putStrLn "nova-cache test suite"-  putStrLn "======================"-  putStrLn ""-  runAll-    [ testBase32,-      testHash,-      testStorePath,-      testNAR,-      testStream,-      testNarInfo,-      testSigning,-      testFileStore,-      testReclaim,-      testValidate,-      testServer,-      testStoredNarCheck-    ]---- ------------------------------------------------------------------------------ Base32 tests--- -----------------------------------------------------------------------------testBase32 :: IO Bool-testBase32 =-  runGroup-    "Base32"-    [ test "encode empty" $-        assertEqual "encode empty" "" (Base32.encode BS.empty),-      test "decode empty" $-        assertRight "decode empty" BS.empty (Base32.decode ""),-      test "encode/decode roundtrip (single byte)" $-        let bs = BS.singleton 0xFF-            encoded = Base32.encode bs-         in assertRight "roundtrip 0xFF" bs (Base32.decode encoded),-      test "encode/decode roundtrip (known SHA-256)" $-        let bs = BS.pack [0 .. 31]-            encoded = Base32.encode bs-         in assertRight "roundtrip 32 bytes" bs (Base32.decode encoded),-      test "encode/decode roundtrip (all zeros)" $-        let bs = BS.replicate 32 0-            encoded = Base32.encode bs-         in assertRight "roundtrip zeros" bs (Base32.decode encoded),-      test "encode/decode roundtrip (all 0xFF)" $-        let bs = BS.replicate 32 0xFF-            encoded = Base32.encode bs-         in assertRight "roundtrip 0xFF*32" bs (Base32.decode encoded),-      test "decode invalid character" $-        assertLeft "invalid char" (Base32.decode "hello!"),-      test "encode length for 32 bytes" $-        -- 32 bytes gives ceil(32*8/5) = ceil(51.2) = 52 chars-        let bs = BS.replicate 32 0x42-            encoded = Base32.encode bs-         in assertEqual "encoded length" 52 (T.length encoded),-      test "nix known vector" $-        -- SHA-256 of the empty string exactly as real Nix renders it.  A-        -- wrong alphabet or bit order stays self-consistent in roundtrips;-        -- only an external vector catches it.-        assertEqual-          "sha256 of empty in nix-base32"-          "sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"-          (Hash.formatNixHash (Hash.hashBytes BS.empty)),-      test "decode rejects nonzero padding bits" $-        -- 52 chars carry 260 bits for a 256-bit value; the 4 spare bits-        -- must be zero in canonical nix-base32.-        assertLeft "nonzero padding" (Base32.decode (T.replicate 52 "z"))-    ]---- ------------------------------------------------------------------------------ Hash tests--- -----------------------------------------------------------------------------testHash :: IO Bool-testHash =-  runGroup-    "Hash"-    [ test "hashBytes deterministic" $-        let h1 = Hash.hashBytes (BS.pack [1, 2, 3])-            h2 = Hash.hashBytes (BS.pack [1, 2, 3])-         in assertEqual "deterministic" h1 h2,-      test "hashBytes different inputs differ" $-        let h1 = Hash.hashBytes (BS.pack [1, 2, 3])-            h2 = Hash.hashBytes (BS.pack [4, 5, 6])-         in assertTrue "different" (h1 /= h2),-      test "hashBytes is 32 bytes" $-        let Hash.NixHash raw = Hash.hashBytes BS.empty-         in assertEqual "32 bytes" 32 (BS.length raw),-      test "formatNixHash prefix" $-        let formatted = Hash.formatNixHash (Hash.hashBytes BS.empty)-         in assertTrue "sha256: prefix" (T.isPrefixOf "sha256:" formatted),-      test "formatNixHash/parseNixHash roundtrip" $-        let h = Hash.hashBytes (BS.pack [42])-            formatted = Hash.formatNixHash h-         in assertRight "roundtrip" h (Hash.parseNixHash formatted),-      test "parseNixHash bad prefix" $-        assertLeft "bad prefix" (Hash.parseNixHash "md5:abc"),-      test "parseNixHash bad base32" $-        assertLeft "bad base32" (Hash.parseNixHash "sha256:!!invalid!!")-    ]---- ------------------------------------------------------------------------------ StorePath tests--- -----------------------------------------------------------------------------testStorePath :: IO Bool-testStorePath =-  runGroup-    "StorePath"-    [ test "parse/render roundtrip" $-        let storeDir = StorePath.defaultStoreDir-            input = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0"-         in case StorePath.parseStorePath storeDir input of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right sp ->-                assertEqual "roundtrip" input (StorePath.renderStorePath storeDir sp),-      test "parse basename only" $-        let storeDir = StorePath.defaultStoreDir-            basename = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0"-         in case StorePath.parseStorePath storeDir basename of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right sp ->-                assertEqual "basename" basename (StorePath.storePathBaseName sp),-      test "parse extracts hash" $-        let storeDir = StorePath.defaultStoreDir-            input = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-test"-         in case StorePath.parseStorePath storeDir input of-              Left _ -> pure False-              Right sp ->-                assertEqual "hash" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" (StorePath.storePathHashString sp),-      test "reject too short" $-        let storeDir = StorePath.defaultStoreDir-         in assertLeft "too short" (StorePath.parseStorePath storeDir "abc-def"),-      test "reject empty name" $-        let storeDir = StorePath.defaultStoreDir-         in assertLeft "empty name" (StorePath.parseStorePath storeDir "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-"),-      test "reject invalid name chars" $-        let storeDir = StorePath.defaultStoreDir-            input = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello world"-         in assertLeft "invalid chars" (StorePath.parseStorePath storeDir input),-      -- Upstream's dot rule: the first dash-separated component may not-      -- be "." or ".."; other dot-leading names stay valid.-      test "reject dot-segment names" $-        let hash = T.replicate 32 "a"-            rejected name = assertLeft (T.unpack name) (StorePath.parseStorePathBaseName (hash <> "-" <> name))-         in do-              ok1 <- rejected "."-              ok2 <- rejected ".."-              ok3 <- rejected ".-x"-              ok4 <- rejected "..-y"-              pure (ok1 && ok2 && ok3 && ok4),-      test "dotfile-style names stay valid" $-        let hash = T.replicate 32 "a"-         in do-              ok1 <- assertTrue "dotfile" (either (const False) (const True) (StorePath.parseStorePathBaseName (hash <> "-.config-1.0")))-              ok2 <- assertTrue "interior dots" (either (const False) (const True) (StorePath.parseStorePathBaseName (hash <> "-x.y.z")))-              pure (ok1 && ok2)-    ]---- ------------------------------------------------------------------------------ NAR tests--- -----------------------------------------------------------------------------testNAR :: IO Bool-testNAR =-  runGroup-    "NAR"-    [ test "serialise/deserialise roundtrip (regular file)" $-        let entry = NAR.NarRegular False (BS.pack [72, 101, 108, 108, 111])-            serialised = NAR.serialise entry-         in assertRight "roundtrip regular" entry (NAR.deserialise serialised),-      test "serialise/deserialise roundtrip (empty file)" $-        let entry = NAR.NarRegular False BS.empty-            serialised = NAR.serialise entry-         in assertRight "roundtrip empty" entry (NAR.deserialise serialised),-      test "serialise/deserialise roundtrip (executable)" $-        let entry = NAR.NarRegular True (BS.pack [0x7F, 0x45, 0x4C, 0x46])-            serialised = NAR.serialise entry-         in assertRight "roundtrip exec" entry (NAR.deserialise serialised),-      test "serialise/deserialise roundtrip (symlink)" $-        let entry = NAR.NarSymlink "/usr/bin/hello"-            serialised = NAR.serialise entry-         in assertRight "roundtrip symlink" entry (NAR.deserialise serialised),-      test "serialise/deserialise roundtrip (directory)" $-        let entry =-              NAR.NarDirectory-                [ ("bar", NAR.NarRegular False (BS.pack [2])),-                  ("foo", NAR.NarRegular False (BS.pack [1]))-                ]-            serialised = NAR.serialise entry-         in assertRight "roundtrip dir" entry (NAR.deserialise serialised),-      test "serialise/deserialise roundtrip (nested directory)" $-        let entry =-              NAR.NarDirectory-                [ ("bin", NAR.NarDirectory [("hello", NAR.NarRegular True (BS.pack [42]))]),-                  ("lib", NAR.NarSymlink "../lib64")-                ]-            serialised = NAR.serialise entry-         in assertRight "roundtrip nested" entry (NAR.deserialise serialised),-      test "directory entries sorted" $-        let entry =-              NAR.NarDirectory-                [ ("zebra", NAR.NarRegular False BS.empty),-                  ("alpha", NAR.NarRegular False BS.empty)-                ]-            serialised = NAR.serialise entry-         in case NAR.deserialise serialised of-              Left err -> do-                putStrLn ("  deserialise failed: " ++ err)-                pure False-              Right (NAR.NarDirectory entries) ->-                assertEqual "sorted" ["alpha", "zebra"] (map fst entries)-              Right other -> do-                putStrLn ("  expected directory, got: " ++ show other)-                pure False,-      test "narHash deterministic" $-        let entry = NAR.NarRegular False (BS.pack [1, 2, 3])-            h1 = NAR.narHash entry-            h2 = NAR.narHash entry-         in assertEqual "deterministic hash" h1 h2,-      test "narHash differs for different content" $-        let h1 = NAR.narHash (NAR.NarRegular False (BS.pack [1]))-            h2 = NAR.narHash (NAR.NarRegular False (BS.pack [2]))-         in assertTrue "different hashes" (h1 /= h2),-      test "deserialise garbage fails" $-        assertLeft "garbage" (NAR.deserialise (BS.pack [0, 0, 0, 0, 0, 0, 0, 0])),-      test "roundtrip edge: empty directory" $-        let entry = NAR.NarDirectory []-         in assertRight "empty dir" entry (NAR.deserialise (NAR.serialise entry)),-      test "roundtrip edge: contents a multiple of 8 (zero padding)" $-        let entry = NAR.NarRegular False (BS.replicate 8 0x41)-         in assertRight "8-byte contents" entry (NAR.deserialise (NAR.serialise entry)),-      test "roundtrip edge: executable empty file" $-        let entry = NAR.NarRegular True BS.empty-         in assertRight "exec empty" entry (NAR.deserialise (NAR.serialise entry)),-      -- Cache-served archives are untrusted input: every name that could-      -- traverse out of a POSIX extraction root must fail the parse.-      test "unsafe directory entry names rejected" $-        let evil name = NAR.serialise (NAR.NarDirectory [(name, NAR.NarRegular False "x")])-            names = ["..", ".", "", "a/b", "a\0b"]-            rejected bytes = either (const True) (const False) (NAR.deserialise bytes)-         in assertTrue "all unsafe names rejected" (all (rejected . evil) names),-      test "duplicate directory entries rejected" $-        let dup =-              NAR.serialise-                ( NAR.NarDirectory-                    [ ("same", NAR.NarRegular False "1"),-                      ("same", NAR.NarRegular False "2")-                    ]-                )-         in assertLeft "duplicate entries" (NAR.deserialise dup),-      test "out-of-order directory entries rejected" $-        assertLeft "unsorted entries" (NAR.deserialise outOfOrderDirNar),-      test "trailing bytes after the root node rejected" $-        let valid = NAR.serialise (NAR.NarRegular False "x")-         in assertLeft "trailing bytes" (NAR.deserialise (valid <> "junk1234")),-      test "nonzero string padding rejected" $-        assertLeft "nonzero padding" (NAR.deserialise badPaddingNar),-      -- The format fixes the executable marker's value as empty; upstream-      -- rejects a nonempty value, and NAR.serialise cannot produce one.-      test "nonempty executable marker rejected" $-        let marked =-              BS.concat-                ( map-                    (narWireStr 0)-                    ["nix-archive-1", "(", "type", "regular", "executable", "X", "contents", "hi", ")"]-                )-         in assertLeft "nonempty marker" (NAR.deserialise marked),-      -- Upstream's restore accepts these on Unix, and real-      -- cache.nixos.org archives carry them (perl man pages named-      -- ExtUtils::MakeMaker.3, kernel trees carrying aux.c): the-      -- parser must too, or substitution fails where upstream-      -- succeeds.  Rejecting them is the store writer's call, via-      -- NAR.isWindowsHazardName, where the target filesystem needs it.-      test "Windows-hazard names parse, as upstream accepts" $-        let entryFor name = NAR.NarDirectory [(name, NAR.NarRegular False "x")]-            names = ["ExtUtils::MakeMaker.3", "aux.c", "a:b", "a\\b", "nul", "foo.", "foo "]-            roundTrips name = NAR.deserialise (NAR.serialise (entryFor name)) == Right (entryFor name)-         in assertTrue "all parse and round-trip" (all roundTrips names),-      -- Windows resolves these names to something other than a file of-      -- this spelling (drive/stream colon, separator backslash, device,-      -- NTFS dot/space strip).-      test "isWindowsHazardName flags the hazard categories" $-        -- Among the devices: COM0/LPT0 are reserved alongside COM1-9,-        -- the device stem is compared with trailing spaces trimmed-        -- ("NUL .txt" still opens the device), and the superscript-        -- digits (here U+00B9 as UTF-8) count as device digits.-        let names =-              [ "C:evil",-                "a:b",-                "a\\b",-                "nul",-                "NUL",-                "com1",-                "nul.txt",-                "foo.",-                "foo ",-                "com0",-                "lpt0",-                "nul .txt",-                "CON .x",-                "com" <> BS.pack [0xC2, 0xB9]-              ]-         in assertTrue "all hazard names flagged" (all NAR.isWindowsHazardName names),-      test "near-miss names parse and pass the hazard predicate" $-        let plain name = NAR.serialise (NAR.NarDirectory [(name, NAR.NarRegular False "x")])-            -- "com" followed by a non-digit non-superscript byte (here-            -- U+00B4, acute accent) is an ordinary name; so are stems-            -- one character too long.-            names = ["nul2", "com10", "conx", "foo.bar", "a.b.c", "lpt00", "com" <> BS.pack [0xC2, 0xB4]]-            accepted bytes = either (const False) (const True) (NAR.deserialise bytes)-            clean name = accepted (plain name) && not (NAR.isWindowsHazardName name)-         in assertTrue "all near-miss names accepted" (all clean names),-      -- Upstream carries names and targets as raw bytes: entries that-      -- do not decode as UTF-8 parse and round-trip.-      test "non-UTF-8 entry name round-trips" $-        let entry = NAR.NarDirectory [(BS.pack [0x66, 0xFF], NAR.NarRegular False "x")]-         in assertRight "raw-byte name" entry (NAR.deserialise (NAR.serialise entry)),-      test "non-UTF-8 symlink target round-trips" $-        let entry = NAR.NarSymlink (BS.pack [0x2F, 0x74, 0x6D, 0x70, 0x2F, 0xFF])-         in assertRight "raw-byte target" entry (NAR.deserialise (NAR.serialise entry)),-      test "entries sort bytewise, non-UTF-8 names included" $-        let entry =-              NAR.NarDirectory-                [ (BS.pack [0xFF], NAR.NarRegular False "hi"),-                  ("b", NAR.NarRegular False "lo")-                ]-         in case NAR.deserialise (NAR.serialise entry) of-              Left err -> do-                putStrLn ("  deserialise failed: " ++ err)-                pure False-              Right (NAR.NarDirectory entries) ->-                assertEqual "byte order" ["b", BS.pack [0xFF]] (map fst entries)-              Right other -> do-                putStrLn ("  expected directory, got: " ++ show other)-                pure False,-      -- The hazard checks are ASCII-structural, so they fire inside-      -- names that do not decode as text.-      test "hazards inside non-UTF-8 names still flagged" $-        let names = ["nul." <> BS.pack [0xFF], BS.pack [0xFF, 0x2E], BS.pack [0xFF] <> ":x"]-         in assertTrue "all hazard bytes flagged" (all NAR.isWindowsHazardName names),-      -- The case-hack strip: a tree materialized with upstream's-      -- collision suffix serialises back under its NAR names.-      test "serialiseFromPathWith strips the case-hack suffix" $ do-        dir <- caseHackFixture "nova-cache-test-casehack"-        BS.writeFile (dir <> "/Foo") "upper"-        BS.writeFile (dir <> "/foo~nix~case~hack~1") "lower"-        entry <- NAR.serialiseFromPathWith NAR.CaseHackEnabled dir-        removeDirectoryRecursive dir-        let expected =-              NAR.NarDirectory-                [ ("Foo", NAR.NarRegular False "upper"),-                  ("foo", NAR.NarRegular False "lower")-                ]-        roundTrip <- assertRight "stripped tree reparses" expected (NAR.deserialise (NAR.serialise entry))-        pure (entry == expected && roundTrip),-      test "serialiseFromPathWith keeps the suffix verbatim when disabled" $ do-        dir <- caseHackFixture "nova-cache-test-casehack-off"-        BS.writeFile (dir <> "/foo~nix~case~hack~1") "kept"-        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-        removeDirectoryRecursive dir-        assertEqual-          "verbatim name"-          (NAR.NarDirectory [("foo~nix~case~hack~1", NAR.NarRegular False "kept")])-          entry,-      test "serialiseFromPathWith fails loudly on an unhack collision" $ do-        dir <- caseHackFixture "nova-cache-test-casehack-clash"-        BS.writeFile (dir <> "/foo") "plain"-        BS.writeFile (dir <> "/foo~nix~case~hack~1") "hacked"-        outcome <- try (NAR.serialiseFromPathWith NAR.CaseHackEnabled dir)-        removeDirectoryRecursive dir-        pure $ case (outcome :: Either SomeException NAR.NarEntry) of-          Left _ -> True-          Right _ -> False,-      -- The resolver is the executable-bit source: its answer lands-      -- verbatim, independent of permissions or extension.-      test "serialiseFromPathOpts takes the exec bit from the resolver" $ do-        dir <- caseHackFixture "nova-cache-test-execresolver"-        BS.writeFile (dir <> "/plain") "p"-        let opts =-              NAR.defaultSerialiseOptions-                { NAR.soCaseHack = NAR.CaseHackDisabled,-                  NAR.soExecBit = \_ -> pure True-                }-        entry <- NAR.serialiseFromPathOpts opts dir-        removeDirectoryRecursive dir-        assertEqual-          "resolver answer"-          (NAR.NarDirectory [("plain", NAR.NarRegular True "p")])-          entry,-      -- A real on-disk exec bit round-trips through the default-      -- resolver; no earlier fixture ever set one.  POSIX-gated:-      -- Windows has no mode bit for the default to read.-      test "a real owner-exec bit round-trips from disk" $-        if System.Info.os == "mingw32"-          then pure True-          else do-            dir <- caseHackFixture "nova-cache-test-realexec"-            BS.writeFile (dir <> "/doc") "txt"-            BS.writeFile (dir <> "/tool") "bin"-            perms <- getPermissions (dir <> "/tool")-            setPermissions (dir <> "/tool") (setOwnerExecutable True perms)-            entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-            removeDirectoryRecursive dir-            assertEqual-              "owner bit from the file mode"-              ( NAR.NarDirectory-                  [ ("doc", NAR.NarRegular False "txt"),-                    ("tool", NAR.NarRegular True "bin")-                  ]-              )-              entry,-      -- The resolver contract, pinned as an exact call set for both-      -- producers: once per regular file, never for the directory,-      -- and always the on-disk spelling - under the case hack, the-      -- suffixed name the walk reads, not the stripped entry name.-      test "the resolver sees the on-disk case-hacked spelling" $ do-        dir <- caseHackFixture "nova-cache-test-execdisk"-        BS.writeFile (dir <> "/Foo") "upper"-        BS.writeFile (dir <> "/foo~nix~case~hack~1") "lower"-        seenWalk <- newIORef []-        seenPlan <- newIORef []-        let recordingOpts ref =-              NAR.defaultSerialiseOptions-                { NAR.soCaseHack = NAR.CaseHackEnabled,-                  NAR.soExecBit = \path ->-                    atomicModifyIORef' ref (\paths -> (path : paths, ())) >> pure False-                }-            baseName = reverse . takeWhile (\c -> c /= '/' && c /= '\\') . reverse-        _ <- NAR.serialiseFromPathOpts (recordingOpts seenWalk) dir-        _ <- NAR.withNarSourceOpts (recordingOpts seenPlan) dir drainSource-        removeDirectoryRecursive dir-        walked <- readIORef seenWalk-        planned <- readIORef seenPlan-        let expected = ["Foo", "foo~nix~case~hack~1"]-        walkOk <--          assertEqual-            "eager walk call set"-            expected-            (sort (map baseName walked))-        planOk <--          assertEqual-            "streaming plan call set"-            expected-            (sort (map baseName planned))-        pure (walkOk && planOk),-      -- A symlink target with a separator serialises to the POSIX-      -- spelling on every platform (nova-nix #112): Windows stores a-      -- reparse point with backslashes, so a target written bin/tool-      -- reads back bin\tool, and an unpack-then-recheck round-trip-      -- diverges from the archive unless the NAR boundary normalises-      -- it.  Both producers must agree with the forward-slash form.-      -- Guarded on symlink privilege, which a Windows runner without-      -- developer mode lacks.-      test "a separator-bearing symlink target serialises with forward slashes" $ do-        dir <- caseHackFixture "nova-cache-test-symlinksep"-        created <- try (createFileLink "bin/tool" (dir <> "/link")) :: IO (Either SomeException ())-        case created of-          Left _ -> removeDirectoryRecursive dir >> pure True-          Right () -> do-            let expectedEntry = NAR.NarDirectory [("link", NAR.NarSymlink "bin/tool")]-            eager <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-            streamed <- NAR.withNarSource NAR.CaseHackDisabled dir drainSource-            removeDirectoryRecursive dir-            eagerOk <- assertEqual "eager target spelling" expectedEntry eager-            streamOk <- assertEqual "streamed target spelling" (NAR.serialise expectedEntry) streamed-            pure (eagerOk && streamOk),-      -- The walk's boundary encoding: a Unicode disk name enters the-      -- archive as its UTF-8 bytes on every platform.-      test "serialiseFromPath encodes a Unicode disk name as UTF-8" $ do-        dir <- caseHackFixture "nova-cache-test-uniname"-        BS.writeFile (dir <> "/caf\233") "au lait"-        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-        removeDirectoryRecursive dir-        assertEqual-          "utf8 name"-          (NAR.NarDirectory [(BS.pack [0x63, 0x61, 0x66, 0xC3, 0xA9], NAR.NarRegular False "au lait")])-          entry,-      -- POSIX names are bytes; one that is not valid UTF-8 must archive-      -- verbatim (it used to be silently rewritten with replacement-      -- characters).  Linux-gated: NTFS and APFS names are Unicode, so-      -- the fixture cannot exist there.  "\56575" is the lone surrogate-      -- GHC's filesystem encoding round-trips to byte 0xFF.-      test "serialiseFromPath carries a non-UTF-8 disk name verbatim" $-        if System.Info.os /= "linux"-          then pure True-          else do-            dir <- caseHackFixture "nova-cache-test-rawname"-            BS.writeFile (dir <> "/f\56575") "raw"-            entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-            removeDirectoryRecursive dir-            assertEqual-              "raw byte name"-              (NAR.NarDirectory [(BS.pack [0x66, 0xFF], NAR.NarRegular False "raw")])-              entry-    ]---- | A fresh, empty fixture directory under the system temp dir.-caseHackFixture :: String -> IO FilePath-caseHackFixture name = do-  tmpBase <- getTemporaryDirectory-  let dir = tmpBase <> "/" <> name-  _ <- try (removeDirectoryRecursive dir) :: IO (Either SomeException ())-  createDirectory dir-  pure dir---- | Encode one NAR wire string with a chosen padding byte.  The spec--- demands zero padding, so a nonzero byte builds archives the parser--- must reject - and 'NAR.serialise' (rightly) cannot produce them.-narWireStr :: Word8 -> ByteString -> ByteString-narWireStr padByte str = lenLE <> str <> BS.replicate padLen padByte-  where-    n = BS.length str-    lenLE = BS.pack [fromIntegral ((n `shiftR` (8 * i)) .&. 0xff) | i <- [0 .. 7]]-    padLen = (8 - n `mod` 8) `mod` 8---- | A directory NAR whose entries arrive out of sorted order - again not--- producible via 'NAR.serialise', which sorts on write.-outOfOrderDirNar :: ByteString-outOfOrderDirNar =-  BS.concat-    ( map-        (narWireStr 0)-        ( ["nix-archive-1", "(", "type", "directory"]-            ++ entryFor "b"-            ++ entryFor "a"-            ++ [")"]-        )-    )-  where-    entryFor name = ["entry", "(", "name", name, "node", "(", "type", "regular", "contents", "", ")", ")"]---- | A regular-file NAR whose contents padding is nonzero.-badPaddingNar :: ByteString-badPaddingNar =-  BS.concat-    [ BS.concat (map (narWireStr 0) ["nix-archive-1", "(", "type", "regular", "contents"]),-      narWireStr 1 "abc",-      narWireStr 0 ")"-    ]---- ------------------------------------------------------------------------------ NAR.Stream tests--- ------------------------------------------------------------------------------- | Drive the streaming parser over the given chunks (all non-empty),--- then signal end of input, collecting events.-runStream :: [ByteString] -> Either String [Stream.NarEvent]-runStream = runStreamFrom Stream.narStream---- | 'runStream' from an explicitly constructed machine, for tests that--- need a bound other than the default.-runStreamFrom :: Stream.NarStep -> [ByteString] -> Either String [Stream.NarEvent]-runStreamFrom start chunks = go start chunks []-  where-    go step pending acc = case step of-      Stream.NarFail err -> Left err-      Stream.NarDone -> Right (reverse acc)-      Stream.NarYield event continue -> go continue pending (event : acc)-      Stream.NarAwait continue -> case pending of-        [] -> go (continue BS.empty) [] acc-        (chunk : rest) -> go (continue chunk) rest acc---- | Entry shapes the chunking-independence test sweeps: every node--- kind, empty and aligned contents, nesting, and a non-UTF-8 name.-streamCorpus :: [NAR.NarEntry]-streamCorpus =-  [ NAR.NarRegular False "hello",-    NAR.NarRegular True BS.empty,-    NAR.NarRegular False (BS.replicate 8 0x41),-    NAR.NarSymlink "/usr/bin/hello",-    NAR.NarDirectory [],-    NAR.NarDirectory-      [ ("bin", NAR.NarDirectory [("hello", NAR.NarRegular True (BS.pack [42]))]),-        ("lib", NAR.NarSymlink "../lib64"),-        (BS.pack [0x66, 0xFF], NAR.NarRegular False "raw")-      ]-  ]---- | Merge consecutive contents slices.  Slice granularity deliberately--- follows the fed chunks, so parses of different chunkings compare by--- structure and content, not by how the input happened to arrive.-coalesceChunks :: [Stream.NarEvent] -> [Stream.NarEvent]-coalesceChunks (Stream.EventRegularChunk a : Stream.EventRegularChunk b : rest) =-  coalesceChunks (Stream.EventRegularChunk (a <> b) : rest)-coalesceChunks (event : rest) = event : coalesceChunks rest-coalesceChunks [] = []---- | Split a byte string into fixed-size pieces.-chunksOf :: Int -> ByteString -> [ByteString]-chunksOf n bs-  | BS.null bs = []-  | otherwise = case BS.splitAt n bs of-      (piece, rest) -> piece : chunksOf n rest---- | Pull a source dry, collecting its chunks (the terminating empty--- chunk excluded).-collectChunks :: IO ByteString -> IO [ByteString]-collectChunks pull = go []-  where-    go acc = do-      chunk <- pull-      if BS.null chunk then pure (reverse acc) else go (chunk : acc)---- | Pull a source dry into one byte string.-drainSource :: IO ByteString -> IO ByteString-drainSource pull = BS.concat <$> collectChunks pull---- | Hash a source chunk by chunk as it is pulled.-hashPull :: Hash.HashContext -> IO ByteString -> IO Hash.NixHash-hashPull ctx pull = do-  chunk <- pull-  if BS.null chunk-    then pure (Hash.hashFinalize ctx)-    else hashPull (Hash.hashUpdate ctx chunk) pull--testStream :: IO Bool-testStream =-  runGroup-    "NAR.Stream"-    [ test "events for a regular file" $-        assertEqual-          "event sequence"-          ( Right-              [ Stream.EventRegularBegin True 2,-                Stream.EventRegularChunk "hi",-                Stream.EventRegularEnd-              ]-          )-          (runStream [NAR.serialise (NAR.NarRegular True "hi")]),-      test "events for a directory arrive entry by entry" $-        let entry = NAR.NarDirectory [("a", NAR.NarSymlink "t"), ("b", NAR.NarRegular False "")]-            expected =-              [ Stream.EventDirectoryBegin,-                Stream.EventEntryBegin "a",-                Stream.EventSymlink "t",-                Stream.EventEntryEnd,-                Stream.EventEntryBegin "b",-                Stream.EventRegularBegin False 0,-                Stream.EventRegularEnd,-                Stream.EventEntryEnd,-                Stream.EventDirectoryEnd-              ]-         in assertEqual "event sequence" (Right expected) (runStream [NAR.serialise entry]),-      test "byte-at-a-time chunking equals whole-input parsing" $-        let normalize = fmap coalesceChunks-            agrees entry =-              let bytes = NAR.serialise entry-               in normalize (runStream [bytes]) == normalize (runStream (map BS.singleton (BS.unpack bytes)))-         in assertTrue "chunking-independent" (all agrees streamCorpus),-      test "contents slices reassemble across misaligned chunks" $-        -- 7-byte chunks sit deliberately askew of the 8-byte wire-        -- alignment, so every length prefix and padding run straddles-        -- a chunk boundary somewhere.-        let payload = BS.pack (concat (replicate 40 [0 .. 7]))-            bytes = NAR.serialise (NAR.NarRegular False payload)-         in case runStream (chunksOf 7 bytes) of-              Left err -> do-                putStrLn ("  streaming parse failed: " ++ err)-                pure False-              Right events ->-                assertEqual-                  "reassembled contents"-                  payload-                  (BS.concat [c | Stream.EventRegularChunk c <- events]),-      test "streaming agrees with deserialise on the malformed corpus" $-        let evil name = NAR.serialise (NAR.NarDirectory [(name, NAR.NarRegular False "x")])-            malformed =-              [ outOfOrderDirNar,-                badPaddingNar,-                NAR.serialise (NAR.NarRegular False "x") <> "junk1234",-                evil "..",-                evil "a/b",-                evil ""-              ]-            bothReject bytes = isLeft (runStream [bytes]) && isLeft (NAR.deserialise bytes)-         in assertTrue "all rejected by both parsers" (all bothReject malformed),-      test "the structural bound applies to streaming, not deserialise" $-        -- deserialise instantiates the machine with its input's length-        -- as the bound, so a name this size stays accepted there; the-        -- streaming default caps structural strings at 64 KiB.-        let longName = BS.replicate 70000 0x61-            bytes = NAR.serialise (NAR.NarDirectory [(longName, NAR.NarRegular False "x")])-         in do-              ok1 <- assertTrue "deserialise accepts" (either (const False) (const True) (NAR.deserialise bytes))-              ok2 <- assertTrue "streaming rejects" (isLeft (runStream [bytes]))-              pure (ok1 && ok2),-      test "a truncated archive fails" $-        let bytes = NAR.serialise (NAR.NarRegular False "some contents here")-         in assertLeft "eof" (runStream [BS.take (BS.length bytes - 9) bytes]),-      -- Regression: under a huge caller bound, a declared length near-      -- maxBound Int once wrapped the padded Int demand negative, so-      -- the machine "read" the string instantly as empty at an unmoved-      -- position and this archive parsed as a complete empty symlink.-      -- The parse must fail, as upstream would at end of input.-      test "a near-maxBound declared length fails under a huge bound" $-        let declared = fromIntegral (maxBound :: Int) :: Word64-            lenPrefix = BS.pack [fromIntegral ((declared `shiftR` (8 * i)) .&. 0xff) | i <- [0 .. 7]]-            bytes =-              BS.concat (map (narWireStr 0) ["nix-archive-1", "(", "type", "symlink", "target"])-                <> lenPrefix-                <> narWireStr 0 ")"-         in assertLeft-              "overflowing padded demand"-              (runStreamFrom (Stream.narStreamBounded (maxBound :: Word64)) [bytes]),-      test "the declared contents size arrives before the bytes" $-        case runStream [NAR.serialise (NAR.NarRegular False (BS.replicate 24 0x2A))] of-          Right (Stream.EventRegularBegin False declared : _) ->-            assertEqual "declared size" 24 declared-          other -> do-            putStrLn ("    expected EventRegularBegin first, got: " ++ show other)-            pure False,-      test "withNarSource streams byte-identically to serialise" $ do-        dir <- caseHackFixture "nova-cache-test-narsource"-        createDirectory (dir <> "/sub")-        -- Larger than two pull chunks, so mid-file continuation runs.-        BS.writeFile (dir <> "/big.bin") (BS.replicate 300000 0x41)-        BS.writeFile (dir <> "/sub/small") "tiny"-        BS.writeFile (dir <> "/zero") ""-        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-        streamed <- NAR.withNarSource NAR.CaseHackDisabled dir drainSource-        removeDirectoryRecursive dir-        assertTrue "bytes equal" (NAR.serialise entry == streamed),-      test "withNarSource under the case-hack matches the strict walk" $ do-        dir <- caseHackFixture "nova-cache-test-narsource-hack"-        BS.writeFile (dir <> "/Foo") "upper"-        BS.writeFile (dir <> "/foo~nix~case~hack~1") "lower"-        entry <- NAR.serialiseFromPathWith NAR.CaseHackEnabled dir-        streamed <- NAR.withNarSource NAR.CaseHackEnabled dir drainSource-        removeDirectoryRecursive dir-        assertTrue "bytes equal" (NAR.serialise entry == streamed),-      -- The streaming planner asks the same resolver the eager walk-      -- does, so a custom executable-bit source streams byte-equal.-      test "withNarSourceOpts streams the resolver's flags byte-identically" $ do-        dir <- caseHackFixture "nova-cache-test-narsource-exec"-        BS.writeFile (dir <> "/doc") "txt"-        BS.writeFile (dir <> "/tool") "bin"-        let opts =-              NAR.defaultSerialiseOptions-                { NAR.soCaseHack = NAR.CaseHackDisabled,-                  NAR.soExecBit = \path -> pure ("tool" `isSuffixOf` path)-                }-        entry <- NAR.serialiseFromPathOpts opts dir-        streamed <- NAR.withNarSourceOpts opts dir drainSource-        removeDirectoryRecursive dir-        flagsOk <--          assertEqual-            "eager flags"-            ( NAR.NarDirectory-                [ ("doc", NAR.NarRegular False "txt"),-                  ("tool", NAR.NarRegular True "bin")-                ]-            )-            entry-        bytesOk <- assertTrue "stream equals serialise" (NAR.serialise entry == streamed)-        pure (flagsOk && bytesOk),-      test "withNarSource keeps returning empty after the end" $ do-        dir <- caseHackFixture "nova-cache-test-narsource-end"-        BS.writeFile (dir <> "/f") "x"-        ends <- NAR.withNarSource NAR.CaseHackDisabled dir $ \pull -> do-          _ <- collectChunks pull-          endA <- pull-          endB <- pull-          pure (endA, endB)-        removeDirectoryRecursive dir-        assertEqual "stable end" ("", "") ends,-      test "pulled chunks parse back to the walked tree" $ do-        dir <- caseHackFixture "nova-cache-test-narsource-loop"-        BS.writeFile (dir <> "/data.bin") (BS.replicate 200000 0x5A)-        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-        chunks <- NAR.withNarSource NAR.CaseHackDisabled dir collectChunks-        removeDirectoryRecursive dir-        ok1 <- assertRight "strict parse of the stream" entry (NAR.deserialise (BS.concat chunks))-        ok2 <- assertTrue "streaming parse of the chunk list" (either (const False) (const True) (runStream chunks))-        pure (ok1 && ok2),-      test "incremental hashing equals whole-input hashing" $-        let payload = BS.pack [0 .. 255]-            pieces = [BS.take 100 payload, BS.take 55 (BS.drop 100 payload), BS.drop 155 payload]-            incremental = Hash.hashFinalize (foldl' Hash.hashUpdate Hash.hashInit pieces)-         in do-              ok1 <- assertEqual "split arbitrarily" (Hash.hashBytes payload) incremental-              ok2 <- assertEqual "empty input" (Hash.hashBytes BS.empty) (Hash.hashFinalize Hash.hashInit)-              pure (ok1 && ok2),-      test "hashing a pulled source equals narHash of the walk" $ do-        dir <- caseHackFixture "nova-cache-test-streamhash"-        BS.writeFile (dir <> "/blob") (BS.replicate 150000 0x07)-        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir-        digest <- NAR.withNarSource NAR.CaseHackDisabled dir (hashPull Hash.hashInit)-        removeDirectoryRecursive dir-        assertEqual "hash matches" (NAR.narHash entry) digest-    ]---- ------------------------------------------------------------------------------ NarInfo tests--- -----------------------------------------------------------------------------sampleNarInfoText :: Text-sampleNarInfoText =-  T.unlines-    [ "StorePath: /nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0",-      "URL: nar/1234abcd.nar.xz",-      "Compression: xz",-      "FileHash: sha256:abcdef1234567890",-      "FileSize: 12345",-      "NarHash: sha256:fedcba0987654321",-      "NarSize: 67890",-      "References: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-glibc-2.38",-      "Deriver: cccccccccccccccccccccccccccccccc-hello-1.0.drv",-      "Sig: cache.example.com:c2lnbmF0dXJl",-      "Sig: backup.example.com:YW5vdGhlcnNpZw=="-    ]--testNarInfo :: IO Bool-testNarInfo =-  runGroup-    "NarInfo"-    [ test "parse sample narinfo" $-        case NarInfo.parseNarInfo sampleNarInfoText of-          Left err -> do-            putStrLn ("  parse failed: " ++ err)-            pure False-          Right ni -> do-            ok1 <- assertEqual "storePath" "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0" (NarInfo.niStorePath ni)-            ok2 <- assertEqual "url" "nar/1234abcd.nar.xz" (NarInfo.niUrl ni)-            ok3 <- assertEqual "compression" "xz" (NarInfo.niCompression ni)-            ok4 <- assertEqual "fileSize" (Just 12345) (NarInfo.niFileSize ni)-            ok5 <- assertEqual "narSize" 67890 (NarInfo.niNarSize ni)-            ok6 <- assertEqual "refs count" 2 (length (NarInfo.niReferences ni))-            ok7 <- assertEqual "deriver" (Just "cccccccccccccccccccccccccccccccc-hello-1.0.drv") (NarInfo.niDeriver ni)-            ok8 <- assertEqual "sigs count" 2 (length (NarInfo.niSigs ni))-            pure (ok1 && ok2 && ok3 && ok4 && ok5 && ok6 && ok7 && ok8),-      test "parse/render roundtrip" $-        case NarInfo.parseNarInfo sampleNarInfoText of-          Left err -> do-            putStrLn ("  parse failed: " ++ err)-            pure False-          Right ni ->-            let rendered = NarInfo.renderNarInfo ni-             in case NarInfo.parseNarInfo rendered of-                  Left err -> do-                    putStrLn ("  re-parse failed: " ++ err)-                    pure False-                  Right reparsed ->-                    assertEqual "roundtrip" ni reparsed,-      test "parse minimal narinfo (no optional fields)" $-        let minimal =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/test.nar.xz",-                  "Compression: xz",-                  "FileHash: sha256:abc",-                  "FileSize: 100",-                  "NarHash: sha256:def",-                  "NarSize: 200",-                  "References: "-                ]-         in case NarInfo.parseNarInfo minimal of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right ni -> do-                ok1 <- assertEqual "deriver" Nothing (NarInfo.niDeriver ni)-                ok2 <- assertEqual "sigs" [] (NarInfo.niSigs ni)-                ok3 <- assertEqual "ca" Nothing (NarInfo.niCA ni)-                ok4 <- assertEqual "refs" [] (NarInfo.niReferences ni)-                pure (ok1 && ok2 && ok3 && ok4),-      test "CRLF-terminated narinfo parses identically" $-        assertEqual-          "crlf tolerated"-          (NarInfo.parseNarInfo sampleNarInfoText)-          (NarInfo.parseNarInfo (T.replace "\n" "\r\n" sampleNarInfoText)),-      test "upstream-optional fields default as upstream" $-        -- Only StorePath, URL, NarHash, NarSize are mandatory upstream;-        -- Compression defaults to bzip2 and FileHash/FileSize stay absent.-        let bare =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/test.nar.xz",-                  "NarHash: sha256:def",-                  "NarSize: 200"-                ]-         in case NarInfo.parseNarInfo bare of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right ni -> do-                ok1 <- assertEqual "compression default" "bzip2" (NarInfo.niCompression ni)-                ok2 <- assertEqual "fileHash absent" Nothing (NarInfo.niFileHash ni)-                ok3 <- assertEqual "fileSize absent" Nothing (NarInfo.niFileSize ni)-                pure (ok1 && ok2 && ok3),-      test "CA field parse/render roundtrip" $-        let withCA = sampleNarInfoText <> "CA: fixed:r:sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73\n"-         in case NarInfo.parseNarInfo withCA of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right ni -> do-                ok1 <--                  assertEqual-                    "ca parsed"-                    (Just "fixed:r:sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73")-                    (NarInfo.niCA ni)-                ok2 <- assertRight "ca survives render" ni (NarInfo.parseNarInfo (NarInfo.renderNarInfo ni))-                pure (ok1 && ok2),-      test "parse missing required key fails" $-        let incomplete = T.unlines ["StorePath: /nix/store/aaaa-test", "URL: nar/test.nar.xz"]-         in assertLeft "missing key" (NarInfo.parseNarInfo incomplete),-      test "parse bad integer fails" $-        let bad =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/test.nar.xz",-                  "Compression: xz",-                  "FileHash: sha256:abc",-                  "FileSize: not-a-number",-                  "NarHash: sha256:def",-                  "NarSize: 200",-                  "References: "-                ]-         in assertLeft "bad integer" (NarInfo.parseNarInfo bad),-      -- Sizes are uint64 on the wire: at most 20 digits.  A longer field-      -- rejects before the quadratic bignum accumulation.-      test "parse rejects an overlong size field" $-        let long =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/test.nar.xz",-                  "NarHash: sha256:def",-                  "NarSize: 123456789012345678901"-                ]-         in assertLeft "overlong size" (NarInfo.parseNarInfo long),-      test "a 20-digit size field still parses" $-        let capped =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/test.nar.xz",-                  "NarHash: sha256:def",-                  "NarSize: 18446744073709551615"-                ]-         in case NarInfo.parseNarInfo capped of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right ni -> assertEqual "uint64 max" 18446744073709551615 (NarInfo.niNarSize ni),-      -- Twenty digits still fit values past 2^64 - 1; upstream's-      -- string2Int<uint64_t> refuses them, so signing one here would-      -- produce a narinfo every real client rejects as corrupt.-      test "a 20-digit size above the uint64 maximum rejects" $-        let overCap =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/test.nar.xz",-                  "NarHash: sha256:def",-                  "NarSize: 18446744073709551616"-                ]-         in assertLeft "size above uint64" (NarInfo.parseNarInfo overCap),-      -- Upstream assigns fields as it reads, so a duplicated scalar key-      -- resolves to the LAST value (Sig stays the repeatable exception).-      test "duplicate scalar keys resolve last-wins" $-        let dup =-              T.unlines-                [ "StorePath: /nix/store/aaaa-test",-                  "URL: nar/first.nar",-                  "URL: nar/last.nar",-                  "Compression: xz",-                  "Compression: zstd",-                  "NarHash: sha256:def",-                  "NarSize: 200"-                ]-         in case NarInfo.parseNarInfo dup of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right ni -> do-                ok1 <- assertEqual "url last" "nar/last.nar" (NarInfo.niUrl ni)-                ok2 <- assertEqual "compression last" "zstd" (NarInfo.niCompression ni)-                pure (ok1 && ok2)-    ]---- ------------------------------------------------------------------------------ Signing tests--- -----------------------------------------------------------------------------testSigning :: IO Bool-testSigning =-  runGroup-    "Signing"-    [ test "fingerprint format" $-        let ni = mkTestNarInfo-            fp = Signing.fingerprint ni-         in do-              ok1 <- assertTrue "starts with 1;" (T.isPrefixOf "1;" fp)-              ok2 <- assertTrue "contains storePath" (T.isInfixOf "/nix/store/" fp)-              pure (ok1 && ok2),-      test "fingerprint renders references as full store paths" $-        let ni = mkTestNarInfo {NarInfo.niReferences = ["00000000000000000000000000000000-glibc-2.40"]}-            fp = Signing.fingerprint ni-         in assertTrue-              "reference is a full /nix/store path in the fingerprint"-              (T.isInfixOf "/nix/store/00000000000000000000000000000000-glibc-2.40" fp),-      test "fingerprint sorts and dedupes references" $-        let ni =-              mkTestNarInfo-                { NarInfo.niReferences =-                    [ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-zlib-1.3",-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-glibc-2.40",-                      "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-zlib-1.3"-                    ]-                }-            fp = Signing.fingerprint ni-         in assertTrue-              "references sorted by basename and deduplicated (C++ Nix parity)"-              -- The leading field separator pins the WHOLE references-              -- field: without it, the pre-fix "zlib,glibc,zlib"-              -- rendering also ends in "...glibc...,...zlib..." and the-              -- assertion cannot fail on a regression to unsorted output.-              ( T.isSuffixOf-                  ";/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-glibc-2.40,/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-zlib-1.3"-                  fp-              ),-      test "SecretKey Show redacts the key bytes" $-        let sk = Signing.SecretKey {Signing.skName = "test-key", Signing.skBytes = BS.pack ([1 .. 32] ++ [33 .. 64])}-         in assertEqual "redacted" "SecretKey \"test-key\" <redacted>" (show sk),-      test "SecretKey equality compares name and bytes" $-        let bytesA = BS.pack ([1 .. 32] ++ [33 .. 64])-            skA = Signing.SecretKey "k" bytesA-            skB = Signing.SecretKey "k" bytesA-            skC = Signing.SecretKey "k" (BS.pack (0 : [2 .. 64]))-         in do-              ok1 <- assertTrue "equal keys" (skA == skB)-              ok2 <- assertTrue "different bytes differ" (skA /= skC)-              pure (ok1 && ok2),-      test "parseSecretKey valid" $-        let keyBytes = BS.pack ([1 .. 32] ++ [33 .. 64])-            keyB64 = TE.decodeUtf8 (B64.encode keyBytes)-            keyStr = "test-key:" <> keyB64-         in case Signing.parseSecretKey keyStr of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right sk ->-                assertEqual "key name" "test-key" (Signing.skName sk),-      test "parsePublicKey valid" $-        let keyBytes = BS.pack [1 .. 32]-            keyB64 = TE.decodeUtf8 (B64.encode keyBytes)-            keyStr = "test-key:" <> keyB64-         in case Signing.parsePublicKey keyStr of-              Left err -> do-                putStrLn ("  parse failed: " ++ err)-                pure False-              Right pk ->-                assertEqual "key name" "test-key" (Signing.pkName pk),-      test "parseSecretKey no colon fails" $-        assertLeft "no colon" (Signing.parseSecretKey "nokeyname"),-      test "empty key name rejected" $-        -- An empty-named key would emit :sig lines no named trust anchor-        -- matches; the misconfiguration must fail at load time.-        let material = TE.decodeUtf8 (B64.encode (BS.pack [1 .. 64]))-         in assertLeft "empty name" (Signing.parseSecretKey (":" <> material)),-      test "empty key material rejected" $-        assertLeft "empty material" (Signing.parsePublicKey "test-key:"),-      test "signature from a different keypair rejected" $ do-        signingKey <- generateTestSecretKey-        otherKey <- generateTestSecretKey-        let verifier = deriveTestPublicKey otherKey-        case Signing.sign signingKey mkTestNarInfo of-          Left err -> do-            putStrLn ("  sign failed: " ++ err)-            pure False-          Right signed ->-            assertTrue "cross-key rejected" (not (Signing.verify verifier mkTestNarInfo signed)),-      test "valid signature under a renamed trust anchor rejected" $ do-        signingKey <- generateTestSecretKey-        let renamed = (deriveTestPublicKey signingKey) {Signing.pkName = "some-other-cache"}-        case Signing.sign signingKey mkTestNarInfo of-          Left err -> do-            putStrLn ("  sign failed: " ++ err)-            pure False-          Right signed ->-            assertTrue "name mismatch rejected" (not (Signing.verify renamed mkTestNarInfo signed)),-      test "malformed signature lines rejected" $ do-        signingKey <- generateTestSecretKey-        let verifier = deriveTestPublicKey signingKey-            wrongSize = "test-key:" <> TE.decodeUtf8 (B64.encode (BS.pack [1 .. 16]))-            badLines = ["test-key:!!!not-base64!!!", wrongSize, "test-key:", "no-colon-at-all"]-        assertTrue-          "all malformed rejected"-          (not (any (Signing.verify verifier mkTestNarInfo) badLines)),-      test "parsePublicKey wrong size fails" $-        let keyStr = "test-key:" <> TE.decodeUtf8 (B64.encode (BS.pack [1 .. 16]))-         in assertLeft "wrong size" (Signing.parsePublicKey keyStr),-      test "sign/verify roundtrip" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-            ni = mkTestNarInfo-        case Signing.sign sk ni of-          Left err -> do-            putStrLn ("  sign failed: " ++ err)-            pure False-          Right sig ->-            assertTrue "verify passes" (Signing.verify pk ni sig),-      test "verify rejects tampered narinfo" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-            ni = mkTestNarInfo-        case Signing.sign sk ni of-          Left err -> do-            putStrLn ("  sign failed: " ++ err)-            pure False-          Right sig ->-            let tampered = ni {NarInfo.niNarSize = 999999}-             in assertFalse "verify rejects tampered" (Signing.verify pk tampered sig),-      test "toPublicKey derives the verifying key" $ do-        sk <- generateTestSecretKey-        case Signing.toPublicKey sk of-          Left err -> do-            putStrLn ("  toPublicKey failed: " ++ err)-            pure False-          Right pk -> do-            okName <- assertEqual "key name carried over" (Signing.skName sk) (Signing.pkName pk)-            okBytes <- assertEqual "matches the stored public half" (BS.drop 32 (Signing.skBytes sk)) (Signing.pkBytes pk)-            case Signing.sign sk mkTestNarInfo of-              Left err -> do-                putStrLn ("  sign failed: " ++ err)-                pure False-              Right sig -> do-                okVerify <- assertTrue "derived key verifies a signature" (Signing.verify pk mkTestNarInfo sig)-                pure (okName && okBytes && okVerify),-      test "renderPublicKey round-trips through parsePublicKey" $ do-        sk <- generateTestSecretKey-        case Signing.toPublicKey sk of-          Left err -> do-            putStrLn ("  toPublicKey failed: " ++ err)-            pure False-          Right pk -> case Signing.parsePublicKey (Signing.renderPublicKey pk) of-            Left err -> do-              putStrLn ("  parse failed: " ++ err)-              pure False-            Right reparsed -> assertEqual "round-trip" pk reparsed,-      test "normalizeKeyText strips BOM and whitespace" $ do-        ok1 <- assertEqual "BOM stripped" "test-key:abc" (Signing.normalizeKeyText ("\xFEFF" <> "test-key:abc"))-        ok2 <- assertEqual "CRLF stripped" "test-key:abc" (Signing.normalizeKeyText "test-key:abc\r\n")-        ok3 <- assertEqual "spaces stripped" "test-key:abc" (Signing.normalizeKeyText "  test-key:abc  ")-        ok4 <- assertEqual "clean text unchanged" "test-key:abc" (Signing.normalizeKeyText "test-key:abc")-        pure (ok1 && ok2 && ok3 && ok4)-    ]---- | Create a test NarInfo for signing tests.-mkTestNarInfo :: NarInfo.NarInfo-mkTestNarInfo =-  NarInfo.NarInfo-    { NarInfo.niStorePath = "/nix/store/aaaa-hello-1.0",-      NarInfo.niUrl = "nar/test.nar.xz",-      NarInfo.niCompression = "xz",-      NarInfo.niFileHash = Just "sha256:abc",-      NarInfo.niFileSize = Just 100,-      NarInfo.niNarHash = "sha256:def",-      NarInfo.niNarSize = 200,-      NarInfo.niReferences = ["aaaa-hello-1.0"],-      NarInfo.niDeriver = Nothing,-      NarInfo.niSigs = [],-      NarInfo.niCA = Nothing-    }---- ------------------------------------------------------------------------------ FileStore tests--- -----------------------------------------------------------------------------testFileStore :: IO Bool-testFileStore =-  runGroup-    "FileStore"-    [ test "narinfo write/read roundtrip" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        let hashKey = "testhash123"-            content = TE.encodeUtf8 ("StorePath: /nix/store/test\n" :: Text)-        wOk <- Store.writeNarInfo store hashKey content-        result <- Store.readNarInfo store hashKey-        removeDirectoryRecursive tmpDir-        ok1 <- assertTrue "write succeeded" wOk-        ok2 <- assertEqual "narinfo roundtrip" (Just content) result-        pure (ok1 && ok2),-      test "nar write/read roundtrip" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        let fileName = "test.nar.xz"-            content = BS.pack [1, 2, 3, 4, 5]-        wOk <- Store.writeNar store fileName content-        result <- Store.readNar store fileName-        removeDirectoryRecursive tmpDir-        ok1 <- assertTrue "write succeeded" wOk-        ok2 <- assertEqual "nar roundtrip" (Just content) result-        pure (ok1 && ok2),-      test "read nonexistent returns Nothing" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        result <- Store.readNarInfo store "nonexistent"-        removeDirectoryRecursive tmpDir-        assertEqual "not found" Nothing result,-      test "cacheInfo defaults" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        let info = Store.getCacheInfo store-        removeDirectoryRecursive tmpDir-        ok1 <- assertEqual "storeDir" "/nix/store" (Store.ciStoreDir info)-        ok2 <- assertTrue "wantMassQuery" (Store.ciWantMassQuery info)-        ok3 <- assertEqual "priority" 50 (Store.ciPriority info)-        pure (ok1 && ok2 && ok3),-      test "sanitizePath rejects traversal" $-        assertEqual "dotdot" Nothing (Store.sanitizePath ".."),-      test "sanitizePath rejects slash" $-        assertEqual "slash" Nothing (Store.sanitizePath "../../etc/passwd"),-      test "sanitizePath rejects backslash" $-        assertEqual "backslash" Nothing (Store.sanitizePath "..\\..\\etc\\passwd"),-      test "sanitizePath rejects empty" $-        assertEqual "empty" Nothing (Store.sanitizePath ""),-      test "sanitizePath accepts valid hash" $-        assertEqual "valid" (Just "abc123def456") (Store.sanitizePath "abc123def456"),-      test "sanitizePath rejects windows device name" $-        assertEqual "device nul" Nothing (Store.sanitizePath "nul"),-      test "sanitizePath rejects the zero-numbered devices" $ do-        ok1 <- assertEqual "com0" Nothing (Store.sanitizePath "com0")-        ok2 <- assertEqual "lpt0" Nothing (Store.sanitizePath "lpt0")-        ok3 <- assertEqual "com10 stays valid" (Just "com10") (Store.sanitizePath "com10")-        pure (ok1 && ok2 && ok3),-      test "sanitizePath rejects dotfile" $-        assertEqual "dotfile" Nothing (Store.sanitizePath ".hidden"),-      test "sanitizePath rejects a trailing dot" $-        assertEqual "trailing dot" Nothing (Store.sanitizePath "foo."),-      test "sanitizePath keeps interior dots valid" $-        assertEqual "interior dots" (Just "foo.nar.xz") (Store.sanitizePath "foo.nar.xz"),-      test "read rejects traversal" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        result <- Store.readNarInfo store "../../etc/passwd"-        removeDirectoryRecursive tmpDir-        assertEqual "blocked" Nothing result,-      test "writeNarInfo rejects traversal" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        ok <- Store.writeNarInfo store "../../etc/passwd" "bad"-        removeDirectoryRecursive tmpDir-        assertFalse "write rejected" ok,-      test "writeNar rejects traversal" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        ok <- Store.writeNar store "../escape.nar" "bad"-        removeDirectoryRecursive tmpDir-        assertFalse "write rejected" ok,-      test "listNarInfoHashes returns stored hashes" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        _ <- Store.writeNarInfo store "hash1" "content1"-        _ <- Store.writeNarInfo store "hash2" "content2"-        hashes <- Store.listNarInfoHashes store-        removeDirectoryRecursive tmpDir-        let sorted = sort hashes-        assertEqual "listed hashes" ["hash1", "hash2"] sorted,-      test "listNarInfoHashes empty store" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        hashes <- Store.listNarInfoHashes store-        removeDirectoryRecursive tmpDir-        assertEqual "empty" [] hashes-    ]---- ------------------------------------------------------------------------------ Reclaim tests--- -----------------------------------------------------------------------------testReclaim :: IO Bool-testReclaim =-  runGroup-    "Reclaim"-    [ test "a nar/<file> URL names that file, compression suffix included" $-        assertEqual-          "target"-          (Reclaim.NamesNar "1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst")-          (narInfoBytesTarget (mkValidNarInfo {NarInfo.niUrl = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst"})),-      test "a URL in any other spelling is unresolvable" $-        let spellings =-              [ "https://cache.example/nar/x.nar",-                "./nar/x.nar",-                "nar/../x.nar",-                "nar/a/x.nar",-                "nar/x.nar?v=1",-                "nar/x%2Enar",-                "nar/.nova1.tmp",-                "nar/",-                "x.nar"-              ]-            resolved = [url | url <- spellings, not (isUnresolvable (narInfoBytesTarget (mkValidNarInfo {NarInfo.niUrl = url})))]-         in assertEqual "spellings that resolved" [] resolved,-      test "an unreadable, undecodable or unparseable narinfo is unresolvable" $ do-        let unreadable = Reclaim.narInfoTarget (Left (userError "permission denied"))-            undecodable = Reclaim.narInfoTarget (Right (BS.pack [0x55, 0x52, 0x4c, 0x3a, 0xff]))-            unparseable = Reclaim.narInfoTarget (Right "StorePath: /nix/store/x\nNarSize: 5\n")-        assertTrue "all unresolvable" (all isUnresolvable [unreadable, undecodable, unparseable]),-      test "property: a nar/<name> URL resolves to the name through the parser" $-        checkProperty $-          forAll genSafeNarName $ \name ->-            narInfoBytesTarget (mkValidNarInfo {NarInfo.niUrl = "nar/" <> name}) === Reclaim.NamesNar name,-      test "property: a referenced NAR is never selected" $-        checkProperty $-          forAll genSelection $ \(minimumAge, references, nars) ->-            let selected = selectAll minimumAge references nars-             in all ((`Set.notMember` references) . Store.nfName) selected,-      test "property: the selection is a subset of the NARs present" $-        checkProperty $-          forAll genSelection $ \(minimumAge, references, nars) ->-            all (`elem` nars) (selectAll minimumAge references nars),-      test "property: every unreferenced NAR is either eligible or kept, never both" $-        checkProperty $-          forAll genSelection $ \(minimumAge, references, nars) ->-            sort (map Store.nfName (selectAll minimumAge references nars))-              === sort [Store.nfName nar | nar <- nars, Set.notMember (Store.nfName nar) references],-      test "property: eligible NARs are at least the minimum age, kept ones younger" $-        checkProperty $-          forAll genSelection $ \(minimumAge, references, nars) ->-            let plan = Reclaim.selectUnreferenced minimumAge referenceTime references nars-             in all ((>= minimumAge) . ageOf) (Reclaim.rpEligible plan)-                  && all ((< minimumAge) . ageOf) (Reclaim.rpTooYoung plan),-      test "property: adding a narinfo never grows the eligible set" $-        checkProperty $-          forAll genSelection $ \(minimumAge, references, nars) ->-            forAll genNarName $ \added ->-              let eligible refs = Reclaim.rpEligible (Reclaim.selectUnreferenced minimumAge referenceTime refs nars)-               in all (`elem` eligible references) (eligible (Set.insert added references)),-      test "property: raising the minimum age never grows the eligible set" $-        checkProperty $-          forAll genSelection $ \(minimumAge, references, nars) ->-            forAll genMinimumAge $ \raise ->-              let eligible age = Reclaim.rpEligible (Reclaim.selectUnreferenced age referenceTime references nars)-               in all (`elem` eligible minimumAge) (eligible (minimumAge + raise)),-      test "property: any unresolvable narinfo blocks the run, and all are reported" $-        checkProperty $-          forAll genSelection $ \(minimumAge, _, nars) ->-            forAll (listOf genTarget) $ \targets ->-              let narInfos = zip [show i | i <- [0 :: Int ..]] targets-                  scan = Reclaim.StoreScan {Reclaim.scanNarInfos = narInfos, Reclaim.scanNars = nars, Reclaim.scanTime = referenceTime}-                  blocking = [path | (path, Reclaim.Unresolvable _) <- narInfos]-               in case Reclaim.planReclaim minimumAge scan of-                    Left (Reclaim.UnresolvedNarInfos unresolved) -> map Reclaim.unresolvedPath (toList unresolved) === blocking-                    Left Reclaim.NoNarInfos -> property (null targets && not (null nars))-                    Right _ -> blocking === [],-      test "property: a store with NARs and no narinfo blocks the run, and an empty store plans nothing" $-        checkProperty $-          forAll genSelection $ \(minimumAge, _, nars) ->-            let scan = Reclaim.StoreScan {Reclaim.scanNarInfos = [], Reclaim.scanNars = nars, Reclaim.scanTime = referenceTime}-                expected = case nars of-                  [] -> Right Reclaim.ReclaimPlan {Reclaim.rpEligible = [], Reclaim.rpTooYoung = []}-                  _ -> Left Reclaim.NoNarInfos-             in Reclaim.planReclaim minimumAge scan === expected,-      test "a NAR exactly the minimum age old is eligible" $-        let nar = narAged "edge.nar" Reclaim.defaultMinimumAge-            plan = Reclaim.selectUnreferenced Reclaim.defaultMinimumAge referenceTime Set.empty [nar]-         in assertEqual "eligible" [nar] (Reclaim.rpEligible plan),-      test "a scan of a store: blocked by a broken narinfo, then reclaimed" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        now <- getCurrentTime-        let monthAgo = addUTCTime (negate (30 * nominalDay)) now-            narPath name = tmpDir ++ "/nar/" ++ name-            putNar name modified = do-              _ <- Store.writeNar store (T.pack name) "nar bytes"-              setModificationTime (narPath name) modified-        _ <- Store.writeNarInfo store "referencing" (TE.encodeUtf8 (NarInfo.renderNarInfo (mkValidNarInfo {NarInfo.niUrl = "nar/ref.nar.zst"})))-        _ <- Store.writeNarInfo store "broken" "not a narinfo\n"-        putNar "ref.nar.zst" monthAgo-        putNar "old.nar.zst" monthAgo-        putNar "young.nar.zst" now-        -- An upload in progress: the temp name a streaming write uses.-        BS.writeFile (narPath ".nova123-4.tmp") "partial"-        setModificationTime (narPath ".nova123-4.tmp") monthAgo-        blocked <- Reclaim.planReclaim Reclaim.defaultMinimumAge <$> Reclaim.scanStore store-        removeFile (tmpDir ++ "/narinfo/broken")-        reclaimable <- Reclaim.planReclaim Reclaim.defaultMinimumAge <$> Reclaim.scanStore store-        let blockedBy = case blocked of-              Left (Reclaim.UnresolvedNarInfos unresolved) -> map Reclaim.unresolvedPath (toList unresolved)-              _ -> []-            eligible = either (const []) (map Store.nfName . Reclaim.rpEligible) reclaimable-            tooYoung = either (const []) (map Store.nfName . Reclaim.rpTooYoung) reclaimable-        removals <- traverse (Store.removeNar store) eligible-        remaining <- sort <$> listDirectory (tmpDir ++ "/nar")-        removeDirectoryRecursive tmpDir-        ok1 <- assertTrue "blocked by the broken narinfo alone" (case blockedBy of [path] -> "broken" `isSuffixOf` path; _ -> False)-        ok2 <- assertEqual "eligible" ["old.nar.zst"] eligible-        ok3 <- assertEqual "kept" ["young.nar.zst"] tooYoung-        ok4 <- assertEqual "removals" [Store.NarRemoved] removals-        ok5 <- assertEqual "remaining" [".nova123-4.tmp", "ref.nar.zst", "young.nar.zst"] remaining-        pure (ok1 && ok2 && ok3 && ok4 && ok5),-      test "a store whose narinfo directory the server recreated blocks the run" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        now <- getCurrentTime-        _ <- Store.writeNar store "old.nar.zst" "nar bytes"-        setModificationTime (tmpDir ++ "/nar/old.nar.zst") (addUTCTime (negate (30 * nominalDay)) now)-        removeDirectoryRecursive (tmpDir ++ "/narinfo")-        restarted <- Store.newFileStore tmpDir-        plan <- Reclaim.planReclaim Reclaim.defaultMinimumAge <$> Reclaim.scanStore restarted-        removeDirectoryRecursive tmpDir-        assertEqual "blocked" (Left Reclaim.NoNarInfos) plan,-      test "a scan of a missing store fails and creates nothing" $ do-        tmpDir <- createTestDir-        let missing = tmpDir ++ "/missing"-        result <- try (Reclaim.scanStore (Store.fileStoreAt missing)) :: IO (Either IOException Reclaim.StoreScan)-        created <- doesDirectoryExist missing-        removeDirectoryRecursive tmpDir-        ok1 <- assertTrue "scan failed" (isLeft result)-        ok2 <- assertFalse "store created" created-        pure (ok1 && ok2),-      test "removeNar reports a missing file and refuses a traversal name" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        missing <- Store.removeNar store "absent.nar"-        traversal <- Store.removeNar store "../escape.nar"-        removeDirectoryRecursive tmpDir-        ok1 <- assertTrue "missing file fails" (case missing of Store.NarRemoveFailed _ -> True; _ -> False)-        ok2 <- assertEqual "traversal" Store.NarRemoveBadPath traversal-        pure (ok1 && ok2)-    ]---- | Run a QuickCheck property quietly, printing its report on failure.-checkProperty :: (Testable prop) => prop -> IO Bool-checkProperty prop = do-  result <- quickCheckWithResult stdArgs {chatty = False} prop-  case result of-    Success {} -> pure True-    Failure {output = report} -> failWith report-    GaveUp {output = report} -> failWith report-    NoExpectedFailure {output = report} -> failWith report-  where-    failWith report = do-      putStrLn ""-      putStr report-      pure False---- | The target of a narinfo rendered and read back as the store holds it.-narInfoBytesTarget :: NarInfo.NarInfo -> Reclaim.NarInfoTarget-narInfoBytesTarget = Reclaim.narInfoTarget . Right . TE.encodeUtf8 . NarInfo.renderNarInfo--isUnresolvable :: Reclaim.NarInfoTarget -> Bool-isUnresolvable (Reclaim.Unresolvable _) = True-isUnresolvable (Reclaim.NamesNar _) = False---- | Everything a selection names, eligible or kept.-selectAll :: NominalDiffTime -> Set Text -> [Store.NarFile] -> [Store.NarFile]-selectAll minimumAge references nars =-  let plan = Reclaim.selectUnreferenced minimumAge referenceTime references nars-   in Reclaim.rpEligible plan ++ Reclaim.rpTooYoung plan---- | The fixed "now" of the pure selection tests.-referenceTime :: UTCTime-referenceTime = UTCTime (fromGregorian 2026 10 9) 0--ageOf :: Store.NarFile -> NominalDiffTime-ageOf nar = diffUTCTime referenceTime (Store.nfModified nar)--narAged :: Text -> NominalDiffTime -> Store.NarFile-narAged name age =-  Store.NarFile-    { Store.nfName = name,-      Store.nfPath = "nar/" ++ T.unpack name,-      Store.nfSize = 0,-      Store.nfModified = addUTCTime (negate age) referenceTime-    }---- | NAR names from a small pool, so references and listings overlap--- often.-narNamePool :: [Text]-narNamePool = [T.pack ("n" ++ show i ++ ".nar.zst") | i <- [0 .. 11 :: Int]]--genNarName :: Gen Text-genNarName = elements narNamePool---- | Whole hours, so ages land on the minimum age exactly now and then;--- the negative end is a modification time in the future (a clock that--- stepped back).-genAge :: Gen NominalDiffTime-genAge = (* 3600) . fromInteger <$> choose (-2, 24 * 30)--genMinimumAge :: Gen NominalDiffTime-genMinimumAge = (* 3600) . fromInteger <$> choose (0, 24 * 21)---- | A listing has distinct names, as a directory does.-genListing :: Gen [Store.NarFile]-genListing = traverse (\name -> narAged name <$> genAge) =<< sublistOf narNamePool--genSelection :: Gen (NominalDiffTime, Set Text, [Store.NarFile])-genSelection = (,,) <$> genMinimumAge <*> (Set.fromList <$> sublistOf narNamePool) <*> genListing--genTarget :: Gen Reclaim.NarInfoTarget-genTarget =-  frequency-    [ (4, Reclaim.NamesNar <$> genNarName),-      (1, pure (Reclaim.Unresolvable "unparseable"))-    ]---- | Names 'Store.sanitizePath' admits.-genSafeNarName :: Gen Text-genSafeNarName =-  (T.pack <$> listOf1 (elements safeNameChars)) `suchThat` (isJust . Store.sanitizePath)-  where-    safeNameChars = ['a' .. 'z'] ++ ['A' .. 'Z'] ++ ['0' .. '9'] ++ "._+-"---- ------------------------------------------------------------------------------ Validate tests--- ------------------------------------------------------------------------------- | Bytes whose SHA-256 is used as the NarHash in 'mkValidNarInfo'.-validNarBytes :: ByteString-validNarBytes = BS.pack [1, 2, 3, 4, 5]---- | Bytes whose SHA-256 is used as the FileHash in 'mkValidNarInfo'.-validFileBytes :: ByteString-validFileBytes = BS.pack [10, 20, 30, 40, 50]---- | A narinfo that passes all field validation.-mkValidNarInfo :: NarInfo.NarInfo-mkValidNarInfo =-  NarInfo.NarInfo-    { NarInfo.niStorePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0",-      NarInfo.niUrl = "nar/test.nar.xz",-      NarInfo.niCompression = "xz",-      NarInfo.niFileHash = Just (Hash.formatNixHash (Hash.hashBytes validFileBytes)),-      NarInfo.niFileSize = Just 5,-      NarInfo.niNarHash = Hash.formatNixHash (Hash.hashBytes validNarBytes),-      NarInfo.niNarSize = 5,-      NarInfo.niReferences = ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0"],-      NarInfo.niDeriver = Nothing,-      NarInfo.niSigs = [],-      NarInfo.niCA = Nothing-    }--testValidate :: IO Bool-testValidate =-  runGroup-    "Validate"-    [ test "validateNarInfo valid" $-        assertEqual-          "valid narinfo"-          (Right mkValidNarInfo)-          (Validate.validateNarInfo mkValidNarInfo),-      test "validateNarInfo negative FileSize" $-        let ni = mkValidNarInfo {NarInfo.niFileSize = Just (-1)}-         in assertEqual-              "negative filesize"-              (Left [Validate.NegativeFileSize (-1)])-              (Validate.validateNarInfo ni),-      test "validateNarInfo negative NarSize" $-        let ni = mkValidNarInfo {NarInfo.niNarSize = -1}-         in assertEqual-              "negative narsize"-              (Left [Validate.NegativeNarSize (-1)])-              (Validate.validateNarInfo ni),-      test "validateNarInfo derivation StorePath rejected" $-        let drvPath = "/nix/store/abc12345678901234567890123456789-foo.drv"-            ni = mkValidNarInfo {NarInfo.niStorePath = drvPath}-         in case Validate.validateNarInfo ni of-              Left [Validate.DerivationStorePath raw] ->-                assertEqual "raw value" drvPath raw-              other -> do-                putStrLn ("    expected Left [DerivationStorePath ..], got: " ++ show other)-                pure False,-      test "validateNarInfo bad StorePath" $-        let ni = mkValidNarInfo {NarInfo.niStorePath = "not-a-store-path"}-         in case Validate.validateNarInfo ni of-              Left [Validate.InvalidStorePath raw _] ->-                assertEqual "raw value" "not-a-store-path" raw-              other -> do-                putStrLn ("    expected Left [InvalidStorePath ..], got: " ++ show other)-                pure False,-      test "validateNarInfo bad FileHash" $-        let ni = mkValidNarInfo {NarInfo.niFileHash = Just "md5:bogus"}-         in case Validate.validateNarInfo ni of-              Left [Validate.InvalidFileHash raw _] ->-                assertEqual "raw value" "md5:bogus" raw-              other -> do-                putStrLn ("    expected Left [InvalidFileHash ..], got: " ++ show other)-                pure False,-      test "validateNarInfo bad NarHash" $-        let ni = mkValidNarInfo {NarInfo.niNarHash = "md5:bogus"}-         in case Validate.validateNarInfo ni of-              Left [Validate.InvalidNarHash raw _] ->-                assertEqual "raw value" "md5:bogus" raw-              other -> do-                putStrLn ("    expected Left [InvalidNarHash ..], got: " ++ show other)-                pure False,-      test "validateNarInfo bad reference" $-        let ni = mkValidNarInfo {NarInfo.niReferences = ["bad"]}-         in case Validate.validateNarInfo ni of-              Left [Validate.InvalidReference raw _] ->-                assertEqual "raw value" "bad" raw-              other -> do-                putStrLn ("    expected Left [InvalidReference ..], got: " ++ show other)-                pure False,-      test "validateNarInfo multiple errors collected" $-        let ni =-              mkValidNarInfo-                { NarInfo.niFileSize = Just (-1),-                  NarInfo.niNarSize = -1,-                  NarInfo.niStorePath = "bad"-                }-         in case Validate.validateNarInfo ni of-              Left errs -> assertTrue "at least 3 errors" (length errs >= 3)-              Right _ -> do-                putStrLn "    expected Left, got Right"-                pure False,-      test "validateNarHash correct" $-        assertEqual-          "correct nar hash"-          (Right ())-          (Validate.validateNarHash mkValidNarInfo validNarBytes),-      test "validateNarHash wrong" $-        case Validate.validateNarHash mkValidNarInfo (BS.pack [99]) of-          Left (Validate.NarHashMismatch _ _) -> pure True-          other -> do-            putStrLn ("    expected Left NarHashMismatch, got: " ++ show other)-            pure False,-      test "validateFileHash correct" $-        assertEqual-          "correct file hash"-          (Right ())-          (Validate.validateFileHash mkValidNarInfo validFileBytes),-      test "validateFileHash wrong" $-        case Validate.validateFileHash mkValidNarInfo (BS.pack [99]) of-          Left (Validate.FileHashMismatch _ _) -> pure True-          other -> do-            putStrLn ("    expected Left FileHashMismatch, got: " ++ show other)-            pure False,-      test "validateSignature valid" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-            ni = mkValidNarInfo-        case Signing.sign sk ni of-          Left err -> do-            putStrLn ("  sign failed: " ++ err)-            pure False-          Right sig ->-            let niSigned = ni {NarInfo.niSigs = [sig]}-             in assertEqual "valid sig" (Right ()) (Validate.validateSignature pk niSigned),-      test "validateSignature invalid" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-            bogusSig = "bogus-key:aW52YWxpZA=="-            ni = mkValidNarInfo {NarInfo.niSigs = [bogusSig]}-        assertEqual-          "invalid sig"-          (Left [Validate.SignatureInvalid bogusSig])-          (Validate.validateSignature pk ni),-      test "validateSignature no sigs" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-        assertEqual-          "no sigs"-          (Left [Validate.NoSignatures])-          (Validate.validateSignature pk mkValidNarInfo),-      test "validateFull all good" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-            ni = mkValidNarInfo-        case Signing.sign sk ni of-          Left err -> do-            putStrLn ("  sign failed: " ++ err)-            pure False-          Right sig ->-            let niSigned = ni {NarInfo.niSigs = [sig]}-             in assertEqual-                  "full valid"-                  (Right ())-                  (Validate.validateFull pk niSigned validNarBytes validFileBytes),-      test "validateFull multiple failures" $ do-        sk <- generateTestSecretKey-        let pk = deriveTestPublicKey sk-            ni = mkValidNarInfo {NarInfo.niFileSize = Just (-1), NarInfo.niNarSize = -1}-        case Validate.validateFull pk ni (BS.pack [99]) (BS.pack [99]) of-          Left errs -> assertTrue "at least 4 errors" (length errs >= 4)-          Right _ -> do-            putStrLn "    expected Left, got Right"-            pure False-    ]---- ------------------------------------------------------------------------------ Server (WAI) tests--- ------------------------------------------------------------------------------- | The write key the authenticated-server tests configure.-serverTestApiKey :: ByteString-serverTestApiKey = "test-api-key"---- | The Authorization header 'serverTestApiKey' expects.-serverAuthHeader :: HTTP.Header-serverAuthHeader = (HTTP.hAuthorization, "Bearer " <> serverTestApiKey)---- | Store-path hash of 'validServerNarInfo' (32 nix-base32 zeros).-validNarInfoHashKey :: Text-validNarInfoHashKey = T.replicate 32 "0"---- | A canonical all-zero sha256 in nix-base32: 52 digits, and the 4--- spare bits of the 260-bit encoding are zero, so it parses.-zeroNarHash :: Text-zeroNarHash = "sha256:" <> T.replicate 52 "0"---- | The file 'validServerNarInfo' names under @nar\/@.-serverNarName :: Text-serverNarName = "test.nar"---- | The bytes 'validServerNarInfo' declares for its NAR.-serverNarBytes :: ByteString-serverNarBytes = "nar-payload-bytes"---- | A narinfo that passes 'Validate.validateNarInfo' end to end, keyed--- under 'validNarInfoHashKey', describing 'serverNarBytes' stored as--- 'serverNarName'.-validServerNarInfo :: NarInfo.NarInfo-validServerNarInfo =-  NarInfo.NarInfo-    { NarInfo.niStorePath = "/nix/store/" <> validNarInfoHashKey <> "-hello-1.0",-      NarInfo.niUrl = "nar/" <> serverNarName,-      NarInfo.niCompression = "none",-      NarInfo.niFileHash = Just (Hash.formatNixHash (Hash.hashBytes serverNarBytes)),-      NarInfo.niFileSize = Just (fromIntegral (BS.length serverNarBytes)),-      NarInfo.niNarHash = zeroNarHash,-      NarInfo.niNarSize = 200,-      NarInfo.niReferences = [validNarInfoHashKey <> "-hello-1.0"],-      NarInfo.niDeriver = Nothing,-      NarInfo.niSigs = [],-      NarInfo.niCA = Nothing-    }---- | Rendered wire form of 'validServerNarInfo'.-validServerNarInfoBytes :: BL.ByteString-validServerNarInfoBytes = renderNarInfoBody validServerNarInfo---- | A narinfo's wire form as a request body.-renderNarInfoBody :: NarInfo.NarInfo -> BL.ByteString-renderNarInfoBody = BL.fromStrict . TE.encodeUtf8 . NarInfo.renderNarInfo---- | Upload bytes as 'serverNarName', authenticated.-putServerNar :: Server.ServerConfig -> ByteString -> IO WT.SResponse-putServerNar cfg bytes = serverRequest cfg "PUT" ["nar", serverNarName] [serverAuthHeader] (BL.fromStrict bytes)---- | Upload a narinfo under 'validNarInfoHashKey', authenticated.-putServerNarInfo :: Server.ServerConfig -> NarInfo.NarInfo -> IO WT.SResponse-putServerNarInfo cfg ni = serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] (renderNarInfoBody ni)---- | Upload a NAR, then a narinfo for it; the narinfo's response, and the--- status a GET of that narinfo answers afterwards.-putNarThenNarInfo :: Server.ServerConfig -> ByteString -> NarInfo.NarInfo -> IO (WT.SResponse, HTTP.Status)-putNarThenNarInfo cfg bytes ni = do-  _ <- putServerNar cfg bytes-  putResp <- putServerNarInfo cfg ni-  getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""-  pure (putResp, WT.simpleStatus getResp)---- | Run one test against a fresh server (configurable write key and--- signing key), removing the store directory afterwards.-withServer :: Maybe ByteString -> Maybe Signing.SecretKey -> (Server.ServerConfig -> IO Bool) -> IO Bool-withServer apiKey sigKey body = do-  tmpDir <- createTestDir-  store <- Store.newFileStore tmpDir-  passed <--    body-      Server.ServerConfig-        { Server.scStore = store,-          Server.scApiKey = apiKey,-          Server.scSigningKey = sigKey,-          Server.scRootResponse = Server.defaultRootResponse-        }-  removeDirectoryRecursive tmpDir-  pure passed---- | 'withServer' with write auth armed and no signing - the common case.-withAuthedServer :: (Server.ServerConfig -> IO Bool) -> IO Bool-withAuthedServer = withServer (Just serverTestApiKey) Nothing---- | Execute a single request against the server.-serverRequest :: Server.ServerConfig -> BS.ByteString -> [Text] -> [HTTP.Header] -> BL.ByteString -> IO WT.SResponse-serverRequest cfg method segments headers body =-  WT.runSession (WT.srequest (WT.SRequest req body)) (Server.cacheApp cfg)-  where-    req =-      defaultRequest-        { requestMethod = method,-          pathInfo = segments,-          requestHeaders = headers-        }---- | Like 'serverRequest', with a declared Content-Length - for the--- reject-before-reading limit checks.-serverRequestSized :: Server.ServerConfig -> BS.ByteString -> [Text] -> [HTTP.Header] -> Word -> IO WT.SResponse-serverRequestSized cfg method segments headers declared =-  WT.runSession (WT.srequest (WT.SRequest req "")) (Server.cacheApp cfg)-  where-    req =-      defaultRequest-        { requestMethod = method,-          pathInfo = segments,-          requestHeaders = headers,-          requestBodyLength = KnownLength (fromIntegral declared)-        }---- | A chunk source yielding the given chunks, then empty (end of input).-chunkSource :: [ByteString] -> IO (IO ByteString)-chunkSource chunks = do-  remaining <- newIORef chunks-  pure $-    atomicModifyIORef' remaining $ \case-      [] -> ([], BS.empty)-      (c : cs) -> (cs, c)---- | Body bytes as strict ByteString, for infix assertions.-strictBody :: WT.SResponse -> ByteString-strictBody = BL.toStrict . WT.simpleBody---- | Take every permission from a file, and say whether that left it--- unreadable.  Windows has no mode bit that stops a read (and a--- read-only file would then resist the test's cleanup), and root reads--- a file whatever its mode, so on either there is no read failure to--- cause.-makeUnreadable :: FilePath -> IO Bool-makeUnreadable path-  | System.Info.os == "mingw32" = pure False-  | otherwise = do-      setPermissions path emptyPermissions-      not . readable <$> getPermissions path--testServer :: IO Bool-testServer =-  runGroup-    "Server"-    [ test "GET / serves the default root response" $-        withServer Nothing Nothing $ \cfg -> do-          resp <- serverRequest cfg "GET" [] [] ""-          ok1 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)-          ok2 <- assertEqual "body" "nova-cache: a Nix binary cache\n" (WT.simpleBody resp)-          pure (ok1 && ok2),-      -- newTTLCache: within the TTL the action runs once; a zero TTL-      -- never satisfies the freshness check, so every call re-runs.-      test "newTTLCache memoizes within the TTL and re-runs past it" $ do-        counter <- newIORef (0 :: Int)-        let bump = atomicModifyIORef' counter (\n -> (n + 1, n + 1))-        cachedHour <- Server.newTTLCache 3600 bump-        firstRead <- cachedHour-        secondRead <- cachedHour-        cachedNever <- Server.newTTLCache 0 bump-        thirdRead <- cachedNever-        fourthRead <- cachedNever-        ok1 <- assertEqual "memoized" (1, 1) (firstRead, secondRead)-        ok2 <- assertEqual "re-run each call" (2, 3) (thirdRead, fourthRead)-        pure (ok1 && ok2),-      test "GET /nix-cache-info renders cache metadata" $-        withServer Nothing Nothing $ \cfg -> do-          resp <- serverRequest cfg "GET" ["nix-cache-info"] [] ""-          ok1 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)-          ok2 <- assertTrue "StoreDir line" (BS.isInfixOf "StoreDir: /nix/store" (strictBody resp))-          pure (ok1 && ok2),-      test "unknown route is 404" $-        withServer Nothing Nothing $ \cfg -> do-          resp <- serverRequest cfg "GET" ["no", "such", "route"] [] ""-          assertEqual "status" HTTP.status404 (WT.simpleStatus resp),-      -- HEAD is routed exactly like GET (upstream clients probe narinfo-      -- existence with HEAD; it used to fall through to 404).-      test "HEAD is served wherever GET is" $-        withServer Nothing Nothing $ \cfg -> do-          okResp <- serverRequest cfg "HEAD" ["nix-cache-info"] [] ""-          missingResp <- serverRequest cfg "HEAD" [validNarInfoHashKey <> ".narinfo"] [] ""-          ok1 <- assertEqual "present" HTTP.status200 (WT.simpleStatus okResp)-          ok2 <- assertEqual "absent" HTTP.status404 (WT.simpleStatus missingResp)-          pure (ok1 && ok2),-      -- Write authentication-      test "PUT narinfo without auth is 401" $-        withAuthedServer $ \cfg -> do-          resp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [] validServerNarInfoBytes-          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),-      test "PUT narinfo with the wrong key is 401" $-        withAuthedServer $ \cfg -> do-          let wrongAuth = (HTTP.hAuthorization, "Bearer not-the-key")-          resp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [wrongAuth] validServerNarInfoBytes-          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),-      test "PUT then GET narinfo roundtrip" $-        withAuthedServer $ \cfg -> do-          _ <- putServerNar cfg serverNarBytes-          putResp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] validServerNarInfoBytes-          getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""-          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)-          ok2 <- assertEqual "GET status" HTTP.status200 (WT.simpleStatus getResp)-          ok3 <- assertTrue "StorePath present" (BS.isInfixOf (TE.encodeUtf8 validNarInfoHashKey) (strictBody getResp))-          ok4 <--            assertEqual-              "revalidatable, never immutable"-              (Just "public, max-age=3600, must-revalidate")-              (lookup HTTP.hCacheControl (WT.simpleHeaders getResp))-          pure (ok1 && ok2 && ok3 && ok4),-      test "PUT narinfo signs when a key is configured" $ do-        sigKey <- generateTestSecretKey-        withServer (Just serverTestApiKey) (Just sigKey) $ \cfg -> do-          _ <- putServerNar cfg serverNarBytes-          putResp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] validServerNarInfoBytes-          getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""-          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)-          ok2 <- assertTrue "Sig line present" (BS.isInfixOf "Sig: test-key:" (strictBody getResp))-          pure (ok1 && ok2),-      -- The confused-deputy gate: a narinfo describing path X cannot be-      -- stored under path Y's key.-      test "PUT narinfo under a mismatched hash is 400" $-        withAuthedServer $ \cfg -> do-          let otherKey = T.replicate 32 "1" <> ".narinfo"-          resp <- serverRequest cfg "PUT" [otherKey] [serverAuthHeader] validServerNarInfoBytes-          assertEqual "status" HTTP.status400 (WT.simpleStatus resp),-      test "PUT malformed narinfo is 400" $-        withAuthedServer $ \cfg -> do-          resp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] "not a narinfo"-          assertEqual "status" HTTP.status400 (WT.simpleStatus resp),-      test "PUT narinfo with an oversized declared length is 413" $-        withAuthedServer $ \cfg -> do-          resp <--            serverRequestSized-              cfg-              "PUT"-              [validNarInfoHashKey <> ".narinfo"]-              [serverAuthHeader]-              (fromIntegral Server.maxNarInfoBodySize + 1)-          assertEqual "status" HTTP.status413 (WT.simpleStatus resp),-      -- A narinfo is signed and stored only over the NAR it describes,-      -- so an upload that went wrong fails here and the next push sends-      -- both again.-      test "PUT narinfo with no NAR stored is 400 and stores nothing" $-        withAuthedServer $ \cfg -> do-          putResp <- putServerNarInfo cfg validServerNarInfo-          getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""-          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)-          ok2 <- assertEqual "body" "NAR nar/test.nar is not stored; upload it before its narinfo" (strictBody putResp)-          ok3 <- assertEqual "GET status" HTTP.status404 (WT.simpleStatus getResp)-          pure (ok1 && ok2 && ok3),-      test "PUT narinfo over a short NAR is 400 and stores nothing" $-        withAuthedServer $ \cfg -> do-          (putResp, getStatus) <- putNarThenNarInfo cfg (BS.take 10 serverNarBytes) validServerNarInfo-          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)-          ok2 <- assertEqual "body" "NAR nar/test.nar is 10 bytes, narinfo FileSize is 17" (strictBody putResp)-          ok3 <- assertEqual "GET status" HTTP.status404 getStatus-          pure (ok1 && ok2 && ok3),-      test "PUT narinfo over a same-size NAR of other content is 400 and stores nothing" $-        withAuthedServer $ \cfg -> do-          let other = BS.map (+ 1) serverNarBytes-          (putResp, getStatus) <- putNarThenNarInfo cfg other validServerNarInfo-          let expected =-                "NAR nar/test.nar hashes to "-                  <> TE.encodeUtf8 (Hash.formatNixHash (Hash.hashBytes other))-                  <> ", narinfo FileHash is "-                  <> TE.encodeUtf8 (Hash.formatNixHash (Hash.hashBytes serverNarBytes))-          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)-          ok2 <- assertEqual "body" expected (strictBody putResp)-          ok3 <- assertEqual "GET status" HTTP.status404 getStatus-          pure (ok1 && ok2 && ok3),-      test "PUT narinfo over its NAR is 200 and stores it" $-        withAuthedServer $ \cfg -> do-          (putResp, getStatus) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo-          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)-          ok2 <- assertEqual "GET status" HTTP.status200 getStatus-          pure (ok1 && ok2),-      test "a refused narinfo is accepted once the right NAR is uploaded again" $-        withAuthedServer $ \cfg -> do-          (refused, _) <- putNarThenNarInfo cfg (BS.take 10 serverNarBytes) validServerNarInfo-          (accepted, getStatus) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo-          ok1 <- assertEqual "first PUT status" HTTP.status400 (WT.simpleStatus refused)-          ok2 <- assertEqual "second PUT status" HTTP.status200 (WT.simpleStatus accepted)-          ok3 <- assertEqual "GET status" HTTP.status200 getStatus-          pure (ok1 && ok2 && ok3),-      test "PUT narinfo whose URL is not under nar/ is 400" $-        withAuthedServer $ \cfg -> do-          let elsewhere = validServerNarInfo {NarInfo.niUrl = "https://cache.example/nar/" <> serverNarName}-          (putResp, getStatus) <- putNarThenNarInfo cfg serverNarBytes elsewhere-          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)-          ok2 <- assertEqual "body" "narinfo URL https://cache.example/nar/test.nar does not name a file under nar/" (strictBody putResp)-          ok3 <- assertEqual "GET status" HTTP.status404 getStatus-          pure (ok1 && ok2 && ok3),-      test "PUT narinfo without FileSize or FileHash is 400" $-        withAuthedServer $ \cfg -> do-          (noSize, _) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo {NarInfo.niFileSize = Nothing}-          (noHash, getStatus) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo {NarInfo.niFileHash = Nothing}-          ok1 <- assertEqual "no FileSize" (HTTP.status400, "narinfo declares no FileSize to check its NAR against") (WT.simpleStatus noSize, strictBody noSize)-          ok2 <- assertEqual "no FileHash" (HTTP.status400, "narinfo declares no FileHash to check its NAR against") (WT.simpleStatus noHash, strictBody noHash)-          ok3 <- assertEqual "GET status" HTTP.status404 getStatus-          pure (ok1 && ok2 && ok3),-      test "PUT narinfo over an unreadable NAR is 500 and stores nothing" $-        withAuthedServer $ \cfg -> do-          _ <- putServerNar cfg serverNarBytes-          unreadable <- makeUnreadable (Store.fsRoot (Server.scStore cfg) ++ "/nar/" ++ T.unpack serverNarName)-          if not unreadable-            then pure True-            else do-              putResp <- putServerNarInfo cfg validServerNarInfo-              getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""-              ok1 <- assertEqual "PUT status" HTTP.status500 (WT.simpleStatus putResp)-              ok2 <- assertEqual "body" "NAR nar/test.nar could not be read" (strictBody putResp)-              ok3 <- assertEqual "GET status" HTTP.status404 (WT.simpleStatus getResp)-              pure (ok1 && ok2 && ok3),-      -- The hash listing is push-tool plumbing: authenticated, uncacheable.-      test "GET /narinfo-hashes without auth is 401" $-        withAuthedServer $ \cfg -> do-          resp <- serverRequest cfg "GET" ["narinfo-hashes"] [] ""-          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),-      test "GET /narinfo-hashes with auth lists hashes, uncacheable" $-        withAuthedServer $ \cfg -> do-          _ <- putServerNar cfg serverNarBytes-          putResp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] validServerNarInfoBytes-          resp <- serverRequest cfg "GET" ["narinfo-hashes"] [serverAuthHeader] ""-          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)-          ok2 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)-          ok3 <- assertTrue "uploaded hash listed" (BS.isInfixOf (TE.encodeUtf8 validNarInfoHashKey) (strictBody resp))-          ok4 <- assertEqual "no-store" (Just "no-store") (lookup HTTP.hCacheControl (WT.simpleHeaders resp))-          pure (ok1 && ok2 && ok3 && ok4),-      test "GET /narinfo-hashes in open mode needs no auth" $-        withServer Nothing Nothing $ \cfg -> do-          resp <- serverRequest cfg "GET" ["narinfo-hashes"] [] ""-          assertEqual "status" HTTP.status200 (WT.simpleStatus resp),-      -- NAR transfer-      test "PUT then GET and HEAD a NAR" $-        withAuthedServer $ \cfg -> do-          putResp <- serverRequest cfg "PUT" ["nar", "test.nar"] [serverAuthHeader] "nar-payload-bytes"-          getResp <- serverRequest cfg "GET" ["nar", "test.nar"] [] ""-          headResp <- serverRequest cfg "HEAD" ["nar", "test.nar"] [] ""-          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)-          ok2 <- assertEqual "GET status" HTTP.status200 (WT.simpleStatus getResp)-          ok3 <- assertEqual "GET body" "nar-payload-bytes" (WT.simpleBody getResp)-          ok4 <--            assertEqual-              "immutable content address"-              (Just "public, max-age=31536000, immutable")-              (lookup HTTP.hCacheControl (WT.simpleHeaders getResp))-          ok5 <- assertEqual "HEAD status" HTTP.status200 (WT.simpleStatus headResp)-          pure (ok1 && ok2 && ok3 && ok4 && ok5),-      test "PUT NAR without auth is 401" $-        withAuthedServer $ \cfg -> do-          resp <- serverRequest cfg "PUT" ["nar", "test.nar"] [] "nar-payload-bytes"-          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),-      test "PUT NAR with a traversal name is 400" $-        withAuthedServer $ \cfg -> do-          resp <- serverRequest cfg "PUT" ["nar", ".."] [serverAuthHeader] "escape"-          assertEqual "status" HTTP.status400 (WT.simpleStatus resp),-      test "PUT NAR with an oversized declared length is 413" $-        withAuthedServer $ \cfg -> do-          resp <--            serverRequestSized-              cfg-              "PUT"-              ["nar", "test.nar"]-              [serverAuthHeader]-              (fromIntegral Server.maxNarBodySize + 1)-          assertEqual "status" HTTP.status413 (WT.simpleStatus resp),-      test "GET absent NAR is 404" $-        withServer Nothing Nothing $ \cfg -> do-          resp <- serverRequest cfg "GET" ["nar", "absent.nar"] [] ""-          assertEqual "status" HTTP.status404 (WT.simpleStatus resp),-      -- Streaming write, at the store layer-      test "writeNarStreaming writes chunks and lands atomically" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        source <- chunkSource ["ab", "cd", "ef"]-        result <- Store.writeNarStreaming store "streamed.nar" 16 source-        stored <- Store.readNar store "streamed.nar"-        located <- Store.narFilePath store "streamed.nar"-        removeDirectoryRecursive tmpDir-        ok1 <- assertEqual "result" Store.NarWriteOk result-        ok2 <- assertEqual "content" (Just "abcdef") stored-        ok3 <- assertTrue "narFilePath resolves" (isJust located)-        pure (ok1 && ok2 && ok3),-      test "writeNarStreaming over the cap deletes the partial file" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        source <- chunkSource ["four", "more", "over"]-        result <- Store.writeNarStreaming store "big.nar" 8 source-        leftovers <- listDirectory (tmpDir ++ "/nar")-        removeDirectoryRecursive tmpDir-        ok1 <- assertEqual "result" Store.NarWriteTooLarge result-        ok2 <- assertEqual "no partial files" [] leftovers-        pure (ok1 && ok2),-      test "writeNarStreaming rejects a traversal name" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        source <- chunkSource ["x"]-        result <- Store.writeNarStreaming store "../escape" 8 source-        removeDirectoryRecursive tmpDir-        assertEqual "result" Store.NarWriteBadPath result,-      test "narFilePath rejects a traversal name" $ do-        tmpDir <- createTestDir-        store <- Store.newFileStore tmpDir-        located <- Store.narFilePath store "../escape"-        removeDirectoryRecursive tmpDir-        assertEqual "path" Nothing located-    ]---- ------------------------------------------------------------------------------ Stored NAR check (pure)--- -----------------------------------------------------------------------------testStoredNarCheck :: IO Bool-testStoredNarCheck =-  runGroup-    "Stored NAR check"-    [ test "a narinfo URL resolves to the file its GET is routed to" $-        let cases =-              [ ("nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst", Just "1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst"),-                ("nar/a%2Bb.nar", Just "a+b.nar"),-                ("nar/x.nar?v=1", Just "x.nar"),-                ("/nar/x.nar", Nothing),-                ("https://cache.example/nar/x.nar", Nothing),-                ("nar/../x.nar", Nothing),-                ("nar/a/x.nar", Nothing),-                ("x.nar", Nothing)-              ]-         in assertEqual-              "mismatches"-              []-              [(url, expected, actual) | (url, expected) <- cases, let actual = Server.narUrlFileName url, actual /= expected],-      test "property: a NAR's request path, as a narinfo URL, resolves to its name" $-        checkProperty $-          forAll (T.pack <$> arbitrary) $ \name ->-            Server.narUrlFileName (relativeRequestPath ["nar", name]) === Just name,-      test "property: a nar/<name> URL claims that name, FileSize and FileHash" $-        checkProperty $-          forAll genClaim $ \claim ->-            Server.narClaim (claimingNarInfo claim) === Right claim,-      test "a narinfo with no FileSize, no usable FileHash or no served URL claims nothing" $ do-        let refusal ni = either Just (const Nothing) (Server.narClaim ni)-            withUrl url = validServerNarInfo {NarInfo.niUrl = url}-        ok1 <- assertEqual "no FileSize" (Just Server.NarFileSizeUndeclared) (refusal validServerNarInfo {NarInfo.niFileSize = Nothing})-        ok2 <- assertEqual "no FileHash" (Just Server.NarFileHashUndeclared) (refusal validServerNarInfo {NarInfo.niFileHash = Nothing})-        ok3 <--          assertTrue-            "unparseable FileHash"-            ( case refusal validServerNarInfo {NarInfo.niFileHash = Just "sha256:abc"} of-                Just (Server.NarFileHashInvalid "sha256:abc" _) -> True-                _ -> False-            )-        ok4 <--          assertEqual-            "names no store could hold"-            [Just (Server.NarUrlNotServed url) | url <- unservedUrls]-            (map (refusal . withUrl) unservedUrls)-        pure (ok1 && ok2 && ok3 && ok4),-      test "property: a stored NAR is accepted exactly when it is the declared file" $-        checkProperty $-          forAll genClaim $ \claim ->-            forAll (genStoredNear claim) $ \stored ->-              isRight (Server.judgeStoredNar claim (Just stored))-                === (stored == Server.StoredNar (Server.ncFileSize claim) (Server.ncFileHash claim)),-      test "property: a NAR of another size is refused on its size" $-        checkProperty $-          forAll genClaim $ \claim ->-            forAll (genFileSize `suchThat` (/= Server.ncFileSize claim)) $ \size ->-              forAll genNixHash $ \digest ->-                Server.judgeStoredNar claim (Just (Server.StoredNar size digest))-                  === Left (Server.NarSizeMismatch (Server.ncFileName claim) (Server.ncFileSize claim) size),-      test "property: a same-size NAR of other content is refused on its hash" $-        checkProperty $-          forAll genClaim $ \claim ->-            forAll (genNixHash `suchThat` (/= Server.ncFileHash claim)) $ \digest ->-              Server.judgeStoredNar claim (Just (Server.StoredNar (Server.ncFileSize claim) digest))-                === Left (Server.NarHashMismatch (Server.ncFileName claim) (Server.ncFileHash claim) digest),-      test "property: an absent NAR is refused as not stored" $-        checkProperty $-          forAll genClaim $ \claim ->-            Server.judgeStoredNar claim Nothing === Left (Server.NarNotStored (Server.ncFileName claim))-    ]---- | URLs that name nothing under @nar\/@ a store could hold: an empty--- name, an upload's temp-file name, and a Windows device.-unservedUrls :: [Text]-unservedUrls = ["nar/", "nar/.nova123-4.tmp", "nar/nul"]---- | The request path a client forms for segments, relative to the cache--- root and percent-encoded as http-types encodes a path.-relativeRequestPath :: [Text] -> Text-relativeRequestPath =-  TE.decodeLatin1 . BL.toStrict . Builder.toLazyByteString . HTTP.encodePathSegmentsRelative---- | 'validServerNarInfo' declaring a claim's name, FileSize and FileHash.-claimingNarInfo :: Server.NarClaim -> NarInfo.NarInfo-claimingNarInfo claim =-  validServerNarInfo-    { NarInfo.niUrl = "nar/" <> Server.ncFileName claim,-      NarInfo.niFileSize = Just (Server.ncFileSize claim),-      NarInfo.niFileHash = Just (Hash.formatNixHash (Server.ncFileHash claim))-    }---- | A FileSize the narinfo parser admits: up to the uint64 maximum.-genFileSize :: Gen Integer-genFileSize = choose (0, 18446744073709551615)--genNixHash :: Gen Hash.NixHash-genNixHash = Hash.NixHash . BS.pack <$> vectorOf 32 arbitrary--genClaim :: Gen Server.NarClaim-genClaim = Server.NarClaim <$> genSafeNarName <*> genFileSize <*> genNixHash---- | A stored NAR that keeps or replaces each of the claim's size and--- hash independently, so the exact file turns up about a quarter of--- the time.-genStoredNear :: Server.NarClaim -> Gen Server.StoredNar-genStoredNear claim =-  Server.StoredNar-    <$> oneof [pure (Server.ncFileSize claim), genFileSize]-    <*> oneof [pure (Server.ncFileHash claim), genNixHash]+{-# LANGUAGE CPP #-}+{-# LANGUAGE LambdaCase #-}++module Main (main) where++import Control.Concurrent (MVar, ThreadId, forkFinally, killThread, myThreadId, newEmptyMVar, putMVar, takeMVar, yield)+import Control.Exception (AsyncException (ThreadKilled, UserInterrupt), Exception, IOException, SomeException, bracket_, displayException, finally, fromException, throwTo, try)+import Control.Monad (guard, replicateM, void, zipWithM)+import qualified Crypto.PubKey.Ed25519 as Ed25519+import Data.Bits (shiftR, (.&.))+import Data.ByteArray (convert)+import Data.ByteString (ByteString)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Base64 as B64+import qualified Data.ByteString.Builder as Builder+import qualified Data.ByteString.Lazy as BL+import Data.Either (isLeft, isRight)+import Data.Foldable (toList, traverse_)+import Data.IORef (atomicModifyIORef', newIORef, readIORef, writeIORef)+import Data.List (isSuffixOf, sort)+import Data.Maybe (isJust)+import Data.Set (Set)+import qualified Data.Set as Set+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import Data.Time.Calendar (fromGregorian)+import Data.Time.Clock (NominalDiffTime, UTCTime (..), addUTCTime, diffUTCTime, getCurrentTime, nominalDay)+import Data.Traversable (for)+import Data.Word (Word64, Word8)+import GHC.Conc (BlockReason (..), ThreadStatus (..), threadStatus)+import qualified Network.HTTP.Types as HTTP+import Network.Wai (RequestBodyLength (..), defaultRequest, pathInfo, queryString, requestBodyLength, requestHeaders, requestMethod)+import qualified Network.Wai.Test as WT+import qualified NovaCache.Base32 as Base32+import qualified NovaCache.Hash as Hash+import qualified NovaCache.NAR as NAR+import qualified NovaCache.NAR.Stream as Stream+import qualified NovaCache.NarInfo as NarInfo+import qualified NovaCache.Reclaim as Reclaim+import qualified NovaCache.Server as Server+import qualified NovaCache.Signing as Signing+import qualified NovaCache.Store as Store+import qualified NovaCache.StorePath as StorePath+import qualified NovaCache.Validate as Validate+import System.Directory (Permissions, createDirectory, createDirectoryLink, createFileLink, doesDirectoryExist, emptyPermissions, getPermissions, getTemporaryDirectory, listDirectory, readable, removeDirectoryRecursive, removeFile, searchable, setModificationTime, setOwnerExecutable, setOwnerReadable, setOwnerSearchable, setPermissions)+import System.Exit (exitFailure, exitSuccess)+import System.IO (hFlush, stdout)+import System.IO.Error (ioeGetErrorString, isPermissionError, isUserError, mkIOError, permissionErrorType)+import qualified System.Info+import Test.QuickCheck (Args (chatty), Gen, Result (Failure, GaveUp, NoExpectedFailure, Success, output), Testable, arbitrary, choose, elements, forAll, frequency, ioProperty, listOf, listOf1, oneof, property, quickCheckWithResult, stdArgs, sublistOf, suchThat, suchThatMap, vectorOf, (.&&.), (===))+#if defined(darwin_HOST_OS)+import Foreign.C.Error (throwErrnoPathIfMinus1_)+import Foreign.C.String (CString)+import Foreign.C.Types (CInt (..))+import System.Directory (getSymbolicLinkTarget)+import System.Posix.Internals (withFilePath)+import System.Posix.Types (CMode (..))+#endif+#if !defined(mingw32_HOST_OS)+import qualified Network.Socket as Socket+import qualified System.Posix.Files as Posix+#endif++-- ---------------------------------------------------------------------------+-- Test harness (hand-rolled, no framework)+-- ---------------------------------------------------------------------------++-- | Run a named test, short-circuit on first failure.+test :: String -> IO Bool -> IO Bool+test name action = do+  putStr ("  " ++ name ++ "... ")+  hFlush stdout+  result <- action+  if result+    then do+      putStrLn "OK"+      pure True+    else do+      putStrLn "FAILED"+      pure False++-- | Assert equality.+assertEqual :: (Eq a, Show a) => String -> a -> a -> IO Bool+assertEqual label expected actual+  | expected == actual = pure True+  | otherwise = do+      putStrLn ""+      putStrLn ("    " ++ label)+      putStrLn ("    expected: " ++ show expected)+      putStrLn ("    actual:   " ++ show actual)+      pure False++-- | Assert a Right value matches.+assertRight :: (Eq a, Show a) => String -> a -> Either String a -> IO Bool+assertRight label expected (Right actual) = assertEqual label expected actual+assertRight label _ (Left err) = do+  putStrLn ""+  putStrLn ("    " ++ label)+  putStrLn ("    expected Right, got Left: " ++ err)+  pure False++-- | Assert a Left (error case).+assertLeft :: (Show a) => String -> Either String a -> IO Bool+assertLeft _ (Left _) = pure True+assertLeft label (Right val) = do+  putStrLn ""+  putStrLn ("    " ++ label)+  putStrLn ("    expected Left, got Right: " ++ show val)+  pure False++-- | Assert a Bool is True.+assertTrue :: String -> Bool -> IO Bool+assertTrue _ True = pure True+assertTrue label False = do+  putStrLn ""+  putStrLn ("    " ++ label ++ ": expected True")+  pure False++-- | Assert a Bool is False.+assertFalse :: String -> Bool -> IO Bool+assertFalse _ False = pure True+assertFalse label True = do+  putStrLn ""+  putStrLn ("    " ++ label ++ ": expected False")+  pure False++-- | Run a group of tests, stopping at first failure.+runGroup :: String -> [IO Bool] -> IO Bool+runGroup name tests = do+  putStrLn (name ++ ":")+  go tests+  where+    go [] = pure True+    go (t : ts) = do+      ok <- t+      if ok then go ts else pure False++-- | Run all test groups.+runAll :: [IO Bool] -> IO ()+runAll groups = do+  results <- sequence groups+  let passed = length (filter id results)+      total = length results+  putStrLn ""+  if and results+    then do+      putStrLn ("All " ++ show total ++ " groups passed.")+      exitSuccess+    else do+      putStrLn (show passed ++ "/" ++ show total ++ " groups passed.")+      exitFailure++main :: IO ()+main = do+  putStrLn "nova-cache test suite"+  putStrLn "======================"+  putStrLn ""+  runAll+    [ testBase32,+      testHash,+      testStorePath,+      testNAR,+      testStream,+      testWalkFailures,+      testNarInfo,+      testSigning,+      testFileStore,+      testReclaim,+      testValidate,+      testServer,+      testStoredNarCheck,+      testMultipartUpload,+      testRemoval+    ]++-- ---------------------------------------------------------------------------+-- Base32 tests+-- ---------------------------------------------------------------------------++testBase32 :: IO Bool+testBase32 =+  runGroup+    "Base32"+    [ test "encode empty" $+        assertEqual "encode empty" "" (Base32.encode BS.empty),+      test "decode empty" $+        assertRight "decode empty" BS.empty (Base32.decode ""),+      test "encode/decode roundtrip (single byte)" $+        let bs = BS.singleton 0xFF+            encoded = Base32.encode bs+         in assertRight "roundtrip 0xFF" bs (Base32.decode encoded),+      test "encode/decode roundtrip (known SHA-256)" $+        let bs = BS.pack [0 .. 31]+            encoded = Base32.encode bs+         in assertRight "roundtrip 32 bytes" bs (Base32.decode encoded),+      test "encode/decode roundtrip (all zeros)" $+        let bs = BS.replicate 32 0+            encoded = Base32.encode bs+         in assertRight "roundtrip zeros" bs (Base32.decode encoded),+      test "encode/decode roundtrip (all 0xFF)" $+        let bs = BS.replicate 32 0xFF+            encoded = Base32.encode bs+         in assertRight "roundtrip 0xFF*32" bs (Base32.decode encoded),+      test "decode invalid character" $+        assertLeft "invalid char" (Base32.decode "hello!"),+      test "encode length for 32 bytes" $+        -- 32 bytes gives ceil(32*8/5) = ceil(51.2) = 52 chars+        let bs = BS.replicate 32 0x42+            encoded = Base32.encode bs+         in assertEqual "encoded length" 52 (T.length encoded),+      test "nix known vector" $+        -- SHA-256 of the empty string exactly as real Nix renders it.  A+        -- wrong alphabet or bit order stays self-consistent in roundtrips;+        -- only an external vector catches it.+        assertEqual+          "sha256 of empty in nix-base32"+          "sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"+          (Hash.formatNixHash (Hash.hashBytes BS.empty)),+      test "decode rejects nonzero padding bits" $+        -- 52 chars carry 260 bits for a 256-bit value; the 4 spare bits+        -- must be zero in canonical nix-base32.+        assertLeft "nonzero padding" (Base32.decode (T.replicate 52 "z"))+    ]++-- ---------------------------------------------------------------------------+-- Hash tests+-- ---------------------------------------------------------------------------++testHash :: IO Bool+testHash =+  runGroup+    "Hash"+    [ test "hashBytes deterministic" $+        let h1 = Hash.hashBytes (BS.pack [1, 2, 3])+            h2 = Hash.hashBytes (BS.pack [1, 2, 3])+         in assertEqual "deterministic" h1 h2,+      test "hashBytes different inputs differ" $+        let h1 = Hash.hashBytes (BS.pack [1, 2, 3])+            h2 = Hash.hashBytes (BS.pack [4, 5, 6])+         in assertTrue "different" (h1 /= h2),+      test "hashBytes is 32 bytes" $+        let Hash.NixHash raw = Hash.hashBytes BS.empty+         in assertEqual "32 bytes" 32 (BS.length raw),+      test "formatNixHash prefix" $+        let formatted = Hash.formatNixHash (Hash.hashBytes BS.empty)+         in assertTrue "sha256: prefix" (T.isPrefixOf "sha256:" formatted),+      test "formatNixHash/parseNixHash roundtrip" $+        let h = Hash.hashBytes (BS.pack [42])+            formatted = Hash.formatNixHash h+         in assertRight "roundtrip" h (Hash.parseNixHash formatted),+      test "parseNixHash bad prefix" $+        assertLeft "bad prefix" (Hash.parseNixHash "md5:abc"),+      test "parseNixHash bad base32" $+        assertLeft "bad base32" (Hash.parseNixHash "sha256:!!invalid!!")+    ]++-- ---------------------------------------------------------------------------+-- StorePath tests+-- ---------------------------------------------------------------------------++testStorePath :: IO Bool+testStorePath =+  runGroup+    "StorePath"+    [ test "parse/render roundtrip" $+        let storeDir = StorePath.defaultStoreDir+            input = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0"+         in case StorePath.parseStorePath storeDir input of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right sp ->+                assertEqual "roundtrip" input (StorePath.renderStorePath storeDir sp),+      test "parse basename only" $+        let storeDir = StorePath.defaultStoreDir+            basename = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0"+         in case StorePath.parseStorePath storeDir basename of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right sp ->+                assertEqual "basename" basename (StorePath.storePathBaseName sp),+      test "parse extracts hash" $+        let storeDir = StorePath.defaultStoreDir+            input = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-test"+         in case StorePath.parseStorePath storeDir input of+              Left _ -> pure False+              Right sp ->+                assertEqual "hash" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" (StorePath.storePathHashString sp),+      test "reject too short" $+        let storeDir = StorePath.defaultStoreDir+         in assertLeft "too short" (StorePath.parseStorePath storeDir "abc-def"),+      test "reject empty name" $+        let storeDir = StorePath.defaultStoreDir+         in assertLeft "empty name" (StorePath.parseStorePath storeDir "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-"),+      test "reject invalid name chars" $+        let storeDir = StorePath.defaultStoreDir+            input = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello world"+         in assertLeft "invalid chars" (StorePath.parseStorePath storeDir input),+      -- Upstream's dot rule: the first dash-separated component may not+      -- be "." or ".."; other dot-leading names stay valid.+      test "reject dot-segment names" $+        let hash = T.replicate 32 "a"+            rejected name = assertLeft (T.unpack name) (StorePath.parseStorePathBaseName (hash <> "-" <> name))+         in do+              ok1 <- rejected "."+              ok2 <- rejected ".."+              ok3 <- rejected ".-x"+              ok4 <- rejected "..-y"+              pure (ok1 && ok2 && ok3 && ok4),+      test "dotfile-style names stay valid" $+        let hash = T.replicate 32 "a"+         in do+              ok1 <- assertTrue "dotfile" (either (const False) (const True) (StorePath.parseStorePathBaseName (hash <> "-.config-1.0")))+              ok2 <- assertTrue "interior dots" (either (const False) (const True) (StorePath.parseStorePathBaseName (hash <> "-x.y.z")))+              pure (ok1 && ok2)+    ]++-- ---------------------------------------------------------------------------+-- NAR tests+-- ---------------------------------------------------------------------------++testNAR :: IO Bool+testNAR =+  runGroup+    "NAR"+    [ test "serialise/deserialise roundtrip (regular file)" $+        let entry = NAR.NarRegular False (BS.pack [72, 101, 108, 108, 111])+            serialised = NAR.serialise entry+         in assertRight "roundtrip regular" entry (NAR.deserialise serialised),+      test "serialise/deserialise roundtrip (empty file)" $+        let entry = NAR.NarRegular False BS.empty+            serialised = NAR.serialise entry+         in assertRight "roundtrip empty" entry (NAR.deserialise serialised),+      test "serialise/deserialise roundtrip (executable)" $+        let entry = NAR.NarRegular True (BS.pack [0x7F, 0x45, 0x4C, 0x46])+            serialised = NAR.serialise entry+         in assertRight "roundtrip exec" entry (NAR.deserialise serialised),+      test "serialise/deserialise roundtrip (symlink)" $+        let entry = NAR.NarSymlink "/usr/bin/hello"+            serialised = NAR.serialise entry+         in assertRight "roundtrip symlink" entry (NAR.deserialise serialised),+      test "serialise/deserialise roundtrip (directory)" $+        let entry =+              NAR.NarDirectory+                [ ("bar", NAR.NarRegular False (BS.pack [2])),+                  ("foo", NAR.NarRegular False (BS.pack [1]))+                ]+            serialised = NAR.serialise entry+         in assertRight "roundtrip dir" entry (NAR.deserialise serialised),+      test "serialise/deserialise roundtrip (nested directory)" $+        let entry =+              NAR.NarDirectory+                [ ("bin", NAR.NarDirectory [("hello", NAR.NarRegular True (BS.pack [42]))]),+                  ("lib", NAR.NarSymlink "../lib64")+                ]+            serialised = NAR.serialise entry+         in assertRight "roundtrip nested" entry (NAR.deserialise serialised),+      test "directory entries sorted" $+        let entry =+              NAR.NarDirectory+                [ ("zebra", NAR.NarRegular False BS.empty),+                  ("alpha", NAR.NarRegular False BS.empty)+                ]+            serialised = NAR.serialise entry+         in case NAR.deserialise serialised of+              Left err -> do+                putStrLn ("  deserialise failed: " ++ err)+                pure False+              Right (NAR.NarDirectory entries) ->+                assertEqual "sorted" ["alpha", "zebra"] (map fst entries)+              Right other -> do+                putStrLn ("  expected directory, got: " ++ show other)+                pure False,+      test "narHash deterministic" $+        let entry = NAR.NarRegular False (BS.pack [1, 2, 3])+            h1 = NAR.narHash entry+            h2 = NAR.narHash entry+         in assertEqual "deterministic hash" h1 h2,+      test "narHash differs for different content" $+        let h1 = NAR.narHash (NAR.NarRegular False (BS.pack [1]))+            h2 = NAR.narHash (NAR.NarRegular False (BS.pack [2]))+         in assertTrue "different hashes" (h1 /= h2),+      test "deserialise garbage fails" $+        assertLeft "garbage" (NAR.deserialise (BS.pack [0, 0, 0, 0, 0, 0, 0, 0])),+      test "roundtrip edge: empty directory" $+        let entry = NAR.NarDirectory []+         in assertRight "empty dir" entry (NAR.deserialise (NAR.serialise entry)),+      test "roundtrip edge: contents a multiple of 8 (zero padding)" $+        let entry = NAR.NarRegular False (BS.replicate 8 0x41)+         in assertRight "8-byte contents" entry (NAR.deserialise (NAR.serialise entry)),+      test "roundtrip edge: executable empty file" $+        let entry = NAR.NarRegular True BS.empty+         in assertRight "exec empty" entry (NAR.deserialise (NAR.serialise entry)),+      -- Cache-served archives are untrusted input: every name that could+      -- traverse out of a POSIX extraction root must fail the parse.+      test "unsafe directory entry names rejected" $+        let evil name = NAR.serialise (NAR.NarDirectory [(name, NAR.NarRegular False "x")])+            names = ["..", ".", "", "a/b", "a\0b"]+            rejected bytes = either (const True) (const False) (NAR.deserialise bytes)+         in assertTrue "all unsafe names rejected" (all (rejected . evil) names),+      test "duplicate directory entries rejected" $+        let dup =+              NAR.serialise+                ( NAR.NarDirectory+                    [ ("same", NAR.NarRegular False "1"),+                      ("same", NAR.NarRegular False "2")+                    ]+                )+         in assertLeft "duplicate entries" (NAR.deserialise dup),+      test "out-of-order directory entries rejected" $+        assertLeft "unsorted entries" (NAR.deserialise outOfOrderDirNar),+      test "trailing bytes after the root node rejected" $+        let valid = NAR.serialise (NAR.NarRegular False "x")+         in assertLeft "trailing bytes" (NAR.deserialise (valid <> "junk1234")),+      test "nonzero string padding rejected" $+        assertLeft "nonzero padding" (NAR.deserialise badPaddingNar),+      -- The format fixes the executable marker's value as empty; upstream+      -- rejects a nonempty value, and NAR.serialise cannot produce one.+      test "nonempty executable marker rejected" $+        let marked =+              BS.concat+                ( map+                    (narWireStr 0)+                    ["nix-archive-1", "(", "type", "regular", "executable", "X", "contents", "hi", ")"]+                )+         in assertLeft "nonempty marker" (NAR.deserialise marked),+      -- Upstream's restore accepts these on Unix, and real+      -- cache.nixos.org archives carry them (perl man pages named+      -- ExtUtils::MakeMaker.3, kernel trees carrying aux.c): the+      -- parser must too, or substitution fails where upstream+      -- succeeds.  Rejecting them is the store writer's call, via+      -- NAR.isWindowsHazardName, where the target filesystem needs it.+      test "Windows-hazard names parse, as upstream accepts" $+        let entryFor name = NAR.NarDirectory [(name, NAR.NarRegular False "x")]+            names = ["ExtUtils::MakeMaker.3", "aux.c", "a:b", "a\\b", "nul", "foo.", "foo "]+            roundTrips name = NAR.deserialise (NAR.serialise (entryFor name)) == Right (entryFor name)+         in assertTrue "all parse and round-trip" (all roundTrips names),+      -- Windows resolves these names to something other than a file of+      -- this spelling (drive/stream colon, separator backslash, device,+      -- NTFS dot/space strip).+      test "isWindowsHazardName flags the hazard categories" $+        -- Among the devices: COM0/LPT0 are reserved alongside COM1-9,+        -- the device stem is compared with trailing spaces trimmed+        -- ("NUL .txt" still opens the device), and the superscript+        -- digits (here U+00B9 as UTF-8) count as device digits.+        let names =+              [ "C:evil",+                "a:b",+                "a\\b",+                "nul",+                "NUL",+                "com1",+                "nul.txt",+                "foo.",+                "foo ",+                "com0",+                "lpt0",+                "nul .txt",+                "CON .x",+                "com" <> BS.pack [0xC2, 0xB9]+              ]+         in assertTrue "all hazard names flagged" (all NAR.isWindowsHazardName names),+      test "near-miss names parse and pass the hazard predicate" $+        let plain name = NAR.serialise (NAR.NarDirectory [(name, NAR.NarRegular False "x")])+            -- "com" followed by a non-digit non-superscript byte (here+            -- U+00B4, acute accent) is an ordinary name; so are stems+            -- one character too long.+            names = ["nul2", "com10", "conx", "foo.bar", "a.b.c", "lpt00", "com" <> BS.pack [0xC2, 0xB4]]+            accepted bytes = either (const False) (const True) (NAR.deserialise bytes)+            clean name = accepted (plain name) && not (NAR.isWindowsHazardName name)+         in assertTrue "all near-miss names accepted" (all clean names),+      -- Upstream carries names and targets as raw bytes: entries that+      -- do not decode as UTF-8 parse and round-trip.+      test "non-UTF-8 entry name round-trips" $+        let entry = NAR.NarDirectory [(BS.pack [0x66, 0xFF], NAR.NarRegular False "x")]+         in assertRight "raw-byte name" entry (NAR.deserialise (NAR.serialise entry)),+      test "non-UTF-8 symlink target round-trips" $+        let entry = NAR.NarSymlink (BS.pack [0x2F, 0x74, 0x6D, 0x70, 0x2F, 0xFF])+         in assertRight "raw-byte target" entry (NAR.deserialise (NAR.serialise entry)),+      test "entries sort bytewise, non-UTF-8 names included" $+        let entry =+              NAR.NarDirectory+                [ (BS.pack [0xFF], NAR.NarRegular False "hi"),+                  ("b", NAR.NarRegular False "lo")+                ]+         in case NAR.deserialise (NAR.serialise entry) of+              Left err -> do+                putStrLn ("  deserialise failed: " ++ err)+                pure False+              Right (NAR.NarDirectory entries) ->+                assertEqual "byte order" ["b", BS.pack [0xFF]] (map fst entries)+              Right other -> do+                putStrLn ("  expected directory, got: " ++ show other)+                pure False,+      -- The hazard checks are ASCII-structural, so they fire inside+      -- names that do not decode as text.+      test "hazards inside non-UTF-8 names still flagged" $+        let names = ["nul." <> BS.pack [0xFF], BS.pack [0xFF, 0x2E], BS.pack [0xFF] <> ":x"]+         in assertTrue "all hazard bytes flagged" (all NAR.isWindowsHazardName names),+      -- The case-hack strip: a tree materialized with upstream's+      -- collision suffix serialises back under its NAR names.+      test "serialiseFromPathWith strips the case-hack suffix" $ do+        dir <- caseHackFixture "nova-cache-test-casehack"+        BS.writeFile (dir <> "/Foo") "upper"+        BS.writeFile (dir <> "/foo~nix~case~hack~1") "lower"+        entry <- NAR.serialiseFromPathWith NAR.CaseHackEnabled dir+        removeDirectoryRecursive dir+        let expected =+              NAR.NarDirectory+                [ ("Foo", NAR.NarRegular False "upper"),+                  ("foo", NAR.NarRegular False "lower")+                ]+        roundTrip <- assertRight "stripped tree reparses" expected (NAR.deserialise (NAR.serialise entry))+        pure (entry == expected && roundTrip),+      test "serialiseFromPathWith keeps the suffix verbatim when disabled" $ do+        dir <- caseHackFixture "nova-cache-test-casehack-off"+        BS.writeFile (dir <> "/foo~nix~case~hack~1") "kept"+        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+        removeDirectoryRecursive dir+        assertEqual+          "verbatim name"+          (NAR.NarDirectory [("foo~nix~case~hack~1", NAR.NarRegular False "kept")])+          entry,+      test "serialiseFromPathWith fails loudly on an unhack collision" $ do+        dir <- caseHackFixture "nova-cache-test-casehack-clash"+        BS.writeFile (dir <> "/foo") "plain"+        BS.writeFile (dir <> "/foo~nix~case~hack~1") "hacked"+        outcome <- try (NAR.serialiseFromPathWith NAR.CaseHackEnabled dir)+        removeDirectoryRecursive dir+        pure $ case (outcome :: Either SomeException NAR.NarEntry) of+          Left _ -> True+          Right _ -> False,+      -- The resolver is the executable-bit source: its answer lands+      -- verbatim, independent of permissions or extension.+      test "serialiseFromPathOpts takes the exec bit from the resolver" $ do+        dir <- caseHackFixture "nova-cache-test-execresolver"+        BS.writeFile (dir <> "/plain") "p"+        let opts =+              NAR.defaultSerialiseOptions+                { NAR.soCaseHack = NAR.CaseHackDisabled,+                  NAR.soExecBit = \_ -> pure True+                }+        entry <- NAR.serialiseFromPathOpts opts dir+        removeDirectoryRecursive dir+        assertEqual+          "resolver answer"+          (NAR.NarDirectory [("plain", NAR.NarRegular True "p")])+          entry,+      -- A real on-disk exec bit round-trips through the default+      -- resolver; no earlier fixture ever set one.  POSIX-gated:+      -- Windows has no mode bit for the default to read.+      test "a real owner-exec bit round-trips from disk" $+        if System.Info.os == "mingw32"+          then pure True+          else do+            dir <- caseHackFixture "nova-cache-test-realexec"+            BS.writeFile (dir <> "/doc") "txt"+            BS.writeFile (dir <> "/tool") "bin"+            perms <- getPermissions (dir <> "/tool")+            setPermissions (dir <> "/tool") (setOwnerExecutable True perms)+            entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+            removeDirectoryRecursive dir+            assertEqual+              "owner bit from the file mode"+              ( NAR.NarDirectory+                  [ ("doc", NAR.NarRegular False "txt"),+                    ("tool", NAR.NarRegular True "bin")+                  ]+              )+              entry,+      -- The resolver contract, pinned as an exact call set for both+      -- producers: once per regular file, never for the directory,+      -- and always the on-disk spelling - under the case hack, the+      -- suffixed name the walk reads, not the stripped entry name.+      test "the resolver sees the on-disk case-hacked spelling" $ do+        dir <- caseHackFixture "nova-cache-test-execdisk"+        BS.writeFile (dir <> "/Foo") "upper"+        BS.writeFile (dir <> "/foo~nix~case~hack~1") "lower"+        seenWalk <- newIORef []+        seenPlan <- newIORef []+        let recordingOpts ref =+              NAR.defaultSerialiseOptions+                { NAR.soCaseHack = NAR.CaseHackEnabled,+                  NAR.soExecBit = \path ->+                    atomicModifyIORef' ref (\paths -> (path : paths, ())) >> pure False+                }+            baseName = reverse . takeWhile (\c -> c /= '/' && c /= '\\') . reverse+        _ <- NAR.serialiseFromPathOpts (recordingOpts seenWalk) dir+        _ <- NAR.withNarSourceOpts (recordingOpts seenPlan) dir drainSource+        removeDirectoryRecursive dir+        walked <- readIORef seenWalk+        planned <- readIORef seenPlan+        let expected = ["Foo", "foo~nix~case~hack~1"]+        walkOk <-+          assertEqual+            "eager walk call set"+            expected+            (sort (map baseName walked))+        planOk <-+          assertEqual+            "streaming plan call set"+            expected+            (sort (map baseName planned))+        pure (walkOk && planOk),+      -- A symlink target with a separator serialises to the POSIX+      -- spelling on every platform: Windows stores a+      -- reparse point with backslashes, so a target written bin/tool+      -- reads back bin\tool, and an unpack-then-recheck round-trip+      -- diverges from the archive unless the NAR boundary normalises+      -- it.  Both producers must agree with the forward-slash form.+      -- Guarded on symlink privilege, which a Windows runner without+      -- developer mode lacks.+      test "a separator-bearing symlink target serialises with forward slashes" $ do+        dir <- caseHackFixture "nova-cache-test-symlinksep"+        created <- try (createFileLink "bin/tool" (dir <> "/link")) :: IO (Either SomeException ())+        case created of+          Left _ -> removeDirectoryRecursive dir >> pure True+          Right () -> do+            let expectedEntry = NAR.NarDirectory [("link", NAR.NarSymlink "bin/tool")]+            eager <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+            streamed <- NAR.withNarSource NAR.CaseHackDisabled dir drainSource+            removeDirectoryRecursive dir+            eagerOk <- assertEqual "eager target spelling" expectedEntry eager+            streamOk <- assertEqual "streamed target spelling" (NAR.serialise expectedEntry) streamed+            pure (eagerOk && streamOk),+      -- The type the walk reads decides the archive's bytes: a symlink+      -- to a directory archives as the symlink, not the directory it+      -- leads to, a symlink walked as the root likewise, and an empty+      -- directory as a directory.  Guarded on symlink privilege, as+      -- above.+      test "a symlink to a directory, a symlink root and an empty directory archive as their own types" $+        withWalkTree $ \root -> do+          createDirectory (root ++ "/empty")+          createDirectory (root ++ "/real")+          BS.writeFile (root ++ "/real/file") "x"+          linked <- try (createDirectoryLink "real" (root ++ "/link")) :: IO (Either IOException ())+          case linked of+            Left _ -> pure True+            Right () -> do+              treeOk <-+                expectArchive+                  ( NAR.NarDirectory+                      [ ("empty", NAR.NarDirectory []),+                        ("link", NAR.NarSymlink "real"),+                        ("real", NAR.NarDirectory [("file", NAR.NarRegular False "x")])+                      ]+                  )+                  root+              rootOk <- expectArchive (NAR.NarSymlink "real") (root ++ "/link")+              pure (treeOk && rootOk),+      -- The walk's boundary encoding: a Unicode disk name enters the+      -- archive as its UTF-8 bytes on every platform.+      test "serialiseFromPath encodes a Unicode disk name as UTF-8" $ do+        dir <- caseHackFixture "nova-cache-test-uniname"+        BS.writeFile (dir <> "/caf\233") "au lait"+        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+        removeDirectoryRecursive dir+        assertEqual+          "utf8 name"+          (NAR.NarDirectory [(BS.pack [0x63, 0x61, 0x66, 0xC3, 0xA9], NAR.NarRegular False "au lait")])+          entry,+      -- POSIX names are bytes; one that is not valid UTF-8 must archive+      -- verbatim, never rewritten with replacement characters.  Linux-gated:+      -- NTFS and APFS names are Unicode, so the fixture cannot exist+      -- there.  "\56575" is the lone surrogate+      -- GHC's filesystem encoding round-trips to byte 0xFF.+      test "serialiseFromPath carries a non-UTF-8 disk name verbatim" $+        if System.Info.os /= "linux"+          then pure True+          else do+            dir <- caseHackFixture "nova-cache-test-rawname"+            BS.writeFile (dir <> "/f\56575") "raw"+            entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+            removeDirectoryRecursive dir+            assertEqual+              "raw byte name"+              (NAR.NarDirectory [(BS.pack [0x66, 0xFF], NAR.NarRegular False "raw")])+              entry+    ]++-- | A fresh, empty fixture directory under the system temp dir.+caseHackFixture :: String -> IO FilePath+caseHackFixture name = do+  tmpBase <- getTemporaryDirectory+  let dir = tmpBase <> "/" <> name+  _ <- try (removeDirectoryRecursive dir) :: IO (Either SomeException ())+  createDirectory dir+  pure dir++-- | Encode one NAR wire string with a chosen padding byte.  The spec+-- demands zero padding, so a nonzero byte builds archives the parser+-- must reject - and 'NAR.serialise' (rightly) cannot produce them.+narWireStr :: Word8 -> ByteString -> ByteString+narWireStr padByte str = lenLE <> str <> BS.replicate padLen padByte+  where+    n = BS.length str+    lenLE = BS.pack [fromIntegral ((n `shiftR` (8 * i)) .&. 0xff) | i <- [0 .. 7]]+    padLen = (8 - n `mod` 8) `mod` 8++-- | A directory NAR whose entries arrive out of sorted order - again not+-- producible via 'NAR.serialise', which sorts on write.+outOfOrderDirNar :: ByteString+outOfOrderDirNar =+  BS.concat+    ( map+        (narWireStr 0)+        ( ["nix-archive-1", "(", "type", "directory"]+            ++ entryFor "b"+            ++ entryFor "a"+            ++ [")"]+        )+    )+  where+    entryFor name = ["entry", "(", "name", name, "node", "(", "type", "regular", "contents", "", ")", ")"]++-- | A regular-file NAR whose contents padding is nonzero.+badPaddingNar :: ByteString+badPaddingNar =+  BS.concat+    [ BS.concat (map (narWireStr 0) ["nix-archive-1", "(", "type", "regular", "contents"]),+      narWireStr 1 "abc",+      narWireStr 0 ")"+    ]++-- ---------------------------------------------------------------------------+-- NAR.Stream tests+-- ---------------------------------------------------------------------------++-- | Drive the streaming parser over the given chunks (all non-empty),+-- then signal end of input, collecting events.+runStream :: [ByteString] -> Either String [Stream.NarEvent]+runStream = runStreamFrom Stream.narStream++-- | 'runStream' from an explicitly constructed machine, for tests that+-- need a bound other than the default.+runStreamFrom :: Stream.NarStep -> [ByteString] -> Either String [Stream.NarEvent]+runStreamFrom start chunks = go start chunks []+  where+    go step pending acc = case step of+      Stream.NarFail err -> Left err+      Stream.NarDone -> Right (reverse acc)+      Stream.NarYield event continue -> go continue pending (event : acc)+      Stream.NarAwait continue -> case pending of+        [] -> go (continue BS.empty) [] acc+        (chunk : rest) -> go (continue chunk) rest acc++-- | Entry shapes the chunking-independence test sweeps: every node+-- kind, empty and aligned contents, nesting, and a non-UTF-8 name.+streamCorpus :: [NAR.NarEntry]+streamCorpus =+  [ NAR.NarRegular False "hello",+    NAR.NarRegular True BS.empty,+    NAR.NarRegular False (BS.replicate 8 0x41),+    NAR.NarSymlink "/usr/bin/hello",+    NAR.NarDirectory [],+    NAR.NarDirectory+      [ ("bin", NAR.NarDirectory [("hello", NAR.NarRegular True (BS.pack [42]))]),+        ("lib", NAR.NarSymlink "../lib64"),+        (BS.pack [0x66, 0xFF], NAR.NarRegular False "raw")+      ]+  ]++-- | Merge consecutive contents slices.  Slice granularity deliberately+-- follows the fed chunks, so parses of different chunkings compare by+-- structure and content, not by how the input happened to arrive.+coalesceChunks :: [Stream.NarEvent] -> [Stream.NarEvent]+coalesceChunks (Stream.EventRegularChunk a : Stream.EventRegularChunk b : rest) =+  coalesceChunks (Stream.EventRegularChunk (a <> b) : rest)+coalesceChunks (event : rest) = event : coalesceChunks rest+coalesceChunks [] = []++-- | Split a byte string into fixed-size pieces.+chunksOf :: Int -> ByteString -> [ByteString]+chunksOf n bs+  | BS.null bs = []+  | otherwise = case BS.splitAt n bs of+      (piece, rest) -> piece : chunksOf n rest++-- | Pull a source dry, collecting its chunks (the terminating empty+-- chunk excluded).+collectChunks :: IO ByteString -> IO [ByteString]+collectChunks pull = go []+  where+    go acc = do+      chunk <- pull+      if BS.null chunk then pure (reverse acc) else go (chunk : acc)++-- | Pull a source dry into one byte string.+drainSource :: IO ByteString -> IO ByteString+drainSource pull = BS.concat <$> collectChunks pull++-- | Hash a source chunk by chunk as it is pulled.+hashPull :: Hash.HashContext -> IO ByteString -> IO Hash.NixHash+hashPull ctx pull = do+  chunk <- pull+  if BS.null chunk+    then pure (Hash.hashFinalize ctx)+    else hashPull (Hash.hashUpdate ctx chunk) pull++testStream :: IO Bool+testStream =+  runGroup+    "NAR.Stream"+    [ test "events for a regular file" $+        assertEqual+          "event sequence"+          ( Right+              [ Stream.EventRegularBegin True 2,+                Stream.EventRegularChunk "hi",+                Stream.EventRegularEnd+              ]+          )+          (runStream [NAR.serialise (NAR.NarRegular True "hi")]),+      test "events for a directory arrive entry by entry" $+        let entry = NAR.NarDirectory [("a", NAR.NarSymlink "t"), ("b", NAR.NarRegular False "")]+            expected =+              [ Stream.EventDirectoryBegin,+                Stream.EventEntryBegin "a",+                Stream.EventSymlink "t",+                Stream.EventEntryEnd,+                Stream.EventEntryBegin "b",+                Stream.EventRegularBegin False 0,+                Stream.EventRegularEnd,+                Stream.EventEntryEnd,+                Stream.EventDirectoryEnd+              ]+         in assertEqual "event sequence" (Right expected) (runStream [NAR.serialise entry]),+      test "byte-at-a-time chunking equals whole-input parsing" $+        let normalize = fmap coalesceChunks+            agrees entry =+              let bytes = NAR.serialise entry+               in normalize (runStream [bytes]) == normalize (runStream (map BS.singleton (BS.unpack bytes)))+         in assertTrue "chunking-independent" (all agrees streamCorpus),+      test "contents slices reassemble across misaligned chunks" $+        -- 7-byte chunks sit deliberately askew of the 8-byte wire+        -- alignment, so every length prefix and padding run straddles+        -- a chunk boundary somewhere.+        let payload = BS.pack (concat (replicate 40 [0 .. 7]))+            bytes = NAR.serialise (NAR.NarRegular False payload)+         in case runStream (chunksOf 7 bytes) of+              Left err -> do+                putStrLn ("  streaming parse failed: " ++ err)+                pure False+              Right events ->+                assertEqual+                  "reassembled contents"+                  payload+                  (BS.concat [c | Stream.EventRegularChunk c <- events]),+      test "streaming agrees with deserialise on the malformed corpus" $+        let evil name = NAR.serialise (NAR.NarDirectory [(name, NAR.NarRegular False "x")])+            malformed =+              [ outOfOrderDirNar,+                badPaddingNar,+                NAR.serialise (NAR.NarRegular False "x") <> "junk1234",+                evil "..",+                evil "a/b",+                evil ""+              ]+            bothReject bytes = isLeft (runStream [bytes]) && isLeft (NAR.deserialise bytes)+         in assertTrue "all rejected by both parsers" (all bothReject malformed),+      test "the structural bound applies to streaming, not deserialise" $+        -- deserialise instantiates the machine with its input's length+        -- as the bound, so a name this size stays accepted there; the+        -- streaming default caps structural strings at 64 KiB.+        let longName = BS.replicate 70000 0x61+            bytes = NAR.serialise (NAR.NarDirectory [(longName, NAR.NarRegular False "x")])+         in do+              ok1 <- assertTrue "deserialise accepts" (either (const False) (const True) (NAR.deserialise bytes))+              ok2 <- assertTrue "streaming rejects" (isLeft (runStream [bytes]))+              pure (ok1 && ok2),+      test "a truncated archive fails" $+        let bytes = NAR.serialise (NAR.NarRegular False "some contents here")+         in assertLeft "eof" (runStream [BS.take (BS.length bytes - 9) bytes]),+      -- Regression: under a huge caller bound, a declared length near+      -- maxBound Int once wrapped the padded Int demand negative, so+      -- the machine "read" the string instantly as empty at an unmoved+      -- position and this archive parsed as a complete empty symlink.+      -- The parse must fail, as upstream would at end of input.+      test "a near-maxBound declared length fails under a huge bound" $+        let declared = fromIntegral (maxBound :: Int) :: Word64+            lenPrefix = BS.pack [fromIntegral ((declared `shiftR` (8 * i)) .&. 0xff) | i <- [0 .. 7]]+            bytes =+              BS.concat (map (narWireStr 0) ["nix-archive-1", "(", "type", "symlink", "target"])+                <> lenPrefix+                <> narWireStr 0 ")"+         in assertLeft+              "overflowing padded demand"+              (runStreamFrom (Stream.narStreamBounded (maxBound :: Word64)) [bytes]),+      test "the declared contents size arrives before the bytes" $+        case runStream [NAR.serialise (NAR.NarRegular False (BS.replicate 24 0x2A))] of+          Right (Stream.EventRegularBegin False declared : _) ->+            assertEqual "declared size" 24 declared+          other -> do+            putStrLn ("    expected EventRegularBegin first, got: " ++ show other)+            pure False,+      test "withNarSource streams byte-identically to serialise" $ do+        dir <- caseHackFixture "nova-cache-test-narsource"+        createDirectory (dir <> "/sub")+        -- Larger than two pull chunks, so mid-file continuation runs.+        BS.writeFile (dir <> "/big.bin") (BS.replicate 300000 0x41)+        BS.writeFile (dir <> "/sub/small") "tiny"+        BS.writeFile (dir <> "/zero") ""+        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+        streamed <- NAR.withNarSource NAR.CaseHackDisabled dir drainSource+        removeDirectoryRecursive dir+        assertTrue "bytes equal" (NAR.serialise entry == streamed),+      test "withNarSource under the case-hack matches the strict walk" $ do+        dir <- caseHackFixture "nova-cache-test-narsource-hack"+        BS.writeFile (dir <> "/Foo") "upper"+        BS.writeFile (dir <> "/foo~nix~case~hack~1") "lower"+        entry <- NAR.serialiseFromPathWith NAR.CaseHackEnabled dir+        streamed <- NAR.withNarSource NAR.CaseHackEnabled dir drainSource+        removeDirectoryRecursive dir+        assertTrue "bytes equal" (NAR.serialise entry == streamed),+      -- The streaming planner asks the same resolver the eager walk+      -- does, so a custom executable-bit source streams byte-equal.+      test "withNarSourceOpts streams the resolver's flags byte-identically" $ do+        dir <- caseHackFixture "nova-cache-test-narsource-exec"+        BS.writeFile (dir <> "/doc") "txt"+        BS.writeFile (dir <> "/tool") "bin"+        let opts =+              NAR.defaultSerialiseOptions+                { NAR.soCaseHack = NAR.CaseHackDisabled,+                  NAR.soExecBit = \path -> pure ("tool" `isSuffixOf` path)+                }+        entry <- NAR.serialiseFromPathOpts opts dir+        streamed <- NAR.withNarSourceOpts opts dir drainSource+        removeDirectoryRecursive dir+        flagsOk <-+          assertEqual+            "eager flags"+            ( NAR.NarDirectory+                [ ("doc", NAR.NarRegular False "txt"),+                  ("tool", NAR.NarRegular True "bin")+                ]+            )+            entry+        bytesOk <- assertTrue "stream equals serialise" (NAR.serialise entry == streamed)+        pure (flagsOk && bytesOk),+      test "withNarSource keeps returning empty after the end" $ do+        dir <- caseHackFixture "nova-cache-test-narsource-end"+        BS.writeFile (dir <> "/f") "x"+        ends <- NAR.withNarSource NAR.CaseHackDisabled dir $ \pull -> do+          _ <- collectChunks pull+          endA <- pull+          endB <- pull+          pure (endA, endB)+        removeDirectoryRecursive dir+        assertEqual "stable end" ("", "") ends,+      -- The pull that opens a file reads its size, so a file cut to+      -- nothing after that pull leaves the next read short, and the+      -- empty chunk after that failure would read as the clean end of+      -- a torn archive.  Writing the file afterwards shows the failure+      -- closed it: GHC refuses a writer while the process holds a+      -- reader open.+      test "every pull after a shrink failure rethrows it" $+        withWalkTree $ \root -> do+          let file = root ++ "/file"+              shrank = Left (userError ("withNarSource: " ++ file ++ " shrank while streaming"))+          BS.writeFile file "contents"+          NAR.withNarSourceOpts walkOptions file $ \pull -> do+            _ <- pull+            _ <- pull+            withTruncated file $ do+              failed <- try pull+              later <- replicateM 2 (try pull)+              closed <- isRight <$> (try (BS.writeFile file "") :: IO (Either IOException ()))+              assertEqual+                "the pull after the cut, the pulls after it, the file closed"+                (shrank, [shrank, shrank], True)+                (failed, later, closed),+      -- The latch holds any failure, not only a shrink: a file that+      -- could not be opened stays failed once it is readable again,+      -- rather than resuming an archive its consumer saw fail.+      test "every pull after a failed open rethrows it" $+        withWalkTree $ \root -> do+          let file = root ++ "/file"+          BS.writeFile file "contents"+          NAR.withNarSourceOpts walkOptions file $ \pull -> do+            _ <- pull+            withoutPermission readable setOwnerReadable file $ do+              refused <- try pull+              getPermissions file >>= setPermissions file . setOwnerReadable True+              later <- try pull+              assertEqual+                "a failed open, then the pull once the file is readable again"+                (True, refused)+                (isLeft refused, later :: Either NAR.NarWalkError ByteString),+      test "pulled chunks parse back to the walked tree" $ do+        dir <- caseHackFixture "nova-cache-test-narsource-loop"+        BS.writeFile (dir <> "/data.bin") (BS.replicate 200000 0x5A)+        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+        chunks <- NAR.withNarSource NAR.CaseHackDisabled dir collectChunks+        removeDirectoryRecursive dir+        ok1 <- assertRight "strict parse of the stream" entry (NAR.deserialise (BS.concat chunks))+        ok2 <- assertTrue "streaming parse of the chunk list" (either (const False) (const True) (runStream chunks))+        pure (ok1 && ok2),+      test "incremental hashing equals whole-input hashing" $+        let payload = BS.pack [0 .. 255]+            pieces = [BS.take 100 payload, BS.take 55 (BS.drop 100 payload), BS.drop 155 payload]+            incremental = Hash.hashFinalize (foldl' Hash.hashUpdate Hash.hashInit pieces)+         in do+              ok1 <- assertEqual "split arbitrarily" (Hash.hashBytes payload) incremental+              ok2 <- assertEqual "empty input" (Hash.hashBytes BS.empty) (Hash.hashFinalize Hash.hashInit)+              pure (ok1 && ok2),+      test "hashing a pulled source equals narHash of the walk" $ do+        dir <- caseHackFixture "nova-cache-test-streamhash"+        BS.writeFile (dir <> "/blob") (BS.replicate 150000 0x07)+        entry <- NAR.serialiseFromPathWith NAR.CaseHackDisabled dir+        digest <- NAR.withNarSource NAR.CaseHackDisabled dir (hashPull Hash.hashInit)+        removeDirectoryRecursive dir+        assertEqual "hash matches" (NAR.narHash entry) digest+    ]++-- ---------------------------------------------------------------------------+-- NAR walk failure tests+-- ---------------------------------------------------------------------------++-- | Each failed access, and each type a NAR cannot hold, is provoked on+-- an entry and walked through both entry points.+testWalkFailures :: IO Bool+testWalkFailures =+  runGroup+    "NAR walk failures"+    [ test "an unreadable file fails the walk as a file read" $+        withWalkTree $ \root -> do+          let locked = root ++ "/sub/locked"+          createDirectory (root ++ "/sub")+          BS.writeFile locked "secret"+          withoutPermission readable setOwnerReadable locked $+            expectWalkFailure NAR.WalkReadFile locked root,+      test "an unlistable directory fails the walk as a directory listing" $+        withWalkTree $ \root -> do+          let sub = root ++ "/sub"+          createDirectory sub+          BS.writeFile (sub ++ "/file") "x"+          withoutPermission readable setOwnerReadable sub $+            expectWalkFailure NAR.WalkListDirectory sub root,+      -- A directory with read but not search permission lists its+      -- entries' names and refuses an lstat of any of them.+      test "an entry of an unsearchable directory fails the walk as a status read" $+        withWalkTree $ \root -> do+          let sub = root ++ "/sub"+          createDirectory sub+          BS.writeFile (sub ++ "/file") "x"+          withoutPermission searchable setOwnerSearchable sub $+            expectWalkFailure NAR.WalkStatus (sub ++ "/file") root,+      test "a symlink whose target cannot be read fails the walk as a readlink" $+        withWalkTree $ \root -> do+          let link = root ++ "/sub/link"+          createDirectory (root ++ "/sub")+          withUnreadableLink link $+            expectWalkFailure NAR.WalkReadLink link root,+      test "a FIFO in a tree is refused as an unsupported type" $+        withWalkTree $ \root -> do+          let pipe = root ++ "/sub/pipe"+          createDirectory (root ++ "/sub")+          withFifo pipe $+            expectUnsupportedType pipe root,+      test "a Unix socket in a tree is refused as an unsupported type" $+        withWalkTree $ \root -> do+          let socketPath = root ++ "/sub/socket"+          createDirectory (root ++ "/sub")+          withUnixSocket socketPath $+            expectUnsupportedType socketPath root,+      -- Making a device takes privilege, and /dev/null is one every+      -- POSIX system has; as the root it goes through the same+      -- classification as an entry below.  Windows has no /dev/null.+      test "a device walked as the root is refused as an unsupported type" $+        if System.Info.os == "mingw32"+          then pure True+          else expectUnsupportedType "/dev/null" "/dev/null",+      -- The resolver is the one place a test can raise an interrupt+      -- mid-walk: no access blocks long enough to be interrupted on+      -- cue.  The handler around each access catches 'IOException' by+      -- its type alone.+      test "an interrupt raised by the resolver mid-walk reaches the caller as itself" $+        withWalkTree $ \root -> do+          BS.writeFile (root ++ "/file") "x"+          let interrupt = myThreadId >>= (`throwTo` UserInterrupt) >> pure False+          outcomes <- walkOutcomes walkOptions {NAR.soExecBit = const interrupt} root+          and+            <$> traverse+              (\(entryPoint, outcome) -> assertEqual (entryPoint ++ " raised") (Just UserInterrupt) (either fromException (const Nothing) outcome))+              outcomes,+      test "a walk failure renders as its IOException, the access as location" $+        let cause = mkIOError permissionErrorType "openBinaryFile" Nothing (Just "/d/tree/sub/locked")+         in assertEqual+              "rendering"+              "/d/tree/sub/locked: reading file: permission denied"+              (displayException (NAR.NarWalkError NAR.WalkReadFile "/d/tree/sub/locked" cause))+    ]++-- | The options every walk-failure test walks with.+walkOptions :: NAR.SerialiseOptions+walkOptions = NAR.defaultSerialiseOptions {NAR.soCaseHack = NAR.CaseHackDisabled}++-- | A fresh tree for one test, removed however the test ends.+withWalkTree :: (FilePath -> IO Bool) -> IO Bool+withWalkTree body = do+  root <- createTestDir+  body root `finally` removeDirectoryRecursive root++-- | What each entry point raised walking a tree, by name.+walkOutcomes :: NAR.SerialiseOptions -> FilePath -> IO [(String, Either SomeException ())]+walkOutcomes opts root = do+  eager <- try (void (NAR.serialiseFromPathOpts opts root))+  streamed <- try (void (NAR.withNarSourceOpts opts root drainSource))+  pure [("serialiseFromPathOpts", eager), ("withNarSourceOpts", streamed)]++-- | Assert that both entry points fail to walk a tree with a+-- 'NAR.NarWalkError' naming the access and the entry, over the+-- permission error the test caused.+expectWalkFailure :: NAR.WalkAccess -> FilePath -> FilePath -> IO Bool+expectWalkFailure access path =+  expectWalkRefusal+    "NarWalkError"+    (\failure -> (NAR.nweAccess failure, NAR.nwePath failure, isPermissionError (NAR.nweCause failure)))+    (access, path, True)++-- | Assert that both entry points refuse a tree with upstream's+-- unsupported-type error, naming the entry.+expectUnsupportedType :: FilePath -> FilePath -> IO Bool+expectUnsupportedType path =+  expectWalkRefusal+    "IOException"+    (\refusal -> (isUserError refusal, ioeGetErrorString refusal))+    (True, "file '" ++ path ++ "' has an unsupported type")++-- | Assert that both entry points archive a path as an entry: the walk+-- builds the entry itself, and the stream pulls its serialised bytes.+expectArchive :: NAR.NarEntry -> FilePath -> IO Bool+expectArchive expected path = do+  walked <- NAR.serialiseFromPathOpts walkOptions path+  streamed <- NAR.withNarSourceOpts walkOptions path drainSource+  walkedOk <- assertEqual (path ++ " walked") expected walked+  streamedOk <- assertEqual (path ++ " streamed") (NAR.serialise expected) streamed+  pure (walkedOk && streamedOk)++-- | Assert that both entry points fail to walk a tree with an exception+-- of the type named, and that what the test reads off it is expected.+expectWalkRefusal :: (Exception e, Eq a, Show a) => String -> (e -> a) -> a -> FilePath -> IO Bool+expectWalkRefusal typeName observe expected root = do+  outcomes <- walkOutcomes walkOptions root+  and <$> traverse check outcomes+  where+    check (entryPoint, outcome) = case outcome of+      Left raised+        | Just refusal <- fromException raised ->+            assertEqual (entryPoint ++ " refusal") expected (observe refusal)+      Left raised -> do+        putStrLn ""+        putStrLn ("    " ++ entryPoint ++ ": expected " ++ typeName ++ ", got: " ++ displayException raised)+        pure False+      Right () -> do+        putStrLn ""+        putStrLn ("    " ++ entryPoint ++ ": expected " ++ typeName ++ ", the walk succeeded")+        pure False++-- | Take one of the owner's permissions from a path and run a check+-- while it is gone, then give it back.  Windows has no mode bit to+-- take, and root's access ignores the mode, so on either there is no+-- refusal to cause and the check is skipped, as 'makeUnreadable''s+-- callers skip theirs.+withoutPermission :: (Permissions -> Bool) -> (Bool -> Permissions -> Permissions) -> FilePath -> IO Bool -> IO Bool+withoutPermission held setHeld path check+  | System.Info.os == "mingw32" = pure True+  | otherwise = do+      original <- getPermissions path+      bracket_ (setPermissions path (setHeld False original)) (setPermissions path original) $ do+        refused <- not . held <$> getPermissions path+        if refused then check else pure True++-- | Create a symlink with no permissions of its own and run a check if+-- that leaves its target unreadable.  macOS checks a symlink's own+-- mode on readlink (root exempt); Linux ignores the mode and Windows+-- has none, so on both no permission refuses a readlink that the lstat+-- before it was allowed, and the check is skipped.+withUnreadableLink :: FilePath -> IO Bool -> IO Bool+#if defined(darwin_HOST_OS)+withUnreadableLink link check = do+  createFileLink "target" link+  withFilePath link (throwErrnoPathIfMinus1_ "lchmod" link . (`c_lchmod` 0))+  refused <- isLeft <$> (try (getSymbolicLinkTarget link) :: IO (Either IOException FilePath))+  if refused then check else pure True++foreign import ccall unsafe "lchmod"+  c_lchmod :: CString -> CMode -> IO CInt+#else+withUnreadableLink _ _ = pure True+#endif++-- | Make a FIFO at a path and run a check.  Windows has no FIFO a+-- directory holds and no file mode the walk reads, so the check is+-- skipped there.+withFifo :: FilePath -> IO Bool -> IO Bool+#if defined(mingw32_HOST_OS)+withFifo _ _ = pure True+#else+withFifo path check = do+  Posix.createNamedPipe path (Posix.unionFileModes Posix.ownerReadMode Posix.ownerWriteMode)+  check+#endif++-- | Bind a Unix socket at a path and run a check on the file the bind+-- leaves, which outlives the socket.  The walk on Windows reads no+-- file mode, so the check is skipped there.+withUnixSocket :: FilePath -> IO Bool -> IO Bool+#if defined(mingw32_HOST_OS)+withUnixSocket _ _ = pure True+#else+withUnixSocket path check = do+  bound <- Socket.socket Socket.AF_UNIX Socket.Stream Socket.defaultProtocol+  Socket.bind bound (Socket.SockAddrUnix path) `finally` Socket.close bound+  check+#endif++-- | Cut a file to nothing and run a check.  A stream holding the file+-- open for reading keeps GHC from giving the same process a handle+-- that writes it, so the cut is truncate(2), which Windows lacks; the+-- check is skipped there.+withTruncated :: FilePath -> IO Bool -> IO Bool+#if defined(mingw32_HOST_OS)+withTruncated _ _ = pure True+#else+withTruncated path check = do+  Posix.setFileSize path 0+  check+#endif++-- ---------------------------------------------------------------------------+-- NarInfo tests+-- ---------------------------------------------------------------------------++sampleNarInfoText :: Text+sampleNarInfoText =+  T.unlines+    [ "StorePath: /nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0",+      "URL: nar/1234abcd.nar.xz",+      "Compression: xz",+      "FileHash: sha256:abcdef1234567890",+      "FileSize: 12345",+      "NarHash: sha256:fedcba0987654321",+      "NarSize: 67890",+      "References: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-glibc-2.38",+      "Deriver: cccccccccccccccccccccccccccccccc-hello-1.0.drv",+      "Sig: cache.example.com:c2lnbmF0dXJl",+      "Sig: backup.example.com:YW5vdGhlcnNpZw=="+    ]++testNarInfo :: IO Bool+testNarInfo =+  runGroup+    "NarInfo"+    [ test "parse sample narinfo" $+        case NarInfo.parseNarInfo sampleNarInfoText of+          Left err -> do+            putStrLn ("  parse failed: " ++ err)+            pure False+          Right ni -> do+            ok1 <- assertEqual "storePath" "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0" (NarInfo.niStorePath ni)+            ok2 <- assertEqual "url" "nar/1234abcd.nar.xz" (NarInfo.niUrl ni)+            ok3 <- assertEqual "compression" "xz" (NarInfo.niCompression ni)+            ok4 <- assertEqual "fileSize" (Just 12345) (NarInfo.niFileSize ni)+            ok5 <- assertEqual "narSize" 67890 (NarInfo.niNarSize ni)+            ok6 <- assertEqual "refs count" 2 (length (NarInfo.niReferences ni))+            ok7 <- assertEqual "deriver" (Just "cccccccccccccccccccccccccccccccc-hello-1.0.drv") (NarInfo.niDeriver ni)+            ok8 <- assertEqual "sigs count" 2 (length (NarInfo.niSigs ni))+            pure (ok1 && ok2 && ok3 && ok4 && ok5 && ok6 && ok7 && ok8),+      test "parse/render roundtrip" $+        case NarInfo.parseNarInfo sampleNarInfoText of+          Left err -> do+            putStrLn ("  parse failed: " ++ err)+            pure False+          Right ni ->+            let rendered = NarInfo.renderNarInfo ni+             in case NarInfo.parseNarInfo rendered of+                  Left err -> do+                    putStrLn ("  re-parse failed: " ++ err)+                    pure False+                  Right reparsed ->+                    assertEqual "roundtrip" ni reparsed,+      test "parse minimal narinfo (no optional fields)" $+        let minimal =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/test.nar.xz",+                  "Compression: xz",+                  "FileHash: sha256:abc",+                  "FileSize: 100",+                  "NarHash: sha256:def",+                  "NarSize: 200",+                  "References: "+                ]+         in case NarInfo.parseNarInfo minimal of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right ni -> do+                ok1 <- assertEqual "deriver" Nothing (NarInfo.niDeriver ni)+                ok2 <- assertEqual "sigs" [] (NarInfo.niSigs ni)+                ok3 <- assertEqual "ca" Nothing (NarInfo.niCA ni)+                ok4 <- assertEqual "refs" [] (NarInfo.niReferences ni)+                pure (ok1 && ok2 && ok3 && ok4),+      test "CRLF-terminated narinfo parses identically" $+        assertEqual+          "crlf tolerated"+          (NarInfo.parseNarInfo sampleNarInfoText)+          (NarInfo.parseNarInfo (T.replace "\n" "\r\n" sampleNarInfoText)),+      test "upstream-optional fields default as upstream" $+        -- Only StorePath, URL, NarHash, NarSize are mandatory upstream;+        -- Compression defaults to bzip2 and FileHash/FileSize stay absent.+        let bare =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/test.nar.xz",+                  "NarHash: sha256:def",+                  "NarSize: 200"+                ]+         in case NarInfo.parseNarInfo bare of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right ni -> do+                ok1 <- assertEqual "compression default" "bzip2" (NarInfo.niCompression ni)+                ok2 <- assertEqual "fileHash absent" Nothing (NarInfo.niFileHash ni)+                ok3 <- assertEqual "fileSize absent" Nothing (NarInfo.niFileSize ni)+                pure (ok1 && ok2 && ok3),+      test "CA field parse/render roundtrip" $+        let withCA = sampleNarInfoText <> "CA: fixed:r:sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73\n"+         in case NarInfo.parseNarInfo withCA of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right ni -> do+                ok1 <-+                  assertEqual+                    "ca parsed"+                    (Just "fixed:r:sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73")+                    (NarInfo.niCA ni)+                ok2 <- assertRight "ca survives render" ni (NarInfo.parseNarInfo (NarInfo.renderNarInfo ni))+                pure (ok1 && ok2),+      test "parse missing required key fails" $+        let incomplete = T.unlines ["StorePath: /nix/store/aaaa-test", "URL: nar/test.nar.xz"]+         in assertLeft "missing key" (NarInfo.parseNarInfo incomplete),+      test "parse bad integer fails" $+        let bad =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/test.nar.xz",+                  "Compression: xz",+                  "FileHash: sha256:abc",+                  "FileSize: not-a-number",+                  "NarHash: sha256:def",+                  "NarSize: 200",+                  "References: "+                ]+         in assertLeft "bad integer" (NarInfo.parseNarInfo bad),+      -- Sizes are uint64 on the wire: at most 20 digits.  A longer field+      -- rejects before the quadratic bignum accumulation.+      test "parse rejects an overlong size field" $+        let long =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/test.nar.xz",+                  "NarHash: sha256:def",+                  "NarSize: 123456789012345678901"+                ]+         in assertLeft "overlong size" (NarInfo.parseNarInfo long),+      test "a 20-digit size field still parses" $+        let capped =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/test.nar.xz",+                  "NarHash: sha256:def",+                  "NarSize: 18446744073709551615"+                ]+         in case NarInfo.parseNarInfo capped of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right ni -> assertEqual "uint64 max" 18446744073709551615 (NarInfo.niNarSize ni),+      -- Twenty digits still fit values past 2^64 - 1; upstream's+      -- string2Int<uint64_t> refuses them, so signing one here would+      -- produce a narinfo every real client rejects as corrupt.+      test "a 20-digit size above the uint64 maximum rejects" $+        let overCap =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/test.nar.xz",+                  "NarHash: sha256:def",+                  "NarSize: 18446744073709551616"+                ]+         in assertLeft "size above uint64" (NarInfo.parseNarInfo overCap),+      -- Upstream assigns fields as it reads, so a duplicated scalar key+      -- resolves to the LAST value (Sig stays the repeatable exception).+      test "duplicate scalar keys resolve last-wins" $+        let dup =+              T.unlines+                [ "StorePath: /nix/store/aaaa-test",+                  "URL: nar/first.nar",+                  "URL: nar/last.nar",+                  "Compression: xz",+                  "Compression: zstd",+                  "NarHash: sha256:def",+                  "NarSize: 200"+                ]+         in case NarInfo.parseNarInfo dup of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right ni -> do+                ok1 <- assertEqual "url last" "nar/last.nar" (NarInfo.niUrl ni)+                ok2 <- assertEqual "compression last" "zstd" (NarInfo.niCompression ni)+                pure (ok1 && ok2)+    ]++-- ---------------------------------------------------------------------------+-- Signing tests+-- ---------------------------------------------------------------------------++testSigning :: IO Bool+testSigning =+  runGroup+    "Signing"+    [ test "fingerprint format" $+        let ni = mkTestNarInfo+            fp = Signing.fingerprint ni+         in do+              ok1 <- assertTrue "starts with 1;" (T.isPrefixOf "1;" fp)+              ok2 <- assertTrue "contains storePath" (T.isInfixOf "/nix/store/" fp)+              pure (ok1 && ok2),+      test "fingerprint renders references as full store paths" $+        let ni = mkTestNarInfo {NarInfo.niReferences = ["00000000000000000000000000000000-glibc-2.40"]}+            fp = Signing.fingerprint ni+         in assertTrue+              "reference is a full /nix/store path in the fingerprint"+              (T.isInfixOf "/nix/store/00000000000000000000000000000000-glibc-2.40" fp),+      test "fingerprint sorts and dedupes references" $+        let ni =+              mkTestNarInfo+                { NarInfo.niReferences =+                    [ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-zlib-1.3",+                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-glibc-2.40",+                      "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-zlib-1.3"+                    ]+                }+            fp = Signing.fingerprint ni+         in assertTrue+              "references sorted by basename and deduplicated (C++ Nix parity)"+              -- The leading field separator pins the WHOLE references+              -- field: without it, the pre-fix "zlib,glibc,zlib"+              -- rendering also ends in "...glibc...,...zlib..." and the+              -- assertion cannot fail on a regression to unsorted output.+              ( T.isSuffixOf+                  ";/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-glibc-2.40,/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-zlib-1.3"+                  fp+              ),+      test "SecretKey Show redacts the key bytes" $+        let sk = Signing.SecretKey {Signing.skName = "test-key", Signing.skBytes = BS.pack ([1 .. 32] ++ [33 .. 64])}+         in assertEqual "redacted" "SecretKey \"test-key\" <redacted>" (show sk),+      test "SecretKey equality compares name and bytes" $+        let bytesA = BS.pack ([1 .. 32] ++ [33 .. 64])+            skA = Signing.SecretKey "k" bytesA+            skB = Signing.SecretKey "k" bytesA+            skC = Signing.SecretKey "k" (BS.pack (0 : [2 .. 64]))+         in do+              ok1 <- assertTrue "equal keys" (skA == skB)+              ok2 <- assertTrue "different bytes differ" (skA /= skC)+              pure (ok1 && ok2),+      test "parseSecretKey valid" $+        let keyBytes = BS.pack ([1 .. 32] ++ [33 .. 64])+            keyB64 = TE.decodeUtf8 (B64.encode keyBytes)+            keyStr = "test-key:" <> keyB64+         in case Signing.parseSecretKey keyStr of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right sk ->+                assertEqual "key name" "test-key" (Signing.skName sk),+      test "parsePublicKey valid" $+        let keyBytes = BS.pack [1 .. 32]+            keyB64 = TE.decodeUtf8 (B64.encode keyBytes)+            keyStr = "test-key:" <> keyB64+         in case Signing.parsePublicKey keyStr of+              Left err -> do+                putStrLn ("  parse failed: " ++ err)+                pure False+              Right pk ->+                assertEqual "key name" "test-key" (Signing.pkName pk),+      test "parseSecretKey no colon fails" $+        assertLeft "no colon" (Signing.parseSecretKey "nokeyname"),+      test "empty key name rejected" $+        -- An empty-named key would emit :sig lines no named trust anchor+        -- matches; the misconfiguration must fail at load time.+        let material = TE.decodeUtf8 (B64.encode (BS.pack [1 .. 64]))+         in assertLeft "empty name" (Signing.parseSecretKey (":" <> material)),+      test "empty key material rejected" $+        assertLeft "empty material" (Signing.parsePublicKey "test-key:"),+      test "signature from a different keypair rejected" $ do+        signingKey <- generateTestSecretKey+        otherKey <- generateTestSecretKey+        let verifier = deriveTestPublicKey otherKey+        case Signing.sign signingKey mkTestNarInfo of+          Left err -> do+            putStrLn ("  sign failed: " ++ err)+            pure False+          Right signed ->+            assertTrue "cross-key rejected" (not (Signing.verify verifier mkTestNarInfo signed)),+      test "valid signature under a renamed trust anchor rejected" $ do+        signingKey <- generateTestSecretKey+        let renamed = (deriveTestPublicKey signingKey) {Signing.pkName = "some-other-cache"}+        case Signing.sign signingKey mkTestNarInfo of+          Left err -> do+            putStrLn ("  sign failed: " ++ err)+            pure False+          Right signed ->+            assertTrue "name mismatch rejected" (not (Signing.verify renamed mkTestNarInfo signed)),+      test "malformed signature lines rejected" $ do+        signingKey <- generateTestSecretKey+        let verifier = deriveTestPublicKey signingKey+            wrongSize = "test-key:" <> TE.decodeUtf8 (B64.encode (BS.pack [1 .. 16]))+            badLines = ["test-key:!!!not-base64!!!", wrongSize, "test-key:", "no-colon-at-all"]+        assertTrue+          "all malformed rejected"+          (not (any (Signing.verify verifier mkTestNarInfo) badLines)),+      test "parsePublicKey wrong size fails" $+        let keyStr = "test-key:" <> TE.decodeUtf8 (B64.encode (BS.pack [1 .. 16]))+         in assertLeft "wrong size" (Signing.parsePublicKey keyStr),+      test "sign/verify roundtrip" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+            ni = mkTestNarInfo+        case Signing.sign sk ni of+          Left err -> do+            putStrLn ("  sign failed: " ++ err)+            pure False+          Right sig ->+            assertTrue "verify passes" (Signing.verify pk ni sig),+      test "verify rejects tampered narinfo" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+            ni = mkTestNarInfo+        case Signing.sign sk ni of+          Left err -> do+            putStrLn ("  sign failed: " ++ err)+            pure False+          Right sig ->+            let tampered = ni {NarInfo.niNarSize = 999999}+             in assertFalse "verify rejects tampered" (Signing.verify pk tampered sig),+      test "toPublicKey derives the verifying key" $ do+        sk <- generateTestSecretKey+        case Signing.toPublicKey sk of+          Left err -> do+            putStrLn ("  toPublicKey failed: " ++ err)+            pure False+          Right pk -> do+            okName <- assertEqual "key name carried over" (Signing.skName sk) (Signing.pkName pk)+            okBytes <- assertEqual "matches the stored public half" (BS.drop 32 (Signing.skBytes sk)) (Signing.pkBytes pk)+            case Signing.sign sk mkTestNarInfo of+              Left err -> do+                putStrLn ("  sign failed: " ++ err)+                pure False+              Right sig -> do+                okVerify <- assertTrue "derived key verifies a signature" (Signing.verify pk mkTestNarInfo sig)+                pure (okName && okBytes && okVerify),+      test "renderPublicKey round-trips through parsePublicKey" $ do+        sk <- generateTestSecretKey+        case Signing.toPublicKey sk of+          Left err -> do+            putStrLn ("  toPublicKey failed: " ++ err)+            pure False+          Right pk -> case Signing.parsePublicKey (Signing.renderPublicKey pk) of+            Left err -> do+              putStrLn ("  parse failed: " ++ err)+              pure False+            Right reparsed -> assertEqual "round-trip" pk reparsed,+      test "normalizeKeyText strips BOM and whitespace" $ do+        ok1 <- assertEqual "BOM stripped" "test-key:abc" (Signing.normalizeKeyText ("\xFEFF" <> "test-key:abc"))+        ok2 <- assertEqual "CRLF stripped" "test-key:abc" (Signing.normalizeKeyText "test-key:abc\r\n")+        ok3 <- assertEqual "spaces stripped" "test-key:abc" (Signing.normalizeKeyText "  test-key:abc  ")+        ok4 <- assertEqual "clean text unchanged" "test-key:abc" (Signing.normalizeKeyText "test-key:abc")+        pure (ok1 && ok2 && ok3 && ok4)+    ]++-- | Create a test NarInfo for signing tests.+mkTestNarInfo :: NarInfo.NarInfo+mkTestNarInfo =+  NarInfo.NarInfo+    { NarInfo.niStorePath = "/nix/store/aaaa-hello-1.0",+      NarInfo.niUrl = "nar/test.nar.xz",+      NarInfo.niCompression = "xz",+      NarInfo.niFileHash = Just "sha256:abc",+      NarInfo.niFileSize = Just 100,+      NarInfo.niNarHash = "sha256:def",+      NarInfo.niNarSize = 200,+      NarInfo.niReferences = ["aaaa-hello-1.0"],+      NarInfo.niDeriver = Nothing,+      NarInfo.niSigs = [],+      NarInfo.niCA = Nothing+    }++-- ---------------------------------------------------------------------------+-- FileStore tests+-- ---------------------------------------------------------------------------++testFileStore :: IO Bool+testFileStore =+  runGroup+    "FileStore"+    [ test "narinfo write/read roundtrip" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        let hashKey = "testhash123"+            content = TE.encodeUtf8 ("StorePath: /nix/store/test\n" :: Text)+        wOk <- Store.writeNarInfo store hashKey content+        result <- Store.readNarInfo store hashKey+        removeDirectoryRecursive tmpDir+        ok1 <- assertTrue "write succeeded" wOk+        ok2 <- assertEqual "narinfo roundtrip" (Just content) result+        pure (ok1 && ok2),+      test "nar write/read roundtrip" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        let fileName = "test.nar.xz"+            content = BS.pack [1, 2, 3, 4, 5]+        wOk <- Store.writeNar store fileName content+        result <- Store.readNar store fileName+        removeDirectoryRecursive tmpDir+        ok1 <- assertTrue "write succeeded" wOk+        ok2 <- assertEqual "nar roundtrip" (Just content) result+        pure (ok1 && ok2),+      test "read nonexistent returns Nothing" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        result <- Store.readNarInfo store "nonexistent"+        removeDirectoryRecursive tmpDir+        assertEqual "not found" Nothing result,+      test "cacheInfo defaults" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        let info = Store.getCacheInfo store+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "storeDir" "/nix/store" (Store.ciStoreDir info)+        ok2 <- assertTrue "wantMassQuery" (Store.ciWantMassQuery info)+        ok3 <- assertEqual "priority" 50 (Store.ciPriority info)+        pure (ok1 && ok2 && ok3),+      test "sanitizePath rejects traversal" $+        assertEqual "dotdot" Nothing (Store.sanitizePath ".."),+      test "sanitizePath rejects slash" $+        assertEqual "slash" Nothing (Store.sanitizePath "../../etc/passwd"),+      test "sanitizePath rejects backslash" $+        assertEqual "backslash" Nothing (Store.sanitizePath "..\\..\\etc\\passwd"),+      test "sanitizePath rejects empty" $+        assertEqual "empty" Nothing (Store.sanitizePath ""),+      test "sanitizePath accepts valid hash" $+        assertEqual "valid" (Just "abc123def456") (Store.sanitizePath "abc123def456"),+      test "sanitizePath rejects windows device name" $+        assertEqual "device nul" Nothing (Store.sanitizePath "nul"),+      test "sanitizePath rejects the zero-numbered devices" $ do+        ok1 <- assertEqual "com0" Nothing (Store.sanitizePath "com0")+        ok2 <- assertEqual "lpt0" Nothing (Store.sanitizePath "lpt0")+        ok3 <- assertEqual "com10 stays valid" (Just "com10") (Store.sanitizePath "com10")+        pure (ok1 && ok2 && ok3),+      test "sanitizePath rejects dotfile" $+        assertEqual "dotfile" Nothing (Store.sanitizePath ".hidden"),+      test "sanitizePath rejects a trailing dot" $+        assertEqual "trailing dot" Nothing (Store.sanitizePath "foo."),+      test "sanitizePath keeps interior dots valid" $+        assertEqual "interior dots" (Just "foo.nar.xz") (Store.sanitizePath "foo.nar.xz"),+      test "read rejects traversal" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        result <- Store.readNarInfo store "../../etc/passwd"+        removeDirectoryRecursive tmpDir+        assertEqual "blocked" Nothing result,+      test "writeNarInfo rejects traversal" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        ok <- Store.writeNarInfo store "../../etc/passwd" "bad"+        removeDirectoryRecursive tmpDir+        assertFalse "write rejected" ok,+      test "writeNar rejects traversal" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        ok <- Store.writeNar store "../escape.nar" "bad"+        removeDirectoryRecursive tmpDir+        assertFalse "write rejected" ok,+      test "listNarInfoHashes returns stored hashes" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        _ <- Store.writeNarInfo store "hash1" "content1"+        _ <- Store.writeNarInfo store "hash2" "content2"+        hashes <- Store.listNarInfoHashes store+        removeDirectoryRecursive tmpDir+        let sorted = sort hashes+        assertEqual "listed hashes" ["hash1", "hash2"] sorted,+      test "listNarInfoHashes empty store" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        hashes <- Store.listNarInfoHashes store+        removeDirectoryRecursive tmpDir+        assertEqual "empty" [] hashes,+      test "removeNarInfo removes a narinfo, reports one that is absent, and refuses a traversal name" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        _ <- Store.writeNarInfo store "hash1" "content1"+        _ <- Store.writeNarInfo store "hash2" "content2"+        outcomes <- traverse (Store.removeNarInfo store) ["hash1", "hash1", "../hash2"]+        hashes <- Store.listNarInfoHashes store+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "outcomes" [Store.NarInfoRemoved, Store.NarInfoAbsent, Store.NarInfoRemoveBadPath] outcomes+        ok2 <- assertEqual "remaining" ["hash2"] hashes+        pure (ok1 && ok2)+    ]++-- ---------------------------------------------------------------------------+-- Reclaim tests+-- ---------------------------------------------------------------------------++testReclaim :: IO Bool+testReclaim =+  runGroup+    "Reclaim"+    [ test "a nar/<file> URL names that file, compression suffix included" $+        assertEqual+          "target"+          (Reclaim.NamesNar "1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst")+          (narInfoBytesTarget (mkValidNarInfo {NarInfo.niUrl = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst"})),+      test "a URL in any other spelling is unresolvable" $+        let spellings =+              [ "https://cache.example/nar/x.nar",+                "./nar/x.nar",+                "nar/../x.nar",+                "nar/a/x.nar",+                "nar/x.nar?v=1",+                "nar/x%2Enar",+                "nar/.nova1.tmp",+                "nar/",+                "x.nar"+              ]+            resolved = [url | url <- spellings, not (isUnresolvable (narInfoBytesTarget (mkValidNarInfo {NarInfo.niUrl = url})))]+         in assertEqual "spellings that resolved" [] resolved,+      test "an unreadable, undecodable or unparseable narinfo is unresolvable" $ do+        let unreadable = Reclaim.narInfoTarget (Left (userError "permission denied"))+            undecodable = Reclaim.narInfoTarget (Right (BS.pack [0x55, 0x52, 0x4c, 0x3a, 0xff]))+            unparseable = Reclaim.narInfoTarget (Right "StorePath: /nix/store/x\nNarSize: 5\n")+        assertTrue "all unresolvable" (all isUnresolvable [unreadable, undecodable, unparseable]),+      test "property: a nar/<name> URL resolves to the name through the parser" $+        checkProperty $+          forAll genSafeNarName $ \name ->+            narInfoBytesTarget (mkValidNarInfo {NarInfo.niUrl = "nar/" <> name}) === Reclaim.NamesNar name,+      test "property: a referenced NAR is never selected" $+        checkProperty $+          forAll genSelection $ \(minimumAge, references, nars) ->+            let selected = selectAll minimumAge references nars+             in all ((`Set.notMember` references) . Store.nfName) selected,+      test "property: the selection is a subset of the NARs present" $+        checkProperty $+          forAll genSelection $ \(minimumAge, references, nars) ->+            all (`elem` nars) (selectAll minimumAge references nars),+      test "property: every unreferenced NAR is either eligible or kept, never both" $+        checkProperty $+          forAll genSelection $ \(minimumAge, references, nars) ->+            sort (map Store.nfName (selectAll minimumAge references nars))+              === sort [Store.nfName nar | nar <- nars, Set.notMember (Store.nfName nar) references],+      test "property: eligible NARs are at least the minimum age, kept ones younger" $+        checkProperty $+          forAll genSelection $ \(minimumAge, references, nars) ->+            let plan = Reclaim.selectUnreferenced minimumAge referenceTime references nars+             in all ((>= minimumAge) . ageOf) (Reclaim.rpEligible plan)+                  && all ((< minimumAge) . ageOf) (Reclaim.rpTooYoung plan),+      test "property: adding a narinfo never grows the eligible set" $+        checkProperty $+          forAll genSelection $ \(minimumAge, references, nars) ->+            forAll genNarName $ \added ->+              let eligible refs = Reclaim.rpEligible (Reclaim.selectUnreferenced minimumAge referenceTime refs nars)+               in all (`elem` eligible references) (eligible (Set.insert added references)),+      test "property: raising the minimum age never grows the eligible set" $+        checkProperty $+          forAll genSelection $ \(minimumAge, references, nars) ->+            forAll genMinimumAge $ \raise ->+              let eligible age = Reclaim.rpEligible (Reclaim.selectUnreferenced age referenceTime references nars)+               in all (`elem` eligible minimumAge) (eligible (minimumAge + raise)),+      test "property: any unresolvable narinfo blocks the run, and all are reported" $+        checkProperty $+          forAll genSelection $ \(minimumAge, _, nars) ->+            forAll (listOf genTarget) $ \targets ->+              let narInfos = zip [show i | i <- [0 :: Int ..]] targets+                  scan = scanOf narInfos nars+                  blocking = [path | (path, Reclaim.Unresolvable _) <- narInfos]+               in case Reclaim.planReclaim minimumAge scan of+                    Left (Reclaim.UnresolvedNarInfos unresolved) -> map Reclaim.unresolvedPath (toList unresolved) === blocking+                    Left Reclaim.NoNarInfos -> property (null targets && not (null nars))+                    Right _ -> blocking === [],+      test "property: a store with NARs and no narinfo blocks the run, and an empty store plans nothing" $+        checkProperty $+          forAll genSelection $ \(minimumAge, _, nars) ->+            let scan = scanOf [] nars+                expected = case nars of+                  [] -> Right Reclaim.ReclaimPlan {Reclaim.rpEligible = [], Reclaim.rpTooYoung = []}+                  _ -> Left Reclaim.NoNarInfos+             in Reclaim.planReclaim minimumAge scan === expected,+      test "property: a run plans its uploads as selectAbandoned splits them, and a blocked run plans nothing" $+        checkProperty $+          forAll genSelection $ \(minimumAge, _, nars) ->+            forAll (listOf genTarget) $ \targets ->+              forAll (listOf genStagedUpload) $ \uploads ->+                let scan = (scanOf (zip [show i | i <- [0 :: Int ..]] targets) nars) {Reclaim.scanStagedUploads = uploads}+                 in Reclaim.planStore minimumAge scan+                      === ( (\narPlan -> Reclaim.StorePlan narPlan (Reclaim.selectAbandoned minimumAge referenceTime uploads))+                              <$> Reclaim.planReclaim minimumAge scan+                          ),+      test "a NAR exactly the minimum age old is eligible" $+        let nar = narAged "edge.nar" Reclaim.defaultMinimumAge+            plan = Reclaim.selectUnreferenced Reclaim.defaultMinimumAge referenceTime Set.empty [nar]+         in assertEqual "eligible" [nar] (Reclaim.rpEligible plan),+      test "a scan of a store: blocked by a broken narinfo, then reclaimed" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        now <- getCurrentTime+        let monthAgo = addUTCTime (negate (30 * nominalDay)) now+            narPath name = tmpDir ++ "/nar/" ++ name+            putNar name modified = do+              _ <- Store.writeNar store (T.pack name) "nar bytes"+              setModificationTime (narPath name) modified+        _ <- Store.writeNarInfo store "referencing" (TE.encodeUtf8 (NarInfo.renderNarInfo (mkValidNarInfo {NarInfo.niUrl = "nar/ref.nar.zst"})))+        _ <- Store.writeNarInfo store "broken" "not a narinfo\n"+        putNar "ref.nar.zst" monthAgo+        putNar "old.nar.zst" monthAgo+        putNar "young.nar.zst" now+        -- An upload in progress: the temp name a streaming write uses.+        BS.writeFile (narPath ".nova123-4.tmp") "partial"+        setModificationTime (narPath ".nova123-4.tmp") monthAgo+        blocked <- Reclaim.planReclaim Reclaim.defaultMinimumAge <$> Reclaim.scanStore store getCurrentTime+        removeFile (tmpDir ++ "/narinfo/broken")+        reclaimable <- Reclaim.planReclaim Reclaim.defaultMinimumAge <$> Reclaim.scanStore store getCurrentTime+        let blockedBy = case blocked of+              Left (Reclaim.UnresolvedNarInfos unresolved) -> map Reclaim.unresolvedPath (toList unresolved)+              _ -> []+            eligible = either (const []) (map Store.nfName . Reclaim.rpEligible) reclaimable+            tooYoung = either (const []) (map Store.nfName . Reclaim.rpTooYoung) reclaimable+        removals <- traverse (Store.removeNar store) eligible+        remaining <- sort <$> listDirectory (tmpDir ++ "/nar")+        removeDirectoryRecursive tmpDir+        ok1 <- assertTrue "blocked by the broken narinfo alone" (case blockedBy of [path] -> "broken" `isSuffixOf` path; _ -> False)+        ok2 <- assertEqual "eligible" ["old.nar.zst"] eligible+        ok3 <- assertEqual "kept" ["young.nar.zst"] tooYoung+        ok4 <- assertEqual "removals" [Store.NarRemoved] removals+        ok5 <- assertEqual "remaining" [".nova123-4.tmp", "ref.nar.zst", "young.nar.zst"] remaining+        pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "a store whose narinfo directory the server recreated blocks the run" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        now <- getCurrentTime+        _ <- Store.writeNar store "old.nar.zst" "nar bytes"+        setModificationTime (tmpDir ++ "/nar/old.nar.zst") (addUTCTime (negate (30 * nominalDay)) now)+        removeDirectoryRecursive (tmpDir ++ "/narinfo")+        restarted <- Store.newFileStore tmpDir+        plan <- Reclaim.planReclaim Reclaim.defaultMinimumAge <$> Reclaim.scanStore restarted getCurrentTime+        removeDirectoryRecursive tmpDir+        assertEqual "blocked" (Left Reclaim.NoNarInfos) plan,+      test "a scan of a store that lost its narinfo directory fails and creates nothing" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        removeDirectoryRecursive (tmpDir ++ "/narinfo")+        result <- tryIO (Reclaim.scanStore store getCurrentTime)+        created <- doesDirectoryExist (tmpDir ++ "/narinfo")+        removeDirectoryRecursive tmpDir+        ok1 <- assertTrue "scan failed" (isLeft result)+        ok2 <- assertFalse "narinfo directory created" created+        pure (ok1 && ok2),+      test "removeNar reports a missing file and refuses a traversal name" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        missing <- Store.removeNar store "absent.nar"+        traversal <- Store.removeNar store "../escape.nar"+        removeDirectoryRecursive tmpDir+        ok1 <- assertTrue "missing file fails" (case missing of Store.NarRemoveFailed _ -> True; _ -> False)+        ok2 <- assertEqual "traversal" Store.NarRemoveBadPath traversal+        pure (ok1 && ok2),+      -- Abandoned multipart uploads+      test "property: a staged upload is eligible exactly when idle for the minimum age" $+        checkProperty $+          forAll genMinimumAge $ \minimumAge ->+            forAll (listOf genStagedUpload) $ \uploads ->+              let plan = Reclaim.selectAbandoned minimumAge referenceTime uploads+               in (sort (map Store.suPath (Reclaim.upEligible plan ++ Reclaim.upTooYoung plan)) === sort (map Store.suPath uploads))+                    .&&. all ((>= minimumAge) . uploadAge) (Reclaim.upEligible plan)+                    .&&. all ((< minimumAge) . uploadAge) (Reclaim.upTooYoung plan),+      test "a reclaim of staged uploads: an idle one is removed, an active one and a stray directory are kept" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        monthAgo <- addUTCTime (negate (30 * nominalDay)) <$> getCurrentTime+        idle <- stageUpload store "idle.nar" ["idle", " bytes"]+        active <- stageUpload store "active.nar" ["active"]+        createDirectory (tmpDir ++ "/uploads/stray")+        setModificationTime (tmpDir ++ "/uploads/stray") monthAgo+        traverse_ (backdateUpload monthAgo) . filter ((== "idle.nar") . Store.suFileName) =<< Store.listStagedUploads store+        reclaimed <- Reclaim.reclaim store Reclaim.defaultMinimumAge getCurrentTime+        remaining <- map Store.suFileName <$> Store.listStagedUploads store+        strayKept <- doesDirectoryExist (tmpDir ++ "/uploads/stray")+        removeDirectoryRecursive tmpDir+        let removals = either (const []) Reclaim.rdUploads reclaimed+            kept = either (const []) (Reclaim.upTooYoung . Reclaim.spUploads . Reclaim.rdPlan) reclaimed+        ok1 <- assertTrue "uploads staged" (isJust idle && isJust active)+        ok2 <- assertEqual "removed" [("idle.nar", 10, Store.UploadRemoved)] [(Store.suFileName u, Store.suSize u, outcome) | (u, outcome) <- removals]+        ok3 <- assertEqual "kept" ["active.nar"] (map Store.suFileName kept)+        ok4 <- assertEqual "remaining" ["active.nar"] remaining+        ok5 <- assertTrue "stray directory kept" strayKept+        pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "a part, or a part in progress, written to an idle upload makes it active again" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        monthAgo <- addUTCTime (negate (30 * nominalDay)) <$> getCurrentTime+        staged <- stageUpload store "resumed.nar" ["first"]+        let ageNow = do+              scan <- Reclaim.scanStore store getCurrentTime+              let plan = Reclaim.selectAbandoned Reclaim.defaultMinimumAge (Reclaim.scanTime scan) (Reclaim.scanStagedUploads scan)+              pure (length (Reclaim.upEligible plan), length (Reclaim.upTooYoung plan))+            backdateAll = traverse_ (backdateUpload monthAgo) =<< Store.listStagedUploads store+        backdateAll+        idle <- ageNow+        written <- traverse (\uid -> writeStorePart store "resumed.nar" uid 2 ["second"] Server.maxNarBodySize) staged+        afterPart <- ageNow+        backdateAll+        -- A part in progress is a temp file in the upload's directory.+        uploads <- Store.listStagedUploads store+        traverse_ (\upload -> BS.writeFile (Store.suPath upload ++ "/.nova123-4.tmp") "partial" >> setModificationTime (Store.suPath upload) monthAgo) uploads+        afterTemp <- ageNow+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "idle (eligible, kept)" (1, 0) idle+        ok2 <- assertEqual "part written" (Just (Just Store.PartWritten)) written+        ok3 <- assertEqual "after a part (eligible, kept)" (0, 1) afterPart+        ok4 <- assertEqual "after a part in progress (eligible, kept)" (0, 1) afterTemp+        pure (ok1 && ok2 && ok3 && ok4),+      test "a store with no uploads directory has no staged uploads" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        removeDirectoryRecursive (tmpDir ++ "/uploads")+        uploads <- Store.listStagedUploads store+        removeDirectoryRecursive tmpDir+        assertEqual "uploads" [] uploads,+      test "removeUpload refuses a traversal name" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        staged <- stageUpload store "kept.nar" ["kept"]+        outcomes <- traverse (Store.removeUpload store "../kept.nar") staged+        remaining <- map Store.suFileName <$> Store.listStagedUploads store+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "outcome" (Just Store.UploadRemoveBadPath) outcomes+        ok2 <- assertEqual "remaining" ["kept.nar"] remaining+        pure (ok1 && ok2)+    ]++-- | Run a QuickCheck property quietly, printing its report on failure.+checkProperty :: (Testable prop) => prop -> IO Bool+checkProperty prop = do+  result <- quickCheckWithResult stdArgs {chatty = False} prop+  case result of+    Success {} -> pure True+    Failure {output = report} -> failWith report+    GaveUp {output = report} -> failWith report+    NoExpectedFailure {output = report} -> failWith report+  where+    failWith report = do+      putStrLn ""+      putStr report+      pure False++-- | The target of a narinfo rendered and read back as the store holds it.+narInfoBytesTarget :: NarInfo.NarInfo -> Reclaim.NarInfoTarget+narInfoBytesTarget = Reclaim.narInfoTarget . Right . TE.encodeUtf8 . NarInfo.renderNarInfo++isUnresolvable :: Reclaim.NarInfoTarget -> Bool+isUnresolvable (Reclaim.Unresolvable _) = True+isUnresolvable (Reclaim.NamesNar _) = False++-- | Everything a selection names, eligible or kept.+selectAll :: NominalDiffTime -> Set Text -> [Store.NarFile] -> [Store.NarFile]+selectAll minimumAge references nars =+  let plan = Reclaim.selectUnreferenced minimumAge referenceTime references nars+   in Reclaim.rpEligible plan ++ Reclaim.rpTooYoung plan++-- | A scan at 'referenceTime' of narinfos and NARs, with no staged+-- upload.+scanOf :: [(FilePath, Reclaim.NarInfoTarget)] -> [Store.NarFile] -> Reclaim.StoreScan+scanOf narInfos nars =+  Reclaim.StoreScan+    { Reclaim.scanNarInfos = narInfos,+      Reclaim.scanNars = nars,+      Reclaim.scanStagedUploads = [],+      Reclaim.scanTime = referenceTime+    }++-- | The fixed "now" of the pure selection tests.+referenceTime :: UTCTime+referenceTime = UTCTime (fromGregorian 2026 10 9) 0++ageOf :: Store.NarFile -> NominalDiffTime+ageOf nar = diffUTCTime referenceTime (Store.nfModified nar)++narAged :: Text -> NominalDiffTime -> Store.NarFile+narAged name age =+  Store.NarFile+    { Store.nfName = name,+      Store.nfPath = "nar/" ++ T.unpack name,+      Store.nfSize = 0,+      Store.nfModified = addUTCTime (negate age) referenceTime+    }++-- | NAR names from a small pool, so references and listings overlap+-- often.+narNamePool :: [Text]+narNamePool = [T.pack ("n" ++ show i ++ ".nar.zst") | i <- [0 .. 11 :: Int]]++genNarName :: Gen Text+genNarName = elements narNamePool++-- | Whole hours, so ages land on the minimum age exactly now and then;+-- the negative end is a modification time in the future (a clock that+-- stepped back).+genAge :: Gen NominalDiffTime+genAge = (* 3600) . fromInteger <$> choose (-2, 24 * 30)++genMinimumAge :: Gen NominalDiffTime+genMinimumAge = (* 3600) . fromInteger <$> choose (0, 24 * 21)++-- | A listing has distinct names, as a directory does.+genListing :: Gen [Store.NarFile]+genListing = traverse (\name -> narAged name <$> genAge) =<< sublistOf narNamePool++genSelection :: Gen (NominalDiffTime, Set Text, [Store.NarFile])+genSelection = (,,) <$> genMinimumAge <*> (Set.fromList <$> sublistOf narNamePool) <*> genListing++genTarget :: Gen Reclaim.NarInfoTarget+genTarget =+  frequency+    [ (4, Reclaim.NamesNar <$> genNarName),+      (1, pure (Reclaim.Unresolvable "unparseable"))+    ]++-- | An upload staged in a directory of its own, idle for a generated age.+genStagedUpload :: Gen Store.StagedUpload+genStagedUpload = do+  uid <- genUploadId+  age <- genAge+  pure+    Store.StagedUpload+      { Store.suFileName = "n.nar",+        Store.suUploadId = uid,+        Store.suPath = "uploads/" ++ T.unpack (Store.renderUploadId uid) ++ "-n.nar",+        Store.suSize = 0,+        Store.suModified = addUTCTime (negate age) referenceTime+      }++uploadAge :: Store.StagedUpload -> NominalDiffTime+uploadAge upload = diffUTCTime referenceTime (Store.suModified upload)++genUploadId :: Gen Store.UploadId+genUploadId = (T.pack <$> vectorOf 32 (elements lowerHexDigits)) `suchThatMap` Store.parseUploadId++lowerHexDigits :: String+lowerHexDigits = "0123456789abcdef"++-- | Begin an upload of a NAR in a store and write its parts in order:+-- the upload, when every step succeeded.+stageUpload :: Store.FileStore -> Text -> [ByteString] -> IO (Maybe Store.UploadId)+stageUpload store name parts = do+  begun <- Store.beginUpload store name+  case begun of+    Nothing -> pure Nothing+    Just uid -> do+      written <- zipWithM (\n bytes -> writeStorePart store name uid n [bytes] Server.maxNarBodySize) [1 ..] parts+      pure (if all (== Just Store.PartWritten) written then Just uid else Nothing)++-- | Write part @n@ of an upload from chunks, under a cap; 'Nothing' when+-- @n@ is not a part number.+writeStorePart :: Store.FileStore -> Text -> Store.UploadId -> Int -> [ByteString] -> Int -> IO (Maybe Store.PartWriteResult)+writeStorePart store name uid n chunks limit = case Store.parsePartNumber (T.pack (show n)) of+  Nothing -> pure Nothing+  Just part -> Just <$> (Store.writeUploadPart store name uid part limit =<< chunkSource chunks)++-- | Set the modification time of an upload's files, then of its+-- directory, which setting a file's time leaves alone.+backdateUpload :: UTCTime -> Store.StagedUpload -> IO ()+backdateUpload time upload = do+  files <- listDirectory (Store.suPath upload)+  traverse_ (\file -> setModificationTime (Store.suPath upload ++ "/" ++ file) time) files+  setModificationTime (Store.suPath upload) time++-- | Names 'Store.sanitizePath' admits.+genSafeNarName :: Gen Text+genSafeNarName =+  (T.pack <$> listOf1 (elements safeNameChars)) `suchThat` (isJust . Store.sanitizePath)+  where+    safeNameChars = ['a' .. 'z'] ++ ['A' .. 'Z'] ++ ['0' .. '9'] ++ "._+-"++-- ---------------------------------------------------------------------------+-- Validate tests+-- ---------------------------------------------------------------------------++-- | Bytes whose SHA-256 is used as the NarHash in 'mkValidNarInfo'.+validNarBytes :: ByteString+validNarBytes = BS.pack [1, 2, 3, 4, 5]++-- | Bytes whose SHA-256 is used as the FileHash in 'mkValidNarInfo'.+validFileBytes :: ByteString+validFileBytes = BS.pack [10, 20, 30, 40, 50]++-- | A narinfo that passes all field validation.+mkValidNarInfo :: NarInfo.NarInfo+mkValidNarInfo =+  NarInfo.NarInfo+    { NarInfo.niStorePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0",+      NarInfo.niUrl = "nar/test.nar.xz",+      NarInfo.niCompression = "xz",+      NarInfo.niFileHash = Just (Hash.formatNixHash (Hash.hashBytes validFileBytes)),+      NarInfo.niFileSize = Just 5,+      NarInfo.niNarHash = Hash.formatNixHash (Hash.hashBytes validNarBytes),+      NarInfo.niNarSize = 5,+      NarInfo.niReferences = ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello-1.0"],+      NarInfo.niDeriver = Nothing,+      NarInfo.niSigs = [],+      NarInfo.niCA = Nothing+    }++testValidate :: IO Bool+testValidate =+  runGroup+    "Validate"+    [ test "validateNarInfo valid" $+        assertEqual+          "valid narinfo"+          (Right mkValidNarInfo)+          (Validate.validateNarInfo mkValidNarInfo),+      test "validateNarInfo negative FileSize" $+        let ni = mkValidNarInfo {NarInfo.niFileSize = Just (-1)}+         in assertEqual+              "negative filesize"+              (Left [Validate.NegativeFileSize (-1)])+              (Validate.validateNarInfo ni),+      test "validateNarInfo negative NarSize" $+        let ni = mkValidNarInfo {NarInfo.niNarSize = -1}+         in assertEqual+              "negative narsize"+              (Left [Validate.NegativeNarSize (-1)])+              (Validate.validateNarInfo ni),+      test "validateNarInfo derivation StorePath rejected" $+        let drvPath = "/nix/store/abc12345678901234567890123456789-foo.drv"+            ni = mkValidNarInfo {NarInfo.niStorePath = drvPath}+         in case Validate.validateNarInfo ni of+              Left [Validate.DerivationStorePath raw] ->+                assertEqual "raw value" drvPath raw+              other -> do+                putStrLn ("    expected Left [DerivationStorePath ..], got: " ++ show other)+                pure False,+      test "validateNarInfo bad StorePath" $+        let ni = mkValidNarInfo {NarInfo.niStorePath = "not-a-store-path"}+         in case Validate.validateNarInfo ni of+              Left [Validate.InvalidStorePath raw _] ->+                assertEqual "raw value" "not-a-store-path" raw+              other -> do+                putStrLn ("    expected Left [InvalidStorePath ..], got: " ++ show other)+                pure False,+      test "validateNarInfo bad FileHash" $+        let ni = mkValidNarInfo {NarInfo.niFileHash = Just "md5:bogus"}+         in case Validate.validateNarInfo ni of+              Left [Validate.InvalidFileHash raw _] ->+                assertEqual "raw value" "md5:bogus" raw+              other -> do+                putStrLn ("    expected Left [InvalidFileHash ..], got: " ++ show other)+                pure False,+      test "validateNarInfo bad NarHash" $+        let ni = mkValidNarInfo {NarInfo.niNarHash = "md5:bogus"}+         in case Validate.validateNarInfo ni of+              Left [Validate.InvalidNarHash raw _] ->+                assertEqual "raw value" "md5:bogus" raw+              other -> do+                putStrLn ("    expected Left [InvalidNarHash ..], got: " ++ show other)+                pure False,+      test "validateNarInfo bad reference" $+        let ni = mkValidNarInfo {NarInfo.niReferences = ["bad"]}+         in case Validate.validateNarInfo ni of+              Left [Validate.InvalidReference raw _] ->+                assertEqual "raw value" "bad" raw+              other -> do+                putStrLn ("    expected Left [InvalidReference ..], got: " ++ show other)+                pure False,+      test "validateNarInfo multiple errors collected" $+        let ni =+              mkValidNarInfo+                { NarInfo.niFileSize = Just (-1),+                  NarInfo.niNarSize = -1,+                  NarInfo.niStorePath = "bad"+                }+         in case Validate.validateNarInfo ni of+              Left errs -> assertTrue "at least 3 errors" (length errs >= 3)+              Right _ -> do+                putStrLn "    expected Left, got Right"+                pure False,+      test "validateNarHash correct" $+        assertEqual+          "correct nar hash"+          (Right ())+          (Validate.validateNarHash mkValidNarInfo validNarBytes),+      test "validateNarHash wrong" $+        case Validate.validateNarHash mkValidNarInfo (BS.pack [99]) of+          Left (Validate.NarHashMismatch _ _) -> pure True+          other -> do+            putStrLn ("    expected Left NarHashMismatch, got: " ++ show other)+            pure False,+      test "validateFileHash correct" $+        assertEqual+          "correct file hash"+          (Right ())+          (Validate.validateFileHash mkValidNarInfo validFileBytes),+      test "validateFileHash wrong" $+        case Validate.validateFileHash mkValidNarInfo (BS.pack [99]) of+          Left (Validate.FileHashMismatch _ _) -> pure True+          other -> do+            putStrLn ("    expected Left FileHashMismatch, got: " ++ show other)+            pure False,+      test "validateSignature valid" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+            ni = mkValidNarInfo+        case Signing.sign sk ni of+          Left err -> do+            putStrLn ("  sign failed: " ++ err)+            pure False+          Right sig ->+            let niSigned = ni {NarInfo.niSigs = [sig]}+             in assertEqual "valid sig" (Right ()) (Validate.validateSignature pk niSigned),+      test "validateSignature invalid" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+            bogusSig = "bogus-key:aW52YWxpZA=="+            ni = mkValidNarInfo {NarInfo.niSigs = [bogusSig]}+        assertEqual+          "invalid sig"+          (Left [Validate.SignatureInvalid bogusSig])+          (Validate.validateSignature pk ni),+      test "validateSignature no sigs" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+        assertEqual+          "no sigs"+          (Left [Validate.NoSignatures])+          (Validate.validateSignature pk mkValidNarInfo),+      test "validateFull all good" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+            ni = mkValidNarInfo+        case Signing.sign sk ni of+          Left err -> do+            putStrLn ("  sign failed: " ++ err)+            pure False+          Right sig ->+            let niSigned = ni {NarInfo.niSigs = [sig]}+             in assertEqual+                  "full valid"+                  (Right ())+                  (Validate.validateFull pk niSigned validNarBytes validFileBytes),+      test "validateFull multiple failures" $ do+        sk <- generateTestSecretKey+        let pk = deriveTestPublicKey sk+            ni = mkValidNarInfo {NarInfo.niFileSize = Just (-1), NarInfo.niNarSize = -1}+        case Validate.validateFull pk ni (BS.pack [99]) (BS.pack [99]) of+          Left errs -> assertTrue "at least 4 errors" (length errs >= 4)+          Right _ -> do+            putStrLn "    expected Left, got Right"+            pure False+    ]++-- ---------------------------------------------------------------------------+-- Server (WAI) tests+-- ---------------------------------------------------------------------------++-- | The write key the authenticated-server tests configure.+serverTestApiKey :: ByteString+serverTestApiKey = "test-api-key"++-- | The Authorization header 'serverTestApiKey' expects.+serverAuthHeader :: HTTP.Header+serverAuthHeader = (HTTP.hAuthorization, "Bearer " <> serverTestApiKey)++-- | Store-path hash of 'validServerNarInfo' (32 nix-base32 zeros).+validNarInfoHashKey :: Text+validNarInfoHashKey = T.replicate 32 "0"++-- | A canonical all-zero sha256 in nix-base32: 52 digits, and the 4+-- spare bits of the 260-bit encoding are zero, so it parses.+zeroNarHash :: Text+zeroNarHash = "sha256:" <> T.replicate 52 "0"++-- | The file 'validServerNarInfo' names under @nar\/@.+serverNarName :: Text+serverNarName = "test.nar"++-- | The bytes 'validServerNarInfo' declares for its NAR.+serverNarBytes :: ByteString+serverNarBytes = "nar-payload-bytes"++-- | A narinfo that passes 'Validate.validateNarInfo' end to end, keyed+-- under 'validNarInfoHashKey', describing 'serverNarBytes' stored as+-- 'serverNarName'.+validServerNarInfo :: NarInfo.NarInfo+validServerNarInfo =+  NarInfo.NarInfo+    { NarInfo.niStorePath = "/nix/store/" <> validNarInfoHashKey <> "-hello-1.0",+      NarInfo.niUrl = "nar/" <> serverNarName,+      NarInfo.niCompression = "none",+      NarInfo.niFileHash = Just (Hash.formatNixHash (Hash.hashBytes serverNarBytes)),+      NarInfo.niFileSize = Just (fromIntegral (BS.length serverNarBytes)),+      NarInfo.niNarHash = zeroNarHash,+      NarInfo.niNarSize = 200,+      NarInfo.niReferences = [validNarInfoHashKey <> "-hello-1.0"],+      NarInfo.niDeriver = Nothing,+      NarInfo.niSigs = [],+      NarInfo.niCA = Nothing+    }++-- | Rendered wire form of 'validServerNarInfo'.+validServerNarInfoBytes :: BL.ByteString+validServerNarInfoBytes = renderNarInfoBody validServerNarInfo++-- | A narinfo's wire form as a request body.+renderNarInfoBody :: NarInfo.NarInfo -> BL.ByteString+renderNarInfoBody = BL.fromStrict . TE.encodeUtf8 . NarInfo.renderNarInfo++-- | Upload bytes as 'serverNarName', authenticated.+putServerNar :: Server.ServerConfig -> ByteString -> IO WT.SResponse+putServerNar cfg bytes = serverRequest cfg "PUT" ["nar", serverNarName] [serverAuthHeader] (BL.fromStrict bytes)++-- | Upload a narinfo under 'validNarInfoHashKey', authenticated.+putServerNarInfo :: Server.ServerConfig -> NarInfo.NarInfo -> IO WT.SResponse+putServerNarInfo cfg ni = serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] (renderNarInfoBody ni)++-- | Upload a NAR, then a narinfo for it; the narinfo's response, and the+-- status a GET of that narinfo answers afterwards.+putNarThenNarInfo :: Server.ServerConfig -> ByteString -> NarInfo.NarInfo -> IO (WT.SResponse, HTTP.Status)+putNarThenNarInfo cfg bytes ni = do+  _ <- putServerNar cfg bytes+  putResp <- putServerNarInfo cfg ni+  getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""+  pure (putResp, WT.simpleStatus getResp)++-- | Run one test against a fresh server (configurable write key and+-- signing key), removing the store directory afterwards.+withServer :: Maybe ByteString -> Maybe Signing.SecretKey -> (Server.ServerConfig -> IO Bool) -> IO Bool+withServer apiKey sigKey body = do+  tmpDir <- createTestDir+  store <- Store.newFileStore tmpDir+  passed <-+    body+      Server.ServerConfig+        { Server.scStore = store,+          Server.scApiKey = apiKey,+          Server.scSigningKey = sigKey,+          Server.scRootResponse = Server.defaultRootResponse+        }+  removeDirectoryRecursive tmpDir+  pure passed++-- | 'withServer' with write auth armed and no signing - the common case.+withAuthedServer :: (Server.ServerConfig -> IO Bool) -> IO Bool+withAuthedServer = withServer (Just serverTestApiKey) Nothing++-- | Execute a single request against the server.+serverRequest :: Server.ServerConfig -> BS.ByteString -> [Text] -> [HTTP.Header] -> BL.ByteString -> IO WT.SResponse+serverRequest cfg method segments headers body =+  WT.runSession (WT.srequest (WT.SRequest req body)) (Server.cacheApp cfg)+  where+    req =+      defaultRequest+        { requestMethod = method,+          pathInfo = segments,+          requestHeaders = headers+        }++-- | Like 'serverRequest', with a declared Content-Length - for the+-- reject-before-reading limit checks.+serverRequestSized :: Server.ServerConfig -> BS.ByteString -> [Text] -> [HTTP.Header] -> Word -> IO WT.SResponse+serverRequestSized cfg method segments headers declared =+  WT.runSession (WT.srequest (WT.SRequest req "")) (Server.cacheApp cfg)+  where+    req =+      defaultRequest+        { requestMethod = method,+          pathInfo = segments,+          requestHeaders = headers,+          requestBodyLength = KnownLength (fromIntegral declared)+        }++-- | A chunk source yielding the given chunks, then empty (end of input).+chunkSource :: [ByteString] -> IO (IO ByteString)+chunkSource chunks = do+  remaining <- newIORef chunks+  pure $+    atomicModifyIORef' remaining $ \case+      [] -> ([], BS.empty)+      (c : cs) -> (cs, c)++-- | Body bytes as strict ByteString, for infix assertions.+strictBody :: WT.SResponse -> ByteString+strictBody = BL.toStrict . WT.simpleBody++-- | Take every permission from a file, and say whether that left it+-- unreadable.  Windows has no mode bit that stops a read (and a+-- read-only file would then resist the test's cleanup), and root reads+-- a file whatever its mode, so on either there is no read failure to+-- cause.+makeUnreadable :: FilePath -> IO Bool+makeUnreadable path+  | System.Info.os == "mingw32" = pure False+  | otherwise = do+      setPermissions path emptyPermissions+      not . readable <$> getPermissions path++testServer :: IO Bool+testServer =+  runGroup+    "Server"+    [ test "GET / serves the default root response" $+        withServer Nothing Nothing $ \cfg -> do+          resp <- serverRequest cfg "GET" [] [] ""+          ok1 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)+          ok2 <- assertEqual "body" "nova-cache: a Nix binary cache\n" (WT.simpleBody resp)+          pure (ok1 && ok2),+      test "GET / serves a deployment's root page as HTML" $+        withServer Nothing Nothing $ \cfg -> do+          let page = "<!DOCTYPE html><title>cache</title>"+          resp <- serverRequest cfg {Server.scRootResponse = Server.htmlRootResponse page} "GET" [] [] ""+          ok1 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)+          ok2 <- assertEqual "content type" (Just "text/html; charset=utf-8") (lookup HTTP.hContentType (WT.simpleHeaders resp))+          ok3 <- assertEqual "body" page (strictBody resp)+          pure (ok1 && ok2 && ok3),+      -- newTTLCache: within the TTL the action runs once; a zero TTL+      -- never satisfies the freshness check, so every call re-runs.+      test "newTTLCache memoizes within the TTL and re-runs past it" $ do+        counter <- newIORef (0 :: Int)+        let bump = atomicModifyIORef' counter (\n -> (n + 1, n + 1))+        cachedHour <- Server.newTTLCache 3600 bump+        firstRead <- cachedHour+        secondRead <- cachedHour+        cachedNever <- Server.newTTLCache 0 bump+        thirdRead <- cachedNever+        fourthRead <- cachedNever+        ok1 <- assertEqual "memoized" (1, 1) (firstRead, secondRead)+        ok2 <- assertEqual "re-run each call" (2, 3) (thirdRead, fourthRead)+        pure (ok1 && ok2),+      test "GET /nix-cache-info renders cache metadata" $+        withServer Nothing Nothing $ \cfg -> do+          resp <- serverRequest cfg "GET" ["nix-cache-info"] [] ""+          ok1 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)+          ok2 <- assertTrue "StoreDir line" (BS.isInfixOf "StoreDir: /nix/store" (strictBody resp))+          pure (ok1 && ok2),+      test "unknown route is 404" $+        withServer Nothing Nothing $ \cfg -> do+          resp <- serverRequest cfg "GET" ["no", "such", "route"] [] ""+          assertEqual "status" HTTP.status404 (WT.simpleStatus resp),+      -- HEAD is routed exactly like GET (upstream clients probe narinfo+      -- existence with HEAD).+      test "HEAD is served wherever GET is" $+        withServer Nothing Nothing $ \cfg -> do+          okResp <- serverRequest cfg "HEAD" ["nix-cache-info"] [] ""+          missingResp <- serverRequest cfg "HEAD" [validNarInfoHashKey <> ".narinfo"] [] ""+          ok1 <- assertEqual "present" HTTP.status200 (WT.simpleStatus okResp)+          ok2 <- assertEqual "absent" HTTP.status404 (WT.simpleStatus missingResp)+          pure (ok1 && ok2),+      -- Write authentication+      test "PUT narinfo without auth is 401" $+        withAuthedServer $ \cfg -> do+          resp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [] validServerNarInfoBytes+          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),+      test "PUT narinfo with the wrong key is 401" $+        withAuthedServer $ \cfg -> do+          let wrongAuth = (HTTP.hAuthorization, "Bearer not-the-key")+          resp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [wrongAuth] validServerNarInfoBytes+          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),+      test "PUT then GET narinfo roundtrip" $+        withAuthedServer $ \cfg -> do+          _ <- putServerNar cfg serverNarBytes+          putResp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] validServerNarInfoBytes+          getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""+          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)+          ok2 <- assertEqual "GET status" HTTP.status200 (WT.simpleStatus getResp)+          ok3 <- assertTrue "StorePath present" (BS.isInfixOf (TE.encodeUtf8 validNarInfoHashKey) (strictBody getResp))+          ok4 <-+            assertEqual+              "revalidatable, never immutable"+              (Just "public, max-age=3600, must-revalidate")+              (lookup HTTP.hCacheControl (WT.simpleHeaders getResp))+          pure (ok1 && ok2 && ok3 && ok4),+      test "PUT narinfo signs when a key is configured" $ do+        sigKey <- generateTestSecretKey+        withServer (Just serverTestApiKey) (Just sigKey) $ \cfg -> do+          _ <- putServerNar cfg serverNarBytes+          putResp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] validServerNarInfoBytes+          getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""+          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)+          ok2 <- assertTrue "Sig line present" (BS.isInfixOf "Sig: test-key:" (strictBody getResp))+          pure (ok1 && ok2),+      -- The confused-deputy gate: a narinfo describing path X cannot be+      -- stored under path Y's key.+      test "PUT narinfo under a mismatched hash is 400" $+        withAuthedServer $ \cfg -> do+          let otherKey = T.replicate 32 "1" <> ".narinfo"+          resp <- serverRequest cfg "PUT" [otherKey] [serverAuthHeader] validServerNarInfoBytes+          assertEqual "status" HTTP.status400 (WT.simpleStatus resp),+      test "PUT malformed narinfo is 400" $+        withAuthedServer $ \cfg -> do+          resp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] "not a narinfo"+          assertEqual "status" HTTP.status400 (WT.simpleStatus resp),+      test "PUT narinfo with an oversized declared length is 413" $+        withAuthedServer $ \cfg -> do+          resp <-+            serverRequestSized+              cfg+              "PUT"+              [validNarInfoHashKey <> ".narinfo"]+              [serverAuthHeader]+              (fromIntegral Server.maxNarInfoBodySize + 1)+          assertEqual "status" HTTP.status413 (WT.simpleStatus resp),+      -- A narinfo is signed and stored only over the NAR it describes,+      -- so an upload that went wrong fails here and the next push sends+      -- both again.+      test "PUT narinfo with no NAR stored is 400 and stores nothing" $+        withAuthedServer $ \cfg -> do+          putResp <- putServerNarInfo cfg validServerNarInfo+          getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""+          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)+          ok2 <- assertEqual "body" "NAR nar/test.nar is not stored; upload it before its narinfo" (strictBody putResp)+          ok3 <- assertEqual "GET status" HTTP.status404 (WT.simpleStatus getResp)+          pure (ok1 && ok2 && ok3),+      test "PUT narinfo over a short NAR is 400 and stores nothing" $+        withAuthedServer $ \cfg -> do+          (putResp, getStatus) <- putNarThenNarInfo cfg (BS.take 10 serverNarBytes) validServerNarInfo+          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)+          ok2 <- assertEqual "body" "NAR nar/test.nar is 10 bytes, narinfo FileSize is 17" (strictBody putResp)+          ok3 <- assertEqual "GET status" HTTP.status404 getStatus+          pure (ok1 && ok2 && ok3),+      test "PUT narinfo over a same-size NAR of other content is 400 and stores nothing" $+        withAuthedServer $ \cfg -> do+          let other = BS.map (+ 1) serverNarBytes+          (putResp, getStatus) <- putNarThenNarInfo cfg other validServerNarInfo+          let expected =+                "NAR nar/test.nar hashes to "+                  <> TE.encodeUtf8 (Hash.formatNixHash (Hash.hashBytes other))+                  <> ", narinfo FileHash is "+                  <> TE.encodeUtf8 (Hash.formatNixHash (Hash.hashBytes serverNarBytes))+          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)+          ok2 <- assertEqual "body" expected (strictBody putResp)+          ok3 <- assertEqual "GET status" HTTP.status404 getStatus+          pure (ok1 && ok2 && ok3),+      test "PUT narinfo over its NAR is 200 and stores it" $+        withAuthedServer $ \cfg -> do+          (putResp, getStatus) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo+          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)+          ok2 <- assertEqual "GET status" HTTP.status200 getStatus+          pure (ok1 && ok2),+      test "a refused narinfo is accepted once the right NAR is uploaded again" $+        withAuthedServer $ \cfg -> do+          (refused, _) <- putNarThenNarInfo cfg (BS.take 10 serverNarBytes) validServerNarInfo+          (accepted, getStatus) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo+          ok1 <- assertEqual "first PUT status" HTTP.status400 (WT.simpleStatus refused)+          ok2 <- assertEqual "second PUT status" HTTP.status200 (WT.simpleStatus accepted)+          ok3 <- assertEqual "GET status" HTTP.status200 getStatus+          pure (ok1 && ok2 && ok3),+      test "PUT narinfo whose URL is not under nar/ is 400" $+        withAuthedServer $ \cfg -> do+          let elsewhere = validServerNarInfo {NarInfo.niUrl = "https://cache.example/nar/" <> serverNarName}+          (putResp, getStatus) <- putNarThenNarInfo cfg serverNarBytes elsewhere+          ok1 <- assertEqual "PUT status" HTTP.status400 (WT.simpleStatus putResp)+          ok2 <- assertEqual "body" "narinfo URL https://cache.example/nar/test.nar does not name a file under nar/" (strictBody putResp)+          ok3 <- assertEqual "GET status" HTTP.status404 getStatus+          pure (ok1 && ok2 && ok3),+      test "PUT narinfo without FileSize or FileHash is 400" $+        withAuthedServer $ \cfg -> do+          (noSize, _) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo {NarInfo.niFileSize = Nothing}+          (noHash, getStatus) <- putNarThenNarInfo cfg serverNarBytes validServerNarInfo {NarInfo.niFileHash = Nothing}+          ok1 <- assertEqual "no FileSize" (HTTP.status400, "narinfo declares no FileSize to check its NAR against") (WT.simpleStatus noSize, strictBody noSize)+          ok2 <- assertEqual "no FileHash" (HTTP.status400, "narinfo declares no FileHash to check its NAR against") (WT.simpleStatus noHash, strictBody noHash)+          ok3 <- assertEqual "GET status" HTTP.status404 getStatus+          pure (ok1 && ok2 && ok3),+      test "PUT narinfo over an unreadable NAR is 500 and stores nothing" $+        withAuthedServer $ \cfg -> do+          _ <- putServerNar cfg serverNarBytes+          unreadable <- makeUnreadable (Store.fsRoot (Server.scStore cfg) ++ "/nar/" ++ T.unpack serverNarName)+          if not unreadable+            then pure True+            else do+              putResp <- putServerNarInfo cfg validServerNarInfo+              getResp <- serverRequest cfg "GET" [validNarInfoHashKey <> ".narinfo"] [] ""+              ok1 <- assertEqual "PUT status" HTTP.status500 (WT.simpleStatus putResp)+              ok2 <- assertEqual "body" "NAR nar/test.nar could not be read" (strictBody putResp)+              ok3 <- assertEqual "GET status" HTTP.status404 (WT.simpleStatus getResp)+              pure (ok1 && ok2 && ok3),+      -- The hash listing is push-tool plumbing: authenticated, uncacheable.+      test "GET /narinfo-hashes without auth is 401" $+        withAuthedServer $ \cfg -> do+          resp <- serverRequest cfg "GET" ["narinfo-hashes"] [] ""+          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),+      test "GET /narinfo-hashes with auth lists hashes, uncacheable" $+        withAuthedServer $ \cfg -> do+          _ <- putServerNar cfg serverNarBytes+          putResp <- serverRequest cfg "PUT" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] validServerNarInfoBytes+          resp <- serverRequest cfg "GET" ["narinfo-hashes"] [serverAuthHeader] ""+          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)+          ok2 <- assertEqual "status" HTTP.status200 (WT.simpleStatus resp)+          ok3 <- assertTrue "uploaded hash listed" (BS.isInfixOf (TE.encodeUtf8 validNarInfoHashKey) (strictBody resp))+          ok4 <- assertEqual "no-store" (Just "no-store") (lookup HTTP.hCacheControl (WT.simpleHeaders resp))+          pure (ok1 && ok2 && ok3 && ok4),+      test "GET /narinfo-hashes in open mode needs no auth" $+        withServer Nothing Nothing $ \cfg -> do+          resp <- serverRequest cfg "GET" ["narinfo-hashes"] [] ""+          assertEqual "status" HTTP.status200 (WT.simpleStatus resp),+      -- NAR transfer+      test "PUT then GET and HEAD a NAR" $+        withAuthedServer $ \cfg -> do+          putResp <- serverRequest cfg "PUT" ["nar", "test.nar"] [serverAuthHeader] "nar-payload-bytes"+          getResp <- serverRequest cfg "GET" ["nar", "test.nar"] [] ""+          headResp <- serverRequest cfg "HEAD" ["nar", "test.nar"] [] ""+          ok1 <- assertEqual "PUT status" HTTP.status200 (WT.simpleStatus putResp)+          ok2 <- assertEqual "GET status" HTTP.status200 (WT.simpleStatus getResp)+          ok3 <- assertEqual "GET body" "nar-payload-bytes" (WT.simpleBody getResp)+          ok4 <-+            assertEqual+              "immutable content address"+              (Just "public, max-age=31536000, immutable")+              (lookup HTTP.hCacheControl (WT.simpleHeaders getResp))+          ok5 <- assertEqual "HEAD status" HTTP.status200 (WT.simpleStatus headResp)+          pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "PUT NAR without auth is 401" $+        withAuthedServer $ \cfg -> do+          resp <- serverRequest cfg "PUT" ["nar", "test.nar"] [] "nar-payload-bytes"+          assertEqual "status" HTTP.status401 (WT.simpleStatus resp),+      test "PUT NAR with a traversal name is 400" $+        withAuthedServer $ \cfg -> do+          resp <- serverRequest cfg "PUT" ["nar", ".."] [serverAuthHeader] "escape"+          assertEqual "status" HTTP.status400 (WT.simpleStatus resp),+      test "PUT NAR with an oversized declared length is 413" $+        withAuthedServer $ \cfg -> do+          resp <-+            serverRequestSized+              cfg+              "PUT"+              ["nar", "test.nar"]+              [serverAuthHeader]+              (fromIntegral Server.maxNarBodySize + 1)+          assertEqual "status" HTTP.status413 (WT.simpleStatus resp),+      test "GET absent NAR is 404" $+        withServer Nothing Nothing $ \cfg -> do+          resp <- serverRequest cfg "GET" ["nar", "absent.nar"] [] ""+          assertEqual "status" HTTP.status404 (WT.simpleStatus resp),+      -- Streaming write, at the store layer+      test "writeNarStreaming writes chunks and lands atomically" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        source <- chunkSource ["ab", "cd", "ef"]+        result <- Store.writeNarStreaming store "streamed.nar" 16 source+        stored <- Store.readNar store "streamed.nar"+        located <- Store.narFilePath store "streamed.nar"+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "result" Store.NarWriteOk result+        ok2 <- assertEqual "content" (Just "abcdef") stored+        ok3 <- assertTrue "narFilePath resolves" (isJust located)+        pure (ok1 && ok2 && ok3),+      test "writeNarStreaming over the cap deletes the partial file" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        source <- chunkSource ["four", "more", "over"]+        result <- Store.writeNarStreaming store "big.nar" 8 source+        leftovers <- listDirectory (tmpDir ++ "/nar")+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "result" Store.NarWriteTooLarge result+        ok2 <- assertEqual "no partial files" [] leftovers+        pure (ok1 && ok2),+      test "writeNarStreaming rejects a traversal name" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        source <- chunkSource ["x"]+        result <- Store.writeNarStreaming store "../escape" 8 source+        removeDirectoryRecursive tmpDir+        assertEqual "result" Store.NarWriteBadPath result,+      test "narFilePath rejects a traversal name" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        located <- Store.narFilePath store "../escape"+        removeDirectoryRecursive tmpDir+        assertEqual "path" Nothing located+    ]++-- ---------------------------------------------------------------------------+-- Stored NAR check (pure)+-- ---------------------------------------------------------------------------++testStoredNarCheck :: IO Bool+testStoredNarCheck =+  runGroup+    "Stored NAR check"+    [ test "a narinfo URL resolves to the file its GET is routed to" $+        let cases =+              [ ("nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst", Just "1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.zst"),+                ("nar/a%2Bb.nar", Just "a+b.nar"),+                ("nar/x.nar?v=1", Just "x.nar"),+                ("/nar/x.nar", Nothing),+                ("https://cache.example/nar/x.nar", Nothing),+                ("nar/../x.nar", Nothing),+                ("nar/a/x.nar", Nothing),+                ("x.nar", Nothing)+              ]+         in assertEqual+              "mismatches"+              []+              [(url, expected, actual) | (url, expected) <- cases, let actual = Server.narUrlFileName url, actual /= expected],+      test "property: a NAR's request path, as a narinfo URL, resolves to its name" $+        checkProperty $+          forAll (T.pack <$> arbitrary) $ \name ->+            Server.narUrlFileName (relativeRequestPath ["nar", name]) === Just name,+      test "property: a nar/<name> URL claims that name, FileSize and FileHash" $+        checkProperty $+          forAll genClaim $ \claim ->+            Server.narClaim (claimingNarInfo claim) === Right claim,+      test "a narinfo with no FileSize, no usable FileHash or no served URL claims nothing" $ do+        let refusal ni = either Just (const Nothing) (Server.narClaim ni)+            withUrl url = validServerNarInfo {NarInfo.niUrl = url}+        ok1 <- assertEqual "no FileSize" (Just Server.NarFileSizeUndeclared) (refusal validServerNarInfo {NarInfo.niFileSize = Nothing})+        ok2 <- assertEqual "no FileHash" (Just Server.NarFileHashUndeclared) (refusal validServerNarInfo {NarInfo.niFileHash = Nothing})+        ok3 <-+          assertTrue+            "unparseable FileHash"+            ( case refusal validServerNarInfo {NarInfo.niFileHash = Just "sha256:abc"} of+                Just (Server.NarFileHashInvalid "sha256:abc" _) -> True+                _ -> False+            )+        ok4 <-+          assertEqual+            "names no store could hold"+            [Just (Server.NarUrlNotServed url) | url <- unservedUrls]+            (map (refusal . withUrl) unservedUrls)+        pure (ok1 && ok2 && ok3 && ok4),+      test "property: a stored NAR is accepted exactly when it is the declared file" $+        checkProperty $+          forAll genClaim $ \claim ->+            forAll (genStoredNear claim) $ \stored ->+              isRight (Server.judgeStoredNar claim (Just stored))+                === (stored == Server.StoredNar (Server.ncFileSize claim) (Server.ncFileHash claim)),+      test "property: a NAR of another size is refused on its size" $+        checkProperty $+          forAll genClaim $ \claim ->+            forAll (genFileSize `suchThat` (/= Server.ncFileSize claim)) $ \size ->+              forAll genNixHash $ \digest ->+                Server.judgeStoredNar claim (Just (Server.StoredNar size digest))+                  === Left (Server.NarSizeMismatch (Server.ncFileName claim) (Server.ncFileSize claim) size),+      test "property: a same-size NAR of other content is refused on its hash" $+        checkProperty $+          forAll genClaim $ \claim ->+            forAll (genNixHash `suchThat` (/= Server.ncFileHash claim)) $ \digest ->+              Server.judgeStoredNar claim (Just (Server.StoredNar (Server.ncFileSize claim) digest))+                === Left (Server.NarHashMismatch (Server.ncFileName claim) (Server.ncFileHash claim) digest),+      test "property: an absent NAR is refused as not stored" $+        checkProperty $+          forAll genClaim $ \claim ->+            Server.judgeStoredNar claim Nothing === Left (Server.NarNotStored (Server.ncFileName claim))+    ]++-- | URLs that name nothing under @nar\/@ a store could hold: an empty+-- name, an upload's temp-file name, and a Windows device.+unservedUrls :: [Text]+unservedUrls = ["nar/", "nar/.nova123-4.tmp", "nar/nul"]++-- | The request path a client forms for segments, relative to the cache+-- root and percent-encoded as http-types encodes a path.+relativeRequestPath :: [Text] -> Text+relativeRequestPath =+  TE.decodeLatin1 . BL.toStrict . Builder.toLazyByteString . HTTP.encodePathSegmentsRelative++-- | 'validServerNarInfo' declaring a claim's name, FileSize and FileHash.+claimingNarInfo :: Server.NarClaim -> NarInfo.NarInfo+claimingNarInfo claim =+  validServerNarInfo+    { NarInfo.niUrl = "nar/" <> Server.ncFileName claim,+      NarInfo.niFileSize = Just (Server.ncFileSize claim),+      NarInfo.niFileHash = Just (Hash.formatNixHash (Server.ncFileHash claim))+    }++-- | A FileSize the narinfo parser admits: up to the uint64 maximum.+genFileSize :: Gen Integer+genFileSize = choose (0, 18446744073709551615)++genNixHash :: Gen Hash.NixHash+genNixHash = Hash.NixHash . BS.pack <$> vectorOf 32 arbitrary++genClaim :: Gen Server.NarClaim+genClaim = Server.NarClaim <$> genSafeNarName <*> genFileSize <*> genNixHash++-- | A stored NAR that keeps or replaces each of the claim's size and+-- hash independently, so the exact file turns up about a quarter of+-- the time.+genStoredNear :: Server.NarClaim -> Gen Server.StoredNar+genStoredNear claim =+  Server.StoredNar+    <$> oneof [pure (Server.ncFileSize claim), genFileSize]+    <*> oneof [pure (Server.ncFileHash claim), genNixHash]++-- ---------------------------------------------------------------------------+-- Multipart upload tests+-- ---------------------------------------------------------------------------++-- | Begin an upload of a NAR, authenticated: the response, and the id+-- its body carries.+beginServerUpload :: Server.ServerConfig -> Text -> IO (WT.SResponse, Text)+beginServerUpload cfg name = do+  resp <- serverRequest cfg "POST" ["nar", name, "uploads"] [serverAuthHeader] ""+  pure (resp, T.strip (TE.decodeLatin1 (strictBody resp)))++-- | Send a part of an upload, authenticated, by the part number as the+-- path spells it.+putServerPart :: Server.ServerConfig -> Text -> Text -> Text -> ByteString -> IO WT.SResponse+putServerPart cfg name uploadId part bytes =+  serverRequest cfg "PUT" ["nar", name, "uploads", uploadId, part] [serverAuthHeader] (BL.fromStrict bytes)++-- | Complete an upload with a body, authenticated.+completeServerUpload :: Server.ServerConfig -> Text -> Text -> ByteString -> IO WT.SResponse+completeServerUpload cfg name uploadId body =+  serverRequest cfg "POST" ["nar", name, "uploads", uploadId] [serverAuthHeader] (BL.fromStrict body)++-- | A completion body, spelled out as the protocol defines it rather+-- than rendered, so a change to the wire format fails here.+totalsBody :: Int -> Int -> ByteString+totalsBody parts size = TE.encodeUtf8 (T.pack ("Parts: " ++ show parts ++ "\nFileSize: " ++ show size ++ "\n"))++-- | Upload bytes as a NAR, one request per part: the statuses of the+-- begin, of each part and of the completion, in order.+uploadInParts :: Server.ServerConfig -> Text -> [ByteString] -> IO [HTTP.Status]+uploadInParts cfg name parts = do+  (begun, uploadId) <- beginServerUpload cfg name+  sent <- zipWithM (putServerPart cfg name uploadId . T.pack . show) [1 :: Int ..] parts+  completed <- completeServerUpload cfg name uploadId (totalsBody (length parts) (sum (map BS.length parts)))+  pure (map WT.simpleStatus (begun : sent ++ [completed]))++-- | The entries of a store subdirectory, sorted.+storeEntries :: Server.ServerConfig -> FilePath -> IO [FilePath]+storeEntries cfg subdir = sort <$> listDirectory (Store.fsRoot (Server.scStore cfg) ++ "/" ++ subdir)++-- | The files each staged upload's directory holds.+stagedFiles :: Server.ServerConfig -> IO [[FilePath]]+stagedFiles cfg = traverse (fmap sort . listDirectory . Store.suPath) =<< Store.listStagedUploads (Server.scStore cfg)++-- | A response's status and body.+statusAndBody :: WT.SResponse -> (HTTP.Status, ByteString)+statusAndBody resp = (WT.simpleStatus resp, strictBody resp)++-- | Split bytes at each length in turn, the remainder last, so there is+-- always one part more than lengths.+splitAtLengths :: [Int] -> ByteString -> [ByteString]+splitAtLengths [] rest = [rest]+splitAtLengths (len : lens) bytes = case BS.splitAt len bytes of+  (piece, rest) -> piece : splitAtLengths lens rest++testMultipartUpload :: IO Bool+testMultipartUpload =+  runGroup+    "Multipart upload"+    [ test "a NAR uploaded in parts is served back byte for byte, and its narinfo is accepted" $+        withAuthedServer $ \cfg -> do+          statuses <- uploadInParts cfg serverNarName ["nar-pay", "load-by", "tes"]+          getResp <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          narInfoResp <- putServerNarInfo cfg validServerNarInfo+          staged <- storeEntries cfg "uploads"+          ok1 <- assertEqual "statuses" (HTTP.status201 : replicate 4 HTTP.status200) statuses+          ok2 <- assertEqual "GET" (HTTP.status200, serverNarBytes) (statusAndBody getResp)+          ok3 <- assertEqual "narinfo status" HTTP.status200 (WT.simpleStatus narInfoResp)+          ok4 <- assertEqual "uploads left staged" [] staged+          pure (ok1 && ok2 && ok3 && ok4),+      test "beginning an upload answers 201, its id, and its path relative to the request's" $+        withAuthedServer $ \cfg -> do+          (begun, uploadId) <- beginServerUpload cfg serverNarName+          (_, otherId) <- beginServerUpload cfg serverNarName+          ok1 <- assertEqual "status" HTTP.status201 (WT.simpleStatus begun)+          ok2 <- assertTrue "id is 32 lowercase hex digits" (T.length uploadId == 32 && T.all (`elem` lowerHexDigits) uploadId)+          ok3 <- assertEqual "body" (TE.encodeUtf8 uploadId <> "\n") (strictBody begun)+          ok4 <- assertEqual "Location" (Just ("uploads/" <> TE.encodeUtf8 uploadId)) (lookup HTTP.hLocation (WT.simpleHeaders begun))+          ok5 <- assertTrue "each upload has its own id" (uploadId /= otherId)+          pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "an upload in progress leaves nothing under nar/" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" "nar-payload"+          _ <- putServerPart cfg serverNarName uploadId "2" "-bytes"+          refused <- completeServerUpload cfg serverNarName uploadId (totalsBody 3 17)+          nars <- storeEntries cfg "nar"+          getResp <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          narInfoResp <- putServerNarInfo cfg validServerNarInfo+          ok1 <- assertEqual "completion" HTTP.status409 (WT.simpleStatus refused)+          ok2 <- assertEqual "nar/" [] nars+          ok3 <- assertEqual "GET status" HTTP.status404 (WT.simpleStatus getResp)+          ok4 <- assertEqual "narinfo" (HTTP.status400, "NAR nar/test.nar is not stored; upload it before its narinfo") (statusAndBody narInfoResp)+          pure (ok1 && ok2 && ok3 && ok4),+      test "every upload request needs the write key" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          let requests headers =+                [ serverRequest cfg "POST" ["nar", serverNarName, "uploads"] headers "",+                  serverRequest cfg "PUT" ["nar", serverNarName, "uploads", uploadId, "1"] headers (BL.fromStrict serverNarBytes),+                  serverRequest cfg "POST" ["nar", serverNarName, "uploads", uploadId] headers (BL.fromStrict (totalsBody 1 17))+                ]+          missing <- traverse (fmap WT.simpleStatus) (requests [])+          wrong <- traverse (fmap WT.simpleStatus) (requests [(HTTP.hAuthorization, "Bearer not-the-key")])+          staged <- stagedFiles cfg+          nars <- storeEntries cfg "nar"+          ok1 <- assertEqual "without a key" (replicate 3 HTTP.status401) missing+          ok2 <- assertEqual "with the wrong key" (replicate 3 HTTP.status401) wrong+          ok3 <- assertEqual "staged: the one authenticated upload, holding no part" [[]] staged+          ok4 <- assertEqual "nar/" [] nars+          pure (ok1 && ok2 && ok3 && ok4),+      test "a part that would leave a gap is 409, and the upload still completes" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          early <- putServerPart cfg serverNarName uploadId "2" "load-by"+          firstPart <- putServerPart cfg serverNarName uploadId "1" "nar-pay"+          skipped <- putServerPart cfg serverNarName uploadId "3" "tes"+          staged <- stagedFiles cfg+          _ <- putServerPart cfg serverNarName uploadId "2" "load-by"+          _ <- putServerPart cfg serverNarName uploadId "3" "tes"+          completed <- completeServerUpload cfg serverNarName uploadId (totalsBody 3 17)+          getResp <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          ok1 <- assertEqual "part 2 first" (HTTP.status409, "upload of nar/test.nar holds 0 parts, so the next is part 1, not part 2") (statusAndBody early)+          ok2 <- assertEqual "part 1" HTTP.status200 (WT.simpleStatus firstPart)+          ok3 <- assertEqual "part 3 after part 1" (HTTP.status409, "upload of nar/test.nar holds 1 part, so the next is part 2, not part 3") (statusAndBody skipped)+          ok4 <- assertEqual "parts held after the refusals" [["1"]] staged+          ok5 <- assertEqual "completion" HTTP.status200 (WT.simpleStatus completed)+          ok6 <- assertEqual "GET body" serverNarBytes (strictBody getResp)+          pure (ok1 && ok2 && ok3 && ok4 && ok5 && ok6),+      test "a part sent again replaces the one held" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" "a first attempt"+          _ <- putServerPart cfg serverNarName uploadId "2" "-bytes"+          resent <- putServerPart cfg serverNarName uploadId "1" "nar-payload"+          completed <- completeServerUpload cfg serverNarName uploadId (totalsBody 2 17)+          getResp <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          ok1 <- assertEqual "part 1 again" HTTP.status200 (WT.simpleStatus resent)+          ok2 <- assertEqual "completion" HTTP.status200 (WT.simpleStatus completed)+          ok3 <- assertEqual "GET body" serverNarBytes (strictBody getResp)+          pure (ok1 && ok2 && ok3),+      test "a part number other than 1 to 10000 in canonical decimal is 400" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          let spellings = ["0", "01", "10001", "-1", "+1", "1.0", "one", ""]+          responses <- traverse (\part -> putServerPart cfg serverNarName uploadId part "x") spellings+          staged <- stagedFiles cfg+          ok1 <- assertEqual "statuses" (map (const HTTP.status400) spellings) (map WT.simpleStatus responses)+          ok2 <- assertEqual "body" "part number must be a whole number from 1 to 10000" (foldMap strictBody (take 1 responses))+          ok3 <- assertEqual "parts held" [[]] staged+          pure (ok1 && ok2 && ok3),+      test "a part declaring more than the NAR cap is 413 before it is read" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          resp <-+            serverRequestSized+              cfg+              "PUT"+              ["nar", serverNarName, "uploads", uploadId, "1"]+              [serverAuthHeader]+              (fromIntegral Server.maxNarBodySize + 1)+          staged <- stagedFiles cfg+          ok1 <- assertEqual "status" HTTP.status413 (WT.simpleStatus resp)+          ok2 <- assertEqual "parts held" [[]] staged+          pure (ok1 && ok2),+      test "an upload's parts are held to the cap together, and a part over it changes nothing" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        staged <- stageUpload store "capped.nar" ["abc"]+        let write n chunks = maybe (pure Nothing) (\uid -> writeStorePart store "capped.nar" uid n chunks 8) staged+        over <- write 2 ["fgh", "ijk"]+        heldAfterOver <- traverse (fmap sort . listDirectory . Store.suPath) =<< Store.listStagedUploads store+        fits <- write 2 ["defgh"]+        regrown <- write 1 ["abcd"]+        completed <- traverse (\uid -> traverse (Store.completeUpload store "capped.nar" uid 8) (totalsOf 2 8)) staged+        stored <- Store.readNar store "capped.nar"+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "part 2 over the cap" (Just Store.PartTooLarge) over+        ok2 <- assertEqual "files held after it" [["1"]] heldAfterOver+        ok3 <- assertEqual "part 2 within the cap" (Just Store.PartWritten) fits+        ok4 <- assertEqual "part 1 again, past the cap with part 2" (Just Store.PartTooLarge) regrown+        ok5 <- assertEqual "completion" (Just (Just Store.UploadCompleted)) completed+        ok6 <- assertEqual "stored" (Just "abcdefgh") stored+        pure (ok1 && ok2 && ok3 && ok4 && ok5 && ok6),+      test "two part writes at once can pass the cap together, and completion refuses what they join to" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        staged <- stageUpload store "raced.nar" ["a", "b"]+        -- Part 2 is written while part 1 streams, so each finds the other+        -- at one byte and is allowed seven under a cap of eight.+        raced <- for staged $ \uid -> do+          secondWrite <- newIORef Nothing+          rest <- chunkSource ["1111111"]+          let source = do+                pending <- readIORef secondWrite+                case pending of+                  Nothing -> writeIORef secondWrite . Just =<< writeStorePart store "raced.nar" uid 2 ["2222222"] 8+                  Just _ -> pure ()+                rest+          first <- traverse (\part -> Store.writeUploadPart store "raced.nar" uid part 8 source) (Store.parsePartNumber "1")+          second <- readIORef secondWrite+          completed <- traverse (Store.completeUpload store "raced.nar" uid 8) (totalsOf 2 14)+          pure (first, second, completed)+        nars <- listDirectory (tmpDir ++ "/nar")+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "both parts written" (Just (Just Store.PartWritten, Just (Just Store.PartWritten))) (fmap (\(first, second, _) -> (first, second)) raced)+        ok2 <- assertEqual "completion" (Just (Just (Store.CompletionTooLarge 14))) (fmap (\(_, _, completed) -> completed) raced)+        ok3 <- assertEqual "nar/" [] nars+        pure (ok1 && ok2 && ok3),+      test "a join that cannot be renamed into nar/ leaves no copy behind" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        staged <- stageUpload store "blocked.nar" ["nar", "-bytes"]+        createDirectory (tmpDir ++ "/nar/blocked.nar")+        completed <- for staged $ \uid -> traverse (tryIO . Store.completeUpload store "blocked.nar" uid Server.maxNarBodySize) (totalsOf 2 9)+        held <- traverse (fmap sort . listDirectory . Store.suPath) =<< Store.listStagedUploads store+        removeDirectoryRecursive tmpDir+        ok1 <- assertTrue "completion failed" (case completed of Just (Just (Left _)) -> True; _ -> False)+        ok2 <- assertEqual "files held" [["1", "2"]] held+        pure (ok1 && ok2),+      test "a temp file left in an upload is neither joined nor left behind" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        staged <- stageUpload store "tidy.nar" ["abc"]+        traverse_ (\upload -> BS.writeFile (Store.suPath upload ++ "/.nova123-4.tmp") "leftover") =<< Store.listStagedUploads store+        completed <- for staged $ \uid -> traverse (Store.completeUpload store "tidy.nar" uid 8) (totalsOf 1 3)+        stored <- Store.readNar store "tidy.nar"+        remaining <- listDirectory (tmpDir ++ "/uploads")+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "completion" (Just (Just Store.UploadCompleted)) completed+        ok2 <- assertEqual "stored" (Just "abc") stored+        ok3 <- assertEqual "uploads/" [] remaining+        pure (ok1 && ok2 && ok3),+      test "a completion declaring a FileSize over the NAR cap is 413" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" serverNarBytes+          resp <- completeServerUpload cfg serverNarName uploadId (totalsBody 1 (Server.maxNarBodySize + 1))+          nars <- storeEntries cfg "nar"+          ok1 <- assertEqual "refusal" (HTTP.status413, "FileSize 4294967297 is over the 4294967296 bytes a NAR may hold") (statusAndBody resp)+          ok2 <- assertEqual "nar/" [] nars+          pure (ok1 && ok2),+      test "a NAR name too long to stage is 400" $+        withAuthedServer $ \cfg -> do+          -- The staging directory is named by the 32-digit id, a dash and+          -- the NAR's name, 256 long here, past the 255 a file name holds.+          resp <- serverRequest cfg "POST" ["nar", T.replicate 219 "a" <> ".nar", "uploads"] [serverAuthHeader] ""+          staged <- storeEntries cfg "uploads"+          ok1 <- assertEqual "refusal" (HTTP.status400, "invalid path") (statusAndBody resp)+          ok2 <- assertEqual "uploads begun" [] staged+          pure (ok1 && ok2),+      test "a request for an upload that is not staged is 404" $+        withAuthedServer $ \cfg -> do+          (_, otherId) <- beginServerUpload cfg "other.nar"+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" serverNarBytes+          _ <- completeServerUpload cfg serverNarName uploadId (totalsBody 1 17)+          let neverIssued = T.replicate 32 "0"+          responses <-+            sequence+              [ completeServerUpload cfg serverNarName neverIssued (totalsBody 1 17),+                putServerPart cfg serverNarName neverIssued "1" "x",+                completeServerUpload cfg serverNarName "not-an-id" (totalsBody 1 17),+                completeServerUpload cfg serverNarName otherId (totalsBody 1 17),+                completeServerUpload cfg serverNarName uploadId (totalsBody 1 17),+                putServerPart cfg serverNarName uploadId "2" "x"+              ]+          ok1 <- assertEqual "statuses" (map (const HTTP.status404) responses) (map WT.simpleStatus responses)+          ok2 <- assertEqual "body" ("no upload " <> TE.encodeUtf8 neverIssued <> " of nar/test.nar") (foldMap strictBody (take 1 responses))+          pure (ok1 && ok2),+      test "a NAR name the store cannot hold is 400 on every upload request" $+        withAuthedServer $ \cfg -> do+          let uploadId = T.replicate 32 "0"+          responses <-+            sequence+              [ serverRequest cfg "POST" ["nar", "..", "uploads"] [serverAuthHeader] "",+                serverRequest cfg "POST" ["nar", ".nova.tmp1", "uploads"] [serverAuthHeader] "",+                putServerPart cfg "nul" uploadId "1" "x",+                completeServerUpload cfg "a/b" uploadId (totalsBody 1 1)+              ]+          staged <- storeEntries cfg "uploads"+          ok1 <- assertEqual "statuses" (map (const HTTP.status400) responses) (map WT.simpleStatus responses)+          ok2 <- assertEqual "uploads begun" [] staged+          pure (ok1 && ok2),+      test "a completion declaring other parts or another size is 409, and the upload stays" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" "nar-payload"+          _ <- putServerPart cfg serverNarName uploadId "2" "-bytes"+          refusals <- traverse (completeServerUpload cfg serverNarName uploadId) [totalsBody 3 17, totalsBody 1 17, totalsBody 2 18, totalsBody 2 16]+          nars <- storeEntries cfg "nar"+          staged <- stagedFiles cfg+          completed <- completeServerUpload cfg serverNarName uploadId (totalsBody 2 17)+          getResp <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          ok1 <-+            assertEqual+              "refusals"+              [ (HTTP.status409, "upload of nar/test.nar holds 2 parts, Parts is 3"),+                (HTTP.status409, "upload of nar/test.nar holds 2 parts, Parts is 1"),+                (HTTP.status409, "upload of nar/test.nar joins to 17 bytes, FileSize is 18"),+                (HTTP.status409, "upload of nar/test.nar joins to 17 bytes, FileSize is 16")+              ]+              (map statusAndBody refusals)+          ok2 <- assertEqual "nar/" [] nars+          ok3 <- assertEqual "files held, no join left behind" [["1", "2"]] staged+          ok4 <- assertEqual "completion" HTTP.status200 (WT.simpleStatus completed)+          ok5 <- assertEqual "GET body" serverNarBytes (strictBody getResp)+          pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "a completion body other than one Parts line and one FileSize line is refused" $ do+        let refused =+              [ "",+                "Parts: 1\n",+                "FileSize: 3\n",+                "Parts: 1\nFileSize: 3\nFileHash: sha256:00\n",+                "Parts: 1\nParts: 1\nFileSize: 3\n",+                "Parts: 01\nFileSize: 3\n",+                "Parts: 1\nFileSize: 03\n",+                "Parts: 1\nFileSize: -3\n",+                "Parts: 1\nFileSize: 3 \n",+                "Parts: 0\nFileSize: 0\n",+                "Parts: 10001\nFileSize: 3\n",+                "Parts 1\nFileSize: 3\n",+                "Parts: 1\n\nFileSize: 3\n",+                "Parts: 1\r\nFileSize: 3\r\n",+                "Parts: 1\nFileSize: 123456789012345678901\n",+                BS.pack [0x50, 0xff]+              ]+            accepted = ["FileSize: 3\nParts: 1\n", "Parts: 1\nFileSize: 3"]+            expected = map (const (totalsOf 1 3)) accepted+        ok1 <- assertEqual "refused bodies accepted" [] [body | body <- refused, isRight (Server.parseUploadTotals body)]+        ok2 <- assertEqual "accepted" expected (map (either (const Nothing) Just . Server.parseUploadTotals) accepted)+        pure (ok1 && ok2),+      test "a malformed completion body is 400, and one over its cap is 413" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" serverNarBytes+          malformed <- completeServerUpload cfg serverNarName uploadId "Parts: 1\n"+          oversized <- completeServerUpload cfg serverNarName uploadId (BS.replicate (Server.maxUploadTotalsBodySize + 1) 0x20)+          nars <- storeEntries cfg "nar"+          ok1 <- assertEqual "malformed" (HTTP.status400, "completion body has no FileSize line") (statusAndBody malformed)+          ok2 <- assertEqual "oversized" HTTP.status413 (WT.simpleStatus oversized)+          ok3 <- assertEqual "nar/" [] nars+          pure (ok1 && ok2 && ok3),+      test "two uploads of one NAR stage apart" $+        withAuthedServer $ \cfg -> do+          (_, firstId) <- beginServerUpload cfg serverNarName+          (_, secondId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName firstId "1" "first"+          _ <- putServerPart cfg serverNarName secondId "1" "second"+          _ <- completeServerUpload cfg serverNarName firstId (totalsBody 1 5)+          afterFirst <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          _ <- completeServerUpload cfg serverNarName secondId (totalsBody 1 6)+          afterSecond <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+          ok1 <- assertEqual "after the first" "first" (strictBody afterFirst)+          ok2 <- assertEqual "after the second" "second" (strictBody afterSecond)+          pure (ok1 && ok2),+      test "property: a NAR uploaded in parts of any sizes is served back byte for byte" $+        withAuthedServer $ \cfg ->+          checkProperty $+            forAll genSplitBytes $ \(bytes, lengths) ->+              ioProperty $ do+                let parts = splitAtLengths lengths bytes+                statuses <- uploadInParts cfg serverNarName parts+                served <- serverRequest cfg "GET" ["nar", serverNarName] [] ""+                pure ((statuses === HTTP.status201 : replicate (length parts + 1) HTTP.status200) .&&. (strictBody served === bytes)),+      test "property: completion totals render and parse back" $+        checkProperty $+          forAll genUploadTotals $ \totals ->+            Server.parseUploadTotals (Server.renderUploadTotals totals) === Right totals,+      test "property: a part number reads back exactly when it is 1 to 10000 in canonical decimal" $+        checkProperty $+          forAll (choose (-3, Store.maxUploadParts + 3)) $ \n ->+            let spelled = T.pack (show n)+                inRange = n >= 1 && n <= Store.maxUploadParts+             in ((Store.renderPartNumber <$> Store.parsePartNumber spelled) === (spelled <$ guard inRange))+                  .&&. (Store.parsePartNumber ("0" <> spelled) === Nothing),+      test "property: an upload id reads back exactly when it is 32 lowercase hex digits" $+        checkProperty $+          forAll genUploadIdText $ \txt ->+            let wellFormed = T.length txt == 32 && T.all (`elem` lowerHexDigits) txt+             in (Store.renderUploadId <$> Store.parseUploadId txt) === (txt <$ guard wellFormed)+    ]++-- | Bytes, and the lengths of the parts to split them into before the+-- last; a length may run past the bytes, which leaves empty parts.+genSplitBytes :: Gen (ByteString, [Int])+genSplitBytes = do+  bytes <- BS.pack <$> listOf arbitrary+  lengths <- listOf (choose (0, 24))+  pure (bytes, lengths)++-- | 'try' at 'IOException'.+tryIO :: IO a -> IO (Either IOException a)+tryIO = try++-- | Totals over every part count and any size up to the uint64 maximum.+genUploadTotals :: Gen Store.UploadTotals+genUploadTotals =+  Store.UploadTotals+    <$> (T.pack . show <$> choose (1, Store.maxUploadParts)) `suchThatMap` Store.parsePartNumber+    <*> genFileSize++-- | Mostly upload ids, and near misses: a digit short or over, or an+-- uppercase digit.+genUploadIdText :: Gen Text+genUploadIdText =+  oneof+    [ T.pack <$> vectorOf 32 (elements lowerHexDigits),+      T.pack <$> vectorOf 31 (elements lowerHexDigits),+      T.pack <$> vectorOf 33 (elements lowerHexDigits),+      T.pack <$> vectorOf 32 (elements (lowerHexDigits ++ "ABCDEF"))+    ]++-- | 'Store.UploadTotals' of a part count and size; 'Nothing' when the+-- count is not a part number.+totalsOf :: Int -> Integer -> Maybe Store.UploadTotals+totalsOf parts size = (`Store.UploadTotals` size) <$> Store.parsePartNumber (T.pack (show parts))++-- ---------------------------------------------------------------------------+-- Removal tests+-- ---------------------------------------------------------------------------++-- | The hash of a second store path, for an entry a test keeps.+otherHashKey :: Text+otherHashKey = T.replicate 32 "1"++-- | 'validServerNarInfo' for the store path of a hash, describing bytes+-- stored under a NAR's name.+entryNarInfo :: Text -> Text -> ByteString -> NarInfo.NarInfo+entryNarInfo hashKey narName bytes =+  validServerNarInfo+    { NarInfo.niStorePath = "/nix/store/" <> hashKey <> "-entry-1.0",+      NarInfo.niUrl = "nar/" <> narName,+      NarInfo.niFileHash = Just (Hash.formatNixHash (Hash.hashBytes bytes)),+      NarInfo.niFileSize = Just (fromIntegral (BS.length bytes)),+      NarInfo.niReferences = []+    }++-- | Upload a NAR, then a narinfo naming it, authenticated: the status+-- the narinfo is answered with.+putEntry :: Server.ServerConfig -> NarInfo.NarInfo -> Text -> ByteString -> IO HTTP.Status+putEntry cfg ni narName bytes = do+  _ <- serverRequest cfg "PUT" ["nar", narName] [serverAuthHeader] (BL.fromStrict bytes)+  WT.simpleStatus <$> putEntryNarInfo cfg ni++-- | Upload a narinfo under its own store path's hash, authenticated.+putEntryNarInfo :: Server.ServerConfig -> NarInfo.NarInfo -> IO WT.SResponse+putEntryNarInfo cfg ni =+  serverRequest cfg "PUT" [T.take 32 (T.drop (T.length "/nix/store/") (NarInfo.niStorePath ni)) <> ".narinfo"] [serverAuthHeader] (renderNarInfoBody ni)++-- | Store an entry the tests keep, so a store holds a narinfo besides+-- the ones a test removes and a reclaim is not blocked for want of one.+putKeptEntry :: Server.ServerConfig -> IO HTTP.Status+putKeptEntry cfg = putEntry cfg (entryNarInfo otherHashKey "kept.nar" "kept-bytes") "kept.nar" "kept-bytes"++-- | Set a stored NAR's modification time a month back, past the default+-- minimum age.+ageNar :: Server.ServerConfig -> Text -> IO ()+ageNar cfg name = do+  monthAgo <- addUTCTime (negate (30 * nominalDay)) <$> getCurrentTime+  setModificationTime (Store.fsRoot (Server.scStore cfg) ++ "/nar/" ++ T.unpack name) monthAgo++-- | A GET's status and body.+getResource :: Server.ServerConfig -> [Text] -> IO (HTTP.Status, ByteString)+getResource cfg segments = statusAndBody <$> serverRequest cfg "GET" segments [] ""++-- | A reclaim request with a query, authenticated.+reclaimQuery :: Server.ServerConfig -> BS.ByteString -> HTTP.Query -> IO WT.SResponse+reclaimQuery cfg method query =+  WT.runSession (WT.srequest (WT.SRequest req "")) (Server.cacheApp cfg)+  where+    req =+      defaultRequest+        { requestMethod = method,+          pathInfo = ["reclaim"],+          requestHeaders = [serverAuthHeader],+          queryString = query+        }++-- | The report a reclaim at the default minimum age answers with,+-- spelled out, so a change to its form fails here.+reclaimReport :: [ByteString] -> ByteString+reclaimReport body = BS.concat (map (<> "\n") ("minimum age: 336 hours" : body))++-- | How a thread stands once it has run as far as it can.+data Settled+  = -- | Waiting in STM, which in these tests is a store's gate.+    AtGate+  | Finished+  | -- | Waiting on an MVar, which nothing in these tests releases.+    BlockedElsewhere+  deriving (Eq, Show)++-- | Yield to a thread until it finishes or waits, and say which.  This+-- suite runs on the single-threaded runtime, where a thread that has+-- neither finished nor blocked is runnable and runs while this one+-- yields, so the loop waits on the thread's own progress and no clock.+settle :: ThreadId -> IO Settled+settle thread = do+  status <- threadStatus thread+  case status of+    ThreadFinished -> pure Finished+    ThreadDied -> pure Finished+    ThreadBlocked BlockedOnSTM -> pure AtGate+    ThreadBlocked BlockedOnMVar -> pure BlockedElsewhere+    ThreadBlocked _ -> yield >> settle thread+    ThreadRunning -> yield >> settle thread++-- | Run an action in a thread of its own: the thread, and where its+-- outcome arrives.+spawn :: IO a -> IO (ThreadId, MVar (Either SomeException a))+spawn action = do+  done <- newEmptyMVar+  thread <- forkFinally action (putMVar done)+  pure (thread, done)++-- | An outcome with its exception, if any, as text.+described :: Either SomeException a -> Either String a+described = either (Left . displayException) Right++-- | Run a reclaim of a server's store at the default minimum age and+-- stop it once it has listed the store, then run @during@ in a thread of+-- its own, and let the reclaim go on once @during@ has run as far as it+-- can: how @during@ stood then, and both outcomes.  The reclaim reads+-- its clock after every listing and before any removal, inside its hold+-- on the gate, so the clock is where it stops: between the listing and+-- the removals, the window no narinfo and no NAR may enter.+duringReclaim :: Server.ServerConfig -> IO a -> IO (Settled, Either String (Either Reclaim.ReclaimBlocked Reclaim.Reclaimed), Either String a)+duringReclaim cfg during = do+  now <- getCurrentTime+  listed <- newEmptyMVar+  resume <- newEmptyMVar+  let clock = putMVar listed () >> takeMVar resume >> pure now+  (_, reclaimed) <- spawn (Reclaim.reclaim (Server.scStore cfg) Reclaim.defaultMinimumAge clock)+  takeMVar listed+  (thread, outcome) <- spawn during+  standing <- settle thread+  putMVar resume ()+  reclaimOutcome <- described <$> takeMVar reclaimed+  duringOutcome <- described <$> takeMVar outcome+  pure (standing, reclaimOutcome, duringOutcome)++-- | The NARs a reclaim removed, by name, with each outcome.+removedNars :: Either String (Either Reclaim.ReclaimBlocked Reclaim.Reclaimed) -> Either String (Either Reclaim.ReclaimBlocked [(Text, Store.NarRemoveResult)])+removedNars = fmap (fmap (\done -> [(Store.nfName nar, outcome) | (nar, outcome) <- Reclaim.rdNars done]))++testRemoval :: IO Bool+testRemoval =+  runGroup+    "Removal"+    [ test "DELETE of a narinfo needs the write key, removes it, and leaves its NAR served" $+        withAuthedServer $ \cfg -> do+          stored <- putEntry cfg validServerNarInfo serverNarName serverNarBytes+          let narInfoPath = [validNarInfoHashKey <> ".narinfo"]+          unauthorized <- serverRequest cfg "DELETE" narInfoPath [] ""+          kept <- fst <$> getResource cfg narInfoPath+          deleted <- serverRequest cfg "DELETE" narInfoPath [serverAuthHeader] ""+          gone <- fst <$> getResource cfg narInfoPath+          again <- serverRequest cfg "DELETE" narInfoPath [serverAuthHeader] ""+          traversal <- serverRequest cfg "DELETE" ["...narinfo"] [serverAuthHeader] ""+          nar <- getResource cfg ["nar", serverNarName]+          ok1 <- assertEqual "stored" HTTP.status200 stored+          ok2 <- assertEqual "without the key" (HTTP.status401, HTTP.status200) (WT.simpleStatus unauthorized, kept)+          ok3 <- assertEqual "deleted" (HTTP.status200, HTTP.status404) (WT.simpleStatus deleted, gone)+          ok4 <- assertEqual "deleted again" HTTP.status404 (WT.simpleStatus again)+          ok5 <- assertEqual "traversal" HTTP.status400 (WT.simpleStatus traversal)+          ok6 <- assertEqual "NAR" (HTTP.status200, serverNarBytes) nar+          pure (ok1 && ok2 && ok3 && ok4 && ok5 && ok6),+      test "an entry deleted, GET /reclaim lists its NAR, and POST /reclaim removes it" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          stored <- putEntry cfg validServerNarInfo serverNarName serverNarBytes+          ageNar cfg serverNarName+          ageNar cfg "kept.nar"+          _ <- serverRequest cfg "DELETE" [validNarInfoHashKey <> ".narinfo"] [serverAuthHeader] ""+          listed <- reclaimQuery cfg "GET" []+          afterListing <- fst <$> getResource cfg ["nar", serverNarName]+          removed <- reclaimQuery cfg "POST" []+          afterRemoval <- fst <$> getResource cfg ["nar", serverNarName]+          kept <- fst <$> getResource cfg ["nar", "kept.nar"]+          ok1 <- assertEqual "stored" HTTP.status200 stored+          ok2 <-+            assertEqual+              "listed"+              ( HTTP.status200,+                reclaimReport+                  [ "unreferenced: nar/test.nar (17 bytes)",+                    "eligible: 1 NAR, 17 bytes; 0 abandoned uploads, 0 bytes",+                    "kept, younger than the minimum age: 0 NARs, 0 uploads"+                  ]+              )+              (statusAndBody listed)+          ok3 <- assertEqual "served after the listing" HTTP.status200 afterListing+          ok4 <-+            assertEqual+              "removed"+              ( HTTP.status200,+                reclaimReport+                  [ "removed: nar/test.nar (17 bytes)",+                    "reclaimed: 1 NAR, 17 bytes; 0 abandoned uploads, 0 bytes",+                    "kept, younger than the minimum age: 0 NARs, 0 uploads"+                  ]+              )+              (statusAndBody removed)+          ok5 <- assertEqual "after the removal" (HTTP.status404, HTTP.status200) (afterRemoval, kept)+          ok6 <- assertEqual "uncacheable" [Just "no-store", Just "no-store"] (map (lookup HTTP.hCacheControl . WT.simpleHeaders) [listed, removed])+          pure (ok1 && ok2 && ok3 && ok4 && ok5 && ok6),+      test "GET and POST /reclaim need the write key" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          _ <- putServerNar cfg serverNarBytes+          ageNar cfg serverNarName+          responses <- traverse (\method -> serverRequest cfg method ["reclaim"] [] "") ["GET", "POST"]+          nar <- fst <$> getResource cfg ["nar", serverNarName]+          ok1 <- assertEqual "statuses" [HTTP.status401, HTTP.status401] (map WT.simpleStatus responses)+          ok2 <- assertEqual "NAR" HTTP.status200 nar+          pure (ok1 && ok2),+      test "min-age-hours sets a reclaim's minimum age, and any other query is 400" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          _ <- putServerNar cfg serverNarBytes+          byDefault <- statusAndBody <$> reclaimQuery cfg "POST" []+          refusals <-+            traverse+              (fmap WT.simpleStatus . reclaimQuery cfg "POST")+              [ [("min-age-hours", Just "01")],+                [("min-age-hours", Just "-1")],+                [("min-age-hours", Just "1.5")],+                [("min-age-hours", Nothing)],+                [("min-age", Just "0")],+                [("min-age-hours", Just "0"), ("min-age-hours", Just "0")],+                [("min-age-hours", Just "0"), ("delete", Nothing)]+              ]+          survived <- fst <$> getResource cfg ["nar", serverNarName]+          atZero <- statusAndBody <$> reclaimQuery cfg "POST" [("min-age-hours", Just "0")]+          gone <- fst <$> getResource cfg ["nar", serverNarName]+          ok1 <-+            assertEqual+              "at the default age"+              ( HTTP.status200,+                reclaimReport+                  [ "reclaimed: 0 NARs, 0 bytes; 0 abandoned uploads, 0 bytes",+                    "kept, younger than the minimum age: 1 NAR, 0 uploads"+                  ]+              )+              byDefault+          ok2 <- assertEqual "refusals" (map (const HTTP.status400) refusals) refusals+          ok3 <- assertEqual "kept through the refusals" HTTP.status200 survived+          ok4 <-+            assertEqual+              "at zero"+              ( HTTP.status200,+                "minimum age: 0 hours\nremoved: nar/test.nar (17 bytes)\nreclaimed: 1 NAR, 17 bytes; 0 abandoned uploads, 0 bytes\nkept, younger than the minimum age: 0 NARs, 0 uploads\n"+              )+              atZero+          ok5 <- assertEqual "removed at zero" HTTP.status404 gone+          pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "property: min-age-hours reads back as that many hours" $+        checkProperty $+          forAll (choose (0, 1000000 :: Integer)) $ \hours ->+            Server.parseMinimumAge [("min-age-hours", Just (TE.encodeUtf8 (T.pack (show hours))))]+              === Right (fromInteger hours * 3600)+              .&&. Server.parseMinimumAge [] === Right Reclaim.defaultMinimumAge,+      test "a narinfo that cannot be resolved blocks a reclaim with 409, which removes nothing" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          _ <- putServerNar cfg serverNarBytes+          ageNar cfg serverNarName+          _ <- Store.writeNarInfo (Server.scStore cfg) "broken" "not a narinfo\n"+          responses <- traverse (\method -> reclaimQuery cfg method []) ["GET", "POST"]+          nar <- fst <$> getResource cfg ["nar", serverNarName]+          ok1 <- assertEqual "statuses" [HTTP.status409, HTTP.status409] (map WT.simpleStatus responses)+          ok2 <- assertTrue "names the narinfo" (all (BS.isPrefixOf "unresolved: broken.narinfo: " . strictBody) responses)+          ok3 <- assertEqual "NAR" HTTP.status200 nar+          pure (ok1 && ok2 && ok3),+      test "POST /reclaim removes an upload idle for the minimum age, named by its request path" $+        withAuthedServer $ \cfg -> do+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" "abandoned"+          monthAgo <- addUTCTime (negate (30 * nominalDay)) <$> getCurrentTime+          traverse_ (backdateUpload monthAgo) =<< Store.listStagedUploads (Server.scStore cfg)+          removed <- reclaimQuery cfg "POST" []+          staged <- storeEntries cfg "uploads"+          completion <- completeServerUpload cfg serverNarName uploadId (totalsBody 1 9)+          ok1 <-+            assertEqual+              "removed"+              ( HTTP.status200,+                reclaimReport+                  [ "removed: nar/test.nar/uploads/" <> TE.encodeUtf8 uploadId <> " (9 bytes)",+                    "reclaimed: 0 NARs, 0 bytes; 1 abandoned upload, 9 bytes",+                    "kept, younger than the minimum age: 0 NARs, 0 uploads"+                  ]+              )+              (statusAndBody removed)+          ok2 <- assertEqual "uploads/" [] staged+          ok3 <- assertEqual "completion" HTTP.status404 (WT.simpleStatus completion)+          pure (ok1 && ok2 && ok3),+      -- The server checks a narinfo's file fields against the file it+      -- names, never the encoding, so the bytes stand in for a zstd+      -- frame.+      test "a narinfo over an entry replaces its NAR with a smaller encoding, and a reclaim removes the old one alone" $+        withAuthedServer $ \cfg -> do+          let uncompressed = BS.replicate 64 0x61+              compressed = "zstd-frame"+              replacement = (entryNarInfo validNarInfoHashKey "small.nar.zst" compressed) {NarInfo.niCompression = "zstd"}+          stored <- putEntry cfg (entryNarInfo validNarInfoHashKey "big.nar" uncompressed) "big.nar" uncompressed+          ageNar cfg "big.nar"+          replaced <- putEntry cfg replacement "small.nar.zst" compressed+          narInfo <- getResource cfg [validNarInfoHashKey <> ".narinfo"]+          oldBefore <- fst <$> getResource cfg ["nar", "big.nar"]+          removed <- reclaimQuery cfg "POST" []+          oldAfter <- fst <$> getResource cfg ["nar", "big.nar"]+          newAfter <- getResource cfg ["nar", "small.nar.zst"]+          ok1 <- assertEqual "stored, then replaced" (HTTP.status200, HTTP.status200) (stored, replaced)+          ok2 <- assertTrue "narinfo names the smaller NAR" (all (`BS.isInfixOf` snd narInfo) ["URL: nar/small.nar.zst\n", "Compression: zstd\n", "FileSize: 10\n"])+          ok3 <- assertEqual "old NAR served until the reclaim" HTTP.status200 oldBefore+          ok4 <-+            assertEqual+              "removed"+              ( HTTP.status200,+                reclaimReport+                  [ "removed: nar/big.nar (64 bytes)",+                    "reclaimed: 1 NAR, 64 bytes; 0 abandoned uploads, 0 bytes",+                    "kept, younger than the minimum age: 0 NARs, 0 uploads"+                  ]+              )+              (statusAndBody removed)+          ok5 <- assertEqual "after the reclaim" (HTTP.status404, (HTTP.status200, compressed)) (oldAfter, newAfter)+          pure (ok1 && ok2 && ok3 && ok4 && ok5),+      -- A narinfo and a reclaim at once+      test "a narinfo stored before a reclaim keeps its NAR, however old" $+        withAuthedServer $ \cfg -> do+          stored <- putEntry cfg validServerNarInfo serverNarName serverNarBytes+          ageNar cfg serverNarName+          reclaimed <- Reclaim.reclaim (Server.scStore cfg) Reclaim.defaultMinimumAge getCurrentTime+          nar <- fst <$> getResource cfg ["nar", serverNarName]+          ok1 <- assertEqual "stored" HTTP.status200 stored+          ok2 <- assertEqual "removed" (Right (Right [])) (removedNars (Right reclaimed))+          ok3 <- assertEqual "NAR" HTTP.status200 nar+          pure (ok1 && ok2 && ok3),+      test "a narinfo sent during a reclaim waits for it, and is refused once its NAR is gone" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          _ <- putServerNar cfg serverNarBytes+          ageNar cfg serverNarName+          (standing, reclaimed, put) <- duringReclaim cfg (putServerNarInfo cfg validServerNarInfo)+          narInfo <- fst <$> getResource cfg [validNarInfoHashKey <> ".narinfo"]+          ok1 <- assertEqual "the narinfo, during the reclaim" AtGate standing+          ok2 <- assertEqual "removed" (Right (Right [(serverNarName, Store.NarRemoved)])) (removedNars reclaimed)+          ok3 <- assertEqual "narinfo" (Right (HTTP.status400, "NAR nar/test.nar is not stored; upload it before its narinfo")) (statusAndBody <$> put)+          ok4 <- assertEqual "narinfo stored" HTTP.status404 narInfo+          pure (ok1 && ok2 && ok3 && ok4),+      -- The signer runs after the narinfo upload has checked its NAR and+      -- before it writes the narinfo, so this one stops there.+      test "a reclaim started between a narinfo upload's check and its write waits, and keeps the NAR" $+        withAuthedServer $ \cfg -> case Server.narClaim validServerNarInfo of+          Left refusal -> assertEqual "claim" Nothing (Just refusal)+          Right claim -> do+            let store = Server.scStore cfg+            _ <- putKeptEntry cfg+            _ <- putServerNar cfg serverNarBytes+            ageNar cfg serverNarName+            checked <- newEmptyMVar+            resume <- newEmptyMVar+            let signer ni = putMVar checked () >> takeMVar resume >> Server.signNarInfo Nothing ni+            (_, written) <- spawn (Server.storeNarInfo store signer validNarInfoHashKey validServerNarInfo claim)+            takeMVar checked+            (reclaiming, reclaimed) <- spawn (Reclaim.reclaim store Reclaim.defaultMinimumAge getCurrentTime)+            standing <- settle reclaiming+            putMVar resume ()+            writeOutcome <- described <$> takeMVar written+            reclaimOutcome <- described <$> takeMVar reclaimed+            nar <- fst <$> getResource cfg ["nar", serverNarName]+            narInfo <- fst <$> getResource cfg [validNarInfoHashKey <> ".narinfo"]+            ok1 <- assertEqual "the reclaim, during the upload" AtGate standing+            ok2 <- assertEqual "upload" (Right Server.NarInfoWritten) writeOutcome+            ok3 <- assertEqual "removed" (Right (Right [])) (removedNars reclaimOutcome)+            ok4 <- assertEqual "NAR and narinfo" (HTTP.status200, HTTP.status200) (nar, narInfo)+            pure (ok1 && ok2 && ok3 && ok4),+      -- A NAR and a reclaim at once+      test "a NAR uploaded during a reclaim lands after it, so the reclaim never removes it" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          _ <- putServerNar cfg "an earlier upload"+          ageNar cfg serverNarName+          (standing, reclaimed, put) <- duringReclaim cfg (putServerNar cfg serverNarBytes)+          nar <- getResource cfg ["nar", serverNarName]+          narInfo <- WT.simpleStatus <$> putServerNarInfo cfg validServerNarInfo+          ok1 <- assertEqual "the upload, during the reclaim" AtGate standing+          ok2 <- assertEqual "removed: the earlier upload" (Right (Right [(serverNarName, Store.NarRemoved)])) (removedNars reclaimed)+          ok3 <- assertEqual "upload" (Right HTTP.status200) (WT.simpleStatus <$> put)+          ok4 <- assertEqual "NAR" (HTTP.status200, serverNarBytes) nar+          ok5 <- assertEqual "narinfo" HTTP.status200 narInfo+          pure (ok1 && ok2 && ok3 && ok4 && ok5),+      test "a multipart upload completed during a reclaim lands after it" $+        withAuthedServer $ \cfg -> do+          _ <- putKeptEntry cfg+          _ <- putServerNar cfg "an earlier upload"+          ageNar cfg serverNarName+          (_, uploadId) <- beginServerUpload cfg serverNarName+          _ <- putServerPart cfg serverNarName uploadId "1" "nar-payload"+          _ <- putServerPart cfg serverNarName uploadId "2" "-bytes"+          (standing, reclaimed, completed) <- duringReclaim cfg (completeServerUpload cfg serverNarName uploadId (totalsBody 2 17))+          nar <- getResource cfg ["nar", serverNarName]+          ok1 <- assertEqual "the completion, during the reclaim" AtGate standing+          ok2 <- assertEqual "removed: the earlier upload" (Right (Right [(serverNarName, Store.NarRemoved)])) (removedNars reclaimed)+          ok3 <- assertEqual "completion" (Right HTTP.status200) (WT.simpleStatus <$> completed)+          ok4 <- assertEqual "NAR" (HTTP.status200, serverNarBytes) nar+          pure (ok1 && ok2 && ok3 && ok4),+      -- The gate itself+      test "a removal waits for the gate's holders, and holds new ones off until it is done" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        events <- newIORef []+        let record event = atomicModifyIORef' events (\done -> (done ++ [event], ()))+        (removalStanding, removalDone, holderStanding, holderDone) <- Store.withoutRemoval store $ do+          (removal, removalDone) <- spawn (Store.withRemoval store (record "removal"))+          removalStanding <- settle removal+          (holder, holderDone) <- spawn (Store.withoutRemoval store (record "new holder"))+          holderStanding <- settle holder+          record "first holder"+          pure (removalStanding, removalDone, holderStanding, holderDone)+        outcomes <- traverse (fmap described . takeMVar) [removalDone, holderDone]+        order <- readIORef events+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "standing, while the first holder holds" (AtGate, AtGate) (removalStanding, holderStanding)+        ok2 <- assertEqual "outcomes" [Right (), Right ()] outcomes+        ok3 <- assertEqual "order" ["first holder", "removal", "new holder" :: String] order+        pure (ok1 && ok2 && ok3),+      test "a removal interrupted while it waits withdraws, and new holders go ahead" $ do+        tmpDir <- createTestDir+        store <- Store.newFileStore tmpDir+        (removalStanding, interrupted, holderStanding) <- Store.withoutRemoval store $ do+          (removal, removalDone) <- spawn (Store.withRemoval store (pure ()))+          removalStanding <- settle removal+          killThread removal+          outcome <- takeMVar removalDone+          (holder, _) <- spawn (Store.withoutRemoval store (pure ()))+          holderStanding <- settle holder+          pure (removalStanding, either fromException (const Nothing) outcome, holderStanding)+        removeDirectoryRecursive tmpDir+        ok1 <- assertEqual "removal, waiting" AtGate removalStanding+        ok2 <- assertEqual "removal, interrupted" (Just ThreadKilled) interrupted+        ok3 <- assertEqual "new holder" Finished holderStanding+        pure (ok1 && ok2 && ok3)+    ]  -- --------------------------------------------------------------------------- -- Helpers