nova-nix-0.10.0.0: src/Nix/Substituter.hs
-- | Binary substituter - download pre-built paths from remote caches.
--
-- == How substitution works
--
-- Before building a derivation, Nix checks if the output already exists
-- in a binary cache. The protocol:
--
-- 1. Compute the output store path hash from the derivation
-- 2. @GET https:\/\/cache.example.com\/\<hash\>.narinfo@
-- 3. If 200: parse the narinfo (NAR hash, size, references, signature)
-- 4. Validate the narinfo's fields, then verify the signature against a
-- trusted public key
-- 5. @GET https:\/\/cache.example.com\/nar\/\<narhash\>.nar.xz@ and, in
-- one bounded streaming pass, decompress, hash, parse, and unpack
-- into the store path
-- 6. Verify the declared NAR hash and size against the streamed bytes,
-- then re-verify the materialized tree from disk
-- 7. Register in the store DB with references from narinfo
--
-- A cache that lacks the path (404, or another status upstream reads as
-- the file not being there) is passed over, and what no cache holds
-- builds locally. A cache that fails is told apart from one that lacks
-- the path, as upstream tells them apart: a failed narinfo query and a
-- substitute that fails to arrive stop the build, unless
-- 'Nix.Realise.BuildFromSource' (upstream's @fallback@) says to build
-- instead ("Nix.Realise").
--
-- The whole sequence runs under an exclusive per-path lock
-- ("Nix.Store.Lock"), upstream's pathlocks protocol: taken before any
-- deletion or download, validity re-checked under it (another process's
-- finished path is adopted without touching disk), and held until the
-- caller's registration transaction commits.
--
-- == Cache priority
--
-- Multiple caches can be configured, checked in priority order:
--
-- @
-- substituters = https:\/\/cache.novavero.ai https:\/\/cache.nixos.org
-- trusted-public-keys = cache.novavero.ai-1:... cache.nixos.org-1:...
-- @
--
-- Our nova-cache server implements this protocol. The narinfo format,
-- NAR serialization, signature verification - all handled by the
-- @nova-cache@ library. This module orchestrates the HTTP requests
-- and store registration.
module Nix.Substituter
( -- * Substitution
SubstResult (..),
trySubstitute,
trySubstituteWith,
queryReferencesWith,
-- * Cache configuration
CacheConfig (..),
defaultCacheConfig,
-- * Failure classification
AttemptFailure (..),
attemptFailureMessage,
catchSync,
compressedBodyCeiling,
downloadCapFor,
-- * Verified NAR bytes
VerifiedNar,
verifyNar,
verifiedNarBytes,
verifiedNarDigest,
-- * Pure helpers (exported for testing)
maxNarInfoBody,
readBodyCapped,
sortCaches,
tryCachesWith,
validateNarInfoFields,
NarInfoAnswer (..),
judgeNarInfo,
verifySigs,
verifyNarHash,
verifyNarSize,
narInfoMatchesPath,
decompressorFor,
decompressNar,
streamingDecompressionSupported,
withDecompressedSource,
cappedBodySource,
materializeNarFromSource,
consumeNarStream,
unpackAndVerify,
clearStaleDestination,
parseReferences,
parseDeriver,
)
where
import Control.Applicative ((<|>))
import Control.Exception (Exception, Handler (..), SomeException, catch, catches, onException, throwIO)
import Control.Monad (void)
import qualified Data.ByteString as BS
import Data.IORef (newIORef, readIORef, writeIORef)
import Data.List (sortBy)
import Data.Ord (comparing)
import Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import qualified Data.Text.IO as TIO
import Data.Word (Word64)
import qualified Network.HTTP.Client as HTTP
import qualified Network.HTTP.Client.TLS as HTTPS
import qualified Network.HTTP.Types.Status as HTTP
import Nix.Compression (NarCompression (..), parseNarCompression)
import Nix.Http (AttemptFailure (..), FetchRetryPolicy, RetryEffects, TransferError (..), attemptFailureMessage, catchSync, defaultFetchRetryPolicy, fetchStatusFailure, ioRetryEffects, retryTransient, statusError, transferFailureHandlers, withTransfer, withUserAgent)
import Nix.Realise (Fallback (..), Offer (..))
import Nix.Store (CaseSensitivity, NarStreamFailure (..), PathLock, Store (..), abortNarUnpack, acquirePathLock, finishNarUnpack, isValid, newNarUnpackSink, releasePathLock, setReadOnly, sinkNarStream, unpackNarEntry, volumeCaseHack)
import Nix.Store.DB (PathRegistration (..))
import qualified Nix.Store.ExecBit as ExecBit
import Nix.Store.Path (StoreDir, StorePath (spHash, spName), parseStorePathBaseName, storePathHashLen, storePathToFilePath)
import qualified NovaCache.Bzip2 as Bzip2
import qualified NovaCache.Hash as Hash
import qualified NovaCache.NAR as NAR
import qualified NovaCache.NarInfo as NarInfo
import qualified NovaCache.Signing as Signing
import qualified NovaCache.Validate as Validate
import qualified NovaCache.Xz as Xz
import qualified NovaCache.Zstd as Zstd
import qualified System.Directory as Dir
import System.IO (stderr)
-- ---------------------------------------------------------------------------
-- Types
-- ---------------------------------------------------------------------------
-- | Configuration for a binary cache.
data CacheConfig = CacheConfig
{ -- | Base URL of the cache (e.g. @https:\/\/cache.novavero.ai@).
ccUrl :: !Text,
-- | Trusted public keys (@name:base64key@). A narinfo is accepted
-- when ANY of them verifies a signature, matching upstream's flat
-- @trusted-public-keys@ set: the keys are not bound to a particular
-- substituter, and one signed by any trusted key is enough.
ccPublicKeys :: ![Text],
-- | Priority (lower = checked first). cache.nixos.org is 40.
ccPriority :: !Int
}
deriving (Eq, Show)
-- | Default cache configuration for cache.nixos.org.
defaultCacheConfig :: CacheConfig
defaultCacheConfig =
CacheConfig
{ ccUrl = "https://cache.nixos.org",
ccPublicKeys = ["cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="],
ccPriority = 40
}
-- | Result of a substitution attempt.
data SubstResult
= -- | Verified and unpacked on disk, NOT yet registered: the carried
-- registration is recorded by the caller, which batches every output
-- of a derivation into one 'Nix.Store.DB.registerPaths' transaction so
-- cross-output reference edges are never dropped. The path's lock
-- rides along STILL HELD, because the exclusion must survive until
-- that transaction commits - released earlier, another process could
-- meet the unpacked-but-unregistered window and delete the tree the
-- row is about to describe. The caller releases it after
-- registration, on every exit path ("Nix.Builder").
SubstSuccess !PathRegistration !PathLock
| -- | The path was already valid under its lock - another process
-- registered it while this one waited - so its work is adopted:
-- nothing was downloaded, nothing touched disk, and no registration
-- is needed. The caller treats this as success. No lock rides
-- along; it was released before returning.
SubstAlreadyValid
| -- | No cache supplied the path, and none failed to. A cache without
-- its narinfo counts, as do one whose narinfo names another path or
-- carries no trusted signature, and one whose NAR is gone: upstream
-- passes over all of them without failing the substitution.
SubstNotFound
| -- | A cache failed to deliver the path and no cache supplied it, with
-- the first failure: the narinfo query, the download, or its
-- verification. Upstream's failed substitution.
SubstError !Text
deriving (Eq, Show)
-- ---------------------------------------------------------------------------
-- Main substitution logic
-- ---------------------------------------------------------------------------
-- | Try to substitute a store path from configured caches.
--
-- Upstream's per-path substitution protocol: take the path's exclusive
-- lock FIRST, before any deletion or download, and re-check validity
-- under it - another process may have registered the path while this
-- one waited, and its finished work must be adopted
-- ('SubstAlreadyValid') rather than deleted and redone. The lock then
-- holds across the delete, the download, materialization, and the
-- on-disk recheck; a successful result carries it still held (see
-- 'SubstSuccess'), and every other exit releases it here.
--
-- Caches are checked in priority order; the first success stops the
-- scan and a failing cache falls through to the remaining ones. On
-- success the path is unpacked and read-only on disk but NOT
-- registered - the caller records the returned t'PathRegistration'.
-- Every narinfo and NAR request runs under upstream's retry policy,
-- 'defaultFetchRetryPolicy'.
trySubstitute :: Store -> [CacheConfig] -> StorePath -> IO SubstResult
trySubstitute = trySubstituteWith defaultFetchRetryPolicy ioRetryEffects
-- | 'trySubstitute' under a given retry policy and effects: a build
-- passes the policy its @builtin:fetchurl@ downloads run under
-- ("Nix.Builder"), and a test passes effects that record rather than
-- sleep.
trySubstituteWith :: FetchRetryPolicy -> RetryEffects IO -> Store -> [CacheConfig] -> StorePath -> IO SubstResult
trySubstituteWith _ _ _ [] _ = pure SubstNotFound
trySubstituteWith policy effects store caches sp = do
-- Reuse the process-global TLS manager (connection pooling / keep-alive)
-- rather than creating a fresh one per call and per output.
manager <- HTTPS.getGlobalManager
lock <- acquirePathLock (stDir store) sp
substituteLocked manager lock `onException` releasePathLock lock
where
substituteLocked manager lock = do
valid <- isValid store sp
if valid
then do
releasePathLock lock
pure SubstAlreadyValid
else do
result <- tryCachesWith (\cache -> tryOneCache (Retrying policy effects) manager store cache sp lock) (sortCaches caches)
case result of
SubstSuccess _ _ -> pure result
other -> do
releasePathLock lock
pure other
-- | What the caches say about a path's references, from the first cache
-- whose narinfo may be used. References arrive before the path that names
-- them ("Nix.Realise"), so this runs ahead of any download, and a narinfo
-- passes the same checks as a substitution before its references are
-- believed.
--
-- The caches are asked in priority order, as upstream's substitution goal
-- asks them (@PathSubstitutionGoal::init@ at 2.24.9). One without the
-- path is passed over. So is one whose narinfo names another path or no
-- trusted key signed, and the scan says why. A query that fails, once its
-- retries are spent or on a narinfo that does not validate, ends the scan
-- with 'QueryFailed' under 'NoFallback', as upstream rethrows it. Under
-- 'BuildFromSource' it is reported and passed over, as upstream logs it
-- and asks the next cache.
queryReferencesWith :: FetchRetryPolicy -> RetryEffects IO -> Fallback -> [CacheConfig] -> StorePath -> IO Offer
queryReferencesWith _ _ _ [] _ = pure NotOffered
queryReferencesWith policy effects fallback caches sp = do
manager <- HTTPS.getGlobalManager
firstAnswer manager (sortCaches caches)
where
firstAnswer _ [] = pure NotOffered
firstAnswer manager (cache : rest) = do
answer <-
queryNarInfo (Retrying policy effects) manager cache sp
`catchSync` \err -> pure (NarInfoFailed ("query exception: " <> T.pack (show err)))
let next = firstAnswer manager rest
failed failure = case fallback of
NoFallback -> pure (QueryFailed (ccUrl cache <> ": " <> spName sp <> ": " <> failure))
BuildFromSource -> reportCache cache sp failure >> next
case answer of
NarInfoUsable narInfo -> either failed (pure . Offered) (parseReferences (NarInfo.niReferences narInfo))
NarInfoAbsent -> next
NarInfoPassedOver reason -> reportCache cache sp reason >> next
NarInfoFailed failure -> failed failure
-- | Say on stderr what one cache did with a path.
reportCache :: CacheConfig -> StorePath -> Text -> IO ()
reportCache cache sp message =
TIO.hPutStrLn stderr (" [subst] " <> ccUrl cache <> ": " <> spName sp <> ": " <> message)
-- | Fold per-cache attempts in priority order. The first success wins and
-- stops the scan. An erroring cache falls through to the remaining ones -
-- a transient failure (DNS, TLS, HTTP 500, unsupported compression) from a
-- higher-priority cache must not mask a hit in the next - and the first
-- error, tagged with its cache URL, is reported only when no cache has the
-- path, as upstream's goal fails only once every substituter has been
-- tried.
tryCachesWith :: (Monad m) => (CacheConfig -> m SubstResult) -> [CacheConfig] -> m SubstResult
tryCachesWith attempt = go Nothing
where
go firstErr [] = pure (maybe SubstNotFound SubstError firstErr)
go firstErr (cache : rest) = do
result <- attempt cache
case result of
SubstSuccess reg lock -> pure (SubstSuccess reg lock)
-- A path found valid mid-scan is terminal like a success:
-- there is nothing left to fetch from any cache.
SubstAlreadyValid -> pure SubstAlreadyValid
SubstNotFound -> go firstErr rest
SubstError err -> go (firstErr <|> Just (ccUrl cache <> ": " <> err)) rest
-- | The retry policy and its effects, carried to every request one
-- substitution makes.
data Retrying = Retrying !FetchRetryPolicy !(RetryEffects IO)
-- | Run one request's attempt under the carried policy.
retrying :: Retrying -> IO (Either AttemptFailure a) -> IO (Either AttemptFailure a)
retrying (Retrying policy effects) = retryTransient policy effects
-- | Attempt substitution from a single cache. Synchronous exceptions
-- become 'SubstError', so the scan falls through to the remaining
-- caches; asynchronous exceptions propagate ('catchSync') - an
-- interrupt mid-download must abort the scan, never continue to the
-- next cache and from there to a local build.
tryOneCache :: Retrying -> HTTP.Manager -> Store -> CacheConfig -> StorePath -> PathLock -> IO SubstResult
tryOneCache retry mgr store cache sp lock =
substituteFromCache retry mgr store cache sp lock
`catchSync` \err -> pure (SubstError ("substitution exception: " <> T.pack (show err)))
-- | Substitution pipeline for a single cache: the narinfo, judged
-- ('queryNarInfo'), then the NAR, streamed ('streamWithRetry'). A narinfo
-- this build passes over is a miss here without a word, because the
-- reference query asks every path first and has said why
-- ('queryReferencesWith').
substituteFromCache :: Retrying -> HTTP.Manager -> Store -> CacheConfig -> StorePath -> PathLock -> IO SubstResult
substituteFromCache retry mgr store cache sp lock = do
answer <- queryNarInfo retry mgr cache sp
case answer of
-- Unsupported compression rejects before the download: the value is
-- known from the narinfo, and a multi-hundred-MB download that can
-- only fail in decompression is pure waste. It is still a failed
-- substitute, as upstream's decompressor fails one. The strict
-- 'decompressorFor' encodes the same support set; the suite pins
-- their agreement.
NarInfoUsable narInfo -> case streamingDecompressionSupported (NarInfo.niCompression narInfo) of
Left err -> pure (SubstError err)
Right () -> streamWithRetry retry mgr store cache sp narInfo lock
NarInfoAbsent -> pure SubstNotFound
NarInfoPassedOver _ -> pure SubstNotFound
NarInfoFailed failure -> pure (SubstError failure)
-- | One cache's narinfo for a path, judged before anything is downloaded.
data NarInfoAnswer
= -- | A narinfo for the path that validates and that a trusted key
-- signed.
NarInfoUsable !NarInfo.NarInfo
| -- | The cache does not hold the path.
NarInfoAbsent
| -- | A narinfo this build passes over, and why: one naming another
-- path, which upstream reads as the path not being in that cache, or
-- one no trusted key signed, which it warns about and skips.
NarInfoPassedOver !Text
| -- | The query failed: the transfer once its retries are spent, a body
-- that does not parse or validate, or a trusted key that does not
-- parse.
NarInfoFailed !Text
deriving (Eq, Show)
-- | Fetch a narinfo from a cache under the substitution's retry policy
-- ('trySubstituteWith') and judge it ('judgeNarInfo').
queryNarInfo :: Retrying -> HTTP.Manager -> CacheConfig -> StorePath -> IO NarInfoAnswer
queryNarInfo retry mgr cache sp = do
fetched <- retrying retry (fetchNarInfoOnce mgr url)
pure $ case fetched of
Left failure -> NarInfoFailed (attemptFailureMessage failure)
Right Nothing -> NarInfoAbsent
Right (Just narInfo) -> judgeNarInfo cache sp narInfo
where
url = ccUrl cache <> "/" <> spHash sp <> ".narinfo"
-- | Whether a served narinfo may be used, decided from the narinfo alone, in
-- upstream's order. The fields validate first, standing in for upstream's
-- narinfo parser, and above all before 'verifySigs' builds the signed
-- fingerprint from them. The narinfo must then describe the requested path
-- (@goodStorePath@ in @Store::queryPathInfo@ at 2.24.9): a misconfigured or
-- hostile cache could return a validly signed narinfo for a different path
-- under this hash's URL. Last come the trusted keys, which upstream parses
-- when it checks the signature, and a signature one of them verifies
-- (@PathSubstitutionGoal::init@).
judgeNarInfo :: CacheConfig -> StorePath -> NarInfo.NarInfo -> NarInfoAnswer
judgeNarInfo cache sp narInfo
| Left err <- validateNarInfoFields narInfo = NarInfoFailed err
| not (narInfoMatchesPath sp narInfo) = NarInfoPassedOver ("narinfo names " <> NarInfo.niStorePath narInfo)
| Left err <- trustedKeys cache = NarInfoFailed err
| otherwise = either NarInfoPassedOver (const (NarInfoUsable narInfo)) (verifySigs cache narInfo)
-- | Validate narinfo field syntax before anything consumes the fields -
-- above all before 'verifySigs' builds the signed fingerprint from them.
-- The fingerprint is delimited text (semicolons between fields, commas
-- between references), so each field must have parsed as well-formed
-- before it is spliced in; a malformed narinfo fails here as a plain
-- parse error instead of flowing onward. The checks are nova-cache's
-- 'Validate.validateNarInfo': store path, references, hash spellings,
-- sizes, and the no-@.drv@ rule.
validateNarInfoFields :: NarInfo.NarInfo -> Either Text ()
validateNarInfoFields narInfo = case Validate.validateNarInfo narInfo of
Right _ -> Right ()
Left errs ->
Left ("invalid narinfo: " <> T.intercalate "; " (map renderValidationError errs))
-- | One 'Validate.ValidationError' in the register the other
-- substitution errors use.
renderValidationError :: Validate.ValidationError -> Text
renderValidationError verr = case verr of
Validate.NegativeFileSize n -> "negative FileSize " <> T.pack (show n)
Validate.NegativeNarSize n -> "negative NarSize " <> T.pack (show n)
Validate.InvalidStorePath raw parseErr -> fieldError "StorePath" raw parseErr
Validate.InvalidFileHash raw parseErr -> fieldError "FileHash" raw parseErr
Validate.InvalidNarHash raw parseErr -> fieldError "NarHash" raw parseErr
Validate.InvalidReference raw parseErr -> fieldError "Reference" raw parseErr
Validate.NarHashMismatch expected actual ->
"NarHash mismatch: declared " <> expected <> ", computed " <> actual
Validate.FileHashMismatch expected actual ->
"FileHash mismatch: declared " <> expected <> ", computed " <> actual
Validate.SignatureInvalid sig -> "signature failed verification: " <> sig
Validate.NoSignatures -> "narinfo has no signatures"
Validate.DerivationStorePath path -> "StorePath names a derivation: " <> path
where
fieldError field raw parseErr =
field <> " '" <> raw <> "' does not parse: " <> T.pack parseErr
-- | NAR bytes that passed 'verifyNar': they hash to the narinfo's
-- signed NarHash and count its signed NarSize. The constructor is not
-- exported, so 'verifyNar' is the only way to obtain one and
-- 'unpackVerifiedNar', the strict pipeline's unpack sink, cannot be
-- reached with bytes that skipped the check: verify-then-unpack is a
-- type, not a statement order. The decoded digest rides along because
-- registration records its canonical spelling and the on-disk recheck
-- compares against it, so neither re-parses the narinfo's NarHash.
-- Only the strict pipeline can carry this: the streaming pipeline
-- writes to disk before the hash can be known (upstream's ordering)
-- and relies on its cleanup contract instead
-- ('materializeNarFromSource').
data VerifiedNar = VerifiedNar !BS.ByteString !Hash.NixHash
deriving (Eq, Show)
-- | The verified bytes.
verifiedNarBytes :: VerifiedNar -> BS.ByteString
verifiedNarBytes (VerifiedNar bytes _) = bytes
-- | The digest the verified bytes hash to, decoded from the narinfo's
-- NarHash.
verifiedNarDigest :: VerifiedNar -> Hash.NixHash
verifiedNarDigest (VerifiedNar _ digest) = digest
-- | The only constructor of a t'VerifiedNar': the hash check, then the
-- size check, with the decoded declared digest carried out. The NAR
-- hash is the content-addressed integrity contract; the signature only
-- attests to the narinfo, not the body, so network corruption or a
-- compromised cache is caught here, before the bytes are trusted.
verifyNar :: NarInfo.NarInfo -> BS.ByteString -> Either Text VerifiedNar
verifyNar narInfo rawNar = do
declared <- verifyNarHash narInfo rawNar
verifyNarSize narInfo rawNar
pure (VerifiedNar rawNar declared)
-- | Verify the NAR hash and size, deserialize, unpack to the store, and
-- set permissions: 'verifyNar', then 'unpackVerifiedNar'. Returns the
-- path's registration for the caller to record; no database write
-- happens here (see 'SubstSuccess'). The strict counterpart of the
-- streaming pipeline, kept as its differential oracle: the suite
-- materializes the same NAR through both and requires identical trees.
unpackAndVerify :: Store -> StorePath -> NarInfo.NarInfo -> BS.ByteString -> IO SubstResult
unpackAndVerify store sp narInfo rawNar =
-- Narinfo metadata is parsed before any disk write: a malformed
-- narinfo must not leave an unpacked-but-unregistered path behind.
case verifiedInputs of
Left err -> pure (SubstError err)
Right (verified, refs, deriver) -> unpackVerifiedNar store sp verified refs deriver
where
verifiedInputs = do
verified <- verifyNar narInfo rawNar
refs <- parseReferences (NarInfo.niReferences narInfo)
deriver <- parseDeriver (stDir store) (NarInfo.niDeriver narInfo)
pure (verified, refs, deriver)
-- | The strict pipeline's unpack sink: deserialize a t'VerifiedNar',
-- materialize it at the store path, seal it read-only, and recheck the
-- tree on disk against the carried digest before building its
-- registration from the references and deriver. Follows the same
-- per-path lock protocol as the live pipeline: the lock is taken
-- before the stale destination is cleared, validity re-checks under
-- it, and a success carries the lock still held.
unpackVerifiedNar :: Store -> StorePath -> VerifiedNar -> [StorePath] -> Maybe Text -> IO SubstResult
unpackVerifiedNar store sp verified refs deriver =
case NAR.deserialise (verifiedNarBytes verified) of
Left err -> pure (SubstError ("NAR deserialisation failed: " <> T.pack err))
Right narEntry -> do
lock <- acquirePathLock (stDir store) sp
result <- unpackLocked narEntry lock `onException` releasePathLock lock
case result of
SubstSuccess _ _ -> pure result
other -> do
releasePathLock lock
pure other
where
digest = verifiedNarDigest verified
destPath = storePathToFilePath (stDir store) sp
unpackLocked narEntry lock = do
alreadyValid <- isValid store sp
if alreadyValid
then pure SubstAlreadyValid
else do
unpackResult <-
fmap
Right
( do
clearStaleDestination destPath
unpackNarEntry (stCaseSensitivity store) destPath narEntry
)
`catchSync` (pure . Left)
case (unpackResult :: Either SomeException (Either Text ())) of
Left err -> pure (SubstError ("unpack failed: " <> T.pack (show err)))
Right (Left err) -> pure (SubstError ("unpack failed: " <> err))
Right (Right ()) -> do
setReadOnly destPath
-- A path registered valid must match its recorded hash ON
-- DISK, not merely in the verified bytes: any divergence
-- the filesystem introduced between the NAR and the
-- materialized tree (name folding, link replication) must
-- surface here, before the row exists. A mismatching tree
-- is removed - left in place it would be adopted by
-- existence checks at this path.
onDisk <- ExecBit.serialiseFromPath (volumeCaseHack (stCaseSensitivity store)) destPath
if Hash.hashBytes (NAR.serialise onDisk) /= digest
then do
Dir.removePathForcibly destPath
pure
( SubstError
( "unpacked tree does not reproduce the declared NAR hash at "
<> T.pack destPath
)
)
else
pure $
SubstSuccess
PathRegistration
{ prPath = sp,
-- The canonical spelling of the verified digest,
-- so the DB converges on one hash spelling
-- regardless of the cache's.
prNarHash = Hash.formatNixHash digest,
-- The verified actual byte count (equal to the
-- declared NarSize per 'verifyNarSize') - no
-- Integer conversion that could wrap.
prNarSize = BS.length (verifiedNarBytes verified),
prDeriver = deriver,
prReferences = refs
}
lock
-- | Verify that the downloaded NAR bytes hash to the narinfo's declared
-- NarHash, returning the decoded digest on success so registration can
-- record its canonical spelling. Compares decoded hash bytes, so any
-- valid encoding of the digest validates.
verifyNarHash :: NarInfo.NarInfo -> BS.ByteString -> Either Text Hash.NixHash
verifyNarHash narInfo rawNar =
case Hash.parseNixHash (NarInfo.niNarHash narInfo) of
Left err -> Left ("invalid narinfo NarHash: " <> T.pack err)
Right declared
| declared == actual -> Right declared
| otherwise ->
Left
( "NAR hash mismatch: narinfo declares "
<> NarInfo.niNarHash narInfo
<> " but downloaded bytes hash to "
<> Hash.formatNixHash actual
)
where
actual = Hash.hashBytes rawNar
-- | Verify that the downloaded NAR's byte count equals the narinfo's
-- declared NarSize. The hash check pins the content, but the size is a
-- separate signed claim that flows into the store DB (and from there
-- into re-pushed narinfos), so a wrong declaration must be rejected
-- rather than recorded.
verifyNarSize :: NarInfo.NarInfo -> BS.ByteString -> Either Text ()
verifyNarSize narInfo rawNar
| toInteger (BS.length rawNar) == NarInfo.niNarSize narInfo = Right ()
| otherwise =
Left
( "NAR size mismatch: narinfo declares "
<> T.pack (show (NarInfo.niNarSize narInfo))
<> " bytes but downloaded "
<> T.pack (show (BS.length rawNar))
)
-- | Whether a served narinfo describes the requested path: the hash component
-- of its declared StorePath must equal the requested path's hash. Only the
-- hash is compared, since the cache may use a different store directory.
narInfoMatchesPath :: StorePath -> NarInfo.NarInfo -> Bool
narInfoMatchesPath sp narInfo =
storePathHashOf (NarInfo.niStorePath narInfo) == Just (spHash sp)
-- | Extract the leading hash from a full store path's basename, if well-formed
-- (@\<hash\>-\<name\>@ with a hash of the expected length).
storePathHashOf :: Text -> Maybe Text
storePathHashOf path =
let base = T.takeWhileEnd (\c -> c /= '/' && c /= '\\') path
(hashPart, rest) = T.splitAt storePathHashLen base
in if T.length hashPart == storePathHashLen && T.isPrefixOf "-" rest
then Just hashPart
else Nothing
-- ---------------------------------------------------------------------------
-- HTTP fetching
-- ---------------------------------------------------------------------------
-- | Cap on a narinfo response body, mirroring nova-cache's server-side
-- @maxNarInfoBodySize@ - the server bounds what it reads, and the
-- client bounds what any cache in its list can make it buffer.
-- Narinfo is small key-value text; 4 MB is far beyond any real one.
maxNarInfoBody :: Int
maxNarInfoBody = 4 * 1024 * 1024
-- | Read an HTTP response body in bounded chunks up to a byte cap -
-- 'Nothing' once the cap is exceeded, so an over-large body aborts
-- mid-stream instead of buffering without limit. The client-side
-- mirror of nova-cache's @readBodyLimited@.
readBodyCapped :: Int -> HTTP.BodyReader -> IO (Maybe BS.ByteString)
readBodyCapped cap reader = go [] 0
where
go chunks !total = do
chunk <- HTTP.brRead reader
if BS.null chunk
then pure (Just (BS.concat (reverse chunks)))
else
let newTotal = total + BS.length chunk
in if newTotal > cap
then pure Nothing
else go (chunk : chunks) newTotal
-- | One narinfo request: @Right Nothing@ when the cache does not hold
-- it. A 'NotFound' or 'Forbidden' answer (404, 410, 401, 403, 407) is
-- the cache saying so, not a failure to retry: upstream's transfer layer
-- retries neither, and its binary cache store reads both as a missing
-- narinfo, so the path is not valid there (@HttpBinaryCacheStore::getFile@
-- at 2.24.9). A narinfo that arrived whole and does not parse, or
-- overruns 'maxNarInfoBody', is the cache's bytes and fails at once.
fetchNarInfoOnce :: HTTP.Manager -> Text -> IO (Either AttemptFailure (Maybe NarInfo.NarInfo))
fetchNarInfoOnce mgr url = fetch `catches` transferFailureHandlers url
where
fetch = do
request <- withUserAgent <$> HTTP.parseRequest (T.unpack url)
withTransfer request mgr $ \response ->
answer (HTTP.responseStatus response) (HTTP.responseBody response)
answer status body
| status /= HTTP.status200 = pure $ case statusError status of
NotFound -> Right Nothing
Forbidden -> Right Nothing
Misc -> Left (fetchStatusFailure url status)
Transient -> Left (fetchStatusFailure url status)
| otherwise = do
capped <- readBodyCapped maxNarInfoBody body
pure $ case capped of
Nothing ->
Left (FatalFailure ("narinfo body exceeds " <> T.pack (show maxNarInfoBody) <> " bytes"))
-- Lenient decode: the body is cache-controlled bytes, and a
-- stray invalid UTF-8 sequence must surface as a narinfo parse
-- error, not an impure UnicodeException (the push side decodes
-- the same way).
Just bytes -> case NarInfo.parseNarInfo (TE.decodeUtf8Lenient bytes) of
Left err -> Left (FatalFailure ("narinfo parse error: " <> T.pack err))
Right narInfo -> Right (Just narInfo)
-- | Stream one substitution end to end, each attempt under the
-- substitution's retry policy ('trySubstituteWith').
--
-- By the time this runs the narinfo has already been fetched and
-- signature-verified, so the cache claims to hold this path. Failures
-- carry their own retry class: a 'TransientFailure' - transport
-- errors, torn or truncated transfers, a status upstream retries,
-- anything a fresh attempt could plausibly complete - consumes retry
-- budget with upstream's backoff, while a 'FatalFailure' - a completed
-- transfer that verifies wrong, a body past its signed ceiling, a
-- status upstream never retries, a write the disk refused - ends the
-- attempt at once: retrying a deterministic failure only delays the
-- outcome. Every attempt starts from a clean slate
-- ('streamNarIntoStore' clears the destination first, and every
-- failure path, exceptions included, removes what it wrote), so a retry
-- restarts the NAR from its first byte, where upstream resumes from the
-- written offset when the server accepts ranges and otherwise retries
-- only a transfer that wrote nothing yet. A NAR the narinfo named and
-- the cache answers 'NotFound' or 'Forbidden' for is gone: one request,
-- as upstream's @SubstituteGone@ (@BinaryCacheStore::narFromPath@ at
-- 2.24.9), reported and then a miss the cache scan passes over, because
-- upstream's substitution goal treats a gone substitute like one that
-- never existed rather than as a failure. A narinfo the cache does not
-- hold is a miss handled earlier in 'queryNarInfo' and never reaches
-- here.
streamWithRetry :: Retrying -> HTTP.Manager -> Store -> CacheConfig -> StorePath -> NarInfo.NarInfo -> PathLock -> IO SubstResult
streamWithRetry retry mgr store cache sp narInfo lock = do
outcome <- retrying retry (streamNarIntoStore mgr cache store sp narInfo)
case outcome of
Right (Just registration) -> pure (SubstSuccess registration lock)
Right Nothing -> do
reportCache cache sp ("file '" <> NarInfo.niUrl narInfo <> "' does not exist in the binary cache")
pure SubstNotFound
Left failure -> pure (SubstError (attemptFailureMessage failure))
-- | Thrown inside the streaming pipeline where a chunk convention has
-- no error channel (the capped body source); converted back to the
-- pipeline's 'Left' at the attempt boundary in 'streamNarIntoStore'.
newtype StreamAbort = StreamAbort Text
deriving (Show)
instance Exception StreamAbort
-- | Convert what one download attempt throws into the pipeline's
-- failure channel, so the retry budget governs it and the caller's
-- cleanup contract holds on every exit: the transfer layer's
-- classification ('transferFailureHandlers'), and a t'StreamAbort'.
-- Nothing else is caught, so an interrupt is never spent as retry
-- budget.
tryAttempt :: Text -> IO (Either AttemptFailure a) -> IO (Either AttemptFailure a)
tryAttempt url action =
action `catches` (Handler streamAborted : transferFailureHandlers url)
where
-- A body past its ceiling: the cap derives from the signed NarSize,
-- so a longer body is the server misdeclaring, not a hiccup.
streamAborted (StreamAbort msg) = pure (Left (FatalFailure msg))
-- | One streaming substitution attempt: download, decompress, hash,
-- parse, and materialize in a single bounded pass, then verify.
-- Memory is bounded by the decoder's buffers and the parser's
-- structural-string cap, never by archive or file size.
--
-- Disk writes begin before the NAR hash can be known - the price of
-- never holding the archive, and exactly upstream's ordering - so
-- every failure path, exceptions included, removes the tree it wrote
-- ('materializeNarFromSource' owns that contract; 'tryAttempt'
-- classifies what escapes it). The narinfo metadata registration
-- needs (declared hash, references, deriver) still parses BEFORE the
-- first byte downloads: a malformed narinfo must not leave an
-- unpacked-but-unregistered path behind. 'Nothing' when the cache no
-- longer holds the NAR ('streamWithRetry').
streamNarIntoStore :: HTTP.Manager -> CacheConfig -> Store -> StorePath -> NarInfo.NarInfo -> IO (Either AttemptFailure (Maybe PathRegistration))
streamNarIntoStore mgr cache store sp narInfo = case preflight of
Left err -> pure (Left (FatalFailure err))
Right (declaredDigest, refs, deriver, downloadCap) -> do
let destPath = storePathToFilePath (stDir store) sp
narUrl = ccUrl cache <> "/" <> NarInfo.niUrl narInfo
clearStaleDestination destPath
tryAttempt narUrl $ do
request <- withUserAgent <$> HTTP.parseRequest (T.unpack narUrl)
withTransfer request mgr $ \response -> do
let status = HTTP.responseStatus response
if status /= HTTP.status200
then pure (unservedNar narUrl status)
else do
source <- cappedBodySource downloadCap (HTTP.responseBody response)
fmap Just <$> materializeNarFromSource store sp narInfo declaredDigest refs deriver source
where
preflight = do
declaredDigest <- case Hash.parseNixHash (NarInfo.niNarHash narInfo) of
Left err -> Left ("invalid narinfo NarHash: " <> T.pack err)
Right digest -> Right digest
refs <- parseReferences (NarInfo.niReferences narInfo)
deriver <- parseDeriver (stDir store) (NarInfo.niDeriver narInfo)
downloadCap <- downloadCapFor narInfo
pure (declaredDigest, refs, deriver, downloadCap)
-- | A NAR response that is not a success. 'NotFound' and 'Forbidden' mean
-- the cache no longer holds the NAR, as upstream's binary cache store
-- reads both (@HttpBinaryCacheStore::getFile@ at 2.24.9); any other status
-- is a failure, retried when upstream retries it.
unservedNar :: Text -> HTTP.Status -> Either AttemptFailure (Maybe PathRegistration)
unservedNar narUrl status = case statusError status of
NotFound -> Right Nothing
Forbidden -> Right Nothing
Misc -> Left (fetchStatusFailure narUrl status)
Transient -> Left (fetchStatusFailure narUrl status)
-- | The byte cap on the compressed NAR body, derived from the SIGNED
-- NarSize: the Ed25519 fingerprint covers StorePath, NarHash, NarSize,
-- and references - not FileSize - so the unsigned FileSize may only
-- LOWER the cap, never raise it. A rewritten FileSize on an otherwise
-- validly-signed narinfo must not buy an unbounded download.
downloadCapFor :: NarInfo.NarInfo -> Either Text Int
downloadCapFor narInfo
| narSize < 0 || narSize > toInteger (maxBound :: Int) =
Left ("narinfo declares an unusable NAR size: " <> T.pack (show narSize))
| otherwise =
let signedCeiling = compressedBodyCeiling narSize
capped = maybe signedCeiling (min signedCeiling . max 0) (NarInfo.niFileSize narInfo)
in Right (fromInteger (min capped (toInteger (maxBound :: Int))))
where
narSize = NarInfo.niNarSize narInfo
-- | The most a compressed NAR body may legitimately exceed its NarSize
-- by: xz and zstd expand incompressible input by well under one
-- percent of framing overhead, so a 1/64 (~1.6%) margin plus a fixed
-- floor for small NARs is generous for any real codec, while a
-- hostile FileSize claiming orders of magnitude more is refused.
compressedBodyCeiling :: Integer -> Integer
compressedBodyCeiling narSize =
narSize + max compressionOverheadFloorBytes (narSize `div` compressionOverheadDivisor)
-- | Fixed overhead floor for small NARs, where framing dominates.
compressionOverheadFloorBytes :: Integer
compressionOverheadFloorBytes = 64 * 1024
-- | Proportional overhead margin: 1/64 of the NarSize.
compressionOverheadDivisor :: Integer
compressionOverheadDivisor = 64
-- | Materialize a NAR from a compressed chunk source into the store
-- path: decompress, hash, parse, and unpack in one pass, then verify
-- the tree on disk and build its registration. The post-download half
-- of 'streamNarIntoStore', taking a plain chunk source so the failure
-- contract is testable without HTTP.
--
-- The cleanup contract: every failing exit - the pipeline's 'Left',
-- a verification mismatch, or an exception (asynchronous included) -
-- removes the destination tree, so no partial or unverified tree ever
-- survives at the store path.
materializeNarFromSource :: Store -> StorePath -> NarInfo.NarInfo -> Hash.NixHash -> [StorePath] -> Maybe Text -> IO BS.ByteString -> IO (Either AttemptFailure PathRegistration)
materializeNarFromSource store sp narInfo declaredDigest refs deriver source =
materialize `onException` Dir.removePathForcibly destPath
where
destPath = storePathToFilePath (stDir store) sp
materialize = do
streamed <-
withDecompressedSource (NarInfo.niNarSize narInfo) (NarInfo.niCompression narInfo) source $
consumeNarStream (stCaseSensitivity store) destPath narInfo declaredDigest
case streamed of
Left err -> do
Dir.removePathForcibly destPath
pure (Left err)
Right narByteCount -> do
setReadOnly destPath
-- A path registered valid must match its recorded hash ON
-- DISK, not merely in the streamed bytes: any divergence the
-- filesystem introduced between the NAR and the materialized
-- tree must surface here, before the row exists. The recheck
-- streams too, so its memory no longer scales with the path.
-- Through 'ExecBit', not nova-cache's walk directly: on Windows
-- the sink writes the exec bit to a stream that walk cannot see,
-- and a verifier reading the flag back out of the file extension
-- would reject exactly the paths this representation exists for.
-- On Unix it is still the streaming hash.
onDiskDigest <- ExecBit.narHashOfPath (volumeCaseHack (stCaseSensitivity store)) destPath
if onDiskDigest /= declaredDigest
then do
Dir.removePathForcibly destPath
pure (Left (FatalFailure ("unpacked tree does not reproduce the declared NAR hash at " <> T.pack destPath)))
else
pure $
Right
PathRegistration
{ prPath = sp,
-- The canonical spelling of the verified digest,
-- so the DB converges on one hash spelling
-- regardless of the cache's.
prNarHash = Hash.formatNixHash declaredDigest,
prNarSize = narByteCount,
prDeriver = deriver,
prReferences = refs
}
-- | Drive the decompressed chunk source through incremental hashing,
-- the streaming NAR parser, and the store's streaming unpack sink,
-- returning the verified NAR byte count. The hash context folds over
-- exactly the bytes the parser consumes, so the digest is of the NAR
-- the tree was built from. The sensitivity is the destination
-- volume's, 'stCaseSensitivity' when the destination is a store path.
consumeNarStream :: CaseSensitivity -> FilePath -> NarInfo.NarInfo -> Hash.NixHash -> IO BS.ByteString -> IO (Either AttemptFailure Int)
consumeNarStream sensitivity destPath narInfo declaredDigest narSource = do
sink <- newNarUnpackSink sensitivity destPath
streamed <- sinkNarStream sink narSource
case streamed of
-- A name or shape the store refuses is a property of the archive,
-- not of this transfer.
Left (NarStreamRefused err) -> pure (Left (FatalFailure err))
-- A truncated body and a torn transfer parse-fail the same way, so
-- the retry budget applies.
Left (NarStreamMalformed msg) -> pure (Left (TransientFailure ("NAR stream parse failed: " <> msg)))
Right (digest, narBytes)
| toInteger narBytes /= NarInfo.niNarSize narInfo -> do
abortNarUnpack sink
-- The grammar completed, so the transfer was whole: a size
-- that still disagrees is the narinfo misdeclaring.
pure
( Left
( FatalFailure
( "NAR size mismatch: narinfo declares "
<> T.pack (show (NarInfo.niNarSize narInfo))
<> " bytes but the stream carried "
<> T.pack (show narBytes)
)
)
)
| digest /= declaredDigest -> do
abortNarUnpack sink
-- Size matched, so the transfer completed; wrong bytes are
-- deterministic corruption, not a hiccup.
pure
( Left
( FatalFailure
( "NAR hash mismatch: narinfo declares "
<> NarInfo.niNarHash narInfo
<> " but downloaded bytes hash to "
<> Hash.formatNixHash digest
)
)
)
| otherwise -> do
finished <- finishNarUnpack sink
pure (either (Left . FatalFailure) (const (Right narBytes)) finished)
-- | Stream an HTTP body as a chunk source bounded by the download cap
-- 'downloadCapFor' derived from the signed NarSize -
-- 'readBodyCapped'\'s discipline without the buffering. Exceeding the
-- cap throws t'StreamAbort'; the attempt boundary converts it back to
-- the pipeline's error channel.
cappedBodySource :: Int -> HTTP.BodyReader -> IO (IO BS.ByteString)
cappedBodySource cap reader = do
countRef <- newIORef 0
pure $ do
chunk <- HTTP.brRead reader
consumed <- readIORef countRef
let total = consumed + BS.length chunk
if total > cap
then throwIO (StreamAbort ("NAR body exceeds its download ceiling (" <> T.pack (show cap) <> " bytes)"))
else do
writeIORef countRef total
pure chunk
-- | Whether the streaming pipeline can decompress a narinfo
-- @Compression@ value, decided from the value alone so unsupported
-- compression rejects before any download. Both this and the strict
-- 'decompressorFor' dispatch through 'parseNarCompression', so the
-- support set exists exactly once.
streamingDecompressionSupported :: Text -> Either Text ()
streamingDecompressionSupported = void . parseNarCompression
-- | Run a consumer over the decompressed view of a chunk source:
-- identity for 'CompressionNone', nova-cache's bounded decoders for
-- 'CompressionXz', 'CompressionZstd', and 'CompressionBzip2' (output
-- capped at the declared NarSize; thrown codec errors convert to the
-- pipeline's error channel here, carrying their retry class from
-- 'xzFailure', 'zstdFailure', and 'bzip2Failure').
withDecompressedSource :: Integer -> Text -> IO BS.ByteString -> (IO BS.ByteString -> IO (Either AttemptFailure a)) -> IO (Either AttemptFailure a)
withDecompressedSource declaredNarSize compression source consume =
case parseNarCompression compression of
Left err -> pure (Left (FatalFailure err))
Right CompressionNone -> consume source
Right CompressionXz -> case xzLimitsFor declaredNarSize of
Left err -> pure (Left (FatalFailure err))
Right limits ->
Xz.withXzSource limits source consume
`catch` \xzErr -> pure (Left (xzFailure xzErr))
Right CompressionZstd -> case zstdLimitsFor declaredNarSize of
Left err -> pure (Left (FatalFailure err))
Right limits ->
Zstd.withZstdSource limits source consume
`catch` \zstdErr -> pure (Left (zstdFailure zstdErr))
Right CompressionBzip2 -> case bzip2LimitsFor declaredNarSize of
Left err -> pure (Left (FatalFailure err))
Right limits ->
Bzip2.withBzip2Source limits source consume
`catch` \bzip2Err -> pure (Left (bzip2Failure bzip2Err))
-- | Classify a decoder failure: a stream error is how truncation and
-- torn transfers surface, so it retries; output or memory past the
-- declared bounds is the served object misdeclaring - deterministic.
xzFailure :: Xz.XzError -> AttemptFailure
xzFailure xzErr = case xzErr of
Xz.XzStreamError _ -> TransientFailure rendered
Xz.XzOutputOverBound _ -> FatalFailure rendered
Xz.XzMemoryOverBound _ -> FatalFailure rendered
where
rendered = renderXzError xzErr
-- | 'xzFailure'\'s zstd counterpart, under the same taxonomy.
zstdFailure :: Zstd.ZstdError -> AttemptFailure
zstdFailure zstdErr = case zstdErr of
Zstd.ZstdStreamError _ -> TransientFailure rendered
Zstd.ZstdOutputOverBound _ -> FatalFailure rendered
where
rendered = renderZstdError zstdErr
-- | 'xzFailure'\'s bzip2 counterpart, under the same taxonomy.
bzip2Failure :: Bzip2.Bzip2Error -> AttemptFailure
bzip2Failure bzip2Err = case bzip2Err of
Bzip2.Bzip2StreamError _ -> TransientFailure rendered
Bzip2.Bzip2OutputOverBound _ -> FatalFailure rendered
where
rendered = renderBzip2Error bzip2Err
-- ---------------------------------------------------------------------------
-- Pure helpers
-- ---------------------------------------------------------------------------
-- | Sort caches by priority (lower = first).
sortCaches :: [CacheConfig] -> [CacheConfig]
sortCaches = sortBy (comparing ccPriority)
-- | Verify narinfo signatures against the cache's trusted public keys.
-- Accepts when at least one signature verifies under at least one key,
-- upstream's any-trusted-key rule.
verifySigs :: CacheConfig -> NarInfo.NarInfo -> Either Text ()
verifySigs cache narInfo =
case NarInfo.niSigs narInfo of
[] -> Left "narinfo has no signatures"
sigs -> do
keys <- trustedKeys cache
if any (\key -> any (Signing.verify key narInfo) sigs) keys
then Right ()
else Left "no valid signature found"
-- | The cache's trusted public keys. A malformed key is an error rather
-- than a silently skipped one: upstream rejects the configuration, so a
-- typo cannot quietly narrow the trusted set to the keys that parsed.
trustedKeys :: CacheConfig -> Either Text [Signing.PublicKey]
trustedKeys = traverse parseKey . ccPublicKeys
where
parseKey raw = case Signing.parsePublicKey raw of
Left err -> Left ("invalid public key: " <> T.pack err)
Right pubKey -> Right pubKey
-- | The whole-buffer decompressor for a narinfo @Compression@ value,
-- decided from the narinfo's declared values alone so unsupported
-- compression rejects before any download. 'CompressionNone' is
-- identity; 'CompressionXz' (cache.nixos.org's format),
-- 'CompressionZstd' (the modern caches'), and 'CompressionBzip2' (the
-- historical caches', and what an absent field means) decompress
-- bounded by the declared NarSize, a signed claim the caller validates
-- before resolving the decompressor. The resolved function runs in IO
-- because the zstd and bzip2 decoders are IO-native (see
-- 'NovaCache.Zstd' and 'NovaCache.Bzip2'); the strict shape follows its
-- codecs. Dispatches through 'parseNarCompression' like the streaming
-- path, so the support set exists exactly once.
decompressorFor :: Integer -> Text -> Either Text (BS.ByteString -> IO (Either Text BS.ByteString))
decompressorFor declaredNarSize compression = do
kind <- parseNarCompression compression
case kind of
CompressionNone -> Right (pure . Right)
CompressionXz -> do
limits <- xzLimitsFor declaredNarSize
Right (pure . either (Left . renderXzError) Right . Xz.decompress limits)
CompressionZstd -> do
limits <- zstdLimitsFor declaredNarSize
Right (fmap (either (Left . renderZstdError) Right) . Zstd.decompress limits)
CompressionBzip2 -> do
limits <- bzip2LimitsFor declaredNarSize
Right (fmap (either (Left . renderBzip2Error) Right) . Bzip2.decompress limits)
-- | Decompress NAR data based on the compression type from narinfo,
-- bounded by the declared NarSize. Support is decided by
-- 'decompressorFor'; this applies the result.
decompressNar :: Integer -> Text -> BS.ByteString -> IO (Either Text BS.ByteString)
decompressNar declaredNarSize compression narData =
case decompressorFor declaredNarSize compression of
Left err -> pure (Left err)
Right decompress -> decompress narData
-- | The bounds for one xz decode: output capped at the narinfo's
-- declared NarSize, decoder memory at nova-cache's default, so a
-- hostile stream can expand to neither more output nor more decoder
-- state than the narinfo promised. Narinfo validation upstream
-- already rejected a negative size; the guard keeps the function
-- total for direct callers, and a size past Word64 cannot name a
-- real NAR.
xzLimitsFor :: Integer -> Either Text Xz.XzLimits
xzLimitsFor declaredNarSize
| declaredNarSize < 0 || declaredNarSize > toInteger (maxBound :: Word64) =
Left ("xz decompression bound out of range: " <> T.pack (show declaredNarSize))
| otherwise =
Right
Xz.XzLimits
{ Xz.xzMaxOutputBytes = fromInteger declaredNarSize,
Xz.xzMaxDecoderMemoryBytes = Xz.defaultXzDecoderMemoryBytes
}
-- | One 'Xz.XzError' in the register the other substitution errors use.
renderXzError :: Xz.XzError -> Text
renderXzError xzErr = case xzErr of
Xz.XzStreamError msg -> "xz stream error: " <> T.pack msg
Xz.XzOutputOverBound bound ->
"xz output exceeds the declared NarSize (" <> T.pack (show bound) <> " bytes)"
Xz.XzMemoryOverBound bound ->
"xz decoder memory over its cap (" <> T.pack (show bound) <> " bytes)"
-- | The bounds for one zstd decode: output capped at the narinfo's
-- declared NarSize; decoder state rides libzstd's built-in window
-- limit (see 'NovaCache.Zstd'). The same totality guard as
-- 'xzLimitsFor'.
zstdLimitsFor :: Integer -> Either Text Zstd.ZstdLimits
zstdLimitsFor declaredNarSize
| declaredNarSize < 0 || declaredNarSize > toInteger (maxBound :: Word64) =
Left ("zstd decompression bound out of range: " <> T.pack (show declaredNarSize))
| otherwise = Right Zstd.ZstdLimits {Zstd.zstdMaxOutputBytes = fromInteger declaredNarSize}
-- | One 'Zstd.ZstdError' in the register the other substitution
-- errors use.
renderZstdError :: Zstd.ZstdError -> Text
renderZstdError zstdErr = case zstdErr of
Zstd.ZstdStreamError msg -> "zstd stream error: " <> T.pack msg
Zstd.ZstdOutputOverBound bound ->
"zstd output exceeds the declared NarSize (" <> T.pack (show bound) <> " bytes)"
-- | The bounds for one bzip2 decode: output capped at the narinfo's
-- declared NarSize. There is no decoder-memory knob to set - bzip2
-- carries no attacker-chosen dictionary size, so decoder state is a
-- small constant of the format (see 'NovaCache.Bzip2'). The same
-- totality guard as 'xzLimitsFor'.
bzip2LimitsFor :: Integer -> Either Text Bzip2.Bzip2Limits
bzip2LimitsFor declaredNarSize
| declaredNarSize < 0 || declaredNarSize > toInteger (maxBound :: Word64) =
Left ("bzip2 decompression bound out of range: " <> T.pack (show declaredNarSize))
| otherwise = Right Bzip2.Bzip2Limits {Bzip2.bzip2MaxOutputBytes = fromInteger declaredNarSize}
-- | One 'Bzip2.Bzip2Error' in the register the other substitution
-- errors use.
renderBzip2Error :: Bzip2.Bzip2Error -> Text
renderBzip2Error bzip2Err = case bzip2Err of
Bzip2.Bzip2StreamError msg -> "bzip2 stream error: " <> T.pack msg
Bzip2.Bzip2OutputOverBound bound ->
"bzip2 output exceeds the declared NarSize (" <> T.pack (show bound) <> " bytes)"
-- | Parse narinfo references (store path basenames, e.g.
-- @abc...-glibc-2.40@) into StorePaths. A malformed token is an error,
-- not filtered: silently dropping a reference registers the path with a
-- hole in its closure, which re-push then publishes as a signed narinfo
-- missing runtime deps, and GC reads as permission to delete them.
parseReferences :: [Text] -> Either Text [StorePath]
parseReferences = traverse parseRef
where
parseRef ref =
maybe (Left ("malformed narinfo reference: " <> ref)) Right (parseStorePathBaseName ref)
-- | The sentinel upstream caches emit for a path whose deriver is unknown.
unknownDeriverSentinel :: Text
unknownDeriverSentinel = "unknown-deriver"
-- | Parse a narinfo Deriver - a store path basename on the wire, or the
-- @unknown-deriver@ sentinel - into the full path text the store DB
-- records (the form "Nix.Push" parses back with 'Nix.Store.Path.parseStorePath').
parseDeriver :: StoreDir -> Maybe Text -> Either Text (Maybe Text)
parseDeriver _ Nothing = Right Nothing
parseDeriver storeDir (Just txt)
| txt == unknownDeriverSentinel = Right Nothing
| otherwise = case parseStorePathBaseName txt of
Just sp -> Right (Just (T.pack (storePathToFilePath storeDir sp)))
Nothing -> Left ("malformed narinfo deriver: " <> txt)
-- ---------------------------------------------------------------------------
-- NAR unpacking
-- ---------------------------------------------------------------------------
-- | Remove a leftover destination tree before unpacking. A crash (or a
-- failed registration) between 'setReadOnly' here and the caller's
-- 'Nix.Store.DB.registerPaths' leaves a read-only, unregistered tree;
-- unpacking over it would then fail with permission-denied on every
-- retry, permanently wedging substitution of that path.
-- 'Dir.removePathForcibly' clears read-only marks and accepts a missing
-- path, so the fresh unpack always starts from a clean slate.
-- Substitution callers reach this only holding the path's exclusive
-- lock ('trySubstitute', 'unpackVerifiedNar'): unlocked, this deletion is
-- exactly the race that lets one process remove a tree another just
-- registered.
clearStaleDestination :: FilePath -> IO ()
clearStaleDestination = Dir.removePathForcibly