kenshou-remote-0.1.0.0: src/Kenshou/Remote/Payload/Publisher.hs
module Kenshou.Remote.Payload.Publisher
( PublishOptions (..),
PreparedPublishOptions (..),
PublishError (..),
PublisherDeps (..),
publishPayload,
publishPayloadWith,
publishPreparedPayload,
publishPreparedPayloadWith,
validateCohortIdentity,
)
where
import Control.Exception (IOException, displayException, finally, try)
import Crypto.Hash.SHA256 qualified as SHA256
import Data.Aeson (eitherDecodeStrict', encode)
import Data.ByteString qualified as ByteString
import Data.ByteString.Base16 qualified as Base16
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as TextEncoding
import Data.Time (getCurrentTime)
import Kenshou.Core.Cohort (CohortDescriptor (..), CohortIdentity (..), CohortName (..), ComponentSpec (..), checkCohort)
import Kenshou.Perf.Request (immutableRevision)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))
import Kenshou.Remote.Payload.Nix (BundleInfo (..), ClosureInfo (..), NixError (..), exportBundle, nixBuild, nixEvalJson)
import Kenshou.Remote.Payload.Publish (BundlePublishError, publishBundle)
import Kenshou.Remote.Store (Bucket (..), ObjectStore)
import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)
import System.Exit (ExitCode (..))
import System.FilePath (takeDirectory, (</>))
import System.IO (hClose, openBinaryTempFile)
import System.Process (readProcessWithExitCode)
data PublishOptions = PublishOptions
{ root :: !FilePath,
cohort :: !Text,
variant :: !Text,
bucket :: !Bucket,
out :: !FilePath,
allowDirty :: !Bool
}
deriving stock (Eq, Show)
data PreparedPublishOptions = PreparedPublishOptions
{ harnessRoot :: !FilePath,
harnessCommit :: !Text,
generatedFlake :: !FilePath,
generatedDescriptor :: !FilePath,
selectedCohort :: !Text,
selectedVariant :: !Text,
controlBucket :: !Bucket,
output :: !FilePath
}
deriving stock (Eq, Show)
data PublishError
= PublishInvalidSelection !Text
| PublishDirtyWorktree
| PublishDescriptorError !Text
| PublishCohortMismatch !Text
| PublishNixError !NixError
| PublishBundleError !BundlePublishError
| PublishIoError !Text
deriving stock (Eq, Show)
data PublisherDeps = PublisherDeps
{ readHarness :: FilePath -> IO (Either PublishError Harness),
evalIdentity :: String -> IO (Either NixError CohortIdentity),
build :: String -> IO (Either NixError FilePath),
export :: FilePath -> FilePath -> IO (Either NixError BundleInfo)
}
defaultDeps :: PublisherDeps
defaultDeps = PublisherDeps gitHarness nixEvalJson nixBuild exportBundle
publishPayload :: ObjectStore -> PublishOptions -> IO (Either PublishError PayloadDescriptor)
publishPayload = publishPayloadWith defaultDeps
publishPayloadWith :: PublisherDeps -> ObjectStore -> PublishOptions -> IO (Either PublishError PayloadDescriptor)
publishPayloadWith deps store options = case flakeAttribute options.cohort options.variant of
Left problem -> pure (Left (PublishInvalidSelection problem))
Right attribute ->
publishAt
deps
store
options
attribute
(options.root <> "#" <> Text.unpack attribute)
(options.root </> "cohort" </> Text.unpack options.cohort <> ".json")
publishPreparedPayload :: ObjectStore -> PreparedPublishOptions -> IO (Either PublishError PayloadDescriptor)
publishPreparedPayload = publishPreparedPayloadWith defaultDeps
publishPreparedPayloadWith :: PublisherDeps -> ObjectStore -> PreparedPublishOptions -> IO (Either PublishError PayloadDescriptor)
publishPreparedPayloadWith deps store prepared
| prepared.selectedVariant `notElem` ["default", "info-table", "profiled"] = pure (Left (PublishInvalidSelection "invalid prepared variant"))
| not ("perf-" `Text.isPrefixOf` prepared.selectedCohort && Text.all (\c -> c `elem` ['a' .. 'z'] || c `elem` ['0' .. '9'] || c == '-') prepared.selectedCohort) = pure (Left (PublishInvalidSelection "prepared cohort must use its generated perf name"))
| not (immutableRevision prepared.harnessCommit) = pure (Left (PublishInvalidSelection "prepared harness commit must be immutable"))
| otherwise = do
harness <- deps.readHarness prepared.harnessRoot
case harness of
Left problem -> pure (Left problem)
Right actual
| actual.dirty -> pure (Left PublishDirtyWorktree)
| actual.revision /= prepared.harnessCommit -> pure (Left (PublishDescriptorError "prepared harness commit differs from the clean build root"))
| otherwise ->
publishAt
deps
store
(PublishOptions prepared.harnessRoot prepared.selectedCohort prepared.selectedVariant prepared.controlBucket prepared.output False)
(preparedAttribute prepared)
(prepared.generatedFlake <> "#" <> Text.unpack (preparedAttribute prepared))
prepared.generatedDescriptor
preparedAttribute :: PreparedPublishOptions -> Text
preparedAttribute prepared =
"packages.x86_64-linux.kenshou-" <> prepared.selectedCohort <> case prepared.selectedVariant of
"default" -> ""
other -> "-" <> other
publishAt :: PublisherDeps -> ObjectStore -> PublishOptions -> Text -> String -> FilePath -> IO (Either PublishError PayloadDescriptor)
publishAt deps store options attribute flake descriptorPath = do
harnessResult <- deps.readHarness options.root
case harnessResult of
Left problem -> pure (Left problem)
Right harness
| harness.dirty && not options.allowDirty -> pure (Left PublishDirtyWorktree)
| otherwise -> do
descriptorResult <- readDescriptor descriptorPath
case descriptorResult of
Left problem -> pure (Left problem)
Right (descriptor, _) | descriptor.descriptorName /= CohortName options.cohort -> pure (Left (PublishDescriptorError "selected descriptor names another cohort"))
Right (descriptor, descriptorSha) -> do
identityResult <- deps.evalIdentity (flake <> ".cohortIdentity")
case identityResult of
Left problem -> pure (Left (PublishNixError problem))
Right identity -> case validateCohortIdentity descriptor descriptorSha identity of
Left problem -> pure (Left (PublishCohortMismatch problem))
Right checked -> do
built <- deps.build flake
case built of
Left problem -> pure (Left (PublishNixError problem))
Right storePath -> withBundleFile options.out \bundlePath -> do
exported <- deps.export storePath bundlePath
case exported of
Left problem -> pure (Left (PublishNixError problem))
Right bundle | bundle.closure.storePath /= storePath -> pure (Left (PublishNixError (NixInvalidOutput "exported closure root differs from the built store path")))
Right bundle -> do
now <- getCurrentTime
let digest = bundle.sha256
object = "payloads/sha256/" <> digest <> ".nar.zst"
payload =
PayloadDescriptor
options.cohort
options.variant
attribute
harness
identity
(CohortCheck checked)
( CellPayload
(Bundle ("gs://" <> options.bucket.unBucket <> "/" <> object) digest bundle.bytes)
(Text.pack storePath)
bundle.closure.narHash
(map Text.pack bundle.closure.closurePaths)
"x86_64-linux"
["bin/kenshou", "cell", "exec"]
)
now
uploaded <- publishBundle store options.bucket payload bundle.file
case uploaded of
Left problem -> pure (Left (PublishBundleError problem))
Right published -> do
written <- writeDescriptor options.out published
pure (published <$ written)
validateCohortIdentity :: CohortDescriptor -> Text -> CohortIdentity -> Either Text Int
validateCohortIdentity descriptor descriptorSha identity
| identity.identityDescriptorSha256 /= descriptorSha = Left "Nix identity descriptor digest differs from the selected descriptor"
| identity.identityResolver /= Just "nix" = Left "payload identity must use the Nix resolver"
| identity.identityOs /= "linux" || identity.identityArch /= "x86_64" = Left "payload identity must target x86_64-linux"
| identity.identityIndexState /= Just descriptor.descriptorIndexState = Left "Nix identity index state differs from the selected descriptor"
| not (null mismatches) = Left (Text.pack (show mismatches))
| otherwise = Right (sum (map (length . (.componentPackages)) descriptor.descriptorComponents))
where
mismatches = checkCohort descriptor identity
flakeAttribute :: Text -> Text -> Either Text Text
flakeAttribute cohort variant
| cohort `notElem` ["released", "head"] = Left "--cohort must be released or head"
| variant `notElem` ["default", "info-table", "profiled"] = Left "--variant must be default, info-table or profiled"
| otherwise = Right ("packages.x86_64-linux.kenshou-" <> cohort <> if variant == "default" then "" else "-" <> variant)
readDescriptor :: FilePath -> IO (Either PublishError (CohortDescriptor, Text))
readDescriptor path = do
contents <- try @IOException (ByteString.readFile path)
pure case contents of
Left problem -> Left (PublishIoError (Text.pack (displayException problem)))
Right bytes -> case eitherDecodeStrict' bytes of
Left problem -> Left (PublishDescriptorError (Text.pack problem))
Right descriptor -> Right (descriptor, TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash bytes)))
gitHarness :: FilePath -> IO (Either PublishError Harness)
gitHarness path = do
revision <- git ["-C", path, "rev-parse", "HEAD"]
status <- git ["-C", path, "status", "--porcelain"]
pure do
full <- Text.strip <$> revision
changed <- status
if Text.length full /= 40 || not (Text.all isHex full)
then Left (PublishDescriptorError "git did not return a full commit revision")
else Right (Harness full (not (Text.null changed)))
where
isHex character = character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']
git :: [String] -> IO (Either PublishError Text)
git arguments = do
result <- try @IOException (readProcessWithExitCode "git" arguments "")
pure case result of
Left problem -> Left (PublishIoError (Text.pack (displayException problem)))
Right (ExitSuccess, output, _) -> Right (Text.pack output)
Right (_, _, errors) -> Left (PublishIoError (Text.pack errors))
withBundleFile :: FilePath -> (FilePath -> IO result) -> IO result
withBundleFile destination operation = do
let directory = takeDirectory destination
createDirectoryIfMissing True directory
(temporary, handle) <- openBinaryTempFile directory ".kenshou-export-"
hClose handle
operation temporary `finally` removeIfPresent temporary
writeDescriptor :: FilePath -> PayloadDescriptor -> IO (Either PublishError ())
writeDescriptor destination descriptor = do
let directory = takeDirectory destination
createDirectoryIfMissing True directory
(temporary, handle) <- openBinaryTempFile directory ".kenshou-descriptor-"
result <- try @IOException do
LazyByteString.hPut handle (encode descriptor) `finally` hClose handle
renameFile temporary destination
case result of
Left problem -> do
removeIfPresent temporary
pure (Left (PublishIoError (Text.pack (displayException problem))))
Right () -> pure (Right ())
removeIfPresent :: FilePath -> IO ()
removeIfPresent path = do
exists <- doesFileExist path
if exists then removeFile path else pure ()