kenshou-core-0.1.0.0: src/Kenshou/Core/Run.hs
module Kenshou.Core.Run
( RunnerConfig (..),
RunOutput (..),
executeRun,
defectDisposition,
)
where
import Control.Exception (SomeException, displayException, fromException, try)
import Control.Monad (when)
import Data.Aeson (ToJSON, Value, encode, object, (.=))
import Data.ByteString qualified as ByteString
import Data.ByteString.Lazy qualified as LazyByteString
import Data.List.NonEmpty (NonEmpty (..))
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (diffUTCTime, getCurrentTime)
import Kenshou.Core.Bundle (Registry, lookupResource, registryMigrationSets)
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))
import Kenshou.Core.Compat (comparisonKey, compatInputs, seriesKey)
import Kenshou.Core.Context
import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))
import Kenshou.Core.Env.Postgres (EnvError (..), PgSettingsSnapshot, PostgresEnv (..), withPostgresEnvKeeping, withPostgresEnvNamed)
import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint, hostValue, kenshouValue, runtimeValue)
import Kenshou.Core.Id (renderRunId, unSeed)
import Kenshou.Core.Log (Severity (Warning), withLogger)
import Kenshou.Core.Manifest (writeManifest)
import Kenshou.Core.Outcome (Outcome (..), outcomeExitCode)
import Kenshou.Core.Resource (ProvisionedResource (..), ResourceName, ResourceProvider (..), renderResourceName)
import Kenshou.Core.RunResult
import Kenshou.Core.RunSpec (CohortExpectation (..), EffectiveRunSpec (..), EnvironmentSpec (..), RunSpec)
import Kenshou.Core.RunSpec.Resolve (SpecError (..), resolveRunSpec)
import Kenshou.Core.Scenario (KnownDefect (..), Scenario (..), ScenarioReport (..), cohortScopeApplies, individualKnownDefects, infrastructureFailureBecause)
import System.Directory (createDirectory, createDirectoryIfMissing, doesDirectoryExist, getCurrentDirectory, renameFile)
import System.FilePath ((</>))
import System.Timeout (timeout)
data RunnerConfig = RunnerConfig
{ registry :: Registry,
outRoot :: FilePath,
cohort :: CohortIdentity,
cellFingerprint :: Maybe Value,
keepEnvironment :: Bool,
strictKnownDefects :: Bool,
argv :: [String]
}
data RunOutput = RunOutput {directory :: FilePath, result :: RunResult, manifestSha256 :: Text}
executeRun :: RunnerConfig -> RunSpec -> IO (Either (NonEmpty SpecError) RunOutput)
executeRun config input = do
resolution <- resolveRunSpec config.registry input
case resolution of
Left problems -> pure (Left problems)
Right (scenario, spec) -> do
let directory = config.outRoot </> Text.unpack (renderRunId spec.runId)
exists <- doesDirectoryExist directory
if exists
then pure (Left (SpecError ("run directory already exists: " <> Text.pack directory) :| []))
else do
createDirectoryIfMissing True config.outRoot
createDirectory directory
createDirectory (directory </> "logs")
atomicEncode (directory </> "run-spec.json") spec
specBytes <- ByteString.readFile (directory </> "run-spec.json")
(result, mediaTypes) <- withLogger (directory </> "logs" </> "harness.jsonl") spec.runId \logger -> do
startedAt <- getCurrentTime
state <- newRunState
let context = RunContext spec.runId spec.scenario spec.knobs spec.dimensions spec.seed spec.phases (Environment Nothing Map.empty Map.empty) spec.environment spec.comparison directory logger state
(report, postgresSnapshot, extraPostgresSnapshots, resourceFingerprints) <- runScenario config scenario spec context
endedAt <- getCurrentTime
host <- collectHostFingerprint
when (host.revision == Nothing) (observe context "harness" Warning "harness revision is unavailable")
(summaries, observations, phaseTimings, mediaTypes) <- readRunState state
workingDirectory <- getCurrentDirectory
let (defect, blocking, exitCode) = defectDisposition config.strictKnownDefects config.cohort scenario report
schemas = maybe [] (.schemas) scenario.requires.postgres
inputs = compatInputs spec postgresSnapshot schemas config.cohort
compatibility = object ["algorithm" .= ("kenshou.compat-key/v1" :: Text), "comparisonKey" .= comparisonKey inputs, "seriesKey" .= seriesKey inputs, "inputs" .= inputs]
fingerprint =
object $
["placement" .= spec.environment.placement, "machineProfile" .= spec.environment.machineProfile, "host" .= hostValue host, "runtime" .= runtimeValue host, "kenshou" .= kenshouValue host, "postgres" .= postgresSnapshot, "extraPostgres" .= extraPostgresSnapshots, "cell" .= config.cellFingerprint]
<> ["resources" .= resourceFingerprints | not (Map.null resourceFingerprints)]
invocation = object ["argv" .= config.argv, "workingDirectory" .= workingDirectory]
result = RunResult spec.runId spec.scenario spec.scenarioRevision scenario.tier report.outcome blocking exitCode report.reason report.failures defect (unSeed spec.seed) (sha256Hex specBytes) spec.comparison startedAt endedAt (realToFrac (diffUTCTime endedAt startedAt)) phaseTimings config.cohort fingerprint compatibility summaries observations invocation
atomicEncode (directory </> "run-result.json") result
pure (result, mediaTypes)
manifest <- writeManifest directory spec.runId mediaTypes
atomicEncode (directory </> "manifest.json") manifest
manifestBytes <- ByteString.readFile (directory </> "manifest.json")
pure (Right (RunOutput directory result (sha256Hex manifestBytes)))
runScenario :: RunnerConfig -> Scenario -> EffectiveRunSpec -> RunContext -> IO (ScenarioReport, Maybe PgSettingsSnapshot, Map.Map Text PgSettingsSnapshot, Map.Map ResourceName Value)
runScenario config scenario spec context
| cohortMismatch config.cohort spec.cohortExpectation = pure (infrastructureFailureBecause "cohort-mismatch", Nothing, Map.empty, Map.empty)
| otherwise = do
nested <- withPrimary scenario.requires.postgres spec.environment.postgres \primary ->
withExtras scenario.requires.extraPostgres spec.environment.extraPostgres Map.empty \extras ->
withResources scenario.requires.resources Map.empty \resources -> do
let environment = Environment primary extras resources
report <- timed context {env = environment}
pure (report, (.snapshot) <$> primary, fmap (.snapshot) extras, fmap (.fingerprint) resources)
let provisioned = nested >>= id >>= id
pure (either (\err -> (infrastructureFailureBecause (Text.pack (show err)), Nothing, Map.empty, Map.empty)) id provisioned)
where
-- Providers are bracketed in declaration order inside the PostgreSQL contexts; a
-- provider failure is an infrastructure failure like a PostgreSQL one.
withResources [] accumulated action = Right <$> action accumulated
withResources (name : rest) accumulated action = case (lookupResource config.registry name, Map.lookup name spec.environment.resources) of
(Nothing, _) -> pure (Left (EnvError ("no registered provider for resource " <> renderResourceName name)))
(_, Nothing) -> pure (Left (EnvError ("resource specification is missing: " <> renderResourceName name)))
(Just ResourceProvider {withLocal = acquire}, Just value) -> do
attempted <- try (acquire context value \provisioned -> withResources rest (Map.insert name provisioned accumulated) action)
pure case attempted of
Left (exception :: SomeException) -> Left (EnvError ("resource " <> renderResourceName name <> " failed: " <> Text.pack (displayException exception)))
Right inner -> inner
withPrimary Nothing _ action = Right <$> action Nothing
withPrimary (Just requirement) (Just postgresSpec) action = withPostgresEnvKeeping (registryMigrationSets config.registry) config.keepEnvironment context.logger context.outDir spec.runId requirement postgresSpec spec.dimensions (action . Just)
withPrimary (Just _) Nothing _ = pure (Left (EnvError "PostgreSQL specification is missing"))
withExtras [] _ accumulated action = Right <$> action accumulated
withExtras ((name, requirement) : rest) postgresSpecs accumulated action = case Map.lookup name postgresSpecs of
Nothing -> pure (Left (EnvError ("extra PostgreSQL specification is missing: " <> name)))
Just postgresSpec -> do
provisioned <- withPostgresEnvNamed (registryMigrationSets config.registry) config.keepEnvironment (Just name) context.logger context.outDir spec.runId requirement postgresSpec spec.dimensions \postgres ->
withExtras rest postgresSpecs (Map.insert name postgres accumulated) action
pure (provisioned >>= id)
timed runContext = do
execution <- try (timeout (spec.timeoutSeconds * 1000000) (scenario.run runContext)) :: IO (Either SomeException (Maybe ScenarioReport))
pure case execution of
Left exception -> case fromException exception of
Just (InfrastructureError message) -> ScenarioReport InfrastructureFailure (Just message) []
Nothing -> ScenarioReport Errored (Just (Text.pack (displayException exception))) []
Right Nothing -> ScenarioReport Errored (Just ("timeout after " <> Text.pack (show spec.timeoutSeconds) <> " s")) []
Right (Just report) -> report
cohortMismatch :: CohortIdentity -> Maybe CohortExpectation -> Bool
cohortMismatch _ Nothing = False
cohortMismatch cohort (Just expectation) = unPlanHash cohort.identityPlanHash /= expectation.planHash
defectDisposition :: Bool -> CohortIdentity -> Scenario -> ScenarioReport -> (Maybe (KnownDefect, KnownDefectStatus), Bool, Int)
defectDisposition strict cohort scenario report = case NonEmpty.nonEmpty applicable of
Nothing -> ordinary
Just defects ->
let status
| report.outcome == Failed && not (null report.failures) && all (`elem` coveredLabels) report.failures = DefectReproduced
| report.outcome == Failed = DefectDifferentFailure
| otherwise = DefectNotReproduced
blocking = report.outcome /= Passed && status /= DefectReproduced
exitCode = if status == DefectReproduced && not strict then 0 else outcomeExitCode report.outcome
identified = case NonEmpty.toList defects of
[single] -> single
_ -> KnownDefectGroup defects
in (Just (identified, status), blocking, exitCode)
where
applicable = filter (cohortScopeApplies cohort . (.appliesTo)) (maybe [] individualKnownDefects scenario.knownDefect)
coveredLabels = concatMap (.expectedFailures) applicable
ordinary = (Nothing, report.outcome /= Passed, outcomeExitCode report.outcome)
atomicEncode :: (ToJSON value) => FilePath -> value -> IO ()
atomicEncode path value = do
let temporary = path <> ".tmp"
LazyByteString.writeFile temporary (encode value <> "\n")
renameFile temporary path