kenshou-check-0.1.0.0: src/Kenshou/Check/Verdict.hs
module Kenshou.Check.Verdict
( InvariantClass (..),
VerdictStatus (..),
InputRef (..),
Replay (..),
RunInfo (..),
Verdict (..),
writeVerdict,
outcomeFromVerdicts,
)
where
import Control.Monad (unless)
import Data.Aeson
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Int (Int64)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime)
import Kenshou.Core.Id (RunId, ScenarioId)
import Kenshou.Core.Outcome (Outcome (..))
import System.Directory (createDirectoryIfMissing, renameFile)
import System.FilePath ((</>))
data InvariantClass = Contract | Implementation deriving stock (Eq, Ord, Show)
data VerdictStatus = Held | Violated | NotEvaluated deriving stock (Eq, Ord, Show)
data InputRef = InputRef {path :: !FilePath, sha256 :: !Text} deriving stock (Eq, Show)
data Replay = Replay
{ seed :: !Word,
size :: !Int,
shrinkPath :: ![Int],
command :: !Text
}
deriving stock (Eq, Show)
data RunInfo = RunInfo {runId :: !RunId, scenario :: !ScenarioId} deriving stock (Eq, Show)
data Verdict = Verdict
{ checker :: !Text,
invariant :: !Text,
cls :: !InvariantClass,
status :: !VerdictStatus,
reason :: !(Maybe Text),
summary :: !Text,
counts :: !(Map Text Int64),
parameters :: !Value,
counterExamples :: ![Value],
counterExamplesTruncated :: !Bool,
inputs :: ![InputRef],
replay :: !(Maybe Replay),
checkedAt :: !UTCTime,
durationMillis :: !Int64
}
deriving stock (Eq, Show)
writeVerdict :: FilePath -> RunInfo -> Verdict -> IO FilePath
writeVerdict directory runInfo verdict = do
unless (all (`Map.member` verdict.counts) ["examined", "violations"]) $
ioError (userError ("verdict " <> Text.unpack verdict.checker <> " lacks required examined/violations counters"))
createDirectoryIfMissing True directory
let path = directory </> sanitise verdict.checker <> ".json"
temporary = path <> ".tmp"
LazyByteString.writeFile temporary (encode (verdictDocument runInfo verdict) <> "\n")
renameFile temporary path
pure path
where
sanitise = fmap (\character -> if character == '/' then '-' else character) . Text.unpack
outcomeFromVerdicts :: [Verdict] -> Outcome
outcomeFromVerdicts verdicts
| any isContractViolation verdicts = Failed
| any isSearchExhausted verdicts = Inconclusive
| any isContractUnevaluated verdicts = Errored
| otherwise = Passed
where
isContractViolation verdict = verdict.cls == Contract && verdict.status == Violated
isSearchExhausted verdict = verdict.cls == Contract && verdict.status == NotEvaluated && verdict.reason == Just "search-budget-exhausted"
isContractUnevaluated verdict = verdict.cls == Contract && verdict.status == NotEvaluated
verdictDocument :: RunInfo -> Verdict -> Value
verdictDocument runInfo verdict =
object
[ "schema" .= ("kenshou.verdict/v1" :: Text),
"runId" .= runInfo.runId,
"scenario" .= runInfo.scenario,
"checker" .= verdict.checker,
"invariant" .= verdict.invariant,
"class" .= classText verdict.cls,
"status" .= statusText verdict.status,
"blocking" .= (verdict.cls == Contract),
"reason" .= verdict.reason,
"summary" .= verdict.summary,
"counts" .= verdict.counts,
"parameters" .= verdict.parameters,
"counterExamples" .= verdict.counterExamples,
"counterExamplesTruncated" .= verdict.counterExamplesTruncated,
"inputs" .= verdict.inputs,
"replay" .= verdict.replay,
"checkedAt" .= verdict.checkedAt,
"durationMillis" .= verdict.durationMillis
]
classText :: InvariantClass -> Text
classText Contract = "contract"
classText Implementation = "implementation"
statusText :: VerdictStatus -> Text
statusText Held = "held"
statusText Violated = "violated"
statusText NotEvaluated = "not-evaluated"
instance ToJSON InputRef where toJSON value = object ["path" .= value.path, "sha256" .= value.sha256]
instance ToJSON Replay where
toJSON value = object ["seed" .= value.seed, "size" .= value.size, "shrinkPath" .= value.shrinkPath, "command" .= value.command]