kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Parity.hs
module Kenshou.Remote.Cell.Parity
( ParityOptions (..),
ParityDifference (..),
ParityReport (..),
compareForParity,
)
where
import Crypto.Hash.SHA256 qualified as SHA256
import Data.Aeson (ToJSON (..), Value (..), eitherDecodeStrict', object, (.=))
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.ByteString.Base16 qualified as Base16
import Data.Foldable (toList)
import Data.List (isPrefixOf, sort)
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as TextEncoding
import System.Directory (doesDirectoryExist, listDirectory)
import System.FilePath (takeExtension, (</>))
newtype ParityOptions = ParityOptions {volatilePaths :: [Text]}
deriving stock (Eq, Show)
data ParityDifference = ParityDifference
{ path :: !Text,
localValue :: !(Maybe Value),
cellValue :: !(Maybe Value)
}
deriving stock (Eq, Show)
instance ToJSON ParityDifference where
toJSON difference =
object
[ "path" .= difference.path,
"local" .= difference.localValue,
"cell" .= difference.cellValue
]
data ParityReport = ParityReport
{ schema :: !Text,
localRunResultSha256 :: !Text,
cellRunResultSha256 :: !Text,
equal :: ![Text],
intentional :: ![ParityDifference],
unexpected :: ![ParityDifference]
}
deriving stock (Eq, Show)
instance ToJSON ParityReport where
toJSON report =
object
[ "schema" .= report.schema,
"localRunResultSha256" .= report.localRunResultSha256,
"cellRunResultSha256" .= report.cellRunResultSha256,
"equal" .= report.equal,
"intentional" .= report.intentional,
"unexpected" .= report.unexpected
]
compareForParity :: ParityOptions -> FilePath -> FilePath -> IO ParityReport
compareForParity options localDir cellDir = do
localBytes <- ByteString.readFile (localDir </> "run-result.json")
cellBytes <- ByteString.readFile (cellDir </> "run-result.json")
localResult <- decode "local run-result.json" localBytes
cellResult <- decode "cell run-result.json" cellBytes
localSpec <- readJson "local run-spec.json" (localDir </> "run-spec.json")
cellSpec <- readJson "cell run-spec.json" (cellDir </> "run-spec.json")
localVerdicts <- readVerdicts localDir
cellVerdicts <- readVerdicts cellDir
let comparisons =
compareJson ["result"] localResult cellResult
<> compareJson ["spec"] localSpec cellSpec
<> compareVerdicts localVerdicts cellVerdicts
allEqual = [renderPath path | (path, Just left, Just right) <- comparisons, left == right]
differences = [ParityDifference (renderPath path) left right | (path, left, right) <- comparisons, left /= right]
intentionalDiffs = filter (isIntentional options . (.path)) differences
unexpectedDiffs = filter (not . isIntentional options . (.path)) differences
pure
ParityReport
{ schema = "kenshou.parity-report/v1",
localRunResultSha256 = digest localBytes,
cellRunResultSha256 = digest cellBytes,
equal = allEqual,
intentional = intentionalDiffs,
unexpected = unexpectedDiffs
}
decode :: Text -> ByteString.ByteString -> IO Value
decode label bytes =
case eitherDecodeStrict' bytes of
Left problem -> ioError (userError (Text.unpack label <> ": " <> problem))
Right value -> pure value
readJson :: Text -> FilePath -> IO Value
readJson label path = ByteString.readFile path >>= decode label
readVerdicts :: FilePath -> IO (Map.Map Text Value)
readVerdicts runDir = do
let verdictDir = runDir </> "verdicts"
exists <- doesDirectoryExist verdictDir
if not exists
then pure Map.empty
else do
names <- sort . filter ((== ".json") . takeExtension) <$> listDirectory verdictDir
Map.fromList
<$> traverse
( \name -> do
value <- readJson (Text.pack name) (verdictDir </> name)
pure (Text.pack name, projectVerdict value)
)
names
projectVerdict :: Value -> Value
projectVerdict (Object value) =
Object (KeyMap.filterWithKey (\key _ -> Key.toText key `elem` ["status", "counts"]) value)
projectVerdict value = value
compareVerdicts :: Map.Map Text Value -> Map.Map Text Value -> [([Text], Maybe Value, Maybe Value)]
compareVerdicts localVerdicts cellVerdicts =
concatMap compareName (Set.toAscList (Map.keysSet localVerdicts `Set.union` Map.keysSet cellVerdicts))
where
compareName name =
case (Map.lookup name localVerdicts, Map.lookup name cellVerdicts) of
(Just localValue, Just cellValue) -> compareJson ["verdicts", name] localValue cellValue
(left, right) -> [(["verdicts", name], left, right)]
compareJson :: [Text] -> Value -> Value -> [([Text], Maybe Value, Maybe Value)]
compareJson prefix localValue cellValue =
[ (path, Map.lookup path localLeaves, Map.lookup path cellLeaves)
| path <- Set.toAscList (Map.keysSet localLeaves `Set.union` Map.keysSet cellLeaves)
]
where
localLeaves = Map.fromList (flatten prefix localValue)
cellLeaves = Map.fromList (flatten prefix cellValue)
flatten :: [Text] -> Value -> [([Text], Value)]
flatten prefix (Object value)
| not (KeyMap.null value) = concatMap (\(key, child) -> flatten (prefix <> [Key.toText key]) child) (KeyMap.toList value)
flatten prefix (Array value)
| not (null value) = concatMap (\(index, child) -> flatten (prefix <> [Text.pack (show index)]) child) (zip [0 :: Int ..] (toList value))
flatten prefix value = [(prefix, value)]
renderPath :: [Text] -> Text
renderPath = Text.intercalate "."
isIntentional :: ParityOptions -> Text -> Bool
isIntentional options rendered =
natural || volatile
where
path = Text.splitOn "." rendered
starts prefix = prefix `isPrefixOf` path
natural =
or
[ starts ["result", "runId"],
starts ["result", "timings"],
starts ["result", "compatibility"],
starts ["result", "invocation"],
starts ["result", "spec", "sha256"],
starts ["result", "spec", "path"],
starts ["result", "cohort", "arch"],
starts ["result", "cohort", "os"],
starts ["result", "cohort", "resolver"],
starts ["result", "cohort", "planHash"],
starts ["result", "cohort", "cabalVersion"],
starts ["result", "fingerprint", "cell"],
starts ["result", "fingerprint", "host"],
starts ["result", "fingerprint", "runtime"],
starts ["result", "fingerprint", "machineProfile"],
starts ["result", "fingerprint", "placement"],
starts ["result", "fingerprint", "postgres"],
starts ["result", "fingerprint", "kenshou", "executableSha256"],
starts ["result", "fingerprint", "kenshou", "revision"],
starts ["spec", "runId"],
starts ["spec", "cohortExpectation"],
starts ["spec", "environment", "placement"],
starts ["spec", "environment", "machineProfile"],
starts ["spec", "environment", "postgres"],
path == ["spec", "labels"],
starts ["spec", "labels", "postgresPlacement"]
]
volatileScope = starts ["result", "summaries", "verdicts"] || starts ["verdicts"]
volatile = volatileScope && any (\selected -> selected == rendered || (selected <> ".") `Text.isPrefixOf` rendered) options.volatilePaths
digest :: ByteString.ByteString -> Text
digest bytes = "sha256:" <> TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash bytes))