packages feed

kenshou-evidence-0.1.0.0: src/Kenshou/Evidence/Publish.hs

module Kenshou.Evidence.Publish
  ( UploadMode (..),
    PublishOptions (..),
    PublishError (..),
    publishRunData,
    publishComparisonData,
  )
where

import Control.Exception (IOException, try)
import Control.Monad (forM, unless, when)
import Control.Monad.Trans.Class (lift)
import Control.Monad.Trans.Except (ExceptT, runExceptT, throwE)
import Data.ByteString qualified as ByteString
import Data.List (sortOn)
import Data.Maybe (catMaybes)
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Id (renderRunId)
import Kenshou.Core.Outcome (Outcome (..))
import Kenshou.Evidence.Source (CellEvidence (..), ComparisonSource (..), ComparisonView (..), RunResultView (..), RunSource (..), VerifiedFile (..))
import Kenshou.Evidence.Store (ObjectStat (..), ObjectStore (..), StoreError, validateObjectUri)
import Kenshou.Evidence.Types (DataKind (..), DataLink (..), Sha256, sha256Bytes)
import Kenshou.Remote.Cell.Docs (CellManifest (..))
import Numeric.Natural (Natural)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)

data UploadMode = UploadMissing | VerifyOnly deriving stock (Eq, Show)

data PublishOptions = PublishOptions
  { baseUri :: !Text,
    uploadMode :: !UploadMode,
    deepVerify :: !Bool,
    linkLogs :: !Bool
  }
  deriving stock (Eq, Show)

data PublishError
  = InvalidBaseUri !Text
  | PublishIo !Text
  | StoreFailure !StoreError
  | MissingObject !Text
  | ObjectMismatch !Text
  deriving stock (Eq, Show)

-- | Publish every file covered by the verified manifest, even if only a
-- subset will be linked directly. The manifest link pins the rest transitively.
publishRunData :: ObjectStore -> PublishOptions -> FilePath -> RunSource -> IO (Either PublishError [DataLink])
publishRunData store options root source = case validateRunBase source options.baseUri of
  Left err -> pure (Left err)
  Right base -> do
    completed <-
      try
        ( runExceptT do
            let runPrefix = base <> "/" <> renderRunId source.result.resultRunId
                entries = sortOn (.relativePath) source.files
            linked <- forM entries $ \entry -> do
              let uri = runPrefix <> "/" <> Text.pack entry.relativePath
                  path = root </> entry.relativePath
              publishOne store options path uri entry.mediaType entry.digest entry.bytes
              pure $ do
                kind <- kindForPath options.linkLogs (source.result.resultOutcome /= Passed) entry.relativePath
                pure DataLink {kind, uri, digest = entry.digest, mediaType = entry.mediaType, bytes = entry.bytes}
            manifestBytes <- lift (ByteString.readFile (root </> "manifest.json"))
            let manifestUri = runPrefix <> "/manifest.json"
                manifestDigest = sha256Bytes manifestBytes
                manifestSize = fromIntegral (ByteString.length manifestBytes)
            publishOne store options (root </> "manifest.json") manifestUri "application/json" manifestDigest manifestSize
            cellLinks <- case source.cellEvidence of
              Nothing -> pure []
              Just cell -> do
                let uri = Text.dropEnd (Text.length "/output") base <> "/manifest.json"
                    file = cell.cellManifestFile
                publishOne store (options {uploadMode = VerifyOnly}) cell.cellManifestPath uri "application/json" file.digest file.bytes
                pure [DataLink {kind = CellManifestData, uri, digest = file.digest, mediaType = "application/json", bytes = file.bytes}]
            pure $
              sortOn (\link -> (link.kind, link.uri)) $
                DataLink {kind = ManifestData, uri = manifestUri, digest = manifestDigest, mediaType = "application/json", bytes = manifestSize} : cellLinks <> catMaybes linked
        ) ::
        IO (Either IOException (Either PublishError [DataLink]))
    pure $ either (Left . PublishIo . Text.pack . show) id completed

publishComparisonData :: ObjectStore -> PublishOptions -> FilePath -> ComparisonSource -> IO (Either PublishError DataLink)
publishComparisonData store options path source = case cleanBase options.baseUri of
  Left err -> pure (Left err)
  Right base -> do
    let entry = source.file
        uri = base <> "/" <> renderRunId source.view.comparisonId <> "/comparison.json"
    completed <- try (runExceptT (publishOne store options path uri entry.mediaType entry.digest entry.bytes)) :: IO (Either IOException (Either PublishError ()))
    pure $ do
      either (Left . PublishIo . Text.pack . show) id completed
      pure DataLink {kind = ComparisonData, uri, digest = entry.digest, mediaType = entry.mediaType, bytes = entry.bytes}

cleanBase :: Text -> Either PublishError Text
cleanBase raw = do
  let base = Text.dropWhileEnd (== '/') raw
  case validateObjectUri (base <> "/probe") of
    Left _ -> Left (InvalidBaseUri raw)
    Right () -> Right base

validateRunBase :: RunSource -> Text -> Either PublishError Text
validateRunBase source raw = do
  base <- cleanBase raw
  case source.cellEvidence of
    Nothing -> pure ()
    Just cell -> do
      let expectedSuffix = "/runs/" <> renderRunId cell.cellManifest.runId <> "/output"
      unless (expectedSuffix `Text.isSuffixOf` base) (Left (InvalidBaseUri raw))
  pure base

publishOne :: ObjectStore -> PublishOptions -> FilePath -> Text -> Text -> Sha256 -> Natural -> ExceptT PublishError IO ()
publishOne store options source uri mediaType expectedDigest expectedSize = do
  contents <- lift (ByteString.readFile source)
  when (fromIntegral (ByteString.length contents) /= expectedSize || sha256Bytes contents /= expectedDigest) (throwE (ObjectMismatch uri))
  result <- lift $ withSystemTempDirectory "kenshou-evidence-publish" $ \scratch -> do
    let snapshot = scratch </> "snapshot"
    ByteString.writeFile snapshot contents
    runExceptT (publishSnapshot snapshot scratch)
  either throwE pure result
  where
    publishSnapshot snapshot scratch = do
      case options.uploadMode of
        UploadMissing -> do
          uploaded <- lift (store.putObjectIfAbsent snapshot uri mediaType)
          either (throwE . StoreFailure) (const (pure ())) uploaded
        VerifyOnly -> pure ()
      observed <- lift (store.statObject uri) >>= either (throwE . StoreFailure) pure
      stat <- maybe (throwE (MissingObject uri)) pure observed
      when (stat.bytes /= expectedSize || maybe False (/= expectedDigest) stat.recordedSha256) (throwE (ObjectMismatch uri))
      -- Cell-owned objects carry their SHA-256 in the sealed manifest, not
      -- GCS custom metadata. Verify their bytes when metadata is absent.
      when (options.deepVerify || stat.recordedSha256 == Nothing) do
        let target = scratch </> "downloaded"
        fetched <- lift (store.fetchObject uri target)
        either (throwE . StoreFailure) pure fetched
        downloaded <- lift (ByteString.readFile target)
        unless (fromIntegral (ByteString.length downloaded) == expectedSize && sha256Bytes downloaded == expectedDigest) (throwE (ObjectMismatch uri))

kindForPath :: Bool -> Bool -> FilePath -> Maybe DataKind
kindForPath linkLogs failed path
  | path == "run-spec.json" = Just RunSpecData
  | path == "run-result.json" = Just RunResultData
  | path == "cell/manifest.json" = Just CellManifestData
  | "samples/" `Text.isPrefixOf` item = Just SamplesData
  | "series/" `Text.isPrefixOf` item = Just SeriesData
  | "verdicts/" `Text.isPrefixOf` item = Just VerdictsData
  | "diagnosis/" `Text.isPrefixOf` item = Just DiagnosisData
  | "logs/" `Text.isPrefixOf` item && (linkLogs || failed) = Just LogsData
  | otherwise = Nothing
  where
    item = Text.pack path