kenshou-evidence-0.1.0.0: src/Kenshou/Evidence/Store.hs
module Kenshou.Evidence.Store
( ObjectStat (..),
ObjectStore (..),
StoreError (..),
PutResult (..),
durableSchemes,
validateObjectUri,
directoryStore,
memoryStore,
gcloudStore,
gcloudStoreWith,
)
where
import Control.Concurrent.MVar (modifyMVar, newMVar, readMVar)
import Control.Exception (IOException, try)
import Data.Aeson (Result (..), Value (..), eitherDecodeStrict', fromJSON)
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString (ByteString)
import Data.ByteString qualified as ByteString
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as Text
import Kenshou.Evidence.Types (Sha256 (..), mkSha256, sha256Bytes)
import Numeric.Natural (Natural)
import System.Directory (createDirectoryIfMissing, doesFileExist)
import System.Exit (ExitCode (..))
import System.FilePath (takeDirectory, (</>))
import System.IO (hFlush)
import System.IO.Error (isAlreadyExistsError)
import System.IO.Temp (withTempFile)
import System.Posix.Files (createLink)
import System.Process (proc, readCreateProcessWithExitCode)
import Text.Read (readMaybe)
data ObjectStat = ObjectStat
{ bytes :: !Natural,
recordedSha256 :: !(Maybe Sha256)
}
deriving stock (Eq, Show)
data StoreError
= InvalidObjectUri !Text
| StoreIo !Text
| ObjectConflict !Text
deriving stock (Eq, Show)
data PutResult = ObjectCreated | ObjectPresent deriving stock (Eq, Show)
data ObjectStore = ObjectStore
{ statObject :: Text -> IO (Either StoreError (Maybe ObjectStat)),
fetchObject :: Text -> FilePath -> IO (Either StoreError ()),
putObjectIfAbsent :: FilePath -> Text -> Text -> IO (Either StoreError PutResult)
}
durableSchemes :: [Text]
durableSchemes = ["gs"]
validateObjectUri :: Text -> Either StoreError ()
validateObjectUri = fmap (const ()) . objectPath ""
-- | The scratch store preserves the gs:// URI in records while keeping the
-- bytes under root/bucket/key. Create-only publication uses an atomic hard link.
directoryStore :: FilePath -> ObjectStore
directoryStore root =
ObjectStore
{ statObject = \uri -> withPath uri $ \path -> ioResult do
exists <- doesFileExist path
if exists then Just . objectStat <$> ByteString.readFile path else pure Nothing,
fetchObject = \uri target -> withPath uri $ \path ->
joinEither <$> ioResult do
exists <- doesFileExist path
if not exists
then pure (Left (StoreIo ("object does not exist: " <> uri)))
else do
createDirectoryIfMissing True (takeDirectory target)
ByteString.readFile path >>= ByteString.writeFile target
pure (Right ()),
putObjectIfAbsent = \source uri _mediaType -> withPath uri $ \path -> do
loaded <- ioResult (ByteString.readFile source)
case loaded of
Left err -> pure (Left err)
Right contents ->
joinEither <$> ioResult do
let parent = takeDirectory path
createDirectoryIfMissing True parent
withTempFile parent ".kenshou-object-" $ \temporary handle -> do
ByteString.hPut handle contents
hFlush handle
linked <- try (createLink temporary path)
case linked of
Right () -> pure (Right ObjectCreated)
Left err
| isAlreadyExistsError err -> do
present <- ByteString.readFile path
pure $ if present == contents then Right ObjectPresent else Left (ObjectConflict uri)
| otherwise -> pure (Left (StoreIo (Text.pack (show (err :: IOException)))))
}
where
withPath :: Text -> (FilePath -> IO (Either StoreError value)) -> IO (Either StoreError value)
withPath uri action = case objectPath root uri of
Left err -> pure (Left err)
Right path -> action path
memoryStore :: IO ObjectStore
memoryStore = do
objects <- newMVar Map.empty
pure
ObjectStore
{ statObject = \uri -> case objectPath "" uri of
Left err -> pure (Left err)
Right _ -> do
contents <- Map.lookup uri <$> readMVar objects
pure (Right (objectStat <$> contents)),
fetchObject = \uri target -> case objectPath "" uri of
Left err -> pure (Left err)
Right _ -> do
contents <- Map.lookup uri <$> readMVar objects
case contents of
Nothing -> pure (Left (StoreIo ("object does not exist: " <> uri)))
Just bytes -> ioResult do
createDirectoryIfMissing True (takeDirectory target)
ByteString.writeFile target bytes,
putObjectIfAbsent = \source uri _mediaType -> case objectPath "" uri of
Left err -> pure (Left err)
Right _ -> do
loaded <- ioResult (ByteString.readFile source)
case loaded of
Left err -> pure (Left err)
Right contents -> modifyMVar objects $ \current ->
case Map.lookup uri current of
Nothing -> pure (Map.insert uri contents current, Right ObjectCreated)
Just old | old == contents -> pure (current, Right ObjectPresent)
Just _ -> pure (current, Left (ObjectConflict uri))
}
-- | The production adapter accepts only the explicitly selected GCP project.
-- The executable seam lets tests exercise the command protocol without cloud
-- credentials or remote mutations.
gcloudStore :: Text -> ObjectStore
gcloudStore = gcloudStoreWith "gcloud"
gcloudStoreWith :: FilePath -> Text -> ObjectStore
gcloudStoreWith executable project = store
where
store =
ObjectStore
{ statObject = \uri -> do
ready <- preflight
case ready of
Left err -> pure (Left err)
Right () -> stat uri,
fetchObject = \uri target -> do
ready <- preflight
case ready of
Left err -> pure (Left err)
Right () -> case validateObjectUri uri of
Left err -> pure (Left err)
Right () -> do
prepared <- ioResult (createDirectoryIfMissing True (takeDirectory target))
case prepared of
Left err -> pure (Left err)
Right () -> do
result <- command ["storage", "cp", Text.unpack uri, target, projectFlag, "--quiet"]
pure $ case result of
Left err -> Left err
Right _ -> Right (),
putObjectIfAbsent = \source uri mediaType -> do
ready <- preflight
case ready of
Left err -> pure (Left err)
Right () -> case validateObjectUri uri of
Left err -> pure (Left err)
Right () -> do
local <- ioResult (ByteString.readFile source)
case local of
Left err -> pure (Left err)
Right contents -> do
let expected = objectStat contents
existing <- stat uri
case existing of
Left err -> pure (Left err)
Right (Just observed) -> pure (if observed == expected then Right ObjectPresent else Left (ObjectConflict uri))
Right Nothing -> do
let Sha256 digest = sha256Bytes contents
uploaded <- command ["storage", "cp", source, Text.unpack uri, "--if-generation-match=0", "--custom-metadata=kenshou-sha256=" <> Text.unpack digest, "--content-type=" <> Text.unpack mediaType, projectFlag, "--quiet"]
after <- stat uri
pure case after of
Left err -> Left err
Right (Just observed)
| observed == expected -> Right (if either (const False) (const True) uploaded then ObjectCreated else ObjectPresent)
| otherwise -> Left (ObjectConflict uri)
Right Nothing -> either Left (const (Left (StoreIo ("uploaded object is not visible: " <> uri)))) uploaded
}
projectFlag = "--project=" <> Text.unpack project
preflight = do
if Text.null project
then pure (Left (StoreIo "GCP project must be explicit"))
else do
active <- command ["config", "get-value", "project"]
pure case active of
Left err -> Left err
Right value
| Text.strip value == project -> Right ()
| otherwise -> Left (StoreIo ("active GCP project differs from requested project " <> project))
stat uri = case validateObjectUri uri of
Left err -> pure (Left err)
Right () -> do
result <- command ["storage", "objects", "describe", Text.unpack uri, "--format=json", projectFlag, "--quiet"]
pure case result of
Left (StoreIo message) | missingObject message -> Right Nothing
Left err -> Left err
Right output -> Just <$> parseGcloudStat output
command arguments = do
executed <- try (readCreateProcessWithExitCode (proc executable arguments) "") :: IO (Either IOException (ExitCode, String, String))
pure case executed of
Left err -> Left (StoreIo (Text.pack (show err)))
Right (ExitSuccess, output, _) -> Right (Text.pack output)
Right (ExitFailure code, _, stderrText) -> Left (StoreIo ("gcloud exited " <> Text.pack (show code) <> ": " <> Text.take 1200 (Text.pack stderrText)))
missingObject :: Text -> Bool
missingObject message = any (`Text.isInfixOf` Text.toCaseFold message) ["not_found", "not found", "404", "no urls matched"]
parseGcloudStat :: Text -> Either StoreError ObjectStat
parseGcloudStat output = do
value <- either (Left . StoreIo . Text.pack) Right (eitherDecodeStrict' (Text.encodeUtf8 output))
case value of
Object fields -> do
sizeValue <- maybe (Left (StoreIo "gcloud object description has no size")) Right (KeyMap.lookup "size" fields)
size <- case sizeValue of
String raw -> parseSize raw
Number _ -> case (fromJSON sizeValue :: Result Integer) of
Success number | number >= 0 -> Right (fromInteger number)
_ -> Left (StoreIo "gcloud object size is invalid")
_ -> Left (StoreIo "gcloud object size is not numeric")
let customMetadata = case KeyMap.lookup "custom_fields" fields of
Just metadataValue -> Just metadataValue
Nothing -> KeyMap.lookup "metadata" fields
digest <- case customMetadata of
Nothing -> Right Nothing
Just (Object metadata) -> case KeyMap.lookup "kenshou-sha256" metadata of
Nothing -> Right Nothing
Just (String raw) -> Just <$> either (Left . StoreIo) Right (mkSha256 raw)
_ -> Left (StoreIo "gcloud object digest metadata is not text")
_ -> Left (StoreIo "gcloud object custom metadata is not an object")
Right ObjectStat {bytes = size, recordedSha256 = digest}
_ -> Left (StoreIo "gcloud object description is not an object")
where
parseSize raw = case readMaybe (Text.unpack raw) :: Maybe Integer of
Just value | value >= 0 -> Right (fromInteger value)
_ -> Left (StoreIo "gcloud object size is invalid")
objectStat :: ByteString -> ObjectStat
objectStat contents =
ObjectStat
{ bytes = fromIntegral (ByteString.length contents),
recordedSha256 = Just (sha256Bytes contents)
}
objectPath :: FilePath -> Text -> Either StoreError FilePath
objectPath root uri = do
rest <- maybe (Left (InvalidObjectUri uri)) Right (Text.stripPrefix "gs://" uri)
let segments = Text.splitOn "/" rest
if length segments < 2 || any invalidSegment segments
then Left (InvalidObjectUri uri)
else Right (foldl (</>) root (Text.unpack <$> segments))
where
invalidSegment part =
Text.null part
|| part == "."
|| part == ".."
|| Text.any (`elem` ['\\', '?', '#']) part
ioResult :: IO value -> IO (Either StoreError value)
ioResult action = do
result <- try action
pure $ case result of
Left err -> Left (StoreIo (Text.pack (show (err :: IOException))))
Right value -> Right value
joinEither :: Either error (Either error value) -> Either error value
joinEither = either Left id