packages feed

kenshou-remote-0.1.0.0: test/LeaseSpec.hs

module LeaseSpec (spec) where

import Control.Concurrent (forkIO, threadDelay)
import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)
import Control.Monad (replicateM)
import Data.Aeson (Value, eitherDecode, encode)
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Either (isRight)
import Data.IORef (atomicModifyIORef', newIORef)
import Data.Maybe (isJust)
import Data.Time (addUTCTime, getCurrentTime)
import Kenshou.Core.Id (newRunId)
import Kenshou.Remote.Cell.Control (CellSnapshot (..), readCellSnapshot)
import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), Quarantine (..), acquireLease, acquireLeaseWithId, leaseHeld, leaseSnapshot, reattachLease, releaseLease, renewLease, requestCancel, resizeLease, withHeartbeat)
import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))
import Kenshou.Remote.Store.File (newFileStore)
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "cell lease protocol" do
  it "reconciles a caller-retained claim without another write or reviving an expired lease" $ withSystemTempDirectory "kenshou-retained-claim" \root -> do
    store <- newFileStore root
    identifier <- newRunId
    handle <- acquireLeaseWithId store cellRef request identifier >>= expectAcquired
    original <- store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json")
    recovered <- acquireLeaseWithId store cellRef request identifier >>= expectAcquired
    (leaseSnapshot recovered >>= pure . (.leaseId)) `shouldReturn` identifier
    store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` original
    now <- store.serverTime
    let future = store {serverTime = pure (addUTCTime 1000 now)}
    expired <- acquireLeaseWithId future cellRef request identifier
    case expired of Busy lease -> lease.leaseId `shouldBe` identifier; _ -> expectationFailure "expired identity was revived"
    store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` original
    releaseLease store cellRef handle `shouldReturn` True

  it "gives exactly one contender a create-only lease" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    replies <- replicateM 12 newEmptyMVar
    mapM_ (\reply -> forkIO (acquireLease store cellRef request >>= putMVar reply)) replies
    outcomes <- mapM takeMVar replies
    length [() | Acquired _ <- outcomes] `shouldBe` 1
    length [() | Busy _ <- outcomes] `shouldBe` 11

  it "renews with the latest generation, reattaches by id, and releases" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef request >>= expectAcquired
    first <- leaseSnapshot handle
    renewLease store cellRef handle `shouldReturn` True
    renewed <- leaseSnapshot handle
    renewed.heartbeatAt `shouldSatisfy` (>= first.heartbeatAt)
    reattached <- reattachLease store cellRef first.leaseId
    isJust reattached `shouldBe` True
    releaseLease store cellRef handle `shouldReturn` True
    leaseHeld handle `shouldReturn` False
    store.statObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing

  it "retains the lease when the cell agent advances runsStarted" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef request >>= expectAcquired
    lease <- leaseSnapshot handle
    let objectName = ObjectName "cells/alpha/lease.json"
    meta <- store.statObject cellRef.controlBucket objectName >>= maybe (expectationFailure "missing lease" >> error "unreachable") pure
    updated <- store.putObject cellRef.controlBucket objectName "application/json" (GenerationIs meta.generation) (encode (lease {runsStarted = 1}))
    updated `shouldSatisfy` \case Written _ -> True; _ -> False
    renewLease store cellRef handle `shouldReturn` True
    refreshed <- leaseSnapshot handle
    refreshed.runsStarted `shouldBe` 1
    releaseLease store cellRef handle `shouldReturn` True

  it "takes over only after server time exceeds updated plus TTL and grace" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    old <- acquireLease store cellRef shortRequest >>= expectAcquired
    let later = store {serverTime = addUTCTime 100 <$> store.serverTime}
    replacement <- acquireLease later cellRef request >>= expectAcquired
    leaseHeld replacement `shouldReturn` True
    renewLease store cellRef old `shouldReturn` False
    releaseLease later cellRef replacement `shouldReturn` True

  it "fences a detached TTL extension against a stale lease generation" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef request >>= expectAcquired
    resizeLease store cellRef handle 900 `shouldReturn` True
    snapshot <- leaseSnapshot handle
    snapshot.ttlSeconds `shouldBe` 900
    stale <- reattachLease store cellRef snapshot.leaseId
    case stale of
      Nothing -> expectationFailure "extended lease disappeared"
      Just old -> do
        renewLease store cellRef handle `shouldReturn` True
        resizeLease store cellRef old 1200 `shouldReturn` False
        leaseHeld old `shouldReturn` False
    releaseLease store cellRef handle `shouldReturn` True

  it "releases a newly won lease when quarantine exists" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    now <- getCurrentTime
    let record = Quarantine "alpha" "reset-failed" Nothing now
    _ <- store.putObject cellRef.controlBucket (ObjectName "cells/alpha/quarantine.json") "application/json" DoesNotExist (encode record)
    outcome <- acquireLease store cellRef request
    case outcome of
      Quarantined observed -> observed `shouldBe` record
      _ -> expectationFailure "expected quarantined cell"
    store.statObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing

  it "keeps a heartbeat and allows release after its thread stops" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef shortRequest >>= expectAcquired
    withHeartbeat store cellRef handle \stillHeld -> do
      threadDelay 1200000
      stillHeld `shouldReturn` True
    releaseLease store cellRef handle `shouldReturn` True

  it "retries one transient heartbeat write failure while the lease remains valid" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef shortRequest >>= expectAcquired
    initial <- leaseSnapshot handle
    failNext <- newIORef True
    let flaky =
          store
            { putObject = \bucket object mediaType precondition bytes -> do
                shouldFail <-
                  if object == ObjectName "cells/alpha/lease.json"
                    then atomicModifyIORef' failNext (\failPending -> (False, failPending))
                    else pure False
                if shouldFail
                  then ioError (userError "transient heartbeat write failure")
                  else store.putObject bucket object mediaType precondition bytes
            }
    withHeartbeat flaky cellRef handle \stillHeld -> do
      threadDelay 1700000
      stillHeld `shouldReturn` True
    renewed <- leaseSnapshot handle
    renewed.heartbeatAt `shouldSatisfy` (> initial.heartbeatAt)
    releaseLease store cellRef handle `shouldReturn` True

  it "cancels only the expected lease unless force is explicit" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef request >>= expectAcquired
    current <- leaseSnapshot handle
    otherId <- newRunId
    requestCancel store cellRef Nothing False `shouldThrow` anyIOException
    requestCancel store cellRef (Just otherId) False `shouldReturn` False
    requestCancel store cellRef (Just current.leaseId) False `shouldReturn` True
    renewLease store cellRef handle `shouldReturn` False
    reattached <- reattachLease store cellRef current.leaseId
    case reattached of
      Just fresh -> releaseLease store cellRef fresh `shouldReturn` True
      Nothing -> expectationFailure "cancelled lease could not be reattached for release"
    second <- acquireLease store cellRef request >>= expectAcquired
    requestCancel store cellRef Nothing True `shouldReturn` True
    leaseHeld second `shouldReturn` True
    renewLease store cellRef second `shouldReturn` False

  it "round-trips the cell lease document" $ withSystemTempDirectory "kenshou-lease" \root -> do
    store <- newFileStore root
    handle <- acquireLease store cellRef request >>= expectAcquired
    lease <- leaseSnapshot handle
    eitherDecode (encode lease) `shouldBe` Right lease

  it "decodes the cell owner's lease and quarantine fixtures and preserves their JSON" do
    leaseBytes <- LazyByteString.readFile "test/golden/cell/cell.lease.v1.json"
    quarantineBytes <- LazyByteString.readFile "test/golden/cell/cell.quarantine.v1.json"
    let decodedLease = eitherDecode leaseBytes :: Either String Lease
        decodedQuarantine = eitherDecode quarantineBytes :: Either String Quarantine
    decodedLease `shouldSatisfy` isRight
    decodedQuarantine `shouldSatisfy` isRight
    fmap (eitherDecode . encode) decodedLease `shouldBe` Right (eitherDecode leaseBytes :: Either String Value)
    fmap (eitherDecode . encode) decodedQuarantine `shouldBe` Right (eitherDecode quarantineBytes :: Either String Value)

  it "reads owner control objects only for an allowed project and matching bucket" $ withSystemTempDirectory "kenshou-cell-control" \root -> do
    store <- newFileStore root
    descriptor <- LazyByteString.readFile "test/golden/cell/cell.descriptor.v1.json"
    let ref = CellRef "alpha" (Bucket "tan-nb-exp-cells-control")
    _ <- store.putObject ref.controlBucket (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor
    snapshot <- readCellSnapshot store ref ["tan-nb-exp"]
    case snapshot of
      Right observed -> do
        observed.lease `shouldBe` Nothing
        observed.quarantine `shouldBe` Nothing
      Left problem -> expectationFailure (show problem)
    readCellSnapshot store ref ["another-project"] `shouldReturn` Left "cell project is outside KENSHOU_GCP_ALLOWED_PROJECTS: tan-nb-exp"
    let wrongBucket = CellRef "alpha" (Bucket "other-control")
    _ <- store.putObject wrongBucket.controlBucket (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor
    readCellSnapshot store wrongBucket ["tan-nb-exp"] `shouldReturn` Left "cell descriptor control bucket differs from selected bucket"

cellRef :: CellRef
cellRef = CellRef "alpha" (Bucket "control")

request :: LeaseRequest
request = LeaseRequest "tester@workstation" "verification" 120

shortRequest :: LeaseRequest
shortRequest = LeaseRequest "tester@workstation" "verification" 1

expectAcquired :: AcquireOutcome -> IO LeaseHandle
expectAcquired (Acquired handle) = pure handle
expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"