packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Lease.hs

module Kenshou.Remote.Cell.Lease
  ( CellRef (..),
    LeaseId,
    Lease (..),
    Quarantine (..),
    LeaseRequest (..),
    LeaseHandle,
    AcquireOutcome (..),
    acquireLease,
    acquireLeaseWithId,
    renewLease,
    resizeLease,
    releaseLease,
    reattachLease,
    reattachActiveLease,
    requestCancel,
    leaseSnapshot,
    leaseHeld,
    withHeartbeat,
    validCellName,
  )
where

import Control.Concurrent (forkIO, killThread, threadDelay)
import Control.Concurrent.MVar (MVar, modifyMVar, modifyMVar_, newMVar, readMVar)
import Control.Exception (SomeAsyncException, SomeException, bracket, displayException, fromException, throwIO, try)
import Control.Monad (unless)
import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecode, encode, object, withObject, (.:), (.=))
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Int (Int64)
import Data.Maybe (isJust)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, addUTCTime, getCurrentTime)
import Kenshou.Core.Id (RunId, newRunId)
import Kenshou.Remote.Store (Bucket, ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))
import System.IO (hPutStrLn, stderr)

data CellRef = CellRef
  { cellName :: !Text,
    controlBucket :: !Bucket
  }
  deriving stock (Eq, Show)

type LeaseId = RunId

data Lease = Lease
  { leaseId :: !LeaseId,
    cell :: !Text,
    owner :: !Text,
    purpose :: !Text,
    ttlSeconds :: !Int,
    acquiredAt :: !UTCTime,
    heartbeatAt :: !UTCTime,
    cancelRequested :: !Bool,
    runsStarted :: !Int
  }
  deriving stock (Eq, Show)

data Quarantine = Quarantine
  { cell :: !Text,
    reason :: !Text,
    runId :: !(Maybe Text),
    at :: !UTCTime
  }
  deriving stock (Eq, Show)

data LeaseRequest = LeaseRequest
  { owner :: !Text,
    purpose :: !Text,
    ttlSeconds :: !Int
  }
  deriving stock (Eq, Show)

data LeaseHandle = LeaseHandle
  { state :: !(MVar (Lease, Int64, Bool))
  }

data AcquireOutcome = Acquired !LeaseHandle | Busy !Lease | Quarantined !Quarantine

instance ToJSON Lease where
  toJSON lease =
    object
      [ "schema" .= ("cell.lease/v1" :: Text),
        "leaseId" .= lease.leaseId,
        "cell" .= lease.cell,
        "owner" .= lease.owner,
        "purpose" .= lease.purpose,
        "ttlSeconds" .= lease.ttlSeconds,
        "acquiredAt" .= lease.acquiredAt,
        "heartbeatAt" .= lease.heartbeatAt,
        "cancelRequested" .= lease.cancelRequested,
        "runsStarted" .= lease.runsStarted
      ]

instance FromJSON Lease where
  parseJSON = withObject "cell lease" \value -> do
    schema <- value .: "schema"
    unless (schema == ("cell.lease/v1" :: Text)) (fail "unsupported cell lease schema")
    lease <- Lease <$> value .: "leaseId" <*> value .: "cell" <*> value .: "owner" <*> value .: "purpose" <*> value .: "ttlSeconds" <*> value .: "acquiredAt" <*> value .: "heartbeatAt" <*> value .: "cancelRequested" <*> value .: "runsStarted"
    unless (validCellName lease.cell && not (Text.null lease.owner) && not (Text.null lease.purpose) && lease.ttlSeconds > 0 && lease.runsStarted >= 0) (fail "invalid cell lease")
    pure lease

instance ToJSON Quarantine where
  toJSON quarantine =
    object
      [ "schema" .= ("cell.quarantine/v1" :: Text),
        "cell" .= quarantine.cell,
        "reason" .= quarantine.reason,
        "runId" .= quarantine.runId,
        "at" .= quarantine.at
      ]

instance FromJSON Quarantine where
  parseJSON = withObject "cell quarantine" \value -> do
    schema <- value .: "schema"
    unless (schema == ("cell.quarantine/v1" :: Text)) (fail "unsupported cell quarantine schema")
    quarantine <- Quarantine <$> value .: "cell" <*> value .: "reason" <*> value .: "runId" <*> value .: "at"
    unless (validCellName quarantine.cell && not (Text.null quarantine.reason)) (fail "invalid cell quarantine")
    pure quarantine

acquireLease :: ObjectStore -> CellRef -> LeaseRequest -> IO AcquireOutcome
acquireLease store ref request = newRunId >>= acquireLeaseWithId store ref request

-- The caller may durably retain this identity before issuing a remote claim.
-- Reconciliation never renews or revives an expired/cancelled claim.
acquireLeaseWithId :: ObjectStore -> CellRef -> LeaseRequest -> LeaseId -> IO AcquireOutcome
acquireLeaseWithId store ref request proposedId = do
  unless (validCellName ref.cellName && not (Text.null request.owner) && not (Text.null request.purpose) && request.ttlSeconds > 0) (ioError (userError "invalid lease request"))
  current <- readCurrent store ref
  now <- store.serverTime
  case current of
    Just (lease, meta)
      | lease.leaseId == proposedId ->
          if not lease.cancelRequested && addUTCTime (fromIntegral lease.ttlSeconds + 30) meta.updated >= now
            then accept lease meta
            else pure (Busy lease)
    _ -> acquire proposedId (0 :: Int)
  where
    acquire identifier attempts = do
      unless (attempts < maxCasAttempts) (ioError (userError "lease claim contention exceeded retry limit"))
      now <- store.serverTime
      let proposed = Lease identifier ref.cellName request.owner request.purpose request.ttlSeconds now now False 0
      result <- store.putObject ref.controlBucket (leaseName ref) "application/json" DoesNotExist (encode proposed)
      case result of
        Written meta -> accept proposed meta
        PreconditionFailed -> do
          observed <- readCurrent store ref
          case observed of
            Nothing -> acquire identifier (attempts + 1)
            Just (current, meta) -> do
              clock <- store.serverTime
              if addUTCTime (fromIntegral current.ttlSeconds + 30) meta.updated < clock
                then do
                  replaced <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs meta.generation) (encode proposed)
                  case replaced of
                    Written next -> accept proposed next
                    PreconditionFailed -> acquire identifier (attempts + 1)
                else pure (Busy current)
    accept lease meta = do
      quarantine <- readQuarantine store ref
      case quarantine of
        Nothing -> Acquired <$> newHandle lease meta.generation
        Just record -> do
          released <- store.deleteObject ref.controlBucket (leaseName ref) (GenerationIs meta.generation)
          unless released (ioError (userError "quarantined cell lease could not be released"))
          pure (Quarantined record)

renewLease :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool
renewLease store ref handle = modifyMVar handle.state \(lease, generation, held) ->
  if not held || lease.cancelRequested
    then pure ((lease, generation, False), False)
    else renewCurrent lease generation (0 :: Int)
  where
    renewCurrent lease generation attempts = do
      unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))
      now <- store.serverTime
      let renewed = lease {heartbeatAt = now}
      result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs generation) (encode renewed)
      case result of
        Written meta -> pure ((renewed, meta.generation, True), True)
        PreconditionFailed -> do
          current <- readCurrent store ref
          case current of
            Just (observed, meta)
              | attempts + 1 < maxCasAttempts,
                observed.leaseId == lease.leaseId,
                not observed.cancelRequested,
                addUTCTime (fromIntegral observed.ttlSeconds + 30) meta.updated >= now ->
                  renewCurrent observed meta.generation (attempts + 1)
            _ -> pure ((lease, generation, False), False)

-- A detached submission needs a lease that outlives its wall-clock budget.
-- Fence the TTL change before publishing its marker.
resizeLease :: ObjectStore -> CellRef -> LeaseHandle -> Int -> IO Bool
resizeLease store ref handle ttl = do
  unless (ttl > 0) (ioError (userError "lease TTL must be positive"))
  modifyMVar handle.state \(lease, generation, held) ->
    if not held || lease.cancelRequested
      then pure ((lease, generation, False), False)
      else do
        unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))
        now <- store.serverTime
        let resized = lease {ttlSeconds = ttl, heartbeatAt = now}
        result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs generation) (encode resized)
        case result of
          Written meta -> pure ((resized, meta.generation, True), True)
          PreconditionFailed -> pure ((lease, generation, False), False)

releaseLease :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool
releaseLease store ref handle = modifyMVar handle.state \(lease, generation, held) ->
  if not held
    then pure ((lease, generation, False), False)
    else do
      unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))
      released <- store.deleteObject ref.controlBucket (leaseName ref) (GenerationIs generation)
      pure ((lease, generation, False), released)

reattachLease :: ObjectStore -> CellRef -> LeaseId -> IO (Maybe LeaseHandle)
reattachLease store ref identifier = do
  current <- readCurrent store ref
  case current of
    Just (lease, meta) | lease.leaseId == identifier -> Just <$> newHandle lease meta.generation
    _ -> pure Nothing

-- Performance recovery must treat an expired or cancelled lease as lost, even
-- when its old object still exists. The unfenced reattachment above remains
-- available to cancellation/release code that needs its original generation.
reattachActiveLease :: ObjectStore -> CellRef -> LeaseId -> IO (Maybe LeaseHandle)
reattachActiveLease store ref identifier = do
  current <- readCurrent store ref
  now <- store.serverTime
  case current of
    Just (lease, meta)
      | lease.leaseId == identifier,
        not lease.cancelRequested,
        addUTCTime (fromIntegral lease.ttlSeconds + 30) meta.updated >= now ->
          Just <$> newHandle lease meta.generation
    _ -> pure Nothing

requestCancel :: ObjectStore -> CellRef -> Maybe LeaseId -> Bool -> IO Bool
requestCancel store ref expected force = do
  unless (force || isJust expected) (ioError (userError "cancelling another owner's lease requires force"))
  attempt (0 :: Int)
  where
    attempt count = do
      unless (count < maxCasAttempts) (ioError (userError "lease cancellation contention exceeded retry limit"))
      current <- readCurrent store ref
      case current of
        Nothing -> pure False
        Just (lease, meta)
          | not force && Just lease.leaseId /= expected -> pure False
          | lease.cancelRequested -> pure True
          | otherwise -> do
              result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs meta.generation) (encode (lease {cancelRequested = True}))
              case result of
                Written _ -> pure True
                PreconditionFailed -> attempt (count + 1)

leaseHeld :: LeaseHandle -> IO Bool
leaseHeld handle = (\(_, _, held) -> held) <$> readMVar handle.state

leaseSnapshot :: LeaseHandle -> IO Lease
leaseSnapshot handle = (\(lease, _, _) -> lease) <$> readMVar handle.state

withHeartbeat :: ObjectStore -> CellRef -> LeaseHandle -> (IO Bool -> IO value) -> IO value
withHeartbeat store ref handle action = bracket start killThread (const (action (leaseHeld handle)))
  where
    start = forkIO loop
    loop = do
      (lease, _, held) <- readMVar handle.state
      if held
        then do
          threadDelay (max 100000 (lease.ttlSeconds * 1000000 `div` 3))
          renew
        else pure ()
    renew = do
      result <- try (renewLease store ref handle) :: IO (Either SomeException Bool)
      case result of
        Right True -> loop
        Right False -> markLost
        Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async
        Left failure -> do
          hPutStrLn stderr ("cell lease heartbeat renewal failed; retrying before TTL: " <> displayException failure)
          current <- leaseSnapshot handle
          now <- getCurrentTime
          if now <= addUTCTime (fromIntegral current.ttlSeconds + 30) current.heartbeatAt
            then threadDelay 1000000 >> renew
            else markLost
    markLost = modifyMVar_ handle.state \(current, generation, _) -> pure (current, generation, False)

newHandle :: Lease -> Int64 -> IO LeaseHandle
newHandle lease generation = LeaseHandle <$> newMVar (lease, generation, True)

readCurrent :: ObjectStore -> CellRef -> IO (Maybe (Lease, ObjectMeta))
readCurrent store ref = do
  stored <- store.getObject ref.controlBucket (leaseName ref)
  case stored of
    Nothing -> pure Nothing
    Just (bytes, meta) -> do
      lease <- decodeDocument "cell lease" bytes
      unless (lease.cell == ref.cellName) (ioError (userError "cell lease names another cell"))
      pure (Just (lease, meta))

readQuarantine :: ObjectStore -> CellRef -> IO (Maybe Quarantine)
readQuarantine store ref = do
  stored <- store.getObject ref.controlBucket (quarantineName ref)
  case stored of
    Nothing -> pure Nothing
    Just (bytes, _) -> do
      record <- decodeDocument "cell quarantine" bytes
      unless (record.cell == ref.cellName) (ioError (userError "cell quarantine names another cell"))
      pure (Just record)

decodeDocument :: (FromJSON value) => String -> LazyByteString.ByteString -> IO value
decodeDocument label bytes = either (ioError . userError . ((label <> ": ") <>)) pure (eitherDecode bytes)

leaseName :: CellRef -> ObjectName
leaseName ref = ObjectName ("cells/" <> ref.cellName <> "/lease.json")

quarantineName :: CellRef -> ObjectName
quarantineName ref = ObjectName ("cells/" <> ref.cellName <> "/quarantine.json")

validCellName :: Text -> Bool
validCellName value =
  Text.length value >= 3
    && Text.length value <= 63
    && Text.head value `elem` (['a' .. 'z'] <> ['0' .. '9'])
    && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "-")) value

maxCasAttempts :: Int
maxCasAttempts = 32