diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## 0.6.0.0 — 2026-09-08 + +### Breaking Changes + +- AI features are configured only through explicit host configuration. `--ai-config FILE` or + `KIOKU_AI_CONFIG` selects it; when neither is present AI is disabled, and credentials in the + environment no longer enable it. The `KIOKU_EMBEDDING_*` and `OPENAI_API_KEY` variables that + previously configured embedding are not consulted. +- `kioku-migrate`-composed deployments move to a 56-migration plan; see the `kioku-migrations` + changelog. Any script asserting on the previous count needs updating before this release. + +### Added + +- `Kioku.Cli.AIConfig`, exposing the `--ai-config` option and the same versioned file boundary + embedded hosts use. +- `kioku worker deferred list` and `kioku worker deferred resume TIMER_ID --ai-config FILE`. + Foreground resume uses the configured execution capabilities and preserves the original parked + work; background workers never receive interactive session ownership. + +### Changed + +- `recall`, `distill`, and the embedding worker all resolve their models through the configured + runtime, so a host that disables a feature disables it uniformly across commands. + ## 0.5.2.0 — 2026-08-31 ### Changed diff --git a/kioku-cli.cabal b/kioku-cli.cabal --- a/kioku-cli.cabal +++ b/kioku-cli.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: kioku-cli -version: 0.5.2.0 +version: 0.6.0.0 synopsis: kioku command-line interface description: Command-line entry point for kioku demos and operational commands. @@ -46,6 +46,7 @@ hs-source-dirs: src exposed-modules: Kioku.Cli + Kioku.Cli.AIConfig Kioku.Cli.Commands.Artifacts Kioku.Cli.Commands.Demo Kioku.Cli.Commands.DemoSession @@ -71,17 +72,20 @@ -- mori://MMZK1526/mmzk-typeid/upstream-issues/mmzk-typeid-kindid-ghc-9-12-4-profiling-coercionkind-panic ghc-prof-options: -fno-opt-coercion build-depends: - , async >=2.2 && <2.3 - , base >=4.21 && <5 - , containers >=0.6 && <0.8 - , directory >=1.3 && <1.4 - , effectful >=2.5 && <2.7 - , kioku-api ^>=0.5.2.0 - , kioku-core ^>=0.5.2.0 + , async >=2.2 && <2.3 + , baikai ^>=0.7.0.0 + , base >=4.21 && <5 + , containers >=0.6 && <0.8 + , directory >=1.3 && <1.4 + , effectful >=2.5 && <2.7 + , keiro ^>=0.16.0.0 + , kioku-api ^>=0.6.0.0 + , kioku-core ^>=0.6.0.0 , kiroku-store ^>=0.8.0.0 - , optparse-applicative >=0.18 && <0.20 - , text >=2.1 && <2.2 - , time >=1.12 && <1.15 + , optparse-applicative >=0.18 && <0.20 + , text >=2.1 && <2.2 + , time >=1.12 && <1.15 + , uuid >=1.3 && <1.4 executable kioku import: warnings, shared @@ -90,7 +94,7 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: , base >=4.21 && <5 - , kioku-cli ^>=0.5.2.0 + , kioku-cli ^>=0.6.0.0 test-suite kioku-cli-test import: warnings, shared @@ -108,16 +112,21 @@ -- process would race the rest of the suite. build-tool-depends: kioku-cli:kioku build-depends: - , base >=4.21 && <5 - , effectful >=2.5 && <2.7 - , kioku-api ^>=0.5.2.0 - , kioku-cli ^>=0.5.2.0 - , kioku-core ^>=0.5.2.0 - , kioku-migrations:test-support ^>=0.5.2.0 + , aeson >=2.2 && <2.3 + , base >=4.21 && <5 + , effectful >=2.5 && <2.7 + , filepath >=1.4 && <1.6 + , keiro ^>=0.16.0.0 + , kioku-api ^>=0.6.0.0 + , kioku-cli ^>=0.6.0.0 + , kioku-core ^>=0.6.0.0 + , kioku-migrations:test-support ^>=0.6.0.0 , kiroku-store ^>=0.8.0.0 , optparse-applicative >=0.18 - , process >=1.6 && <1.7 + , process >=1.6 && <1.7 , tasty >=1.5 , tasty-hunit >=0.10 + , temporary >=1.3 , text >=2.1 - , time >=1.12 && <1.15 + , time >=1.12 && <1.15 + , uuid >=1.3 && <1.4 diff --git a/src/Kioku/Cli/AIConfig.hs b/src/Kioku/Cli/AIConfig.hs new file mode 100644 --- /dev/null +++ b/src/Kioku/Cli/AIConfig.hs @@ -0,0 +1,8 @@ +-- | CLI assembly delegates to the same versioned file boundary as embedded hosts. +module Kioku.Cli.AIConfig (aiConfigOption, loadAIRuntime, parseAIConfig) where + +import Kioku.AI.File (loadAIRuntime, parseAIConfig) +import Options.Applicative qualified as Opt + +aiConfigOption :: Opt.Parser (Maybe FilePath) +aiConfigOption = Opt.optional (Opt.strOption (Opt.long "ai-config" <> Opt.metavar "FILE" <> Opt.help "Explicit AI configuration (otherwise KIOKU_AI_CONFIG; absent means disabled)")) diff --git a/src/Kioku/Cli/Commands/Distill.hs b/src/Kioku/Cli/Commands/Distill.hs --- a/src/Kioku/Cli/Commands/Distill.hs +++ b/src/Kioku/Cli/Commands/Distill.hs @@ -7,13 +7,15 @@ where import Data.Text qualified as Text +import Kioku.AI.Config (AIFeature (CandidateEmbedding)) import Kioku.App (runAppIO, withNoopAppEnv) +import Kioku.Cli.AIConfig (aiConfigOption, loadAIRuntime) import Kioku.Cli.Context (cliMemoryContext) import Kioku.Cli.Options (boundedIntReader) import Kioku.Distill.L1 (L1Outcome (..), L1RunMode (..), L1Summary (..), distillSessionL1, recallCandidates, scopedScanCandidates) import Kioku.Distill.Runtime (newDistillRuntime) import Kioku.Id (SessionId, idText, parseId) -import Kioku.Memory.Embedding (EmbeddingConfig (..), resolveEmbeddingConfig, toEmbeddingModel) +import Kioku.Memory.Embedding (resolveEmbeddingConfig) import Kioku.Recall.Capability (detectVectorCapability) import Kiroku.Store.Connection (defaultConnectionSettings) import Options.Applicative @@ -26,7 +28,8 @@ { sessionId :: !SessionId, candidateSource :: !CandidateSource, candidateLimit :: !Int, - force :: !Bool + force :: !Bool, + aiConfig :: !(Maybe FilePath) } deriving stock (Eq, Show) @@ -64,25 +67,21 @@ ( long "force" <> help "Re-run even when the session has no turns newer than the last successful pass" ) + <*> aiConfigOption runDistill :: DistillOptions -> IO () runDistill opts = do connStr <- requireEnv "PG_CONNECTION_STRING" - rt <- newDistillRuntime + ai <- loadAIRuntime True opts.aiConfig + let rt = newDistillRuntime ai Nothing context <- cliMemoryContext - recallConfig <- - case opts.candidateSource of - CandidateScan -> pure Nothing - CandidateRecall -> Just <$> resolveEmbeddingConfig withNoopAppEnv (defaultConnectionSettings (Text.pack connStr)) \env -> do result <- runAppIO env do - finder <- - case (opts.candidateSource, recallConfig) of - (CandidateRecall, Just config) -> do - capability <- detectVectorCapability config.dimensions - pure (recallCandidates (toEmbeddingModel config) capability opts.candidateLimit) - _ -> - pure (scopedScanCandidates opts.candidateLimit) + finder <- case (opts.candidateSource, resolveEmbeddingConfig ai CandidateEmbedding) of + (CandidateRecall, Right _) -> do + capability <- detectVectorCapability 1536 + pure (recallCandidates ai capability opts.candidateLimit) + _ -> pure (scopedScanCandidates opts.candidateLimit) distillSessionL1 context (runMode opts) rt finder opts.sessionId case result of Left storeErr -> ioError (userError ("kioku distill store error: " <> show storeErr)) diff --git a/src/Kioku/Cli/Commands/Recall.hs b/src/Kioku/Cli/Commands/Recall.hs --- a/src/Kioku/Cli/Commands/Recall.hs +++ b/src/Kioku/Cli/Commands/Recall.hs @@ -37,10 +37,10 @@ import Kioku.Api.Scope (MemoryScope (..), Namespace (..), ScopeKind (..)) import Kioku.Api.Types (MemoryRecord (..)) import Kioku.App (runAppIO, withNoopAppEnv) +import Kioku.Cli.AIConfig (aiConfigOption, loadAIRuntime) import Kioku.Cli.Context (cliMemoryContext) import Kioku.Cli.Options (boundedIntReader) import Kioku.Cli.Scope (parseNamespaceOnly, parseScope) -import Kioku.Memory.Embedding (EmbeddingConfig (..), resolveEmbeddingConfig, toEmbeddingModel) import Kioku.Recall ( RecallHit (..), RecallStrategy (..), @@ -63,7 +63,8 @@ target :: !RecallTarget, strategy :: !RecallStrategy, limit :: !Int, - showScores :: !Bool + showScores :: !Bool, + aiConfig :: !(Maybe FilePath) } deriving stock (Eq, Show) @@ -90,6 +91,7 @@ ( long "show-scores" <> help "Print fused scores and component ranks" ) + <*> aiConfigOption -- | Exactly one of the three target flags, and never two. -- @@ -168,7 +170,7 @@ runRecall :: RecallOptions -> IO () runRecall opts = do connStr <- requireEnv "PG_CONNECTION_STRING" - config <- resolveEmbeddingConfig + ai <- loadAIRuntime False opts.aiConfig context <- cliMemoryContext request <- case mkRecallQuery opts.target opts.query opts.strategy opts.limit of @@ -185,10 +187,9 @@ <> Text.unpack (memorySpaceIdText (memoryContextSpace context)) ) withNoopAppEnv (defaultConnectionSettings (Text.pack connStr)) \env -> do - let model = toEmbeddingModel config result <- runAppIO env do - capability <- detectVectorCapability config.dimensions - recall model capability context request + capability <- detectVectorCapability 1536 + recall ai capability context request case result of Left storeErr -> ioError (userError ("kioku recall store error: " <> show storeErr)) Right (Left recallErr) -> ioError (userError ("kioku recall error: " <> show recallErr)) diff --git a/src/Kioku/Cli/Commands/Worker.hs b/src/Kioku/Cli/Commands/Worker.hs --- a/src/Kioku/Cli/Commands/Worker.hs +++ b/src/Kioku/Cli/Commands/Worker.hs @@ -5,19 +5,27 @@ ) where +import Baikai.Embedding (EmbeddingModel) import Control.Concurrent (threadDelay) import Control.Concurrent.Async (race) import Control.Exception (SomeException, displayException, try) import Data.Text qualified as Text import Data.Time (getCurrentTime) +import Data.UUID qualified as UUID import Effectful (Eff, IOE, (:>)) +import Keiro.Timer qualified as Timer +import Kioku.AI.Config (AIFeature (MemoryEmbedding)) +import Kioku.AI.Runtime (AIRuntime) import Kioku.Api.Access (MemorySpaceId, memorySpaceIdText, mkMemorySpaceId) import Kioku.App (AppEffects, AppEnv, runAppIO, withNoopAppEnv) +import Kioku.Cli.AIConfig (aiConfigOption, loadAIRuntime) import Kioku.Cli.Context (cliContextProvider) import Kioku.Distill.L1 (FindMergeCandidates, recallCandidates) import Kioku.Distill.Runtime (newDistillRuntime) +import Kioku.Distill.Timer.Deferred +import Kioku.Distill.Timer.Outcome (FireOutcome (..)) import Kioku.Distill.Timer.Worker (drainKiokuTimers, runKiokuTimerWorkerOnce) -import Kioku.Memory.Embedding (EmbeddingConfig (..), resolveEmbeddingConfig, toEmbeddingModel) +import Kioku.Memory.Embedding (resolveEmbeddingConfig) import Kioku.Memory.Embedding.Worker ( EmbeddingBackfillScope (..), backfillMissingEmbeddings, @@ -44,20 +52,28 @@ -- operator run a backfill, see a count, and never learn that the other spaces are still -- unsearchable. data WorkerOptions - = WorkerContinuous + = WorkerConfigured !FilePath !WorkerOptions + | WorkerContinuous | WorkerBackfill !EmbeddingBackfillScope + | WorkerDeferredList + | WorkerDeferredResume !Timer.TimerId | WorkerTimersOnce deriving stock (Eq, Show) workerOptionsParser :: Parser WorkerOptions -workerOptionsParser = - ( flag' - WorkerBackfill - ( long "backfill" - <> help "Run one embedding backfill pass and exit (conflicts with --timers-once)" - ) - <*> backfillScopeParser - ) +workerOptionsParser = (\config mode -> maybe mode (`WorkerConfigured` mode) config) <$> aiConfigOption <*> workerModeParser + +workerModeParser :: Parser WorkerOptions +workerModeParser = + hsubparser + (command "deferred" (info deferredParser (progDesc "List or resume authorized deferred timers"))) + <|> ( flag' + WorkerBackfill + ( long "backfill" + <> help "Run one embedding backfill pass and exit (conflicts with --timers-once)" + ) + <*> backfillScopeParser + ) <|> flag' WorkerTimersOnce ( long "timers-once" @@ -65,6 +81,21 @@ ) <|> pure WorkerContinuous +deferredParser :: Parser WorkerOptions +deferredParser = + hsubparser + ( command "list" (info (pure WorkerDeferredList) (progDesc "List deferred distillation work")) + <> command + "resume" + ( info + (WorkerDeferredResume <$> argument timerIdReader (metavar "TIMER_ID")) + (progDesc "Resume one deferred timer using foreground AI capabilities") + ) + ) + where + timerIdReader = eitherReader $ \raw -> + maybe (Left "TIMER_ID must be a UUID") (Right . Timer.TimerId) (UUID.fromString raw) + backfillScopeParser :: Parser EmbeddingBackfillScope backfillScopeParser = maybe BackfillEverySpace BackfillOneSpace @@ -84,22 +115,36 @@ Right space -> Right space runWorker :: WorkerOptions -> IO () -runWorker opts = do +runWorker opts = case opts of + WorkerConfigured path mode -> runConfiguredWorker (Just path) mode + _ -> runConfiguredWorker Nothing opts + +runConfiguredWorker :: Maybe FilePath -> WorkerOptions -> IO () +runConfiguredWorker path opts = do + ai <- loadAIRuntime (case opts of WorkerDeferredResume _ -> True; _ -> False) path + case opts of + WorkerBackfill _ -> either (dieWorker . show) (const (pure ())) (resolveEmbeddingConfig ai MemoryEmbedding) + _ -> pure () connStr <- requireEnv "PG_CONNECTION_STRING" - config <- resolveEmbeddingConfig let settings = defaultConnectionSettings (Text.pack connStr) withStore settings $ \st -> - withNoopAppEnv settings \env -> - case opts of - WorkerTimersOnce -> runTimerOnce env config - WorkerBackfill scope -> withCapability env config \capability -> - runBackfill env capability config scope - WorkerContinuous -> withCapability env config \capability -> - runContinuousWorker env st capability config + withNoopAppEnv settings \env -> case opts of + WorkerDeferredList -> runDeferredList env + WorkerDeferredResume tid -> runDeferredResume ai env tid + WorkerTimersOnce -> runTimerOnce ai env + WorkerBackfill scope -> case resolveEmbeddingConfig ai MemoryEmbedding of + Left err -> dieWorker (show err) + Right model -> withCapability env model $ \capability -> runBackfill ai env capability scope + WorkerContinuous -> case resolveEmbeddingConfig ai MemoryEmbedding of + Left _ -> do + putStrLn "Memory embeddings disabled by AI policy; running timer worker only." + runTimerLoop ai env VectorExtensionUnavailable + Right model -> withCapability env model $ \capability -> runContinuousWorker ai env st capability model + WorkerConfigured nested mode -> runConfiguredWorker (Just nested) mode -withCapability :: AppEnv -> EmbeddingConfig -> (VectorCapability -> IO a) -> IO a -withCapability env config k = do - result <- runAppIO env (detectVectorCapability config.dimensions) +withCapability :: AppEnv -> EmbeddingModel -> (VectorCapability -> IO a) -> IO a +withCapability env _config k = do + result <- runAppIO env (detectVectorCapability 1536) case result of Left storeErr -> ioError (userError ("kioku worker store error: " <> show storeErr)) Right capability -> k capability @@ -111,25 +156,25 @@ -- so no capability gating is needed here. mergeCandidateFinder :: (IOE :> es, Store :> es) => - EmbeddingConfig -> + AIRuntime -> VectorCapability -> FindMergeCandidates es -mergeCandidateFinder config capability = - recallCandidates (toEmbeddingModel config) capability mergeCandidateLimit +mergeCandidateFinder ai capability = + recallCandidates ai capability mergeCandidateLimit mergeCandidateLimit :: Int mergeCandidateLimit = 8 -runBackfill :: AppEnv -> VectorCapability -> EmbeddingConfig -> EmbeddingBackfillScope -> IO () -runBackfill env capability config scope = do +runBackfill :: AIRuntime -> AppEnv -> VectorCapability -> EmbeddingBackfillScope -> IO () +runBackfill ai env capability scope = do -- Refuse before any event is touched: a backfill under a mismatched dimension count would -- embed every memory in the store and fail the ::vector cast on every single one. case capability of VectorDimensionMismatch configured actual -> dieWorker (dimensionMismatchMessage configured actual) _ -> pure () - let model = toEmbeddingModel config - result <- runAppIO env (backfillMissingEmbeddings capability (mkEmbeddingWorkerEnv model config.dimensions) scope) + embeddingEnv <- either (ioError . userError . show) pure (mkEmbeddingWorkerEnv ai) + result <- runAppIO env (backfillMissingEmbeddings capability embeddingEnv scope) case result of Left storeErr -> ioError (userError ("kioku worker backfill store error: " <> show storeErr)) Right count -> @@ -152,18 +197,17 @@ -- 'race' makes both directions loud: whichever pipeline stops first ends the -- race, and the process exits non-zero with a reason so a supervisor restarts -- it. Neither side is expected to return at all. -runContinuousWorker :: AppEnv -> KirokuStore -> VectorCapability -> EmbeddingConfig -> IO () -runContinuousWorker env store capability config = do - let model = toEmbeddingModel config +runContinuousWorker :: AIRuntime -> AppEnv -> KirokuStore -> VectorCapability -> EmbeddingModel -> IO () +runContinuousWorker ai env store capability _config = do contexts <- cliContextProvider @(Eff AppEffects) case capability of VectorAvailable -> do - startupBackfill env capability config + startupBackfill ai env capability outcome <- try @SomeException $ race - (runTimerLoop env capability config) - (runAppIO env (runEmbeddingWorkerHost store contexts capability model config.dimensions)) + (runTimerLoop ai env capability) + (runAppIO env (runEmbeddingWorkerHost store contexts capability ai)) case outcome of -- A halted processor can tear its own machinery down hard enough to -- surface as an exception rather than a clean return (shibuya's halt path @@ -181,16 +225,16 @@ dieWorker "embedding worker stopped (processor halted or subscription ended)" VectorExtensionUnavailable -> do putStrLn "pgvector is not available; recall will run FTS-only; running kioku timer worker only." - runTimerLoop env capability config + runTimerLoop ai env capability VectorColumnsUnavailable missing -> do putStrLn ("pgvector columns are missing (" <> Text.unpack (Text.intercalate ", " missing) <> "); running kioku timer worker only.") - runTimerLoop env capability config + runTimerLoop ai env capability -- Loud, but not fatal. Every embedding write would fail on the ::vector cast, so there -- is no point starting the embedding host — but distillation timers have nothing to do -- with embeddings, and killing the whole worker would stop them too. VectorDimensionMismatch configured actual -> do hPutStrLn stderr ("kioku worker: " <> dimensionMismatchMessage configured actual <> "; running kioku timer worker only.") - runTimerLoop env capability config + runTimerLoop ai env capability -- | A dimension mismatch would otherwise be discovered one failed event at a time, forever. dimensionMismatchMessage :: Int -> Int -> String @@ -209,14 +253,15 @@ -- It covers every space, not @KIOKU_MEMORY_SPACE@: this process is about to subscribe to every -- space's memory events, so recovering only one space's would leave the others' recall degraded -- with nothing to say so. -startupBackfill :: AppEnv -> VectorCapability -> EmbeddingConfig -> IO () -startupBackfill env capability config = do +startupBackfill :: AIRuntime -> AppEnv -> VectorCapability -> IO () +startupBackfill ai env capability = do + embeddingEnv <- either (ioError . userError . show) pure (mkEmbeddingWorkerEnv ai) result <- runAppIO env ( backfillMissingEmbeddings capability - (mkEmbeddingWorkerEnv (toEmbeddingModel config) config.dimensions) + embeddingEnv BackfillEverySpace ) case result of @@ -230,14 +275,14 @@ hPutStrLn stderr ("kioku worker: " <> msg <> "; exiting") exitWith (ExitFailure 1) -runTimerOnce :: AppEnv -> EmbeddingConfig -> IO () -runTimerOnce env config = do - rt <- newDistillRuntime +runTimerOnce :: AIRuntime -> AppEnv -> IO () +runTimerOnce ai env = do + let rt = newDistillRuntime ai Nothing contexts <- cliContextProvider @(Eff AppEffects) now <- getCurrentTime result <- runAppIO env do - capability <- detectVectorCapability config.dimensions - runKiokuTimerWorkerOnce Nothing contexts rt (mergeCandidateFinder config capability) now + capability <- detectVectorCapability 1536 + runKiokuTimerWorkerOnce Nothing contexts rt (mergeCandidateFinder ai capability) now case result of Left storeErr -> ioError (userError ("kioku timer worker store error: " <> show storeErr)) Right Nothing -> putStrLn "No due kioku distillation timers." @@ -255,13 +300,13 @@ -- This never returns normally, so 'race' seeing it finish genuinely means -- something impossible happened. Non-store exceptions propagate to 'race', which -- is equally loud. -runTimerLoop :: AppEnv -> VectorCapability -> EmbeddingConfig -> IO () -runTimerLoop env capability config = do - rt <- newDistillRuntime +runTimerLoop :: AIRuntime -> AppEnv -> VectorCapability -> IO () +runTimerLoop ai env capability = do + let rt = newDistillRuntime ai Nothing contexts <- cliContextProvider @(Eff AppEffects) putStrLn "kioku timer worker started." let go failures = do - result <- runAppIO env (drainKiokuTimers Nothing contexts rt (mergeCandidateFinder config capability)) + result <- runAppIO env (drainKiokuTimers Nothing contexts rt (mergeCandidateFinder ai capability)) case result of Left storeErr -> do hPutStrLn stderr ("kioku timer worker: store error (will retry): " <> show storeErr) @@ -288,3 +333,41 @@ case found of Just envValue -> pure envValue Nothing -> ioError (userError (name <> " is not set")) + +runDeferredList :: AppEnv -> IO () +runDeferredList env = do + contexts <- cliContextProvider @(Eff AppEffects) + let go cursor = do + result <- runAppIO env (listDeferredTimers contexts (Timer.DeadTimerPageRequest 100 cursor)) + case result of + Left err -> dieWorker (show err) + Right (Left err) -> dieWorker (show err) + Right (Right page) -> do + mapM_ render page.entries + maybe (pure ()) (go . Just) page.nextAfterTimerId + render entry = do + let Timer.TimerId uuid = entry.timer.timerId + putStrLn + ( UUID.toString uuid + <> " space=" + <> Text.unpack (memorySpaceIdText entry.memorySpace) + <> " features=" + <> show entry.features + <> " attempts=" + <> show entry.timer.attempts + <> " reason=" + <> Text.unpack entry.reason + ) + putStrLn (" Resume: kioku worker deferred resume " <> UUID.toString uuid <> " --ai-config FILE") + go Nothing + +runDeferredResume :: AIRuntime -> AppEnv -> Timer.TimerId -> IO () +runDeferredResume ai env tid = do + contexts <- cliContextProvider @(Eff AppEffects) + result <- runAppIO env do + capability <- detectVectorCapability 1536 + resumeDeferredTimer contexts (newDistillRuntime ai Nothing) (mergeCandidateFinder ai capability) tid + case result of + Right (DeferredFinished (FireCompleted _)) -> putStrLn "Completed the original deferred timer." + Right outcome -> dieWorker (show outcome <> "; unfinished work remains parked; inspect configuration, authorization, and attempt ceiling before retrying") + Left err -> dieWorker (show err) diff --git a/test/Kioku/Cli/ParserSpec.hs b/test/Kioku/Cli/ParserSpec.hs --- a/test/Kioku/Cli/ParserSpec.hs +++ b/test/Kioku/Cli/ParserSpec.hs @@ -6,6 +6,8 @@ import Data.List (isInfixOf) import Data.Text qualified as Text +import Data.UUID qualified as UUID +import Keiro.Timer (TimerId (..)) import Kioku.Api.Access (mkMemorySpaceId) import Kioku.Api.Scope (MemoryScope (..), Namespace (..), ScopeKind (..)) import Kioku.Cli.Commands.Demo (DemoOptions (..), demoOptionsParser, demoScope) @@ -303,7 +305,16 @@ workerModeTests = testGroup "worker one-shot modes are mutually exclusive" - [ testCase "no flags means the continuous worker" do + [ testCase "deferred resume accepts a UUID and trailing AI config" do + parseWith workerOptionsParser ["deferred", "resume", UUID.toString UUID.nil, "--ai-config", "ai.json"] + @?= Right (WorkerConfigured "ai.json" (WorkerDeferredResume (TimerId UUID.nil))), + testCase "deferred listing parses" do + parseWith workerOptionsParser ["deferred", "list"] @?= Right WorkerDeferredList, + testCase "deferred resume rejects malformed timer IDs" do + case parseWith workerOptionsParser ["deferred", "resume", "bad"] of + Left _ -> pure () + Right _ -> assertBool "invalid UUID accepted" False, + testCase "no flags means the continuous worker" do parseWith workerOptionsParser [] @?= Right WorkerContinuous, testCase "--backfill covers every space unless one is named" do parseWith workerOptionsParser ["--backfill"] @?= Right (WorkerBackfill BackfillEverySpace), diff --git a/test/Kioku/Cli/RecallEndToEndSpec.hs b/test/Kioku/Cli/RecallEndToEndSpec.hs --- a/test/Kioku/Cli/RecallEndToEndSpec.hs +++ b/test/Kioku/Cli/RecallEndToEndSpec.hs @@ -16,12 +16,15 @@ module Kioku.Cli.RecallEndToEndSpec (tests) where import Control.Monad.IO.Class (liftIO) +import Data.Aeson (toJSON) import Data.List (isInfixOf) import Data.Text (Text) import Data.Text qualified as Text import Data.Time (UTCTime, getCurrentTime) +import Data.UUID qualified as UUID import Effectful (Eff, IOE, (:>)) import Effectful.Error.Static (Error) +import Keiro.Timer qualified as Timer import Kioku.Api.Access ( MemoryAccessContext, MemoryActor (..), @@ -35,6 +38,7 @@ import Kioku.Api.Scope (MemoryScope (..), Namespace (..), ScopeKind (..)) import Kioku.Api.Types (Confidence (..), MemoryType (..)) import Kioku.App (runAppIO, withNoopAppEnv) +import Kioku.Distill.L2 (SceneTimerPayload (..), l2SceneProcessManagerName) import Kioku.Id (genMemoryId) import Kioku.Memory qualified as Memory import Kioku.Memory.Domain (RecordMemoryData (..)) @@ -43,8 +47,11 @@ import Kiroku.Store.Effect (Store) import Kiroku.Store.Effect.Resource (KirokuStoreResource) import Kiroku.Store.Error (StoreError) +import Kiroku.Store.Transaction (runTransaction) import System.Environment (getEnvironment) import System.Exit (ExitCode (..)) +import System.FilePath (()) +import System.IO.Temp (withSystemTempDirectory) import System.Process (CreateProcess (..), proc, readCreateProcessWithExitCode) import Test.Tasty (TestTree, testGroup) import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=)) @@ -53,9 +60,27 @@ tests = testGroup "kioku recall end to end" - [ testCase "each flag reaches the database as its own target, inside one space" targetsReachPostgres + [ testCase "deferred commands list, refuse disabled execution, and complete original work" deferredCommands, + testCase "explicit AI file overrides environment and credentials do not enable AI" aiFilePrecedence, + testCase "each flag reaches the database as its own target, inside one space" targetsReachPostgres ] +aiFilePrecedence :: IO () +aiFilePrecedence = withSystemTempDirectory "kioku-ai-config-test" $ \dir -> do + let path = dir "disabled.json" + writeFile path "{\"version\":1}" + inherited <- getEnvironment + let variables = + filter (\(name, _) -> name `notElem` ["KIOKU_AI_CONFIG", "PG_CONNECTION_STRING", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"]) inherited + <> [("KIOKU_AI_CONFIG", dir "missing.json"), ("PG_CONNECTION_STRING", "host=/nonexistent connect_timeout=1"), ("ANTHROPIC_API_KEY", "sentinel-not-a-credential")] + (code, _, err) <- readCreateProcessWithExitCode (proc "kioku" ["worker", "--backfill", "--ai-config", path]) {env = Just variables} "" + assertBool "disabled backfill fails" (code /= ExitSuccess) + assertContains "explicit file wins" "AIDisabled MemoryEmbedding" err + assertMissing "credential redaction" "sentinel-not-a-credential" err + (environmentCode, _, environmentError) <- readCreateProcessWithExitCode (proc "kioku" ["worker", "--backfill"]) {env = Just variables} "" + assertBool "environment file is used without flag" (environmentCode /= ExitSuccess) + assertContains "environment fallback" "missing.json" environmentError + -- | Two spaces holding the same namespace and the same two scopes, with content that names which -- space and which scope it came from — so a row appearing in the wrong answer is unmistakable. alphaSpace :: MemorySpaceId @@ -167,17 +192,20 @@ -- The connection string and memory space are replaced rather than added, so a developer's -- exported @PG_CONNECTION_STRING@ cannot redirect the test at their own database. runKioku :: Text -> MemorySpaceId -> [String] -> IO (ExitCode, String, String) -runKioku connStr space args = do +runKioku = runKiokuAt Nothing + +runKiokuAt :: Maybe FilePath -> Text -> MemorySpaceId -> [String] -> IO (ExitCode, String, String) +runKiokuAt directory connStr space args = do inherited <- getEnvironment let overridden = [ (name, value) | (name, value) <- inherited, - name `notElem` ["PG_CONNECTION_STRING", "KIOKU_MEMORY_SPACE"] + name `notElem` ["PG_CONNECTION_STRING", "KIOKU_MEMORY_SPACE", "KIOKU_AI_CONFIG", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"] ] <> [ ("PG_CONNECTION_STRING", Text.unpack connStr), ("KIOKU_MEMORY_SPACE", Text.unpack (memorySpaceIdText space)) ] - readCreateProcessWithExitCode (proc "kioku" args) {env = Just overridden} "" + readCreateProcessWithExitCode (proc "kioku" args) {env = Just overridden, cwd = directory} "" assertContains :: String -> String -> String -> IO () assertContains label needle haystack = @@ -200,3 +228,45 @@ spaceNamed :: Text -> MemorySpaceId spaceNamed raw = either (error . Text.unpack) id (mkMemorySpaceId raw) + +-- No memories exist in this scope, so valid foreground scene work completes +-- without invoking a model. The core timer suite separately exercises the real +-- interactive manifest/result handoff during a contested foreground claim. +deferredCommands :: IO () +deferredCommands = withKiokuMigratedDatabase $ \connStr -> + withSystemTempDirectory "kioku-deferred-cli" $ \dir -> do + let tid = Timer.TimerId UUID.nil + disabled = dir "disabled.json" + interactive = dir "interactive.json" + writeFile disabled "{\"version\":1}" + writeFile interactive "{\"version\":1,\"permissions\":[\"interactive\"],\"distillation\":{\"mode\":\"interactive\",\"provider\":\"claude\",\"model\":\"fixture\",\"workingDir\":\".\"}}" + now <- getCurrentTime + withNoopAppEnv (defaultConnectionSettings connStr) $ \app -> do + seeded <- runAppIO app $ do + runTransaction + ( Timer.scheduleTimerTx + ( Timer.TimerRequest + tid + l2SceneProcessManagerName + "scene-fixture" + now + (toJSON (SceneTimerPayload alphaSpace globalScope)) + ) + ) + Timer.deadLetterTimer tid "kioku:deferred:interactive-unavailable feature=scene" + seeded @?= Right True + (listed, output, _) <- runKiokuAt (Just dir) connStr alphaSpace ["worker", "deferred", "list", "--ai-config", disabled] + listed @?= ExitSuccess + assertContains "list timer" (UUID.toString UUID.nil) output + assertContains "list space" (Text.unpack (memorySpaceIdText alphaSpace)) output + (refused, _, diagnostic) <- runKiokuAt (Just dir) connStr alphaSpace ["worker", "deferred", "resume", UUID.toString UUID.nil, "--ai-config", disabled] + assertBool "disabled resume refuses" (refused /= ExitSuccess) + assertContains "disabled reason" "AIDisabled Scene" diagnostic + afterRefusal <- runAppIO app (Timer.lookupTimer tid) + fmap (fmap (.attempts)) afterRefusal @?= Right (Just 0) + (completed, done, err) <- runKiokuAt (Just dir) connStr alphaSpace ["worker", "deferred", "resume", UUID.toString UUID.nil, "--ai-config", interactive] + assertBool ("foreground failed: " <> err) (completed == ExitSuccess) + assertContains "completion" "Completed the original deferred timer" done + final <- runAppIO app (Timer.lookupTimer tid) + fmap (fmap (.status)) final @?= Right (Just Timer.Fired) + fmap (fmap (.attempts)) final @?= Right (Just 1)