diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.7.3.0 — 2026-10-28 + +### New Features + +* Migration `0017` installs the immutable store identity and empty append-only event-compaction ledger, with named constraints and mutation guards. The manifest now has seventeen entries; ordinary hot tables are unchanged. Existing sixteen-step databases gain one pending migration and schema comment `Managed by pg-migrate component kiroku through 0017`; migration-count fixtures must advance to seventeen. + ## 0.7.2.0 — 2026-10-26 ### New Features diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ `kiroku-store-migrations` owns Kiroku's PostgreSQL schema as one native `pg-migrate` component named `kiroku`. The component embeds an ordered manifest -and sixteen SQL payloads, so applications can compose it with other libraries +and seventeen SQL payloads, so applications can compose it with other libraries without copying Kiroku SQL. The first seven payloads are immutable historical -Codd bytes; `0008` through `0016` are native-only forward migrations. +Codd bytes; `0008` through `0017` are native-only forward migrations. ## Public API @@ -183,3 +183,8 @@ checksum for databases that applied the withdrawn 0.3.2.0/0.3.2.1 payload. Required before migrating such a database onto 0.4.0.0 or later; see the changelog. + +Migration `0017` adds the immutable, clone-shared store identity and append-only +compaction ledger. Apply it before using Store 0.13 compaction or identity APIs. +Existing sixteen-entry native stores gain one pending migration. See +[selective compaction](../docs/user/compaction.md) for the operator workflow. diff --git a/kiroku-store-migrations.cabal b/kiroku-store-migrations.cabal --- a/kiroku-store-migrations.cabal +++ b/kiroku-store-migrations.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: kiroku-store-migrations -version: 0.7.2.0 +version: 0.7.3.0 synopsis: Schema migrations for kiroku-store description: Native pg-migrate component, Codd history mapping, and migration executable diff --git a/migrations/0017.sql b/migrations/0017.sql new file mode 100644 --- /dev/null +++ b/migrations/0017.sql @@ -0,0 +1,108 @@ +-- add store identity and event-compaction ledger + +-- A schema-installed, immutable identity for this store. Generated once, on the +-- PostgreSQL side, the first time this migration runs. A restored clone shares +-- its source's identity by construction; that is intended, so a maintenance +-- manifest rehearsed on a clone still names the production store. +CREATE TABLE kiroku.store_identity ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE, + store_id UUID NOT NULL DEFAULT kiroku.uuidv7(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chk_store_identity_singleton CHECK (singleton) +); + +INSERT INTO kiroku.store_identity (singleton) +VALUES (TRUE) +ON CONFLICT DO NOTHING; + +-- Durable, append-only record of every applied selective compaction. Rows are +-- written only by kiroku-store's compaction apply transaction; the table ships +-- empty and its full contract is fixed here so the feature needs one schema +-- release. manifest_digest and report_digest are raw SHA-256 bytes. +CREATE TABLE kiroku.event_compactions ( + compaction_id UUID PRIMARY KEY DEFAULT kiroku.uuidv7(), + manifest_digest BYTEA NOT NULL UNIQUE, + report_digest BYTEA NOT NULL, + store_id UUID NOT NULL, + operation TEXT NOT NULL, + dead_letter_policy TEXT NOT NULL, + causation_policy TEXT NOT NULL, + selected_events BIGINT NOT NULL, + home_memberships BIGINT NOT NULL, + global_memberships BIGINT NOT NULL, + link_memberships BIGINT NOT NULL, + dead_letters_removed BIGINT NOT NULL, + causation_dependents BIGINT NOT NULL, + lowest_global_position BIGINT NOT NULL, + highest_global_position BIGINT NOT NULL, + affected_streams JSONB NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + applied_by TEXT NOT NULL DEFAULT session_user, + CONSTRAINT chk_event_compactions_manifest_digest_bytes + CHECK (octet_length(manifest_digest) = 32), + CONSTRAINT chk_event_compactions_report_digest_bytes + CHECK (octet_length(report_digest) = 32), + CONSTRAINT chk_event_compactions_operation_bytes + CHECK (octet_length(operation) BETWEEN 1 AND 512), + CONSTRAINT chk_event_compactions_policies + CHECK (dead_letter_policy IN ('refuse', 'remove') + AND causation_policy IN ('refuse', 'allow')), + CONSTRAINT chk_event_compactions_counts + CHECK (selected_events > 0 + AND home_memberships = selected_events + AND global_memberships = selected_events + AND link_memberships >= 0 + AND dead_letters_removed >= 0 + AND causation_dependents >= 0), + CONSTRAINT chk_event_compactions_positions + CHECK (0 < lowest_global_position + AND lowest_global_position <= highest_global_position) +); + +CREATE INDEX ix_event_compactions_applied_at + ON kiroku.event_compactions (applied_at DESC, compaction_id); + +-- Immutability and gated destruction, reusing the functions 0001 installed. +-- Nothing here touches events, stream_events, or streams. +CREATE TRIGGER no_update_store_identity + BEFORE UPDATE ON kiroku.store_identity + FOR EACH ROW EXECUTE FUNCTION kiroku.prevent_mutation(); + +CREATE TRIGGER no_delete_store_identity + BEFORE DELETE ON kiroku.store_identity + FOR EACH ROW EXECUTE FUNCTION kiroku.protect_deletion(); + +CREATE TRIGGER no_truncate_store_identity + BEFORE TRUNCATE ON kiroku.store_identity + FOR EACH STATEMENT EXECUTE FUNCTION kiroku.protect_truncation(); + +CREATE TRIGGER no_update_event_compactions + BEFORE UPDATE ON kiroku.event_compactions + FOR EACH ROW EXECUTE FUNCTION kiroku.prevent_mutation(); + +CREATE TRIGGER no_delete_event_compactions + BEFORE DELETE ON kiroku.event_compactions + FOR EACH ROW EXECUTE FUNCTION kiroku.protect_deletion(); + +CREATE TRIGGER no_truncate_event_compactions + BEFORE TRUNCATE ON kiroku.event_compactions + FOR EACH STATEMENT EXECUTE FUNCTION kiroku.protect_truncation(); + +COMMENT ON TABLE kiroku.store_identity IS + 'Single immutable row identifying this store; a restored clone shares its source identity.'; +COMMENT ON COLUMN kiroku.store_identity.store_id IS + 'UUIDv7 generated once by migration 0017; never updated.'; + +COMMENT ON TABLE kiroku.event_compactions IS + 'Append-only ledger of applied selective compactions keyed by manifest digest; rows are audit evidence and are never updated.'; +COMMENT ON COLUMN kiroku.event_compactions.manifest_digest IS + 'Raw SHA-256 of the canonical manifest encoding; unique so an exact manifest can be applied once.'; +COMMENT ON COLUMN kiroku.event_compactions.report_digest IS + 'Raw SHA-256 of the canonical report encoding recorded at apply time.'; +COMMENT ON COLUMN kiroku.event_compactions.affected_streams IS + 'JSON array of {"stream", "head_version"} objects sorted by stream name, recording the verified head of every affected stream.'; +COMMENT ON COLUMN kiroku.event_compactions.applied_by IS + 'PostgreSQL session_user that applied the manifest; recorded by default, not supplied by the caller.'; + +COMMENT ON SCHEMA kiroku IS + 'Managed by pg-migrate component kiroku through 0017'; diff --git a/migrations/manifest b/migrations/manifest --- a/migrations/manifest +++ b/migrations/manifest @@ -14,3 +14,4 @@ 0014.sql 0015.sql 0016.sql +0017.sql diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -44,7 +44,7 @@ main :: IO () main = hspec $ do describe "native Kiroku migration definition" $ do - it "tracks the sixteen native files in manifest order" $ do + it "tracks the seventeen native files in manifest order" $ do directory <- findMigrationsDirectory manifest <- Text.lines <$> Text.IO.readFile (directory "manifest") manifest `shouldBe` Text.pack <$> nativeMigrationFiles @@ -57,7 +57,7 @@ bytes <- ByteString.readFile (directory nativeName) lookup legacyName lockEntries `shouldBe` Just (checksumText bytes) - it "builds component kiroku and a sixteen-migration plan" $ do + it "builds component kiroku and a seventeen-migration plan" $ do component <- requireRight kirokuMigrations component `seq` pure () plan <- requirePlan @@ -93,7 +93,7 @@ `shouldReturn` "0007-existing.sql\n" describe "fresh native databases" $ do - it "applies all sixteen, verifies strictly, and reports AlreadyApplied on rerun" $ do + it "applies all seventeen, verifies strictly, and reports AlreadyApplied on rerun" $ do plan <- requirePlan result <- withMigratedDatabase plan $ \connection -> do assertSchema connection @@ -246,6 +246,43 @@ Text.isInfixOf "CTE Scan" planText `shouldBe` False either (expectationFailure . show) pure result + describe "compaction schema" $ do + it "publishes the exact store identity and ledger columns" $ do + plan <- requirePlan + result <- withMigratedDatabase plan $ \connection -> do + columns <- useSession connection (Session.statement () compactionColumnsStatement) + columns + `shouldBe` [ ("event_compactions", "compaction_id", "uuid", True), + ("event_compactions", "manifest_digest", "bytea", True), + ("event_compactions", "report_digest", "bytea", True), + ("event_compactions", "store_id", "uuid", True), + ("event_compactions", "operation", "text", True), + ("event_compactions", "dead_letter_policy", "text", True), + ("event_compactions", "causation_policy", "text", True), + ("event_compactions", "selected_events", "bigint", True), + ("event_compactions", "home_memberships", "bigint", True), + ("event_compactions", "global_memberships", "bigint", True), + ("event_compactions", "link_memberships", "bigint", True), + ("event_compactions", "dead_letters_removed", "bigint", True), + ("event_compactions", "causation_dependents", "bigint", True), + ("event_compactions", "lowest_global_position", "bigint", True), + ("event_compactions", "highest_global_position", "bigint", True), + ("event_compactions", "affected_streams", "jsonb", True), + ("event_compactions", "applied_at", "timestamp with time zone", True), + ("event_compactions", "applied_by", "text", True), + ("store_identity", "singleton", "boolean", True), + ("store_identity", "store_id", "uuid", True), + ("store_identity", "created_at", "timestamp with time zone", True) + ] + either (expectationFailure . show) pure result + + it "installs one identity row, named constraints, the ledger index, six protection triggers, and leaves hot tables unchanged" $ do + plan <- requirePlan + result <- withMigratedDatabase plan $ \connection -> do + facts <- useSession connection (Session.statement () compactionFactsStatement) + facts `shouldBe` (1, 0, 10, True, 6, 6) + either (expectationFailure . show) pure result + describe "history retention schema" $ do it "publishes the exact coordinator and lease table columns" $ do plan <- requirePlan @@ -287,11 +324,11 @@ withConnection settings $ \connection -> useSession connection (Session.script "INSERT INTO kiroku.subscriptions(subscription_name, last_seen, target_kind, target_category) VALUES ('legacy-category', 42, 'category', 'pm')") upgraded <- runMigrationPlan defaultRunOptions settings plan >>= requireMigration - reportOutcomes upgraded `shouldBe` replicate 15 AlreadyApplied <> [AppliedNow] + reportOutcomes upgraded `shouldBe` replicate 15 AlreadyApplied <> replicate (length nativeMigrationFiles - 15) AppliedNow withConnection settings $ \connection -> useSession connection (Session.script "INSERT INTO kiroku.subscriptions(subscription_name, last_seen, target_kind, target_category) VALUES ('namespace', 0, 'category_namespace', 'pm'); DO $$ BEGIN IF NOT EXISTS(SELECT FROM kiroku.subscriptions WHERE subscription_name='legacy-category' AND last_seen=42 AND target_kind='category' AND target_category='pm') THEN RAISE EXCEPTION 'binding changed'; END IF; END $$") rerun <- runMigrationPlan defaultRunOptions settings plan >>= requireMigration - reportOutcomes rerun `shouldBe` replicate 16 AlreadyApplied + reportOutcomes rerun `shouldBe` replicate (length nativeMigrationFiles) AlreadyApplied it "applies the pending tail in a session that never ran the bootstrap" $ do plan <- requirePlan @@ -473,7 +510,7 @@ pendingIds <- traverse (requireRight . migrationId "kiroku") - ["0008-schema-management-comment", "0009", "0010", "0011", "0012", "0013", "0014", "0015", "0016"] + ["0008-schema-management-comment", "0009", "0010", "0011", "0012", "0013", "0014", "0015", "0016", "0017"] verifiedBeforeCanary <- verifyMigrationPlan defaultRunOptions settings plan >>= requireMigration case verifiedBeforeCanary of VerificationReport verificationIssues _ _ _ -> @@ -515,7 +552,8 @@ "0013.sql", "0014.sql", "0015.sql", - "0016.sql" + "0016.sql", + "0017.sql" ] -- | The plan truncated to its first @count@ migrations, read from the checked-in @@ -658,7 +696,7 @@ (EXISTS (SELECT 1 FROM pg_catalog.pg_constraint WHERE conname = 'chk_streams_stream_name_length')), (EXISTS (SELECT 1 FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid = a.attrelid JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'kiroku' AND c.relname = 'streams' AND a.attname = 'truncate_before' AND NOT a.attisdropped)), (EXISTS (SELECT 1 FROM pg_catalog.pg_indexes WHERE schemaname = 'kiroku' AND indexname = 'ix_streams_browse_name' AND indexdef LIKE '%COLLATE "C"%' AND indexdef LIKE '%WHERE (stream_id <> 0)%')), - (obj_description(to_regnamespace('kiroku'), 'pg_namespace') = 'Managed by pg-migrate component kiroku through 0016') + (obj_description(to_regnamespace('kiroku'), 'pg_namespace') = 'Managed by pg-migrate component kiroku through 0017') ) AS checks(ok) """ Encoders.noParams @@ -1268,3 +1306,71 @@ """ Encoders.noParams (Decoders.singleRow ((,,,) <$> Decoders.column (Decoders.nonNullable Decoders.text) <*> Decoders.column (Decoders.nullable Decoders.text) <*> Decoders.column (Decoders.nonNullable Decoders.int8) <*> Decoders.column (Decoders.nonNullable Decoders.bool))) + +compactionColumnsStatement :: Statement () [(Text, Text, Text, Bool)] +compactionColumnsStatement = + Statement.preparable + """ + SELECT relation.relname::text, + attribute.attname::text, + pg_catalog.format_type(attribute.atttypid, attribute.atttypmod), + attribute.attnotnull + FROM pg_catalog.pg_class AS relation + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = relation.relnamespace + JOIN pg_catalog.pg_attribute AS attribute + ON attribute.attrelid = relation.oid + WHERE namespace.nspname = 'kiroku' + AND relation.relname IN ('event_compactions', 'store_identity') + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ORDER BY relation.relname, attribute.attnum + """ + Encoders.noParams + ( Decoders.rowList + ( (,,,) + <$> column Decoders.text + <*> column Decoders.text + <*> column Decoders.text + <*> column Decoders.bool + ) + ) + where + column = Decoders.column . Decoders.nonNullable + +compactionFactsStatement :: Statement () (Int64, Int64, Int64, Bool, Int64, Int64) +compactionFactsStatement = + Statement.preparable + """ + SELECT + (SELECT count(*) FROM kiroku.store_identity WHERE singleton), + (SELECT count(*) FROM kiroku.event_compactions), + (SELECT count(*) FROM pg_catalog.pg_constraint + WHERE connamespace = 'kiroku'::regnamespace AND conname IN ( + 'store_identity_pkey', 'chk_store_identity_singleton', + 'event_compactions_pkey', 'event_compactions_manifest_digest_key', + 'chk_event_compactions_manifest_digest_bytes', 'chk_event_compactions_report_digest_bytes', + 'chk_event_compactions_operation_bytes', 'chk_event_compactions_policies', + 'chk_event_compactions_counts', 'chk_event_compactions_positions')), + EXISTS (SELECT 1 FROM pg_catalog.pg_indexes WHERE schemaname = 'kiroku' + AND indexname = 'ix_event_compactions_applied_at' AND indexdef LIKE '%applied_at DESC%'), + (SELECT count(*) FROM pg_catalog.pg_trigger + WHERE tgrelid IN ('kiroku.store_identity'::regclass, 'kiroku.event_compactions'::regclass) + AND NOT tgisinternal AND tgname LIKE 'no_%'), + (SELECT count(*) FROM pg_catalog.pg_trigger + WHERE tgrelid IN ('kiroku.events'::regclass, 'kiroku.stream_events'::regclass, 'kiroku.streams'::regclass) + AND NOT tgisinternal AND tgname IN ('protect_replay_history_delete', 'protect_replay_history_truncate')) + """ + Encoders.noParams + ( Decoders.singleRow + ( (,,,,,) + <$> col Decoders.int8 + <*> col Decoders.int8 + <*> col Decoders.int8 + <*> col Decoders.bool + <*> col Decoders.int8 + <*> col Decoders.int8 + ) + ) + where + col = Decoders.column . Decoders.nonNullable