diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog for clickhouse-migrate-test-support + +## 0.1.0.0 — 2026-10-16 + +Initial release. + +### New Features + +- Isolated native ClickHouse server fixtures for consumers' tests + (`ClickHouseMigrate.TestSupport.Server`). +- Fault-injection proxy (plain and TLS) with per-query and per-ledger-stage + faults — dropped, truncated, delayed or redirected responses, late errors, + closed connections and malformed history — plus request evidence that excludes + SQL and credentials (`ClickHouseMigrate.TestSupport.FaultProxy`). +- Conservative host-local exclusive-access adapter for development and tests + (`ClickHouseMigrate.TestSupport.ExclusiveAccess`); not distributed fencing. diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2026, shinzui +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# clickhouse-migrate-test-support + +Owned native ClickHouse fixtures and controlled HTTP faults for qualification. This +optional package is not in the production core closure. It requires Python 3 and the +exact digest-checked Apple Silicon macOS server artifact in packaged matrix data; +it allocates temporary roots, owned process groups and loopback ports. Successful +fixtures remove only owned data; failures preserve diagnostics. No shared server is used. + +The host-local development coordinator uses an OS file lock and persistent dirty +marker. All administrators must share the host/canonical root/target key. Later entry +requires explicit external quiescence even after normal release; lock release or an +empty process/history read is insufficient. It is not distributed fencing. + +The default-off integration flag compiles public API suites, embedded captured +consumer data, CLI handlers and executable stream/cancellation checks. Transport +qualification also needs the separate default-off core internal-test-driver artifact. +Tests never run a server during normal Nix builds. Directly invoking a Cabal-built +suite requires `clickhouse_migrate_test_support_datadir` set to this package directory; +`cabal test` supplies it automatically and installed Nix artifacts use their data path. +The workspace's release matrix script runs all twelve required groups serially. diff --git a/clickhouse-migrate-test-support.cabal b/clickhouse-migrate-test-support.cabal new file mode 100644 --- /dev/null +++ b/clickhouse-migrate-test-support.cabal @@ -0,0 +1,107 @@ +cabal-version: 3.12 +name: clickhouse-migrate-test-support +version: 0.1.0.0 +synopsis: Isolated server fixtures for ClickHouse migration tests +description: + Isolated server fixtures for ClickHouse migration tests. + Validated public APIs with explicit application-owned configuration and scope. + +license: BSD-3-Clause +license-file: LICENSE +author: shinzui +maintainer: shinzui@users.noreply.github.com +category: Database +build-type: Simple +extra-doc-files: + CHANGELOG.md + README.md + +data-files: + test/data/clickhouse-matrix.json + test/fixtures/kansoku/**/*.sql + test/fixtures/kansoku/*.json + test/fixtures/kansoku/original/migrate.sh + test/scripts/native_fixture.py + +extra-source-files: + test/scripts/transport_smoke.py + +tested-with: ghc ==9.12.4 + +source-repository head + type: git + location: https://github.com/shinzui/clickhouse-migrate.git + subdir: clickhouse-migrate-test-support + +common common + default-language: GHC2024 + default-extensions: + DeriveAnyClass + DuplicateRecordFields + OverloadedLabels + OverloadedStrings + + ghc-options: -Wall + build-depends: + base >=4.21 && <4.22, + clickhouse-migrate ==0.1.*, + + build-depends: + aeson >=2.2.4.1 && <2.4, + bytestring >=0.12.2 && <0.13, + cryptohash-sha256 >=0.11.102.1 && <0.12, + directory >=1.3.10 && <1.4, + filelock >=0.1.1.8 && <0.2, + filepath >=1.5.4 && <1.6, + process >=1.6.25 && <1.7, + text >=2.1 && <2.2, + +library + import: common + hs-source-dirs: src + exposed-modules: + ClickHouseMigrate.TestSupport + ClickHouseMigrate.TestSupport.ExclusiveAccess + ClickHouseMigrate.TestSupport.FaultProxy + ClickHouseMigrate.TestSupport.Server + + other-modules: + ClickHouseMigrate.TestSupport.Internal.Fixture + Paths_clickhouse_migrate_test_support + + autogen-modules: Paths_clickhouse_migrate_test_support + +flag integration + description: Run owned real-server qualification (explicit driver path required) + default: False + manual: True + +test-suite clickhouse-migrate-integration + import: common + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: Main.hs + other-modules: + AdoptionFaults + AdoptionIntegration + CLIIntegration + ConsumerFixtures + FailureBoundaries + LedgerIntegration + Paths_clickhouse_migrate_test_support + RepairBoundaries + RepairIntegration + ReviewRemediation + RunnerIntegration + SurvivingWork + + autogen-modules: Paths_clickhouse_migrate_test_support + ghc-options: -threaded + build-depends: + clickhouse-migrate-cli ==0.1.*, + clickhouse-migrate-embed ==0.1.*, + clickhouse-migrate-test-support, + containers >=0.7 && <0.9, + + if !flag(integration) + buildable: False diff --git a/src/ClickHouseMigrate/TestSupport.hs b/src/ClickHouseMigrate/TestSupport.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/TestSupport.hs @@ -0,0 +1,9 @@ +-- | Owned server and fault fixtures for migration qualification. +module ClickHouseMigrate.TestSupport + ( module ClickHouseMigrate.TestSupport.Server, + module ClickHouseMigrate.TestSupport.FaultProxy, + ) +where + +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server diff --git a/src/ClickHouseMigrate/TestSupport/ExclusiveAccess.hs b/src/ClickHouseMigrate/TestSupport/ExclusiveAccess.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/TestSupport/ExclusiveAccess.hs @@ -0,0 +1,46 @@ +-- | Conservative HOST-LOCAL development/test coordination. All administrators +-- must share this host, canonical lock root and target key. Not distributed fencing. +module ClickHouseMigrate.TestSupport.ExclusiveAccess (hostLocalExclusiveAccess) where + +import ClickHouseMigrate.ExclusiveAccess +import ClickHouseMigrate.Ledger (QuiescenceEvidence, quiescenceTarget) +import ClickHouseMigrate.Prelude +import ClickHouseMigrate.Target (targetKeyText) +import Control.Concurrent (threadDelay) +import Control.Exception (mask, onException) +import Crypto.Hash.SHA256 qualified as SHA256 +import Data.ByteString qualified as BS +import Data.Text.Encoding qualified as Encoding +import Numeric (showHex) +import System.Directory (canonicalizePath, createDirectoryIfMissing, doesFileExist) +import System.FileLock (SharedExclusive (Exclusive), tryLockFile, unlockFile) +import System.FilePath (()) + +-- | A persistent marker survives BOTH normal release and process death: OS lock +-- release cannot prove server quiescence. A later acquisition requires explicit +-- target-bound external evidence, even after a clean client exit. This deliberate +-- inconvenience prevents this development adapter from silently authorizing a +-- successor after a lost response. Evidence remains the caller's responsibility. +hostLocalExclusiveAccess :: FilePath -> Int -> Maybe QuiescenceEvidence -> IO (Either AccessError ExclusiveAccess) +hostLocalExclusiveAccess directory micros evidence = do + createDirectoryIfMissing True directory + root <- canonicalizePath directory + let acquire target = mask $ \restore -> do + let digest = concatMap (\byte -> let value = showHex byte "" in replicate (2 - length value) '0' ++ value) (BS.unpack (SHA256.hash (Encoding.encodeUtf8 (targetKeyText target)))) + lockPath = root digest ++ ".lock" + marker = root digest ++ ".requires-quiescence" + wait = + tryLockFile lockPath Exclusive >>= \case + Nothing -> restore (threadDelay 10000) >> wait + Just lease -> pure lease + lease <- wait + ( do + dirty <- doesFileExist marker + when dirty $ case evidence of + Just assertion | quiescenceTarget assertion == target -> pure () + _ -> rejectAccessAcquisition PredecessorQuiescenceRequired + BS.writeFile marker (Encoding.encodeUtf8 (targetKeyText target)) + pure lease + ) + `onException` unlockFile lease + pure (externallySerialized "host-local-os-file-lock" "one host and canonical lock root; complete target key; persistent quiescence marker" micros acquire unlockFile (const (pure True))) diff --git a/src/ClickHouseMigrate/TestSupport/FaultProxy.hs b/src/ClickHouseMigrate/TestSupport/FaultProxy.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/TestSupport/FaultProxy.hs @@ -0,0 +1,126 @@ +-- | Fault controls for an owned fixture, with request evidence excluding SQL/auth. +module ClickHouseMigrate.TestSupport.FaultProxy + ( FaultProxy, + FaultMode (..), + LedgerStage (..), + setLedgerFault, + setLedgerStepFault, + addLedgerFault, + releaseFault, + withFaultProxy, + withTlsFaultProxy, + proxyEndpoint, + setFault, + setFaultForQuery, + proxyEvents, + proxyConnections, + redirectTo, + ) +where + +import ClickHouseMigrate.Prelude +import ClickHouseMigrate.TestSupport.Internal.Fixture +import Control.Exception (bracket) +import Data.Aeson (Value, object, (.=)) +import Data.Aeson qualified + +data FaultMode = Pass | DelayHistoryBody | AppendLateError | OversizedHistoryRow | HistoryGap | HistoryIdentityConflict | HistoryDuplicateConflict | DropResponse | TruncateResponse | LateError | Redirect | CloseConnection | CloseReusedConnection | DelayResponse | DelayFirstResponse | HoldBeforeForward | UnsupportedVersion | PrereleaseVersion | WeakenedPolicy | WeakenedStorage | SwitchIdentityAfterResponse + deriving stock (Eq, Show) + +data FaultProxy = FaultProxy !Fixture !Int !Text + +-- Shares the fixture owner's lifetime; the owner closes every proxy and server. +withFaultProxy :: Server -> (FaultProxy -> IO a) -> IO a +withFaultProxy server = withProxy server False + +withTlsFaultProxy :: Server -> (FaultProxy -> IO a) -> IO a +withTlsFaultProxy server = withProxy server True + +withProxy :: Server -> Bool -> (FaultProxy -> IO a) -> IO a +withProxy (Server fixture) tls = bracket acquire release + where + acquire = do + response <- fixtureRpc fixture (object ["operation" .= ("proxy-start" :: Text), "tls" .= tls]) + identity <- field "id" response + endpoint <- field "endpoint" response + pure (FaultProxy fixture identity endpoint) + release (FaultProxy _ identity _) = void $ fixtureRpc fixture (object ["operation" .= ("proxy-stop" :: Text), "id" .= identity]) + +proxyEndpoint :: FaultProxy -> Text +proxyEndpoint (FaultProxy _ _ endpoint) = endpoint + +setFault :: FaultProxy -> FaultMode -> IO () +setFault (FaultProxy fixture identity _) mode = void $ fixtureRpc fixture (object ["operation" .= ("proxy-mode" :: Text), "id" .= identity, "mode" .= modeName mode]) + +proxyEvents :: FaultProxy -> IO Value +proxyEvents (FaultProxy fixture identity _) = fixtureRpc fixture (object ["operation" .= ("proxy-events" :: Text), "id" .= identity]) + +redirectTo :: FaultProxy -> Text -> IO () +redirectTo (FaultProxy fixture identity _) endpoint = void $ fixtureRpc fixture (object ["operation" .= ("proxy-redirect" :: Text), "id" .= identity, "endpoint" .= endpoint]) + +modeName :: FaultMode -> Text +modeName Pass = "pass" +modeName DelayHistoryBody = "delay-history-body" +modeName AppendLateError = "append-late-error" +modeName OversizedHistoryRow = "oversized-history-row" +modeName HistoryGap = "history-gap" +modeName HistoryIdentityConflict = "history-identity-conflict" +modeName HistoryDuplicateConflict = "history-duplicate-conflict" +modeName DropResponse = "drop" +modeName TruncateResponse = "truncate" +modeName LateError = "late-error" +modeName Redirect = "redirect" +modeName CloseConnection = "close" +modeName CloseReusedConnection = "close-second" +modeName DelayResponse = "delay" +modeName DelayFirstResponse = "delay-first" +modeName HoldBeforeForward = "hold-before" +modeName UnsupportedVersion = "unsupported-version" +modeName PrereleaseVersion = "prerelease-version" +modeName WeakenedPolicy = "weakened-policy" +modeName WeakenedStorage = "weakened-storage" +modeName SwitchIdentityAfterResponse = "switch-identity" + +setFaultForQuery :: FaultProxy -> Text -> FaultMode -> IO () +setFaultForQuery (FaultProxy fixture identity _) query mode = void $ fixtureRpc fixture (object ["operation" .= ("proxy-mode" :: Text), "id" .= identity, "mode" .= modeName mode, "queryId" .= query]) + +proxyConnections :: FaultProxy -> IO Int +proxyConnections (FaultProxy fixture identity _) = fixtureRpc fixture (object ["operation" .= ("proxy-open-connections" :: Text), "id" .= identity]) >>= fieldResult + where + fieldResult value = case Data.Aeson.fromJSON value of Data.Aeson.Success number -> pure number; Data.Aeson.Error _ -> fail "Invalid proxy connection count" + +-- SQL bodies are inspected privately only to select a protocol stage; they are +-- never included in proxy events or diagnostics. +data LedgerStage = HistoryRead | DatabaseCreation | TableCreation | Initialization | MigrationStart | StepStart | StepSuccess | MigrationCompletion | FailureRecording | RecoveryAuthorization | StepAcceptance | AdoptionStart | AdoptionEntry | AdoptionCompletion + deriving stock (Eq, Show) + +stageName :: LedgerStage -> Text +stageName = \case + HistoryRead -> "history-read" + DatabaseCreation -> "ddl-database" + TableCreation -> "ddl-table" + Initialization -> "initialized" + MigrationStart -> "migration-started" + StepStart -> "step-started" + StepSuccess -> "step-succeeded" + MigrationCompletion -> "migration-completed" + FailureRecording -> "execution-failed" + RecoveryAuthorization -> "recovery-authorized" + StepAcceptance -> "step-accepted" + AdoptionStart -> "adoption-started" + AdoptionEntry -> "adoption-entry" + AdoptionCompletion -> "adoption-completed" + +setLedgerFault :: FaultProxy -> LedgerStage -> FaultMode -> IO () +setLedgerFault (FaultProxy fixture identity _) stage mode = void $ fixtureRpc fixture (object ["operation" .= ("proxy-mode" :: Text), "id" .= identity, "mode" .= modeName mode, "stage" .= stageName stage]) + +releaseFault :: FaultProxy -> IO () +releaseFault (FaultProxy fixture identity _) = void $ fixtureRpc fixture (object ["operation" .= ("proxy-release" :: Text), "id" .= identity]) + +-- | Select a named step without exposing its SQL in recorded evidence. +setLedgerStepFault :: FaultProxy -> LedgerStage -> Text -> FaultMode -> IO () +setLedgerStepFault (FaultProxy fixture identity _) stage step mode = void $ fixtureRpc fixture (object ["operation" .= ("proxy-mode" :: Text), "id" .= identity, "mode" .= modeName mode, "stage" .= stageName stage, "step" .= step]) + +-- | Overlay a ledger fault while retaining a query-specific application fault. +addLedgerFault :: FaultProxy -> LedgerStage -> FaultMode -> IO () +addLedgerFault (FaultProxy fixture identity _) stage mode = void $ fixtureRpc fixture (object ["operation" .= ("proxy-rule" :: Text), "id" .= identity, "mode" .= modeName mode, "stage" .= stageName stage]) diff --git a/src/ClickHouseMigrate/TestSupport/Internal/Fixture.hs b/src/ClickHouseMigrate/TestSupport/Internal/Fixture.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/TestSupport/Internal/Fixture.hs @@ -0,0 +1,116 @@ +-- | Private RPC/process ownership for native fixtures. Stdout is protocol only. +module ClickHouseMigrate.TestSupport.Internal.Fixture + ( Fixture, + Server (..), + withFixture, + fixtureRpc, + fixtureEndpoint, + fixtureRoot, + fixtureVersion, + field, + ) +where + +import ClickHouseMigrate.Prelude +import Control.Concurrent.MVar (MVar, newMVar, withMVar) +import Control.Exception (SomeAsyncException, SomeException, fromException, mask, onException, throwIO, try) +import Data.Aeson (FromJSON, Key, Result (..), Value (..), fromJSON, object, (.=)) +import Data.Aeson qualified as Aeson +import Data.Aeson.KeyMap qualified as KM +import Data.ByteString.Char8 qualified as BS +import Data.ByteString.Lazy.Char8 qualified as Lazy +import Data.Text qualified as Text +import Paths_clickhouse_migrate_test_support (getDataFileName) +import System.IO (Handle, hClose, hFlush, openTempFile) +import System.Process (CreateProcess (..), ProcessHandle, StdStream (..), createProcess, proc, terminateProcess, waitForProcess) +import System.Timeout (timeout) + +data Server = Server !Fixture + +data Fixture = Fixture !Handle !Handle !ProcessHandle !(MVar ()) !Text !FilePath !Text + +field :: (FromJSON a) => Key -> Value -> IO a +field key (Object value) = case KM.lookup key value of + Just member -> case fromJSON member of + Success result -> pure result + Error _ -> fail "Invalid fixture response field" + Nothing -> fail "Missing fixture response field" +field _ _ = fail "Invalid fixture response object" + +withFixture :: Text -> (Fixture -> IO a) -> IO a +withFixture entry action = mask $ \restore -> do + script <- getDataFileName "test/scripts/native_fixture.py" + matrix <- getDataFileName "test/data/clickhouse-matrix.json" + (logPath, logHandle) <- openTempFile "/tmp" "clickhouse-migrate-fixture.log" + pipes <- + createProcess + ( (proc "python3" [script, "--matrix", matrix, "--entry", Text.unpack entry]) + { std_in = CreatePipe, + std_out = CreatePipe, + std_err = UseHandle logHandle + } + ) + `onException` hClose logHandle + case pipes of + (Just input, Just output, _, process) -> do + ready <- try @SomeException $ do + line <- timeout 40000000 (BS.hGetLine output) + value <- maybe (fail "Fixture startup deadline expired") (either (const (fail "Fixture startup failed")) pure . Aeson.eitherDecodeStrict') line + endpoint <- field "endpoint" value + root <- field "root" value + version <- field "version" value + gate <- newMVar () + pure (Fixture input output process gate endpoint root version) + case ready of + Left exception -> do + terminateProcess process + _ <- timeout 20000000 (waitForProcess process) + hClose input + hClose output + hClose logHandle + case fromException exception :: Maybe SomeAsyncException of + Just _ -> throwIO exception + Nothing -> fail ("Owned fixture startup failed; evidence: " ++ logPath) + Right fixture -> do + result <- try @SomeException (restore (action fixture)) + cleanup <- try @SomeException $ do + _ <- fixtureRpc fixture (object ["operation" .= ("stop" :: Text), "success" .= either (const False) (const True) result]) + stopped <- timeout 20000000 (waitForProcess process) + case stopped of + Nothing -> terminateProcess process >> void (timeout 20000000 (waitForProcess process)) >> fail "Fixture cleanup deadline expired" + Just _ -> pure () + case cleanup of + Left _ -> terminateProcess process >> void (timeout 20000000 (waitForProcess process)) + Right () -> pure () + hClose input + hClose output + hClose logHandle + case result of + Left exception -> throwIO exception + Right value -> case cleanup of + Left exception -> case fromException exception :: Maybe SomeAsyncException of + Just _ -> throwIO exception + Nothing -> fail ("Owned fixture cleanup failed; evidence: " ++ logPath) + Right () -> pure value + _ -> hClose logHandle >> fail "Owned fixture pipes unavailable" + +fixtureRpc :: Fixture -> Value -> IO Value +fixtureRpc (Fixture input output _ gate _ root _) request = withMVar gate $ \() -> do + Lazy.hPutStrLn input (Aeson.encode request) + -- ByteString writes to a Handle still use its buffering policy. + hFlush input + let deadline = case request of Object fields | KM.lookup "operation" fields == Just (String "restart") -> 40000000; _ -> 15000000 + line <- timeout deadline (BS.hGetLine output) + reply <- maybe (fail ("Fixture RPC deadline expired; evidence: " ++ root)) (either (const (fail "Invalid fixture protocol")) pure . Aeson.eitherDecodeStrict') line + case reply of + Object members -> maybe (fail ("Owned fixture operation failed; evidence: " ++ root)) pure (KM.lookup "result" members) + _ -> fail "Invalid fixture protocol object" + +fixtureEndpoint :: Fixture -> Text +fixtureEndpoint (Fixture _ _ _ _ endpoint _ _) = endpoint + +fixtureRoot :: Fixture -> FilePath +fixtureRoot (Fixture _ _ _ _ _ root _) = root + +fixtureVersion :: Fixture -> Text +fixtureVersion (Fixture _ _ _ _ _ _ version) = version diff --git a/src/ClickHouseMigrate/TestSupport/Server.hs b/src/ClickHouseMigrate/TestSupport/Server.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/TestSupport/Server.hs @@ -0,0 +1,34 @@ +-- | Disposable native ClickHouse server. Never uses a shared developer database. +module ClickHouseMigrate.TestSupport.Server + ( Server, + withCandidateServer, + serverEndpoint, + serverLogDirectory, + serverVersion, + queryServer, + restartCandidateServer, + ) +where + +import ClickHouseMigrate.Prelude +import ClickHouseMigrate.TestSupport.Internal.Fixture +import Data.Aeson (Value, object, (.=)) + +withCandidateServer :: Text -> (Server -> IO a) -> IO a +withCandidateServer entry action = withFixture entry (action . Server) + +serverEndpoint :: Server -> Text +serverEndpoint (Server fixture) = fixtureEndpoint fixture + +serverLogDirectory :: Server -> FilePath +serverLogDirectory (Server fixture) = fixtureRoot fixture + +serverVersion :: Server -> Text +serverVersion (Server fixture) = fixtureVersion fixture + +queryServer :: Server -> Text -> IO Value +queryServer (Server fixture) sql = fixtureRpc fixture (object ["operation" .= ("query" :: Text), "sql" .= sql]) + +-- | SIGKILL/restart the owned candidate using the same temporary data root. +restartCandidateServer :: Server -> IO () +restartCandidateServer (Server fixture) = void $ fixtureRpc fixture (object ["operation" .= ("restart" :: Text)]) diff --git a/test/AdoptionFaults.hs b/test/AdoptionFaults.hs new file mode 100644 --- /dev/null +++ b/test/AdoptionFaults.hs @@ -0,0 +1,265 @@ +module AdoptionFaults (adoptionFaultCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Adoption qualified as A +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.ExclusiveAccess qualified as Access +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.ExclusiveAccess (hostLocalExclusiveAccess) +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Concurrent (threadDelay) +import Control.Exception (AsyncException (ThreadKilled), SomeAsyncException, finally, throwIO, try) +import Control.Monad (forM_, void, when) +import Crypto.Hash.SHA256 qualified as SHA +import Data.Aeson (Value) +import Data.ByteString qualified as BS +import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE +import Data.Text qualified as T +import Data.Text.IO qualified as TextIO +import Numeric (showHex) +import RunnerIntegration (assert, field, fixture, fixtureWithLimits, fromJson, planFor, provider, right, scalar) +import System.FilePath (()) +import System.Timeout (timeout) + +historyOf :: R.InspectionReport -> R.HistoryReport +historyOf (R.InspectionReport snapshot _ _) = snapshot + +positionOf :: R.InspectionReport -> IO L.LedgerPosition +positionOf (R.InspectionReport (R.HistoryReport _ _ _ (Just position) _ _ _ _) _ _) = pure position +positionOf _ = fail "Missing adoption ledger position" + +requestFor :: MigrationPlan -> TargetKey -> LedgerId -> L.AuditContext -> L.QuiescenceEvidence -> L.ExpectedLedgerState -> T.Text -> IO A.AdoptionRequest +requestFor plan key ledger audit quiet expected source = do + row <- right (A.legacyRow "001.sql" "2020-01-01T00:00:00Z" Nothing) + let first = NE.head (componentMigrations (NE.head (inspectPlan plan))) + mapped <- right (A.attestedMapping "001.sql" "app" "one" (migrationChecksumHex (migrationChecksum first)) ["fixture:reviewed-complete-two-statement-mapping"]) + right (A.adoptionRequest key expected (Just (L.authorizeInitialization key ledger audit)) quiet audit source "fixture:immutable-legacy-snapshot" (row :| []) (mapped :| [])) + +prepare :: Server -> T.Text -> IO () +prepare server label = do + void (queryServer server ("CREATE TABLE default." <> label <> " (id UInt8, secret String) ENGINE=MergeTree ORDER BY id")) + void (queryServer server ("INSERT INTO default." <> label <> " VALUES (7,'SQL_SECRET_MARKER')")) + +counts :: FaultProxy -> IO (Int, Int) +counts proxy = do + rows <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") rows :: IO [Maybe T.Text] + pure (length (filter (== Just "application-sql") stages), length (filter (== Just "adoption-entry") stages)) + +save :: Server -> T.Text -> FaultProxy -> R.InspectionReport -> IO () +save server label proxy snapshot = do + requests <- proxyEvents proxy + TextIO.writeFile (serverLogDirectory server T.unpack label ++ ".trace") (T.pack (show requests) <> "\n" <> T.pack (show snapshot)) + +adoptionFaultCases :: Server -> IO () +adoptionFaultCases server = do + access <- provider + audit <- right (L.auditContext "adoption fault reviewer" "controlled complete effects and drained predecessor requests" ["owned fixture; parent excludes every outside administrator"]) + forM_ (zip [0 :: Int ..] [(stage, mode) | stage <- [Initialization, AdoptionStart, AdoptionEntry, AdoptionCompletion], mode <- [DropResponse, LateError]]) $ \(index, (stage, mode)) -> withFaultProxy server $ \proxy -> do + let label = "adoption_append_" <> T.pack (show index) + prepare server label + plan <- planFor label 7 + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + quiet <- right (L.attestQuiescence key audit) + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + observed <- I.status I.Observe config plan >>= right + request <- requestFor plan key ledger audit quiet (L.ExpectAbsent key) "synthetic-legacy" + adoption <- right (A.planAdoption plan observed request) + setLedgerFault proxy stage mode + failed <- A.applyAdoption (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) plan adoption + receipt <- case failed of + Left (R.RunFailure (R.LedgerRequestFailed evidence _ (Just receipt)) _ []) | evidence == (if mode == LateError then L.ObservedFailure else L.UncertainSubmission) -> pure receipt + other -> fail ("Wrong adoption append failure " ++ show other) + assert "Adoption append error leaked SQL" (not ("SQL_SECRET_MARKER" `T.isInfixOf` T.pack (show failed))) + setFault proxy Pass + fresh <- I.status (I.Guarded access) bound plan >>= right + save server label proxy fresh + let expectedEvents = case stage of Initialization -> 1; AdoptionStart -> 2; AdoptionEntry -> 3; _ -> 4 + assert "Failed adoption append did not commit exactly once" (case historyOf fresh of R.HistoryReport _ _ _ _ _ events _ _ -> length events == expectedEvents) + _ <- L.reconcileAppend bound access receipt >>= \result -> case L.result result of Right (L.PresentIdentical, _) -> pure (); _ -> fail "Exact failed adoption append not reconcilable" + if stage == AdoptionCompletion + then do + _ <- A.applyAdoption (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan adoption >>= right + pure () + else do + stale <- A.resumeAdoption (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan adoption + assert "Original absence request resumed after state moved" (case stale of Left (R.RunFailure (R.AdoptionRejected A.AdoptionExpectedStateMismatch) _ _) -> True; _ -> False) + when (stage /= Initialization) $ do + blocked <- Run.up (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit Nothing) plan + assert "Ordinary up consumed incomplete adoption" (case blocked of Left (R.RunFailure (R.LedgerFailed L.UnresolvedHistory) _ _) -> True; _ -> False) + current <- positionOf fresh + changed <- requestFor plan key ledger audit quiet (L.ExpectPosition current) "changed-source" + when (stage /= Initialization) $ do + assert "Changed interrupted batch planned" (case A.planAdoption plan fresh changed of Left A.AdoptionBatchConflict -> True; _ -> False) + next <- requestFor plan key ledger audit quiet (L.ExpectPosition current) "synthetic-legacy" >>= right . A.planAdoption plan fresh + _ <- (if stage == Initialization then A.applyAdoption else A.resumeAdoption) (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan next >>= right + pure () + _ <- I.verify I.Observe bound (I.verifyOptions True) plan >>= right + assert "Adoption recovery submitted SQL or duplicated an entry" . (== (0, 1)) =<< counts proxy + assert "Adoption recovery replayed predecessor INSERT" . (== 1) =<< scalar server ("SELECT toUInt32(count()) AS value FROM default." <> label) + putStrLn ("adoption append " ++ show stage ++ "/" ++ show mode ++ ": exact reconciliation; fresh explicit continuation; no SQL or duplicate entry") + forM_ (zip [0 :: Int ..] [E.BeforeAdoptionStart, E.AdoptionStarted, E.BeforeAdoptionEntry, E.AdoptionEntryRecorded, E.BeforeAdoptionComplete, E.AdoptionCompleted]) $ \(index, phase) -> withFaultProxy server $ \proxy -> do + let label = "adoption_callback_" <> T.pack (show index) + prepare server label + plan <- planFor label 7 + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + quiet <- right (L.attestQuiescence key audit) + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + observed <- I.status I.Observe config plan >>= right + request <- requestFor plan key ledger audit quiet (L.ExpectAbsent key) "synthetic-legacy" + adoption <- right (A.planAdoption plan observed request) + let sink (E.RunEvent actual _ _ _ _ _ _) = when (actual == phase) (fail "adoption observer SQL_SECRET_MARKER") + failed <- A.applyAdoption (Run.mutationContext config access sink E.redactedDiagnostics) plan adoption + assert "Adoption observer failure hidden" (case failed of Left (R.RunFailure (R.CallbackFailed _) _ []) -> True; _ -> False) + fresh <- I.status (I.Guarded access) bound plan >>= right + save server label proxy fresh + let expectedEvents = case phase of E.BeforeAdoptionStart -> 1; E.AdoptionStarted -> 2; E.BeforeAdoptionEntry -> 2; E.AdoptionEntryRecorded -> 3; E.BeforeAdoptionComplete -> 3; _ -> 4 + assert "Adoption callback lost acknowledged state" (case failed of Left (R.RunFailure _ (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ _ _ events _ _) _ _ _) _) -> length events == expectedEvents; _ -> False) + if phase == E.AdoptionCompleted + then do _ <- A.applyAdoption (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan adoption >>= right; pure () + else do + current <- positionOf fresh + next <- requestFor plan key ledger audit quiet (L.ExpectPosition current) "synthetic-legacy" >>= right . A.planAdoption plan fresh + _ <- (if phase == E.BeforeAdoptionStart then A.applyAdoption else A.resumeAdoption) (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan next >>= right + pure () + assert "Adoption callback recovery submitted SQL/duplicate entry" . (== (0, 1)) =<< counts proxy + putStrLn ("adoption observer " ++ show phase ++ ": last acknowledgement retained; explicit continuation; zero SQL") + -- Resume must refuse a nonexistent batch before even bootstrap DDL. + withFaultProxy server $ \proxy -> do + let label = "adoption_absent_resume" + plan <- planFor label 7 + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + quiet <- right (L.attestQuiescence key audit) + config <- fixture server label (proxyEndpoint proxy) FirstUse + observed <- I.status I.Observe config plan >>= right + request <- requestFor plan key ledger audit quiet (L.ExpectAbsent key) "synthetic-legacy" + adoption <- right (A.planAdoption plan observed request) + refused <- A.resumeAdoption (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) plan adoption + assert "Resume initialized a missing batch" (case refused of Left (R.RunFailure (R.AdoptionRejected A.AdoptionBatchConflict) _ _) -> True; _ -> False) + present <- scalar server "SELECT toUInt32(count()) AS value FROM system.databases WHERE name='adoption_absent_resume_history'" + assert "Invalid adoption resume created objects" (present == 0) + putStrLn "adoption absent resume: complete fresh validation rejects before bootstrap mutation" + -- Real predecessor checksum fixture: read its retained algorithm/digest, rather + -- than relabeling a current-file hash as historical evidence. + withFaultProxy server $ \proxy -> do + let label = "adoption_verified" + prepare server label + plan <- planFor label 7 + let migration = NE.head (componentMigrations (NE.head (inspectPlan plan))) + historical = mconcat [stepSql step <> ";\n" | step <- NE.toList (migrationSteps migration)] + storedHash = T.pack (concatMap (\byte -> let value = showHex byte "" in replicate (2 - length value) '0' ++ value) (BS.unpack (SHA.hash historical))) + void (queryServer server "CREATE TABLE default.predecessor_checksums (algorithm String, checksum String) ENGINE=Memory") + void (queryServer server ("INSERT INTO default.predecessor_checksums VALUES ('sha256','" <> storedHash <> "')")) + rows <- queryServer server "SELECT algorithm,checksum FROM default.predecessor_checksums" >>= fromJson :: IO [Value] + (algorithm, retained) <- case rows of [row] -> (,) <$> field "algorithm" row <*> field "checksum" row; _ -> fail "Missing actual predecessor checksum row" + checksum <- right (A.sourceChecksum algorithm retained) + row <- right (A.legacyRow "001.sql" "2020-01-01T00:00:00Z" (Just checksum)) + mapping <- right (A.verifiedMapping "001.sql" "app" "one" (migrationChecksumHex (migrationChecksum migration)) historical ["fixture:retained-predecessor-checksum-row"]) + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + quiet <- right (L.attestQuiescence key audit) + config <- fixture server label (proxyEndpoint proxy) FirstUse + observed <- I.status I.Observe config plan >>= right + request <- right (A.adoptionRequest key (L.ExpectAbsent key) (Just (L.authorizeInitialization key ledger audit)) quiet audit "predecessor-with-sha256" "fixture:immutable-historical-source-and-checksum-row" (row :| []) (mapping :| [])) + adoption <- right (A.planAdoption plan observed request) + result <- A.applyAdoption (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) plan adoption >>= right + assert "Stored checksum evidence downgraded/conflated" (case result of R.RunReport _ _ _ _ _ [R.MigrationSummary _ R.Adopted _ steps] [] _ -> all (\case R.StepSummary _ _ R.StepAccepted _ (Just (R.EvidenceSummary L.PredecessorChecksum _)) -> True; _ -> False) steps; _ -> False) + assert "Checksum adoption submitted application SQL" . (== (0, 1)) =<< counts proxy + putStrLn "adoption historical checksum: actual predecessor algorithm/digest row and matching historical bytes; evidence retained separately from observed execution" + -- Async observer cancellation escapes after cleanup; the durable entry remains. + withFaultProxy server $ \proxy -> do + let label = "adoption_async" + prepare server label + plan <- planFor label 7 + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + quiet <- right (L.attestQuiescence key audit) + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + observed <- I.status I.Observe config plan >>= right + request <- requestFor plan key ledger audit quiet (L.ExpectAbsent key) "synthetic-legacy" + adoption <- right (A.planAdoption plan observed request) + let sink (E.RunEvent phase _ _ _ _ _ _) = when (phase == E.AdoptionEntryRecorded) (throwIO ThreadKilled) + cancelled <- try @SomeAsyncException (A.applyAdoption (Run.mutationContext config access sink E.redactedDiagnostics) plan adoption) + assert "Adoption cancellation became synchronous result" (case cancelled of Left _ -> True; _ -> False) + fresh <- I.status I.Observe bound plan >>= right + assert "Cancelled adoption lost its entry" (case historyOf fresh of R.HistoryReport _ _ _ _ _ events _ True -> length events == 3; _ -> False) + current <- positionOf fresh + next <- requestFor plan key ledger audit quiet (L.ExpectPosition current) "synthetic-legacy" >>= right . A.planAdoption plan fresh + _ <- A.resumeAdoption (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan next >>= right + cleanup <- right (Access.externallySerialized "owned-parent-release-fault" "all requests drained before next job" 1000000 (const (pure ())) (const (fail "release private message")) (const (pure True))) + final <- A.applyAdoption (Run.mutationContext bound cleanup E.quietEvents E.redactedDiagnostics) plan adoption + assert "Read-only committed adoption hid cleanup failure/progress" (case final of Left (R.RunFailure R.CleanupFailed (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ _ _ events _ False) _ _ _) secondary) -> length events == 4 && not (null secondary); _ -> False) + assert "Cancelled/read-only adoption submitted SQL or duplicate entry" . (== (0, 1)) =<< counts proxy + putStrLn "adoption async observer: rethrown after cleanup, committed entry preserved; fresh resume; read-only reapply retains cleanup-only failure" + delayedEntry server access audit + +-- An empty observation while a proxy-held ledger request is not yet forwarded +-- must not admit a mutating successor. Only the fixture parent can release and +-- prove the complete controlled request drain. +delayedEntry :: Server -> Access.ExclusiveAccess -> L.AuditContext -> IO () +delayedEntry server parentJob audit = withFaultProxy server $ \proxy -> do + let label = "adoption_late_entry" + root = serverLogDirectory server "adoption-late-locks" + prepare server label + plan <- planFor label 7 + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + quiet <- right (L.attestQuiescence key audit) + short <- right (httpLimits 1000000 1048576) + config <- fixtureWithLimits short server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + local <- hostLocalExclusiveAccess root 1000000 Nothing >>= right + observed <- I.status I.Observe config plan >>= right + request <- requestFor plan key ledger audit quiet (L.ExpectAbsent key) "synthetic-legacy" + adoption <- right (A.planAdoption plan observed request) + setLedgerFault proxy AdoptionEntry HoldBeforeForward + ( do + stopped <- A.applyAdoption (Run.mutationContext config local E.quietEvents E.redactedDiagnostics) plan adoption + receipt <- case stopped of Left (R.RunFailure (R.LedgerRequestFailed L.UncertainSubmission _ (Just receipt)) _ []) -> pure receipt; _ -> fail "Held adoption entry did not become uncertain" + empty <- I.status I.Observe bound plan >>= right + assert "Held entry arrived before explicit release" (case historyOf empty of R.HistoryReport _ _ _ _ _ events _ True -> length events == 2; _ -> False) + reconciled <- L.reconcileAppend bound parentJob receipt + assert "Held entry not observed as absent" (case L.result reconciled of Right (L.NotObserved, _) -> True; _ -> False) + active <- scalar server ("SELECT toUInt32(count()) AS value FROM system.processes WHERE query_id='" <> queryIdText (L.appendQueryId receipt) <> "'") + assert "Unforwarded request unexpectedly active on server" (active == 0) + current <- positionOf empty + premature <- requestFor plan key ledger audit quiet (L.ExpectPosition current) "synthetic-legacy" >>= right . A.planAdoption plan empty + -- Negative attempt: neither the old assertion nor absent rows establish new + -- quiescence. The real host-local marker must refuse entry after OS unlock. + blocked <- hostLocalExclusiveAccess root 1000000 Nothing >>= right + refused <- A.resumeAdoption (Run.mutationContext bound blocked E.quietEvents E.redactedDiagnostics) plan premature + assert "NotObserved admitted premature adoption resume" (case refused of Left (R.RunFailure (R.AccessFailed Access.PredecessorQuiescenceRequired) _ _) -> True; _ -> False) + assert "Premature successor submitted a duplicate entry" . (== (0, 1)) =<< counts proxy + releaseFault proxy + let wait message action = do + done <- timeout 8000000 (let loop = action >>= \ready -> if ready then pure () else threadDelay 20000 >> loop in loop) + assert message (done == Just ()) + wait "Late entry did not arrive" $ do + snapshot <- I.status I.Observe bound plan >>= right + pure (case historyOf snapshot of R.HistoryReport _ _ _ _ _ events _ True -> length events == 3; _ -> False) + wait "Controlled original proxy request failed to drain" ((== 0) <$> proxyConnections proxy) + setFault proxy Pass + evidence <- right (L.auditContext "post-drain operator" "original client ended; parent released and drained every controlled request" ["owned proxy has zero open connections; exact late event present after synchronous response"]) + established <- right (L.attestQuiescence key evidence) + access <- hostLocalExclusiveAccess root 1000000 (Just established) >>= right + fresh <- I.status I.Observe bound plan >>= right + save server label proxy fresh + position <- positionOf fresh + next <- requestFor plan key ledger evidence established (L.ExpectPosition position) "synthetic-legacy" >>= right . A.planAdoption plan fresh + _ <- A.resumeAdoption (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan next >>= right + _ <- I.verify I.Observe bound (I.verifyOptions True) plan >>= right + assert "Late entry recovery replayed SQL or entry" . (== (0, 1)) =<< counts proxy + putStrLn "adoption late entry: absent history/process row and NotObserved precede one commit; host marker blocks successor; explicit complete drain permits resume without duplicate or premature completion" + ) + `finally` releaseFault proxy diff --git a/test/AdoptionIntegration.hs b/test/AdoptionIntegration.hs new file mode 100644 --- /dev/null +++ b/test/AdoptionIntegration.hs @@ -0,0 +1,74 @@ +module AdoptionIntegration (adoptionEarlyCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Adoption qualified as A +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Monad (void) +import Data.Aeson (Value) +import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE +import Data.Set qualified as Set +import Data.Text qualified as T +import RunnerIntegration (assert, field, fixture, fromJson, provider, right, scalar) + +plans :: IO (MigrationPlan, MigrationPlan) +plans = do + create <- right (sqlStep "create" "CREATE TABLE default.adoption_early (id UInt8) ENGINE=Memory") + seed <- right (sqlStep "seed" "INSERT INTO default.adoption_early VALUES (1)") + one <- right (sqlMigration "one" (create :| [seed])) + add <- right (sqlStep "next" "INSERT INTO default.adoption_early VALUES (9)") + two <- right (sqlMigration "two" (add :| [])) + component <- right (migrationComponent "app" Set.empty (one :| [])) + extended <- right (migrationComponent "app" Set.empty (one :| [two])) + (,) <$> right (migrationPlan (component :| [])) <*> right (migrationPlan (extended :| [])) + +adoptionEarlyCases :: Server -> IO () +adoptionEarlyCases server = withFaultProxy server $ \proxy -> do + access <- provider + audit <- right (L.auditContext "adoption fixture operator" "reviewed synthetic filename-only legacy baseline" ["fixture:legacy-table-and-seed-independent-observation"]) + ledger <- newLedgerId + key <- right (targetKey "runner-tests/adoption_early") + quiet <- right (L.attestQuiescence key audit) + config <- fixture server "adoption_early" (proxyEndpoint proxy) FirstUse + bound <- fixture server "adoption_early" (proxyEndpoint proxy) (ExistingLedger ledger) + (plan, extended) <- plans + -- Independent old-runner effects exist before native history. Adoption must + -- not execute either CREATE or INSERT, even though both are valid definitions. + void (queryServer server "CREATE TABLE default.adoption_early (id UInt8) ENGINE=Memory") + void (queryServer server "INSERT INTO default.adoption_early SETTINGS async_insert=0 VALUES (1)") + observed <- I.status I.Observe config extended >>= right + row <- right (A.legacyRow "001.sql" "2020-01-01T00:00:00Z" Nothing) + let first = NE.head (componentMigrations (NE.head (inspectPlan plan))) + mapping <- right (A.attestedMapping "001.sql" "app" "one" (migrationChecksumHex (migrationChecksum first)) ["fixture:reviewed-two-statement-mapping"]) + request <- right (A.adoptionRequest key (L.ExpectAbsent key) (Just (L.authorizeInitialization key ledger audit)) quiet audit "synthetic-filename-ledger" "fixture:immutable-source-snapshot" (row :| []) (mapping :| [])) + adoption <- right (A.planAdoption extended observed request) + let context = Run.mutationContext config access E.quietEvents E.redactedDiagnostics + existing = Run.mutationContext bound access E.quietEvents E.redactedDiagnostics + adopted <- A.applyAdoption context extended adoption >>= right + assert "Adoption did not preserve attested step evidence" (case adopted of R.RunReport _ _ _ _ _ [R.MigrationSummary _ R.Adopted _ [R.StepSummary _ _ R.StepAccepted _ (Just (R.EvidenceSummary L.OperatorAttestation _)), R.StepSummary _ _ R.StepAccepted _ (Just (R.EvidenceSummary L.OperatorAttestation _))], R.MigrationSummary _ R.Pending _ _] [] (Just actual) -> actual == ledger; _ -> False) + assert "Adoption replayed old INSERT" . (== 1) =<< scalar server "SELECT toUInt32(count()) AS value FROM default.adoption_early" + received <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") received :: IO [Maybe T.Text] + assert "Adoption executed application SQL" (null (filter (== Just "application-sql") stages)) + let writes = length (filter (\stage -> stage `elem` map Just ["initialized", "adoption-started", "adoption-entry", "adoption-completed"]) stages) + assert "Adoption immutable batch incomplete" (writes == 4) + _ <- I.verify I.Observe bound (I.verifyOptions False) extended >>= right + _ <- A.applyAdoption existing extended adoption >>= right + _ <- Run.up existing (Run.upOptions audit Nothing) extended >>= right + assert "Next ordinary up did not execute only new migration" . (== 2) =<< scalar server "SELECT toUInt32(count()) AS value FROM default.adoption_early" + -- The original compiled prefix still supports an inspection-only identical + -- adoption after newer native history, without claiming that old plan current. + _ <- A.applyAdoption existing plan adoption >>= right + _ <- A.resumeAdoption existing plan adoption >>= right + after <- proxyEvents proxy >>= fromJson :: IO [Value] + afterStages <- mapM (field "stage") after :: IO [Maybe T.Text] + assert "Committed identical adoption added events" (length (filter (\stage -> stage `elem` map Just ["initialized", "adoption-started", "adoption-entry", "adoption-completed"]) afterStages) == writes) + assert "Committed adoption replayed native SQL" (length (filter (== Just "application-sql") afterStages) == 1) + _ <- I.verify I.Observe bound (I.verifyOptions True) extended >>= right + putStrLn "adoption early: reviewed two-step baseline; no SQL; immutable attestation; exactly one subsequent native migration; identical committed batch remains read-only after newer history" diff --git a/test/CLIIntegration.hs b/test/CLIIntegration.hs new file mode 100644 --- /dev/null +++ b/test/CLIIntegration.hs @@ -0,0 +1,258 @@ +module CLIIntegration (cliCases, exampleCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.CLI +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Concurrent (threadDelay) +import Control.Exception (AsyncException (ThreadKilled), SomeAsyncException, throwIO, try) +import Control.Monad (forM_, void, when) +import Data.Aeson (Value, object, (.=)) +import Data.Aeson qualified as JSON +import Data.ByteString qualified as BS +import Data.ByteString.Lazy qualified as Lazy +import Data.Maybe (isJust) +import Data.Text qualified as T +import Data.Text.Encoding qualified as Encoding +import RunnerIntegration (assert, field, fixture, fixtureWithLimits, fromJson, planFor, provider, right, scalar) +import System.Environment (getEnv, getEnvironment) +import System.Exit (ExitCode (..)) +import System.FilePath (()) +import System.IO (hClose) +import System.Process (CreateProcess (..), StdStream (..), createProcess, proc, readCreateProcessWithExitCode, terminateProcess, waitForProcess) +import System.Timeout (timeout) + +auditArgs :: TargetKey -> [String] +auditArgs key = ["--expect-target", T.unpack (targetKeyText key), "--actor", "CLI fixture", "--reason", "owned predecessor drained", "--evidence", "fixture/controlled-drain", "--quiescent"] + +dispatch :: CommandEnvironment -> MigrationPlan -> [String] -> IO CommandResult +dispatch environment plan args = case parseInvocation args of Right (Invocation _ command) -> handleCommand environment plan command; Left _ -> fail "CLI integration argument error" + +successful :: CommandResult -> IO CommandReport +successful (CommandResult _ (Right report)) = pure report +successful failure = fail (T.unpack (Encoding.decodeUtf8 (renderJson failure))) + +checkpoint :: CommandResult -> IO (L.LedgerPosition, L.AttemptId, Maybe R.StepAttempt) +checkpoint result = case result of + CommandResult _ (Right (StatusReport (R.InspectionReport (R.HistoryReport _ _ _ (Just position) _ _ _ _) (R.MigrationSummary _ _ (Just attempt) steps : _) _))) -> pure (position, attempt, case [selected | R.StepSummary _ _ R.StepUncertain (Just selected) _ <- steps] of selected : _ -> Just selected; _ -> Nothing) + _ -> fail "CLI checkpoint missing" + +repairArgs :: TargetKey -> String -> (L.LedgerPosition, L.AttemptId, Maybe R.StepAttempt) -> [String] +repairArgs key operation (position, attempt, selected) = ["repair", operation] ++ auditArgs key ++ ["--expect-position", T.unpack (L.ledgerPositionText position), "--attempt", T.unpack (L.attemptIdText attempt)] ++ if operation `elem` ["accept-step", "retry-step"] then case selected of Just (R.StepAttempt owner name query) -> ["--step-attempt", T.unpack (L.attemptIdText owner), "--step", T.unpack name, "--query-id", T.unpack (queryIdText query)]; _ -> [] else [] + +applicationCount :: FaultProxy -> IO Int +applicationCount proxy = do + rows <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") rows :: IO [Maybe T.Text] + pure (length (filter (== Just "application-sql") stages)) + +cliCases :: Server -> IO () +cliCases server = do + access <- provider + forM_ ["accept-step", "retry-step", "resume"] $ \operation -> withFaultProxy server $ \proxy -> do + let label = "cli_" <> T.replace "-" "_" (T.pack operation) + ledger <- newLedgerId + first <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + plan <- planFor label 7 + let key = configuredTargetKey first + environment config sink = commandEnvironment (Just config) (Just access) sink E.redactedDiagnostics AdministrativeCommands + clean = environment bound E.quietEvents + forM_ [["status"], ["history"], ["verify"]] $ \args -> void (dispatch (environment first E.quietEvents) plan args >>= successful) + pending <- dispatch (environment first E.quietEvents) plan ["verify", "--require-current"] + assert "CLI require-current allowed pending" (case pending of CommandResult _ (Left (InspectionFailed (R.InspectionFailure R.VerificationNotCurrent _ _))) -> True; _ -> False) + noInit <- dispatch clean plan (["up"] ++ auditArgs key) + assert "CLI implicitly initialized" (case noInit of CommandResult _ (Left (RunFailed (R.RunFailure R.LedgerRequired _ _))) -> True; _ -> False) + let stop (E.RunEvent phase _ _ _ step query _) = + when (step == Just "insert") $ + if operation == "resume" + then when (phase == E.BeforeStepStart) (fail "stop before pending step SQL_SECRET_MARKER") + else when (phase == E.BeforeSql) $ case query of Just identity -> setFaultForQuery proxy (queryIdText identity) DropResponse; _ -> fail "Missing CLI query" + partial <- dispatch (environment first stop) plan (["up"] ++ auditArgs key ++ ["--initialize-ledger", T.unpack (ledgerIdText ledger)]) + let bytes = renderJson partial + assert "CLI failed to preserve partial failure" (case partial of CommandResult _ (Left (RunFailed (R.RunFailure _ (R.RunReport _ _ _ _ _ [R.MigrationSummary _ R.InProgress _ (R.StepSummary _ _ R.StepObservedSuccess _ _ : _)] _ _) _))) -> True; _ -> False) + assert "CLI result leaked SQL" (not (BS.isInfixOf "SQL_SECRET_MARKER" bytes) && not (BS.isInfixOf "test-password" bytes)) + _ <- right (JSON.eitherDecodeStrict' bytes :: Either String Value) + setFault proxy Pass + before <- dispatch clean plan ["status", "--guarded"] >>= checkpoint + recovered <- dispatch clean plan (repairArgs key operation before) >>= successful + stale <- dispatch clean plan (repairArgs key operation before) + assert "CLI stale repair accepted" (case stale of CommandResult _ (Left (RunFailed (R.RunFailure (R.StalePosition _ _) _ _))) -> True; _ -> False) + when (operation /= "resume") $ do + after <- dispatch clean plan ["status", "--guarded"] >>= checkpoint + void (dispatch clean plan (repairArgs key "finalize" after) >>= successful) + when (operation == "accept-step") $ assert "CLI accepted result lost weaker evidence" ("operator-attestation" `BS.isInfixOf` renderJson (CommandResult "repair.accept-step" (Right recovered))) + void (dispatch clean plan ["verify", "--require-current"] >>= successful) + void (dispatch clean plan (["up"] ++ auditArgs key) >>= successful) + count <- scalar server ("SELECT toUInt32(count()) AS value FROM default." <> label) + assert "CLI replay/repair effects incorrect" (count == if operation == "retry-step" then 2 else 1) + assert "CLI request count incorrect" . (== if operation == "retry-step" then 3 else 2) =<< applicationCount proxy + withFaultProxy server $ \proxy -> do + ledger <- newLedgerId + first <- fixture server "cli_adoption" (proxyEndpoint proxy) FirstUse + bound <- fixture server "cli_adoption" (proxyEndpoint proxy) (ExistingLedger ledger) + plan <- planFor "cli_adoption" 7 + -- Independent synthetic predecessor effect; this is not library execution. + void (queryServer server "CREATE TABLE default.cli_adoption (id UInt8, secret String) ENGINE=MergeTree ORDER BY id") + void (queryServer server "INSERT INTO default.cli_adoption VALUES (7,'SQL_SECRET_MARKER')") + let key = configuredTargetKey first + file = serverLogDirectory server "cli-adoption.json" + digest = case planReport plan of [PlanComponent _ _ [PlanMigration _ _ checksum _]] -> checksum; _ -> error "Unexpected adoption fixture plan" + input = object ["schemaVersion" .= (1 :: Int), "source" .= ("synthetic-cli-predecessor" :: T.Text), "snapshotReference" .= ("fixture/retained-snapshot" :: T.Text), "rows" .= [object ["id" .= ("001.sql" :: T.Text), "appliedAt" .= ("2020-01-01" :: T.Text)]], "mappings" .= [object ["row" .= ("001.sql" :: T.Text), "component" .= ("app" :: T.Text), "migration" .= ("one" :: T.Text), "checksum" .= digest, "evidence" .= object ["level" .= ("operator-attestation" :: T.Text), "references" .= (["fixture/reviewed-entire-prefix"] :: [T.Text])]]]] + environment config = commandEnvironment (Just config) (Just access) E.quietEvents E.redactedDiagnostics AdministrativeCommands + args operation expected = ["adopt", operation] ++ auditArgs key ++ expected ++ ["--initialize-ledger", T.unpack (ledgerIdText ledger), "--request", file] + BS.writeFile file (Lazy.toStrict (JSON.encode input)) + void (dispatch (environment first) plan (args "plan" ["--expect-absent"]) >>= successful) + missing <- dispatch (environment first) plan (args "resume" ["--expect-absent"]) + assert "CLI resumed absent adoption batch" (case missing of CommandResult _ (Left (RunFailed _)) -> True; _ -> False) + setLedgerFault proxy AdoptionEntry DropResponse + partial <- dispatch (environment first) plan (args "apply" ["--expect-absent"]) + assert "CLI adoption lost append uncertainty" (case partial of CommandResult _ (Left (RunFailed (R.RunFailure (R.LedgerRequestFailed L.UncertainSubmission _ (Just _)) _ _))) -> True; _ -> False) + setFault proxy Pass + stale <- dispatch (environment bound) plan (args "resume" ["--expect-absent"]) + assert "CLI stale adoption plan accepted" (case stale of CommandResult _ (Left (AdoptionRejected _ _)) -> True; _ -> False) + snapshot <- dispatch (environment bound) plan ["status", "--guarded"] >>= successful + position <- case snapshot of StatusReport (R.InspectionReport (R.HistoryReport _ _ _ (Just value) _ _ _ _) _ _) -> pure value; _ -> fail "Missing adoption checkpoint" + void (dispatch (environment bound) plan (args "resume" ["--expect-position", T.unpack (L.ledgerPositionText position)]) >>= successful) + void (dispatch (environment bound) plan (args "apply" ["--expect-absent"]) >>= successful) + assert "CLI adoption executed SQL" . (== 0) =<< applicationCount proxy + withFaultProxy server $ \proxy -> do + -- A delayed ledger acknowledgement has a separate receipt and no automatic + -- retry. Release the controlled gate and establish complete drain ourselves. + limits <- right (httpLimits 1000000 1048576) + first <- fixtureWithLimits limits server "cli_delayed" (proxyEndpoint proxy) FirstUse + ledger <- newLedgerId + plan <- planFor "cli_delayed" 7 + let key = configuredTargetKey first + environment sink = commandEnvironment (Just first) (Just access) sink E.redactedDiagnostics AdministrativeCommands + setLedgerStepFault proxy StepSuccess "create" HoldBeforeForward + delayed <- dispatch (environment E.quietEvents) plan (["up"] ++ auditArgs key ++ ["--initialize-ledger", T.unpack (ledgerIdText ledger)]) + assert "CLI delayed append receipt missing" (case delayed of CommandResult _ (Left (RunFailed (R.RunFailure (R.LedgerRequestFailed L.UncertainSubmission _ (Just _)) _ _))) -> True; _ -> False) + assert "CLI delayed JSON lost receipt" ("chm-receipt1:" `BS.isInfixOf` renderJson delayed) + releaseFault proxy + drained <- timeout 5000000 (let wait = proxyConnections proxy >>= \n -> if n == 0 then pure () else threadDelay 10000 >> wait in wait) + assert "CLI delayed request did not drain" (isJust drained) + withFaultProxy server $ \proxy -> do + first <- fixture server "cli_cancel" (proxyEndpoint proxy) FirstUse + ledger <- newLedgerId + plan <- planFor "cli_cancel" 7 + let cancel (E.RunEvent phase _ _ _ _ _ _) = when (phase == E.MigrationStarted) (throwIO ThreadKilled) + environment = commandEnvironment (Just first) (Just access) cancel E.redactedDiagnostics AdministrativeCommands + result <- try @SomeAsyncException (dispatch environment plan (["up"] ++ auditArgs (configuredTargetKey first) ++ ["--initialize-ledger", T.unpack (ledgerIdText ledger)])) + assert "CLI converted cancellation into result" (case result of Left _ -> True; _ -> False) + putStrLn "CLI: native inspection/current distinction, explicit init, all four repair handlers, stale refusal, partial JSON, delayed receipt and async propagation" + +-- This invokes the application executable, capturing stdout and stderr separately. +-- It never talks to a pre-existing server or changes an external repository. +exampleCases :: Server -> IO () +exampleCases server = do + executable <- getEnv "CLICKHOUSE_MIGRATE_EXAMPLE" + inherited <- filter (not . ("CHM_" `T.isPrefixOf`) . T.pack . fst) <$> getEnvironment + let offline = (proc executable ["plan", "--json"]) {env = Just inherited} + (offlineExit, offlineOut, offlineErr) <- readCreateProcessWithExitCode offline "" + assert "Offline example failed without config" (offlineExit == ExitSuccess && null offlineErr) + offlineValue <- right (JSON.eitherDecodeStrict' (Encoding.encodeUtf8 (T.pack offlineOut)) :: Either String Value) + result <- field "result" offlineValue + components <- field "components" result :: IO [Value] + names <- mapM (field "name") components :: IO [T.Text] + assert "Independent components/order missing" (names == ["catalog", "analytics"]) + withFaultProxy server $ \proxy -> do + ledger <- newLedgerId + first <- fixture server "example" (proxyEndpoint proxy) FirstUse + let key = configuredTargetKey first + identityRows <- queryServer server "SELECT toString(serverUUID()) AS identity" >>= fromJson :: IO [Value] + identity <- case identityRows of [row] -> field "identity" row; _ -> fail "Missing example server identity" + let configuration = [("CHM_TARGET", T.unpack (targetKeyText key)), ("CHM_ENDPOINT", T.unpack (proxyEndpoint proxy)), ("CHM_SERVER_ID", identity), ("CHM_LEDGER_DATABASE", "example_history"), ("CHM_PASSWORD", "test-password"), ("CHM_LOCK_ROOT", serverLogDirectory server "example-locks")] + runWith configured binding args = do + (exit, out, err) <- readCreateProcessWithExitCode ((proc executable args) {env = Just (configured ++ binding ++ inherited)}) "" + value <- right (JSON.eitherDecodeStrict' (Encoding.encodeUtf8 (T.pack out)) :: Either String Value) + assert "Example stdout/stderr leaked sensitive values" (not ("test-password" `T.isInfixOf` T.pack (out ++ err)) && not ("CREATE TABLE" `T.isInfixOf` T.pack (out ++ err))) + pure (exit, value, err) + run = runWith configuration + bound = [("CHM_LEDGER_ID", T.unpack (ledgerIdText ledger))] + upArgs = ["up", "--json"] ++ auditArgs key + (initialExit, _, progress) <- run [] (["--progress"] ++ upArgs ++ ["--initialize-ledger", T.unpack (ledgerIdText ledger)]) + assert "Fresh example up/stream separation failed" (initialExit == ExitSuccess && "progress:" `T.isInfixOf` T.pack progress) + before <- applicationCount proxy + (secondExit, _, _) <- run bound upArgs + assert "Second example up failed" (secondExit == ExitSuccess) + assert "Second example up replayed SQL" . (== before) =<< applicationCount proxy + forM_ [["status", "--json"], ["history", "--json"], ["verify", "--require-current", "--json"]] $ \args -> do (exit, _, err) <- run bound args; assert "Example inspection failed" (exit == ExitSuccess && null err) + -- The same executable recovers an interrupted durable completion and then + -- adopts the retained complete two-component prefix into a separate ledger. + void (queryServer server "DROP TABLE default.chm_example_events SYNC") + void (queryServer server "DROP TABLE default.chm_example_catalog SYNC") + repairLedger <- newLedgerId + let repairKey = rightKey "example/repair" + configured label = map (\(name, value) -> if name == "CHM_LEDGER_DATABASE" then (name, "example_" ++ label ++ "_history") else if name == "CHM_TARGET" then (name, "example/" ++ label) else (name, value)) configuration + repairBound = [("CHM_LEDGER_ID", T.unpack (ledgerIdText repairLedger))] + setLedgerStepFault proxy StepSuccess "seed" DropResponse + (partialExit, partial, _) <- runWith (configured "repair") [] (["up", "--json"] ++ auditArgs repairKey ++ ["--initialize-ledger", T.unpack (ledgerIdText repairLedger)]) + assert "Example failed completion looked successful" (partialExit == ExitFailure 1) + failure <- field "error" partial + primary <- field "primary" failure + receipt <- field "pendingAppend" primary :: IO (Maybe T.Text) + assert "Example lost exact completion receipt" (isJust receipt) + setFault proxy Pass + (_, observed, _) <- runWith (configured "repair") repairBound ["status", "--json"] + observedResult <- field "result" observed >>= field "report" + observation <- field "observation" observedResult + position <- field "position" observation :: IO T.Text + summaries <- field "migrations" observedResult :: IO [Value] + active <- case summaries of firstSummary : _ -> field "attempt" firstSummary; _ -> fail "Missing example repair attempt" + let finalArgs = ["repair", "finalize", "--json"] ++ auditArgs repairKey ++ ["--expect-position", T.unpack position, "--attempt", active] ++ maybe [] (\token -> ["--pending-append", T.unpack token]) receipt + beforeFinal <- applicationCount proxy + (finalExit, _, _) <- runWith (configured "repair") repairBound finalArgs + assert "Example finalize failed" (finalExit == ExitSuccess) + assert "Example finalize executed SQL" . (== beforeFinal) =<< applicationCount proxy + (staleExit, _, _) <- runWith (configured "repair") repairBound finalArgs + assert "Example stale finalize accepted" (staleExit == ExitFailure 1) + (tailExit, _, _) <- runWith (configured "repair") repairBound (["up", "--json"] ++ auditArgs repairKey) + assert "Example failed to complete appended component" (tailExit == ExitSuccess) + adoptLedger <- newLedgerId + entries <- + mapM + ( \component -> do + name <- field "name" component :: IO T.Text + migrations <- field "migrations" component :: IO [Value] + digest <- case migrations of + [migration] -> field "checksum" migration + _ -> fail "Example migration count" + pure (name, digest :: T.Text) + ) + components + let adoptKey = rightKey "example/adopt" + file = serverLogDirectory server "example-adoption.json" + input = object ["schemaVersion" .= (1 :: Int), "source" .= ("owned-example-predecessor" :: T.Text), "snapshotReference" .= ("fixture/example-completed-prefix" :: T.Text), "rows" .= [object ["id" .= name, "appliedAt" .= ("2020-01-01" :: T.Text)] | (name, _) <- entries], "mappings" .= [object ["row" .= name, "component" .= name, "migration" .= ("0001" :: T.Text), "checksum" .= digest, "evidence" .= object ["level" .= ("operator-attestation" :: T.Text), "references" .= (["fixture/complete-prefix-review"] :: [T.Text])]] | (name, digest) <- entries]] + adoptArgs operation = ["adopt", operation, "--json"] ++ auditArgs adoptKey ++ ["--expect-absent", "--initialize-ledger", T.unpack (ledgerIdText adoptLedger), "--request", file] + BS.writeFile file (Lazy.toStrict (JSON.encode input)) + beforeAdopt <- applicationCount proxy + forM_ ["plan", "apply", "resume"] $ \operation -> do + (adoptExit, _, _) <- runWith (configured "adopt") (if operation == "resume" then [("CHM_LEDGER_ID", T.unpack (ledgerIdText adoptLedger))] else []) (adoptArgs operation) + assert "Example adoption command failed" (adoptExit == ExitSuccess) + assert "Example adoption executed SQL" . (== beforeAdopt) =<< applicationCount proxy + -- Deliberate process cancellation while a ledger INSERT is held. Client + -- cleanup cannot itself establish server drain; the fixture releases it. + let cancelConfig = map (\(name, value) -> if name == "CHM_LEDGER_DATABASE" then (name, "example_cancel_history") else if name == "CHM_TARGET" then (name, "example/cancel") else (name, value)) configuration + canceledLedger <- newLedgerId + baselineRows <- proxyEvents proxy >>= fromJson :: IO [Value] + baselineStages <- mapM (field "stage") baselineRows :: IO [Maybe T.Text] + let baseline = length (filter (== Just "migration-started") baselineStages) + setLedgerFault proxy MigrationStart HoldBeforeForward + (Nothing, Just output, Just errors, process) <- createProcess ((proc executable (["up", "--json"] ++ auditArgs (rightKey "example/cancel") ++ ["--initialize-ledger", T.unpack (ledgerIdText canceledLedger)])) {env = Just (cancelConfig ++ inherited), std_out = CreatePipe, std_err = CreatePipe}) + started <- timeout 10000000 (let wait = do rows <- proxyEvents proxy >>= fromJson :: IO [Value]; stages <- mapM (field "stage") rows :: IO [Maybe T.Text]; if length (filter (== Just "migration-started") stages) > baseline then pure () else threadDelay 10000 >> wait in wait) + assert "Canceled subprocess never reached controlled request" (isJust started) + terminateProcess process + ended <- timeout 5000000 (waitForProcess process) + hClose output + hClose errors + assert "Canceled subprocess did not exit" (isJust ended) + releaseFault proxy + drained <- timeout 5000000 (let wait = proxyConnections proxy >>= \n -> if n == 0 then pure () else threadDelay 10000 >> wait in wait) + assert "Canceled subprocess left client/proxy request open" (isJust drained) + putStrLn "example: offline composed plan; application-owned config; fresh/no-op up; inspection; separate stdout/progress; killed subprocess and controlled drain" + where + rightKey = either (error . show) id . targetKey diff --git a/test/ConsumerFixtures.hs b/test/ConsumerFixtures.hs new file mode 100644 --- /dev/null +++ b/test/ConsumerFixtures.hs @@ -0,0 +1,183 @@ +{-# LANGUAGE TemplateHaskell #-} + +module ConsumerFixtures (consumerFixtureCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Adoption qualified as A +import ClickHouseMigrate.Embed (embedComponent) +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Monad (forM, forM_, void) +import Crypto.Hash.SHA256 qualified as SHA +import Data.Aeson (Value) +import Data.Aeson qualified as JSON +import Data.ByteString qualified as BS +import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE +import Data.Set qualified as Set +import Data.Text qualified as T +import Data.Text.Encoding qualified as Encoding +import Data.Text.IO qualified as TextIO +import Numeric (showHex) +import Paths_clickhouse_migrate_test_support (getDataFileName) +import RunnerIntegration (assert, field, fixture, fromJson, provider, right, scalar) +import System.FilePath (()) + +firstComponent :: MigrationComponent +firstComponent = $(embedComponent "test/fixtures/kansoku/manifest-first.json") + +capturedComponent :: MigrationComponent +capturedComponent = $(embedComponent "test/fixtures/kansoku/manifest.json") + +fixtureFile :: FilePath -> IO FilePath +fixtureFile name = getDataFileName ("test/fixtures/kansoku/" ++ name) + +readJson :: FilePath -> IO Value +readJson name = fixtureFile name >>= BS.readFile >>= either (const (fail "Invalid frozen consumer evidence")) pure . JSON.eitherDecodeStrict' + +hex :: BS.ByteString -> T.Text +hex = T.pack . concatMap (\byte -> let rendered = showHex byte "" in replicate (2 - length rendered) '0' ++ rendered) . BS.unpack . SHA.hash + +-- Verify frozen byte evidence independently from embedding before server work. +checkCapture :: IO [Value] +checkCapture = do + evidence <- readJson "provenance.json" + source <- field "sourceProject" evidence :: IO T.Text + revision <- field "sourceRevision" evidence :: IO T.Text + assert "Consumer capture lacks canonical immutable origin" (source == "mori://shinzui/kansoku" && revision == "138d9a316931bbf46781757a29ab6c7fdd90f870") + runner <- field "runner" evidence + runnerBytes <- field "fixturePath" runner >>= fixtureFile >>= BS.readFile + runnerHash <- field "sha256" runner + assert "Original runner hash changed" (hex runnerBytes == runnerHash) + files <- field "files" evidence :: IO [Value] + assert "Captured history incomplete" (length files == 25) + forM_ files $ \file -> do + original <- field "original" file >>= fixtureFile >>= BS.readFile + digest <- field "sha256" file + assert "Original source hash changed" (hex original == digest) + steps <- field "steps" file :: IO [Value] + chunks <- forM steps $ \step -> do + bytes <- field "file" step >>= fixtureFile >>= BS.readFile + expected <- field "sha256" step + start <- field "byteStart" step + end <- field "byteEndExclusive" step + assert "Reviewed source range changed" (hex bytes == expected && BS.take (end - start) (BS.drop start original) == bytes) + pure bytes + assert "Source ranges lose or duplicate bytes" (BS.concat chunks == original) + adaptations <- field "nativeAdaptations" evidence :: IO [Value] + assert "Unreviewed native adaptation count" (length adaptations == 3) + forM_ adaptations $ \adaptation -> do + sourceStep <- field "sourceStep" adaptation + original <- field "file" sourceStep >>= fixtureFile >>= BS.readFile + expected <- field "sourceStepSha256" adaptation + assert "Assertion source changed" (hex original == expected) + native <- field "nativeSteps" adaptation :: IO [Value] + forM_ native $ \step -> do + bytes <- field "file" step >>= fixtureFile >>= BS.readFile + digest <- field "sha256" step + assert "Reviewed native step changed" (hex bytes == digest) + pure files + +schemaObservation :: Server -> IO Value +schemaObservation server = queryServer server "SELECT table,name,type,default_kind,default_expression FROM system.columns WHERE database='analytics' ORDER BY table,position" + +applicationCount :: FaultProxy -> IO Int +applicationCount proxy = do + rows <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") rows :: IO [Maybe T.Text] + pure (length (filter (== Just "application-sql") stages)) + +consumerFixtureCases :: Server -> IO () +consumerFixtureCases server = do + files <- checkCapture + access <- provider + audit <- right (L.auditContext "consumer fixture reviewer" "explicit reviewed native baseline and assertion translations" ["test/fixtures/kansoku/provenance.json: immutable source/ranges/transformations", "owned isolated server; all predecessor requests drained"]) + let runFresh label plan = withFaultProxy server $ \proxy -> do + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + result <- Run.up (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + report <- case result of + Right value -> pure value + Left failure@(R.RunFailure primary _ _) -> do + TextIO.writeFile (serverLogDirectory server T.unpack label ++ ".trace") (T.pack (show failure)) + fail ("consumer native replay: " ++ show primary) + assert "Fresh replay unexpectedly adopted evidence" (case report of R.RunReport _ _ _ _ _ migrations _ _ -> all (\(R.MigrationSummary _ state _ steps) -> state == R.Applied && all (\(R.StepSummary _ _ step _ _) -> step == R.StepObservedSuccess) steps) migrations) + pure =<< applicationCount proxy + first <- right (migrationPlan (firstComponent :| [])) + count <- runFresh "consumer_first" first + assert "First real multi-statement file was not one complete native migration" (count == 2) + assert "First source file created no real application object" . (== 1) =<< scalar server "SELECT toUInt32(count()) AS value FROM system.tables WHERE database='analytics' AND name='events_raw'" + void (queryServer server "DROP DATABASE analytics SYNC") + putStrLn "consumer first mapping: real immutable 0001 source file, two explicit steps, actual application table" + complete <- right (migrationPlan (capturedComponent :| [])) + fullCount <- runFresh "consumer_fresh" complete + assert "Fresh native statement count does not match reviewed manifest" (fullCount == 257) + freshSchema <- schemaObservation server + assert "Native assertion cleanup left application scratch tables" . (== 0) =<< scalar server "SELECT toUInt32(count()) AS value FROM system.tables WHERE database='analytics' AND startsWith(name,'__chm_legacy_assert_')" + void (queryServer server "DROP DATABASE analytics SYNC") + putStrLn "consumer fresh: all 25 native migrations / 257 reviewed steps applied; assertions checked; scratch tables cleaned" + -- Independently replay the original source ranges, then store synthetic legacy + -- filename/time history as the predecessor did. No native ledger exists yet. + void (queryServer server "CREATE DATABASE analytics") + void (queryServer server "CREATE TABLE analytics.schema_migrations (filename String, applied_at DateTime64(3,'UTC')) ENGINE=MergeTree ORDER BY filename") + rows <- readJson "legacy-history.json" >>= fromJson :: IO [Value] + assert "Legacy evidence row count changed" (length rows == length files) + forM_ (zip files rows) $ \(file, row) -> do + steps <- field "steps" file :: IO [Value] + forM_ steps $ \step -> field "file" step >>= fixtureFile >>= BS.readFile >>= either (const (fail "Invalid captured UTF-8")) pure . Encoding.decodeUtf8' >>= void . queryServer server + filename <- field "filename" row :: IO T.Text + applied <- field "appliedAt" row :: IO T.Text + -- Values come from the checked-in synthetic fixture, not external input. + assert "Unexpected synthetic evidence spelling" (not (T.any (== '\'') filename) && not (T.any (== '\'') applied)) + void (queryServer server ("INSERT INTO analytics.schema_migrations VALUES ('" <> filename <> "', parseDateTime64BestEffort('" <> applied <> "'))")) + legacySchema <- queryServer server "SELECT table,name,type,default_kind,default_expression FROM system.columns WHERE database='analytics' AND table!='schema_migrations' ORDER BY table,position" + assert "Fresh native and original legacy schema observations differ" (freshSchema == legacySchema) + observedRows <- queryServer server "SELECT filename,toString(applied_at) AS appliedAt FROM analytics.schema_migrations ORDER BY filename" >>= fromJson :: IO [Value] + assert "Legacy reader lost filename/time rows" (length observedRows == 25) + provenanceBytes <- fixtureFile "provenance.json" >>= BS.readFile + let evidenceReference = "test/fixtures/kansoku/provenance.json#sha256:" <> hex provenanceBytes + withFaultProxy server $ \proxy -> do + ledger <- newLedgerId + let label = "consumer_adopt" + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + quiet <- right (L.attestQuiescence key audit) + legacy <- forM observedRows $ \row -> do + filename <- field "filename" row + applied <- field "appliedAt" row + right (A.legacyRow filename applied Nothing) + let definitions = NE.toList (componentMigrations (NE.head (inspectPlan complete))) + mappings <- forM (zip observedRows definitions) $ \(row, migration) -> do + filename <- field "filename" row + right (A.attestedMapping filename "kansoku" (migrationNameText (migrationIdName (migrationId migration))) (migrationChecksumHex (migrationChecksum migration)) ["test/fixtures/kansoku/provenance.json: complete file mapping and author obligations"]) + observation <- I.status I.Observe config complete >>= right + legacyRows <- maybe (fail "Empty legacy evidence") pure (NE.nonEmpty legacy) + mappedRows <- maybe (fail "Empty legacy mapping") pure (NE.nonEmpty mappings) + request <- right (A.adoptionRequest key (L.ExpectAbsent key) (Just (L.authorizeInitialization key ledger audit)) quiet audit "mori://shinzui/kansoku" evidenceReference legacyRows mappedRows) + adoption <- right (A.planAdoption complete observation request) + _ <- A.applyAdoption (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) complete adoption >>= right + assert "Consumer adoption executed application SQL" . (== 0) =<< applicationCount proxy + _ <- I.verify I.Observe bound (I.verifyOptions True) complete >>= right + let originalMigrations = NE.toList (componentMigrations (NE.head (inspectPlan complete))) + -- Rebuild through public smart constructors: embedding remains the source of + -- exact original SQL bytes and names, with one explicit appended migration. + authored <- forM originalMigrations $ \migration -> do + steps <- forM (NE.toList (migrationSteps migration)) $ \step -> right (sqlStep (stepNameText (stepViewName step)) (stepSql step)) + nonempty <- maybe (fail "Empty captured migration") pure (NE.nonEmpty steps) + right (sqlMigration (migrationNameText (migrationIdName (migrationId migration))) nonempty) + nextStep <- right (sqlStep "add-probe" "ALTER TABLE analytics.events_raw ADD COLUMN clickhouse_migrate_probe UInt8") + next <- right (sqlMigration "0026_native_probe" (nextStep :| [])) + nonempty <- maybe (fail "Empty captured component") pure (NE.nonEmpty (authored ++ [next])) + extended <- right (migrationComponent "kansoku" Set.empty nonempty) >>= \component -> right (migrationPlan (component :| [])) + _ <- Run.up (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit Nothing) extended >>= right + assert "Consumer cutover replayed accepted legacy work" . (== 1) =<< applicationCount proxy + _ <- I.verify I.Observe bound (I.verifyOptions True) extended >>= right + assert "Subsequent native migration effect missing" . (== 1) =<< scalar server "SELECT toUInt32(count()) AS value FROM system.columns WHERE database='analytics' AND table='events_raw' AND name='clickhouse_migrate_probe'" + putStrLn "consumer existing history: original 253 statements replayed independently; schema observations match; 25 filename-only rows adopted without SQL; one subsequent native migration observed" diff --git a/test/FailureBoundaries.hs b/test/FailureBoundaries.hs new file mode 100644 --- /dev/null +++ b/test/FailureBoundaries.hs @@ -0,0 +1,203 @@ +module FailureBoundaries (failureBoundaryCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.ExclusiveAccess qualified as Access +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Exception (finally) +import Control.Monad (forM_, when) +import Data.Aeson (Value) +import Data.IORef (modifyIORef', newIORef, readIORef, writeIORef) +import Data.Text qualified as T +import Data.Text.IO qualified as TIO +import RunnerIntegration (assert, field, fixture, fromJson, planFor, provider, right) +import System.FilePath (()) + +-- Every public observer boundary, separately for both statements. Numbers are +-- independently specified durable-event and application-request expectations. +callbacks :: [(E.EventPhase, Maybe T.Text, Int, Int)] +callbacks = + [ (E.RunBeginning, Nothing, 0, 0), + (E.BeforeMigrationStart, Nothing, 1, 0), + (E.MigrationStarted, Nothing, 2, 0), + (E.BeforeStepStart, Just "create", 2, 0), + (E.StepStarted, Just "create", 3, 0), + (E.BeforeSql, Just "create", 3, 0), + (E.SqlObserved, Just "create", 3, 1), + (E.BeforeStepSuccess, Just "create", 3, 1), + (E.StepSucceeded, Just "create", 4, 1), + (E.BeforeStepStart, Just "insert", 4, 1), + (E.StepStarted, Just "insert", 5, 1), + (E.BeforeSql, Just "insert", 5, 1), + (E.SqlObserved, Just "insert", 5, 2), + (E.BeforeStepSuccess, Just "insert", 5, 2), + (E.StepSucceeded, Just "insert", 6, 2), + (E.BeforeMigrationComplete, Nothing, 6, 2), + (E.MigrationCompleted, Nothing, 7, 2), + (E.RunCompleted, Nothing, 7, 2) + ] + +-- Every ordinary dependent append, including bootstrap and both step positions. +acknowledgements :: [(LedgerStage, Maybe T.Text, Int, Int)] +acknowledgements = + [ (Initialization, Nothing, 0, 0), + (MigrationStart, Nothing, 1, 0), + (StepStart, Just "create", 2, 0), + (StepSuccess, Just "create", 3, 1), + (StepStart, Just "insert", 4, 1), + (StepSuccess, Just "insert", 5, 2), + (MigrationCompletion, Nothing, 6, 2) + ] + +historyEvents :: R.RunReport -> [R.HistoryEvent] +historyEvents (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ _ _ events _ _) _ _ _) = events + +applicationCount :: FaultProxy -> IO Int +applicationCount proxy = do + rows <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") rows :: IO [Maybe T.Text] + pure (length (filter (== Just "application-sql") stages)) + +failureBoundaryCases :: Server -> IO () +failureBoundaryCases server = do + access <- provider + audit <- right (L.auditContext "boundary fixture" "explicit isolated bootstrap" ["parent owns fixture and drains all requests before successor mutation"]) + forM_ (zip [0 :: Int ..] callbacks) $ \(number, (phase, selected, expectedEvents, expectedSql)) -> withFaultProxy server $ \proxy -> do + let label = "callback_" <> T.pack (show number) + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + trace <- newIORef [] + saved <- newIORef "No result" + let sink event@(E.RunEvent actual _ _ _ step _ _) = do + modifyIORef' trace (++ [event]) + when (actual == phase && step == selected) (fail "callback SQL_SECRET_MARKER") + save = do + events <- readIORef trace + requests <- proxyEvents proxy + report <- readIORef saved + TIO.writeFile (serverLogDirectory server T.unpack label ++ ".trace") (T.pack (show events) <> "\n" <> T.pack (show requests) <> "\n" <> report) + ( do + result <- Run.up (Run.mutationContext config access sink E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + writeIORef saved (T.pack (show result)) + report <- case result of Left (R.RunFailure (R.CallbackFailed _) report []) -> pure report; _ -> fail ("Wrong callback result: " ++ show result) + assert "Callback failure lost acknowledged state" (length (historyEvents report) == expectedEvents) + assert "Observer exception text leaked" (not ("SQL_SECRET_MARKER" `T.isInfixOf` T.pack (show result))) + assert "Callback allowed later application SQL" . (== expectedSql) =<< applicationCount proxy + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + observed <- I.history (I.Guarded access) bound + when (expectedEvents > 0) $ do + snapshot <- right observed + writeIORef saved (T.pack (show result) <> "\n" <> T.pack (show snapshot)) + assert "Durable history diverged from acknowledged callback checkpoint" (case snapshot of R.HistoryReport _ _ _ _ _ actual _ _ -> length actual == expectedEvents) + again <- Run.up (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit Nothing) plan + if expectedEvents >= 2 && expectedEvents < 7 + then assert "Ordinary up used callback interruption as resume permission" (case again of Left (R.RunFailure (R.LedgerFailed L.UnresolvedHistory) _ _) -> True; _ -> False) + else + if expectedEvents == 0 + then assert "Missing ledger was implicitly initialized" (case again of Left _ -> True; _ -> False) + else do + _ <- right again + pure () + let after = if expectedEvents == 1 then 2 else expectedSql + assert "Restart replayed interrupted/completed SQL" . (== after) =<< applicationCount proxy + putStrLn ("callback boundary " ++ show phase ++ "/" ++ show selected ++ ": ack=" ++ show expectedEvents ++ ", sql=" ++ show expectedSql) + ) + `finally` save + forM_ (zip [0 :: Int ..] acknowledgements) $ \(number, (stage, step, expectedEvents, expectedSql)) -> withFaultProxy server $ \proxy -> do + let label = "append_loss_" <> T.pack (show number) + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + maybe (setLedgerFault proxy stage DropResponse) (\name -> setLedgerStepFault proxy stage name DropResponse) step + result <- Run.up (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + requests <- proxyEvents proxy + TIO.writeFile (serverLogDirectory server T.unpack label ++ ".trace") (T.pack (show result) <> "\n" <> T.pack (show requests)) + report <- case result of Left (R.RunFailure (R.LedgerRequestFailed L.UncertainSubmission _ (Just _)) report []) -> pure report; _ -> fail ("Missing uncertain append receipt " ++ show result) + assert "Unknown ledger append advanced acknowledged report" (length (historyEvents report) == expectedEvents) + assert "SQL submitted before dependent acknowledgement" . (== expectedSql) =<< applicationCount proxy + setFault proxy Pass + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + observed <- I.history (I.Guarded access) bound >>= right + assert "Dropped ledger response did not persist exactly one event" (case observed of R.HistoryReport _ _ _ _ _ actual _ _ -> length actual == expectedEvents + 1) + again <- Run.up (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit Nothing) plan + if stage == Initialization + then do _ <- right again; pure () + else + if stage == MigrationCompletion + then do _ <- right again; pure () + else assert "Restart replayed after ledger acknowledgement loss" (case again of Left (R.RunFailure (R.LedgerFailed L.UnresolvedHistory) _ _) -> True; _ -> False) + assert "Acknowledgement loss caused application replay" . (== if stage == Initialization then 2 else expectedSql) =<< applicationCount proxy + putStrLn ("ledger ack loss " ++ show stage ++ "/" ++ show step ++ ": ack=" ++ show expectedEvents ++ ", sql=" ++ show expectedSql) + -- An observed error may follow an effect. An uncertain failure-record INSERT + -- must remain secondary, and no observer exception can replace the SQL error. + forM_ [False, True] $ \loseRecording -> withFaultProxy server $ \proxy -> do + let label = if loseRecording then "failure_record_loss" else "failure_observer" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + let sink (E.RunEvent phase _ _ _ step query _) = do + when (phase == E.BeforeSql && step == Just "insert") $ do + maybe (fail "Missing query") (\q -> setFaultForQuery proxy (queryIdText q) LateError) query + when loseRecording (addLedgerFault proxy FailureRecording DropResponse) + when (phase == E.SqlObserved && step == Just "insert") (fail "observer SQL_SECRET_MARKER") + result <- Run.up (Run.mutationContext config access sink E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + case result of + Left (R.RunFailure (R.SqlFailed _ _ L.ObservedFailure _) report secondary) -> do + assert "Primary SQL error replaced or checkpoint lost" (length (historyEvents report) == if loseRecording then 5 else 6) + assert "Secondary observer error lost" (any (\case R.ObserverError _ -> True; _ -> False) secondary) + assert "Secondary recording uncertainty lost" (any (\case R.RecordingError (R.LedgerRequestFailed L.UncertainSubmission _ (Just _)) -> True; _ -> False) secondary == loseRecording) + _ -> fail ("Wrong composite failure " ++ show result) + assert "Failure recording caused SQL replay" . (== 2) =<< applicationCount proxy + putStrLn ("SQL observed failure + observer failure + uncertain recording=" ++ show loseRecording ++ ": primary and checkpoint preserved") + -- Loss after observed SQL prevents a checkpoint write; release faults preserve + -- successful work as a cleanup-only failure. Neither is replay permission. + forM_ [False, True] $ \loseLease -> withFaultProxy server $ \proxy -> do + let label = if loseLease then "lease_after_sql" else "release_after_complete" + alive <- newIORef True + custom <- right (Access.externallySerialized "owned-parent-with-injected-fault" "all controlled requests drained before next job" 1000000 (const (pure ())) (const (when (not loseLease) (fail "release secret"))) (const (readIORef alive))) + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + let sink (E.RunEvent phase _ _ _ step _ _) = when (loseLease && phase == E.SqlObserved && step == Just "insert") (writeIORef alive False) + result <- Run.up (Run.mutationContext config custom sink E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + case result of + Left (R.RunFailure primary report secondary) -> + if loseLease + then do + assert "Lease loss allowed success append" (length (historyEvents report) == 5) + assert "Wrong lease failure" (case primary of R.AccessFailed Access.AccessLost -> True; _ -> False) + else do + assert "Cleanup fault erased completed work" (length (historyEvents report) == 7) + assert "Release failure did not remain separate" (primary == R.CleanupFailed && not (null secondary)) + _ -> fail ("Injected access fault disappeared " ++ show result) + assert "Access fault replayed SQL" . (== 2) =<< applicationCount proxy + putStrLn ("access boundary: lease loss=" ++ show loseLease ++ "; checkpoint and cleanup evidence preserved") + forM_ [False, True] $ \failDiagnostic -> withFaultProxy server $ \proxy -> do + let label = if failDiagnostic then "diagnostic_failure" else "diagnostic_optin" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + delivered <- newIORef [] + let diagnostic (E.SqlDiagnostic reference query sql) = do + modifyIORef' delivered (++ [(reference, query, sql)]) + when failDiagnostic (fail "diagnostic SQL_SECRET_MARKER") + result <- Run.up (Run.mutationContext config access E.quietEvents (E.sqlDiagnostics diagnostic)) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + if failDiagnostic + then assert "Diagnostic exception failed to stop before SQL" (case result of Left (R.RunFailure (R.CallbackFailed R.SqlDiagnosticCallback) report []) -> length (historyEvents report) == 3; _ -> False) + else do _ <- right result; pure () + assert "Opt-in diagnostic exception leaked into report" (not ("SQL_SECRET_MARKER" `T.isInfixOf` T.pack (show result))) + observed <- readIORef delivered + assert "Explicit diagnostic sink did not receive exact SQL" (length observed == if failDiagnostic then 1 else 2) + assert "Diagnostic failure allowed SQL submission" . (== if failDiagnostic then 0 else 2) =<< applicationCount proxy + putStrLn ("explicit SQL diagnostics: throwing sink=" ++ show failDiagnostic ++ "; safe default report and exact opt-in delivery") diff --git a/test/LedgerIntegration.hs b/test/LedgerIntegration.hs new file mode 100644 --- /dev/null +++ b/test/LedgerIntegration.hs @@ -0,0 +1,395 @@ +module LedgerIntegration (ledgerFaultCases) where + +import ClickHouseMigrate.ExclusiveAccess qualified as Access +import ClickHouseMigrate.Ledger qualified as Ledger +import ClickHouseMigrate.Prelude +import ClickHouseMigrate.Target qualified as Target +import ClickHouseMigrate.TestSupport.ExclusiveAccess (hostLocalExclusiveAccess) +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Concurrent (threadDelay) +import Data.Aeson (FromJSON, Key, Value (..), object, (.=)) +import Data.Aeson qualified as Aeson +import Data.Aeson.KeyMap qualified as KM +import Data.ByteString qualified as BS +import Data.ByteString.Lazy qualified as Lazy +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Encoding +import System.Environment (getEnvironment, getExecutablePath) +import System.Exit (ExitCode (..)) +import System.FilePath (()) +import System.Process (CreateProcess (..), proc, readCreateProcessWithExitCode) +import System.Timeout (timeout) + +assert :: String -> Bool -> IO () +assert label predicate = unless predicate (fail ("ledger integration: " ++ label)) + +right :: (Show e) => Either e a -> IO a +right = either (fail . show) pure + +field :: (FromJSON a) => Key -> Value -> IO a +field key (Object members) = maybe (fail "Missing ledger test field") convert (KM.lookup key members) +field _ _ = fail "Invalid ledger test object" + +convert :: (FromJSON a) => Value -> IO a +convert value = case Aeson.fromJSON value of Aeson.Success result -> pure result; Aeson.Error _ -> fail "Invalid ledger test field type" + +data FixtureTarget = FixtureTarget !Server !Text !Target.TargetKey !Target.ServerId !Target.LedgerId + +newTarget :: Server -> Text -> IO FixtureTarget +newTarget server namespace = do + _ <- right (Target.ledgerNamespace namespace) + key <- right (Target.targetKey ("test-target/" <> namespace)) + rows <- queryServer server "SELECT toString(serverUUID()) AS server_id" >>= convert :: IO [Value] + identity <- case rows of [row] -> field "server_id" row >>= right . Target.serverId; _ -> fail "Missing server identity" + ledger <- Target.newLedgerId + pure (FixtureTarget server namespace key identity ledger) + +configFor :: FixtureTarget -> Text -> Target.LedgerBinding -> IO Target.TargetConfig +configFor (FixtureTarget _ namespace key serverId _) endpoint binding = + Target.targetConfig key <$> right (Target.directEndpoint endpoint) <*> pure serverId <*> right (Target.ledgerNamespace namespace) <*> pure binding <*> right (Target.basicCredentials "default" "test-password") <*> pure Target.defaultHttpLimits + +fixtureProvider :: IO Access.ExclusiveAccess +-- The parent owns the entire isolated job. Local callback return never admits a +-- mutating successor: the parent waits for every controlled request to drain. +-- Diagnostic reconciliation is allowed while the same outer job still excludes +-- all other administrators; it never grants sequence reuse or SQL continuation. +fixtureProvider = right (Access.externallySerialized "owned-fixture-serialized-job" "complete owned target; parent excludes mutating successors until all prior requests drain" 1000000 (const (pure ())) (const (pure ())) (const (pure True))) + +initialize :: FixtureTarget -> IO Ledger.LedgerPosition +initialize target@(FixtureTarget server _ key _ ledger) = do + config <- configFor target (serverEndpoint server) Target.FirstUse + provider <- fixtureProvider + audit <- right (Ledger.auditContext "fixture operator" "explicit bootstrap/resumption after controlled requests finished" ["fault proxy has consumed the full upstream response; predecessor client scope closed"]) + quiet <- right (Ledger.attestQuiescence key audit) + report <- Ledger.initializeLedger config provider (Ledger.authorizeInitialization key ledger audit) quiet + assert "Bootstrap cleanup failed" (null (Ledger.cleanupErrors report)) + right (Ledger.result report) + +runProtocol :: FilePath -> FixtureTarget -> Text -> String -> IO Value +runProtocol = runProtocolWithLimit 10000000 + +runProtocolWithLimit :: Int -> FilePath -> FixtureTarget -> Text -> String -> IO Value +runProtocolWithLimit micros driver (FixtureTarget server namespace _ identity ledger) endpoint operation = do + let file = serverLogDirectory server Text.unpack namespace ++ "-" ++ operation ++ ".sql" + BS.writeFile file "" + inherited <- getEnvironment + let overrides = [("CHM_EXPECTED_SERVER_ID", Text.unpack (Target.serverIdText identity)), ("CHM_LEDGER_NAMESPACE", Text.unpack namespace), ("CHM_LEDGER_ID", Text.unpack (Target.ledgerIdText ledger)), ("CHM_TEST_USER", "default"), ("CHM_TEST_PASSWORD", "test-password")] + configured = overrides ++ filter (\(key, _) -> key `notElem` ("CHM_EXPECTED_LEDGER_ID" : map fst overrides)) inherited + (status, output, _) <- readCreateProcessWithExitCode ((proc driver [operation, Text.unpack endpoint, "ledger-protocol", file, show micros, "1048576"]) {env = Just configured}) "" + assert "Private protocol process failed" (status == ExitSuccess) + value <- either (const (fail "Invalid private ledger protocol")) pure (Aeson.eitherDecodeStrict' (Encoding.encodeUtf8 (Text.pack output))) + scope <- field "scopeError" value :: IO Value + verification <- field "verificationError" value :: IO Value + cleanup <- field "cleanupErrors" value :: IO [Value] + access <- field "accessCleanup" value :: IO [Value] + assert "Scope/admission/cleanup failure" (scope == Null && verification == Null && null cleanup && null access) + pure value + +store :: Value -> IO Value +store = field "store" + +success :: Value -> IO Value +success value = store value >>= field "success" + +readHistory :: FilePath -> FixtureTarget -> IO Value +readHistory driver target@(FixtureTarget server _ _ _ _) = runProtocol driver target (serverEndpoint server) "ledger-read" >>= success + +scalar :: Server -> Text -> IO Int +scalar server sql = do + rows <- queryServer server sql >>= convert :: IO [Value] + case rows of [row] -> field "value" row; _ -> fail "Invalid independent ledger observation" + +effect :: FixtureTarget -> IO Bool +effect (FixtureTarget server namespace _ _ _) = do + rows <- queryServer server ("EXISTS TABLE default." <> namespace <> "_effects") >>= convert :: IO [Value] + case rows of [row] -> (== (1 :: Int)) <$> field "result" row; _ -> fail "Invalid effect observation" + +rowCount :: FixtureTarget -> IO Int +rowCount (FixtureTarget server namespace _ _ _) = scalar server ("SELECT toUInt32(count()) AS value FROM `" <> namespace <> "`.events") + +reconcile :: FixtureTarget -> Ledger.AppendReceipt -> IO (Ledger.Reconciliation, Ledger.ExpectedLedgerState) +reconcile target@(FixtureTarget server _ _ _ _) receipt = do + config <- configFor target (serverEndpoint server) Target.FirstUse + provider <- fixtureProvider + report <- Ledger.reconcileAppend config provider receipt + assert "Reconciliation cleanup failed" (null (Ledger.cleanupErrors report)) + right (Ledger.result report) + +ledgerFaultCases :: FilePath -> Server -> IO () +ledgerFaultCases driver server = do + unauthorized <- newTarget server "no_authorization" + rejected <- runProtocol driver unauthorized (serverEndpoint server) "ledger-no-authorization" >>= store + field "ledgerError" rejected >>= assert "Missing ledger was implicitly initialized" . (== ("InitializationNotAuthorized" :: Text)) + absent <- readHistory driver unauthorized + field "observation" absent >>= assert "Read-only inspection initialized storage" . (== ("missing" :: Text)) + forM_Indexed + [ (DatabaseCreation, "ddl-database", DropResponse, 0, False), + (TableCreation, "ddl-table", DropResponse, 0, False), + (Initialization, "initialized", DropResponse, 1, False), + (MigrationStart, "migration-started", DropResponse, 2, False), + (StepStart, "step-started", DropResponse, 3, False), + (StepSuccess, "step-succeeded", DropResponse, 4, True), + (MigrationCompletion, "migration-completed", DropResponse, 5, True), + (StepStart, "step-started", LateError, 3, False) + ] + $ \index (stage, label, mode, count, hasEffect) -> do + target <- newTarget server ("ack_fault_" <> Text.pack (show index)) + withFaultProxy server $ \proxy -> do + setLedgerFault proxy stage mode + report <- runProtocol driver target (proxyEndpoint proxy) "ledger-sequence" + detail <- store report + request <- field "request" detail :: IO Value + field "outcome" request >>= assert "Wrong append evidence" . (== (if mode == LateError then "observed-server-failure" else "submission-uncertain" :: Text)) + received <- proxyEvents proxy >>= convert :: IO [Value] + labels <- mapM (field "stage") received :: IO [Maybe Text] + assert "Faulted request replayed or followed by another request" (length (filter (== Just label) labels) == 1 && last labels == Just label) + assert "Application SQL replayed or preceded acknowledged intent" (length (filter (== Just "application-sql") labels) == if hasEffect then 1 else 0) + assert "Forbidden SQL after unacknowledged intent" . (== hasEffect) =<< effect target + if count == 0 + then do + position <- initialize target + assert "Resumed empty bootstrap changed identity" (Target.ledgerIdText (targetLedger target) `Text.isInfixOf` Ledger.ledgerPositionText position) + assert "DDL bootstrap resumption wrote extra events" . (== 1) =<< rowCount target + else do + encoded <- field "receipt" detail :: IO Text + receipt <- right (Ledger.parseAppendReceipt encoded) + (classification, _) <- reconcile target receipt + assert "Exact event not reconciled" (classification == Ledger.PresentIdentical) + assert "Reconciliation duplicated a row" . (== count) =<< rowCount target + inspected <- readHistory driver target + field "eventCount" inspected >>= assert "Wrong event prefix" . (== count) + field "unresolved" inspected >>= assert "Interrupted state incorrectly resolved" . (== (count `elem` [2, 3, 4])) + when (count > 1) (field "poisoned" report >>= assert "Unacknowledged append allowed another store write") + -- Tokens are observations, not permission; differing digest/unknown + -- identity returns a read-only classification at the same position. + let parts = Text.splitOn ":" encoded + case parts of + [version, key, ledger, number, event, digest, query] -> do + wrong <- right (Ledger.parseAppendReceipt (Text.intercalate ":" [version, key, ledger, number, event, if digest == Text.replicate 64 "b" then Text.replicate 64 "c" else Text.replicate 64 "b", query])) + (conflict, _) <- reconcile target wrong + assert "Conflicting receipt accepted" (conflict == Ledger.Conflicting) + missing <- right (Ledger.parseAppendReceipt (Text.intercalate ":" [version, key, ledger, "999999", "00000000-0000-4000-8000-000000999999", digest, query])) + (notObserved, _) <- reconcile target missing + assert "NotObserved result wrong" (notObserved == Ledger.NotObserved) + assert "Read-only reconciliation mutated history" . (== count) =<< rowCount target + _ -> fail "Unexpected validated receipt format" + putStrLn ("ledger fault " ++ show stage ++ "/" ++ show mode ++ ": one submission; no successor write; exact guarded reconciliation") + protocolFamilyCases driver server + delayedEventCases driver server + delayedInitialization server + bindingAndSchemaCases driver server + corruptionCases driver server + durable <- newTarget server "restart_history" + before <- runProtocol driver durable (serverEndpoint server) "ledger-sequence" >>= success + restartCandidateServer server + after <- readHistory driver durable + assert "Acknowledged history changed after owned server SIGKILL/restart" (before == after) + putStrLn "ledger durability: acknowledged intent/outcomes and pinned server UUID survive owned SIGKILL/restart; no power-loss claim" + where + targetLedger (FixtureTarget _ _ _ _ ledger) = ledger + +forM_Indexed :: [a] -> (Int -> a -> IO ()) -> IO () +forM_Indexed values action = sequence_ (zipWith action [0 ..] values) + +bindingAndSchemaCases :: FilePath -> Server -> IO () +bindingAndSchemaCases driver server = do + target@(FixtureTarget _ namespace key serverId ledger) <- newTarget server "binding_history" + first <- initialize target + second <- initialize target + assert "Reopening bootstrap changed UUID/position" (first == second) + foreignLedger <- Target.newLedgerId + audit <- right (Ledger.auditContext "fixture" "test foreign identity" ["owned fixture and no outstanding requests"]) + quiet <- right (Ledger.attestQuiescence key audit) + provider <- fixtureProvider + wrong <- configFor target (serverEndpoint server) (Target.ExistingLedger foreignLedger) + rejected <- Ledger.initializeLedger wrong provider (Ledger.authorizeInitialization key foreignLedger audit) quiet + assert "Existing UUID mismatch accepted" (Ledger.result rejected == Left (Ledger.LedgerRejected Ledger.LedgerIdentityMismatch)) + wrongKey <- right (Target.targetKey "different-target") + wrongConfig <- Target.targetConfig wrongKey <$> right (Target.directEndpoint (serverEndpoint server)) <*> pure serverId <*> right (Target.ledgerNamespace namespace) <*> pure Target.FirstUse <*> right (Target.basicCredentials "default" "test-password") <*> pure Target.defaultHttpLimits + wrongQuiet <- right (Ledger.attestQuiescence wrongKey audit) + misrouted <- Ledger.initializeLedger wrongConfig provider (Ledger.authorizeInitialization wrongKey ledger audit) wrongQuiet + assert "Target binding mismatch accepted" (Ledger.result misrouted == Left (Ledger.LedgerRejected Ledger.LedgerIdentityMismatch)) + invalidDatabase <- newTarget server "wrong_database" + _ <- queryServer server "CREATE DATABASE wrong_database ENGINE=Memory" + databaseRejected <- runProtocol driver invalidDatabase (serverEndpoint server) "ledger-sequence" >>= store + field "ledgerError" databaseRejected >>= assert "Non-Atomic database accepted" . (== ("InvalidLedgerSchema" :: Text)) + forM_Indexed [("wrong_engine", "Memory", "event String"), ("wrong_columns", "MergeTree ORDER BY sequence SETTINGS fsync_after_insert=1, fsync_part_directory=1, storage_policy='default'", "event UInt64")] $ \_ (name, engine, column) -> do + invalid <- newTarget server name + _ <- queryServer server ("CREATE DATABASE `" <> name <> "` ENGINE=Atomic") + _ <- queryServer server ("CREATE TABLE `" <> name <> "`.events (sequence UInt64,event_id UUID,ledger_id UUID," <> column <> ") ENGINE=" <> engine) + result <- runProtocol driver invalid (serverEndpoint server) "ledger-sequence" >>= store + field "ledgerError" result >>= assert "Wrong engine/schema accepted" . (== ("InvalidLedgerSchema" :: Text)) + assert "Invalid schema executed application SQL" . not =<< effect invalid + remote@(FixtureTarget _ remoteName remoteKey _ remoteLedger) <- newTarget server "remote_storage" + withFaultProxy server $ \proxy -> do + setFault proxy WeakenedStorage + config <- configFor remote (proxyEndpoint proxy) Target.FirstUse + remoteQuiet <- right (Ledger.attestQuiescence remoteKey audit) + result <- Ledger.initializeLedger config provider (Ledger.authorizeInitialization remoteKey remoteLedger audit) remoteQuiet + assert "Unqualified remote storage accepted" (Ledger.result result == Left (Ledger.LedgerRejected Ledger.UnsupportedLedgerStorage)) + count <- scalar server ("SELECT toUInt32(count()) AS value FROM system.databases WHERE name='" <> remoteName <> "'") + assert "Remote storage rejection created objects" (count == 0) + putStrLn "ledger preflight: absent authorization, foreign ledger/target, wrong engine/columns and unqualified remote storage reject before application writes" + +corruptionCases :: FilePath -> Server -> IO () +corruptionCases driver server = forM_Indexed ["duplicate", "gap", "sequence-conflict", "identity-conflict", "unknown-version", "foreign-ledger", "missing-initialization"] $ \index corruption -> do + target@(FixtureTarget _ namespace _ _ _) <- newTarget server ("corruption_" <> Text.pack (show index)) + _ <- runProtocol driver target (serverEndpoint server) "ledger-sequence" >>= success + rows <- queryServer server ("SELECT toString(sequence) AS sequence,toString(event_id) AS event_id,toString(ledger_id) AS ledger_id,event FROM `" <> namespace <> "`.events ORDER BY sequence SETTINGS max_result_rows=0,result_overflow_mode='throw'") >>= convert :: IO [Value] + latest <- case reverse rows of row : _ -> pure row; _ -> fail "Missing corruption fixture" + numberText <- field "sequence" latest :: IO Text + identity <- field "event_id" latest :: IO Text + ledger <- field "ledger_id" latest :: IO Text + original <- field "event" latest :: IO Text + let old = read (Text.unpack numberText) :: Int + newNumber = if corruption `elem` ["gap", "identity-conflict"] then old + 2 else if corruption == "foreign-ledger" then old + 1 else old + newId = if corruption `elem` ["gap", "sequence-conflict", "foreign-ledger"] then "00000000-0000-4000-8000-000000999998" else identity + newLedger = if corruption == "foreign-ledger" then "00000000-0000-4000-8000-000000999997" else ledger + bytes = + Text.replace ("\"eventId\":\"" <> identity <> "\"") ("\"eventId\":\"" <> newId <> "\"") $ + Text.replace ("\"ledgerId\":\"" <> ledger <> "\"") ("\"ledgerId\":\"" <> newLedger <> "\"") $ + Text.replace ("\"sequence\":" <> numberText) ("\"sequence\":" <> Text.pack (show newNumber)) original + changed = if corruption == "unknown-version" then Text.replace "\"version\":1" "\"version\":2" bytes else bytes + payload = object ["sequence" .= newNumber, "event_id" .= newId, "ledger_id" .= newLedger, "event" .= changed] + _ <- + if corruption == "missing-initialization" + then + queryServer server ("ALTER TABLE `" <> namespace <> "`.events DELETE WHERE sequence=1 SETTINGS mutations_sync=1") + else queryServer server ("INSERT INTO `" <> namespace <> "`.events SETTINGS async_insert=0 FORMAT JSONEachRow\n" <> Encoding.decodeUtf8 (Lazy.toStrict (Aeson.encode payload))) + value <- runProtocol driver target (serverEndpoint server) "ledger-read" >>= store + if corruption == "duplicate" + then do + state <- field "success" value + field "eventCount" state >>= assert "Exact duplicate did not collapse" . (== (5 :: Int)) + else do + code <- field "ledgerError" value :: IO Text + let expected = case corruption of "gap" -> "SequenceGap"; "sequence-conflict" -> "SequenceConflict"; "identity-conflict" -> "EventIdentityConflict"; "foreign-ledger" -> "LedgerIdentityMismatch"; "missing-initialization" -> "MissingInitialization"; _ -> "InvalidEvent" + assert ("Corrupt history accepted: " ++ Text.unpack corruption) (code == expected) + putStrLn ("ledger corruption " ++ Text.unpack corruption ++ ": strict full-history fold; no winner selected") + +protocolFamilyCases :: FilePath -> Server -> IO () +protocolFamilyCases driver server = do + forM_Indexed + [ (RecoveryAuthorization, "recovery-authorized", "ledger-recovery-sequence", 5, True), + (StepAcceptance, "step-accepted", "ledger-recovery-sequence", 6, True), + (AdoptionStart, "adoption-started", "ledger-adoption-sequence", 2, False), + (AdoptionEntry, "adoption-entry", "ledger-adoption-sequence", 3, False), + (AdoptionCompletion, "adoption-completed", "ledger-adoption-sequence", 4, False) + ] + $ \index (stage, label, operation, count, recovery) -> do + target@(FixtureTarget _ namespace _ _ _) <- newTarget server ("family_fault_" <> Text.pack (show index)) + when recovery (void (queryServer server ("CREATE TABLE default." <> namespace <> "_effects (id UInt8) ENGINE=MergeTree ORDER BY id"))) + withFaultProxy server $ \proxy -> do + setLedgerFault proxy stage DropResponse + report <- runProtocol driver target (proxyEndpoint proxy) operation + detail <- store report + receipt <- field "receipt" detail >>= right . Ledger.parseAppendReceipt + field "poisoned" report >>= assert "Family append did not stop store" + requests <- proxyEvents proxy >>= convert :: IO [Value] + labels <- mapM (field "stage") requests :: IO [Maybe Text] + assert "Family event replay or successor" (length (filter (== Just label) labels) == 1 && last labels == Just label) + assert "Adoption executed SQL or recovery replayed query" (length (filter (== Just "application-sql") labels) == if recovery then 1 else 0) + (classification, _) <- reconcile target receipt + assert "Family event not reconciled" (classification == Ledger.PresentIdentical) + assert "Family reconciliation wrote history" . (== count) =<< rowCount target + state <- readHistory driver target + field "unresolved" state >>= assert "Interrupted family incorrectly resolved" . (== (stage /= AdoptionCompletion)) + putStrLn ("ledger family " ++ show stage ++ ": one event; exact reconciliation; no implicit continuation") + complete@(FixtureTarget _ namespace _ _ _) <- newTarget server "complete_recovery" + _ <- queryServer server ("CREATE TABLE default." <> namespace <> "_effects (id UInt8) ENGINE=MergeTree ORDER BY id") + state <- runProtocol driver complete (serverEndpoint server) "ledger-recovery-sequence" >>= success + field "eventCount" state >>= assert "Recovery protocol missing evidence" . (== (8 :: Int)) + field "unresolved" state >>= assert "Complete recovery remains unresolved" . not + putStrLn "ledger family: failure, recovery authorization, accepted step and explicit finalize roundtrip without replaying SQL" + +-- Hold the first INSERT after the client has sent it but before the server sees +-- it. An empty independent read (and absent system.processes entry) cannot make +-- its sequence or a new ledger UUID safe to reuse. +delayedInitialization :: Server -> IO () +delayedInitialization server = do + binary <- getExecutablePath + FixtureTarget _ namespace _ identity ledger <- newTarget server "delayed_initialization" + key <- right (Target.targetKey ("ledger-tests/" <> namespace)) + let target = FixtureTarget server namespace key identity ledger + root = serverLogDirectory server "delayed-locks" + command endpoint candidate evidence = proc binary ["exclusive-child", Text.unpack endpoint, Text.unpack (Target.serverIdText identity), Text.unpack namespace, Text.unpack (Target.ledgerIdText candidate), root, "1000000", evidence] + launch micros candidate = do + inherited <- getEnvironment + let configured = ("CHM_CHILD_HTTP_MICROS", micros) : filter ((/= "CHM_CHILD_HTTP_MICROS") . fst) inherited + (status, output, _) <- readCreateProcessWithExitCode (candidate {env = Just configured}) "" + assert "Delayed bootstrap child failed" (status == ExitSuccess) + value <- either (const (fail "Invalid delayed child protocol")) pure (Aeson.eitherDecodeStrict' (Encoding.encodeUtf8 (Text.pack output))) + cleanup <- field "cleanupErrors" value :: IO [Value] + assert "Delayed child cleanup failed" (null cleanup) + pure value + waitFor action = do + ready <- action + unless ready (threadDelay 10000 >> waitFor action) + withFaultProxy server $ \proxy -> do + setLedgerFault proxy Initialization HoldBeforeForward + failed <- launch "1000000" (command (proxyEndpoint proxy) ledger "yes") + field "outcome" failed >>= assert "Held INSERT did not remain uncertain" . (== ("request-failed" :: Text)) + field "evidence" failed >>= assert "Held INSERT was called absent" . (== ("UncertainSubmission" :: Text)) + receipt <- field "receipt" failed >>= right . Ledger.parseAppendReceipt + assert "First event arrived before controlled release" . (== 0) =<< rowCount target + active <- scalar server ("SELECT toUInt32(count()) AS value FROM system.processes WHERE query_id='" <> Target.queryIdText (Ledger.appendQueryId receipt) <> "'") + assert "Held INSERT already active on server" (active == 0) + different <- Target.newLedgerId + blocked <- launch "10000000" (command (serverEndpoint server) different "no") + field "code" blocked >>= assert "Empty read allowed different bootstrap UUID/sequence" . (== ("PredecessorQuiescenceRequired" :: Text)) + assert "Successor wrote before predecessor quiescence" . (== 0) =<< rowCount target + releaseFault proxy + arrived <- timeout 5000000 (waitFor ((== 1) <$> rowCount target)) + drained <- timeout 5000000 (waitFor ((== 0) <$> proxyConnections proxy)) + assert "Delayed first event never arrived/drained" (isJust arrived && isJust drained) + -- The parent now knows every controlled request completed and the original + -- client process ended, and explicitly attests that evidence to the SAME OS + -- coordinator. This is stronger than observing an empty row/process query. + audit <- right (Ledger.auditContext "fixture recovery operator" "all controlled predecessor requests drained" ["original client process exited; proxy consumed sole upstream response and closed its handler"]) + quiet <- right (Ledger.attestQuiescence key audit) + provider <- hostLocalExclusiveAccess root 1000000 (Just quiet) >>= right + config <- configFor target (serverEndpoint server) Target.FirstUse + inspected <- Ledger.reconcileAppend config provider receipt + assert "Delayed event not reconciled" (case Ledger.result inspected of Right (Ledger.PresentIdentical, _) -> null (Ledger.cleanupErrors inspected); _ -> False) + resumed <- Ledger.initializeLedger config provider (Ledger.authorizeInitialization key ledger audit) quiet + assert "Bootstrap UUID not retained after reconciliation" (case Ledger.result resumed of Right position -> Target.ledgerIdText ledger `Text.isInfixOf` Ledger.ledgerPositionText position && null (Ledger.cleanupErrors resumed); _ -> False) + requests <- proxyEvents proxy >>= convert :: IO [Value] + stages <- mapM (field "stage") requests :: IO [Maybe Text] + assert "Held initialization was replayed" (length (filter (== Just "initialized") stages) == 1) + assert "Reconciled bootstrap duplicated event" . (== 1) =<< rowCount target + putStrLn "delayed first event: empty row/process reads; different-UUID successor blocked; later single commit reconciled only after explicit proven quiescence" + +-- The parent retains the externally serialized job through diagnostics/drain; +-- no new mutating job is admitted on the basis of NotObserved. This exercises +-- the same append boundary for intent, completion and adoption payloads. +delayedEventCases :: FilePath -> Server -> IO () +delayedEventCases driver server = forM_Indexed [(StepStart, "step-started", "ledger-sequence", 2, False), (StepSuccess, "step-succeeded", "ledger-sequence", 3, True), (AdoptionEntry, "adoption-entry", "ledger-adoption-sequence", 2, False)] $ \index (stage, label, operation, before, effected) -> do + target <- newTarget server ("delayed_event_" <> Text.pack (show index)) + withFaultProxy server $ \proxy -> do + setLedgerFault proxy stage HoldBeforeForward + failed <- runProtocolWithLimit 1000000 driver target (proxyEndpoint proxy) operation + field "poisoned" failed >>= assert "Delayed append did not stop store" + detail <- store failed + pending <- field "request" detail :: IO Value + field "outcome" pending >>= assert "Delayed append classified as absent" . (== ("submission-uncertain" :: Text)) + receipt <- field "receipt" detail >>= right . Ledger.parseAppendReceipt + assert "Delayed event already committed" . (== before) =<< rowCount target + (notObserved, observed) <- reconcile target receipt + assert "In-flight absent event not classified read-only" (notObserved == Ledger.NotObserved && case observed of Ledger.ExpectPosition _ -> True; _ -> False) + assert "NotObserved mutated or reused sequence" . (== before) =<< rowCount target + assert "Delayed boundary application effect wrong" . (== effected) =<< effect target + releaseFault proxy + let wait action = action >>= \ready -> unless ready (threadDelay 10000 >> wait action) + arrived <- timeout 5000000 (wait ((== before + 1) <$> rowCount target)) + drained <- timeout 5000000 (wait ((== 0) <$> proxyConnections proxy)) + assert "Delayed event never committed/drained" (isJust arrived && isJust drained) + (present, _) <- reconcile target receipt + assert "Delayed event did not reconcile exactly" (present == Ledger.PresentIdentical) + state <- readHistory driver target + field "unresolved" state >>= assert "Late event silently resumed migration/adoption" + requests <- proxyEvents proxy >>= convert :: IO [Value] + stages <- mapM (field "stage") requests :: IO [Maybe Text] + assert "Delayed event or SQL replayed" (length (filter (== Just label) stages) == 1 && length (filter (== Just "application-sql") stages) == if effected then 1 else 0) + assert "Reconciliation appended anything" . (== before + 1) =<< rowCount target + putStrLn ("delayed " ++ show stage ++ ": NotObserved under retained job; one late commit; unresolved state retained; no successor admitted before drain") diff --git a/test/Main.hs b/test/Main.hs new file mode 100644 --- /dev/null +++ b/test/Main.hs @@ -0,0 +1,487 @@ +module Main (main) where + +import AdoptionFaults (adoptionFaultCases) +import AdoptionIntegration (adoptionEarlyCases) +import CLIIntegration (cliCases, exampleCases) +import ClickHouseMigrate.ExclusiveAccess qualified as Access +import ClickHouseMigrate.Ledger qualified as Ledger +import ClickHouseMigrate.Prelude +import ClickHouseMigrate.Target qualified as Target +import ClickHouseMigrate.TestSupport.ExclusiveAccess (hostLocalExclusiveAccess) +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import ConsumerFixtures (consumerFixtureCases) +import Control.Concurrent (forkFinally, killThread, newEmptyMVar, putMVar, takeMVar, threadDelay, tryReadMVar) +import Control.Exception (bracket, finally, throwIO) +import Data.Aeson (FromJSON, Key, Object, Result (..), Value (..), fromJSON, object, (.=)) +import Data.Aeson qualified as Aeson +import Data.Aeson.KeyMap qualified as KM +import Data.ByteString qualified as BS +import Data.ByteString.Char8 qualified as Char8 +import Data.ByteString.Lazy qualified as Lazy +import Data.IORef (newIORef, readIORef, writeIORef) +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Encoding +import FailureBoundaries (failureBoundaryCases) +import LedgerIntegration (ledgerFaultCases) +import RepairBoundaries (repairBoundaryCases) +import RepairIntegration (repairEarlyCases) +import ReviewRemediation (remediationCases) +import RunnerIntegration (runnerEarlyCases, runnerValidationCases) +import SurvivingWork (runnerChild, survivingWorkCases) +import System.Environment (getArgs, getEnv, getEnvironment, getExecutablePath, lookupEnv) +import System.Exit (ExitCode (..)) +import System.FilePath (()) +import System.IO (hClose) +import System.Process (CreateProcess (..), StdStream (..), createProcess, getProcessExitCode, proc, readCreateProcessWithExitCode, terminateProcess, waitForProcess) +import System.Timeout (timeout) + +assert :: String -> Bool -> IO () +assert label condition = unless condition (fail label) + +field :: (FromJSON a) => Key -> Value -> IO a +field key (Object value) = maybe (fail "Missing result field") convert (KM.lookup key value) +field _ _ = fail "Invalid result object" + +convert :: (FromJSON a) => Value -> IO a +convert value = case fromJSON value of Success result -> pure result; Error _ -> fail "Invalid result type" + +main :: IO () +main = do + args <- getArgs + case args of + ["runner-child", endpoint, identity, label, ledger, root] -> runnerChild endpoint identity label ledger root + ["exclusive-child", endpoint, identity, namespace, ledger, root, micros, evidence] -> exclusiveChild endpoint identity namespace ledger root micros evidence + _ -> integrationMain args + +integrationMain :: [String] -> IO () +integrationMain args = do + driver <- if args `elem` [["exclusive"], ["runner"], ["repair"], ["boundaries"], ["survivors"], ["adoption"], ["consumer"], ["cli"], ["example"], ["remediation"]] then pure "" else getEnv "CLICKHOUSE_MIGRATE_TEST_DRIVER" + finished <- timeout (if args == ["consumer"] then 600000000 else 180000000) $ withCandidateServer "lts-native" $ \server -> do + putStrLn ("Candidate ClickHouse " ++ Text.unpack (serverVersion server)) + case args of + [] -> transportCases driver server + ["transport"] -> transportCases driver server + ["completion"] -> completionCases driver server + ["ledger"] -> ledgerCases driver server + ["consumer"] -> consumerFixtureCases server + ["cli"] -> cliCases server + ["example"] -> exampleCases server + ["adoption"] -> adoptionEarlyCases server >> adoptionFaultCases server + ["remediation"] -> remediationCases server + ["runner"] -> runnerEarlyCases server >> runnerValidationCases server + ["repair"] -> repairEarlyCases server >> repairBoundaryCases server + ["survivors"] -> survivingWorkCases server + ["boundaries"] -> failureBoundaryCases server + ["exclusive"] -> exclusiveCases server + _ -> fail "Unknown integration group" + case finished of Nothing -> fail "Integration watchdog expired"; Just () -> pure () + +runDriver :: FilePath -> Server -> Text -> String -> String -> ByteString -> Int -> Int -> IO Value +runDriver = runDriverExpected Nothing + +runDriverExpected :: Maybe Text -> FilePath -> Server -> Text -> String -> String -> ByteString -> Int -> Int -> IO Value +runDriverExpected expected driver server endpoint operation identity sql micros bound = do + let file = serverLogDirectory server identity ++ ".sql" + BS.writeFile file sql + inherited <- getEnvironment + let configured = case expected of + Nothing -> inherited + Just value -> ("CHM_EXPECTED_SERVER_ID", Text.unpack value) : filter ((/= "CHM_EXPECTED_SERVER_ID") . fst) inherited + (status, output, _) <- readCreateProcessWithExitCode ((proc driver [operation, Text.unpack endpoint, identity, file, show micros, show bound]) {env = Just configured}) "" + assert "Private driver failed" (status == ExitSuccess) + report <- either (const (fail "Invalid private driver protocol")) pure (Aeson.eitherDecodeStrict' (Encoding.encodeUtf8 (Text.pack output))) + scope <- field "scopeError" report :: IO Value + cleanup <- field "cleanupErrors" report :: IO [Value] + assert "Unexpected scope failure" (scope == Null) + unless (operation `elem` ["cleanup-failure", "cleanup-timeout"]) (assert "Unexpected cleanup failure" (null cleanup)) + let rendered = Encoding.decodeUtf8 (Lazy.toStrict (Aeson.encode report)) + assert "Credentials or raw diagnostic escaped" (not ("test-password" `Text.isInfixOf` rendered) && not ("SQL_SECRET_MARKER" `Text.isInfixOf` rendered)) + pure report + +outcomes :: Value -> IO [Value] +outcomes = field "outcomes" + +expectFirst :: Text -> Value -> IO () +expectFirst expected report = + outcomes report >>= \case + first : _ -> field "outcome" first >>= assert "Wrong transport evidence classification" . (== expected) + _ -> fail "Missing transport evidence" + +exists :: Server -> Text -> IO Bool +exists server name = do + value <- queryServer server ("EXISTS TABLE default." <> name) + rows <- convert value :: IO [Value] + case rows of + [row] -> (== (1 :: Int)) <$> field "result" row + _ -> fail "Invalid independent effect observation" + +transportCases :: FilePath -> Server -> IO () +transportCases driver server = do + let cases = + [ (DropResponse, "lost_response", "submission-uncertain"), + (TruncateResponse, "truncated_response", "submission-uncertain"), + (LateError, "late_error", "observed-server-failure"), + (DelayResponse, "timed_out_response", "submission-uncertain") + ] + mapM_ + ( \(mode, name, expected) -> withFaultProxy server $ \proxy -> do + setFault proxy mode + let sql = Encoding.encodeUtf8 ("CREATE TABLE default." <> name <> " (id UInt8) ENGINE=Memory") + micros = if mode == DelayResponse then 150000 else 3000000 + report <- runDriver driver server (proxyEndpoint proxy) "command" (Text.unpack name) sql micros 1048576 + expectFirst expected report + assert "Committed effect missing" =<< exists server name + events <- proxyEvents proxy >>= convert :: IO [Object] + assert "POST was replayed" (length events == 1) + putStrLn (Text.unpack name ++ ": " ++ Text.unpack expected ++ "; actual effect present; requests=1") + ) + cases + withFaultProxy server $ \origin -> withFaultProxy server $ \destination -> do + redirectTo origin (proxyEndpoint destination) + setFault origin Redirect + report <- runDriver driver server (proxyEndpoint origin) "command" "redirect" "CREATE TABLE default.redirect_trap (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "observed-server-failure" report + destinationEvents <- proxyEvents destination >>= convert :: IO [Object] + originEvents <- proxyEvents origin >>= convert :: IO [Object] + assert "Redirect was followed" (length originEvents == 1 && null destinationEvents) + present <- exists server "redirect_trap" + assert "Redirect destination executed SQL" (not present) + putStrLn "redirect: rejected; origin requests=1; destination requests=0" + withFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "command" "empty_request" "" 3000000 1048576 + expectFirst "not-submitted" report + events <- proxyEvents proxy >>= convert :: IO [Object] + assert "Invalid request reached network" (null events) + putStrLn "empty request: positively not submitted; requests=0" + withFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "read" "bounded_response" "SELECT repeat('x', 2048) AS value" 3000000 32 + expectFirst "submission-uncertain" report + events <- proxyEvents proxy >>= convert :: IO [Object] + assert "Oversized response caused replay" (length events == 1) + putStrLn "bounded response: no false success; requests=1" + + withFaultProxy server $ \proxy -> do + setFault proxy CloseReusedConnection + report <- runDriver driver server (proxyEndpoint proxy) "reuse" "reused_connection" "CREATE TABLE IF NOT EXISTS default.reused_connection (id UInt8) ENGINE=Memory" 3000000 1048576 + results <- outcomes report + case results of + [first, second] -> do + firstKind <- field "outcome" first :: IO Text + secondKind <- field "outcome" second :: IO Text + assert "Reused connection evidence wrong" (firstKind == "acknowledged-success" && secondKind == "submission-uncertain") + _ -> fail "Missing reuse evidence" + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Reused POST replayed" (length events == 2) + connections <- mapM (field "connection") events :: IO [Int] + assert "Test did not actually reuse a socket" (case connections of [a, b] -> a == b; _ -> False) + putStrLn "reused connection: acknowledged then uncertain; one socket; requests=2 for two authorizations" + withTlsFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "command" "untrusted_tls" "CREATE TABLE default.untrusted_tls (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "not-submitted" report + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Untrusted TLS received HTTP SQL" (null events) + present <- exists server "untrusted_tls" + assert "Untrusted TLS executed SQL" (not present) + putStrLn "untrusted TLS: certificate rejected before HTTP; requests=0" + + fullRead <- runDriver driver server (serverEndpoint server) "read" "complete_machine_read" "SELECT number FROM numbers(3) ORDER BY number" 3000000 1048576 + expectFirst "acknowledged-success" fullRead + readOutcomes <- outcomes fullRead + readRows <- case readOutcomes of [row] -> field "rows" row :: IO [Value]; _ -> fail "Missing complete read" + assert "Profile silently truncated a machine read" (length readRows == 3) + putStrLn "machine read: profile result_overflow_mode=break/max_result_rows=1 cannot silently truncate history" + + -- A checked transport verifies every fresh connection, not just its first request. + identityRows <- queryServer server "SELECT toString(serverUUID()) AS server_id" >>= convert :: IO [Value] + actualId <- case identityRows of [row] -> field "server_id" row; _ -> fail "Missing server identity" + direct <- runDriverExpected (Just actualId) driver server (serverEndpoint server) "guarded-command" "guarded_direct" "CREATE TABLE default.guarded_direct (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "acknowledged-success" direct + assert "Direct target effect absent" =<< exists server "guarded_direct" + putStrLn "direct native admission: explicit identity encoding; acknowledged application effect" + withFaultProxy server $ \proxy -> do + report <- runDriverExpected (Just actualId) driver server (proxyEndpoint proxy) "guarded-command" "guarded_success" "CREATE TABLE default.guarded_success (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "acknowledged-success" report + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Unexpected admission/request count" (length events == 3) + paths <- mapM (field "path") events :: IO [Text] + assert "Completion policy missing on independent request" (all (\path -> all (`Text.isInfixOf` path) ["async_insert=0", "mutations_sync=1", "alter_sync=1", "wait_end_of_query=1"]) paths) + putStrLn "guarded target: fresh-connection admission + explicit probe + one SQL; fixed policy on all three" + withFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "guarded-command" "wrong_identity" "CREATE TABLE default.wrong_identity (id UInt8) ENGINE=Memory" 3000000 1048576 + verification <- field "verificationError" report :: IO Value + assert "Wrong identity accepted" (verification /= Null) + present <- exists server "wrong_identity" + assert "Wrong identity executed application SQL" (not present) + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Wrong identity reached application request" (length events == 1) + putStrLn "wrong identity: rejected on new connection before application SQL" + mapM_ + ( \mode -> withFaultProxy server $ \proxy -> do + setFault proxy mode + report <- runDriverExpected (Just actualId) driver server (proxyEndpoint proxy) "guarded-command" "unsupported_admission" "CREATE TABLE default.unsupported_admission (id UInt8) ENGINE=Memory" 3000000 1048576 + verification <- field "verificationError" report :: IO Value + assert "Unsupported server/policy accepted" (verification /= Null) + present <- exists server "unsupported_admission" + assert "Unsupported admission executed SQL" (not present) + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Unsupported admission was retried" (length events == 1) + putStrLn (show mode ++ ": admission rejected; no application SQL") + ) + [UnsupportedVersion, PrereleaseVersion, WeakenedPolicy] + withFaultProxy server $ \proxy -> do + setFaultForQuery proxy "guarded_drop" DropResponse + report <- runDriverExpected (Just actualId) driver server (proxyEndpoint proxy) "guarded-command" "guarded_drop" "CREATE TABLE default.guarded_drop (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "submission-uncertain" report + assert "Guarded effect missing" =<< exists server "guarded_drop" + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Guarded application POST replayed" (length events == 3) + putStrLn "guarded lost response: two admission reads, exactly one application SQL, effect retained" + withFaultProxy server $ \proxy -> do + setFaultForQuery proxy "reconnect_identity" SwitchIdentityAfterResponse + report <- runDriverExpected (Just actualId) driver server (proxyEndpoint proxy) "guarded-reuse" "reconnect_identity" "CREATE TABLE IF NOT EXISTS default.reconnect_identity (id UInt8) ENGINE=Memory" 3000000 1048576 + results <- outcomes report + kinds <- mapM (field "outcome") results :: IO [Text] + assert "Fresh connection bypassed identity admission" (kinds == ["acknowledged-success", "not-submitted"]) + events <- proxyEvents proxy >>= convert :: IO [Value] + paths <- mapM (field "path") events :: IO [Text] + assert "Second application SQL was sent to changed identity" (length events == 4 && not (any (Text.isInfixOf "query_id=reconnect_identity-second&") paths)) + putStrLn "reconnect: changed identity blocks second application SQL before its bytes are submitted" + withFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "after-scope" "closed_scope" "CREATE TABLE default.closed_scope (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "not-submitted" report + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Closed capability submitted SQL" (null events) + putStrLn "closed scope: escaped private handle cannot submit" + + withFaultProxy server $ \proxy -> do + setFault proxy DropResponse + report <- runDriver driver server (proxyEndpoint proxy) "cleanup-failure" "cleanup_fault" "CREATE TABLE default.cleanup_fault (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "submission-uncertain" report + failures <- field "cleanupErrors" report :: IO [Value] + codes <- mapM (field "code") failures :: IO [Text] + assert "Cleanup replaced primary failure" (codes == ["CleanupFailed"]) + putStrLn "cleanup failure: primary uncertainty and separate cleanup error retained" + withFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "cleanup-timeout" "cleanup_timeout" "CREATE TABLE default.cleanup_timeout (id UInt8) ENGINE=Memory" 3000000 1048576 + expectFirst "acknowledged-success" report + failures <- field "cleanupErrors" report :: IO [Value] + codes <- mapM (field "code") failures :: IO [Text] + assert "Cleanup deadline was hidden or treated as SQL failure" (codes == ["CleanupTimedOut"]) + putStrLn "cleanup timeout: bounded and distinct from acknowledged SQL" + withFaultProxy server $ \proxy -> do + let file = serverLogDirectory server "live_cleanup.sql" + BS.writeFile file "CREATE TABLE default.live_cleanup (id UInt8) ENGINE=Memory" + (_, output, _, process) <- createProcess (proc driver ["linger", Text.unpack (proxyEndpoint proxy), "live_cleanup", file, "3000000", "1048576"]) {std_out = CreatePipe} + case output of + Nothing -> terminateProcess process >> fail "Missing driver pipe" + Just handle -> + ( do + line <- Char8.hGetLine handle + report <- either (const (fail "Invalid live driver response")) pure (Aeson.eitherDecodeStrict' line) + expectFirst "acknowledged-success" report + threadDelay 100000 + status <- getProcessExitCode process + assert "Cleanup test waited for process exit" (status == Nothing) + active <- proxyConnections proxy + assert "Sockets remained open after HTTP scope exit" (active == 0) + void (waitForProcess process) + putStrLn "socket cleanup: zero open connections while driver process is still alive" + ) + `finally` (hClose handle >> terminateProcess process >> void (waitForProcess process)) + +completionCases :: FilePath -> Server -> IO () +completionCases driver server = do + let send identity sql = do + report <- runDriver driver server (serverEndpoint server) "command" identity sql 10000000 1048576 + expectFirst "acknowledged-success" report + pure report + _ <- send "completion_create" "CREATE TABLE default.completion (id UInt32, value UInt32) ENGINE=MergeTree ORDER BY id" + _ <- send "synchronous_insert" "INSERT INTO default.completion VALUES (1,10)" + count <- scalar server "SELECT toUInt32(count()) AS value FROM default.completion" + assert "INSERT returned before synchronous visibility" (count == 1) + _ <- send "local_alter" "ALTER TABLE default.completion ADD COLUMN note String DEFAULT 'ok'" + columns <- scalar server "SELECT toUInt32(count()) AS value FROM system.columns WHERE database='default' AND table='completion' AND name='note'" + assert "Local ALTER not complete" (columns == 1) + putStrLn "completion: async_insert=0 overrides async profile; INSERT visible immediately; local ALTER metadata complete" + void (queryServer server "SYSTEM STOP MERGES default.completion") + done <- newEmptyMVar + let mutation = runDriver driver server (serverEndpoint server) "command" "mutation_barrier" "ALTER TABLE default.completion UPDATE value=value+1 WHERE id=1" 10000000 1048576 + waitPending 0 = fail "Mutation never entered the controlled queue" + waitPending n = do + pending <- scalar server "SELECT toUInt32(count()) AS value FROM system.mutations WHERE database='default' AND table='completion' AND NOT is_done" + if pending > 0 then pure () else threadDelay 50000 >> waitPending (n - 1) + ( bracket (forkFinally mutation (putMVar done)) killThread $ \_ -> do + waitPending (40 :: Int) + threadDelay 300000 + early <- tryReadMVar done + assert "Mutation returned before local completion" (case early of Nothing -> True; _ -> False) + void (queryServer server "SYSTEM START MERGES default.completion") + result <- takeMVar done >>= either throwIO pure + expectFirst "acknowledged-success" result + value <- scalar server "SELECT toUInt32(value) AS value FROM default.completion WHERE id=1" + assert "Mutation acknowledged before effects" (value == 11) + ) + `finally` void (queryServer server "SYSTEM START MERGES default.completion") + putStrLn "mutation completion: request blocks while merges are stopped, then acknowledges only after the update" + _ <- send "delayed_create" "CREATE TABLE default.delayed_ledger_event (id UInt8) ENGINE=Memory" + withFaultProxy server $ \proxy -> do + report <- runDriver driver server (proxyEndpoint proxy) "command" "delayed_ledger_insert" "INSERT INTO default.delayed_ledger_event SELECT toUInt8(sleep(2))" 150000 1048576 + expectFirst "submission-uncertain" report + empty <- scalar server "SELECT toUInt32(count()) AS value FROM default.delayed_ledger_event" + active <- scalar server "SELECT toUInt32(count()) AS value FROM system.processes WHERE query_id='delayed_ledger_insert'" + assert "Test did not observe empty history with an in-flight write" (empty == 0 && active == 1) + waitRows server "SELECT toUInt32(count()) AS value FROM default.delayed_ledger_event" 60 + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Delayed INSERT replayed" (length events == 1) + putStrLn "delayed ledger-style INSERT: empty read while request active; row arrives later; requests=1" + _ <- send "cancel_create" "CREATE TABLE default.cancelled_insert (id UInt8) ENGINE=Memory" + withFaultProxy server $ \proxy -> do + let file = serverLogDirectory server "cancelled_insert.sql" + BS.writeFile file "INSERT INTO default.cancelled_insert SELECT toUInt8(sleep(2))" + (status, output, _) <- readCreateProcessWithExitCode (proc driver ["cancel", Text.unpack (proxyEndpoint proxy), "cancelled_insert", file, "5000000", "1048576"]) "" + assert "Cancellation became ordinary success" (status /= ExitSuccess) + assert "Cancellation diagnostics leaked credentials" (not ("test-password" `Text.isInfixOf` Text.pack output)) + active <- scalar server "SELECT toUInt32(count()) AS value FROM system.processes WHERE query_id='cancelled_insert'" + empty <- scalar server "SELECT toUInt32(count()) AS value FROM default.cancelled_insert" + assert "Server request did not survive client cancellation" (active == 1 && empty == 0) + waitRows server "SELECT toUInt32(count()) AS value FROM default.cancelled_insert" 60 + events <- proxyEvents proxy >>= convert :: IO [Value] + assert "Cancellation replayed INSERT" (length events == 1) + putStrLn "cancellation: async exception rethrown after client cleanup; server work survives and later commits once" + +scalar :: Server -> Text -> IO Int +scalar server sql = + queryServer server sql >>= convert >>= \case + [row] -> field "value" row + _ -> fail "Invalid independent scalar observation" + +waitRows :: Server -> Text -> Int -> IO () +waitRows _ _ 0 = fail "Delayed effect never completed" +waitRows server sql remaining = do + count <- scalar server sql + if count == 1 then pure () else threadDelay 50000 >> waitRows server sql (remaining - 1) + +right :: (Show e) => Either e a -> IO a +right = either (fail . show) pure + +-- Separate child processes exercise the public provider/initialization API. +exclusiveChild :: String -> String -> String -> String -> FilePath -> String -> String -> IO () +exclusiveChild endpoint identity namespace ledger root micros evidence = do + key <- right (Target.targetKey ("ledger-tests/" <> Text.pack namespace)) + credentials <- right (Target.basicCredentials "default" "test-password") + requestMicros <- maybe 10000000 read <$> lookupEnv "CHM_CHILD_HTTP_MICROS" + config <- Target.targetConfig key <$> right (Target.directEndpoint (Text.pack endpoint)) <*> right (Target.serverId (Text.pack identity)) <*> right (Target.ledgerNamespace (Text.pack namespace)) <*> pure Target.FirstUse <*> pure credentials <*> right (Target.httpLimits requestMicros 1048576) + uuid <- right (Target.ledgerId (Text.pack ledger)) + audit <- right (Ledger.auditContext "test child administrator" "owned fixture handoff" ["owned process and all server requests independently quiescent"]) + quiet <- right (Ledger.attestQuiescence key audit) + provider <- hostLocalExclusiveAccess root (read micros) (if evidence == "yes" then Just quiet else Nothing) >>= right + report <- Ledger.initializeLedger config provider (Ledger.authorizeInitialization key uuid audit) quiet + let status = case Ledger.result report of + Right position -> object ["outcome" .= ("initialized" :: Text), "position" .= Ledger.ledgerPositionText position] + Left (Ledger.AccessRejected problem) -> object ["outcome" .= ("access-rejected" :: Text), "code" .= show problem] + Left (Ledger.RequestFailed requestEvidence failure receipt) -> object ["outcome" .= ("request-failed" :: Text), "code" .= show (Ledger.code failure), "evidence" .= show requestEvidence, "receipt" .= fmap Ledger.appendReceiptText receipt] + Left (Ledger.InitializationFailed ledgerIdentity stage query requestEvidence failure) -> object ["outcome" .= ("ddl-failed" :: Text), "code" .= show (Ledger.code failure), "evidence" .= show requestEvidence, "ledgerId" .= ledgerIdentity, "stage" .= stage, "queryId" .= Target.queryIdText query] + Left problem -> object ["outcome" .= ("rejected" :: Text), "code" .= show problem] + let value = case status of Object fields -> Object (KM.insert "cleanupErrors" (Aeson.toJSON (map show (Ledger.cleanupErrors report))) fields); other -> other + Lazy.putStr (Aeson.encode value) + +ledgerCases :: FilePath -> Server -> IO () +ledgerCases driver server = do + identityRows <- queryServer server "SELECT toString(serverUUID()) AS server_id" >>= convert :: IO [Value] + actualId <- case identityRows of [row] -> field "server_id" row; _ -> fail "Missing server identity" + let file = serverLogDirectory server "ledger-smoke.sql" + BS.writeFile file "" + inherited <- getEnvironment + let configured = ("CHM_EXPECTED_SERVER_ID", Text.unpack actualId) : filter ((/= "CHM_EXPECTED_SERVER_ID") . fst) inherited + (exit, output, _) <- readCreateProcessWithExitCode ((proc driver ["ledger-smoke", Text.unpack (serverEndpoint server), "ledger-smoke", file, "10000000", "1048576"]) {env = Just configured}) "" + assert "Private ledger roundtrip failed" (exit == ExitSuccess) + value <- either (const (fail "Invalid ledger driver protocol")) pure (Aeson.eitherDecodeStrict' (Encoding.encodeUtf8 (Text.pack output))) + field "roundtrip" value >>= assert "Ledger byte roundtrip mismatch" + putStrLn "ledger: direct native initialization + immutable append/read roundtrip; quoted/newline audit text preserved" + ledgerFaultCases driver server + +exclusiveCases :: Server -> IO () +exclusiveCases server = do + binary <- getExecutablePath + identityRows <- queryServer server "SELECT toString(serverUUID()) AS server_id" >>= convert :: IO [Value] + actualId <- case identityRows of [row] -> field "server_id" row; _ -> fail "Missing server identity" + let root = serverLogDirectory server "locks" + ledger = "01234567-89ab-4def-8123-456789abcdef" + child endpoint namespace micros evidence = proc binary ["exclusive-child", Text.unpack endpoint, Text.unpack actualId, namespace, ledger, root, micros, evidence] + launch command = do + (status, output, _) <- readCreateProcessWithExitCode command "" + assert "Public lock child failed" (status == ExitSuccess) + either (const (fail "Invalid lock child protocol")) pure (Aeson.eitherDecodeStrict' (Encoding.encodeUtf8 (Text.pack output))) + outcome expected value = field "outcome" value >>= assert "Wrong public lock outcome" . (== (expected :: Text)) + waitForPost proxy = do + events <- proxyEvents proxy >>= convert :: IO [Value] + if null events then threadDelay 10000 >> waitForPost proxy else pure () + withFaultProxy server $ \proxy -> do + setFault proxy DelayFirstResponse + (input, output, _, process) <- createProcess (child (proxyEndpoint proxy) "concurrent_history" "5000000" "yes") {std_out = CreatePipe} + let close = do + mapM_ hClose input + mapM_ hClose output + terminateProcess process + void (waitForProcess process) + ( do + ready <- timeout 5000000 (waitForPost proxy) + assert "First administrator never entered" (isJust ready) + second <- launch (child (serverEndpoint server) "concurrent_history" "100000" "yes") + outcome "access-rejected" second + field "code" second >>= assert "Second process entered while first held lock" . (== ("AcquisitionTimedOut" :: Text)) + first <- case output of + Just handle -> do + bytes <- BS.hGetContents handle + either (const (fail "Invalid first child result")) pure (Aeson.eitherDecodeStrict' bytes) + Nothing -> fail "Missing child stdout" + void (waitForProcess process) + outcome "initialized" first + blocked <- launch (child (serverEndpoint server) "concurrent_history" "1000000" "no") + outcome "access-rejected" blocked + field "code" blocked >>= assert "OS unlock was mistaken for quiescence" . (== ("PredecessorQuiescenceRequired" :: Text)) + resumed <- launch (child (serverEndpoint server) "concurrent_history" "1000000" "yes") + outcome "initialized" resumed + before <- field "position" first :: IO Text + after <- field "position" resumed :: IO Text + assert "Initialized ledger was rewritten" (before == after) + putStrLn "exclusive: two processes excluded; timeout typed; clean OS unlock still requires quiescence; same UUID/position retained" + ) + `finally` close + withFaultProxy server $ \proxy -> do + setFault proxy DelayFirstResponse + (_, output, _, process) <- createProcess (child (proxyEndpoint proxy) "crashed_history" "1000000" "yes") {std_out = CreatePipe} + ready <- timeout 5000000 (waitForPost proxy) + assert "Crash test owner never acquired" (isJust ready) + terminateProcess process + void (waitForProcess process) + mapM_ hClose output + blocked <- launch (child (serverEndpoint server) "crashed_history" "1000000" "no") + outcome "access-rejected" blocked + field "code" blocked >>= assert "Process death silently authorized successor" . (== ("PredecessorQuiescenceRequired" :: Text)) + putStrLn "exclusive: process death releases OS lock but persistent marker blocks successor without explicit evidence" + -- Lease loss is checked at the actual socket write, including admission probes. + key <- right (Target.targetKey "ledger-tests/lease_lost") + credentials <- right (Target.basicCredentials "default" "test-password") + uuid <- right (Target.ledgerId (Text.pack ledger)) + audit <- right (Ledger.auditContext "lease test" "forced loss" ["owned fixture, no predecessor requests"]) + quiet <- right (Ledger.attestQuiescence key audit) + withFaultProxy server $ \proxy -> do + setFault proxy DelayFirstResponse + config <- Target.targetConfig key <$> right (Target.directEndpoint (proxyEndpoint proxy)) <*> right (Target.serverId actualId) <*> right (Target.ledgerNamespace "lease_lost") <*> pure Target.FirstUse <*> pure credentials <*> pure Target.defaultHttpLimits + alive <- newIORef True + provider <- right (Access.externallySerialized "controlled-test-lease" "complete isolated test target" 1000000 (const (pure ())) (const (pure ())) (const (readIORef alive))) + finished <- newEmptyMVar + bracket (forkFinally (Ledger.initializeLedger config provider (Ledger.authorizeInitialization key uuid audit) quiet) (putMVar finished)) killThread $ \_ -> do + ready <- timeout 5000000 (waitForPost proxy) + assert "Lease-loss probe was never submitted" (isJust ready) + writeIORef alive False + report <- takeMVar finished >>= either throwIO pure + assert "Lease loss did not stop command" (case Ledger.result report of Left (Ledger.RequestFailed Ledger.PositiveNonSubmission failure _) -> Ledger.code failure == Ledger.AdministrativeAccessLost; _ -> False) + requests <- proxyEvents proxy >>= convert :: IO [Value] + assert "A request escaped after lease loss during admission" (length requests == 1) + rows <- queryServer server "SELECT toUInt8(count()) AS count FROM system.databases WHERE name='lease_lost'" >>= convert :: IO [Value] + count <- case rows of [row] -> field "count" row :: IO Int; _ -> fail "Missing lease observation" + assert "Lost lease created a database" (count == 0) + putStrLn "exclusive: loss DURING connection admission prevents the next HTTP write; no initialization DDL; no server-side fencing claimed" diff --git a/test/RepairBoundaries.hs b/test/RepairBoundaries.hs new file mode 100644 --- /dev/null +++ b/test/RepairBoundaries.hs @@ -0,0 +1,137 @@ +module RepairBoundaries (repairBoundaryCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Repair qualified as Repair +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Monad (forM_, when) +import Data.Aeson (Value) +import Data.Text qualified as T +import Data.Text.IO qualified as TIO +import RunnerIntegration (assert, field, fixture, fromJson, planFor, provider, right) +import System.FilePath (()) + +checkpoint :: R.RunReport -> IO (L.LedgerPosition, L.AttemptId, [R.StepSummary]) +checkpoint (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just position) _ _ _ _) [R.MigrationSummary _ _ (Just attempt) steps] _ _) = pure (position, attempt, steps) +checkpoint report = fail ("Missing active checkpoint " ++ show report) + +selected :: [R.StepSummary] -> IO R.StepAttempt +selected [_, R.StepSummary "insert" _ _ (Just attempt) _] = pure attempt +selected _ = fail "Missing selected attempt" + +count :: FaultProxy -> IO Int +count proxy = do + rows <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") rows :: IO [Maybe T.Text] + pure (length (filter (== Just "application-sql") stages)) + +repairBoundaryCases :: Server -> IO () +repairBoundaryCases server = do + access <- provider + audit <- right (L.auditContext "repair boundary fixture" "independently controlled effects and request drain" ["parent owns fixture, clients ended, all predecessor requests drained"]) + -- All four authorizations, lost response and interrupted immediately after ack. + forM_ [(action, lost) | action <- ["resume", "retry", "accept", "finalize"], lost <- [False, True]] $ \(action, lost) -> withFaultProxy server $ \proxy -> do + let label = "repair_boundary_" <> action <> if lost then "_lost" else "_callback" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + let stop (E.RunEvent phase _ _ _ step _ _) = + when ((action == "resume" && phase == E.BeforeStepStart && step == Just "insert") || (action == "retry" && phase == E.BeforeSql && step == Just "insert") || (action == "accept" && phase == E.BeforeStepSuccess && step == Just "insert") || (action == "finalize" && phase == E.BeforeMigrationComplete)) (fail "owned interruption") + initial <- + Run.up (Run.mutationContext config access stop E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan >>= \case + Left (R.RunFailure (R.CallbackFailed _) report []) -> pure report + other -> fail ("Did not establish repair state " ++ show other) + (position, attempt, steps) <- checkpoint initial + quiet <- right (L.attestQuiescence key audit) + evidence <- right (Repair.recoveryEvidence audit quiet []) + request <- + right =<< case action of + "resume" -> pure (Repair.resumeRequest position attempt evidence) + "retry" -> do step <- selected steps; pure (Repair.retryStepRequest position attempt step evidence) + "accept" -> do step <- selected steps; pure (Repair.acceptStepRequest position attempt step evidence) + _ -> pure (Repair.finalizeRequest position attempt evidence) + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + let interrupted (E.RunEvent phase _ _ _ _ _ _) = when (phase == E.RecoveryAuthorized && not lost) (fail "authorization observer stopped") + context = Run.mutationContext bound access interrupted E.redactedDiagnostics + quietContext = Run.mutationContext bound access E.quietEvents E.redactedDiagnostics + before <- count proxy + when lost (setLedgerFault proxy RecoveryAuthorization DropResponse) + stopped <- Repair.repair context plan request + receipt <- case stopped of + Left (R.RunFailure (R.LedgerRequestFailed L.UncertainSubmission _ (Just receipt)) _ []) | lost -> pure [receipt] + Left (R.RunFailure (R.CallbackFailed _) _ []) | not lost -> pure [] + _ -> fail ("Wrong authorization interruption " ++ show stopped) + assert "Unacknowledged/interrupted authorization executed SQL" . (== before) =<< count proxy + setFault proxy Pass + observed <- I.status (I.Guarded access) bound plan >>= right + let R.InspectionReport snapshot current _ = observed + R.HistoryReport _ _ _ currentPosition _ events _ _ = snapshot + requests <- proxyEvents proxy + TIO.writeFile (serverLogDirectory server T.unpack label ++ ".trace") (T.pack (show stopped) <> "\n" <> T.pack (show requests) <> "\n" <> T.pack (show observed)) + assert "Authorization absent from actual history" (case reverse events of R.HistoryEvent _ _ _ _ _ _ _ _ _ (R.RecoveryAuthorization {}) : _ -> True; _ -> False) + ordinary <- Run.up quietContext (Run.upOptions audit Nothing) plan + assert "Ordinary up consumed standing authorization" (case ordinary of Left (R.RunFailure (R.LedgerFailed L.UnresolvedHistory) _ _) -> True; _ -> False) + stale <- Repair.repair quietContext plan request + assert "Interrupted request was reusable without fresh state" (case stale of Left (R.RunFailure (R.StalePosition _ _) _ _) -> True; _ -> False) + newPosition <- maybe (fail "Missing new position") pure currentPosition + (newAttempt, newSteps) <- case current of [R.MigrationSummary _ _ (Just value) values] -> pure (value, values); _ -> fail "Missing new active attempt" + assert "Authorization did not change attempt" (newAttempt /= attempt) + freshEvidence <- right (Repair.recoveryEvidence audit quiet receipt) + fresh <- case action of + "accept" -> do + step <- selected newSteps + -- Even though active attempt changed, the selected query's original + -- owner remains the original attempt; substituting the new one fails. + let R.StepAttempt _ name query = step + wrong <- right (Repair.acceptStepRequest newPosition newAttempt (R.StepAttempt newAttempt name query) freshEvidence) + rejected <- Repair.repair quietContext plan wrong + assert "Selected old query accepted with fabricated attempt owner" (case rejected of Left (R.RunFailure (R.StepAttemptMismatch _) _ _) -> True; _ -> False) + right (Repair.acceptStepRequest newPosition newAttempt step freshEvidence) + "finalize" -> right (Repair.finalizeRequest newPosition newAttempt freshEvidence) + _ -> right (Repair.resumeRequest newPosition newAttempt freshEvidence) + recovered <- Repair.repair quietContext plan fresh >>= right + if action == "accept" + then do + (finalPosition, finalAttempt, _) <- checkpoint recovered + finalize <- right (Repair.finalizeRequest finalPosition finalAttempt freshEvidence) + _ <- Repair.repair quietContext plan finalize >>= right + pure () + else pure () + _ <- I.verify I.Observe bound (I.verifyOptions True) plan >>= right + assert "Recovery replayed completed steps or executed acceptance SQL" . (== 2) =<< count proxy + putStrLn ("repair authorization " ++ T.unpack action ++ ": lost response=" ++ show lost ++ "; ordinary restart blocked; fresh request succeeds without replay") + -- Acceptance and finalization have their own dependent append acknowledgements. + forM_ [(StepAcceptance, True), (StepAcceptance, False), (MigrationCompletion, True)] $ \(stage, lost) -> withFaultProxy server $ \proxy -> do + let label = if stage == StepAcceptance then (if lost then "accept_append_lost" else "accepted_callback") else "finalize_append_lost" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + let stop (E.RunEvent phase _ _ _ step _ _) = when ((stage == StepAcceptance && phase == E.BeforeStepSuccess && step == Just "insert") || (stage == MigrationCompletion && phase == E.BeforeMigrationComplete)) (fail "owned interruption") + initial <- Run.up (Run.mutationContext config access stop E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan >>= \case Left (R.RunFailure _ report _) -> pure report; _ -> fail "Missing initial interruption" + (position, attempt, steps) <- checkpoint initial + quiet <- right (L.attestQuiescence key audit) + evidence <- right (Repair.recoveryEvidence audit quiet []) + request <- if stage == StepAcceptance then selected steps >>= \step -> right (Repair.acceptStepRequest position attempt step evidence) else right (Repair.finalizeRequest position attempt evidence) + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + when lost (setLedgerFault proxy stage DropResponse) + let acceptedSink (E.RunEvent phase _ _ _ _ _ _) = when (not lost && phase == E.StepAccepted) (fail "accepted callback") + result <- Repair.repair (Run.mutationContext bound access acceptedSink E.redactedDiagnostics) plan request + assert + "Dependent repair append/observer failure was hidden" + ( case result of + Left (R.RunFailure (R.LedgerRequestFailed L.UncertainSubmission _ (Just _)) _ []) -> lost + Left (R.RunFailure (R.CallbackFailed _) report []) -> not lost && case report of R.RunReport _ _ _ _ _ [R.MigrationSummary _ R.InProgress _ [_, R.StepSummary _ _ R.StepAccepted _ _]] _ _ -> True; _ -> False + _ -> False + ) + assert "Acceptance/finalize sent application SQL" . (== 2) =<< count proxy + setFault proxy Pass + observed <- I.history I.Observe bound >>= right + assert "Repair append did not commit once" (case observed of R.HistoryReport _ _ _ _ _ events _ _ -> case reverse events of R.HistoryEvent _ _ _ _ _ _ _ _ _ body : _ -> R.historyKind body == if stage == StepAcceptance then "step-accepted" else "migration-completed"; _ -> False) + putStrLn ("repair dependent append loss: " ++ show stage ++ "; lost response=" ++ show lost ++ "; no SQL submitted") diff --git a/test/RepairIntegration.hs b/test/RepairIntegration.hs new file mode 100644 --- /dev/null +++ b/test/RepairIntegration.hs @@ -0,0 +1,199 @@ +module RepairIntegration (repairEarlyCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Event qualified as Event +import ClickHouseMigrate.Inspection qualified as Inspect +import ClickHouseMigrate.Ledger qualified as Ledger +import ClickHouseMigrate.Repair qualified as Repair +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Monad (forM_, when) +import Data.Aeson (Value) +import Data.IORef (modifyIORef', newIORef, readIORef) +import Data.List.NonEmpty (NonEmpty (..)) +import Data.Set qualified as Set +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Encoding +import RunnerIntegration (assert, field, fixture, fromJson, planFor, provider, right, scalar) + +planWithTail :: Text.Text -> IO MigrationPlan +planWithTail label = do + create <- right (sqlStep "create" (Encoding.encodeUtf8 ("CREATE TABLE default." <> label <> " (id UInt8, secret String) ENGINE=MergeTree ORDER BY id"))) + insert <- right (sqlStep "insert" (Encoding.encodeUtf8 ("INSERT INTO default." <> label <> " VALUES (7,'SQL_SECRET_MARKER')"))) + later <- right (sqlStep "later" (Encoding.encodeUtf8 ("ALTER TABLE default." <> label <> " ADD COLUMN later UInt8"))) + one <- right (sqlMigration "one" (create :| [insert, later])) + next <- right (sqlStep "next" (Encoding.encodeUtf8 ("INSERT INTO default." <> label <> " VALUES (9,'next',0)"))) + two <- right (sqlMigration "two" (next :| [])) + component <- right (migrationComponent "app" Set.empty (one :| [two])) + right (migrationPlan (component :| [])) + +checkpoint :: R.RunReport -> IO (Ledger.LedgerPosition, Ledger.AttemptId, [R.StepSummary]) +checkpoint (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just position) _ _ _ _) (R.MigrationSummary _ _ (Just attempt) steps : _) _ _) = pure (position, attempt, steps) +checkpoint _ = fail "Missing repair checkpoint/attempt" + +selected :: [R.StepSummary] -> IO R.StepAttempt +selected (_ : R.StepSummary "insert" _ _ (Just attempt) _ : _) = pure attempt +selected _ = fail "Missing exact second step attempt" + +evidenceFor :: Text.Text -> IO Repair.RecoveryEvidence +evidenceFor label = do + key <- right (targetKey ("runner-tests/" <> label)) + audit <- right (Ledger.auditContext "recovery operator" "independently established safety/effects for this controlled operation" ["owned fixture: client ended and every controlled server request completed"]) + quiet <- right (Ledger.attestQuiescence key audit) + right (Repair.recoveryEvidence audit quiet []) + +-- Public consumer: construct all four variants before expanding crash cases. +compileRequests :: Ledger.LedgerPosition -> Ledger.AttemptId -> R.StepAttempt -> Repair.RecoveryEvidence -> IO () +compileRequests position attempt step evidence = do + _ <- right (Repair.resumeRequest position attempt evidence) + _ <- right (Repair.retryStepRequest position attempt step evidence) + _ <- right (Repair.acceptStepRequest position attempt step evidence) + _ <- right (Repair.finalizeRequest position attempt evidence) + assert "Attempt token cannot roundtrip" (Ledger.parseAttemptId (Ledger.attemptIdText attempt) == Right attempt) + +applicationCount :: FaultProxy -> IO Int +applicationCount proxy = do + rows <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") rows :: IO [Maybe Text.Text] + pure (length (filter (== Just "application-sql") stages)) + +repairEarlyCases :: Server -> IO () +repairEarlyCases server = do + oversizedRecovery server + access <- provider + audit <- right (Ledger.auditContext "runner fixture" "explicit first use" ["owned server; parent serializes whole job"]) + withFaultProxy server $ \proxy -> do + let label = "repair_retry" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planWithTail label + let stop (Event.RunEvent phase _ _ _ step _ _) = when (phase == Event.BeforeSql && step == Just "insert") (fail "synthetic callback failure SQL_SECRET_MARKER") + stopped <- Run.up (Run.mutationContext config access stop Event.redactedDiagnostics) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) plan + initial <- case stopped of Left failure@(R.RunFailure (R.CallbackFailed _) report _) -> assert "Callback exception leaked SQL" (not ("SQL_SECRET_MARKER" `Text.isInfixOf` Text.pack (show failure))) >> pure report; _ -> fail "Before-SQL callback did not stop" + (position, attempt, steps) <- checkpoint initial + step <- selected steps + evidence <- evidenceFor label + compileRequests position attempt step evidence + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + let context = Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics + unsafeResume <- right (Repair.resumeRequest position attempt evidence) >>= Repair.repair context plan + assert "Resume replayed uncertain step" (case unsafeResume of Left (R.RunFailure (R.RepairNotAllowed R.UncertainStepRemains) _ _) -> True; _ -> False) + retry <- right (Repair.retryStepRequest position attempt step evidence) + retried <- Repair.repair context plan retry >>= right + (after, active, checkpoints) <- checkpoint retried + assert "Retry did not create a fresh attempt" (active /= attempt && after /= position) + assert "Retry continued later steps" (case checkpoints of [R.StepSummary "create" _ R.StepObservedSuccess _ _, R.StepSummary "insert" _ R.StepObservedSuccess (Just fresh) _, R.StepSummary "later" _ R.StepPending _ _] -> fresh /= step; _ -> False) + count <- scalar server ("SELECT toUInt32(count()) AS value FROM default." <> label) + assert "Retry SQL not submitted exactly once" (count == 1) + assert "Retry submitted other SQL" . (== 2) =<< applicationCount proxy + stale <- Repair.repair context plan retry + assert "Old retry request accepted after checkpoint moved" (case stale of Left (R.RunFailure (R.StalePosition _ _) _ _) -> True; _ -> False) + oversized <- right (Ledger.auditContext "operator" "combined recovery audits" (replicate 16 (Text.replicate 2048 "x"))) + oversizedQuiet <- right (Ledger.attestQuiescence key oversized) + oversizedEvidence <- right (Repair.recoveryEvidence oversized oversizedQuiet []) + refusedResume <- right (Repair.resumeRequest after active oversizedEvidence) >>= Repair.repair context plan + assert "resume authorized oversized metadata" (case refusedResume of Left (R.RunFailure (R.RecoveryEventTooLarge R.RecoveryAuthorizationEvent) _ _) -> True; _ -> False) + resumed <- right (Repair.resumeRequest after active evidence) >>= Repair.repair context plan >>= right + let R.RunReport _ _ _ _ _ migrations _ _ = resumed + assert "Resume continued another migration" (case migrations of [R.MigrationSummary _ R.Applied _ _, R.MigrationSummary _ R.Pending _ _] -> True; _ -> False) + assert "Resume replayed completed checkpoints" . (== 3) =<< applicationCount proxy + columns <- scalar server ("SELECT toUInt32(count()) AS value FROM system.columns WHERE database='default' AND table='" <> label <> "' AND name='later'") + assert "Resume failed to execute pending suffix" (columns == 1) + compatible <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions False) plan >>= right + assert "Pending tail is not compatible" (case compatible of R.VerificationReport _ False -> True; _ -> False) + current <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions True) plan + assert "Require-current accepted pending tail" (case current of Left (R.InspectionFailure R.VerificationNotCurrent _ _) -> True; _ -> False) + _ <- Run.up context (Run.upOptions audit Nothing) plan >>= right + assert "Following ordinary up did not execute only the next migration" . (== 4) =<< applicationCount proxy + putStrLn "repair: retry only selected step; fresh query/attempt; stale request refused; resume preserves checkpoints and stops at selected migration" + withFaultProxy server $ \proxy -> do + let label = "repair_accept" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + let stop (Event.RunEvent phase _ _ _ step _ _) = when (phase == Event.BeforeStepSuccess && step == Just "insert") (fail "stop after SQL acknowledgement") + stopped <- Run.up (Run.mutationContext config access stop Event.redactedDiagnostics) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) plan + initial <- case stopped of Left (R.RunFailure (R.CallbackFailed _) report _) -> pure report; _ -> fail "After-SQL callback did not stop" + (position, attempt, steps) <- checkpoint initial + step <- selected steps + assert "Expected effect not independently present" . (== 1) =<< scalar server ("SELECT toUInt32(count()) AS value FROM default." <> label <> " WHERE id=7") + evidence <- evidenceFor label + compileRequests position attempt step evidence + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + let context = Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics + accept <- right (Repair.acceptStepRequest position attempt step evidence) + accepted <- Repair.repair context plan accept >>= right + (after, active, checkpoints) <- checkpoint accepted + assert "Accepted completion conflated with observed success" (case checkpoints of [R.StepSummary _ _ R.StepObservedSuccess _ _, R.StepSummary _ _ R.StepAccepted _ (Just (R.EvidenceSummary Ledger.OperatorAttestation _))] -> True; _ -> False) + assert "Accept executed SQL or silently finalized" . (== 2) =<< applicationCount proxy + let R.RunReport _ _ _ _ _ migrations _ _ = accepted + assert "Accept implicitly completed migration" (case migrations of [R.MigrationSummary _ R.InProgress _ _] -> True; _ -> False) + stale <- Repair.repair context plan accept + assert "Repeated accept was not stale" (case stale of Left (R.RunFailure (R.StalePosition _ _) _ _) -> True; _ -> False) + final <- right (Repair.finalizeRequest after active evidence) >>= Repair.repair context plan >>= right + assert "Finalize submitted application SQL" . (== 2) =<< applicationCount proxy + _ <- Inspect.verify (Inspect.Guarded access) bound (Inspect.verifyOptions True) plan >>= right + let R.RunReport _ _ _ _ _ finalMigrations _ _ = final + assert "Finalization erased acceptance evidence" (case finalMigrations of [R.MigrationSummary _ R.Applied _ [_, R.StepSummary _ _ R.StepAccepted _ _]] -> True; _ -> False) + putStrLn "repair: accept records attestation without SQL/continuation; explicit finalize submits no SQL and preserves accepted-vs-observed evidence" + +-- REV-1/F3: authorization must not precede acceptance envelope validation. +oversizedRecovery :: Server -> IO () +oversizedRecovery server = withFaultProxy server $ \proxy -> do + access <- provider + audit <- right (Ledger.auditContext "review fixture" "before SQL interruption" ["owned server"]) + ledger <- newLedgerId + let label = "repair_oversized" + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + key <- right (targetKey ("runner-tests/" <> label)) + create <- right (sqlStep "create" "CREATE TABLE default.repair_oversized (id UInt8) ENGINE=Memory") + one <- right (sqlMigration "one" (create :| [])) + component <- right (migrationComponent "app" Set.empty (one :| [])) + plan <- right (migrationPlan (component :| [])) + let stop (Event.RunEvent phase _ _ _ _ _ _) = when (phase == Event.BeforeSql) (fail "review interruption") + stopped <- Run.up (Run.mutationContext config access stop Event.redactedDiagnostics) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) plan + initial <- case stopped of Left (R.RunFailure (R.CallbackFailed _) report _) -> pure report; _ -> fail "Expected interrupted single step" + (position, attempt, steps) <- checkpoint initial + step <- case steps of [R.StepSummary _ _ _ (Just exactStep) _] -> pure exactStep; _ -> fail "Missing selected step" + quiet <- right (Ledger.attestQuiescence key audit) + trace <- newIORef [] + let sink (Event.RunEvent phase _ _ _ _ _ _) = modifyIORef' trace (++ [phase]) + context = Run.mutationContext bound access sink Event.redactedDiagnostics + -- Authorization fits; duplicated accepted references do not. + big <- right (Ledger.auditContext "review" "oversized accepted evidence" (replicate 24 (Text.replicate 2000 "x"))) + evidence <- right (Repair.recoveryEvidence big quiet []) + request <- right (Repair.acceptStepRequest position attempt step evidence) + before <- Inspect.history Inspect.Observe bound >>= right + rejected <- Repair.repair context plan request + assert "wrong size rejection" (case rejected of Left (R.RunFailure (R.RecoveryEventTooLarge R.RecoveryAcceptanceEvent) _ _) -> True; _ -> False) + after <- Inspect.history Inspect.Observe bound >>= right + assert "size rejection advanced original position/attempt" (before == after) + assert "rejected request submitted SQL" . (== 0) =<< applicationCount proxy + phases <- readIORef trace + assert "preparation failure emitted recovery acknowledgement" (Event.RecoveryAuthorized `notElem` phases && Event.StepAccepted `notElem` phases) + -- Combined outer/quiescence evidence must reject all action variants. + medium <- right (Ledger.auditContext "review" "two valid audits exceeding combined envelope" (replicate 16 (Text.replicate 2048 "x"))) + largeQuiet <- right (Ledger.attestQuiescence key medium) + combined <- right (Repair.recoveryEvidence medium largeQuiet []) + forM_ [Repair.retryStepRequest position attempt step combined, Repair.acceptStepRequest position attempt step combined] $ \input -> do + result <- right input >>= Repair.repair context plan + assert "oversized authorization not typed" (case result of Left (R.RunFailure (R.RecoveryEventTooLarge R.RecoveryAuthorizationEvent) _ _) -> True; _ -> False) + hugeQuietAudit <- right (Ledger.auditContext "operator" "oversized quiescence only" (replicate 32 (Text.replicate 2048 "x"))) + hugeQuiet <- right (Ledger.attestQuiescence key hugeQuietAudit) + hugeEvidence <- right (Repair.recoveryEvidence audit hugeQuiet []) + refusedQuiet <- right (Repair.acceptStepRequest position attempt step hugeEvidence) >>= Repair.repair context plan + assert "oversized quiescence not rejected" (case refusedQuiet of Left (R.RunFailure (R.RecoveryEventTooLarge R.RecoveryAuthorizationEvent) _ _) -> True; _ -> False) + replacement <- right (Repair.recoveryEvidence audit quiet []) >>= right . Repair.acceptStepRequest position attempt step + accepted <- Repair.repair context plan replacement >>= right + (nextPosition, nextAttempt, _) <- checkpoint accepted + final <- right (Repair.finalizeRequest nextPosition nextAttempt combined) >>= Repair.repair context plan + assert "finalize authorized oversized metadata" (case final of Left (R.RunFailure (R.RecoveryEventTooLarge R.RecoveryAuthorizationEvent) _ _) -> True; _ -> False) + _ <- right (Repair.finalizeRequest nextPosition nextAttempt =<< Repair.recoveryEvidence audit quiet []) >>= Repair.repair context plan >>= right + _ <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions True) plan >>= right + assert "accept/finalize submitted application SQL" . (== 0) =<< applicationCount proxy + putStrLn "recovery size: oversized acceptance preserves (3,3) events, position and attempt; combined audits reject authorization; replacement against original position accepts and finalizes without SQL" diff --git a/test/ReviewRemediation.hs b/test/ReviewRemediation.hs new file mode 100644 --- /dev/null +++ b/test/ReviewRemediation.hs @@ -0,0 +1,225 @@ +-- Native review regressions through public commands on one disposable server. +module ReviewRemediation (remediationCases) where + +import ClickHouseMigrate +import ClickHouseMigrate.Adoption qualified as A +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Repair qualified as Repair +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Concurrent (forkFinally, killThread, newEmptyMVar, putMVar, takeMVar, threadDelay) +import Control.Exception (SomeAsyncException, fromException) +import Control.Monad (forM_, void, when) +import Data.Aeson (Value) +import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE +import Data.Maybe (isJust) +import Data.Set qualified as Set +import Data.Text qualified as T +import RunnerIntegration (assert, field, fixture, fixtureWithLimits, fromJson, provider, right, scalar) +import System.Timeout (timeout) + +remediationCases :: Server -> IO () +remediationCases server = do + access <- provider + audit <- right (L.auditContext "review fixture" "ordinary control evidence" ["owned server"]) + big <- right (L.auditContext "review fixture" "capacity reproduction" (replicate 24 (T.replicate 2000 "x"))) + let makeMigration name count sql = do + steps <- mapM (\n -> right (sqlStep ("s" <> T.pack (show n)) sql)) [1 .. count :: Int] + right (sqlMigrationFromList name steps) + context config = Run.mutationContext config access E.quietEvents E.redactedDiagnostics + run config evidence auth plan = Run.up (context config) (Run.upOptions evidence auth) plan >>= right + -- The exact original 12-step / ~1.3 MiB reproduction with default limits. + smallLedger <- newLedgerId + small <- fixture server "stream_original" (serverEndpoint server) FirstUse + smallBound <- fixture server "stream_original" (serverEndpoint server) (ExistingLedger smallLedger) + original <- makeMigration "large" 12 "CREATE TABLE IF NOT EXISTS default.stream_original (id UInt8) ENGINE=Memory" + originalComponent <- right (migrationComponent "capacity" Set.empty (original :| [])) + originalPlan <- right (migrationPlan (originalComponent :| [])) + _ <- run small big (Just (L.authorizeInitialization (configuredTargetKey small) smallLedger audit)) originalPlan + originalHistory <- I.history I.Observe smallBound >>= right + assert "original complete history" (case originalHistory of R.HistoryReport _ _ _ _ _ events _ False -> length events == 27; _ -> False) + largeAdoption server + putStrLn "stream: original 12-step large-audit history returns all 27 events with default limits" + withFaultProxy server $ \proxy -> do + ledger <- newLedgerId + let label = "review_combined" + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + a1 <- makeMigration "one" 1 "CREATE TABLE default.review_combined (id UInt8) ENGINE=Memory" + b1 <- makeMigration "one" 1 "CREATE TABLE default.review_combined_b (id UInt8) ENGINE=Memory" + upgrades <- mapM (\n -> makeMigration ("upgrade-" <> T.pack (show n)) 18 "INSERT INTO default.review_combined VALUES (1)") [1 .. 10 :: Int] + a <- right (migrationComponent "a" Set.empty (a1 :| [])) + aAll <- right (migrationComponent "a" Set.empty (a1 :| upgrades)) + b <- right (migrationComponent "b" (Set.singleton "a") (b1 :| [])) + initial <- right (migrationPlan (a :| [b])) + let ident c m = migrationIdFromNames <$> componentName c <*> migrationName m + order <- mapM (right . uncurry ident) ([("a", "one"), ("b", "one")] ++ [("a", "upgrade-" <> T.pack (show n)) | n <- [1 .. 10 :: Int]]) + full <- right (scheduledMigrationPlanFromList [aAll, b] order) + _ <- run config audit (Just (L.authorizeInitialization (configuredTargetKey config) ledger audit)) initial + _ <- run bound big Nothing full + _ <- I.verify I.Observe bound (I.verifyOptions True) full >>= right + observed <- I.history I.Observe bound >>= right + let R.HistoryReport _ _ _ position _ events _ unresolved = observed + assert "large complete event count" (length events == 389 && not unresolved) + records <- proxyEvents proxy >>= fromJson :: IO [Value] + historyRecords <- filterMHistory records + sizes <- mapM (field "responseBytes") historyRecords :: IO [Int] + rows <- mapM (field "maxRowBytes") historyRecords :: IO [Int] + assert "fixture did not exceed old maximum" (maximum sizes > 16777216) + assert "individual row exceeded default bound" (maximum rows <= 1048576) + largeSnapshot <- I.status I.Observe bound full >>= right + largePosition <- maybe (fail "Missing large history position") pure position + quietForAdoption <- right (L.attestQuiescence (configuredTargetKey bound) audit) + source <- right (A.legacyRow "a.sql" "2020-01-01T00:00:00Z" Nothing) + let firstView = NE.head (inspectSchedule full) + mapped <- right (A.attestedMapping "a.sql" "a" "one" (migrationChecksumHex (migrationChecksum firstView)) ["reviewed native prefix"]) + nonfresh <- right (A.adoptionRequest (configuredTargetKey bound) (L.ExpectPosition largePosition) Nothing quietForAdoption audit "new-source" "immutable-source" (source :| []) (mapped :| [])) + assert "adoption accepted large native history" (case A.planAdoption full largeSnapshot nonfresh of Left A.AdoptionNativeConflict -> True; _ -> False) + _ <- run bound audit Nothing full + assert "large schedule effects" . (== 180) =<< scalar server "SELECT toUInt32(count()) AS value FROM default.review_combined" + putStrLn ("stream: wire bytes=" ++ show (maximum sizes) ++ "; max row=" ++ show (maximum rows) ++ "; events=" ++ show (length events) ++ "; status/history/currentness/no-op up pass") + -- Exact physical duplicates remain transport-visible and project identically. + void (queryServer server "INSERT INTO review_combined_history.events SELECT * FROM review_combined_history.events ORDER BY sequence LIMIT 1") + duplicate <- I.history I.Observe bound >>= right + assert "exact duplicates changed logical position" (case duplicate of R.HistoryReport _ _ _ after _ allEvents _ _ -> after == position && length allEvents == length events) + -- Pending up and explicit recovery consume the same >16 MiB observation. + pending <- makeMigration "pending" 1 "INSERT INTO default.review_combined VALUES (2)" + tailMigration <- makeMigration "tail" 1 "INSERT INTO default.review_combined VALUES (3)" + pendingComponent <- right (migrationComponent "a" Set.empty (a1 :| (upgrades ++ [pending]))) + completeComponent <- right (migrationComponent "a" Set.empty (a1 :| (upgrades ++ [pending, tailMigration]))) + pendingId <- right (ident "a" "pending") + tailId <- right (ident "a" "tail") + pendingPlan <- right (scheduledMigrationPlanFromList [pendingComponent, b] (order ++ [pendingId])) + completePlan <- right (scheduledMigrationPlanFromList [completeComponent, b] (order ++ [pendingId, tailId])) + _ <- run bound audit Nothing pendingPlan + assert "large history pending up" . (== 181) =<< scalar server "SELECT toUInt32(count()) AS value FROM default.review_combined" + let stop (E.RunEvent phase _ _ migration _ _ _) = when (phase == E.BeforeSql && migration == Just "tail") (fail "combined gate interruption") + stopped <- Run.up (Run.mutationContext bound access stop E.redactedDiagnostics) (Run.upOptions audit Nothing) completePlan + (originalPosition, attempt, selected) <- case stopped of + Left (R.RunFailure (R.CallbackFailed _) (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just current) _ _ _ _) migrations _ _) _) -> + case last migrations of R.MigrationSummary _ _ (Just attempt) [R.StepSummary _ _ _ (Just selected) _] -> pure (current, attempt, selected); _ -> fail "Missing combined selected step" + _ -> fail "Combined tail did not stop" + quiet <- right (L.attestQuiescence (configuredTargetKey bound) audit) + oversized <- right (Repair.recoveryEvidence big quiet []) >>= right . Repair.acceptStepRequest originalPosition attempt selected + beforeRecovery <- mutationCount proxy + rejected <- Repair.repair (context bound) completePlan oversized + assert "combined size error" (case rejected of Left (R.RunFailure (R.RecoveryEventTooLarge R.RecoveryAcceptanceEvent) _ _) -> True; _ -> False) + afterRecovery <- mutationCount proxy + assert "combined size rejection mutated" (beforeRecovery == afterRecovery) + unchanged <- I.history I.Observe bound >>= right + assert "combined size rejection changed position" (case unchanged of R.HistoryReport _ _ _ current _ _ _ _ -> current == Just originalPosition) + evidence <- right (Repair.recoveryEvidence audit quiet []) + request <- right (Repair.acceptStepRequest originalPosition attempt selected evidence) + accepted <- Repair.repair (context bound) completePlan request >>= right + (acceptedPosition, acceptedAttempt) <- case accepted of + R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just current) _ _ _ _) migrations _ _ -> + case last migrations of R.MigrationSummary _ _ (Just currentAttempt) _ -> pure (current, currentAttempt); _ -> fail "Missing accepted attempt" + _ -> fail "Missing accepted position" + finalize <- right (Repair.finalizeRequest acceptedPosition acceptedAttempt evidence) + _ <- Repair.repair (context bound) completePlan finalize >>= right + _ <- I.verify I.Observe bound (I.verifyOptions True) completePlan >>= right + putStrLn "combined: large-history pending up executes once; interrupted tail rejects oversized acceptance without position change, accepts replacement and explicitly finalizes" + forM_ [TruncateResponse, AppendLateError, OversizedHistoryRow, HistoryGap, HistoryIdentityConflict, HistoryDuplicateConflict] $ \mode -> do + before <- mutationCount proxy + setLedgerFault proxy HistoryRead mode + result <- Run.up (context bound) (Run.upOptions audit Nothing) completePlan + let expected = case result of + Left (R.RunFailure (R.LedgerFailed problem) _ _) -> case mode of + HistoryGap -> problem == L.SequenceGap + HistoryIdentityConflict -> problem == L.EventIdentityConflict + HistoryDuplicateConflict -> problem == L.EventIdentityConflict + _ -> False + Left (R.RunFailure (R.LedgerRequestFailed requestEvidence (L.TransportError code _ _) Nothing) _ _) -> case mode of + TruncateResponse -> requestEvidence == L.UncertainSubmission && code == L.ResponseInterrupted + AppendLateError -> requestEvidence == L.ObservedFailure && code == L.ServerRejected + OversizedHistoryRow -> requestEvidence == L.UncertainSubmission && code == L.ResponseTooLarge + _ -> False + _ -> False + let observedError = case result of Left (R.RunFailure problem _ _) -> show problem; Right _ -> "success" + assert ("wrong stream failure: " ++ show mode ++ ": " ++ observedError) expected + after <- mutationCount proxy + assert "failed stream allowed mutation" (before == after) + setFault proxy Pass + setLedgerFault proxy HistoryRead DelayHistoryBody + limits <- right (httpLimits 500000 1048576) + short <- fixtureWithLimits limits server label (proxyEndpoint proxy) (ExistingLedger ledger) + expired <- I.history I.Observe short + assert "stream deadline did not stop read" (case expired of Left (R.InspectionFailure (R.LedgerRequestFailed _ (L.TransportError L.DeadlineExpired _ _) _) _ _) -> True; _ -> False) + beforeChild <- historyPrefixCount proxy + done <- newEmptyMVar + child <- forkFinally (I.history I.Observe bound) (putMVar done) + -- The delay occurs after headers and half the body have been flushed. + let waitForRead remaining = do + count <- historyPrefixCount proxy + if count > beforeChild then pure () else if remaining == (0 :: Int) then fail "Child never consumed a streamed body prefix" else threadDelay 10000 >> waitForRead (remaining - 1) + waitForRead 100 + killThread child + cancelled <- timeout 3000000 (takeMVar done) + assert "stream cancellation converted to normal result" (case cancelled of Just (Left exception) -> isJust (fromException exception :: Maybe SomeAsyncException); _ -> False) + setFault proxy Pass + _ <- I.verify I.Observe bound (I.verifyOptions True) completePlan >>= right + putStrLn "stream: framed truncation, trailing error, oversized row and late gap/identity/duplicate conflicts discard prefix and allow no writes" + where + filterMHistory records = do + stages <- mapM (field "stage") records :: IO [Maybe T.Text] + pure [record | (record, Just "history-read") <- zip records stages] + +mutationCount :: FaultProxy -> IO Int +mutationCount proxy = do + records <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") records :: IO [Maybe T.Text] + pure (length [() | Just stage <- stages, stage /= "history-read", stage /= "read-only"]) + +-- The adoption consumer must also read a large interrupted/committed batch. +largeAdoption :: Server -> IO () +largeAdoption server = withFaultProxy server $ \proxy -> do + access <- provider + ledger <- newLedgerId + let label = "stream_adoption" + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + audit <- right (L.auditContext "adoption reviewer" "large reviewed batch" (replicate 24 (T.replicate 2000 "x"))) + quiet <- right (L.attestQuiescence (configuredTargetKey config) audit) + step <- right (sqlStep "create" "CREATE TABLE IF NOT EXISTS default.stream_adoption (id UInt8) ENGINE=Memory") + migrations <- mapM (\n -> right (sqlMigration ("m-" <> T.pack (show n)) (step :| []))) [1 .. 360 :: Int] + component <- right (migrationComponentFromList "app" Set.empty migrations) + plan <- right (migrationPlan (component :| [])) + let views = NE.toList (inspectSchedule plan) + names = ["source-" <> T.pack (show n) | n <- [1 .. 360 :: Int]] + rows <- mapM (\name -> right (A.legacyRow name "2020-01-01T00:00:00Z" Nothing)) names + mappings <- mapM (\(name, view) -> right (A.attestedMapping name "app" (migrationNameText (migrationIdName (migrationId view))) (migrationChecksumHex (migrationChecksum view)) ["reviewed immutable fixture"])) (zip names views) + let build expected = case (rows, mappings) of + (r : rs, m : ms) -> right (A.adoptionRequest (configuredTargetKey config) expected (Just (L.authorizeInitialization (configuredTargetKey config) ledger audit)) quiet audit "large-fixture" "immutable-source" (r :| rs) (m :| ms)) + _ -> fail "Missing large adoption fixture" + context = Run.mutationContext bound access E.quietEvents E.redactedDiagnostics + stop (E.RunEvent phase _ _ _ _ _ _) = when (phase == E.BeforeAdoptionComplete) (fail "interrupt before commit") + observation <- I.status I.Observe config plan >>= right + request <- build (L.ExpectAbsent (configuredTargetKey config)) + adoption <- right (A.planAdoption plan observation request) + stopped <- A.applyAdoption (Run.mutationContext config access stop E.redactedDiagnostics) plan adoption + assert "large adoption interruption lost" (case stopped of Left (R.RunFailure (R.CallbackFailed _) _ _) -> True; _ -> False) + current <- I.status I.Observe bound plan >>= right + position <- case current of R.InspectionReport (R.HistoryReport _ _ _ (Just value) _ events _ True) _ _ | length events == 362 -> pure value; _ -> fail "Large interrupted adoption incomplete" + resumedRequest <- build (L.ExpectPosition position) + resumed <- right (A.planAdoption plan current resumedRequest) + _ <- A.resumeAdoption context plan resumed >>= right + _ <- A.applyAdoption context plan adoption >>= right + _ <- I.verify I.Observe bound (I.verifyOptions True) plan >>= right + records <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") records :: IO [Maybe T.Text] + sizes <- mapM (field "responseBytes") [record | (record, Just "history-read") <- zip records stages] :: IO [Int] + assert "adoption fixture below old maximum" (maximum sizes > 16777216) + assert "large adoption executed SQL/duplicated entries" (length (filter (== Just "adoption-entry") stages) == 360 && Just "application-sql" `notElem` stages) + putStrLn ("stream: adoption wire bytes=" ++ show (maximum sizes) ++ "; 360 entries; interrupted resume and committed identical read are complete and submit no SQL") + +historyPrefixCount :: FaultProxy -> IO Int +historyPrefixCount proxy = do + records <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") records :: IO [Maybe T.Text] + prefixes <- mapM (field "bodyPrefixSent") records :: IO [Bool] + pure (length [() | (Just "history-read", True) <- zip stages prefixes]) diff --git a/test/RunnerIntegration.hs b/test/RunnerIntegration.hs new file mode 100644 --- /dev/null +++ b/test/RunnerIntegration.hs @@ -0,0 +1,265 @@ +module RunnerIntegration (runnerEarlyCases, runnerValidationCases, fixture, fixtureWithLimits, provider, planFor, scalar, assert, right, field, fromJson) where + +import ClickHouseMigrate +import ClickHouseMigrate.Adoption qualified as Adoption +import ClickHouseMigrate.Event qualified as Event +import ClickHouseMigrate.ExclusiveAccess qualified as Access +import ClickHouseMigrate.Inspection qualified as Inspect +import ClickHouseMigrate.Ledger qualified as Ledger +import ClickHouseMigrate.Repair qualified as Repair +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Monad (forM_, unless, void, when) +import Data.Aeson (FromJSON, Value) +import Data.Aeson qualified as Aeson +import Data.Aeson.KeyMap qualified as KM +import Data.ByteString qualified as BS +import Data.IORef (modifyIORef', newIORef, readIORef, writeIORef) +import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE +import Data.Maybe (isJust) +import Data.Set qualified as Set +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Encoding +import System.FilePath (()) + +assert :: String -> Bool -> IO () +assert label predicate = unless predicate (fail ("runner early integration: " ++ label)) + +right :: (Show e) => Either e a -> IO a +right = either (fail . show) pure + +field :: (FromJSON a) => Aeson.Key -> Value -> IO a +field key value = case Aeson.fromJSON value :: Aeson.Result Aeson.Object of + Aeson.Success row -> case KM.lookup key row of + Just member -> case Aeson.fromJSON member of + Aeson.Success result -> pure result + _ -> fail "Invalid runner field" + _ -> fail "Missing runner field" + _ -> fail "Invalid runner object" + +fixture :: Server -> Text.Text -> Text.Text -> LedgerBinding -> IO TargetConfig +fixture = fixtureWithLimits defaultHttpLimits + +fixtureWithLimits :: HttpLimits -> Server -> Text.Text -> Text.Text -> LedgerBinding -> IO TargetConfig +fixtureWithLimits limits server label endpoint binding = do + rows <- queryServer server "SELECT toString(serverUUID()) AS server_id" >>= fromJson :: IO [Value] + identity <- case rows of [row] -> field "server_id" row; _ -> fail "Missing server identity" + targetConfig <$> right (targetKey ("runner-tests/" <> label)) <*> right (directEndpoint endpoint) <*> right (serverId identity) <*> right (ledgerNamespace (label <> "_history")) <*> pure binding <*> right (basicCredentials "default" "test-password") <*> pure limits + +fromJson :: (FromJSON a) => Value -> IO a +fromJson value = case Aeson.fromJSON value of Aeson.Success result -> pure result; _ -> fail "Invalid independent runner observation" + +provider :: IO Access.ExclusiveAccess +provider = right (Access.externallySerialized "owned-fixture-serialized-job" "parent owns target and admits no mutating successor before all predecessor requests drain" 1000000 (const (pure ())) (const (pure ())) (const (pure True))) + +planFor :: Text.Text -> Int -> IO MigrationPlan +planFor label value = do + create <- right (sqlStep "create" (Encoding.encodeUtf8 ("CREATE TABLE default." <> label <> " (id UInt8, secret String) ENGINE=MergeTree ORDER BY id"))) + insert <- right (sqlStep "insert" (Encoding.encodeUtf8 ("INSERT INTO default." <> label <> " VALUES (" <> Text.pack (show value) <> ",'SQL_SECRET_MARKER')"))) + migration <- right (sqlMigration "one" (create :| [insert])) + component <- right (migrationComponent "app" Set.empty (migration :| [])) + right (migrationPlan (component :| [])) + +scalar :: Server -> Text.Text -> IO Int +scalar server sql = do + rows <- queryServer server sql >>= fromJson :: IO [Value] + case rows of [row] -> field "value" row; _ -> fail "Invalid independent scalar" + +runnerEarlyCases :: Server -> IO () +runnerEarlyCases server = do + scheduleUpgrade server + audit <- right (Ledger.auditContext "runner fixture" "explicit isolated first run" ["owned server and controlled parent job"]) + access <- provider + -- Required first producer/consumer check, before the full crash matrix. + withFaultProxy server $ \proxy -> do + let label = "runner_early" + ledger <- newLedgerId + config <- fixture server label (proxyEndpoint proxy) FirstUse + key <- right (targetKey ("runner-tests/" <> label)) + trace <- newIORef [] + let sink event@(Event.RunEvent phase _ _ _ step query _) = do + modifyIORef' trace (++ [event]) + when (phase == Event.BeforeSql && step == Just "insert") $ case query of + Just identity -> setFaultForQuery proxy (queryIdText identity) DropResponse + _ -> fail "Step intent lacks its query identity" + context = Run.mutationContext config access sink Event.redactedDiagnostics + plan <- planFor label 7 + result <- Run.up context (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) plan + failure <- case result of Left value -> pure value; Right _ -> fail "Lost response was acknowledged" + let R.RunFailure primary report secondary = failure + R.RunReport _ _ _ acquired snapshot migrations observations initialized = report + assert "Primary lost-response identity missing" (case primary of R.SqlFailed (R.StepReference "app" "one" "insert" _ _) _ Ledger.UncertainSubmission _ -> True; _ -> False) + assert "Acknowledged partial progress missing" (acquired && null secondary && initialized == Just ledger && length observations == 2) + assert "Earlier successful step was lost" (case migrations of [R.MigrationSummary _ R.InProgress (Just _) [R.StepSummary "create" _ R.StepObservedSuccess _ _, R.StepSummary "insert" _ R.StepUncertain _ _]] -> True; _ -> False) + assert "SQL leaked in default failure/report" (not ("SQL_SECRET_MARKER" `Text.isInfixOf` Text.pack (show failure))) + count <- scalar server ("SELECT toUInt32(count()) AS value FROM default." <> label) + assert "Lost response did not commit exactly one effect" (count == 1) + let R.HistoryReport _ _ _ position _ events _ unresolved = snapshot + assert "Latest acknowledged checkpoint wrong" (isJust position && length events == 5 && unresolved) + token <- maybe (fail "Missing checkpoint") pure position + assert "Expected-position token not roundtrippable" (Ledger.parseLedgerPosition (Ledger.ledgerPositionText token) == Right token) + setFault proxy Pass + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + guarded <- Inspect.status (Inspect.Guarded access) bound plan >>= right + assert "Guarded inspection lost state" (case guarded of R.InspectionReport (R.HistoryReport _ _ _ observed (R.GuardedObservation _) _ _ _) _ issues -> observed == position && not (null issues); _ -> False) + repeated <- Run.up (Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit Nothing) plan + assert "Ordinary up resumed unresolved history" (case repeated of Left (R.RunFailure (R.LedgerFailed Ledger.UnresolvedHistory) _ _) -> True; _ -> False) + received <- proxyEvents proxy >>= fromJson :: IO [Value] + kinds <- mapM (field "stage") received :: IO [Maybe Text.Text] + assert "Ordinary restart replayed an application statement" (length (filter (== Just "application-sql") kinds) == 2) + callbacks <- readIORef trace + assert "Completion callback preceded ledger acknowledgement" (length [() | Event.RunEvent Event.StepSucceeded _ _ _ _ _ _ <- callbacks] == 1 && null [() | Event.RunEvent Event.MigrationCompleted _ _ _ _ _ _ <- callbacks]) + putStrLn "runner early case: lost second response; one effect; first checkpoint retained; guarded status; ordinary up refuses replay" + withFaultProxy server $ \proxy -> do + let label = "runner_normal" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + plan <- planFor label 7 + first <- Run.up (Run.mutationContext config access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) plan >>= right + let R.RunReport _ _ _ _ snapshot _ _ returned = first + persisted <- case returned of + Nothing -> fail "Bootstrap did not return its ledger identity" + Just value -> do + let file = serverLogDirectory server "runner-ledger-id.txt" + BS.writeFile file (Encoding.encodeUtf8 (ledgerIdText value)) + BS.readFile file >>= right . ledgerId . Encoding.decodeUtf8 + assert "Normal run did not complete" (case snapshot of R.HistoryReport _ _ (R.LedgerInitializedAt actual) (Just _) _ _ [R.MigrationSummary _ R.Applied _ _] False -> actual == ledger; _ -> False) + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger persisted) + _ <- Run.up (Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit Nothing) plan >>= right + received <- proxyEvents proxy >>= fromJson :: IO [Value] + kinds <- mapM (field "stage") received :: IO [Maybe Text.Text] + assert "Completed migration replayed" (length (filter (== Just "application-sql") kinds) == 2) + _ <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions True) plan >>= right + changed <- planFor label 8 + blocked <- Run.up (Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit Nothing) changed + assert "Modified applied SQL accepted" (case blocked of Left (R.RunFailure (R.IncompatibleHistory _) _ _) -> True; _ -> False) + putStrLn "runner normal case: ordered two-step completion, verify current, second run skips, changed applied SQL blocks" + +runnerValidationCases :: Server -> IO () +runnerValidationCases server = do + audit <- right (Ledger.auditContext "validation fixture" "owned isolated target" ["parent serializes and drains all requests"]) + access <- provider + withFaultProxy server $ \proxy -> do + let label = "runtime_validation" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + touched <- newIORef False + counting <- right (Access.externallySerialized "preflight-counter" "owned fixture" 1000000 (const (writeIORef touched True)) (const (pure ())) (const (pure True))) + forbidden <- right (sqlStep "reserved" "CREATE TABLE runtime_validation_history.forbidden (id UInt8) ENGINE=Memory") + migration <- right (sqlMigration "forbidden" (forbidden :| [])) + component <- right (migrationComponent "app" Set.empty (migration :| [])) + bad <- right (migrationPlan (component :| [])) + rejected <- Run.up (Run.mutationContext config counting Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) bad + assert "Dynamic namespace was not rejected in pure preflight" (case rejected of Left (R.RunFailure (R.LedgerNamespaceInSql _) _ _) -> True; _ -> False) + assert "Invalid plan acquired access" . not =<< readIORef touched + empty <- proxyEvents proxy >>= fromJson :: IO [Value] + assert "Invalid plan contacted server" (null empty) + valid <- planFor label 7 + missing <- Inspect.status Inspect.Observe bound valid >>= right + assert "Missing inspection created ledger" (case missing of R.InspectionReport (R.HistoryReport _ _ R.LedgerMissing _ R.UnguardedObservation [] _ _) _ _ -> True; _ -> False) + _ <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions False) valid >>= right + repeated <- Run.up (Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit Nothing) valid + assert "Recurring run initialized absent history" (case repeated of Left (R.RunFailure R.LedgerRequired _ _) -> True; _ -> False) + -- A two-component prefix retains dependency and global ordering evidence. + seedStep <- right (sqlStep "seed" "CREATE TABLE default.validation_seed (id UInt8) ENGINE=Memory") + seedMigration <- right (sqlMigration "seed" (seedStep :| [])) + seed <- right (migrationComponent "seed" Set.empty (seedMigration :| [])) + appStep <- right (sqlStep "app" "CREATE TABLE default.validation_app (id UInt8) ENGINE=Memory") + appMigration <- right (sqlMigration "app" (appStep :| [])) + dependent <- right (migrationComponent "app" (Set.singleton "seed") (appMigration :| [])) + independent <- right (migrationComponent "app" Set.empty (appMigration :| [])) + original <- right (migrationPlan (seed :| [dependent])) + _ <- Run.up (Run.mutationContext config access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) original >>= right + changedDependencies <- right (migrationPlan (seed :| [independent])) + reordered <- right (migrationPlan (independent :| [seed])) + unknownStored <- right (migrationPlan (seed :| [])) + forM_ [changedDependencies, reordered, unknownStored] $ \changed -> do + result <- Run.up (Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit Nothing) changed + assert "Incompatible global history permitted mutation" (case result of Left (R.RunFailure (R.IncompatibleHistory _) _ _) -> True; _ -> False) + verification <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions False) changed + assert "Verify accepted changed global prefix" (case verification of Left (R.InspectionFailure (R.IncompatibleHistory _) _ _) -> True; _ -> False) + _ <- Inspect.history Inspect.Observe bound >>= right + pure () + void (queryServer server "INSERT INTO runtime_validation_history.events SELECT sequence,event_id,ledger_id,concat(event,' ') FROM runtime_validation_history.events WHERE sequence=1 SETTINGS async_insert=0") + corrupt <- Run.up (Run.mutationContext bound access Event.quietEvents Event.redactedDiagnostics) (Run.upOptions audit Nothing) original + assert "Conflicting physical history permitted execution" (case corrupt of Left (R.RunFailure (R.LedgerFailed _) _ _) -> True; _ -> False) + requests <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") requests :: IO [Maybe Text.Text] + assert "Validation failures submitted application SQL" (length (filter (== Just "application-sql") stages) == 2) + putStrLn "runtime validation: pure namespace preflight; absent observation without initialization; changed dependencies/order/unknown stored item/corruption refuse SQL" + +-- Public producer/consumer proof, including original-prefix rejection. +scheduleUpgrade :: Server -> IO () +scheduleUpgrade server = do + access <- provider + audit <- right (Ledger.auditContext "schedule fixture" "append earlier component upgrade" ["owned disposable server"]) + ledger <- newLedgerId + let label = "schedule_upgrade" + config <- fixture server label (serverEndpoint server) FirstUse + bound <- fixture server label (serverEndpoint server) (ExistingLedger ledger) + key <- right (targetKey ("runner-tests/" <> label)) + stepA <- right (sqlStep "create" "CREATE TABLE default.schedule_a (id UInt8) ENGINE=Memory") + stepB <- right (sqlStep "create" "CREATE TABLE default.schedule_b (id UInt8) ENGINE=Memory") + stepA2 <- right (sqlStep "insert" "INSERT INTO default.schedule_a VALUES (2)") + a1 <- right (sqlMigration "one" (stepA :| [])) + b1 <- right (sqlMigration "one" (stepB :| [])) + a2 <- right (sqlMigration "two" (stepA2 :| [])) + a <- right (migrationComponent "a" Set.empty (a1 :| [])) + upgraded <- right (migrationComponent "a" Set.empty (a1 :| [a2])) + b <- right (migrationComponent "b" (Set.singleton "a") (b1 :| [])) + initial <- right (migrationPlan (a :| [b])) + let ident c m = migrationIdFromNames <$> componentName c <*> migrationName m + ids <- mapM (right . uncurry ident) [("a", "one"), ("b", "one"), ("a", "two")] + full <- right (scheduledMigrationPlanFromList [upgraded, b] ids) + let context target = Run.mutationContext target access Event.quietEvents Event.redactedDiagnostics + _ <- Run.up (context config) (Run.upOptions audit (Just (Ledger.authorizeInitialization key ledger audit))) initial >>= right + -- Old grouping would put A.two before the already-recorded B.one. + grouped <- right (migrationPlan (upgraded :| [b])) + refused <- Run.up (context bound) (Run.upOptions audit Nothing) grouped + assert "changed prefix accepted" (case refused of Left (R.RunFailure (R.IncompatibleHistory _) _ _) -> True; _ -> False) + result <- Run.up (context bound) (Run.upOptions audit Nothing) full >>= right + assert "upgrade reran old migrations" (case result of R.RunReport _ _ _ _ _ _ observations _ -> length observations == 1) + count <- scalar server "SELECT toUInt32(count()) AS value FROM default.schedule_a" + assert "A.two missing or replayed" (count == 1) + repeated <- Run.up (context bound) (Run.upOptions audit Nothing) full >>= right + assert "repeat submitted SQL" (case repeated of R.RunReport _ _ _ _ _ _ observations _ -> null observations) + _ <- Inspect.verify Inspect.Observe bound (Inspect.verifyOptions True) full >>= right + putStrLn "schedule: A.one B.one preserved; only appended A.two executes; changed prefix refused; repeat no-op; require-current passes" + + -- Adoption maps the explicit prefix A.one, B.one, leaving A.two pending. + adoptLedger <- newLedgerId + let adoptLabel = "schedule_adoption" + adoptConfig <- fixture server adoptLabel (serverEndpoint server) FirstUse + adoptBound <- fixture server adoptLabel (serverEndpoint server) (ExistingLedger adoptLedger) + adoptKey <- right (targetKey ("runner-tests/" <> adoptLabel)) + quiet <- right (Ledger.attestQuiescence adoptKey audit) + let views = take 2 (NE.toList (inspectSchedule full)) + rows <- mapM (\name -> right (Adoption.legacyRow name "2020-01-01T00:00:00Z" Nothing)) ["a.sql", "b.sql"] + mappings <- mapM (\(name, view) -> right (Adoption.attestedMapping name (componentNameText (migrationIdComponent (migrationId view))) "one" (migrationChecksumHex (migrationChecksum view)) ["reviewed schedule prefix"])) (zip ["a.sql", "b.sql"] views) + request <- case (rows, mappings) of + (r : rs, m : ms) -> right (Adoption.adoptionRequest adoptKey (Ledger.ExpectAbsent adoptKey) (Just (Ledger.authorizeInitialization adoptKey adoptLedger audit)) quiet audit "schedule-fixture" "immutable-prefix" (r :| rs) (m :| ms)) + _ -> fail "Missing adoption fixture" + observed <- Inspect.status Inspect.Observe adoptConfig full >>= right + adoption <- right (Adoption.planAdoption full observed request) + _ <- Adoption.applyAdoption (context adoptConfig) full adoption >>= right + let stop (Event.RunEvent phase _ component migration _ _ _) = when (phase == Event.BeforeSql && component == Just "a" && migration == Just "two") (fail "interrupt appended schedule migration") + stopped <- Run.up (Run.mutationContext adoptBound access stop Event.redactedDiagnostics) (Run.upOptions audit Nothing) full + (position, attempt, selected) <- case stopped of + Left (R.RunFailure (R.CallbackFailed _) (R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just position) _ _ _ _) [_, _, R.MigrationSummary (R.DefinitionSummary 2 0 "a" _ "two" _ _) _ (Just attempt) [R.StepSummary _ _ _ (Just selected) _]] _ _) _) -> pure (position, attempt, selected) + _ -> fail "Interleaved recovery selected wrong definition" + evidence <- right (Repair.recoveryEvidence audit quiet []) + retry <- right (Repair.retryStepRequest position attempt selected evidence) + retried <- Repair.repair (context adoptBound) full retry >>= right + (nextPosition, nextAttempt) <- case retried of + R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just position') _ _ _ _) [_, _, R.MigrationSummary _ _ (Just attempt') _] _ _ -> pure (position', attempt') + _ -> fail "Missing recovered schedule migration" + finalize <- right (Repair.finalizeRequest nextPosition nextAttempt evidence) + _ <- Repair.repair (context adoptBound) full finalize >>= right + _ <- Inspect.verify Inspect.Observe adoptBound (Inspect.verifyOptions True) full >>= right + putStrLn "schedule: adoption maps A.one B.one; interrupted A.two repairs ordinal 2/component 0 and finalizes" diff --git a/test/SurvivingWork.hs b/test/SurvivingWork.hs new file mode 100644 --- /dev/null +++ b/test/SurvivingWork.hs @@ -0,0 +1,149 @@ +module SurvivingWork (survivingWorkCases, runnerChild) where + +import ClickHouseMigrate +import ClickHouseMigrate.Event qualified as E +import ClickHouseMigrate.ExclusiveAccess qualified as Access +import ClickHouseMigrate.Inspection qualified as I +import ClickHouseMigrate.Ledger qualified as L +import ClickHouseMigrate.Repair qualified as Repair +import ClickHouseMigrate.Report qualified as R +import ClickHouseMigrate.Runner qualified as Run +import ClickHouseMigrate.TestSupport.ExclusiveAccess (hostLocalExclusiveAccess) +import ClickHouseMigrate.TestSupport.FaultProxy +import ClickHouseMigrate.TestSupport.Server +import Control.Concurrent (forkFinally, killThread, newEmptyMVar, putMVar, takeMVar, threadDelay) +import Control.Exception (SomeAsyncException, finally, fromException) +import Control.Monad (forM_, void) +import Data.Aeson (Value) +import Data.IORef (atomicModifyIORef', newIORef) +import Data.List.NonEmpty (NonEmpty (..)) +import Data.Set qualified as Set +import Data.Text qualified as T +import Data.Text.Encoding qualified as Encoding +import RunnerIntegration (assert, field, fixture, fromJson, provider, right, scalar) +import System.Environment (getExecutablePath) +import System.Exit (ExitCode (..)) +import System.FilePath (()) +import System.Process (createProcess, proc, terminateProcess, waitForProcess) +import System.Timeout (timeout) + +mutationPlan :: T.Text -> IO MigrationPlan +mutationPlan label = do + metadata <- right (sqlStep "metadata" (Encoding.encodeUtf8 ("ALTER TABLE default." <> label <> " ADD COLUMN note String"))) + update <- right (sqlStep "update" (Encoding.encodeUtf8 ("ALTER TABLE default." <> label <> " UPDATE value=7 WHERE id=1"))) + migration <- right (sqlMigration "one" (metadata :| [update])) + component <- right (migrationComponent "app" Set.empty (migration :| [])) + right (migrationPlan (component :| [])) + +auditContext :: IO L.AuditContext +auditContext = right (L.auditContext "surviving-work fixture" "owned isolated server; explicit administration" ["parent retains exclusion until client gone and mutation effects plus all requests drained"]) + +runnerChild :: String -> String -> String -> String -> String -> IO () +runnerChild endpoint identity labelText ledgerText root = do + let label = T.pack labelText + key <- right (targetKey ("runner-tests/" <> label)) + ledger <- right (ledgerId (T.pack ledgerText)) + config <- targetConfig key <$> right (directEndpoint (T.pack endpoint)) <*> right (serverId (T.pack identity)) <*> right (ledgerNamespace (label <> "_history")) <*> pure FirstUse <*> right (basicCredentials "default" "test-password") <*> pure defaultHttpLimits + access <- hostLocalExclusiveAccess root 1000000 Nothing >>= right + plan <- mutationPlan label + audit <- auditContext + result <- Run.up (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan + fail ("Controlled client returned before kill: " ++ show result) + +waitUntil :: String -> IO Bool -> IO () +waitUntil label predicate = do + finished <- timeout 8000000 loop + assert label (finished == Just ()) + where + loop = predicate >>= \done -> if done then pure () else threadDelay 20000 >> loop + +survivingWorkCases :: Server -> IO () +survivingWorkCases server = do + forM_ [False, True] $ \asyncCancellation -> withFaultProxy server $ \proxy -> do + let label = if asyncCancellation then "async_survivor" else "killed_survivor" + root = serverLogDirectory server T.unpack label ++ "-locks" + ledger <- newLedgerId + key <- right (targetKey ("runner-tests/" <> label)) + audit <- auditContext + plan <- mutationPlan label + config <- fixture server label (proxyEndpoint proxy) FirstUse + bound <- fixture server label (proxyEndpoint proxy) (ExistingLedger ledger) + void (queryServer server ("CREATE TABLE default." <> label <> " (id UInt8, value UInt8) ENGINE=MergeTree ORDER BY id")) + void (queryServer server ("INSERT INTO default." <> label <> " SETTINGS async_insert=0 VALUES (1,0)")) + void (queryServer server ("SYSTEM STOP MERGES default." <> label)) + let activeMutation = (> 0) <$> scalar server ("SELECT toUInt32(count()) AS value FROM system.mutations WHERE database='default' AND table='" <> label <> "' AND NOT is_done") + startMerges = void (queryServer server ("SYSTEM START MERGES default." <> label)) + ( do + stopOnce <- + if asyncCancellation + then do + access <- hostLocalExclusiveAccess root 1000000 Nothing >>= right + done <- newEmptyMVar + thread <- forkFinally (Run.up (Run.mutationContext config access E.quietEvents E.redactedDiagnostics) (Run.upOptions audit (Just (L.authorizeInitialization key ledger audit))) plan) (putMVar done) + pure $ do + killThread thread + result <- takeMVar done + assert + "Async cancellation became a normal RunFailure" + ( case result of + Left exception -> case fromException exception :: Maybe SomeAsyncException of + Just _ -> True + _ -> False + _ -> False + ) + else do + binary <- getExecutablePath + rows <- queryServer server "SELECT toString(serverUUID()) AS server_id" >>= fromJson :: IO [Value] + identity <- case rows of [row] -> field "server_id" row; _ -> fail "Missing server UUID" + (_, _, _, process) <- createProcess (proc binary ["runner-child", T.unpack (proxyEndpoint proxy), T.unpack identity, T.unpack label, T.unpack (ledgerIdText ledger), root]) + pure $ do + terminateProcess process + exitCode <- waitForProcess process + assert "Killed client reported success" (exitCode /= ExitSuccess) + stopped <- newIORef False + let stopClient = do + already <- atomicModifyIORef' stopped (\done -> (True, done)) + if already then pure () else stopOnce + ( do + waitUntil "Mutation did not start before client cancellation" activeMutation + observation <- I.status I.Observe bound plan >>= right + let R.InspectionReport (R.HistoryReport _ _ _ position _ _ _ _) migrations _ = observation + (expected, attempt, selected) <- case (position, migrations) of + (Just expected, [R.MigrationSummary _ R.InProgress (Just attempt) [R.StepSummary _ _ R.StepObservedSuccess _ _, R.StepSummary "update" _ R.StepUncertain (Just selected) _]]) -> pure (expected, attempt, selected) + _ -> fail ("Missing active mutation intent " ++ show observation) + let R.StepAttempt _ _ query = selected + activeQuery = (> 0) <$> scalar server ("SELECT toUInt32(count()) AS value FROM system.processes WHERE query_id='" <> queryIdText query <> "'") + assert "Controlled mutation query not active" =<< activeQuery + -- Negative test: deliberately supply false external evidence and a test + -- provider that allows overlap. Positive process evidence must still win. + broken <- provider + falseQuiet <- right (L.attestQuiescence key audit) + falseEvidence <- right (Repair.recoveryEvidence audit falseQuiet []) + unsafe <- right (Repair.acceptStepRequest expected attempt selected falseEvidence) + rejected <- Repair.repair (Run.mutationContext bound broken E.quietEvents E.redactedDiagnostics) plan unsafe + assert "Recovery accepted a positively active predecessor" (case rejected of Left (R.RunFailure (R.PriorQueriesActive queries) _ _) -> query `elem` queries; _ -> False) + stopClient + assert "Server mutation did not survive client shutdown" =<< activeMutation + blocked <- hostLocalExclusiveAccess root 1000000 Nothing >>= right + ordinary <- Run.up (Run.mutationContext bound blocked E.quietEvents E.redactedDiagnostics) (Run.upOptions audit Nothing) plan + assert "OS unlock was mistaken for server quiescence" (case ordinary of Left (R.RunFailure (R.AccessFailed Access.PredecessorQuiescenceRequired) _ _) -> True; _ -> False) + startMerges + waitUntil "Surviving mutation never committed" ((== 7) <$> scalar server ("SELECT toUInt32(value) AS value FROM default." <> label <> " WHERE id=1")) + waitUntil "Original query failed to drain" (not <$> activeQuery) + waitUntil "Background mutation failed to drain" (not <$> activeMutation) + quiet <- right (L.attestQuiescence key audit) + access <- hostLocalExclusiveAccess root 1000000 (Just quiet) >>= right + evidence <- right (Repair.recoveryEvidence audit quiet []) + accepted <- right (Repair.acceptStepRequest expected attempt selected evidence) >>= Repair.repair (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan >>= right + (after, current) <- case accepted of R.RunReport _ _ _ _ (R.HistoryReport _ _ _ (Just after) _ _ _ _) [R.MigrationSummary _ _ (Just current) _] _ _ -> pure (after, current); _ -> fail "Missing accepted attempt" + final <- right (Repair.finalizeRequest after current evidence) + _ <- Repair.repair (Run.mutationContext bound access E.quietEvents E.redactedDiagnostics) plan final >>= right + _ <- I.verify I.Observe bound (I.verifyOptions True) plan >>= right + requests <- proxyEvents proxy >>= fromJson :: IO [Value] + stages <- mapM (field "stage") requests :: IO [Maybe T.Text] + assert "Killed-client recovery replayed application SQL" (length (filter (== Just "application-sql") stages) == 2) + putStrLn ("surviving server mutation: async cancellation=" ++ show asyncCancellation ++ "; active-query refusal, host marker, external drain, accept/finalize without replay") + ) + `finally` stopClient + ) + `finally` startMerges diff --git a/test/data/clickhouse-matrix.json b/test/data/clickhouse-matrix.json new file mode 100644 --- /dev/null +++ b/test/data/clickhouse-matrix.json @@ -0,0 +1,43 @@ +{ + "matrixVersion": 1, + "checkedAt": "2026-10-16", + "policy": "Only exact qualified patches; no prereleases, Cloud, proxies, distributed or replicated execution", + "entries": [ + { + "id": "lts-native", + "version": "26.8.28.14", + "tag": "v26.8.28.14-lts", + "commit": "3694fe07040369cef78c937311cbaac5f0555484", + "platform": "aarch64-darwin", + "url": "https://github.com/ClickHouse/ClickHouse/releases/download/v26.8.28.14-lts/clickhouse-macos-aarch64", + "sha256": "ab5a04de4a6b705e0f1f02e6a0e0a44feff8b4455b6dfcc4a03b7571f98fbb60", + "status": "native-qualified", + "tagObject": "cb8b2f96f32feb4203ab9edbf4f03b274bbb6952", + "qualification": { + "date": "2026-10-16", + "compiler": "ghc-9.12.4", + "host": "macOS 26.7 arm64", + "requiredGroups": 12, + "artifactVariants": [ + "extracted sibling Cabal distributions", + "Nix public suite, private driver and example" + ], + "scope": "Native protocol qualification; no production OS, Linux, power-loss or uptime certification", + "startupCaveat": "A separate attempt exited SIGSEGV before readiness; cause unknown; preserved in docs/release-evidence.md" + } + } + ], + "unqualifiedReleases": [ + { + "version": "26.8.29.10", + "tag": "v26.8.29.10-lts", + "tagObject": "0dd3d23efb25729261b3ef78b43cf5b872f79b04", + "commit": "7b8b36217c67c466bd4f903c15430b04e467b25a", + "reason": "Published during qualification; untested and rejected by exact runtime allowlist" + }, + { + "version": "26.9.19.9", + "reason": "Current stable in official package index; untested and rejected" + } + ] +} diff --git a/test/fixtures/kansoku/legacy-history.json b/test/fixtures/kansoku/legacy-history.json new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/legacy-history.json @@ -0,0 +1,127 @@ +[ + { + "filename": "0001_events_raw.sql", + "appliedAt": "2020-01-01T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0002_identity.sql", + "appliedAt": "2020-01-02T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0003_rollups.sql", + "appliedAt": "2020-01-03T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0004_events_raw_v2.sql", + "appliedAt": "2020-01-04T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0005_final_free_resolution.sql", + "appliedAt": "2020-01-05T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0006_rollups_v2.sql", + "appliedAt": "2020-01-06T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0007_drop_events_raw_v1.sql", + "appliedAt": "2020-01-07T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0008_account_scoped_metrics.sql", + "appliedAt": "2020-01-08T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0009_typed_properties.sql", + "appliedAt": "2020-01-09T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0010_resolved_event_properties.sql", + "appliedAt": "2020-01-10T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0011_events_raw_identifiers.sql", + "appliedAt": "2020-01-11T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0012_identity_signals.sql", + "appliedAt": "2020-01-12T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0013_retire_identity_map.sql", + "appliedAt": "2020-01-13T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0014_experiment_exposures.sql", + "appliedAt": "2020-01-14T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0015_experiment_exposure_string_casts.sql", + "appliedAt": "2020-01-15T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0016_events_raw_schema_version.sql", + "appliedAt": "2020-01-16T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0017_replay_safe_sources.sql", + "appliedAt": "2020-01-17T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0018_conservative_identity_resolution.sql", + "appliedAt": "2020-01-18T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0019_web_enrichment.sql", + "appliedAt": "2020-01-19T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0020_replay_safe_sessions.sql", + "appliedAt": "2020-01-20T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0021_governed_current_user_traits.sql", + "appliedAt": "2020-01-21T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0022_applied_retention_policy.sql", + "appliedAt": "2020-01-22T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0023_stable_reader_surface.sql", + "appliedAt": "2020-01-23T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0024_cohort_members.sql", + "appliedAt": "2020-01-24T00:00:00Z", + "evidence": "operator-attestation" + }, + { + "filename": "0025_organization_analytics.sql", + "appliedAt": "2020-01-25T00:00:00Z", + "evidence": "operator-attestation" + } +] diff --git a/test/fixtures/kansoku/manifest-first.json b/test/fixtures/kansoku/manifest-first.json new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/manifest-first.json @@ -0,0 +1,21 @@ +{ + "manifestVersion": 1, + "component": "kansoku", + "dependencies": [], + "migrations": [ + { + "name": "0001_events_raw", + "description": "Reviewed complete source file 0001_events_raw.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0001_events_raw/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0001_events_raw/statement-02.sql" + } + ] + } + ] +} diff --git a/test/fixtures/kansoku/manifest.json b/test/fixtures/kansoku/manifest.json new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/manifest.json @@ -0,0 +1,1185 @@ +{ + "manifestVersion": 1, + "component": "kansoku", + "dependencies": [], + "migrations": [ + { + "name": "0001_events_raw", + "description": "Reviewed complete source file 0001_events_raw.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0001_events_raw/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0001_events_raw/statement-02.sql" + } + ] + }, + { + "name": "0002_identity", + "description": "Reviewed complete source file 0002_identity.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0002_identity/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0002_identity/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0002_identity/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0002_identity/statement-04.sql" + } + ] + }, + { + "name": "0003_rollups", + "description": "Reviewed complete source file 0003_rollups.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0003_rollups/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0003_rollups/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0003_rollups/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0003_rollups/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0003_rollups/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0003_rollups/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0003_rollups/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0003_rollups/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0003_rollups/statement-09.sql" + } + ] + }, + { + "name": "0004_events_raw_v2", + "description": "Reviewed complete source file 0004_events_raw_v2.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0004_events_raw_v2/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0004_events_raw_v2/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0004_events_raw_v2/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0004_events_raw_v2/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0004_events_raw_v2/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0004_events_raw_v2/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0004_events_raw_v2/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0004_events_raw_v2/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0004_events_raw_v2/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0004_events_raw_v2/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0004_events_raw_v2/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0004_events_raw_v2/statement-12.sql" + }, + { + "name": "statement-13", + "file": "steps/0004_events_raw_v2/statement-13.sql" + }, + { + "name": "statement-14", + "file": "steps/0004_events_raw_v2/statement-14.sql" + }, + { + "name": "statement-15", + "file": "steps/0004_events_raw_v2/statement-15.sql" + }, + { + "name": "statement-16", + "file": "steps/0004_events_raw_v2/statement-16.sql" + } + ] + }, + { + "name": "0005_final_free_resolution", + "description": "Reviewed complete source file 0005_final_free_resolution.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0005_final_free_resolution/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0005_final_free_resolution/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0005_final_free_resolution/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0005_final_free_resolution/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0005_final_free_resolution/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0005_final_free_resolution/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0005_final_free_resolution/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0005_final_free_resolution/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0005_final_free_resolution/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0005_final_free_resolution/statement-10.sql" + } + ] + }, + { + "name": "0006_rollups_v2", + "description": "Reviewed complete source file 0006_rollups_v2.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0006_rollups_v2/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0006_rollups_v2/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0006_rollups_v2/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0006_rollups_v2/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0006_rollups_v2/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0006_rollups_v2/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0006_rollups_v2/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0006_rollups_v2/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0006_rollups_v2/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0006_rollups_v2/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0006_rollups_v2/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0006_rollups_v2/statement-12.sql" + }, + { + "name": "statement-13", + "file": "steps/0006_rollups_v2/statement-13.sql" + }, + { + "name": "statement-14", + "file": "steps/0006_rollups_v2/statement-14.sql" + }, + { + "name": "statement-15", + "file": "steps/0006_rollups_v2/statement-15.sql" + }, + { + "name": "statement-16", + "file": "steps/0006_rollups_v2/statement-16.sql" + }, + { + "name": "statement-17", + "file": "steps/0006_rollups_v2/statement-17.sql" + }, + { + "name": "statement-18", + "file": "steps/0006_rollups_v2/statement-18.sql" + }, + { + "name": "statement-19", + "file": "steps/0006_rollups_v2/statement-19.sql" + }, + { + "name": "statement-20", + "file": "steps/0006_rollups_v2/statement-20.sql" + } + ] + }, + { + "name": "0007_drop_events_raw_v1", + "description": "Reviewed complete source file 0007_drop_events_raw_v1.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0007_drop_events_raw_v1/statement-01.sql" + } + ] + }, + { + "name": "0008_account_scoped_metrics", + "description": "Reviewed complete source file 0008_account_scoped_metrics.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0008_account_scoped_metrics/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0008_account_scoped_metrics/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0008_account_scoped_metrics/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0008_account_scoped_metrics/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0008_account_scoped_metrics/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0008_account_scoped_metrics/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0008_account_scoped_metrics/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0008_account_scoped_metrics/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0008_account_scoped_metrics/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0008_account_scoped_metrics/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0008_account_scoped_metrics/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0008_account_scoped_metrics/statement-12.sql" + }, + { + "name": "statement-13", + "file": "steps/0008_account_scoped_metrics/statement-13.sql" + }, + { + "name": "statement-14", + "file": "steps/0008_account_scoped_metrics/statement-14.sql" + }, + { + "name": "statement-15", + "file": "steps/0008_account_scoped_metrics/statement-15.sql" + }, + { + "name": "statement-16", + "file": "steps/0008_account_scoped_metrics/statement-16.sql" + }, + { + "name": "statement-17", + "file": "steps/0008_account_scoped_metrics/statement-17.sql" + }, + { + "name": "statement-18", + "file": "steps/0008_account_scoped_metrics/statement-18.sql" + }, + { + "name": "statement-19", + "file": "steps/0008_account_scoped_metrics/statement-19.sql" + }, + { + "name": "statement-20", + "file": "steps/0008_account_scoped_metrics/statement-20.sql" + }, + { + "name": "statement-21", + "file": "steps/0008_account_scoped_metrics/statement-21.sql" + }, + { + "name": "statement-22", + "file": "steps/0008_account_scoped_metrics/statement-22.sql" + } + ] + }, + { + "name": "0009_typed_properties", + "description": "Reviewed complete source file 0009_typed_properties.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0009_typed_properties/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0009_typed_properties/statement-02.sql" + } + ] + }, + { + "name": "0010_resolved_event_properties", + "description": "Reviewed complete source file 0010_resolved_event_properties.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0010_resolved_event_properties/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0010_resolved_event_properties/statement-02.sql" + } + ] + }, + { + "name": "0011_events_raw_identifiers", + "description": "Reviewed complete source file 0011_events_raw_identifiers.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0011_events_raw_identifiers/statement-01.sql" + } + ] + }, + { + "name": "0012_identity_signals", + "description": "Reviewed complete source file 0012_identity_signals.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0012_identity_signals/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0012_identity_signals/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0012_identity_signals/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0012_identity_signals/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0012_identity_signals/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0012_identity_signals/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0012_identity_signals/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0012_identity_signals/statement-08.sql" + } + ] + }, + { + "name": "0013_retire_identity_map", + "description": "Reviewed complete source file 0013_retire_identity_map.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0013_retire_identity_map/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0013_retire_identity_map/statement-02.sql" + } + ] + }, + { + "name": "0014_experiment_exposures", + "description": "Reviewed complete source file 0014_experiment_exposures.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0014_experiment_exposures/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0014_experiment_exposures/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0014_experiment_exposures/statement-03.sql" + } + ] + }, + { + "name": "0015_experiment_exposure_string_casts", + "description": "Reviewed complete source file 0015_experiment_exposure_string_casts.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0015_experiment_exposure_string_casts/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0015_experiment_exposure_string_casts/statement-02.sql" + } + ] + }, + { + "name": "0016_events_raw_schema_version", + "description": "Reviewed complete source file 0016_events_raw_schema_version.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0016_events_raw_schema_version/statement-01.sql" + } + ] + }, + { + "name": "0017_replay_safe_sources", + "description": "Reviewed complete source file 0017_replay_safe_sources.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0017_replay_safe_sources/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0017_replay_safe_sources/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0017_replay_safe_sources/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0017_replay_safe_sources/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0017_replay_safe_sources/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0017_replay_safe_sources/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0017_replay_safe_sources/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0017_replay_safe_sources/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0017_replay_safe_sources/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0017_replay_safe_sources/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0017_replay_safe_sources/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0017_replay_safe_sources/statement-12.sql" + }, + { + "name": "statement-13-prepare", + "file": "native-assertions/statement-13-prepare.sql" + }, + { + "name": "statement-13-check", + "file": "native-assertions/statement-13-check.sql" + }, + { + "name": "statement-13-cleanup", + "file": "native-assertions/statement-13-cleanup.sql" + }, + { + "name": "statement-14-prepare", + "file": "native-assertions/statement-14-prepare.sql" + }, + { + "name": "statement-14-check", + "file": "native-assertions/statement-14-check.sql" + }, + { + "name": "statement-14-cleanup", + "file": "native-assertions/statement-14-cleanup.sql" + }, + { + "name": "statement-15", + "file": "steps/0017_replay_safe_sources/statement-15.sql" + }, + { + "name": "statement-16", + "file": "steps/0017_replay_safe_sources/statement-16.sql" + }, + { + "name": "statement-17", + "file": "steps/0017_replay_safe_sources/statement-17.sql" + }, + { + "name": "statement-18", + "file": "steps/0017_replay_safe_sources/statement-18.sql" + }, + { + "name": "statement-19", + "file": "steps/0017_replay_safe_sources/statement-19.sql" + }, + { + "name": "statement-20", + "file": "steps/0017_replay_safe_sources/statement-20.sql" + } + ] + }, + { + "name": "0018_conservative_identity_resolution", + "description": "Reviewed complete source file 0018_conservative_identity_resolution.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0018_conservative_identity_resolution/statement-01.sql" + }, + { + "name": "statement-02", + "file": "native-assertions/local-materialize-policy.sql" + }, + { + "name": "statement-03", + "file": "steps/0018_conservative_identity_resolution/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0018_conservative_identity_resolution/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0018_conservative_identity_resolution/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0018_conservative_identity_resolution/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0018_conservative_identity_resolution/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0018_conservative_identity_resolution/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0018_conservative_identity_resolution/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0018_conservative_identity_resolution/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0018_conservative_identity_resolution/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0018_conservative_identity_resolution/statement-12.sql" + }, + { + "name": "statement-13", + "file": "steps/0018_conservative_identity_resolution/statement-13.sql" + }, + { + "name": "statement-14", + "file": "steps/0018_conservative_identity_resolution/statement-14.sql" + }, + { + "name": "statement-15", + "file": "steps/0018_conservative_identity_resolution/statement-15.sql" + }, + { + "name": "statement-16", + "file": "steps/0018_conservative_identity_resolution/statement-16.sql" + }, + { + "name": "statement-17", + "file": "steps/0018_conservative_identity_resolution/statement-17.sql" + }, + { + "name": "statement-18", + "file": "steps/0018_conservative_identity_resolution/statement-18.sql" + }, + { + "name": "statement-19", + "file": "steps/0018_conservative_identity_resolution/statement-19.sql" + }, + { + "name": "statement-20", + "file": "steps/0018_conservative_identity_resolution/statement-20.sql" + }, + { + "name": "statement-21", + "file": "steps/0018_conservative_identity_resolution/statement-21.sql" + }, + { + "name": "statement-22", + "file": "steps/0018_conservative_identity_resolution/statement-22.sql" + }, + { + "name": "statement-23", + "file": "steps/0018_conservative_identity_resolution/statement-23.sql" + }, + { + "name": "statement-24", + "file": "steps/0018_conservative_identity_resolution/statement-24.sql" + }, + { + "name": "statement-25", + "file": "steps/0018_conservative_identity_resolution/statement-25.sql" + }, + { + "name": "statement-26", + "file": "steps/0018_conservative_identity_resolution/statement-26.sql" + }, + { + "name": "statement-27", + "file": "steps/0018_conservative_identity_resolution/statement-27.sql" + }, + { + "name": "statement-28", + "file": "steps/0018_conservative_identity_resolution/statement-28.sql" + }, + { + "name": "statement-29", + "file": "steps/0018_conservative_identity_resolution/statement-29.sql" + }, + { + "name": "statement-30", + "file": "steps/0018_conservative_identity_resolution/statement-30.sql" + }, + { + "name": "statement-31", + "file": "steps/0018_conservative_identity_resolution/statement-31.sql" + }, + { + "name": "statement-32", + "file": "steps/0018_conservative_identity_resolution/statement-32.sql" + }, + { + "name": "statement-33", + "file": "steps/0018_conservative_identity_resolution/statement-33.sql" + }, + { + "name": "statement-34", + "file": "steps/0018_conservative_identity_resolution/statement-34.sql" + }, + { + "name": "statement-35", + "file": "steps/0018_conservative_identity_resolution/statement-35.sql" + }, + { + "name": "statement-36", + "file": "steps/0018_conservative_identity_resolution/statement-36.sql" + }, + { + "name": "statement-37", + "file": "steps/0018_conservative_identity_resolution/statement-37.sql" + }, + { + "name": "statement-38", + "file": "steps/0018_conservative_identity_resolution/statement-38.sql" + }, + { + "name": "statement-39", + "file": "steps/0018_conservative_identity_resolution/statement-39.sql" + }, + { + "name": "statement-40", + "file": "steps/0018_conservative_identity_resolution/statement-40.sql" + }, + { + "name": "statement-41", + "file": "steps/0018_conservative_identity_resolution/statement-41.sql" + }, + { + "name": "statement-42", + "file": "steps/0018_conservative_identity_resolution/statement-42.sql" + }, + { + "name": "statement-43", + "file": "steps/0018_conservative_identity_resolution/statement-43.sql" + }, + { + "name": "statement-44", + "file": "steps/0018_conservative_identity_resolution/statement-44.sql" + }, + { + "name": "statement-45", + "file": "steps/0018_conservative_identity_resolution/statement-45.sql" + }, + { + "name": "statement-46", + "file": "steps/0018_conservative_identity_resolution/statement-46.sql" + }, + { + "name": "statement-47", + "file": "steps/0018_conservative_identity_resolution/statement-47.sql" + }, + { + "name": "statement-48", + "file": "steps/0018_conservative_identity_resolution/statement-48.sql" + }, + { + "name": "statement-49", + "file": "steps/0018_conservative_identity_resolution/statement-49.sql" + }, + { + "name": "statement-50", + "file": "steps/0018_conservative_identity_resolution/statement-50.sql" + }, + { + "name": "statement-51", + "file": "steps/0018_conservative_identity_resolution/statement-51.sql" + }, + { + "name": "statement-52", + "file": "steps/0018_conservative_identity_resolution/statement-52.sql" + }, + { + "name": "statement-53", + "file": "steps/0018_conservative_identity_resolution/statement-53.sql" + }, + { + "name": "statement-54", + "file": "steps/0018_conservative_identity_resolution/statement-54.sql" + }, + { + "name": "statement-55", + "file": "steps/0018_conservative_identity_resolution/statement-55.sql" + }, + { + "name": "statement-56", + "file": "steps/0018_conservative_identity_resolution/statement-56.sql" + }, + { + "name": "statement-57", + "file": "steps/0018_conservative_identity_resolution/statement-57.sql" + }, + { + "name": "statement-58", + "file": "steps/0018_conservative_identity_resolution/statement-58.sql" + }, + { + "name": "statement-59", + "file": "steps/0018_conservative_identity_resolution/statement-59.sql" + }, + { + "name": "statement-60", + "file": "steps/0018_conservative_identity_resolution/statement-60.sql" + }, + { + "name": "statement-61", + "file": "steps/0018_conservative_identity_resolution/statement-61.sql" + }, + { + "name": "statement-62", + "file": "steps/0018_conservative_identity_resolution/statement-62.sql" + }, + { + "name": "statement-63", + "file": "steps/0018_conservative_identity_resolution/statement-63.sql" + }, + { + "name": "statement-64", + "file": "steps/0018_conservative_identity_resolution/statement-64.sql" + } + ] + }, + { + "name": "0019_web_enrichment", + "description": "Reviewed complete source file 0019_web_enrichment.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0019_web_enrichment/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0019_web_enrichment/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0019_web_enrichment/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0019_web_enrichment/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0019_web_enrichment/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0019_web_enrichment/statement-06.sql" + } + ] + }, + { + "name": "0020_replay_safe_sessions", + "description": "Reviewed complete source file 0020_replay_safe_sessions.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0020_replay_safe_sessions/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0020_replay_safe_sessions/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0020_replay_safe_sessions/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0020_replay_safe_sessions/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0020_replay_safe_sessions/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0020_replay_safe_sessions/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0020_replay_safe_sessions/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0020_replay_safe_sessions/statement-08.sql" + } + ] + }, + { + "name": "0021_governed_current_user_traits", + "description": "Reviewed complete source file 0021_governed_current_user_traits.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0021_governed_current_user_traits/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0021_governed_current_user_traits/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0021_governed_current_user_traits/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0021_governed_current_user_traits/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0021_governed_current_user_traits/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0021_governed_current_user_traits/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0021_governed_current_user_traits/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0021_governed_current_user_traits/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0021_governed_current_user_traits/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0021_governed_current_user_traits/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0021_governed_current_user_traits/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0021_governed_current_user_traits/statement-12.sql" + }, + { + "name": "statement-13", + "file": "steps/0021_governed_current_user_traits/statement-13.sql" + } + ] + }, + { + "name": "0022_applied_retention_policy", + "description": "Reviewed complete source file 0022_applied_retention_policy.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0022_applied_retention_policy/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0022_applied_retention_policy/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0022_applied_retention_policy/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0022_applied_retention_policy/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0022_applied_retention_policy/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0022_applied_retention_policy/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0022_applied_retention_policy/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0022_applied_retention_policy/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0022_applied_retention_policy/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0022_applied_retention_policy/statement-10.sql" + }, + { + "name": "statement-11", + "file": "steps/0022_applied_retention_policy/statement-11.sql" + }, + { + "name": "statement-12", + "file": "steps/0022_applied_retention_policy/statement-12.sql" + }, + { + "name": "statement-13", + "file": "steps/0022_applied_retention_policy/statement-13.sql" + }, + { + "name": "statement-14", + "file": "steps/0022_applied_retention_policy/statement-14.sql" + }, + { + "name": "statement-15", + "file": "steps/0022_applied_retention_policy/statement-15.sql" + }, + { + "name": "statement-16", + "file": "steps/0022_applied_retention_policy/statement-16.sql" + }, + { + "name": "statement-17", + "file": "steps/0022_applied_retention_policy/statement-17.sql" + }, + { + "name": "statement-18", + "file": "steps/0022_applied_retention_policy/statement-18.sql" + }, + { + "name": "statement-19", + "file": "steps/0022_applied_retention_policy/statement-19.sql" + }, + { + "name": "statement-20", + "file": "steps/0022_applied_retention_policy/statement-20.sql" + }, + { + "name": "statement-21", + "file": "steps/0022_applied_retention_policy/statement-21.sql" + }, + { + "name": "statement-22", + "file": "steps/0022_applied_retention_policy/statement-22.sql" + }, + { + "name": "statement-23", + "file": "steps/0022_applied_retention_policy/statement-23.sql" + } + ] + }, + { + "name": "0023_stable_reader_surface", + "description": "Reviewed complete source file 0023_stable_reader_surface.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0023_stable_reader_surface/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0023_stable_reader_surface/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0023_stable_reader_surface/statement-03.sql" + } + ] + }, + { + "name": "0024_cohort_members", + "description": "Reviewed complete source file 0024_cohort_members.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0024_cohort_members/statement-01.sql" + } + ] + }, + { + "name": "0025_organization_analytics", + "description": "Reviewed complete source file 0025_organization_analytics.sql", + "steps": [ + { + "name": "statement-01", + "file": "steps/0025_organization_analytics/statement-01.sql" + }, + { + "name": "statement-02", + "file": "steps/0025_organization_analytics/statement-02.sql" + }, + { + "name": "statement-03", + "file": "steps/0025_organization_analytics/statement-03.sql" + }, + { + "name": "statement-04", + "file": "steps/0025_organization_analytics/statement-04.sql" + }, + { + "name": "statement-05", + "file": "steps/0025_organization_analytics/statement-05.sql" + }, + { + "name": "statement-06", + "file": "steps/0025_organization_analytics/statement-06.sql" + }, + { + "name": "statement-07", + "file": "steps/0025_organization_analytics/statement-07.sql" + }, + { + "name": "statement-08", + "file": "steps/0025_organization_analytics/statement-08.sql" + }, + { + "name": "statement-09", + "file": "steps/0025_organization_analytics/statement-09.sql" + }, + { + "name": "statement-10", + "file": "steps/0025_organization_analytics/statement-10.sql" + } + ] + } + ] +} diff --git a/test/fixtures/kansoku/native-assertions/local-materialize-policy.sql b/test/fixtures/kansoku/native-assertions/local-materialize-policy.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/local-materialize-policy.sql @@ -0,0 +1,2 @@ + +ALTER TABLE analytics.events_raw MATERIALIZE COLUMN identity_context_key; diff --git a/test/fixtures/kansoku/native-assertions/statement-13-check.sql b/test/fixtures/kansoku/native-assertions/statement-13-check.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/statement-13-check.sql @@ -0,0 +1,16 @@ +INSERT INTO analytics.__chm_legacy_assert_0017_13 + + +-- Verify complete replacement groups before publishing either table. +SELECT throwIf(count() > 0, 'canonical daily-event rebuild mismatch') +FROM +( + SELECT account_id, toDate(occurred_at) AS day, event_name, count() AS n + FROM analytics.events_canonical GROUP BY account_id, day, event_name +) AS canonical +FULL OUTER JOIN +( + SELECT account_id, day, event_name, uniqExactMerge(events) AS n + FROM analytics.daily_event_counts_rebuild GROUP BY account_id, day, event_name +) AS replacement USING (account_id, day, event_name) +WHERE ifNull(canonical.n, 0) != ifNull(replacement.n, 0); diff --git a/test/fixtures/kansoku/native-assertions/statement-13-cleanup.sql b/test/fixtures/kansoku/native-assertions/statement-13-cleanup.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/statement-13-cleanup.sql @@ -0,0 +1,1 @@ +DROP TABLE analytics.__chm_legacy_assert_0017_13 SYNC; diff --git a/test/fixtures/kansoku/native-assertions/statement-13-prepare.sql b/test/fixtures/kansoku/native-assertions/statement-13-prepare.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/statement-13-prepare.sql @@ -0,0 +1,1 @@ +CREATE TABLE analytics.__chm_legacy_assert_0017_13 (result UInt8) ENGINE=Memory; diff --git a/test/fixtures/kansoku/native-assertions/statement-14-check.sql b/test/fixtures/kansoku/native-assertions/statement-14-check.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/statement-14-check.sql @@ -0,0 +1,16 @@ +INSERT INTO analytics.__chm_legacy_assert_0017_14 + + +SELECT throwIf(count() > 0, 'canonical page-view rebuild mismatch') +FROM +( + SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, count() AS n + FROM analytics.events_canonical WHERE event_name = 'page_view' + GROUP BY account_id, day, route +) AS canonical +FULL OUTER JOIN +( + SELECT account_id, day, route, uniqExactMerge(views) AS n + FROM analytics.page_views_by_route_rebuild GROUP BY account_id, day, route +) AS replacement USING (account_id, day, route) +WHERE ifNull(canonical.n, 0) != ifNull(replacement.n, 0); diff --git a/test/fixtures/kansoku/native-assertions/statement-14-cleanup.sql b/test/fixtures/kansoku/native-assertions/statement-14-cleanup.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/statement-14-cleanup.sql @@ -0,0 +1,1 @@ +DROP TABLE analytics.__chm_legacy_assert_0017_14 SYNC; diff --git a/test/fixtures/kansoku/native-assertions/statement-14-prepare.sql b/test/fixtures/kansoku/native-assertions/statement-14-prepare.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/native-assertions/statement-14-prepare.sql @@ -0,0 +1,1 @@ +CREATE TABLE analytics.__chm_legacy_assert_0017_14 (result UInt8) ENGINE=Memory; diff --git a/test/fixtures/kansoku/original/0001_events_raw.sql b/test/fixtures/kansoku/original/0001_events_raw.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0001_events_raw.sql @@ -0,0 +1,32 @@ +CREATE DATABASE IF NOT EXISTS analytics; + +CREATE TABLE IF NOT EXISTS analytics.events_raw +( + event_id String, + event_name LowCardinality(String), + source LowCardinality(String), + source_service Nullable(String), + occurred_at DateTime64(3, 'UTC'), + received_at DateTime64(3, 'UTC'), + anonymous_id Nullable(String), + user_id String, + organization_id Nullable(String), + account_id String, + session_id Nullable(String), + url Nullable(String), + referrer Nullable(String), + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + utm_content Nullable(String), + utm_term Nullable(String), + correlation_id Nullable(String), + causation_id Nullable(String), + source_event_id Nullable(String), + source_global_position Nullable(UInt64), + properties String, + context String +) +ENGINE = ReplacingMergeTree(received_at) +PARTITION BY toYYYYMM(occurred_at) +ORDER BY (account_id, user_id, event_name, occurred_at, event_id); diff --git a/test/fixtures/kansoku/original/0002_identity.sql b/test/fixtures/kansoku/original/0002_identity.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0002_identity.sql @@ -0,0 +1,52 @@ +CREATE DATABASE IF NOT EXISTS analytics; + +-- Latest anonymous_id -> user_id mapping, one row per anonymous_id after merges. +-- `version` is the event's received_at; the newest mapping wins. +CREATE TABLE IF NOT EXISTS analytics.identity_map +( + anonymous_id String, + user_id String, + version DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(version) +ORDER BY anonymous_id; + +-- Fill identity_map from identify/alias events as they arrive in events_raw. +-- Fires on each inserted block; ignores identity events missing either id. +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_identity_map +TO analytics.identity_map +AS +SELECT + anonymous_id, + user_id, + received_at AS version +FROM analytics.events_raw +WHERE event_name IN ('identify', 'alias') + AND anonymous_id != '' + AND user_id != ''; + +-- Query-time resolution: every raw event with its canonical user id. +-- Reads events_raw FINAL so duplicate raw rows collapse before row-level use. +-- The subquery collapses identity_map to the latest user_id per anonymous_id. +-- NOTE: nullIf(m.user_id, '') is load-bearing. With ClickHouse's default +-- join_use_nulls = 0, a LEFT JOIN miss fills m.user_id with '' (NOT NULL); +-- without the nullIf, coalesce would stop there and never reach the +-- anonymous-id fallback. The trailing '' keeps the type plain String. +CREATE VIEW IF NOT EXISTS analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e FINAL +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/original/0003_rollups.sql b/test/fixtures/kansoku/original/0003_rollups.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0003_rollups.sql @@ -0,0 +1,103 @@ +CREATE DATABASE IF NOT EXISTS analytics; + +-- ---------- Daily event counts, by event_name ---------- +CREATE TABLE IF NOT EXISTS analytics.daily_event_counts +( + day Date, + event_name String, + events AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, event_name); + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + uniqExactState(toString(event_id)) AS events +FROM analytics.events_raw +GROUP BY day, event_name; + +-- ---------- Daily active users (canonical, resolved at insert time) ---------- +-- NOTE: nullIf(m.user_id, '') is load-bearing (join_use_nulls = 0 fills a LEFT +-- JOIN miss with '', not NULL); the trailing '' keeps the argument a plain +-- String so the state stays AggregateFunction(uniqExact, String). +CREATE TABLE IF NOT EXISTS analytics.dau +( + day Date, + users AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY day; + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(e.occurred_at) AS day, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY day; + +-- ---------- Weekly active users (canonical), keyed on the Monday of the week ---------- +CREATE TABLE IF NOT EXISTS analytics.wau +( + week Date, + users AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY week; + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(e.occurred_at) AS week, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY week; + +-- ---------- Page views by route ---------- +-- url is Nullable(String) in EP-3's schema; coalesce(url, '') keeps path() +-- non-Nullable so the insert into the String route column cannot fail. +CREATE TABLE IF NOT EXISTS analytics.page_views_by_route +( + day Date, + route String, + views AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, route); + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + toDate(occurred_at) AS day, + path(coalesce(url, '')) AS route, + uniqExactState(toString(event_id)) AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/original/0004_events_raw_v2.sql b/test/fixtures/kansoku/original/0004_events_raw_v2.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0004_events_raw_v2.sql @@ -0,0 +1,160 @@ +-- Schema v2 for the raw events table: sort key (account_id, event_name, +-- occurred_at, event_id) and Nullable(String) -> String DEFAULT ''. +-- Strategy: build events_raw_v2, copy, EXCHANGE names, keep the old table +-- (now named events_raw_v2) until 0007 drops it after verification. +-- Every object reading events_raw is dropped first and recreated against the +-- new table with UNCHANGED v1 semantics; 0005/0006 change semantics later. + +DROP TABLE IF EXISTS analytics.mv_identity_map; +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; +DROP TABLE IF EXISTS analytics.mv_dau; +DROP TABLE IF EXISTS analytics.mv_wau; +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; +DROP VIEW IF EXISTS analytics.events_resolved; +DROP TABLE IF EXISTS analytics.events_raw_v2; + +CREATE TABLE analytics.events_raw_v2 +( + event_id String, + event_name LowCardinality(String), + source LowCardinality(String), + source_service String DEFAULT '', + occurred_at DateTime64(3, 'UTC'), + received_at DateTime64(3, 'UTC'), + anonymous_id String DEFAULT '', + user_id String, + organization_id String DEFAULT '', + account_id String, + session_id String DEFAULT '', + url String DEFAULT '', + referrer String DEFAULT '', + utm_source LowCardinality(String) DEFAULT '', + utm_medium LowCardinality(String) DEFAULT '', + utm_campaign LowCardinality(String) DEFAULT '', + utm_content String DEFAULT '', + utm_term String DEFAULT '', + correlation_id String DEFAULT '', + causation_id String DEFAULT '', + source_event_id String DEFAULT '', + source_global_position Nullable(UInt64), + properties String, + context String +) +ENGINE = ReplacingMergeTree(received_at) +PARTITION BY toYYYYMM(occurred_at) +ORDER BY (account_id, event_name, occurred_at, event_id); + +INSERT INTO analytics.events_raw_v2 + (event_id, event_name, source, source_service, occurred_at, received_at, + anonymous_id, user_id, organization_id, account_id, session_id, url, + referrer, utm_source, utm_medium, utm_campaign, utm_content, utm_term, + correlation_id, causation_id, source_event_id, source_global_position, + properties, context) +SELECT + event_id, event_name, source, coalesce(source_service, ''), occurred_at, + received_at, coalesce(anonymous_id, ''), user_id, + coalesce(organization_id, ''), account_id, coalesce(session_id, ''), + coalesce(url, ''), coalesce(referrer, ''), coalesce(utm_source, ''), + coalesce(utm_medium, ''), coalesce(utm_campaign, ''), + coalesce(utm_content, ''), coalesce(utm_term, ''), + coalesce(correlation_id, ''), coalesce(causation_id, ''), + coalesce(source_event_id, ''), source_global_position, properties, context +FROM analytics.events_raw; + +-- Atomic name swap: analytics.events_raw is now the v2 table; the old v1 +-- table lives on under the name analytics.events_raw_v2 as the rollback copy. +EXCHANGE TABLES analytics.events_raw AND analytics.events_raw_v2; + +-- ---- Recreate dependents against the new table, v1 semantics unchanged ---- + +CREATE MATERIALIZED VIEW analytics.mv_identity_map +TO analytics.identity_map +AS +SELECT + anonymous_id, + user_id, + received_at AS version +FROM analytics.events_raw +WHERE event_name IN ('identify', 'alias') + AND anonymous_id != '' + AND user_id != ''; + +-- Still FINAL here; 0005 removes it. +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e FINAL +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + uniqExactState(toString(event_id)) AS events +FROM analytics.events_raw +GROUP BY day, event_name; + +-- Still joining the identity map here; 0005 replaces these two MVs. +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(e.occurred_at) AS day, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY day; + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(e.occurred_at) AS week, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY week; + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + toDate(occurred_at) AS day, + path(url) AS route, + uniqExactState(toString(event_id)) AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/original/0005_final_free_resolution.sql b/test/fixtures/kansoku/original/0005_final_free_resolution.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0005_final_free_resolution.sql @@ -0,0 +1,74 @@ +-- FINAL-free identity resolution and join-free DAU/WAU. +-- events_resolved consumers aggregate with uniqExact/min, which are +-- insensitive to the transient duplicate rows FINAL would remove; row-COUNT +-- consumers must protect themselves (count DISTINCT event_id). +-- mv_dau/mv_wau stop joining the identity map at insert time and count raw +-- identities (user_id if present, else anonymous_id); the stitched number +-- stays available at query time through events_resolved. + +DROP VIEW IF EXISTS analytics.events_resolved; + +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; + +DROP TABLE IF EXISTS analytics.mv_dau; +DROP TABLE IF EXISTS analytics.mv_wau; + +-- Rebuild dau/wau contents under the new (raw-identity) semantics so the +-- tables are not a mix of stitched and unstitched day states. Column types +-- are unchanged in this migration (still AggregateFunction(uniqExact, String)); +-- 0006 replaces the state type. +TRUNCATE TABLE analytics.dau; +TRUNCATE TABLE analytics.wau; + +INSERT INTO analytics.dau (day, users) +SELECT + toDate(occurred_at) AS day, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; + +INSERT INTO analytics.wau (week, users) +SELECT + toMonday(occurred_at) AS week, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; + +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(occurred_at) AS day, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(occurred_at) AS week, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/original/0006_rollups_v2.sql b/test/fixtures/kansoku/original/0006_rollups_v2.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0006_rollups_v2.sql @@ -0,0 +1,125 @@ +-- Rollup states v2: bounded-size states instead of exact-distinct states that +-- stored every id. events/views: AggregateFunction(uniqExact, String) over +-- event_id -> AggregateFunction(count) (dedup makes duplicate rows rare and +-- transient, so count-vs-distinct drift is negligible; exactness lives in the +-- raw table). dau/wau users: uniqExact -> uniqCombined sketch (~0.8% error at +-- scale, exact for small sets). Tables are rebuilt (state types cannot be +-- ALTERed) and backfilled from events_raw. + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; +DROP TABLE IF EXISTS analytics.mv_dau; +DROP TABLE IF EXISTS analytics.mv_wau; +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; +DROP TABLE IF EXISTS analytics.daily_event_counts; +DROP TABLE IF EXISTS analytics.dau; +DROP TABLE IF EXISTS analytics.wau; +DROP TABLE IF EXISTS analytics.page_views_by_route; + +CREATE TABLE analytics.daily_event_counts +( + day Date, + event_name String, + events AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, event_name); + +CREATE TABLE analytics.dau +( + day Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY day; + +CREATE TABLE analytics.wau +( + week Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY week; + +CREATE TABLE analytics.page_views_by_route +( + day Date, + route String, + views AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, route); + +-- Backfill from the (already deduplicated-by-merges) raw table. +INSERT INTO analytics.daily_event_counts (day, event_name, events) +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY day, event_name; + +INSERT INTO analytics.dau (day, users) +SELECT + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; + +INSERT INTO analytics.wau (week, users) +SELECT + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; + +INSERT INTO analytics.page_views_by_route (day, route, views) +SELECT + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY day, event_name; + +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/original/0007_drop_events_raw_v1.sql b/test/fixtures/kansoku/original/0007_drop_events_raw_v1.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0007_drop_events_raw_v1.sql @@ -0,0 +1,3 @@ +-- Final step of the events_raw v2 rewrite (0004): drop the retained v1 table. +-- 0004's row-count verification must have passed before this file is created. +DROP TABLE IF EXISTS analytics.events_raw_v2; diff --git a/test/fixtures/kansoku/original/0008_account_scoped_metrics.sql b/test/fixtures/kansoku/original/0008_account_scoped_metrics.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0008_account_scoped_metrics.sql @@ -0,0 +1,153 @@ +-- Account-scoped metrics. Keep the FINAL-free identity resolution and the +-- bounded v2 aggregate states while adding account_id to every query source. + +DROP VIEW IF EXISTS analytics.events_resolved; + +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; +DROP TABLE IF EXISTS analytics.mv_dau; +DROP TABLE IF EXISTS analytics.mv_wau; +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; +DROP TABLE IF EXISTS analytics.daily_event_counts; +DROP TABLE IF EXISTS analytics.dau; +DROP TABLE IF EXISTS analytics.wau; +DROP TABLE IF EXISTS analytics.page_views_by_route; + +CREATE TABLE analytics.daily_event_counts +( + account_id String, + day Date, + event_name String, + events AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, event_name); + +CREATE TABLE analytics.dau +( + account_id String, + day Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day); + +CREATE TABLE analytics.wau +( + account_id String, + week Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, week); + +CREATE TABLE analytics.page_views_by_route +( + account_id String, + day Date, + route String, + views AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, route); + +INSERT INTO analytics.daily_event_counts (account_id, day, event_name, events) +SELECT + account_id, + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY account_id, day, event_name; + +INSERT INTO analytics.dau (account_id, day, users) +SELECT + account_id, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; + +INSERT INTO analytics.wau (account_id, week, users) +SELECT + account_id, + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; + +INSERT INTO analytics.page_views_by_route (account_id, day, route, views) +SELECT + account_id, + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + account_id, + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY account_id, day, event_name; + +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + account_id, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + account_id, + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + account_id, + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/original/0009_typed_properties.sql b/test/fixtures/kansoku/original/0009_typed_properties.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0009_typed_properties.sql @@ -0,0 +1,8 @@ +-- EP-17 Milestone 1 chose a server-side native JSON materialized column. +-- Kansoku.ClickHouse.EventRow serializes properties from a parsed Aeson Object, +-- so every properties value written by the application is valid JSON. +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS properties_json JSON + MATERIALIZED CAST(properties, 'JSON'); + +ALTER TABLE analytics.events_raw MATERIALIZE COLUMN properties_json; diff --git a/test/fixtures/kansoku/original/0010_resolved_event_properties.sql b/test/fixtures/kansoku/original/0010_resolved_event_properties.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0010_resolved_event_properties.sql @@ -0,0 +1,24 @@ +-- Property-grouped unique-user metrics need the same typed property projection +-- as raw-event metrics while retaining canonical identity stitching. +DROP VIEW IF EXISTS analytics.events_resolved; + +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user, + e.properties_json AS properties_json +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/original/0011_events_raw_identifiers.sql b/test/fixtures/kansoku/original/0011_events_raw_identifiers.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0011_events_raw_identifiers.sql @@ -0,0 +1,5 @@ +-- EP-18 M1: persist the identifiers map (kind -> value) on raw events. +-- The ClickHaskell wire carries a JSON-object String; '{}' keeps old rows +-- valid for server-side JSON extraction without rewriting existing parts. +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS identifiers String DEFAULT '{}' AFTER context; diff --git a/test/fixtures/kansoku/original/0012_identity_signals.sql b/test/fixtures/kansoku/original/0012_identity_signals.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0012_identity_signals.sql @@ -0,0 +1,140 @@ +-- EP-18 M2: retain immutable identity-binding evidence and resolve anonymous +-- events through account-scoped, confidence-ranked identifier signals. +CREATE TABLE IF NOT EXISTS analytics.identity_signals +( + account_id String, + signal_kind LowCardinality(String), + signal_value String, + user_id String, + observed_at DateTime64(3, 'UTC'), + event_id String +) +ENGINE = ReplacingMergeTree +ORDER BY (account_id, signal_kind, signal_value, user_id, observed_at, event_id) +TTL toDateTime(observed_at) + INTERVAL 180 DAY DELETE + WHERE signal_kind = 'fingerprint'; + +-- Cookie bindings retain the old identify/alias-only semantics. Explicit +-- identifier-map entries are binding evidence on any event carrying user_id. +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_identity_signals +TO analytics.identity_signals +AS +SELECT + account_id, + sig.1 AS signal_kind, + sig.2 AS signal_value, + user_id, + received_at AS observed_at, + event_id +FROM analytics.events_raw +ARRAY JOIN arrayFilter( + kv -> kv.1 != '' AND kv.2 != '', + arrayConcat( + [(if(event_name IN ('identify', 'alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig +WHERE user_id != ''; + +-- Backfill the evidence that predates the materialized view. Replaying this +-- statement produces byte-identical rows, which the full ORDER BY key dedups. +INSERT INTO analytics.identity_signals + (account_id, signal_kind, signal_value, user_id, observed_at, event_id) +SELECT + account_id, + sig.1 AS signal_kind, + sig.2 AS signal_value, + user_id, + received_at AS observed_at, + event_id +FROM analytics.events_raw +ARRAY JOIN arrayFilter( + kv -> kv.1 != '' AND kv.2 != '', + arrayConcat( + [(if(event_name IN ('identify', 'alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig +WHERE user_id != ''; + +-- Policy is view-time, so changing confidence later re-scores all history +-- without mutating evidence rows. +CREATE VIEW IF NOT EXISTS analytics.identity_bindings +AS +SELECT + account_id, + signal_kind, + signal_value, + user_id, + observed_at, + multiIf( + signal_kind = 'email_sha256', 0.95, + signal_kind = 'cookie', 0.90, + signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, + 0.0) AS confidence +FROM analytics.identity_signals; + +-- Conflicting bindings stay visible rather than being collapsed by a +-- latest-wins table. +CREATE VIEW IF NOT EXISTS analytics.identity_conflicts +AS +SELECT + account_id, + signal_kind, + signal_value, + groupUniqArray(user_id) AS user_ids, + uniqExact(user_id) AS user_count, + count() AS observation_count, + max(observed_at) AS last_observed_at +FROM analytics.identity_signals +GROUP BY account_id, signal_kind, signal_value +HAVING user_count > 1; + +-- Resolve explicit user_id first, then the strongest and newest binding +-- reachable from the event's cookie or identifiers, then the raw cookie. +-- The ordered seven-column output is the contract recorded in ADR 0004. +CREATE VIEW IF NOT EXISTS analytics.events_resolved_v3 +AS +WITH event_signal_users AS +( + SELECT + s.account_id AS account_id, + s.event_id AS event_id, + argMax(b.user_id, tuple(b.confidence, b.observed_at)) AS stitched_user + FROM + ( + SELECT + account_id, + event_id, + sig.1 AS signal_kind, + sig.2 AS signal_value + FROM analytics.events_raw + ARRAY JOIN arrayFilter( + kv -> kv.2 != '', + arrayConcat( + [('cookie', anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig + WHERE user_id = '' + ) AS s + INNER JOIN analytics.identity_bindings AS b + ON b.account_id = s.account_id + AND b.signal_kind = s.signal_kind + AND b.signal_value = s.signal_value + WHERE b.confidence >= 0.50 + GROUP BY s.account_id, s.event_id +) +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + multiIf( + e.user_id != '', e.user_id, + r.stitched_user != '', r.stitched_user, + e.anonymous_id) AS canonical_user, + e.properties_json AS properties_json +FROM analytics.events_raw AS e +LEFT JOIN event_signal_users AS r + ON r.account_id = e.account_id AND r.event_id = e.event_id; + +DROP VIEW IF EXISTS analytics.events_resolved; +CREATE VIEW analytics.events_resolved +AS SELECT * FROM analytics.events_resolved_v3; diff --git a/test/fixtures/kansoku/original/0013_retire_identity_map.sql b/test/fixtures/kansoku/original/0013_retire_identity_map.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0013_retire_identity_map.sql @@ -0,0 +1,5 @@ +-- EP-18 M3: identity_signals and identity_bindings supersede the old +-- latest-wins cookie map. Migration 0012 backfilled equivalent cookie evidence, +-- and the pre-retirement stability battery preserved canonical-user results. +DROP TABLE IF EXISTS analytics.mv_identity_map; +DROP TABLE IF EXISTS analytics.identity_map; diff --git a/test/fixtures/kansoku/original/0014_experiment_exposures.sql b/test/fixtures/kansoku/original/0014_experiment_exposures.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0014_experiment_exposures.sql @@ -0,0 +1,53 @@ +-- Experiment exposure rollup: unique users per account x experiment x variant x day. +-- +-- Identity is the raw identity (user_id if present, else anonymous_id), matching +-- the join-free dau/wau rollups. An insert-time materialized view cannot apply +-- query-time identity resolution without freezing stale bindings. The exact, +-- stitched conversion and funnel endpoints read analytics.events_resolved. +-- +-- IF NOT EXISTS plus the union semantics of uniqCombined states make a retry +-- after a partial failure safe: a repeated backfill only merges the same users. + +CREATE TABLE IF NOT EXISTS analytics.experiment_exposures +( + account_id String, + experiment LowCardinality(String), + variant LowCardinality(String), + day Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, experiment, variant, day); + +INSERT INTO analytics.experiment_exposures (account_id, experiment, variant, day, users) +SELECT + account_id, + toString(getSubcolumn(properties_json, 'experiment')) AS experiment, + toString(getSubcolumn(properties_json, 'variant')) AS variant, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE event_name = 'experiment_viewed' + AND (user_id != '' OR anonymous_id != '') + AND match(toString(getSubcolumn(properties_json, 'experiment')), '^[a-z0-9_.:-]{1,128}$') + AND match(toString(getSubcolumn(properties_json, 'variant')), '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; + +-- Inserts made between the backfill above and this trigger creation are not +-- observed. EP-19 lands before production experiment traffic; in development, +-- reset and replay migrations if that window matters. +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_experiment_exposures +TO analytics.experiment_exposures +AS +SELECT + account_id, + toString(getSubcolumn(properties_json, 'experiment')) AS experiment, + toString(getSubcolumn(properties_json, 'variant')) AS variant, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE event_name = 'experiment_viewed' + AND (user_id != '' OR anonymous_id != '') + AND match(toString(getSubcolumn(properties_json, 'experiment')), '^[a-z0-9_.:-]{1,128}$') + AND match(toString(getSubcolumn(properties_json, 'variant')), '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/original/0015_experiment_exposure_string_casts.sql b/test/fixtures/kansoku/original/0015_experiment_exposure_string_casts.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0015_experiment_exposure_string_casts.sql @@ -0,0 +1,23 @@ +-- Correct EP-19's materialized-view input-block behavior without editing the +-- already-applied 0014 migration. When a block contains no experiment paths, +-- getSubcolumn over JSON can retain Dynamic even after toString; ClickHouse 26 +-- rejects that alias as a GROUP BY key before the event-name filter removes the +-- rows. An explicit outer String cast makes every ordinary event block safe. + +DROP TABLE IF EXISTS analytics.mv_experiment_exposures; + +CREATE MATERIALIZED VIEW analytics.mv_experiment_exposures +TO analytics.experiment_exposures +AS +SELECT + account_id, + CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, + CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE event_name = 'experiment_viewed' + AND (user_id != '' OR anonymous_id != '') + AND match(CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String'), '^[a-z0-9_.:-]{1,128}$') + AND match(CAST(toString(getSubcolumn(properties_json, 'variant')), 'String'), '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/original/0016_events_raw_schema_version.sql b/test/fixtures/kansoku/original/0016_events_raw_schema_version.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0016_events_raw_schema_version.sql @@ -0,0 +1,2 @@ +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS schema_version UInt16 DEFAULT 1 AFTER identifiers; diff --git a/test/fixtures/kansoku/original/0017_replay_safe_sources.sql b/test/fixtures/kansoku/original/0017_replay_safe_sources.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0017_replay_safe_sources.sql @@ -0,0 +1,175 @@ +-- EP-30: one immutable logical event before analytical joins/aggregations. +-- Administrative migration: keep the writer stopped throughout. A failed run +-- can be reapplied: replacement states are rebuilt from canonical retained rows, +-- rather than appended to partially populated aggregate tables. +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *, properties_json FROM analytics.events_raw FINAL; + +CREATE OR REPLACE VIEW analytics.identity_bindings +AS +SELECT + account_id, + signal_kind, + signal_value, + user_id, + observed_at, + multiIf( + signal_kind = 'email_sha256', 0.95, + signal_kind = 'cookie', 0.90, + signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, + 0.0) AS confidence +FROM analytics.identity_signals FINAL; + +-- Conflicting bindings stay visible rather than being collapsed by a +-- latest-wins table. +CREATE OR REPLACE VIEW analytics.identity_conflicts +AS +SELECT + account_id, + signal_kind, + signal_value, + groupUniqArray(user_id) AS user_ids, + uniqExact(user_id) AS user_count, + count() AS observation_count, + max(observed_at) AS last_observed_at +FROM analytics.identity_signals FINAL +GROUP BY account_id, signal_kind, signal_value +HAVING user_count > 1; + +CREATE OR REPLACE VIEW analytics.events_resolved_v3 +AS +WITH event_signal_users AS +( + SELECT + s.account_id AS account_id, + s.event_id AS event_id, + argMax(b.user_id, tuple(b.confidence, b.observed_at)) AS stitched_user + FROM + ( + SELECT + account_id, + event_id, + sig.1 AS signal_kind, + sig.2 AS signal_value + FROM analytics.events_canonical + ARRAY JOIN arrayFilter( + kv -> kv.2 != '', + arrayConcat( + [('cookie', anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig + WHERE user_id = '' + ) AS s + INNER JOIN analytics.identity_bindings AS b + ON b.account_id = s.account_id + AND b.signal_kind = s.signal_kind + AND b.signal_value = s.signal_value + WHERE b.confidence >= 0.50 + GROUP BY s.account_id, s.event_id +) +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + multiIf( + e.user_id != '', e.user_id, + r.stitched_user != '', r.stitched_user, + e.anonymous_id) AS canonical_user, + e.properties_json AS properties_json +FROM analytics.events_canonical AS e +LEFT JOIN event_signal_users AS r + ON r.account_id = e.account_id AND r.event_id = e.event_id; + + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; +DROP TABLE IF EXISTS analytics.daily_event_counts_rebuild; +DROP TABLE IF EXISTS analytics.page_views_by_route_rebuild; + +CREATE TABLE analytics.daily_event_counts_rebuild +( + account_id String, + day Date, + event_name String, + events AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, event_name); + +CREATE TABLE analytics.page_views_by_route_rebuild +( + account_id String, + day Date, + route String, + views AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, route); + +INSERT INTO analytics.daily_event_counts_rebuild +SELECT account_id, toDate(occurred_at) AS day, event_name, + uniqExactState(event_id) AS events +FROM analytics.events_canonical +GROUP BY account_id, day, event_name; + +INSERT INTO analytics.page_views_by_route_rebuild +SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, + uniqExactState(event_id) AS views +FROM analytics.events_canonical +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; + +-- Verify complete replacement groups before publishing either table. +SELECT throwIf(count() > 0, 'canonical daily-event rebuild mismatch') +FROM +( + SELECT account_id, toDate(occurred_at) AS day, event_name, count() AS n + FROM analytics.events_canonical GROUP BY account_id, day, event_name +) AS canonical +FULL OUTER JOIN +( + SELECT account_id, day, event_name, uniqExactMerge(events) AS n + FROM analytics.daily_event_counts_rebuild GROUP BY account_id, day, event_name +) AS replacement USING (account_id, day, event_name) +WHERE ifNull(canonical.n, 0) != ifNull(replacement.n, 0); + +SELECT throwIf(count() > 0, 'canonical page-view rebuild mismatch') +FROM +( + SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, count() AS n + FROM analytics.events_canonical WHERE event_name = 'page_view' + GROUP BY account_id, day, route +) AS canonical +FULL OUTER JOIN +( + SELECT account_id, day, route, uniqExactMerge(views) AS n + FROM analytics.page_views_by_route_rebuild GROUP BY account_id, day, route +) AS replacement USING (account_id, day, route) +WHERE ifNull(canonical.n, 0) != ifNull(replacement.n, 0); + +-- Each swap is atomic. The writer stays stopped across both. On interruption, +-- discard/rebuild the temporary tables and swap again from canonical data; +-- this works whether the public name has its old or its replacement schema. +EXCHANGE TABLES analytics.daily_event_counts AND analytics.daily_event_counts_rebuild; +EXCHANGE TABLES analytics.page_views_by_route AND analytics.page_views_by_route_rebuild; +DROP TABLE analytics.daily_event_counts_rebuild; +DROP TABLE analytics.page_views_by_route_rebuild; + +-- Trigger inputs are the current inserted block. Distinct-ID states compose +-- across arbitrary retry batches, parts, restarts and old archive replays. +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts AS +SELECT account_id, toDate(occurred_at) AS day, event_name, + uniqExactState(event_id) AS events +FROM analytics.events_raw +GROUP BY account_id, day, event_name; + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route AS +SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, + uniqExactState(event_id) AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/original/0018_conservative_identity_resolution.sql b/test/fixtures/kansoku/original/0018_conservative_identity_resolution.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0018_conservative_identity_resolution.sql @@ -0,0 +1,293 @@ +-- EP-32: writer-stopped identity evidence/index rebuild and conservative exact resolution. +-- Reapply after interruption with the writer still stopped. Canonical raw facts +-- are authoritative; max/unique state rebuilds never append incompatible history. +-- Length-prefixed components are lossless, including separators and UTF-8. +-- Persist this key so metrics read/hash one string instead of five key vectors. +ALTER TABLE analytics.events_raw ADD COLUMN IF NOT EXISTS identity_context_key String MATERIALIZED concat(toString(length(account_id)),':',account_id,toString(length(source_service)),':',source_service,toString(length(session_id)),':',session_id,toString(length(anonymous_id)),':',anonymous_id,toString(length(identifiers)),':',identifiers); +ALTER TABLE analytics.events_raw MATERIALIZE COLUMN identity_context_key SETTINGS mutations_sync=2; +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *,properties_json,identity_context_key FROM analytics.events_raw FINAL; + +DROP TABLE IF EXISTS analytics.mv_identity_signal_users; +DROP TABLE IF EXISTS analytics.mv_identity_session_users; +DROP TABLE IF EXISTS analytics.mv_identity_signals; +DROP TABLE IF EXISTS analytics.identity_signals_rebuild; +CREATE TABLE analytics.identity_signals_rebuild +( + account_id String, signal_kind LowCardinality(String), signal_value String, + user_id String, observed_at DateTime64(3, 'UTC'), event_id String, + received_at DateTime64(3, 'UTC'), source_service String, session_id String +) +ENGINE = ReplacingMergeTree +ORDER BY (account_id, signal_kind, signal_value, user_id, observed_at, event_id) +TTL toDateTime(observed_at) + INTERVAL 180 DAY + INTERVAL 1 SECOND DELETE WHERE signal_kind = 'fingerprint'; + +INSERT INTO analytics.identity_signals_rebuild +SELECT account_id, sig.1, sig.2, user_id, occurred_at, event_id, received_at, source_service, session_id +FROM analytics.events_canonical +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.1 != '' AND kv.2 != '', + arrayConcat([(if(event_name IN ('identify','identify_once','alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String')))) AS sig +WHERE user_id != ''; +EXCHANGE TABLES analytics.identity_signals AND analytics.identity_signals_rebuild; +DROP TABLE analytics.identity_signals_rebuild; + +CREATE MATERIALIZED VIEW analytics.mv_identity_signals TO analytics.identity_signals AS +SELECT account_id, sig.1 AS signal_kind, sig.2 AS signal_value, user_id, + occurred_at AS observed_at, event_id, received_at, source_service, session_id +FROM analytics.events_raw +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.1 != '' AND kv.2 != '', + arrayConcat([(if(event_name IN ('identify','identify_once','alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String')))) AS sig +WHERE user_id != ''; + +-- Compact exact indexes retain one latest timestamp per observed signal/user, +-- and one per cookie/session/user. Max is replay-safe; evidence remains auditable. +DROP TABLE IF EXISTS analytics.mv_identity_signal_users; +DROP TABLE IF EXISTS analytics.mv_identity_session_users; +CREATE TABLE IF NOT EXISTS analytics.identity_signal_users +( + account_id String, signal_kind LowCardinality(String), signal_value String, user_id String, + observed_at SimpleAggregateFunction(max, DateTime64(3, 'UTC')) +) +ENGINE = AggregatingMergeTree ORDER BY (account_id, signal_kind, signal_value, user_id) +TTL toDateTime(observed_at) + INTERVAL 180 DAY + INTERVAL 1 SECOND DELETE WHERE signal_kind = 'fingerprint'; +CREATE TABLE IF NOT EXISTS analytics.identity_session_users +( + account_id String, source_service String, session_id String, signal_value String, user_id String, + observed_at SimpleAggregateFunction(max, DateTime64(3, 'UTC')) +) +ENGINE = AggregatingMergeTree ORDER BY (account_id, source_service, session_id, signal_value, user_id); +DROP TABLE IF EXISTS analytics.identity_signal_users_rebuild; +DROP TABLE IF EXISTS analytics.identity_session_users_rebuild; +CREATE TABLE analytics.identity_signal_users_rebuild AS analytics.identity_signal_users; +CREATE TABLE analytics.identity_session_users_rebuild AS analytics.identity_session_users; +INSERT INTO analytics.identity_signal_users_rebuild +SELECT account_id, signal_kind, signal_value, user_id, max(observed_at) +FROM analytics.identity_signals FINAL GROUP BY account_id, signal_kind, signal_value, user_id; +INSERT INTO analytics.identity_session_users_rebuild +SELECT account_id, source_service, session_id, signal_value, user_id, max(observed_at) +FROM analytics.identity_signals FINAL WHERE signal_kind = 'cookie' AND session_id != '' +GROUP BY account_id, source_service, session_id, signal_value, user_id; +EXCHANGE TABLES analytics.identity_signal_users AND analytics.identity_signal_users_rebuild; +EXCHANGE TABLES analytics.identity_session_users AND analytics.identity_session_users_rebuild; +DROP TABLE analytics.identity_signal_users_rebuild; +DROP TABLE analytics.identity_session_users_rebuild; +CREATE MATERIALIZED VIEW analytics.mv_identity_signal_users TO analytics.identity_signal_users AS +SELECT account_id, signal_kind, signal_value, user_id, max(observed_at) AS observed_at +FROM analytics.identity_signals GROUP BY account_id, signal_kind, signal_value, user_id; +CREATE MATERIALIZED VIEW analytics.mv_identity_session_users TO analytics.identity_session_users AS +SELECT account_id, source_service, session_id, signal_value, user_id, max(observed_at) AS observed_at +FROM analytics.identity_signals WHERE signal_kind = 'cookie' AND session_id != '' +GROUP BY account_id, source_service, session_id, signal_value, user_id; + +-- Identity depends on a context, not each individual event. Keep an exact +-- replay-safe per-day signal index so repeated session activity does not need +-- raw JSON expansion on every read. Raw candidates still bound returned facts. +DROP TABLE IF EXISTS analytics.mv_identity_context_signals; +CREATE TABLE IF NOT EXISTS analytics.identity_context_signals +( + account_id String, day Date, + context_key Tuple(String,String,String,String), + signal_kind LowCardinality(String), signal_value String +) +ENGINE = ReplacingMergeTree ORDER BY (account_id,day,context_key,signal_kind,signal_value); +DROP TABLE IF EXISTS analytics.identity_context_signals_rebuild; +CREATE TABLE analytics.identity_context_signals_rebuild AS analytics.identity_context_signals; +INSERT INTO analytics.identity_context_signals_rebuild +SELECT account_id,toDate(occurred_at),tuple(source_service,session_id,anonymous_id,identifiers),sig.1,sig.2 +FROM analytics.events_canonical +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.2!='', + arrayConcat([('cookie',anonymous_id)],JSONExtractKeysAndValues(identifiers,'String')))) AS sig +WHERE user_id=''; +EXCHANGE TABLES analytics.identity_context_signals AND analytics.identity_context_signals_rebuild; +DROP TABLE analytics.identity_context_signals_rebuild; +CREATE MATERIALIZED VIEW analytics.mv_identity_context_signals TO analytics.identity_context_signals AS +SELECT account_id,toDate(occurred_at) AS day,tuple(source_service,session_id,anonymous_id,identifiers) AS context_key, + sig.1 AS signal_kind,sig.2 AS signal_value +FROM analytics.events_raw +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.2!='', + arrayConcat([('cookie',anonymous_id)],JSONExtractKeysAndValues(identifiers,'String')))) AS sig +WHERE user_id=''; + +CREATE OR REPLACE VIEW analytics.identity_bindings AS +SELECT account_id, signal_kind, signal_value, user_id, observed_at, + multiIf(signal_kind IN ('email_hmac','email_sha256'), 0.95, + signal_kind = 'cookie', 0.90, signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, 0.0) AS confidence +FROM analytics.identity_signals FINAL +WHERE signal_kind != 'fingerprint' OR observed_at >= now64(3) - INTERVAL 180 DAY; +CREATE OR REPLACE VIEW analytics.identity_conflicts AS +SELECT account_id, signal_kind, signal_value, arraySort(groupUniqArray(user_id)) AS user_ids, + uniqExact(user_id) AS user_count, count() AS observation_count, max(observed_at) AS last_observed_at +FROM analytics.identity_bindings GROUP BY account_id, signal_kind, signal_value HAVING user_count > 1; + +-- Parameterized candidates are bounded BEFORE mapped-signal expansion. Evidence +-- is indexed separately and is intentionally not bounded by the candidate dates. +-- all_accounts is an explicit administrative/public-SQL flag, never inferred +-- from an empty account ID. HTTP always passes zero. +CREATE OR REPLACE VIEW analytics.events_resolved_internal AS +WITH +candidates AS +( + SELECT * FROM analytics.events_canonical + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND occurred_at >= {scope_from:DateTime64(3, 'UTC')} + AND occurred_at < {scope_to:DateTime64(3, 'UTC')} +), +candidate_signals AS MATERIALIZED +( + SELECT DISTINCT account_id, context_key, context_key.1 AS source_service, context_key.2 AS session_id, + signal_kind, signal_value + FROM analytics.identity_context_signals FINAL + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND day >= toDate({scope_from:DateTime64(3, 'UTC')}) + AND toDateTime64(day,3,'UTC') < {scope_to:DateTime64(3, 'UTC')} +), +bindings AS +( + SELECT account_id, signal_kind, signal_value, uniqExact(user_id) AS users, min(user_id) AS unique_user, + multiIf(signal_kind IN ('email_hmac','email_sha256'), 0.95, + signal_kind = 'cookie', 0.90, signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, 0.0) AS confidence + FROM analytics.identity_signal_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,signal_kind,signal_value) IN (SELECT account_id,signal_kind,signal_value FROM candidate_signals) + AND (signal_kind != 'fingerprint' OR observed_at >= now64(3) - INTERVAL 180 DAY) + GROUP BY account_id, signal_kind, signal_value +), +session_users AS +( + SELECT account_id, source_service, session_id, signal_value, uniqExact(user_id) AS users, + min(user_id) AS unique_user + FROM analytics.identity_session_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,source_service,session_id,signal_value) IN + (SELECT account_id,source_service,session_id,signal_value FROM candidate_signals WHERE signal_kind='cookie' AND session_id!='') + GROUP BY account_id, source_service, session_id, signal_value +), +matches AS +( + SELECT m.account_id AS account_id, m.context_key AS context_key, + if(m.signal_kind = 'cookie' AND s.users > 0, s.unique_user, b.unique_user) AS candidate_user, + b.confidence AS confidence + FROM candidate_signals AS m + LEFT JOIN bindings AS b ON b.account_id=m.account_id AND b.signal_kind=m.signal_kind AND b.signal_value=m.signal_value + LEFT JOIN session_users AS s ON m.signal_kind='cookie' AND s.account_id=m.account_id + AND s.source_service=m.source_service AND s.session_id=m.session_id AND s.signal_value=m.signal_value + WHERE if(m.signal_kind='cookie' AND s.users > 0, s.users, b.users) = 1 AND b.confidence >= 0.50 +), +best AS +( + SELECT account_id, context_key, groupArray(tuple(candidate_user,confidence)) AS hits, + arrayMax(arrayMap(h -> h.2,hits)) AS winning_confidence, + arrayDistinct(arrayMap(h -> h.1,arrayFilter(h -> h.2=winning_confidence,hits))) AS winners, + if(length(winners)=1,winners[1],'') AS stitched_user + FROM matches GROUP BY account_id, context_key +), +subjects AS +( + SELECT e.* EXCEPT(account_id), e.account_id AS account_id, + multiIf(e.user_id!='','user',m.stitched_user!='','user',e.anonymous_id!='','anonymous','') AS subject_kind, + multiIf(e.user_id!='',e.user_id,m.stitched_user!='',m.stitched_user,e.anonymous_id) AS subject_id + FROM candidates AS e + LEFT JOIN best AS m ON m.account_id=e.account_id AND m.context_key=tuple(e.source_service,e.session_id,e.anonymous_id,e.identifiers) +) +SELECT *, if(subject_kind='', '', concat(subject_kind, ':', subject_id)) AS canonical_user +FROM subjects +SETTINGS enable_materialized_cte=1; + +CREATE OR REPLACE VIEW analytics.events_resolved_v3 AS +SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json +FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='', + scope_from=toDateTime64('1970-01-01 00:00:00',3,'UTC'),scope_to=toDateTime64('2100-01-01 00:00:00',3,'UTC')); +CREATE OR REPLACE VIEW analytics.events_resolved AS SELECT * FROM analytics.events_resolved_v3; + +DROP TABLE IF EXISTS analytics.mv_dau; +DROP TABLE IF EXISTS analytics.dau_rebuild; +CREATE TABLE analytics.dau_rebuild AS analytics.dau; +INSERT INTO analytics.dau_rebuild (account_id,day,users) +SELECT account_id, toDate(occurred_at) AS day, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_canonical WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; +EXCHANGE TABLES analytics.dau AND analytics.dau_rebuild; +DROP TABLE analytics.dau_rebuild; +CREATE MATERIALIZED VIEW analytics.mv_dau TO analytics.dau AS +SELECT account_id, toDate(occurred_at) AS day, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_raw WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; + +DROP TABLE IF EXISTS analytics.mv_wau; +DROP TABLE IF EXISTS analytics.wau_rebuild; +CREATE TABLE analytics.wau_rebuild AS analytics.wau; +INSERT INTO analytics.wau_rebuild (account_id,week,users) +SELECT account_id, toMonday(occurred_at) AS week, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_canonical WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; +EXCHANGE TABLES analytics.wau AND analytics.wau_rebuild; +DROP TABLE analytics.wau_rebuild; +CREATE MATERIALIZED VIEW analytics.mv_wau TO analytics.wau AS +SELECT account_id, toMonday(occurred_at) AS week, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_raw WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; + +DROP TABLE IF EXISTS analytics.mv_experiment_exposures; +DROP TABLE IF EXISTS analytics.experiment_exposures_rebuild; +CREATE TABLE analytics.experiment_exposures_rebuild AS analytics.experiment_exposures; +INSERT INTO analytics.experiment_exposures_rebuild (account_id,experiment,variant,day,users) +SELECT account_id, toDate(occurred_at) AS day, CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_canonical WHERE event_name = 'experiment_viewed' AND (user_id != '' OR anonymous_id != '') AND match(experiment, '^[a-z0-9_.:-]{1,128}$') AND match(variant, '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; +EXCHANGE TABLES analytics.experiment_exposures AND analytics.experiment_exposures_rebuild; +DROP TABLE analytics.experiment_exposures_rebuild; +CREATE MATERIALIZED VIEW analytics.mv_experiment_exposures TO analytics.experiment_exposures AS +SELECT account_id, toDate(occurred_at) AS day, CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_raw WHERE event_name = 'experiment_viewed' AND (user_id != '' OR anonymous_id != '') AND match(experiment, '^[a-z0-9_.:-]{1,128}$') AND match(variant, '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; + +-- Exact context snapshots. The connection refresh deadline is the earliest +-- future fingerprint expiry in its evidence. Parameters defer temporary binding. +CREATE OR REPLACE VIEW analytics.identity_context_resolution AS +WITH + (a,k,v) -> arrayDistinct(arrayMap(p -> p.1, + arrayFilter(p -> k!='fingerprint' OR p.2>=now64(3)-INTERVAL 180 DAY, + joinGet('kansoku_signal_cache','observations',a,k,v)))) AS global_users, + (a,svc,sid,v) -> joinGet('kansoku_session_cache','users',a,svc,sid,v) AS session_users, + k -> multiIf(k IN ('email_hmac','email_sha256'),0.95,k='cookie',0.90,k='device',0.80,k='fingerprint',0.50,0.0) AS confidence, + (a,svc,sid,k,v) -> if(k='cookie' AND sid!='' AND notEmpty(session_users(a,svc,sid,v)), + session_users(a,svc,sid,v),global_users(a,k,v)) AS bound_users +SELECT * EXCEPT(targets,best_confidence,winners),if(length(winners)=1,winners[1],'') AS stitched_user +FROM +( + SELECT *,arrayMax(arrayMap(h -> h.2,targets)) AS best_confidence, + arrayDistinct(arrayMap(h -> h.1[1],arrayFilter(h -> h.2=best_confidence,targets))) AS winners + FROM + ( + SELECT *,arrayFilter(h -> length(h.1)=1 AND h.2>=0.50, + arrayMap(sig -> tuple(bound_users(account_id,source_service,session_id,sig.1,sig.2),confidence(sig.1)), + arrayDistinct(arrayFilter(sig -> sig.2!='',arrayConcat([('cookie',anonymous_id)], + JSONExtractKeysAndValues(identifiers,'String')))))) AS targets + FROM + ( + SELECT DISTINCT account_id,context_key.1 AS source_service,context_key.2 AS session_id, + context_key.3 AS anonymous_id,context_key.4 AS identifiers + FROM analytics.identity_context_signals FINAL + WHERE ({scope_all_accounts:UInt8}=1 OR account_id={scope_account:String}) + AND day>={scope_from:Date} AND day<{scope_to:Date} + + ) + ) +); + +-- HTTP-only relation. The physical query connection refreshes this exact +-- snapshot on index revisions and before its earliest fingerprint expiry. +CREATE OR REPLACE VIEW analytics.events_resolved_http AS +SELECT *,canonical_user, + if(startsWith(canonical_user,'user:'),'user',if(startsWith(canonical_user,'anonymous:'),'anonymous','')) AS subject_kind, + substring(canonical_user,length(subject_kind)+2) AS subject_id +FROM analytics.events_canonical +ARRAY JOIN [if(user_id!='',concat('user:',user_id), + joinGet('kansoku_context_cache','canonical_user',identity_context_key))] AS canonical_user +WHERE ({scope_all_accounts:UInt8}=1 OR account_id={scope_account:String}) + AND throwIf(joinGet('kansoku_identity_clock','deadline',toUInt8(0))<=toUnixTimestamp64Milli(now64(3)), 'identity snapshot expired')=0 + AND occurred_at>={scope_from:DateTime64(3,'UTC')} AND occurred_at<{scope_to:DateTime64(3,'UTC')}; diff --git a/test/fixtures/kansoku/original/0019_web_enrichment.sql b/test/fixtures/kansoku/original/0019_web_enrichment.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0019_web_enrichment.sql @@ -0,0 +1,106 @@ +-- EP-33: additive private enrichment; no producer-chosen categorical columns. +-- Existing development facts retain unknown/version-zero enrichment. Reclassification +-- is an explicit rebuild operation, never an implicit change to accepted events. +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS browser_family LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS os_family LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS device_class LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS is_bot Bool DEFAULT false, + ADD COLUMN IF NOT EXISTS channel LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS country_code LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS enrichment_version UInt16 DEFAULT 0, + ADD COLUMN IF NOT EXISTS context_json JSON(max_dynamic_paths=256) + MATERIALIZED CAST(if(isValidJSON(context),context,'{}') AS JSON(max_dynamic_paths=256)); + +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *,properties_json,identity_context_key,context_json FROM analytics.events_raw FINAL; + +-- Refresh view headers after adding raw columns; keep the public seven-column shape. +CREATE OR REPLACE VIEW analytics.events_resolved_internal AS +WITH +candidates AS +( + SELECT * FROM analytics.events_canonical + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND occurred_at >= {scope_from:DateTime64(3, 'UTC')} + AND occurred_at < {scope_to:DateTime64(3, 'UTC')} +), +candidate_signals AS MATERIALIZED +( + SELECT DISTINCT account_id, context_key, context_key.1 AS source_service, context_key.2 AS session_id, + signal_kind, signal_value + FROM analytics.identity_context_signals FINAL + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND day >= toDate({scope_from:DateTime64(3, 'UTC')}) + AND toDateTime64(day,3,'UTC') < {scope_to:DateTime64(3, 'UTC')} +), +bindings AS +( + SELECT account_id, signal_kind, signal_value, uniqExact(user_id) AS users, min(user_id) AS unique_user, + multiIf(signal_kind IN ('email_hmac','email_sha256'), 0.95, + signal_kind = 'cookie', 0.90, signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, 0.0) AS confidence + FROM analytics.identity_signal_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,signal_kind,signal_value) IN (SELECT account_id,signal_kind,signal_value FROM candidate_signals) + AND (signal_kind != 'fingerprint' OR observed_at >= now64(3) - INTERVAL 180 DAY) + GROUP BY account_id, signal_kind, signal_value +), +session_users AS +( + SELECT account_id, source_service, session_id, signal_value, uniqExact(user_id) AS users, + min(user_id) AS unique_user + FROM analytics.identity_session_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,source_service,session_id,signal_value) IN + (SELECT account_id,source_service,session_id,signal_value FROM candidate_signals WHERE signal_kind='cookie' AND session_id!='') + GROUP BY account_id, source_service, session_id, signal_value +), +matches AS +( + SELECT m.account_id AS account_id, m.context_key AS context_key, + if(m.signal_kind = 'cookie' AND s.users > 0, s.unique_user, b.unique_user) AS candidate_user, + b.confidence AS confidence + FROM candidate_signals AS m + LEFT JOIN bindings AS b ON b.account_id=m.account_id AND b.signal_kind=m.signal_kind AND b.signal_value=m.signal_value + LEFT JOIN session_users AS s ON m.signal_kind='cookie' AND s.account_id=m.account_id + AND s.source_service=m.source_service AND s.session_id=m.session_id AND s.signal_value=m.signal_value + WHERE if(m.signal_kind='cookie' AND s.users > 0, s.users, b.users) = 1 AND b.confidence >= 0.50 +), +best AS +( + SELECT account_id, context_key, groupArray(tuple(candidate_user,confidence)) AS hits, + arrayMax(arrayMap(h -> h.2,hits)) AS winning_confidence, + arrayDistinct(arrayMap(h -> h.1,arrayFilter(h -> h.2=winning_confidence,hits))) AS winners, + if(length(winners)=1,winners[1],'') AS stitched_user + FROM matches GROUP BY account_id, context_key +), +subjects AS +( + SELECT e.* EXCEPT(account_id), e.account_id AS account_id, + multiIf(e.user_id!='','user',m.stitched_user!='','user',e.anonymous_id!='','anonymous','') AS subject_kind, + multiIf(e.user_id!='',e.user_id,m.stitched_user!='',m.stitched_user,e.anonymous_id) AS subject_id + FROM candidates AS e + LEFT JOIN best AS m ON m.account_id=e.account_id AND m.context_key=tuple(e.source_service,e.session_id,e.anonymous_id,e.identifiers) +) +SELECT *, if(subject_kind='', '', concat(subject_kind, ':', subject_id)) AS canonical_user +FROM subjects +SETTINGS enable_materialized_cte=1; + +CREATE OR REPLACE VIEW analytics.events_resolved_v3 AS +SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json +FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='', + scope_from=toDateTime64('1970-01-01 00:00:00',3,'UTC'),scope_to=toDateTime64('2100-01-01 00:00:00',3,'UTC')); +CREATE OR REPLACE VIEW analytics.events_resolved AS SELECT * FROM analytics.events_resolved_v3; + + +CREATE OR REPLACE VIEW analytics.events_resolved_http AS +SELECT *,canonical_user, + if(startsWith(canonical_user,'user:'),'user',if(startsWith(canonical_user,'anonymous:'),'anonymous','')) AS subject_kind, + substring(canonical_user,length(subject_kind)+2) AS subject_id +FROM analytics.events_canonical +ARRAY JOIN [if(user_id!='',concat('user:',user_id), + joinGet('kansoku_context_cache','canonical_user',identity_context_key))] AS canonical_user +WHERE ({scope_all_accounts:UInt8}=1 OR account_id={scope_account:String}) + AND throwIf(joinGet('kansoku_identity_clock','deadline',toUInt8(0))<=toUnixTimestamp64Milli(now64(3)), 'identity snapshot expired')=0 + AND occurred_at>={scope_from:DateTime64(3,'UTC')} AND occurred_at<{scope_to:DateTime64(3,'UTC')}; diff --git a/test/fixtures/kansoku/original/0020_replay_safe_sessions.sql b/test/fixtures/kansoku/original/0020_replay_safe_sessions.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0020_replay_safe_sessions.sql @@ -0,0 +1,53 @@ +-- EP-34: merge whole retained sessions, never per-block start-day groups. +-- The operator keeps writers stopped during migrations/rebuilds. Every state +-- is idempotent for immutable event IDs: sets, min/max and deterministic argmin/max. +DROP TABLE IF EXISTS analytics.mv_sessions; +DROP TABLE IF EXISTS analytics.sessions_rebuild; +CREATE TABLE analytics.sessions_rebuild +ENGINE = AggregatingMergeTree +ORDER BY (account_id,source_service,session_id) +AS +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM analytics.events_canonical +WHERE source='web' AND session_id!='' AND 0 +GROUP BY account_id,source_service,session_id; + +INSERT INTO analytics.sessions_rebuild +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM analytics.events_canonical +WHERE source='web' AND session_id!='' +GROUP BY account_id,source_service,session_id; + +CREATE TABLE IF NOT EXISTS analytics.sessions AS analytics.sessions_rebuild; +EXCHANGE TABLES analytics.sessions AND analytics.sessions_rebuild; +DROP TABLE analytics.sessions_rebuild; + +CREATE MATERIALIZED VIEW analytics.mv_sessions TO analytics.sessions AS +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM analytics.events_raw +WHERE source='web' AND session_id!='' +GROUP BY account_id,source_service,session_id; diff --git a/test/fixtures/kansoku/original/0021_governed_current_user_traits.sql b/test/fixtures/kansoku/original/0021_governed_current_user_traits.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0021_governed_current_user_traits.sql @@ -0,0 +1,102 @@ +-- EP-40: governed current traits. Keep ingest/writers fenced during DDL/rebuilds. +-- Identify v1 is identity-only. Eligibility mirrors the shared registry and +-- checks actual stored schema_version and the immutable server validation marker. +-- Exact decimal integrality is validated before Float64/native JSON conversion. +-- Strings are stored decoded; numeric values use the metrics Float64 contract. +ALTER TABLE analytics.events_raw ADD COLUMN IF NOT EXISTS governed_traits_valid Bool DEFAULT false; + +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *,properties_json,identity_context_key,context_json FROM analytics.events_raw FINAL; + +CREATE TABLE IF NOT EXISTS analytics.trait_event_erasures +(account_id String,event_id String) +ENGINE=ReplacingMergeTree ORDER BY (account_id,event_id); + +CREATE TABLE IF NOT EXISTS analytics.user_properties +(account_id String,user_id String,name String, + value AggregateFunction(argMax,Tuple(String,String,Float64),Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,user_id,name); + +CREATE TABLE IF NOT EXISTS analytics.user_properties_once +(account_id String,user_id String,name String, + value AggregateFunction(argMin,Tuple(String,String,Float64),Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,user_id,name); + +CREATE OR REPLACE VIEW analytics.user_trait_events AS +WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_canonical +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1); + +DROP TABLE IF EXISTS analytics.mv_user_properties; +CREATE MATERIALIZED VIEW analytics.mv_user_properties TO analytics.user_properties AS +SELECT account_id,user_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_raw +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify' GROUP BY account_id,user_id,name; + +INSERT INTO analytics.user_properties +SELECT account_id,user_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (SELECT * FROM analytics.user_trait_events) WHERE event_name='identify' GROUP BY account_id,user_id,name; + +DROP TABLE IF EXISTS analytics.mv_user_properties_once; +CREATE MATERIALIZED VIEW analytics.mv_user_properties_once TO analytics.user_properties_once AS +SELECT account_id,user_id,name,argMinState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_raw +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify_once' GROUP BY account_id,user_id,name; + +INSERT INTO analytics.user_properties_once +SELECT account_id,user_id,name,argMinState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (SELECT * FROM analytics.user_trait_events) WHERE event_name='identify_once' GROUP BY account_id,user_id,name; + +CREATE OR REPLACE VIEW analytics.user_traits AS +SELECT account_id,user_id,name,selected.1 AS trait_type,selected.2 AS text_value,selected.3 AS number_value +FROM ( + SELECT account_id,user_id,name,argMax(candidate,priority) AS selected + FROM ( + SELECT account_id,user_id,name,argMaxMerge(value) AS candidate,toUInt8(1) AS priority + FROM analytics.user_properties GROUP BY account_id,user_id,name + UNION ALL + SELECT account_id,user_id,name,argMinMerge(value) AS candidate,toUInt8(0) AS priority + FROM analytics.user_properties_once GROUP BY account_id,user_id,name + ) GROUP BY account_id,user_id,name +); diff --git a/test/fixtures/kansoku/original/0022_applied_retention_policy.sql b/test/fixtures/kansoku/original/0022_applied_retention_policy.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0022_applied_retention_policy.sql @@ -0,0 +1,124 @@ +-- Applied cutoff advances only through fenced kansoku-admin apply-retention. +-- Physical TTL cleanup is separate from the immediate canonical predicate. +CREATE TABLE IF NOT EXISTS analytics.retention_policy +( + singleton UInt8, + revision UInt64, + cutoff DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(revision) ORDER BY singleton; +INSERT INTO analytics.retention_policy VALUES (1,0,'1970-01-01 00:00:00'); +CREATE OR REPLACE VIEW analytics.events_canonical AS +SELECT *, properties_json, identity_context_key, context_json +FROM analytics.events_raw FINAL +WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy); + +-- Every raw-triggered projection filters the applied cutoff before updating state. +-- Keep the raw table inside the subquery so native insert-block triggers still fire. + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts TO analytics.daily_event_counts AS +SELECT account_id, toDate(occurred_at) AS day, event_name, + uniqExactState(event_id) AS events +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +GROUP BY account_id, day, event_name; + +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route TO analytics.page_views_by_route AS +SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, + uniqExactState(event_id) AS views +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; + +DROP TABLE IF EXISTS analytics.mv_identity_signals; +CREATE MATERIALIZED VIEW analytics.mv_identity_signals TO analytics.identity_signals AS +SELECT account_id, sig.1 AS signal_kind, sig.2 AS signal_value, user_id, + occurred_at AS observed_at, event_id, received_at, source_service, session_id +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.1 != '' AND kv.2 != '', + arrayConcat([(if(event_name IN ('identify','identify_once','alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String')))) AS sig +WHERE user_id != ''; + +DROP TABLE IF EXISTS analytics.mv_identity_context_signals; +CREATE MATERIALIZED VIEW analytics.mv_identity_context_signals TO analytics.identity_context_signals AS +SELECT account_id,toDate(occurred_at) AS day,tuple(source_service,session_id,anonymous_id,identifiers) AS context_key, + sig.1 AS signal_kind,sig.2 AS signal_value +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.2!='', + arrayConcat([('cookie',anonymous_id)],JSONExtractKeysAndValues(identifiers,'String')))) AS sig +WHERE user_id=''; + +DROP TABLE IF EXISTS analytics.mv_dau; +CREATE MATERIALIZED VIEW analytics.mv_dau TO analytics.dau AS +SELECT account_id, toDate(occurred_at) AS day, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; + +DROP TABLE IF EXISTS analytics.mv_wau; +CREATE MATERIALIZED VIEW analytics.mv_wau TO analytics.wau AS +SELECT account_id, toMonday(occurred_at) AS week, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; + +DROP TABLE IF EXISTS analytics.mv_experiment_exposures; +CREATE MATERIALIZED VIEW analytics.mv_experiment_exposures TO analytics.experiment_exposures AS +SELECT account_id, toDate(occurred_at) AS day, CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) WHERE event_name = 'experiment_viewed' AND (user_id != '' OR anonymous_id != '') AND match(experiment, '^[a-z0-9_.:-]{1,128}$') AND match(variant, '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; + +DROP TABLE IF EXISTS analytics.mv_sessions; +CREATE MATERIALIZED VIEW analytics.mv_sessions TO analytics.sessions AS +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +WHERE source='web' AND session_id!='' +GROUP BY account_id,source_service,session_id; + +DROP TABLE IF EXISTS analytics.mv_user_properties; +CREATE MATERIALIZED VIEW analytics.mv_user_properties TO analytics.user_properties AS +SELECT account_id,user_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify' GROUP BY account_id,user_id,name; + +DROP TABLE IF EXISTS analytics.mv_user_properties_once; +CREATE MATERIALIZED VIEW analytics.mv_user_properties_once TO analytics.user_properties_once AS +SELECT account_id,user_id,name,argMinState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify_once' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/original/0023_stable_reader_surface.sql b/test/fixtures/kansoku/original/0023_stable_reader_surface.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0023_stable_reader_surface.sql @@ -0,0 +1,16 @@ +-- The seven-column contract remains frozen; SELECT rights on these deliberate +-- projections do not require rights on the changing identity/storage relations. +ALTER TABLE analytics.events_resolved_v3 MODIFY SQL SECURITY DEFINER DEFINER = CURRENT_USER; +ALTER TABLE analytics.events_resolved MODIFY SQL SECURITY DEFINER DEFINER = CURRENT_USER; + +-- Explicit bounded candidate parameters preserve conservative current identity +-- semantics without the HTTP connection's private Join-table cache. +CREATE OR REPLACE VIEW analytics.events_analytics_v1 +DEFINER = CURRENT_USER SQL SECURITY DEFINER +AS SELECT + event_id, event_name, occurred_at, url, account_id, canonical_user, properties_json, + source, source_service, session_id, is_bot, browser_family, os_family, + device_class, channel, country_code +FROM analytics.events_resolved_internal( + scope_all_accounts=0, scope_account={account:String}, + scope_from={from_time:DateTime64(3, 'UTC')}, scope_to={to_time:DateTime64(3, 'UTC')}); diff --git a/test/fixtures/kansoku/original/0024_cohort_members.sql b/test/fixtures/kansoku/original/0024_cohort_members.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0024_cohort_members.sql @@ -0,0 +1,10 @@ +CREATE TABLE analytics.cohort_members +( + account_id String, + cohort_id String, + user_id String, + version DateTime64(3, 'UTC'), + present UInt8 +) +ENGINE = ReplacingMergeTree(version) +ORDER BY (account_id, cohort_id, user_id); diff --git a/test/fixtures/kansoku/original/0025_organization_analytics.sql b/test/fixtures/kansoku/original/0025_organization_analytics.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/0025_organization_analytics.sql @@ -0,0 +1,67 @@ +-- EP-45: append the organization without changing the first seven columns. +-- Rollback (after reverting consumers): +-- CREATE OR REPLACE VIEW analytics.events_resolved_v3 DEFINER = CURRENT_USER SQL SECURITY DEFINER AS +-- SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json +-- FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='',scope_from=toDateTime64('1970-01-01',3,'UTC'),scope_to=toDateTime64('2100-01-01',3,'UTC')); +-- CREATE OR REPLACE VIEW analytics.events_resolved DEFINER = CURRENT_USER SQL SECURITY DEFINER AS SELECT * FROM analytics.events_resolved_v3; +CREATE OR REPLACE VIEW analytics.events_resolved_v3 +DEFINER = CURRENT_USER SQL SECURITY DEFINER AS +SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json,organization_id +FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='',scope_from=toDateTime64('1970-01-01',3,'UTC'),scope_to=toDateTime64('2100-01-01',3,'UTC')); +CREATE OR REPLACE VIEW analytics.events_resolved +DEFINER = CURRENT_USER SQL SECURITY DEFINER AS SELECT * FROM analytics.events_resolved_v3; + +CREATE TABLE IF NOT EXISTS analytics.organization_properties +(account_id String,organization_id String,name String, + value AggregateFunction(argMax,Tuple(String,String,Float64),Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,organization_id,name); +CREATE OR REPLACE VIEW analytics.organization_trait_events AS +WITH toFloat64OrNull(toString(getSubcolumn(properties_json,'seats'))) AS seat_number +SELECT account_id,organization_id,event_id,occurred_at,received_at, +trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_canonical +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('industry','string',ifNull(dynamicElement(getSubcolumn(properties_json,'industry'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('seats','number','',ifNull(seat_number,toFloat64(0)))] AS trait +WHERE event_name='group' AND schema_version=2 AND organization_id!='' AND governed_traits_valid +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') +AND (NOT JSONHas(properties,'industry') OR dynamicType(getSubcolumn(properties_json,'industry'))='String') +AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') +AND (NOT JSONHas(properties,'seats') OR (dynamicType(getSubcolumn(properties_json,'seats')) IN ('Int64','UInt64','Float64') AND isFinite(seat_number) AND seat_number>=0 AND floor(seat_number)=seat_number)) +AND JSONHas(properties,trait.1); +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_organization_properties TO analytics.organization_properties AS +SELECT account_id,organization_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH toFloat64OrNull(toString(getSubcolumn(properties_json,'seats'))) AS seat_number +SELECT account_id,organization_id,event_id,occurred_at,received_at, +trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_raw +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('industry','string',ifNull(dynamicElement(getSubcolumn(properties_json,'industry'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('seats','number','',ifNull(seat_number,toFloat64(0)))] AS trait +WHERE event_name='group' AND schema_version=2 AND organization_id!='' AND governed_traits_valid +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') +AND (NOT JSONHas(properties,'industry') OR dynamicType(getSubcolumn(properties_json,'industry'))='String') +AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') +AND (NOT JSONHas(properties,'seats') OR (dynamicType(getSubcolumn(properties_json,'seats')) IN ('Int64','UInt64','Float64') AND isFinite(seat_number) AND seat_number>=0 AND floor(seat_number)=seat_number)) +AND JSONHas(properties,trait.1)) GROUP BY account_id,organization_id,name; +INSERT INTO analytics.organization_properties +SELECT account_id,organization_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (SELECT * FROM analytics.organization_trait_events) GROUP BY account_id,organization_id,name; +CREATE OR REPLACE VIEW analytics.organization_traits AS +SELECT account_id,organization_id,name,selected.1 AS trait_type,selected.2 AS text_value,selected.3 AS number_value +FROM (SELECT account_id,organization_id,name,argMaxMerge(value) AS selected FROM analytics.organization_properties GROUP BY account_id,organization_id,name); + +CREATE TABLE IF NOT EXISTS analytics.user_organizations +(account_id String,user_id String, + organization_id AggregateFunction(argMax,String,Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,user_id); +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_user_organizations TO analytics.user_organizations AS +SELECT account_id,user_id,argMaxState(r.organization_id,tuple(occurred_at,received_at,event_id)) AS organization_id +FROM analytics.events_raw AS r +WHERE event_name='group' AND r.organization_id!='' AND user_id!='' +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +GROUP BY account_id,user_id; +INSERT INTO analytics.user_organizations +SELECT account_id,user_id,argMaxState(r.organization_id,tuple(occurred_at,received_at,event_id)) AS organization_id +FROM analytics.events_canonical AS r +WHERE event_name='group' AND r.organization_id!='' AND user_id!='' +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +GROUP BY account_id,user_id; diff --git a/test/fixtures/kansoku/original/migrate.sh b/test/fixtures/kansoku/original/migrate.sh new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/original/migrate.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# ClickHouse migration runner. +# +# Applies db/clickhouse/migrations/*.sql in filename order, skipping files +# already recorded in analytics.schema_migrations, and recording each applied +# file. RULE: a migration file is IMMUTABLE once applied anywhere — never edit +# an applied file; add a new numbered file instead. The ledger keys on the +# filename, so an edited file would silently not re-run. +set -euo pipefail + +port="${CLICKHOUSE_TCP_PORT:?CLICKHOUSE_TCP_PORT must be set (enter nix develop)}" + +ch() { + clickhouse-client --port "$port" "$@" +} + +ch --query "CREATE DATABASE IF NOT EXISTS analytics" +ch --query " + CREATE TABLE IF NOT EXISTS analytics.schema_migrations + ( + filename String, + applied_at DateTime64(3, 'UTC') DEFAULT now64(3) + ) + ENGINE = MergeTree + ORDER BY filename" + +for f in db/clickhouse/migrations/*.sql; do + name="$(basename "$f")" + applied="$(ch --query "SELECT count() FROM analytics.schema_migrations WHERE filename = '$name'")" + if [ "$applied" = "0" ]; then + echo "applying $name" + ch --multiquery < "$f" + ch --query "INSERT INTO analytics.schema_migrations (filename) VALUES ('$name')" + else + echo "skipping $name (already applied)" + fi +done diff --git a/test/fixtures/kansoku/provenance.json b/test/fixtures/kansoku/provenance.json new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/provenance.json @@ -0,0 +1,2054 @@ +{ + "fixtureVersion": 1, + "sourceProject": "mori://shinzui/kansoku", + "sourceRevision": "138d9a316931bbf46781757a29ab6c7fdd90f870", + "artifactUris": "File-level URI pending; all source paths below are relative to sourceProject at sourceRevision.", + "evidenceLimit": "Synthetic filename/time rows. The predecessor stored no checksums. All imported completion is operator-attested.", + "boundaryPolicy": "Reviewed complete statements; contiguous original byte ranges, concatenating exactly to each original file. No runtime SQL splitting.", + "files": [ + { + "sourcePath": "db/clickhouse/migrations/0001_events_raw.sql", + "original": "original/0001_events_raw.sql", + "sha256": "27fd9656256fe707f368e9a8428a389d932ae3a5e4ec84d63f34273a3c1f4ff5", + "migration": "0001_events_raw", + "steps": [ + { + "name": "statement-01", + "file": "steps/0001_events_raw/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 40, + "sha256": "2200b1521d968e229dab8e8cb415fa555e87a91b83a3b1a2c115884f914368dd" + }, + { + "name": "statement-02", + "file": "steps/0001_events_raw/statement-02.sql", + "byteStart": 40, + "byteEndExclusive": 1286, + "sha256": "515b45b027db0136eeb73dea220f422c892e88d32c534048a820636164f5b05b" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0002_identity.sql", + "original": "original/0002_identity.sql", + "sha256": "1909ac790fe3b9e91ffb035070cdbf3e08100a6a8d0914b59f34458c153622ca", + "migration": "0002_identity", + "steps": [ + { + "name": "statement-01", + "file": "steps/0002_identity/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 40, + "sha256": "2200b1521d968e229dab8e8cb415fa555e87a91b83a3b1a2c115884f914368dd" + }, + { + "name": "statement-02", + "file": "steps/0002_identity/statement-02.sql", + "byteStart": 40, + "byteEndExclusive": 391, + "sha256": "66da20e0ef3fe8a442f9783fc0fd1f736801b2666b897a7229017741d745570f" + }, + { + "name": "statement-03", + "file": "steps/0002_identity/statement-03.sql", + "byteStart": 391, + "byteEndExclusive": 819, + "sha256": "f55eae735ae80c032bd12c890e110d6255602a867ccefd24ec092b48e938cfa8" + }, + { + "name": "statement-04", + "file": "steps/0002_identity/statement-04.sql", + "byteStart": 819, + "byteEndExclusive": 1849, + "sha256": "fbf24c751a8b25ef92565677ef0bce550c3ef19172bbb098b48f30e8375b9596" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0003_rollups.sql", + "original": "original/0003_rollups.sql", + "sha256": "bd7eea194dc1c9d1b5471254f5c38bbbe6f14aab71f5ee76942dbc0ade242d37", + "migration": "0003_rollups", + "steps": [ + { + "name": "statement-01", + "file": "steps/0003_rollups/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 40, + "sha256": "2200b1521d968e229dab8e8cb415fa555e87a91b83a3b1a2c115884f914368dd" + }, + { + "name": "statement-02", + "file": "steps/0003_rollups/statement-02.sql", + "byteStart": 40, + "byteEndExclusive": 314, + "sha256": "0e42118dd75c726f1faeaf0967d52819874a24691b73eb50f33b39be475e8707" + }, + { + "name": "statement-03", + "file": "steps/0003_rollups/statement-03.sql", + "byteStart": 314, + "byteEndExclusive": 591, + "sha256": "94580b6d5a28cb9d2d1eda365a9c3ba365f215cb6a4184d4c171f70f652cc10f" + }, + { + "name": "statement-04", + "file": "steps/0003_rollups/statement-04.sql", + "byteStart": 591, + "byteEndExclusive": 1048, + "sha256": "a5c21e8a4e1d4b105d0539b54c861840f331b04881d421c74aa5dfaad6ce5bb3" + }, + { + "name": "statement-05", + "file": "steps/0003_rollups/statement-05.sql", + "byteStart": 1048, + "byteEndExclusive": 1615, + "sha256": "35cd78a4d3fb0894df9973d0b72818de1678ac107ef7518bff4994da1431b3fc" + }, + { + "name": "statement-06", + "file": "steps/0003_rollups/statement-06.sql", + "byteStart": 1615, + "byteEndExclusive": 1859, + "sha256": "d66684cead4205bfe810187dd21b73e7061689eeccadddb4fd3d4406a9ce6241" + }, + { + "name": "statement-07", + "file": "steps/0003_rollups/statement-07.sql", + "byteStart": 1859, + "byteEndExclusive": 2430, + "sha256": "630c86c366e8531fef350a246d86f8e87820b0ecd2a777c578f43f40902f0b7b" + }, + { + "name": "statement-08", + "file": "steps/0003_rollups/statement-08.sql", + "byteStart": 2430, + "byteEndExclusive": 2819, + "sha256": "464cb34015c26486256e33d2fd7091bb4c63f905adbe9539e741171851aea9c8" + }, + { + "name": "statement-09", + "file": "steps/0003_rollups/statement-09.sql", + "byteStart": 2819, + "byteEndExclusive": 3132, + "sha256": "7bc25383a76daee9063cc03ef4573e9a929863d130e6105723d0a8389180e966" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0004_events_raw_v2.sql", + "original": "original/0004_events_raw_v2.sql", + "sha256": "43728707a636a89ebb75331fa9209e195526ffe38aac986d311d8d83dde852f9", + "migration": "0004_events_raw_v2", + "steps": [ + { + "name": "statement-01", + "file": "steps/0004_events_raw_v2/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 488, + "sha256": "e925844c3c5307aa5a372b391371aa09e4511b23b0a2ac352db396244147d56f" + }, + { + "name": "statement-02", + "file": "steps/0004_events_raw_v2/statement-02.sql", + "byteStart": 488, + "byteEndExclusive": 542, + "sha256": "6c88ccc4859d6acb07f8dc00983739ca7e01dd055a65fd4dca83a4d42b6d034f" + }, + { + "name": "statement-03", + "file": "steps/0004_events_raw_v2/statement-03.sql", + "byteStart": 542, + "byteEndExclusive": 581, + "sha256": "44fb80ddb087677362f03695d3c224415c7c6ca438f11769fdb64fe4599d0fff" + }, + { + "name": "statement-04", + "file": "steps/0004_events_raw_v2/statement-04.sql", + "byteStart": 581, + "byteEndExclusive": 620, + "sha256": "b0a09200b707ea3195cc1344ed7c7e69d5c2243dc72ae271908f0a147dc0a899" + }, + { + "name": "statement-05", + "file": "steps/0004_events_raw_v2/statement-05.sql", + "byteStart": 620, + "byteEndExclusive": 675, + "sha256": "4f71e2f306d6183b80a458e7f489a96e413e60e881ca7a10ae9f5ac30fb9e20b" + }, + { + "name": "statement-06", + "file": "steps/0004_events_raw_v2/statement-06.sql", + "byteStart": 675, + "byteEndExclusive": 722, + "sha256": "95cb7fd530ccc9acc2511af48a004778320575895175eb422e30f29de67e2db3" + }, + { + "name": "statement-07", + "file": "steps/0004_events_raw_v2/statement-07.sql", + "byteStart": 722, + "byteEndExclusive": 768, + "sha256": "0c0ca6de649a412195e2bab5f4380037569cd59e97c510dee7d2e68011ee1758" + }, + { + "name": "statement-08", + "file": "steps/0004_events_raw_v2/statement-08.sql", + "byteStart": 768, + "byteEndExclusive": 2055, + "sha256": "649cbf5bedcfadf01d82943e4b33759154b36d97c2aa2df57fd6cf8b7c15e11e" + }, + { + "name": "statement-09", + "file": "steps/0004_events_raw_v2/statement-09.sql", + "byteStart": 2055, + "byteEndExclusive": 2986, + "sha256": "bf4e7b8126e26002bcbdca9c7ea1a7ae8e5c3c4ac328da7622f0425ee282f26d" + }, + { + "name": "statement-10", + "file": "steps/0004_events_raw_v2/statement-10.sql", + "byteStart": 2986, + "byteEndExclusive": 3206, + "sha256": "65a0a5948a305b0072a055280b2e8d277bba94649b5848727fdcbb5f70bf1722" + }, + { + "name": "statement-11", + "file": "steps/0004_events_raw_v2/statement-11.sql", + "byteStart": 3206, + "byteEndExclusive": 3546, + "sha256": "2711a6e187c4ad417213f6a36d9c59328c74374f8ce8a3ec9616f0e8685ad020" + }, + { + "name": "statement-12", + "file": "steps/0004_events_raw_v2/statement-12.sql", + "byteStart": 3546, + "byteEndExclusive": 4081, + "sha256": "e558d86d0e053a7a3944a809c902b9f2e09abff8931f1ad269d04b244195262b" + }, + { + "name": "statement-13", + "file": "steps/0004_events_raw_v2/statement-13.sql", + "byteStart": 4081, + "byteEndExclusive": 4344, + "sha256": "93e23240859bee8b51442adb4bf4dbfbbfc738865c38b73cee4b2a2faa9b558d" + }, + { + "name": "statement-14", + "file": "steps/0004_events_raw_v2/statement-14.sql", + "byteStart": 4344, + "byteEndExclusive": 4966, + "sha256": "bed7b992766b36c7f6236224d7fa8066171edca436dd3b0819077d2fe737e796" + }, + { + "name": "statement-15", + "file": "steps/0004_events_raw_v2/statement-15.sql", + "byteStart": 4966, + "byteEndExclusive": 5523, + "sha256": "97b8aa6fb8de8a6102db07953036606130b2b1e92bf0463520850930b382f143" + }, + { + "name": "statement-16", + "file": "steps/0004_events_raw_v2/statement-16.sql", + "byteStart": 5523, + "byteEndExclusive": 5808, + "sha256": "fa0734a8d2385a70b6d82e7a99a309ac8a9299f2a2cb0eb86440e9626d903e11" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0005_final_free_resolution.sql", + "original": "original/0005_final_free_resolution.sql", + "sha256": "7c98dcf31b841886d64ec66c8ff1bab7c0c574a2762131fa075e4fd8a7083638", + "migration": "0005_final_free_resolution", + "steps": [ + { + "name": "statement-01", + "file": "steps/0005_final_free_resolution/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 523, + "sha256": "e7c197abce3859612e25c48619633f4a7b68270bdc73e5487d1e4da999185614" + }, + { + "name": "statement-02", + "file": "steps/0005_final_free_resolution/statement-02.sql", + "byteStart": 523, + "byteEndExclusive": 1014, + "sha256": "b581f371f8374595ad3c91032eaa4a513413161c8bc4552aa38e9e2dfe8db20b" + }, + { + "name": "statement-03", + "file": "steps/0005_final_free_resolution/statement-03.sql", + "byteStart": 1014, + "byteEndExclusive": 1054, + "sha256": "443d5d63f067f954c1d2ce84fd22ee02db1844b6b9b13b02d432257593ed7347" + }, + { + "name": "statement-04", + "file": "steps/0005_final_free_resolution/statement-04.sql", + "byteStart": 1054, + "byteEndExclusive": 1093, + "sha256": "b0a09200b707ea3195cc1344ed7c7e69d5c2243dc72ae271908f0a147dc0a899" + }, + { + "name": "statement-05", + "file": "steps/0005_final_free_resolution/statement-05.sql", + "byteStart": 1093, + "byteEndExclusive": 1388, + "sha256": "9b8e1da0ec936f78afa1f716475403c6c0a9cf3072509327c782fc34b520a255" + }, + { + "name": "statement-06", + "file": "steps/0005_final_free_resolution/statement-06.sql", + "byteStart": 1388, + "byteEndExclusive": 1418, + "sha256": "1099b4262d7fa05b8e723d4893b3ab764bc2431a27ee02c13cc43b44a576dd61" + }, + { + "name": "statement-07", + "file": "steps/0005_final_free_resolution/statement-07.sql", + "byteStart": 1418, + "byteEndExclusive": 1649, + "sha256": "6e38915566d68fba72693f56fd064809701c0bd739977c99bd4f26ae5497b211" + }, + { + "name": "statement-08", + "file": "steps/0005_final_free_resolution/statement-08.sql", + "byteStart": 1649, + "byteEndExclusive": 1885, + "sha256": "666dcb5054b7d256d53006287f11e15eaaccb24ba198f6adbac9859a38990a50" + }, + { + "name": "statement-09", + "file": "steps/0005_final_free_resolution/statement-09.sql", + "byteStart": 1885, + "byteEndExclusive": 2139, + "sha256": "2d411849b7e67dad57c8d85e653ef1f8ad14ef6330fd5c92e6a05c2f63adec25" + }, + { + "name": "statement-10", + "file": "steps/0005_final_free_resolution/statement-10.sql", + "byteStart": 2139, + "byteEndExclusive": 2398, + "sha256": "677b1c367a38fdc7ad7070759c2c9765033fd5974bf195955a768c852081153b" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0006_rollups_v2.sql", + "original": "original/0006_rollups_v2.sql", + "sha256": "48fcfd193523c73dcef347c3ee0a947faa86e0bb24980ee385349d5450465336", + "migration": "0006_rollups_v2", + "steps": [ + { + "name": "statement-01", + "file": "steps/0006_rollups_v2/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 563, + "sha256": "f2979112e546e014527f78945d8a5119a44681071366a2542a725b8059ec0d29" + }, + { + "name": "statement-02", + "file": "steps/0006_rollups_v2/statement-02.sql", + "byteStart": 563, + "byteEndExclusive": 602, + "sha256": "44fb80ddb087677362f03695d3c224415c7c6ca438f11769fdb64fe4599d0fff" + }, + { + "name": "statement-03", + "file": "steps/0006_rollups_v2/statement-03.sql", + "byteStart": 602, + "byteEndExclusive": 641, + "sha256": "b0a09200b707ea3195cc1344ed7c7e69d5c2243dc72ae271908f0a147dc0a899" + }, + { + "name": "statement-04", + "file": "steps/0006_rollups_v2/statement-04.sql", + "byteStart": 641, + "byteEndExclusive": 696, + "sha256": "4f71e2f306d6183b80a458e7f489a96e413e60e881ca7a10ae9f5ac30fb9e20b" + }, + { + "name": "statement-05", + "file": "steps/0006_rollups_v2/statement-05.sql", + "byteStart": 696, + "byteEndExclusive": 747, + "sha256": "d95932232a2f6816ec4871f73ab6830131368bd42f0479ac78b6812161cf7bf4" + }, + { + "name": "statement-06", + "file": "steps/0006_rollups_v2/statement-06.sql", + "byteStart": 747, + "byteEndExclusive": 783, + "sha256": "17a2615750a0f9028a7d611e71fa1976f1116c0ce19bdf5650af163eb646e7a8" + }, + { + "name": "statement-07", + "file": "steps/0006_rollups_v2/statement-07.sql", + "byteStart": 783, + "byteEndExclusive": 819, + "sha256": "b421613250ac36c03b6f292706ce3cef7b5f0590bff588e41827de71d5b09add" + }, + { + "name": "statement-08", + "file": "steps/0006_rollups_v2/statement-08.sql", + "byteStart": 819, + "byteEndExclusive": 871, + "sha256": "9d234a0a7f9e45e8a10a09d281f669b204b907a8b157ece6a306235377d34180" + }, + { + "name": "statement-09", + "file": "steps/0006_rollups_v2/statement-09.sql", + "byteStart": 871, + "byteEndExclusive": 1060, + "sha256": "5914aa5a052b7a7db9b8abf13a311888e01397025a39a769fb99718681c38528" + }, + { + "name": "statement-10", + "file": "steps/0006_rollups_v2/statement-10.sql", + "byteStart": 1060, + "byteEndExclusive": 1202, + "sha256": "410d21c8f39ad4cea79020ade2f0f214731b7e3176a65d64fffb028c08f8e9be" + }, + { + "name": "statement-11", + "file": "steps/0006_rollups_v2/statement-11.sql", + "byteStart": 1202, + "byteEndExclusive": 1345, + "sha256": "a22ff9b0f8394e21c7972da45a87598548c4df558d6dcaf80f5be5765f1e1920" + }, + { + "name": "statement-12", + "file": "steps/0006_rollups_v2/statement-12.sql", + "byteStart": 1345, + "byteEndExclusive": 1515, + "sha256": "5f44fd2950989d3044a86a58948d31451665aaa9fd91ec5e1cb59938fc4497ad" + }, + { + "name": "statement-13", + "file": "steps/0006_rollups_v2/statement-13.sql", + "byteStart": 1515, + "byteEndExclusive": 1796, + "sha256": "4074174cb07bac34a7e6c6e7415dcc53558308f3970cb83d54ed97c560758179" + }, + { + "name": "statement-14", + "file": "steps/0006_rollups_v2/statement-14.sql", + "byteStart": 1796, + "byteEndExclusive": 2030, + "sha256": "1318f014323c51c7b559d601acfd26b94d6c0d5cae0ebab1c3b839c15066adb8" + }, + { + "name": "statement-15", + "file": "steps/0006_rollups_v2/statement-15.sql", + "byteStart": 2030, + "byteEndExclusive": 2269, + "sha256": "56d5d41fc997f9a5df66ac40242736ece8c79b4c8c7af83b0e606054a3e972a8" + }, + { + "name": "statement-16", + "file": "steps/0006_rollups_v2/statement-16.sql", + "byteStart": 2269, + "byteEndExclusive": 2499, + "sha256": "da771222bd639789ecf25061a198b81928e40af485f8d85bf24f16f6301019a1" + }, + { + "name": "statement-17", + "file": "steps/0006_rollups_v2/statement-17.sql", + "byteStart": 2499, + "byteEndExclusive": 2740, + "sha256": "d9744db60d418fa6322bab20e6d517c26b7b75a3a8bec21566f9fc852bd8f578" + }, + { + "name": "statement-18", + "file": "steps/0006_rollups_v2/statement-18.sql", + "byteStart": 2740, + "byteEndExclusive": 2997, + "sha256": "5f6450e0d8a0c2c0682529b798e722318ad17abc9bbd36420997382536cbf17f" + }, + { + "name": "statement-19", + "file": "steps/0006_rollups_v2/statement-19.sql", + "byteStart": 2997, + "byteEndExclusive": 3258, + "sha256": "de1abe08b446a70c8aea56a3683da0dbc8b2d26ab07087f6c1f4a589d5c2fc68" + }, + { + "name": "statement-20", + "file": "steps/0006_rollups_v2/statement-20.sql", + "byteStart": 3258, + "byteEndExclusive": 3521, + "sha256": "49a454f9ed3da6c8f56c2a8a8f6371b00dd172c9c52ffffd85ba5973bff76098" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0007_drop_events_raw_v1.sql", + "original": "original/0007_drop_events_raw_v1.sql", + "sha256": "d6a529078a02c3a4216c5fe2063d76624539f71e497cf96032771bebd9543758", + "migration": "0007_drop_events_raw_v1", + "steps": [ + { + "name": "statement-01", + "file": "steps/0007_drop_events_raw_v1/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 204, + "sha256": "d6a529078a02c3a4216c5fe2063d76624539f71e497cf96032771bebd9543758" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0008_account_scoped_metrics.sql", + "original": "original/0008_account_scoped_metrics.sql", + "sha256": "67d301ca66c789fb837c6abf1d6515cc8a3b36e49e00add57aa170c71272b568", + "migration": "0008_account_scoped_metrics", + "steps": [ + { + "name": "statement-01", + "file": "steps/0008_account_scoped_metrics/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 200, + "sha256": "41259c6f0d556ee80b7ae443199487e1d1d7c1169d5c149fb29ee464f272e40e" + }, + { + "name": "statement-02", + "file": "steps/0008_account_scoped_metrics/statement-02.sql", + "byteStart": 200, + "byteEndExclusive": 723, + "sha256": "1b8d830e3995c1cfea18f5fe8a8b239fb6411ce49dc6ebf70a999b5356fac444" + }, + { + "name": "statement-03", + "file": "steps/0008_account_scoped_metrics/statement-03.sql", + "byteStart": 723, + "byteEndExclusive": 778, + "sha256": "679a5d3bc433f7e0267684ea99ce9019595e2a4de00aa172d2a4990c604e049b" + }, + { + "name": "statement-04", + "file": "steps/0008_account_scoped_metrics/statement-04.sql", + "byteStart": 778, + "byteEndExclusive": 817, + "sha256": "44fb80ddb087677362f03695d3c224415c7c6ca438f11769fdb64fe4599d0fff" + }, + { + "name": "statement-05", + "file": "steps/0008_account_scoped_metrics/statement-05.sql", + "byteStart": 817, + "byteEndExclusive": 856, + "sha256": "b0a09200b707ea3195cc1344ed7c7e69d5c2243dc72ae271908f0a147dc0a899" + }, + { + "name": "statement-06", + "file": "steps/0008_account_scoped_metrics/statement-06.sql", + "byteStart": 856, + "byteEndExclusive": 911, + "sha256": "4f71e2f306d6183b80a458e7f489a96e413e60e881ca7a10ae9f5ac30fb9e20b" + }, + { + "name": "statement-07", + "file": "steps/0008_account_scoped_metrics/statement-07.sql", + "byteStart": 911, + "byteEndExclusive": 962, + "sha256": "d95932232a2f6816ec4871f73ab6830131368bd42f0479ac78b6812161cf7bf4" + }, + { + "name": "statement-08", + "file": "steps/0008_account_scoped_metrics/statement-08.sql", + "byteStart": 962, + "byteEndExclusive": 998, + "sha256": "17a2615750a0f9028a7d611e71fa1976f1116c0ce19bdf5650af163eb646e7a8" + }, + { + "name": "statement-09", + "file": "steps/0008_account_scoped_metrics/statement-09.sql", + "byteStart": 998, + "byteEndExclusive": 1034, + "sha256": "b421613250ac36c03b6f292706ce3cef7b5f0590bff588e41827de71d5b09add" + }, + { + "name": "statement-10", + "file": "steps/0008_account_scoped_metrics/statement-10.sql", + "byteStart": 1034, + "byteEndExclusive": 1086, + "sha256": "9d234a0a7f9e45e8a10a09d281f669b204b907a8b157ece6a306235377d34180" + }, + { + "name": "statement-11", + "file": "steps/0008_account_scoped_metrics/statement-11.sql", + "byteStart": 1086, + "byteEndExclusive": 1310, + "sha256": "b1266d26814f40e444c0a5b0d28c10a0008966d0c549139e75ffc0547fbee0a9" + }, + { + "name": "statement-12", + "file": "steps/0008_account_scoped_metrics/statement-12.sql", + "byteStart": 1310, + "byteEndExclusive": 1499, + "sha256": "f1134ede8135292a9538260d673ecc1d543162d76d8d6e5ce990b64d49070db7" + }, + { + "name": "statement-13", + "file": "steps/0008_account_scoped_metrics/statement-13.sql", + "byteStart": 1499, + "byteEndExclusive": 1689, + "sha256": "11a4d8fa04ff63de5f50370fcd9a733e2a8654e64a6a033dd5c212ac7723f88a" + }, + { + "name": "statement-14", + "file": "steps/0008_account_scoped_metrics/statement-14.sql", + "byteStart": 1689, + "byteEndExclusive": 1909, + "sha256": "1059e1b40e195e2f5ff6fd7a20d12495c89bd0a459ab29b80b96feb76abf9ca5" + }, + { + "name": "statement-15", + "file": "steps/0008_account_scoped_metrics/statement-15.sql", + "byteStart": 1909, + "byteEndExclusive": 2165, + "sha256": "2168c08bafa7b7bb8f13af770346404ccfe46d7a4febc54dd6088623284cfa33" + }, + { + "name": "statement-16", + "file": "steps/0008_account_scoped_metrics/statement-16.sql", + "byteStart": 2165, + "byteEndExclusive": 2439, + "sha256": "c3cd888beae99b31df1b5f74c3f93515ee06676b7a87febf396ce39d7aaa28bc" + }, + { + "name": "statement-17", + "file": "steps/0008_account_scoped_metrics/statement-17.sql", + "byteStart": 2439, + "byteEndExclusive": 2718, + "sha256": "3a3e2ec23e7fe73ab59583a18abe27949eaafd9d00e4d480a00152978179427d" + }, + { + "name": "statement-18", + "file": "steps/0008_account_scoped_metrics/statement-18.sql", + "byteStart": 2718, + "byteEndExclusive": 2988, + "sha256": "5b377539e9afa9172113b5bdb14ada1ba65dde6f41e96b003a56fc96ab29e0fd" + }, + { + "name": "statement-19", + "file": "steps/0008_account_scoped_metrics/statement-19.sql", + "byteStart": 2988, + "byteEndExclusive": 3257, + "sha256": "1621e5d6e10d6c1c91160014e977ab197b1184abcf02eb80675d005379e468da" + }, + { + "name": "statement-20", + "file": "steps/0008_account_scoped_metrics/statement-20.sql", + "byteStart": 3257, + "byteEndExclusive": 3542, + "sha256": "3c1822e7b0e61d9d6320067126587bb45699c95e36f96f12c8478dbf7a4ec194" + }, + { + "name": "statement-21", + "file": "steps/0008_account_scoped_metrics/statement-21.sql", + "byteStart": 3542, + "byteEndExclusive": 3831, + "sha256": "2f143ada90b95dfb28248cc598d8f32c09a93abe77eebf37a0e0b30752886f98" + }, + { + "name": "statement-22", + "file": "steps/0008_account_scoped_metrics/statement-22.sql", + "byteStart": 3831, + "byteEndExclusive": 4122, + "sha256": "e6705b25d769d9e4d22d760b8d6fa2fe72cea0b2d8efdaf347910f1ad05508bc" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0009_typed_properties.sql", + "original": "original/0009_typed_properties.sql", + "sha256": "e800f9d12be889710debbc70e5eb7894d949269d2da7e3c2955e9709230a5604", + "migration": "0009_typed_properties", + "steps": [ + { + "name": "statement-01", + "file": "steps/0009_typed_properties/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 351, + "sha256": "140b8352562ce4210ce58f5bc52f2e9a16d2781e78b4a056821b28a1b7cd64a7" + }, + { + "name": "statement-02", + "file": "steps/0009_typed_properties/statement-02.sql", + "byteStart": 351, + "byteEndExclusive": 422, + "sha256": "21b7acf530d6efddcb9a5a74bc1e3c19ff41bdbddc6d1cc628ea7c74e648d790" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0010_resolved_event_properties.sql", + "original": "original/0010_resolved_event_properties.sql", + "sha256": "22a2f23981a3bded3d02742a30e653a644f28d8648a26e11b0c53601f7d05031", + "migration": "0010_resolved_event_properties", + "steps": [ + { + "name": "statement-01", + "file": "steps/0010_resolved_event_properties/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 196, + "sha256": "b0dd44d38d8169f5d7faacb8ee86d18015741ef5d2162a3b99da990411bcd25f" + }, + { + "name": "statement-02", + "file": "steps/0010_resolved_event_properties/statement-02.sql", + "byteStart": 196, + "byteEndExclusive": 762, + "sha256": "fde5546f55d8282e52630d0e34b1a5f470b4824043bff403452a42949148fcf2" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0011_events_raw_identifiers.sql", + "original": "original/0011_events_raw_identifiers.sql", + "sha256": "63bc883bc44628383f0149cf65b1a9c71d93ae6f5a51b6df1d36301e1f0b12c2", + "migration": "0011_events_raw_identifiers", + "steps": [ + { + "name": "statement-01", + "file": "steps/0011_events_raw_identifiers/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 331, + "sha256": "63bc883bc44628383f0149cf65b1a9c71d93ae6f5a51b6df1d36301e1f0b12c2" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0012_identity_signals.sql", + "original": "original/0012_identity_signals.sql", + "sha256": "43d619f703a24d04513df3ea19067973b2914c11053e2ae257dc108e4e36d8ce", + "migration": "0012_identity_signals", + "steps": [ + { + "name": "statement-01", + "file": "steps/0012_identity_signals/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 588, + "sha256": "fe1f9b17e4a31a5a125fd614b0d4ccf686419d2ac064b700a3525ddbeaac3272" + }, + { + "name": "statement-02", + "file": "steps/0012_identity_signals/statement-02.sql", + "byteStart": 588, + "byteEndExclusive": 1248, + "sha256": "026dfa2584db280e6da58de3e6be8ad7595d28c2a6aa89ad5425bb4ec26fabfb" + }, + { + "name": "statement-03", + "file": "steps/0012_identity_signals/statement-03.sql", + "byteStart": 1248, + "byteEndExclusive": 1925, + "sha256": "8597ee15d041aa626b3cbf1781f18b3be295f47e34e7790bdbe1eef58130a474" + }, + { + "name": "statement-04", + "file": "steps/0012_identity_signals/statement-04.sql", + "byteStart": 1925, + "byteEndExclusive": 2430, + "sha256": "8084c688e58f7d0dc82e3a9cfeb33e0b0912285b4983576a09ec878a86b9d36f" + }, + { + "name": "statement-05", + "file": "steps/0012_identity_signals/statement-05.sql", + "byteStart": 2430, + "byteEndExclusive": 2895, + "sha256": "dee1d130a47f3cf92e106958823499b8a0f4152c5e1b02d33ed4e1304077d309" + }, + { + "name": "statement-06", + "file": "steps/0012_identity_signals/statement-06.sql", + "byteStart": 2895, + "byteEndExclusive": 4487, + "sha256": "da3265228a38f95349f9188f0b2e0949ceac0cf224fa88b6d68afb52cbd24dd8" + }, + { + "name": "statement-07", + "file": "steps/0012_identity_signals/statement-07.sql", + "byteStart": 4487, + "byteEndExclusive": 4535, + "sha256": "8e2fdcbe6573b942d6f82e1293cd5d1c7cd39fb2233394265e47966affdf85f8" + }, + { + "name": "statement-08", + "file": "steps/0012_identity_signals/statement-08.sql", + "byteStart": 4535, + "byteEndExclusive": 4621, + "sha256": "c928db83dcf676f7da3506c5e49e14ce57157f838d78f8537ce2936e0a557b8e" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0013_retire_identity_map.sql", + "original": "original/0013_retire_identity_map.sql", + "sha256": "743acdfd7eb87733ce7faccbe9268ea814234c42d5325a043bd6143db227461f", + "migration": "0013_retire_identity_map", + "steps": [ + { + "name": "statement-01", + "file": "steps/0013_retire_identity_map/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 276, + "sha256": "39752da99771bf42038077799fdc150df42bed7b6e9ae9c09c18f19cf550e0fb" + }, + { + "name": "statement-02", + "file": "steps/0013_retire_identity_map/statement-02.sql", + "byteStart": 276, + "byteEndExclusive": 322, + "sha256": "708b7ef94e1b5b94250e89ad7f7eab3423942d41773f784f504c30202f8e0602" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0014_experiment_exposures.sql", + "original": "original/0014_experiment_exposures.sql", + "sha256": "066db9c83cc844188ca73544a67054f6b63f0a1c36e2eeab98d438c4a94a35e6", + "migration": "0014_experiment_exposures", + "steps": [ + { + "name": "statement-01", + "file": "steps/0014_experiment_exposures/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 885, + "sha256": "695538e452c37b98b3369fd1ad489056f2c919fccb2cc071328252691ceed3d0" + }, + { + "name": "statement-02", + "file": "steps/0014_experiment_exposures/statement-02.sql", + "byteStart": 885, + "byteEndExclusive": 1580, + "sha256": "547f8eed15e971d537db2d6bf05b2f03e52e5f1c2f639a68521e7278db3903c9" + }, + { + "name": "statement-03", + "file": "steps/0014_experiment_exposures/statement-03.sql", + "byteStart": 1580, + "byteEndExclusive": 2508, + "sha256": "15f7b2d46fbbdb64d57cf76a36282e14b350aaff7bd1ff277464132a7b179f79" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0015_experiment_exposure_string_casts.sql", + "original": "original/0015_experiment_exposure_string_casts.sql", + "sha256": "c9958643c9f018f68fd07fad1425e4dd94d890ec190b6787c1d134e127b11f74", + "migration": "0015_experiment_exposure_string_casts", + "steps": [ + { + "name": "statement-01", + "file": "steps/0015_experiment_exposure_string_casts/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 451, + "sha256": "8a3cba65e710d02ef987e3cbecacfa5a3822f8acf44ff113087ee942b13ddcb6" + }, + { + "name": "statement-02", + "file": "steps/0015_experiment_exposure_string_casts/statement-02.sql", + "byteStart": 451, + "byteEndExclusive": 1218, + "sha256": "f3264ef53649d0d682f0a7a565ca60c7e1253a200061bdab2e7e23eb1c24d1d6" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0016_events_raw_schema_version.sql", + "original": "original/0016_events_raw_schema_version.sql", + "sha256": "375f7a8fae2d4cf6bb53a579e93e4f5ebb15ffe91923759a436b2def2e1d2df3", + "migration": "0016_events_raw_schema_version", + "steps": [ + { + "name": "statement-01", + "file": "steps/0016_events_raw_schema_version/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 113, + "sha256": "375f7a8fae2d4cf6bb53a579e93e4f5ebb15ffe91923759a436b2def2e1d2df3" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0017_replay_safe_sources.sql", + "original": "original/0017_replay_safe_sources.sql", + "sha256": "6986149ed18b17dd781ac5dbe1215c8c32a39c36f392dbe4573e556f8f31f1ba", + "migration": "0017_replay_safe_sources", + "steps": [ + { + "name": "statement-01", + "file": "steps/0017_replay_safe_sources/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 412, + "sha256": "822cbdccab6b4f31df20c0de0083606a051cc57ae22ca170c4e0d8b801dd3909" + }, + { + "name": "statement-02", + "file": "steps/0017_replay_safe_sources/statement-02.sql", + "byteStart": 412, + "byteEndExclusive": 810, + "sha256": "b701f1c87c342fe46f5361410f22e3a8289e96de0aed589dece01f51fe38643b" + }, + { + "name": "statement-03", + "file": "steps/0017_replay_safe_sources/statement-03.sql", + "byteStart": 810, + "byteEndExclusive": 1278, + "sha256": "426961a6ba43d5d6de7a94bbe88731a638930036ddce3bb5480013a3e37fff7c" + }, + { + "name": "statement-04", + "file": "steps/0017_replay_safe_sources/statement-04.sql", + "byteStart": 1278, + "byteEndExclusive": 2659, + "sha256": "e627342c27a91249c94ca14970bf43740f40c35fd1ded53d8b86731908091530" + }, + { + "name": "statement-05", + "file": "steps/0017_replay_safe_sources/statement-05.sql", + "byteStart": 2659, + "byteEndExclusive": 2715, + "sha256": "22545ab8b2268b8be3936f313948939b9a4e855d8b808d2cd4a684790f6490da" + }, + { + "name": "statement-06", + "file": "steps/0017_replay_safe_sources/statement-06.sql", + "byteStart": 2715, + "byteEndExclusive": 2770, + "sha256": "4f71e2f306d6183b80a458e7f489a96e413e60e881ca7a10ae9f5ac30fb9e20b" + }, + { + "name": "statement-07", + "file": "steps/0017_replay_safe_sources/statement-07.sql", + "byteStart": 2770, + "byteEndExclusive": 2829, + "sha256": "5471c547503e83cfd4729b206ac6d1966d83e1d065f15e2d1517e95706fb65b0" + }, + { + "name": "statement-08", + "file": "steps/0017_replay_safe_sources/statement-08.sql", + "byteStart": 2829, + "byteEndExclusive": 2889, + "sha256": "9a9321fe6a0b194e67d3f12fcaa8391633896b30b3010b1f870b3edad69a41ab" + }, + { + "name": "statement-09", + "file": "steps/0017_replay_safe_sources/statement-09.sql", + "byteStart": 2889, + "byteEndExclusive": 3122, + "sha256": "36eba8e1983624f4f3fa6046924c196dc417d283d6d2141e1a14bcdb65e2baeb" + }, + { + "name": "statement-10", + "file": "steps/0017_replay_safe_sources/statement-10.sql", + "byteStart": 3122, + "byteEndExclusive": 3345, + "sha256": "1eff7df66f163846a62266e891ff8b2c55b855dc6a2e69a64c3128a1576b31e3" + }, + { + "name": "statement-11", + "file": "steps/0017_replay_safe_sources/statement-11.sql", + "byteStart": 3345, + "byteEndExclusive": 3566, + "sha256": "290cd2ae6ebb6b076ab7f43d1f99bda23cc76a1ca006a6c28fc51651e259cffe" + }, + { + "name": "statement-12", + "file": "steps/0017_replay_safe_sources/statement-12.sql", + "byteStart": 3566, + "byteEndExclusive": 3821, + "sha256": "bba8ce814eef1e9635af0cc5da9f4a32ee09215ab78f6ef75badd1801ea6d85a" + }, + { + "name": "statement-13", + "file": "steps/0017_replay_safe_sources/statement-13.sql", + "byteStart": 3821, + "byteEndExclusive": 4413, + "sha256": "86a29561ca2b4537b06acdfebbee2f4c7140de5b25d19076f1200a06ca5fd7d2" + }, + { + "name": "statement-14", + "file": "steps/0017_replay_safe_sources/statement-14.sql", + "byteStart": 4413, + "byteEndExclusive": 4956, + "sha256": "f46011bb58ae8532437f1619a01e50818644e77925d47711741b1d82af9e96fc" + }, + { + "name": "statement-15", + "file": "steps/0017_replay_safe_sources/statement-15.sql", + "byteStart": 4956, + "byteEndExclusive": 5276, + "sha256": "6ce9de15254fc7517f4d750b9c560417bad7b1b5ffdb139556feb82018ff6baa" + }, + { + "name": "statement-16", + "file": "steps/0017_replay_safe_sources/statement-16.sql", + "byteStart": 5276, + "byteEndExclusive": 5365, + "sha256": "72632740aa0c84d1ef1acf38eff9aff6ad3dfed642fe97d6a1cf606082160dc7" + }, + { + "name": "statement-17", + "file": "steps/0017_replay_safe_sources/statement-17.sql", + "byteStart": 5365, + "byteEndExclusive": 5414, + "sha256": "06ba8a4f9f016d2ed1d1c2b518197ed3efed92759e747f5f93f09b44204eb15e" + }, + { + "name": "statement-18", + "file": "steps/0017_replay_safe_sources/statement-18.sql", + "byteStart": 5414, + "byteEndExclusive": 5464, + "sha256": "283993b525fa5c927a3c6f908b5297996b891174310eab2c61129b470092004b" + }, + { + "name": "statement-19", + "file": "steps/0017_replay_safe_sources/statement-19.sql", + "byteStart": 5464, + "byteEndExclusive": 5875, + "sha256": "3a4e2ac47f84d3116d232accc42926e52cac9b4c72ef58aecf2484a379c97006" + }, + { + "name": "statement-20", + "file": "steps/0017_replay_safe_sources/statement-20.sql", + "byteStart": 5875, + "byteEndExclusive": 6169, + "sha256": "90d8007b4c23e990d3beeda8b7d1376073d9780d5c20e63c792779036179676e" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0018_conservative_identity_resolution.sql", + "original": "original/0018_conservative_identity_resolution.sql", + "sha256": "c144311cda6d59dc08ba92e7384daea85f852742ac40cabbd90592f249a40bb6", + "migration": "0018_conservative_identity_resolution", + "steps": [ + { + "name": "statement-01", + "file": "steps/0018_conservative_identity_resolution/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 754, + "sha256": "ed98df17dc8bf29970b0c5f7bf7ff97c4ba4a8c9160910bfb70e99976010002a" + }, + { + "name": "statement-02", + "file": "steps/0018_conservative_identity_resolution/statement-02.sql", + "byteStart": 754, + "byteEndExclusive": 854, + "sha256": "24c96eed70c96fcbc65a2ca6468819a80eb10a2580dbe893cff789248df76569" + }, + { + "name": "statement-03", + "file": "steps/0018_conservative_identity_resolution/statement-03.sql", + "byteStart": 854, + "byteEndExclusive": 986, + "sha256": "b563a7ef286a459b3713eb8536f37a9df004a559cc3873e6ed38b7dccfdf8912" + }, + { + "name": "statement-04", + "file": "steps/0018_conservative_identity_resolution/statement-04.sql", + "byteStart": 986, + "byteEndExclusive": 1044, + "sha256": "e32677f9bf3db048e79f2ebf3d99cc4eaf75bd0f3ef0cd46182a67433c9c10d7" + }, + { + "name": "statement-05", + "file": "steps/0018_conservative_identity_resolution/statement-05.sql", + "byteStart": 1044, + "byteEndExclusive": 1102, + "sha256": "443907dcd239d6e4a3957da9f6f685057ab8ad852641f28c6f757d0677ba8cbd" + }, + { + "name": "statement-06", + "file": "steps/0018_conservative_identity_resolution/statement-06.sql", + "byteStart": 1102, + "byteEndExclusive": 1154, + "sha256": "14fc59c2068b842ed9c2af4a115f41903d32091d33a4be00c722f9b03984df6e" + }, + { + "name": "statement-07", + "file": "steps/0018_conservative_identity_resolution/statement-07.sql", + "byteStart": 1154, + "byteEndExclusive": 1211, + "sha256": "cbd692e2b85912df80ea1974aebf696d2676586cbee38b1011eede5cadd3629f" + }, + { + "name": "statement-08", + "file": "steps/0018_conservative_identity_resolution/statement-08.sql", + "byteStart": 1211, + "byteEndExclusive": 1711, + "sha256": "c9de2bc0747012817955cfd54d6080ab0a478f9c1aefef57e99e9965eb69cd86" + }, + { + "name": "statement-09", + "file": "steps/0018_conservative_identity_resolution/statement-09.sql", + "byteStart": 1711, + "byteEndExclusive": 2165, + "sha256": "a05d0013c180527c7a6801af88d66eb88323482e44ec6240edb04ba283ceb27a" + }, + { + "name": "statement-10", + "file": "steps/0018_conservative_identity_resolution/statement-10.sql", + "byteStart": 2165, + "byteEndExclusive": 2248, + "sha256": "d9f8741b505699b90ea6fd84242aea8c90fdbab7ca4992a4840d63ab5b32ef18" + }, + { + "name": "statement-11", + "file": "steps/0018_conservative_identity_resolution/statement-11.sql", + "byteStart": 2248, + "byteEndExclusive": 2295, + "sha256": "ecf93eb49ee58819e86e8c32760ebaf1a622e1ff78bc2609cbf69a7f77c042a2" + }, + { + "name": "statement-12", + "file": "steps/0018_conservative_identity_resolution/statement-12.sql", + "byteStart": 2295, + "byteEndExclusive": 2837, + "sha256": "818eb0558d34ef841ee345a1fd42addc3ed258dcbda3fff7d43245e5fd0b08b9" + }, + { + "name": "statement-13", + "file": "steps/0018_conservative_identity_resolution/statement-13.sql", + "byteStart": 2837, + "byteEndExclusive": 3058, + "sha256": "5f8870e835d49acca2e0461d8adf6a4cc23044e218781a474595e160cbfe1191" + }, + { + "name": "statement-14", + "file": "steps/0018_conservative_identity_resolution/statement-14.sql", + "byteStart": 3058, + "byteEndExclusive": 3116, + "sha256": "443907dcd239d6e4a3957da9f6f685057ab8ad852641f28c6f757d0677ba8cbd" + }, + { + "name": "statement-15", + "file": "steps/0018_conservative_identity_resolution/statement-15.sql", + "byteStart": 3116, + "byteEndExclusive": 3539, + "sha256": "d8baba86546a21d549518ee9e2294cc4966170ad82d83c92f1e59e0fc9a6ebff" + }, + { + "name": "statement-16", + "file": "steps/0018_conservative_identity_resolution/statement-16.sql", + "byteStart": 3539, + "byteEndExclusive": 3876, + "sha256": "022646100dc0e0620cbb3a61e76eec97606b11ea1dfffbadbadad64c5944ce18" + }, + { + "name": "statement-17", + "file": "steps/0018_conservative_identity_resolution/statement-17.sql", + "byteStart": 3876, + "byteEndExclusive": 3938, + "sha256": "b2c47359f4bcc7151c49df0704fdceb8a2fc2f4fe6746c77bfc1ccf5895d7f50" + }, + { + "name": "statement-18", + "file": "steps/0018_conservative_identity_resolution/statement-18.sql", + "byteStart": 3938, + "byteEndExclusive": 4001, + "sha256": "b08d28b0efb788aeb4d7d96ab40ac3e0cf844177b8cda7fef499256ff5f1fe25" + }, + { + "name": "statement-19", + "file": "steps/0018_conservative_identity_resolution/statement-19.sql", + "byteStart": 4001, + "byteEndExclusive": 4090, + "sha256": "aab26a36422d7050b5d4760d1af372593cbae19b68112bb4c8a01c014b8d7595" + }, + { + "name": "statement-20", + "file": "steps/0018_conservative_identity_resolution/statement-20.sql", + "byteStart": 4090, + "byteEndExclusive": 4181, + "sha256": "01d40420c262550d8a7e87cdffe5fb65717db8cbd94b14dbdc14c252f7bad020" + }, + { + "name": "statement-21", + "file": "steps/0018_conservative_identity_resolution/statement-21.sql", + "byteStart": 4181, + "byteEndExclusive": 4400, + "sha256": "6c160a107c4a57d3eafad42c5c5c50bf303a5088e2fd738cd9076daf5fb59a30" + }, + { + "name": "statement-22", + "file": "steps/0018_conservative_identity_resolution/statement-22.sql", + "byteStart": 4400, + "byteEndExclusive": 4700, + "sha256": "9d2cf338807f22c9c22e0efc732629b4b561d3101aabc91bac56fbc95a328c5c" + }, + { + "name": "statement-23", + "file": "steps/0018_conservative_identity_resolution/statement-23.sql", + "byteStart": 4700, + "byteEndExclusive": 4793, + "sha256": "65e41b39d5eec9ac866f3ecf4513555acf9a8ea174460d87c32d357981e2f292" + }, + { + "name": "statement-24", + "file": "steps/0018_conservative_identity_resolution/statement-24.sql", + "byteStart": 4793, + "byteEndExclusive": 4888, + "sha256": "6b19772a4972d37a5d039daf7e270793610299d6b5a35e21bf4339865d620ab3" + }, + { + "name": "statement-25", + "file": "steps/0018_conservative_identity_resolution/statement-25.sql", + "byteStart": 4888, + "byteEndExclusive": 4940, + "sha256": "3c5df9d4375a298787ff0a89913d6583abe5b95c120682f4dec3f2eaa69e63d1" + }, + { + "name": "statement-26", + "file": "steps/0018_conservative_identity_resolution/statement-26.sql", + "byteStart": 4940, + "byteEndExclusive": 4993, + "sha256": "3fd2c6c57cc450c31c7ca4ed8e48e9eb0f42898981a7ac4cfe0a6094e567a32a" + }, + { + "name": "statement-27", + "file": "steps/0018_conservative_identity_resolution/statement-27.sql", + "byteStart": 4993, + "byteEndExclusive": 5267, + "sha256": "0a7ce0fb9f5cff8ca053461b947bc0f27a837d7e21a617d69fedc694aa32b161" + }, + { + "name": "statement-28", + "file": "steps/0018_conservative_identity_resolution/statement-28.sql", + "byteStart": 5267, + "byteEndExclusive": 5623, + "sha256": "ddc87d6f3aa8bb404534983bc28b891c3591e43a95ff16c84631be90d8045064" + }, + { + "name": "statement-29", + "file": "steps/0018_conservative_identity_resolution/statement-29.sql", + "byteStart": 5623, + "byteEndExclusive": 5918, + "sha256": "ed9cba13eedba2f97061e234696ef8631a6ac2aaa49d778f4a9fcb0573e47cac" + }, + { + "name": "statement-30", + "file": "steps/0018_conservative_identity_resolution/statement-30.sql", + "byteStart": 5918, + "byteEndExclusive": 6221, + "sha256": "7ff55958678db3153e47d13752730992ba69e02bd786fc78d9c0c1f95813bdb3" + }, + { + "name": "statement-31", + "file": "steps/0018_conservative_identity_resolution/statement-31.sql", + "byteStart": 6221, + "byteEndExclusive": 6286, + "sha256": "77bd330c6692ff3b1931fd42c4e971c8fad598a6ed524471ecc29e56ddc553ef" + }, + { + "name": "statement-32", + "file": "steps/0018_conservative_identity_resolution/statement-32.sql", + "byteStart": 6286, + "byteEndExclusive": 6381, + "sha256": "a5689563993c6c92cf88d634a2eb472b14ea6209f638307efbe981937cf80ca5" + }, + { + "name": "statement-33", + "file": "steps/0018_conservative_identity_resolution/statement-33.sql", + "byteStart": 6381, + "byteEndExclusive": 6746, + "sha256": "5aa5d9fdf1cacd2955c94a0f266d62331b2a5f77499ef7b9324a156994bd724e" + }, + { + "name": "statement-34", + "file": "steps/0018_conservative_identity_resolution/statement-34.sql", + "byteStart": 6746, + "byteEndExclusive": 6845, + "sha256": "57ca937b6999a38177df6149384ac43dc8ca6e4c6c8723c9471b53c2affd1686" + }, + { + "name": "statement-35", + "file": "steps/0018_conservative_identity_resolution/statement-35.sql", + "byteStart": 6845, + "byteEndExclusive": 6900, + "sha256": "653fa3202bcf27f29ff99c74b1e958a818f6c6ca3db1856974c40c1018ef767b" + }, + { + "name": "statement-36", + "file": "steps/0018_conservative_identity_resolution/statement-36.sql", + "byteStart": 6900, + "byteEndExclusive": 7369, + "sha256": "800a2df81d74b57b46155d8995dfa3fcbb37277d78e1aebd7331a352fcd489e3" + }, + { + "name": "statement-37", + "file": "steps/0018_conservative_identity_resolution/statement-37.sql", + "byteStart": 7369, + "byteEndExclusive": 7823, + "sha256": "7c1839a3ef84d8d547c22fc1661c6ae5e5d864507b1b704802c9f5ea885bd1c6" + }, + { + "name": "statement-38", + "file": "steps/0018_conservative_identity_resolution/statement-38.sql", + "byteStart": 7823, + "byteEndExclusive": 8183, + "sha256": "3bab2b00970d3d0dd38fb65709358f25d03c81afce738dfcd12ec428c6b8db46" + }, + { + "name": "statement-39", + "file": "steps/0018_conservative_identity_resolution/statement-39.sql", + "byteStart": 8183, + "byteEndExclusive": 12202, + "sha256": "51c390bb5a1ebe4cd181c518e1dbd0404827c199ffa1db399cd1b9d0447318c9" + }, + { + "name": "statement-40", + "file": "steps/0018_conservative_identity_resolution/statement-40.sql", + "byteStart": 12202, + "byteEndExclusive": 12536, + "sha256": "73ffdc3d709659e81a327eb53d0b94e130c97ec50ce94658f215f01c1affa32d" + }, + { + "name": "statement-41", + "file": "steps/0018_conservative_identity_resolution/statement-41.sql", + "byteStart": 12536, + "byteEndExclusive": 12632, + "sha256": "6b8bcdfaff3b06e33fc0dd8ada9115a664e4c176786e97cee4f4e163f96d2633" + }, + { + "name": "statement-42", + "file": "steps/0018_conservative_identity_resolution/statement-42.sql", + "byteStart": 12632, + "byteEndExclusive": 12672, + "sha256": "443d5d63f067f954c1d2ce84fd22ee02db1844b6b9b13b02d432257593ed7347" + }, + { + "name": "statement-43", + "file": "steps/0018_conservative_identity_resolution/statement-43.sql", + "byteStart": 12672, + "byteEndExclusive": 12716, + "sha256": "423e43b9ab2442e6d60ec554befe02f11b60098ac2d4aa377fedf167f04c2680" + }, + { + "name": "statement-44", + "file": "steps/0018_conservative_identity_resolution/statement-44.sql", + "byteStart": 12716, + "byteEndExclusive": 12769, + "sha256": "72349be7608bc0924b1124a2c515ceb8e832e4c499ca1d69061e8daccf32ef92" + }, + { + "name": "statement-45", + "file": "steps/0018_conservative_identity_resolution/statement-45.sql", + "byteStart": 12769, + "byteEndExclusive": 13075, + "sha256": "7c867182dbd5dbd3956d3d6dff71ebec237f54f32fcac458f3b6cdb24cbd1e32" + }, + { + "name": "statement-46", + "file": "steps/0018_conservative_identity_resolution/statement-46.sql", + "byteStart": 13075, + "byteEndExclusive": 13132, + "sha256": "2718142da482b111b5073e8c1c69f5931cf54ffcced94ef0f7cec9ee7823ece3" + }, + { + "name": "statement-47", + "file": "steps/0018_conservative_identity_resolution/statement-47.sql", + "byteStart": 13132, + "byteEndExclusive": 13166, + "sha256": "4aeb5779bc2c94246fcb8a5cf49380e0729e41e4377179a06fa9d67812b1d919" + }, + { + "name": "statement-48", + "file": "steps/0018_conservative_identity_resolution/statement-48.sql", + "byteStart": 13166, + "byteEndExclusive": 13471, + "sha256": "94d2adf2fd520aea72ba6d9ac31235d6c49c881c878e262870b4efb6421105f5" + }, + { + "name": "statement-49", + "file": "steps/0018_conservative_identity_resolution/statement-49.sql", + "byteStart": 13471, + "byteEndExclusive": 13511, + "sha256": "1fd107ff3a23e8a4a4b5f5aaaf513dce511c61a2f2b86f73b0cc0260f1ba0021" + }, + { + "name": "statement-50", + "file": "steps/0018_conservative_identity_resolution/statement-50.sql", + "byteStart": 13511, + "byteEndExclusive": 13555, + "sha256": "71943b13a12cfa88c67bd8c97fcbcd401f9d45b66b1792217e7cf4267592d7ce" + }, + { + "name": "statement-51", + "file": "steps/0018_conservative_identity_resolution/statement-51.sql", + "byteStart": 13555, + "byteEndExclusive": 13608, + "sha256": "26c73ca127d7fcd55463fe78edd35a0cdf4036aa13ffa0a30942698c94588323" + }, + { + "name": "statement-52", + "file": "steps/0018_conservative_identity_resolution/statement-52.sql", + "byteStart": 13608, + "byteEndExclusive": 13919, + "sha256": "847698ae0be9c5abb83b1671e64229cd1c190d5ca20dd88ffda8f205a2208225" + }, + { + "name": "statement-53", + "file": "steps/0018_conservative_identity_resolution/statement-53.sql", + "byteStart": 13919, + "byteEndExclusive": 13976, + "sha256": "4cc8e95401b842ab7ef0206a0331f38eb1061ac3bf93229e5c56b7073a942c8f" + }, + { + "name": "statement-54", + "file": "steps/0018_conservative_identity_resolution/statement-54.sql", + "byteStart": 13976, + "byteEndExclusive": 14010, + "sha256": "47d5b2a7739171280b6418d2e2c19f52ec6fea48ce031c59e53a5351b5dd695b" + }, + { + "name": "statement-55", + "file": "steps/0018_conservative_identity_resolution/statement-55.sql", + "byteStart": 14010, + "byteEndExclusive": 14319, + "sha256": "b7ffb1ff83cbbe27bc09319782f8d9c5cb9b619eff2bd1cb26906e4012d4d0e1" + }, + { + "name": "statement-56", + "file": "steps/0018_conservative_identity_resolution/statement-56.sql", + "byteStart": 14319, + "byteEndExclusive": 14376, + "sha256": "b1adb450cb134b370b5e29b8ac4c2a497685ea697c7c31ebe71b965b88797d21" + }, + { + "name": "statement-57", + "file": "steps/0018_conservative_identity_resolution/statement-57.sql", + "byteStart": 14376, + "byteEndExclusive": 14437, + "sha256": "e2d1fd73b5ad3d3242c184e11cdbaecbde6d816030d11dd495c896ee03e50acd" + }, + { + "name": "statement-58", + "file": "steps/0018_conservative_identity_resolution/statement-58.sql", + "byteStart": 14437, + "byteEndExclusive": 14524, + "sha256": "9c78716015c60a0d580175df68b147c4ce2135f9b7e7b83115111b0ccbe0c261" + }, + { + "name": "statement-59", + "file": "steps/0018_conservative_identity_resolution/statement-59.sql", + "byteStart": 14524, + "byteEndExclusive": 15181, + "sha256": "a21fdc4cc13bef969c916ff475293a014b9664823a855b999d21406204fb9cba" + }, + { + "name": "statement-60", + "file": "steps/0018_conservative_identity_resolution/statement-60.sql", + "byteStart": 15181, + "byteEndExclusive": 15272, + "sha256": "7e70f31dc37fb2d71ef73d0736c1653151fb2b63b81cda258b1871d7504486e6" + }, + { + "name": "statement-61", + "file": "steps/0018_conservative_identity_resolution/statement-61.sql", + "byteStart": 15272, + "byteEndExclusive": 15323, + "sha256": "204bddededcb0b9326792495f2311b9fc62b567e9e945f62ed320041cfaeed27" + }, + { + "name": "statement-62", + "file": "steps/0018_conservative_identity_resolution/statement-62.sql", + "byteStart": 15323, + "byteEndExclusive": 15977, + "sha256": "7cf838187300d708ba1ca7f82db39d6c59def839e83a54f4624af3ddaa35b922" + }, + { + "name": "statement-63", + "file": "steps/0018_conservative_identity_resolution/statement-63.sql", + "byteStart": 15977, + "byteEndExclusive": 17922, + "sha256": "66446388e235450c2cddef01c3a8791271b8ebafe1d27774edf4a2d5400e2ac2" + }, + { + "name": "statement-64", + "file": "steps/0018_conservative_identity_resolution/statement-64.sql", + "byteStart": 17922, + "byteEndExclusive": 18843, + "sha256": "27aac20101349d88dd700e2e549c38d73e0190f07a01f685c21abcb81abcdd5a" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0019_web_enrichment.sql", + "original": "original/0019_web_enrichment.sql", + "sha256": "28f118103db1932769a286be7bca316ebaebeae9453660b51b71e5651f3b6b23", + "migration": "0019_web_enrichment", + "steps": [ + { + "name": "statement-01", + "file": "steps/0019_web_enrichment/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 976, + "sha256": "40e16ba91843cdb7c61a953a31ecb3d8ba58f7c8d30bafbe726590673d9ce12c" + }, + { + "name": "statement-02", + "file": "steps/0019_web_enrichment/statement-02.sql", + "byteStart": 976, + "byteEndExclusive": 1122, + "sha256": "40fa8cb3264da8fa33ec187383080c5308b4a49cd68a93d97daf60a0ae2dc204" + }, + { + "name": "statement-03", + "file": "steps/0019_web_enrichment/statement-03.sql", + "byteStart": 1122, + "byteEndExclusive": 4932, + "sha256": "0479cd227452539b803c9932dac31ae09c6389a5e47ba492f1e13e228416b8c1" + }, + { + "name": "statement-04", + "file": "steps/0019_web_enrichment/statement-04.sql", + "byteStart": 4932, + "byteEndExclusive": 5266, + "sha256": "73ffdc3d709659e81a327eb53d0b94e130c97ec50ce94658f215f01c1affa32d" + }, + { + "name": "statement-05", + "file": "steps/0019_web_enrichment/statement-05.sql", + "byteStart": 5266, + "byteEndExclusive": 5362, + "sha256": "6b8bcdfaff3b06e33fc0dd8ada9115a664e4c176786e97cee4f4e163f96d2633" + }, + { + "name": "statement-06", + "file": "steps/0019_web_enrichment/statement-06.sql", + "byteStart": 5362, + "byteEndExclusive": 6135, + "sha256": "e4e83d2bbe22de75a44b26c520c7d4b8cfd893da401ba4733811ecf5d4b903c7" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0020_replay_safe_sessions.sql", + "original": "original/0020_replay_safe_sessions.sql", + "sha256": "2520352091378f9bdee1af3be0d36340422c3b82c3d7fc6f6d5291386c4c354b", + "migration": "0020_replay_safe_sessions", + "steps": [ + { + "name": "statement-01", + "file": "steps/0020_replay_safe_sessions/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 282, + "sha256": "47ffe989ef67ba8e6aeab77deb5d5b295507cca9e90f6cb8f7f54c6d7b89ba97" + }, + { + "name": "statement-02", + "file": "steps/0020_replay_safe_sessions/statement-02.sql", + "byteStart": 282, + "byteEndExclusive": 331, + "sha256": "1b0a43b0df3d05f023a3418b2312fddd02404455dc0b2c48f103aab9c4b71284" + }, + { + "name": "statement-03", + "file": "steps/0020_replay_safe_sessions/statement-03.sql", + "byteStart": 331, + "byteEndExclusive": 1856, + "sha256": "2cb643e6c3f5a6ade3f7e227ccc970ac7e5958af1b5dff2c53fb5e32dda39699" + }, + { + "name": "statement-04", + "file": "steps/0020_replay_safe_sessions/statement-04.sql", + "byteStart": 1856, + "byteEndExclusive": 3294, + "sha256": "6c7e0abd95806ac6133f6e9b06cb4f44b47dbc98b677f09189beedbfbe2f6243" + }, + { + "name": "statement-05", + "file": "steps/0020_replay_safe_sessions/statement-05.sql", + "byteStart": 3294, + "byteEndExclusive": 3372, + "sha256": "9dc88926229e3cea7d8c4da45c830b2e34484466f33280d7ac255e72b5db0092" + }, + { + "name": "statement-06", + "file": "steps/0020_replay_safe_sessions/statement-06.sql", + "byteStart": 3372, + "byteEndExclusive": 3439, + "sha256": "881333c358dc4110d39d2c378e2eb7a60b290377a407f398f2c1718a04559eba" + }, + { + "name": "statement-07", + "file": "steps/0020_replay_safe_sessions/statement-07.sql", + "byteStart": 3439, + "byteEndExclusive": 3478, + "sha256": "8b856bb5c7a6d73a202a5902891b3b54460d4dfd89781682f992d35d60b998bd" + }, + { + "name": "statement-08", + "file": "steps/0020_replay_safe_sessions/statement-08.sql", + "byteStart": 3478, + "byteEndExclusive": 4944, + "sha256": "e7d14097d26691cefb9a014bab9934ab5318cc5bbb4b816e982ed52ceca3ecf7" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0021_governed_current_user_traits.sql", + "original": "original/0021_governed_current_user_traits.sql", + "sha256": "41f30678ff7d119c74e33f08186fe95de443be23787227dc00de8a54c466bd37", + "migration": "0021_governed_current_user_traits", + "steps": [ + { + "name": "statement-01", + "file": "steps/0021_governed_current_user_traits/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 503, + "sha256": "6ab6b1f0262a47a3d41d0278e588a3ac0dc7f22cd0b492444cb1e43599fe7859" + }, + { + "name": "statement-02", + "file": "steps/0021_governed_current_user_traits/statement-02.sql", + "byteStart": 503, + "byteEndExclusive": 649, + "sha256": "40fa8cb3264da8fa33ec187383080c5308b4a49cd68a93d97daf60a0ae2dc204" + }, + { + "name": "statement-03", + "file": "steps/0021_governed_current_user_traits/statement-03.sql", + "byteStart": 649, + "byteEndExclusive": 802, + "sha256": "ea0d33a822f1b26c57d6e77fa17e1ee1b66e61a18153288c1e1967597da1d9f0" + }, + { + "name": "statement-04", + "file": "steps/0021_governed_current_user_traits/statement-04.sql", + "byteStart": 802, + "byteEndExclusive": 1084, + "sha256": "ca059120b30f584af967c712e43ee09eeb58b50b9264f056237128ebfb5adc71" + }, + { + "name": "statement-05", + "file": "steps/0021_governed_current_user_traits/statement-05.sql", + "byteStart": 1084, + "byteEndExclusive": 1371, + "sha256": "0085e15acbfd384b429669f27159553dda7596f5d1e2ef9e1ceda2725c5fc302" + }, + { + "name": "statement-06", + "file": "steps/0021_governed_current_user_traits/statement-06.sql", + "byteStart": 1371, + "byteEndExclusive": 3428, + "sha256": "c1b09457357fd41322c85aae2207786a29df0ef9df95850aa294a7ce1bfd726d" + }, + { + "name": "statement-07", + "file": "steps/0021_governed_current_user_traits/statement-07.sql", + "byteStart": 3428, + "byteEndExclusive": 3480, + "sha256": "5d8e4a15ab1a37c15fd75519f4043e4a01a504450b620a6ba79c50aa6be873ed" + }, + { + "name": "statement-08", + "file": "steps/0021_governed_current_user_traits/statement-08.sql", + "byteStart": 3480, + "byteEndExclusive": 5765, + "sha256": "2e843855eea13db66e07ad48c8c1306cf0fc82cc1549a2f9b087915b9271ed51" + }, + { + "name": "statement-09", + "file": "steps/0021_governed_current_user_traits/statement-09.sql", + "byteStart": 5765, + "byteEndExclusive": 6050, + "sha256": "6dbe5ed0a4d6327617292bd579b6e439ec4d93999fa47cac34371f5685250219" + }, + { + "name": "statement-10", + "file": "steps/0021_governed_current_user_traits/statement-10.sql", + "byteStart": 6050, + "byteEndExclusive": 6107, + "sha256": "68689bf69aab12437a8b267708220fdaea2ab2d257aca97c1ed91b13705ad5a9" + }, + { + "name": "statement-11", + "file": "steps/0021_governed_current_user_traits/statement-11.sql", + "byteStart": 6107, + "byteEndExclusive": 8407, + "sha256": "fd948d50d1c9608397cc60839472c137bf3bfe580973bfffd406331500209e09" + }, + { + "name": "statement-12", + "file": "steps/0021_governed_current_user_traits/statement-12.sql", + "byteStart": 8407, + "byteEndExclusive": 8702, + "sha256": "5b953b5597e4fc7343f09481367751e02c16ed4d36ea2e5c64354da384631aab" + }, + { + "name": "statement-13", + "file": "steps/0021_governed_current_user_traits/statement-13.sql", + "byteStart": 8702, + "byteEndExclusive": 9323, + "sha256": "f1df13525a3a80e658b70bfe103d4539144c08a44944fb12087ec2e00a7d3030" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0022_applied_retention_policy.sql", + "original": "original/0022_applied_retention_policy.sql", + "sha256": "8c594b02c0b8e15d8a9369a81db13e3501289c3f326b5d4b3554dc241bb48492", + "migration": "0022_applied_retention_policy", + "steps": [ + { + "name": "statement-01", + "file": "steps/0022_applied_retention_policy/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 343, + "sha256": "439c59e4a0fc73124be599936103ce84354f313f58d2e3f7bd1bbaf3a7c7bc78" + }, + { + "name": "statement-02", + "file": "steps/0022_applied_retention_policy/statement-02.sql", + "byteStart": 343, + "byteEndExclusive": 418, + "sha256": "a8ef37c97783872b83129695847ae9bf1b7ef521cf6063954edc01530b7266cf" + }, + { + "name": "statement-03", + "file": "steps/0022_applied_retention_policy/statement-03.sql", + "byteStart": 418, + "byteEndExclusive": 652, + "sha256": "683faeed379d8f12d5cac5fde691f112255d7ab2abc02e64bde8172264982cc0" + }, + { + "name": "statement-04", + "file": "steps/0022_applied_retention_policy/statement-04.sql", + "byteStart": 652, + "byteEndExclusive": 878, + "sha256": "b15a030ea29a55d48eba51c2c4b1dcb407769a0b4c42ff7b3bbe96608ede6bfe" + }, + { + "name": "statement-05", + "file": "steps/0022_applied_retention_policy/statement-05.sql", + "byteStart": 878, + "byteEndExclusive": 1290, + "sha256": "060ed9ea75de9aba3ab03d508653a6de75ede7a1ddb50729bf1ba1f4ffb9a924" + }, + { + "name": "statement-06", + "file": "steps/0022_applied_retention_policy/statement-06.sql", + "byteStart": 1290, + "byteEndExclusive": 1346, + "sha256": "25d0d037df001c0523e4b7e06cc8abd93936c12bd9111655a74ac05c02a43697" + }, + { + "name": "statement-07", + "file": "steps/0022_applied_retention_policy/statement-07.sql", + "byteStart": 1346, + "byteEndExclusive": 1793, + "sha256": "5fd7d817996ced9ce7abe3ffda9aec8aa9f4af54995d94dd3197ac83fc13b4b6" + }, + { + "name": "statement-08", + "file": "steps/0022_applied_retention_policy/statement-08.sql", + "byteStart": 1793, + "byteEndExclusive": 1846, + "sha256": "f13f4ee4825e6c88ce7eeb3c7172fc3350586e107dd9cc410ea9d5935b56c95a" + }, + { + "name": "statement-09", + "file": "steps/0022_applied_retention_policy/statement-09.sql", + "byteStart": 1846, + "byteEndExclusive": 2542, + "sha256": "db71bd5deb9aadc92fd1f6bcde5352f12874ad5e6062ec08ebea1fc1d0a76ec7" + }, + { + "name": "statement-10", + "file": "steps/0022_applied_retention_policy/statement-10.sql", + "byteStart": 2542, + "byteEndExclusive": 2603, + "sha256": "fb3a5644c55700b8917d379e585c09f77571ddeee8da6f3272f84c033f55bf16" + }, + { + "name": "statement-11", + "file": "steps/0022_applied_retention_policy/statement-11.sql", + "byteStart": 2603, + "byteEndExclusive": 3227, + "sha256": "db08f44aca7913cad7b85837adbd6a61fe601cbb3076679f362357d39dc554a7" + }, + { + "name": "statement-12", + "file": "steps/0022_applied_retention_policy/statement-12.sql", + "byteStart": 3227, + "byteEndExclusive": 3267, + "sha256": "443d5d63f067f954c1d2ce84fd22ee02db1844b6b9b13b02d432257593ed7347" + }, + { + "name": "statement-13", + "file": "steps/0022_applied_retention_policy/statement-13.sql", + "byteStart": 3267, + "byteEndExclusive": 3727, + "sha256": "bf57155c90270f36de7bb03e8c177ca658f05296b08d9c61c69e13e2d4d8d75d" + }, + { + "name": "statement-14", + "file": "steps/0022_applied_retention_policy/statement-14.sql", + "byteStart": 3727, + "byteEndExclusive": 3767, + "sha256": "1fd107ff3a23e8a4a4b5f5aaaf513dce511c61a2f2b86f73b0cc0260f1ba0021" + }, + { + "name": "statement-15", + "file": "steps/0022_applied_retention_policy/statement-15.sql", + "byteStart": 3767, + "byteEndExclusive": 4231, + "sha256": "31db10695085e15272ff3000152c199cee877ca157c1af6606a1d4e85e44b03c" + }, + { + "name": "statement-16", + "file": "steps/0022_applied_retention_policy/statement-16.sql", + "byteStart": 4231, + "byteEndExclusive": 4288, + "sha256": "b1adb450cb134b370b5e29b8ac4c2a497685ea697c7c31ebe71b965b88797d21" + }, + { + "name": "statement-17", + "file": "steps/0022_applied_retention_policy/statement-17.sql", + "byteStart": 4288, + "byteEndExclusive": 5097, + "sha256": "a04ace24afd85d2b0a8fecd316e12834f5353c14847557512fd1c415f7fe8991" + }, + { + "name": "statement-18", + "file": "steps/0022_applied_retention_policy/statement-18.sql", + "byteStart": 5097, + "byteEndExclusive": 5142, + "sha256": "050286d1aa00612a94d7b5eb974415db9d6275fd9e2d5306a4466a24cc7a4f08" + }, + { + "name": "statement-19", + "file": "steps/0022_applied_retention_policy/statement-19.sql", + "byteStart": 5142, + "byteEndExclusive": 6761, + "sha256": "650d2cccdec98854b5c022ab9f60caa163ea4e64d871c9eb4b1737b0a513598e" + }, + { + "name": "statement-20", + "file": "steps/0022_applied_retention_policy/statement-20.sql", + "byteStart": 6761, + "byteEndExclusive": 6813, + "sha256": "5d8e4a15ab1a37c15fd75519f4043e4a01a504450b620a6ba79c50aa6be873ed" + }, + { + "name": "statement-21", + "file": "steps/0022_applied_retention_policy/statement-21.sql", + "byteStart": 6813, + "byteEndExclusive": 9253, + "sha256": "f83d7d1c32b0decabac71f33edeee5621fba1a1ce84a9d0b1dd95b022bc1aad3" + }, + { + "name": "statement-22", + "file": "steps/0022_applied_retention_policy/statement-22.sql", + "byteStart": 9253, + "byteEndExclusive": 9310, + "sha256": "68689bf69aab12437a8b267708220fdaea2ab2d257aca97c1ed91b13705ad5a9" + }, + { + "name": "statement-23", + "file": "steps/0022_applied_retention_policy/statement-23.sql", + "byteStart": 9310, + "byteEndExclusive": 11766, + "sha256": "0bcbed90f0e94f79e9463ad8dd1ebe984fe489c492653e02319df6ef28bd3f30" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0023_stable_reader_surface.sql", + "original": "original/0023_stable_reader_surface.sql", + "sha256": "d22c39530bc31bfac1cb60f4dd7dde9d57b74fd900a2b3e9e84fe65c16924bbd", + "migration": "0023_stable_reader_surface", + "steps": [ + { + "name": "statement-01", + "file": "steps/0023_stable_reader_surface/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 252, + "sha256": "23774b5f1a7fc28b23a692e2d4a89dc1200c5541a5f62952ea9ef48b329ee8a2" + }, + { + "name": "statement-02", + "file": "steps/0023_stable_reader_surface/statement-02.sql", + "byteStart": 252, + "byteEndExclusive": 342, + "sha256": "b77660907711f64306a7bd6ddfb367695ad0c1c391b70b26594ae56eb86f267a" + }, + { + "name": "statement-03", + "file": "steps/0023_stable_reader_surface/statement-03.sql", + "byteStart": 342, + "byteEndExclusive": 994, + "sha256": "c34aaa0d84e9606217d415cc109eb7ecc300d5e553c96e2eeadee3df3a8e8a97" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0024_cohort_members.sql", + "original": "original/0024_cohort_members.sql", + "sha256": "c8effe8505363b52006b4b74eec7d552c6282837ab983b542901ee5564836911", + "migration": "0024_cohort_members", + "steps": [ + { + "name": "statement-01", + "file": "steps/0024_cohort_members/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 239, + "sha256": "c8effe8505363b52006b4b74eec7d552c6282837ab983b542901ee5564836911" + } + ] + }, + { + "sourcePath": "db/clickhouse/migrations/0025_organization_analytics.sql", + "original": "original/0025_organization_analytics.sql", + "sha256": "4989c4d4fb49d546175935a2560968d8ba8a8406bd4be5b4cb74fbfcd2527a2b", + "migration": "0025_organization_analytics", + "steps": [ + { + "name": "statement-01", + "file": "steps/0025_organization_analytics/statement-01.sql", + "byteStart": 0, + "byteEndExclusive": 992, + "sha256": "92d4b445f5504a545489ae87933d40a4df7de614fd28e75e2653aa489b885655" + }, + { + "name": "statement-02", + "file": "steps/0025_organization_analytics/statement-02.sql", + "byteStart": 992, + "byteEndExclusive": 1132, + "sha256": "de995d491eb69bbeb94aec0b1b751dd019ff274599937926c4ebb75733e829a1" + }, + { + "name": "statement-03", + "file": "steps/0025_organization_analytics/statement-03.sql", + "byteStart": 1132, + "byteEndExclusive": 1438, + "sha256": "d10383c4984f4c7ffd187888dfa3d462b07513211b68dbfb7118dcb3327ba88e" + }, + { + "name": "statement-04", + "file": "steps/0025_organization_analytics/statement-04.sql", + "byteStart": 1438, + "byteEndExclusive": 2916, + "sha256": "f447958c6d3c330e2e440828d1b41dd1b0423e20d3b54c33b6c5f5bed7eeac61" + }, + { + "name": "statement-05", + "file": "steps/0025_organization_analytics/statement-05.sql", + "byteStart": 2916, + "byteEndExclusive": 4633, + "sha256": "5bb8b78303f5e6ccce88bf8fd7d65dbe8e3277d763074fbb0ef5b6bbd9545197" + }, + { + "name": "statement-06", + "file": "steps/0025_organization_analytics/statement-06.sql", + "byteStart": 4633, + "byteEndExclusive": 4921, + "sha256": "0e9ee9c4e9913449d363256b800f2f641d9cf95b5decb7537bd9f82de826a01c" + }, + { + "name": "statement-07", + "file": "steps/0025_organization_analytics/statement-07.sql", + "byteStart": 4921, + "byteEndExclusive": 5251, + "sha256": "a9a210b39318105e5e8993722563773b7dbfcace073f0c7a0b0970dfa40907d1" + }, + { + "name": "statement-08", + "file": "steps/0025_organization_analytics/statement-08.sql", + "byteStart": 5251, + "byteEndExclusive": 5507, + "sha256": "30c46b6108d09b9aa9967f28ebb5c78386202673749bd514786365d07625bde9" + }, + { + "name": "statement-09", + "file": "steps/0025_organization_analytics/statement-09.sql", + "byteStart": 5507, + "byteEndExclusive": 5954, + "sha256": "8a42ba765ea819c2a4a7190e3c51ee19c02085a78b9eabeef7ef06a60e785713" + }, + { + "name": "statement-10", + "file": "steps/0025_organization_analytics/statement-10.sql", + "byteStart": 5954, + "byteEndExclusive": 6343, + "sha256": "dccb389d1e2e6063505ad371bc7d661c421bcada36f1ea1b2d87af6bf348b624" + } + ] + } + ], + "runner": { + "sourcePath": "db/clickhouse/migrate.sh", + "sha256": "dd43aac1d1ef1a10671a16371c046b2fc6c10bd83865fb006614690c424eede4", + "fixturePath": "original/migrate.sh" + }, + "nativeAdaptations": [ + { + "sourceStep": { + "name": "statement-13", + "file": "steps/0017_replay_safe_sources/statement-13.sql" + }, + "sourceStepSha256": "86a29561ca2b4537b06acdfebbee2f4c7140de5b25d19076f1200a06ca5fd7d2", + "nativeSteps": [ + { + "name": "statement-13-prepare", + "file": "native-assertions/statement-13-prepare.sql", + "sha256": "87d5c68cb9c66e9e00fbc9c277b3686ce3fb7b034a4d2fb6f2430212052d6586" + }, + { + "name": "statement-13-check", + "file": "native-assertions/statement-13-check.sql", + "sha256": "10a5aa19258c8c0c3f475acb58e68cd5755941df3a8b172d428ede69b7579b46" + }, + { + "name": "statement-13-cleanup", + "file": "native-assertions/statement-13-cleanup.sql", + "sha256": "f9a129a327b7adf94894d43c88a8cba22dc99c8c08f23c99851e9bb66f7bc356" + } + ], + "reason": "Standalone SELECT is outside v1. Preserve the exact throwIf query as INSERT SELECT into a named application Memory table, with explicit prepare and cleanup steps. A failure stops before cleanup and requires ordinary explicit recovery; there is no automatic SQL cleanup." + }, + { + "sourceStep": { + "name": "statement-14", + "file": "steps/0017_replay_safe_sources/statement-14.sql" + }, + "sourceStepSha256": "f46011bb58ae8532437f1619a01e50818644e77925d47711741b1d82af9e96fc", + "nativeSteps": [ + { + "name": "statement-14-prepare", + "file": "native-assertions/statement-14-prepare.sql", + "sha256": "b6b9ca6e1a1e6c41eaa9e52279df2ba7a70b484fc7398f6fa9cdef3d9f3ee437" + }, + { + "name": "statement-14-check", + "file": "native-assertions/statement-14-check.sql", + "sha256": "64ff25c9988722341dce0ecbaad87f1e3d907dadf5016744be031c00c5a813de" + }, + { + "name": "statement-14-cleanup", + "file": "native-assertions/statement-14-cleanup.sql", + "sha256": "419e7def20782b0895b9bd6f0519371aff50eefa51157f9b0a572523ab0ddb98" + } + ], + "reason": "Standalone SELECT is outside v1. Preserve the exact throwIf query as INSERT SELECT into a named application Memory table, with explicit prepare and cleanup steps. A failure stops before cleanup and requires ordinary explicit recovery; there is no automatic SQL cleanup." + }, + { + "sourceStep": { + "name": "statement-02", + "file": "steps/0018_conservative_identity_resolution/statement-02.sql" + }, + "sourceStepSha256": "24c96eed70c96fcbc65a2ca6468819a80eb10a2580dbe893cff789248df76569", + "nativeSteps": [ + { + "name": "statement-02", + "file": "native-assertions/local-materialize-policy.sql", + "sha256": "dba2fb505bee58a42b071bacd2b6ec61f432172eadc82ae1bd63580d9f810ca6" + } + ], + "reason": "Remove the query-level mutations_sync=2 override. The qualified native target is one non-replicated local server and v1 always supplies its immutable mutations_sync=1 policy. This makes no replica-completion claim." + } + ], + "nativeManifestPolicy": "Original source byte ranges remain exact. The native manifest references those chunks except the three explicitly recorded native adaptations. Current native hashes establish the accepted baseline onward, never historical source-checksum verification." +} diff --git a/test/fixtures/kansoku/steps/0001_events_raw/statement-01.sql b/test/fixtures/kansoku/steps/0001_events_raw/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0001_events_raw/statement-01.sql @@ -0,0 +1,1 @@ +CREATE DATABASE IF NOT EXISTS analytics; diff --git a/test/fixtures/kansoku/steps/0001_events_raw/statement-02.sql b/test/fixtures/kansoku/steps/0001_events_raw/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0001_events_raw/statement-02.sql @@ -0,0 +1,32 @@ + + +CREATE TABLE IF NOT EXISTS analytics.events_raw +( + event_id String, + event_name LowCardinality(String), + source LowCardinality(String), + source_service Nullable(String), + occurred_at DateTime64(3, 'UTC'), + received_at DateTime64(3, 'UTC'), + anonymous_id Nullable(String), + user_id String, + organization_id Nullable(String), + account_id String, + session_id Nullable(String), + url Nullable(String), + referrer Nullable(String), + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + utm_content Nullable(String), + utm_term Nullable(String), + correlation_id Nullable(String), + causation_id Nullable(String), + source_event_id Nullable(String), + source_global_position Nullable(UInt64), + properties String, + context String +) +ENGINE = ReplacingMergeTree(received_at) +PARTITION BY toYYYYMM(occurred_at) +ORDER BY (account_id, user_id, event_name, occurred_at, event_id); diff --git a/test/fixtures/kansoku/steps/0002_identity/statement-01.sql b/test/fixtures/kansoku/steps/0002_identity/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0002_identity/statement-01.sql @@ -0,0 +1,1 @@ +CREATE DATABASE IF NOT EXISTS analytics; diff --git a/test/fixtures/kansoku/steps/0002_identity/statement-02.sql b/test/fixtures/kansoku/steps/0002_identity/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0002_identity/statement-02.sql @@ -0,0 +1,12 @@ + + +-- Latest anonymous_id -> user_id mapping, one row per anonymous_id after merges. +-- `version` is the event's received_at; the newest mapping wins. +CREATE TABLE IF NOT EXISTS analytics.identity_map +( + anonymous_id String, + user_id String, + version DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(version) +ORDER BY anonymous_id; diff --git a/test/fixtures/kansoku/steps/0002_identity/statement-03.sql b/test/fixtures/kansoku/steps/0002_identity/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0002_identity/statement-03.sql @@ -0,0 +1,15 @@ + + +-- Fill identity_map from identify/alias events as they arrive in events_raw. +-- Fires on each inserted block; ignores identity events missing either id. +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_identity_map +TO analytics.identity_map +AS +SELECT + anonymous_id, + user_id, + received_at AS version +FROM analytics.events_raw +WHERE event_name IN ('identify', 'alias') + AND anonymous_id != '' + AND user_id != ''; diff --git a/test/fixtures/kansoku/steps/0002_identity/statement-04.sql b/test/fixtures/kansoku/steps/0002_identity/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0002_identity/statement-04.sql @@ -0,0 +1,27 @@ + + +-- Query-time resolution: every raw event with its canonical user id. +-- Reads events_raw FINAL so duplicate raw rows collapse before row-level use. +-- The subquery collapses identity_map to the latest user_id per anonymous_id. +-- NOTE: nullIf(m.user_id, '') is load-bearing. With ClickHouse's default +-- join_use_nulls = 0, a LEFT JOIN miss fills m.user_id with '' (NOT NULL); +-- without the nullIf, coalesce would stop there and never reach the +-- anonymous-id fallback. The trailing '' keeps the type plain String. +CREATE VIEW IF NOT EXISTS analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e FINAL +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-01.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-01.sql @@ -0,0 +1,1 @@ +CREATE DATABASE IF NOT EXISTS analytics; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-02.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-02.sql @@ -0,0 +1,11 @@ + + +-- ---------- Daily event counts, by event_name ---------- +CREATE TABLE IF NOT EXISTS analytics.daily_event_counts +( + day Date, + event_name String, + events AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, event_name); diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-03.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-03.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + uniqExactState(toString(event_id)) AS events +FROM analytics.events_raw +GROUP BY day, event_name; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-04.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-04.sql @@ -0,0 +1,13 @@ + + +-- ---------- Daily active users (canonical, resolved at insert time) ---------- +-- NOTE: nullIf(m.user_id, '') is load-bearing (join_use_nulls = 0 fills a LEFT +-- JOIN miss with '', not NULL); the trailing '' keeps the argument a plain +-- String so the state stays AggregateFunction(uniqExact, String). +CREATE TABLE IF NOT EXISTS analytics.dau +( + day Date, + users AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY day; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-05.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-05.sql @@ -0,0 +1,20 @@ + + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(e.occurred_at) AS day, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY day; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-06.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-06.sql @@ -0,0 +1,10 @@ + + +-- ---------- Weekly active users (canonical), keyed on the Monday of the week ---------- +CREATE TABLE IF NOT EXISTS analytics.wau +( + week Date, + users AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY week; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-07.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-07.sql @@ -0,0 +1,20 @@ + + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(e.occurred_at) AS week, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-08.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-08.sql @@ -0,0 +1,13 @@ + + +-- ---------- Page views by route ---------- +-- url is Nullable(String) in EP-3's schema; coalesce(url, '') keeps path() +-- non-Nullable so the insert into the String route column cannot fail. +CREATE TABLE IF NOT EXISTS analytics.page_views_by_route +( + day Date, + route String, + views AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, route); diff --git a/test/fixtures/kansoku/steps/0003_rollups/statement-09.sql b/test/fixtures/kansoku/steps/0003_rollups/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0003_rollups/statement-09.sql @@ -0,0 +1,12 @@ + + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + toDate(occurred_at) AS day, + path(coalesce(url, '')) AS route, + uniqExactState(toString(event_id)) AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-01.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-01.sql @@ -0,0 +1,8 @@ +-- Schema v2 for the raw events table: sort key (account_id, event_name, +-- occurred_at, event_id) and Nullable(String) -> String DEFAULT ''. +-- Strategy: build events_raw_v2, copy, EXCHANGE names, keep the old table +-- (now named events_raw_v2) until 0007 drops it after verification. +-- Every object reading events_raw is dropped first and recreated against the +-- new table with UNCHANGED v1 semantics; 0005/0006 change semantics later. + +DROP TABLE IF EXISTS analytics.mv_identity_map; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-02.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-02.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-03.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-03.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_dau; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-04.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-04.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_wau; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-05.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-05.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-06.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-06.sql @@ -0,0 +1,2 @@ + +DROP VIEW IF EXISTS analytics.events_resolved; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-07.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-07.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.events_raw_v2; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-08.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-08.sql @@ -0,0 +1,32 @@ + + +CREATE TABLE analytics.events_raw_v2 +( + event_id String, + event_name LowCardinality(String), + source LowCardinality(String), + source_service String DEFAULT '', + occurred_at DateTime64(3, 'UTC'), + received_at DateTime64(3, 'UTC'), + anonymous_id String DEFAULT '', + user_id String, + organization_id String DEFAULT '', + account_id String, + session_id String DEFAULT '', + url String DEFAULT '', + referrer String DEFAULT '', + utm_source LowCardinality(String) DEFAULT '', + utm_medium LowCardinality(String) DEFAULT '', + utm_campaign LowCardinality(String) DEFAULT '', + utm_content String DEFAULT '', + utm_term String DEFAULT '', + correlation_id String DEFAULT '', + causation_id String DEFAULT '', + source_event_id String DEFAULT '', + source_global_position Nullable(UInt64), + properties String, + context String +) +ENGINE = ReplacingMergeTree(received_at) +PARTITION BY toYYYYMM(occurred_at) +ORDER BY (account_id, event_name, occurred_at, event_id); diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-09.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-09.sql @@ -0,0 +1,18 @@ + + +INSERT INTO analytics.events_raw_v2 + (event_id, event_name, source, source_service, occurred_at, received_at, + anonymous_id, user_id, organization_id, account_id, session_id, url, + referrer, utm_source, utm_medium, utm_campaign, utm_content, utm_term, + correlation_id, causation_id, source_event_id, source_global_position, + properties, context) +SELECT + event_id, event_name, source, coalesce(source_service, ''), occurred_at, + received_at, coalesce(anonymous_id, ''), user_id, + coalesce(organization_id, ''), account_id, coalesce(session_id, ''), + coalesce(url, ''), coalesce(referrer, ''), coalesce(utm_source, ''), + coalesce(utm_medium, ''), coalesce(utm_campaign, ''), + coalesce(utm_content, ''), coalesce(utm_term, ''), + coalesce(correlation_id, ''), coalesce(causation_id, ''), + coalesce(source_event_id, ''), source_global_position, properties, context +FROM analytics.events_raw; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-10.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-10.sql @@ -0,0 +1,5 @@ + + +-- Atomic name swap: analytics.events_raw is now the v2 table; the old v1 +-- table lives on under the name analytics.events_raw_v2 as the rollback copy. +EXCHANGE TABLES analytics.events_raw AND analytics.events_raw_v2; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-11.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-11.sql @@ -0,0 +1,15 @@ + + +-- ---- Recreate dependents against the new table, v1 semantics unchanged ---- + +CREATE MATERIALIZED VIEW analytics.mv_identity_map +TO analytics.identity_map +AS +SELECT + anonymous_id, + user_id, + received_at AS version +FROM analytics.events_raw +WHERE event_name IN ('identify', 'alias') + AND anonymous_id != '' + AND user_id != ''; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-12.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-12.sql @@ -0,0 +1,21 @@ + + +-- Still FINAL here; 0005 removes it. +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e FINAL +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-13.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-13.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + uniqExactState(toString(event_id)) AS events +FROM analytics.events_raw +GROUP BY day, event_name; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-14.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-14.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-14.sql @@ -0,0 +1,21 @@ + + +-- Still joining the identity map here; 0005 replaces these two MVs. +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(e.occurred_at) AS day, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY day; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-15.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-15.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-15.sql @@ -0,0 +1,20 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(e.occurred_at) AS week, + uniqExactState(coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '')) AS users +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id +WHERE coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-16.sql b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-16.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0004_events_raw_v2/statement-16.sql @@ -0,0 +1,12 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + toDate(occurred_at) AS day, + path(url) AS route, + uniqExactState(toString(event_id)) AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-01.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-01.sql @@ -0,0 +1,9 @@ +-- FINAL-free identity resolution and join-free DAU/WAU. +-- events_resolved consumers aggregate with uniqExact/min, which are +-- insensitive to the transient duplicate rows FINAL would remove; row-COUNT +-- consumers must protect themselves (count DISTINCT event_id). +-- mv_dau/mv_wau stop joining the identity map at insert time and count raw +-- identities (user_id if present, else anonymous_id); the stitched number +-- stays available at query time through events_resolved. + +DROP VIEW IF EXISTS analytics.events_resolved; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-02.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-02.sql @@ -0,0 +1,20 @@ + + +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-03.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-03.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_dau; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-04.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-04.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_wau; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-05.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-05.sql @@ -0,0 +1,7 @@ + + +-- Rebuild dau/wau contents under the new (raw-identity) semantics so the +-- tables are not a mix of stitched and unstitched day states. Column types +-- are unchanged in this migration (still AggregateFunction(uniqExact, String)); +-- 0006 replaces the state type. +TRUNCATE TABLE analytics.dau; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-06.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-06.sql @@ -0,0 +1,2 @@ + +TRUNCATE TABLE analytics.wau; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-07.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-07.sql @@ -0,0 +1,9 @@ + + +INSERT INTO analytics.dau (day, users) +SELECT + toDate(occurred_at) AS day, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-08.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-08.sql @@ -0,0 +1,9 @@ + + +INSERT INTO analytics.wau (week, users) +SELECT + toMonday(occurred_at) AS week, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-09.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-09.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(occurred_at) AS day, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; diff --git a/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-10.sql b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0005_final_free_resolution/statement-10.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(occurred_at) AS week, + uniqExactState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-01.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-01.sql @@ -0,0 +1,9 @@ +-- Rollup states v2: bounded-size states instead of exact-distinct states that +-- stored every id. events/views: AggregateFunction(uniqExact, String) over +-- event_id -> AggregateFunction(count) (dedup makes duplicate rows rare and +-- transient, so count-vs-distinct drift is negligible; exactness lives in the +-- raw table). dau/wau users: uniqExact -> uniqCombined sketch (~0.8% error at +-- scale, exact for small sets). Tables are rebuilt (state types cannot be +-- ALTERed) and backfilled from events_raw. + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-02.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-02.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_dau; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-03.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-03.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_wau; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-04.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-04.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-05.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-05.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-06.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-06.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.dau; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-07.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-07.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.wau; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-08.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-08.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-09.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-09.sql @@ -0,0 +1,10 @@ + + +CREATE TABLE analytics.daily_event_counts +( + day Date, + event_name String, + events AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, event_name); diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-10.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-10.sql @@ -0,0 +1,9 @@ + + +CREATE TABLE analytics.dau +( + day Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY day; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-11.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-11.sql @@ -0,0 +1,9 @@ + + +CREATE TABLE analytics.wau +( + week Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY week; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-12.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-12.sql @@ -0,0 +1,10 @@ + + +CREATE TABLE analytics.page_views_by_route +( + day Date, + route String, + views AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (day, route); diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-13.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-13.sql @@ -0,0 +1,10 @@ + + +-- Backfill from the (already deduplicated-by-merges) raw table. +INSERT INTO analytics.daily_event_counts (day, event_name, events) +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY day, event_name; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-14.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-14.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-14.sql @@ -0,0 +1,9 @@ + + +INSERT INTO analytics.dau (day, users) +SELECT + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-15.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-15.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-15.sql @@ -0,0 +1,9 @@ + + +INSERT INTO analytics.wau (week, users) +SELECT + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-16.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-16.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-16.sql @@ -0,0 +1,10 @@ + + +INSERT INTO analytics.page_views_by_route (day, route, views) +SELECT + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-17.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-17.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-17.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY day, event_name; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-18.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-18.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-18.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY day; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-19.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-19.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-19.sql @@ -0,0 +1,11 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY week; diff --git a/test/fixtures/kansoku/steps/0006_rollups_v2/statement-20.sql b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-20.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0006_rollups_v2/statement-20.sql @@ -0,0 +1,12 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY day, route; diff --git a/test/fixtures/kansoku/steps/0007_drop_events_raw_v1/statement-01.sql b/test/fixtures/kansoku/steps/0007_drop_events_raw_v1/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0007_drop_events_raw_v1/statement-01.sql @@ -0,0 +1,3 @@ +-- Final step of the events_raw v2 rewrite (0004): drop the retained v1 table. +-- 0004's row-count verification must have passed before this file is created. +DROP TABLE IF EXISTS analytics.events_raw_v2; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-01.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-01.sql @@ -0,0 +1,4 @@ +-- Account-scoped metrics. Keep the FINAL-free identity resolution and the +-- bounded v2 aggregate states while adding account_id to every query source. + +DROP VIEW IF EXISTS analytics.events_resolved; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-02.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-02.sql @@ -0,0 +1,21 @@ + + +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-03.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-03.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-04.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-04.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_dau; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-05.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-05.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_wau; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-06.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-06.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-07.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-07.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-08.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-08.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.dau; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-09.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-09.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.wau; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-10.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-10.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-11.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-11.sql @@ -0,0 +1,11 @@ + + +CREATE TABLE analytics.daily_event_counts +( + account_id String, + day Date, + event_name String, + events AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, event_name); diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-12.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-12.sql @@ -0,0 +1,10 @@ + + +CREATE TABLE analytics.dau +( + account_id String, + day Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day); diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-13.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-13.sql @@ -0,0 +1,10 @@ + + +CREATE TABLE analytics.wau +( + account_id String, + week Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, week); diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-14.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-14.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-14.sql @@ -0,0 +1,11 @@ + + +CREATE TABLE analytics.page_views_by_route +( + account_id String, + day Date, + route String, + views AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, route); diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-15.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-15.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-15.sql @@ -0,0 +1,10 @@ + + +INSERT INTO analytics.daily_event_counts (account_id, day, event_name, events) +SELECT + account_id, + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY account_id, day, event_name; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-16.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-16.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-16.sql @@ -0,0 +1,10 @@ + + +INSERT INTO analytics.dau (account_id, day, users) +SELECT + account_id, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-17.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-17.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-17.sql @@ -0,0 +1,10 @@ + + +INSERT INTO analytics.wau (account_id, week, users) +SELECT + account_id, + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-18.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-18.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-18.sql @@ -0,0 +1,11 @@ + + +INSERT INTO analytics.page_views_by_route (account_id, day, route, views) +SELECT + account_id, + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-19.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-19.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-19.sql @@ -0,0 +1,12 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts +AS +SELECT + account_id, + toDate(occurred_at) AS day, + event_name AS event_name, + countState() AS events +FROM analytics.events_raw +GROUP BY account_id, day, event_name; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-20.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-20.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-20.sql @@ -0,0 +1,12 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_dau +TO analytics.dau +AS +SELECT + account_id, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-21.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-21.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-21.sql @@ -0,0 +1,12 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_wau +TO analytics.wau +AS +SELECT + account_id, + toMonday(occurred_at) AS week, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; diff --git a/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-22.sql b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-22.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0008_account_scoped_metrics/statement-22.sql @@ -0,0 +1,13 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route +AS +SELECT + account_id, + toDate(occurred_at) AS day, + path(url) AS route, + countState() AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/steps/0009_typed_properties/statement-01.sql b/test/fixtures/kansoku/steps/0009_typed_properties/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0009_typed_properties/statement-01.sql @@ -0,0 +1,6 @@ +-- EP-17 Milestone 1 chose a server-side native JSON materialized column. +-- Kansoku.ClickHouse.EventRow serializes properties from a parsed Aeson Object, +-- so every properties value written by the application is valid JSON. +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS properties_json JSON + MATERIALIZED CAST(properties, 'JSON'); diff --git a/test/fixtures/kansoku/steps/0009_typed_properties/statement-02.sql b/test/fixtures/kansoku/steps/0009_typed_properties/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0009_typed_properties/statement-02.sql @@ -0,0 +1,3 @@ + + +ALTER TABLE analytics.events_raw MATERIALIZE COLUMN properties_json; diff --git a/test/fixtures/kansoku/steps/0010_resolved_event_properties/statement-01.sql b/test/fixtures/kansoku/steps/0010_resolved_event_properties/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0010_resolved_event_properties/statement-01.sql @@ -0,0 +1,3 @@ +-- Property-grouped unique-user metrics need the same typed property projection +-- as raw-event metrics while retaining canonical identity stitching. +DROP VIEW IF EXISTS analytics.events_resolved; diff --git a/test/fixtures/kansoku/steps/0010_resolved_event_properties/statement-02.sql b/test/fixtures/kansoku/steps/0010_resolved_event_properties/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0010_resolved_event_properties/statement-02.sql @@ -0,0 +1,22 @@ + + +CREATE VIEW analytics.events_resolved +AS +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + coalesce(nullIf(e.user_id, ''), nullIf(m.user_id, ''), nullIf(e.anonymous_id, ''), '') AS canonical_user, + e.properties_json AS properties_json +FROM analytics.events_raw AS e +LEFT JOIN +( + SELECT + anonymous_id, + argMax(user_id, version) AS user_id + FROM analytics.identity_map + GROUP BY anonymous_id +) AS m +ON e.anonymous_id = m.anonymous_id; diff --git a/test/fixtures/kansoku/steps/0011_events_raw_identifiers/statement-01.sql b/test/fixtures/kansoku/steps/0011_events_raw_identifiers/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0011_events_raw_identifiers/statement-01.sql @@ -0,0 +1,5 @@ +-- EP-18 M1: persist the identifiers map (kind -> value) on raw events. +-- The ClickHaskell wire carries a JSON-object String; '{}' keeps old rows +-- valid for server-side JSON extraction without rewriting existing parts. +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS identifiers String DEFAULT '{}' AFTER context; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-01.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-01.sql @@ -0,0 +1,15 @@ +-- EP-18 M2: retain immutable identity-binding evidence and resolve anonymous +-- events through account-scoped, confidence-ranked identifier signals. +CREATE TABLE IF NOT EXISTS analytics.identity_signals +( + account_id String, + signal_kind LowCardinality(String), + signal_value String, + user_id String, + observed_at DateTime64(3, 'UTC'), + event_id String +) +ENGINE = ReplacingMergeTree +ORDER BY (account_id, signal_kind, signal_value, user_id, observed_at, event_id) +TTL toDateTime(observed_at) + INTERVAL 180 DAY DELETE + WHERE signal_kind = 'fingerprint'; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-02.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-02.sql @@ -0,0 +1,21 @@ + + +-- Cookie bindings retain the old identify/alias-only semantics. Explicit +-- identifier-map entries are binding evidence on any event carrying user_id. +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_identity_signals +TO analytics.identity_signals +AS +SELECT + account_id, + sig.1 AS signal_kind, + sig.2 AS signal_value, + user_id, + received_at AS observed_at, + event_id +FROM analytics.events_raw +ARRAY JOIN arrayFilter( + kv -> kv.1 != '' AND kv.2 != '', + arrayConcat( + [(if(event_name IN ('identify', 'alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig +WHERE user_id != ''; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-03.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-03.sql @@ -0,0 +1,20 @@ + + +-- Backfill the evidence that predates the materialized view. Replaying this +-- statement produces byte-identical rows, which the full ORDER BY key dedups. +INSERT INTO analytics.identity_signals + (account_id, signal_kind, signal_value, user_id, observed_at, event_id) +SELECT + account_id, + sig.1 AS signal_kind, + sig.2 AS signal_value, + user_id, + received_at AS observed_at, + event_id +FROM analytics.events_raw +ARRAY JOIN arrayFilter( + kv -> kv.1 != '' AND kv.2 != '', + arrayConcat( + [(if(event_name IN ('identify', 'alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig +WHERE user_id != ''; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-04.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-04.sql @@ -0,0 +1,19 @@ + + +-- Policy is view-time, so changing confidence later re-scores all history +-- without mutating evidence rows. +CREATE VIEW IF NOT EXISTS analytics.identity_bindings +AS +SELECT + account_id, + signal_kind, + signal_value, + user_id, + observed_at, + multiIf( + signal_kind = 'email_sha256', 0.95, + signal_kind = 'cookie', 0.90, + signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, + 0.0) AS confidence +FROM analytics.identity_signals; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-05.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-05.sql @@ -0,0 +1,17 @@ + + +-- Conflicting bindings stay visible rather than being collapsed by a +-- latest-wins table. +CREATE VIEW IF NOT EXISTS analytics.identity_conflicts +AS +SELECT + account_id, + signal_kind, + signal_value, + groupUniqArray(user_id) AS user_ids, + uniqExact(user_id) AS user_count, + count() AS observation_count, + max(observed_at) AS last_observed_at +FROM analytics.identity_signals +GROUP BY account_id, signal_kind, signal_value +HAVING user_count > 1; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-06.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-06.sql @@ -0,0 +1,49 @@ + + +-- Resolve explicit user_id first, then the strongest and newest binding +-- reachable from the event's cookie or identifiers, then the raw cookie. +-- The ordered seven-column output is the contract recorded in ADR 0004. +CREATE VIEW IF NOT EXISTS analytics.events_resolved_v3 +AS +WITH event_signal_users AS +( + SELECT + s.account_id AS account_id, + s.event_id AS event_id, + argMax(b.user_id, tuple(b.confidence, b.observed_at)) AS stitched_user + FROM + ( + SELECT + account_id, + event_id, + sig.1 AS signal_kind, + sig.2 AS signal_value + FROM analytics.events_raw + ARRAY JOIN arrayFilter( + kv -> kv.2 != '', + arrayConcat( + [('cookie', anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig + WHERE user_id = '' + ) AS s + INNER JOIN analytics.identity_bindings AS b + ON b.account_id = s.account_id + AND b.signal_kind = s.signal_kind + AND b.signal_value = s.signal_value + WHERE b.confidence >= 0.50 + GROUP BY s.account_id, s.event_id +) +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + multiIf( + e.user_id != '', e.user_id, + r.stitched_user != '', r.stitched_user, + e.anonymous_id) AS canonical_user, + e.properties_json AS properties_json +FROM analytics.events_raw AS e +LEFT JOIN event_signal_users AS r + ON r.account_id = e.account_id AND r.event_id = e.event_id; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-07.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-07.sql @@ -0,0 +1,3 @@ + + +DROP VIEW IF EXISTS analytics.events_resolved; diff --git a/test/fixtures/kansoku/steps/0012_identity_signals/statement-08.sql b/test/fixtures/kansoku/steps/0012_identity_signals/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0012_identity_signals/statement-08.sql @@ -0,0 +1,3 @@ + +CREATE VIEW analytics.events_resolved +AS SELECT * FROM analytics.events_resolved_v3; diff --git a/test/fixtures/kansoku/steps/0013_retire_identity_map/statement-01.sql b/test/fixtures/kansoku/steps/0013_retire_identity_map/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0013_retire_identity_map/statement-01.sql @@ -0,0 +1,4 @@ +-- EP-18 M3: identity_signals and identity_bindings supersede the old +-- latest-wins cookie map. Migration 0012 backfilled equivalent cookie evidence, +-- and the pre-retirement stability battery preserved canonical-user results. +DROP TABLE IF EXISTS analytics.mv_identity_map; diff --git a/test/fixtures/kansoku/steps/0013_retire_identity_map/statement-02.sql b/test/fixtures/kansoku/steps/0013_retire_identity_map/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0013_retire_identity_map/statement-02.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.identity_map; diff --git a/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-01.sql b/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-01.sql @@ -0,0 +1,20 @@ +-- Experiment exposure rollup: unique users per account x experiment x variant x day. +-- +-- Identity is the raw identity (user_id if present, else anonymous_id), matching +-- the join-free dau/wau rollups. An insert-time materialized view cannot apply +-- query-time identity resolution without freezing stale bindings. The exact, +-- stitched conversion and funnel endpoints read analytics.events_resolved. +-- +-- IF NOT EXISTS plus the union semantics of uniqCombined states make a retry +-- after a partial failure safe: a repeated backfill only merges the same users. + +CREATE TABLE IF NOT EXISTS analytics.experiment_exposures +( + account_id String, + experiment LowCardinality(String), + variant LowCardinality(String), + day Date, + users AggregateFunction(uniqCombined, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, experiment, variant, day); diff --git a/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-02.sql b/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-02.sql @@ -0,0 +1,15 @@ + + +INSERT INTO analytics.experiment_exposures (account_id, experiment, variant, day, users) +SELECT + account_id, + toString(getSubcolumn(properties_json, 'experiment')) AS experiment, + toString(getSubcolumn(properties_json, 'variant')) AS variant, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE event_name = 'experiment_viewed' + AND (user_id != '' OR anonymous_id != '') + AND match(toString(getSubcolumn(properties_json, 'experiment')), '^[a-z0-9_.:-]{1,128}$') + AND match(toString(getSubcolumn(properties_json, 'variant')), '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-03.sql b/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0014_experiment_exposures/statement-03.sql @@ -0,0 +1,20 @@ + + +-- Inserts made between the backfill above and this trigger creation are not +-- observed. EP-19 lands before production experiment traffic; in development, +-- reset and replay migrations if that window matters. +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_experiment_exposures +TO analytics.experiment_exposures +AS +SELECT + account_id, + toString(getSubcolumn(properties_json, 'experiment')) AS experiment, + toString(getSubcolumn(properties_json, 'variant')) AS variant, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE event_name = 'experiment_viewed' + AND (user_id != '' OR anonymous_id != '') + AND match(toString(getSubcolumn(properties_json, 'experiment')), '^[a-z0-9_.:-]{1,128}$') + AND match(toString(getSubcolumn(properties_json, 'variant')), '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/steps/0015_experiment_exposure_string_casts/statement-01.sql b/test/fixtures/kansoku/steps/0015_experiment_exposure_string_casts/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0015_experiment_exposure_string_casts/statement-01.sql @@ -0,0 +1,7 @@ +-- Correct EP-19's materialized-view input-block behavior without editing the +-- already-applied 0014 migration. When a block contains no experiment paths, +-- getSubcolumn over JSON can retain Dynamic even after toString; ClickHouse 26 +-- rejects that alias as a GROUP BY key before the event-name filter removes the +-- rows. An explicit outer String cast makes every ordinary event block safe. + +DROP TABLE IF EXISTS analytics.mv_experiment_exposures; diff --git a/test/fixtures/kansoku/steps/0015_experiment_exposure_string_casts/statement-02.sql b/test/fixtures/kansoku/steps/0015_experiment_exposure_string_casts/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0015_experiment_exposure_string_casts/statement-02.sql @@ -0,0 +1,17 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_experiment_exposures +TO analytics.experiment_exposures +AS +SELECT + account_id, + CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, + CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, + toDate(occurred_at) AS day, + uniqCombinedState(if(user_id != '', user_id, anonymous_id)) AS users +FROM analytics.events_raw +WHERE event_name = 'experiment_viewed' + AND (user_id != '' OR anonymous_id != '') + AND match(CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String'), '^[a-z0-9_.:-]{1,128}$') + AND match(CAST(toString(getSubcolumn(properties_json, 'variant')), 'String'), '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/steps/0016_events_raw_schema_version/statement-01.sql b/test/fixtures/kansoku/steps/0016_events_raw_schema_version/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0016_events_raw_schema_version/statement-01.sql @@ -0,0 +1,2 @@ +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS schema_version UInt16 DEFAULT 1 AFTER identifiers; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-01.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-01.sql @@ -0,0 +1,6 @@ +-- EP-30: one immutable logical event before analytical joins/aggregations. +-- Administrative migration: keep the writer stopped throughout. A failed run +-- can be reapplied: replacement states are rebuilt from canonical retained rows, +-- rather than appended to partially populated aggregate tables. +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *, properties_json FROM analytics.events_raw FINAL; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-02.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-02.sql @@ -0,0 +1,17 @@ + + +CREATE OR REPLACE VIEW analytics.identity_bindings +AS +SELECT + account_id, + signal_kind, + signal_value, + user_id, + observed_at, + multiIf( + signal_kind = 'email_sha256', 0.95, + signal_kind = 'cookie', 0.90, + signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, + 0.0) AS confidence +FROM analytics.identity_signals FINAL; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-03.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-03.sql @@ -0,0 +1,17 @@ + + +-- Conflicting bindings stay visible rather than being collapsed by a +-- latest-wins table. +CREATE OR REPLACE VIEW analytics.identity_conflicts +AS +SELECT + account_id, + signal_kind, + signal_value, + groupUniqArray(user_id) AS user_ids, + uniqExact(user_id) AS user_count, + count() AS observation_count, + max(observed_at) AS last_observed_at +FROM analytics.identity_signals FINAL +GROUP BY account_id, signal_kind, signal_value +HAVING user_count > 1; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-04.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-04.sql @@ -0,0 +1,46 @@ + + +CREATE OR REPLACE VIEW analytics.events_resolved_v3 +AS +WITH event_signal_users AS +( + SELECT + s.account_id AS account_id, + s.event_id AS event_id, + argMax(b.user_id, tuple(b.confidence, b.observed_at)) AS stitched_user + FROM + ( + SELECT + account_id, + event_id, + sig.1 AS signal_kind, + sig.2 AS signal_value + FROM analytics.events_canonical + ARRAY JOIN arrayFilter( + kv -> kv.2 != '', + arrayConcat( + [('cookie', anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String'))) AS sig + WHERE user_id = '' + ) AS s + INNER JOIN analytics.identity_bindings AS b + ON b.account_id = s.account_id + AND b.signal_kind = s.signal_kind + AND b.signal_value = s.signal_value + WHERE b.confidence >= 0.50 + GROUP BY s.account_id, s.event_id +) +SELECT + e.event_id AS event_id, + e.event_name AS event_name, + e.occurred_at AS occurred_at, + e.url AS url, + e.account_id AS account_id, + multiIf( + e.user_id != '', e.user_id, + r.stitched_user != '', r.stitched_user, + e.anonymous_id) AS canonical_user, + e.properties_json AS properties_json +FROM analytics.events_canonical AS e +LEFT JOIN event_signal_users AS r + ON r.account_id = e.account_id AND r.event_id = e.event_id; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-05.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-05.sql @@ -0,0 +1,4 @@ + + + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-06.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-06.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-07.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-07.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.daily_event_counts_rebuild; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-08.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-08.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.page_views_by_route_rebuild; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-09.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-09.sql @@ -0,0 +1,11 @@ + + +CREATE TABLE analytics.daily_event_counts_rebuild +( + account_id String, + day Date, + event_name String, + events AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, event_name); diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-10.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-10.sql @@ -0,0 +1,11 @@ + + +CREATE TABLE analytics.page_views_by_route_rebuild +( + account_id String, + day Date, + route String, + views AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +ORDER BY (account_id, day, route); diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-11.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-11.sql @@ -0,0 +1,7 @@ + + +INSERT INTO analytics.daily_event_counts_rebuild +SELECT account_id, toDate(occurred_at) AS day, event_name, + uniqExactState(event_id) AS events +FROM analytics.events_canonical +GROUP BY account_id, day, event_name; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-12.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-12.sql @@ -0,0 +1,8 @@ + + +INSERT INTO analytics.page_views_by_route_rebuild +SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, + uniqExactState(event_id) AS views +FROM analytics.events_canonical +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-13.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-13.sql @@ -0,0 +1,15 @@ + + +-- Verify complete replacement groups before publishing either table. +SELECT throwIf(count() > 0, 'canonical daily-event rebuild mismatch') +FROM +( + SELECT account_id, toDate(occurred_at) AS day, event_name, count() AS n + FROM analytics.events_canonical GROUP BY account_id, day, event_name +) AS canonical +FULL OUTER JOIN +( + SELECT account_id, day, event_name, uniqExactMerge(events) AS n + FROM analytics.daily_event_counts_rebuild GROUP BY account_id, day, event_name +) AS replacement USING (account_id, day, event_name) +WHERE ifNull(canonical.n, 0) != ifNull(replacement.n, 0); diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-14.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-14.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-14.sql @@ -0,0 +1,15 @@ + + +SELECT throwIf(count() > 0, 'canonical page-view rebuild mismatch') +FROM +( + SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, count() AS n + FROM analytics.events_canonical WHERE event_name = 'page_view' + GROUP BY account_id, day, route +) AS canonical +FULL OUTER JOIN +( + SELECT account_id, day, route, uniqExactMerge(views) AS n + FROM analytics.page_views_by_route_rebuild GROUP BY account_id, day, route +) AS replacement USING (account_id, day, route) +WHERE ifNull(canonical.n, 0) != ifNull(replacement.n, 0); diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-15.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-15.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-15.sql @@ -0,0 +1,6 @@ + + +-- Each swap is atomic. The writer stays stopped across both. On interruption, +-- discard/rebuild the temporary tables and swap again from canonical data; +-- this works whether the public name has its old or its replacement schema. +EXCHANGE TABLES analytics.daily_event_counts AND analytics.daily_event_counts_rebuild; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-16.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-16.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-16.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.page_views_by_route AND analytics.page_views_by_route_rebuild; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-17.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-17.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-17.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.daily_event_counts_rebuild; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-18.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-18.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-18.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.page_views_by_route_rebuild; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-19.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-19.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-19.sql @@ -0,0 +1,10 @@ + + +-- Trigger inputs are the current inserted block. Distinct-ID states compose +-- across arbitrary retry batches, parts, restarts and old archive replays. +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts +TO analytics.daily_event_counts AS +SELECT account_id, toDate(occurred_at) AS day, event_name, + uniqExactState(event_id) AS events +FROM analytics.events_raw +GROUP BY account_id, day, event_name; diff --git a/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-20.sql b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-20.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0017_replay_safe_sources/statement-20.sql @@ -0,0 +1,9 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route +TO analytics.page_views_by_route AS +SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, + uniqExactState(event_id) AS views +FROM analytics.events_raw +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-01.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-01.sql @@ -0,0 +1,6 @@ +-- EP-32: writer-stopped identity evidence/index rebuild and conservative exact resolution. +-- Reapply after interruption with the writer still stopped. Canonical raw facts +-- are authoritative; max/unique state rebuilds never append incompatible history. +-- Length-prefixed components are lossless, including separators and UTF-8. +-- Persist this key so metrics read/hash one string instead of five key vectors. +ALTER TABLE analytics.events_raw ADD COLUMN IF NOT EXISTS identity_context_key String MATERIALIZED concat(toString(length(account_id)),':',account_id,toString(length(source_service)),':',source_service,toString(length(session_id)),':',session_id,toString(length(anonymous_id)),':',anonymous_id,toString(length(identifiers)),':',identifiers); diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-02.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-02.sql @@ -0,0 +1,2 @@ + +ALTER TABLE analytics.events_raw MATERIALIZE COLUMN identity_context_key SETTINGS mutations_sync=2; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-03.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-03.sql @@ -0,0 +1,3 @@ + +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *,properties_json,identity_context_key FROM analytics.events_raw FINAL; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-04.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-04.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_identity_signal_users; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-05.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-05.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_identity_session_users; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-06.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-06.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_identity_signals; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-07.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-07.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.identity_signals_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-08.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-08.sql @@ -0,0 +1,10 @@ + +CREATE TABLE analytics.identity_signals_rebuild +( + account_id String, signal_kind LowCardinality(String), signal_value String, + user_id String, observed_at DateTime64(3, 'UTC'), event_id String, + received_at DateTime64(3, 'UTC'), source_service String, session_id String +) +ENGINE = ReplacingMergeTree +ORDER BY (account_id, signal_kind, signal_value, user_id, observed_at, event_id) +TTL toDateTime(observed_at) + INTERVAL 180 DAY + INTERVAL 1 SECOND DELETE WHERE signal_kind = 'fingerprint'; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-09.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-09.sql @@ -0,0 +1,9 @@ + + +INSERT INTO analytics.identity_signals_rebuild +SELECT account_id, sig.1, sig.2, user_id, occurred_at, event_id, received_at, source_service, session_id +FROM analytics.events_canonical +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.1 != '' AND kv.2 != '', + arrayConcat([(if(event_name IN ('identify','identify_once','alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String')))) AS sig +WHERE user_id != ''; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-10.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-10.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.identity_signals AND analytics.identity_signals_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-11.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-11.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.identity_signals_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-12.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-12.sql @@ -0,0 +1,10 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_identity_signals TO analytics.identity_signals AS +SELECT account_id, sig.1 AS signal_kind, sig.2 AS signal_value, user_id, + occurred_at AS observed_at, event_id, received_at, source_service, session_id +FROM analytics.events_raw +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.1 != '' AND kv.2 != '', + arrayConcat([(if(event_name IN ('identify','identify_once','alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String')))) AS sig +WHERE user_id != ''; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-13.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-13.sql @@ -0,0 +1,5 @@ + + +-- Compact exact indexes retain one latest timestamp per observed signal/user, +-- and one per cookie/session/user. Max is replay-safe; evidence remains auditable. +DROP TABLE IF EXISTS analytics.mv_identity_signal_users; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-14.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-14.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-14.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.mv_identity_session_users; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-15.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-15.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-15.sql @@ -0,0 +1,8 @@ + +CREATE TABLE IF NOT EXISTS analytics.identity_signal_users +( + account_id String, signal_kind LowCardinality(String), signal_value String, user_id String, + observed_at SimpleAggregateFunction(max, DateTime64(3, 'UTC')) +) +ENGINE = AggregatingMergeTree ORDER BY (account_id, signal_kind, signal_value, user_id) +TTL toDateTime(observed_at) + INTERVAL 180 DAY + INTERVAL 1 SECOND DELETE WHERE signal_kind = 'fingerprint'; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-16.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-16.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-16.sql @@ -0,0 +1,7 @@ + +CREATE TABLE IF NOT EXISTS analytics.identity_session_users +( + account_id String, source_service String, session_id String, signal_value String, user_id String, + observed_at SimpleAggregateFunction(max, DateTime64(3, 'UTC')) +) +ENGINE = AggregatingMergeTree ORDER BY (account_id, source_service, session_id, signal_value, user_id); diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-17.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-17.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-17.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.identity_signal_users_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-18.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-18.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-18.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.identity_session_users_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-19.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-19.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-19.sql @@ -0,0 +1,2 @@ + +CREATE TABLE analytics.identity_signal_users_rebuild AS analytics.identity_signal_users; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-20.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-20.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-20.sql @@ -0,0 +1,2 @@ + +CREATE TABLE analytics.identity_session_users_rebuild AS analytics.identity_session_users; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-21.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-21.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-21.sql @@ -0,0 +1,4 @@ + +INSERT INTO analytics.identity_signal_users_rebuild +SELECT account_id, signal_kind, signal_value, user_id, max(observed_at) +FROM analytics.identity_signals FINAL GROUP BY account_id, signal_kind, signal_value, user_id; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-22.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-22.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-22.sql @@ -0,0 +1,5 @@ + +INSERT INTO analytics.identity_session_users_rebuild +SELECT account_id, source_service, session_id, signal_value, user_id, max(observed_at) +FROM analytics.identity_signals FINAL WHERE signal_kind = 'cookie' AND session_id != '' +GROUP BY account_id, source_service, session_id, signal_value, user_id; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-23.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-23.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-23.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.identity_signal_users AND analytics.identity_signal_users_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-24.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-24.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-24.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.identity_session_users AND analytics.identity_session_users_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-25.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-25.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-25.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.identity_signal_users_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-26.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-26.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-26.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.identity_session_users_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-27.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-27.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-27.sql @@ -0,0 +1,4 @@ + +CREATE MATERIALIZED VIEW analytics.mv_identity_signal_users TO analytics.identity_signal_users AS +SELECT account_id, signal_kind, signal_value, user_id, max(observed_at) AS observed_at +FROM analytics.identity_signals GROUP BY account_id, signal_kind, signal_value, user_id; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-28.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-28.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-28.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_identity_session_users TO analytics.identity_session_users AS +SELECT account_id, source_service, session_id, signal_value, user_id, max(observed_at) AS observed_at +FROM analytics.identity_signals WHERE signal_kind = 'cookie' AND session_id != '' +GROUP BY account_id, source_service, session_id, signal_value, user_id; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-29.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-29.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-29.sql @@ -0,0 +1,6 @@ + + +-- Identity depends on a context, not each individual event. Keep an exact +-- replay-safe per-day signal index so repeated session activity does not need +-- raw JSON expansion on every read. Raw candidates still bound returned facts. +DROP TABLE IF EXISTS analytics.mv_identity_context_signals; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-30.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-30.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-30.sql @@ -0,0 +1,8 @@ + +CREATE TABLE IF NOT EXISTS analytics.identity_context_signals +( + account_id String, day Date, + context_key Tuple(String,String,String,String), + signal_kind LowCardinality(String), signal_value String +) +ENGINE = ReplacingMergeTree ORDER BY (account_id,day,context_key,signal_kind,signal_value); diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-31.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-31.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-31.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.identity_context_signals_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-32.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-32.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-32.sql @@ -0,0 +1,2 @@ + +CREATE TABLE analytics.identity_context_signals_rebuild AS analytics.identity_context_signals; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-33.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-33.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-33.sql @@ -0,0 +1,7 @@ + +INSERT INTO analytics.identity_context_signals_rebuild +SELECT account_id,toDate(occurred_at),tuple(source_service,session_id,anonymous_id,identifiers),sig.1,sig.2 +FROM analytics.events_canonical +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.2!='', + arrayConcat([('cookie',anonymous_id)],JSONExtractKeysAndValues(identifiers,'String')))) AS sig +WHERE user_id=''; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-34.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-34.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-34.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.identity_context_signals AND analytics.identity_context_signals_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-35.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-35.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-35.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.identity_context_signals_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-36.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-36.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-36.sql @@ -0,0 +1,8 @@ + +CREATE MATERIALIZED VIEW analytics.mv_identity_context_signals TO analytics.identity_context_signals AS +SELECT account_id,toDate(occurred_at) AS day,tuple(source_service,session_id,anonymous_id,identifiers) AS context_key, + sig.1 AS signal_kind,sig.2 AS signal_value +FROM analytics.events_raw +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.2!='', + arrayConcat([('cookie',anonymous_id)],JSONExtractKeysAndValues(identifiers,'String')))) AS sig +WHERE user_id=''; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-37.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-37.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-37.sql @@ -0,0 +1,9 @@ + + +CREATE OR REPLACE VIEW analytics.identity_bindings AS +SELECT account_id, signal_kind, signal_value, user_id, observed_at, + multiIf(signal_kind IN ('email_hmac','email_sha256'), 0.95, + signal_kind = 'cookie', 0.90, signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, 0.0) AS confidence +FROM analytics.identity_signals FINAL +WHERE signal_kind != 'fingerprint' OR observed_at >= now64(3) - INTERVAL 180 DAY; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-38.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-38.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-38.sql @@ -0,0 +1,5 @@ + +CREATE OR REPLACE VIEW analytics.identity_conflicts AS +SELECT account_id, signal_kind, signal_value, arraySort(groupUniqArray(user_id)) AS user_ids, + uniqExact(user_id) AS user_count, count() AS observation_count, max(observed_at) AS last_observed_at +FROM analytics.identity_bindings GROUP BY account_id, signal_kind, signal_value HAVING user_count > 1; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-39.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-39.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-39.sql @@ -0,0 +1,76 @@ + + +-- Parameterized candidates are bounded BEFORE mapped-signal expansion. Evidence +-- is indexed separately and is intentionally not bounded by the candidate dates. +-- all_accounts is an explicit administrative/public-SQL flag, never inferred +-- from an empty account ID. HTTP always passes zero. +CREATE OR REPLACE VIEW analytics.events_resolved_internal AS +WITH +candidates AS +( + SELECT * FROM analytics.events_canonical + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND occurred_at >= {scope_from:DateTime64(3, 'UTC')} + AND occurred_at < {scope_to:DateTime64(3, 'UTC')} +), +candidate_signals AS MATERIALIZED +( + SELECT DISTINCT account_id, context_key, context_key.1 AS source_service, context_key.2 AS session_id, + signal_kind, signal_value + FROM analytics.identity_context_signals FINAL + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND day >= toDate({scope_from:DateTime64(3, 'UTC')}) + AND toDateTime64(day,3,'UTC') < {scope_to:DateTime64(3, 'UTC')} +), +bindings AS +( + SELECT account_id, signal_kind, signal_value, uniqExact(user_id) AS users, min(user_id) AS unique_user, + multiIf(signal_kind IN ('email_hmac','email_sha256'), 0.95, + signal_kind = 'cookie', 0.90, signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, 0.0) AS confidence + FROM analytics.identity_signal_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,signal_kind,signal_value) IN (SELECT account_id,signal_kind,signal_value FROM candidate_signals) + AND (signal_kind != 'fingerprint' OR observed_at >= now64(3) - INTERVAL 180 DAY) + GROUP BY account_id, signal_kind, signal_value +), +session_users AS +( + SELECT account_id, source_service, session_id, signal_value, uniqExact(user_id) AS users, + min(user_id) AS unique_user + FROM analytics.identity_session_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,source_service,session_id,signal_value) IN + (SELECT account_id,source_service,session_id,signal_value FROM candidate_signals WHERE signal_kind='cookie' AND session_id!='') + GROUP BY account_id, source_service, session_id, signal_value +), +matches AS +( + SELECT m.account_id AS account_id, m.context_key AS context_key, + if(m.signal_kind = 'cookie' AND s.users > 0, s.unique_user, b.unique_user) AS candidate_user, + b.confidence AS confidence + FROM candidate_signals AS m + LEFT JOIN bindings AS b ON b.account_id=m.account_id AND b.signal_kind=m.signal_kind AND b.signal_value=m.signal_value + LEFT JOIN session_users AS s ON m.signal_kind='cookie' AND s.account_id=m.account_id + AND s.source_service=m.source_service AND s.session_id=m.session_id AND s.signal_value=m.signal_value + WHERE if(m.signal_kind='cookie' AND s.users > 0, s.users, b.users) = 1 AND b.confidence >= 0.50 +), +best AS +( + SELECT account_id, context_key, groupArray(tuple(candidate_user,confidence)) AS hits, + arrayMax(arrayMap(h -> h.2,hits)) AS winning_confidence, + arrayDistinct(arrayMap(h -> h.1,arrayFilter(h -> h.2=winning_confidence,hits))) AS winners, + if(length(winners)=1,winners[1],'') AS stitched_user + FROM matches GROUP BY account_id, context_key +), +subjects AS +( + SELECT e.* EXCEPT(account_id), e.account_id AS account_id, + multiIf(e.user_id!='','user',m.stitched_user!='','user',e.anonymous_id!='','anonymous','') AS subject_kind, + multiIf(e.user_id!='',e.user_id,m.stitched_user!='',m.stitched_user,e.anonymous_id) AS subject_id + FROM candidates AS e + LEFT JOIN best AS m ON m.account_id=e.account_id AND m.context_key=tuple(e.source_service,e.session_id,e.anonymous_id,e.identifiers) +) +SELECT *, if(subject_kind='', '', concat(subject_kind, ':', subject_id)) AS canonical_user +FROM subjects +SETTINGS enable_materialized_cte=1; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-40.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-40.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-40.sql @@ -0,0 +1,6 @@ + + +CREATE OR REPLACE VIEW analytics.events_resolved_v3 AS +SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json +FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='', + scope_from=toDateTime64('1970-01-01 00:00:00',3,'UTC'),scope_to=toDateTime64('2100-01-01 00:00:00',3,'UTC')); diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-41.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-41.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-41.sql @@ -0,0 +1,2 @@ + +CREATE OR REPLACE VIEW analytics.events_resolved AS SELECT * FROM analytics.events_resolved_v3; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-42.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-42.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-42.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_dau; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-43.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-43.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-43.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.dau_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-44.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-44.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-44.sql @@ -0,0 +1,2 @@ + +CREATE TABLE analytics.dau_rebuild AS analytics.dau; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-45.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-45.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-45.sql @@ -0,0 +1,5 @@ + +INSERT INTO analytics.dau_rebuild (account_id,day,users) +SELECT account_id, toDate(occurred_at) AS day, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_canonical WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-46.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-46.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-46.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.dau AND analytics.dau_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-47.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-47.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-47.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.dau_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-48.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-48.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-48.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_dau TO analytics.dau AS +SELECT account_id, toDate(occurred_at) AS day, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_raw WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-49.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-49.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-49.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_wau; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-50.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-50.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-50.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.wau_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-51.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-51.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-51.sql @@ -0,0 +1,2 @@ + +CREATE TABLE analytics.wau_rebuild AS analytics.wau; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-52.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-52.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-52.sql @@ -0,0 +1,5 @@ + +INSERT INTO analytics.wau_rebuild (account_id,week,users) +SELECT account_id, toMonday(occurred_at) AS week, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_canonical WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-53.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-53.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-53.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.wau AND analytics.wau_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-54.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-54.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-54.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.wau_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-55.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-55.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-55.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_wau TO analytics.wau AS +SELECT account_id, toMonday(occurred_at) AS week, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_raw WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-56.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-56.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-56.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_experiment_exposures; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-57.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-57.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-57.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.experiment_exposures_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-58.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-58.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-58.sql @@ -0,0 +1,2 @@ + +CREATE TABLE analytics.experiment_exposures_rebuild AS analytics.experiment_exposures; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-59.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-59.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-59.sql @@ -0,0 +1,5 @@ + +INSERT INTO analytics.experiment_exposures_rebuild (account_id,experiment,variant,day,users) +SELECT account_id, toDate(occurred_at) AS day, CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_canonical WHERE event_name = 'experiment_viewed' AND (user_id != '' OR anonymous_id != '') AND match(experiment, '^[a-z0-9_.:-]{1,128}$') AND match(variant, '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-60.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-60.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-60.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.experiment_exposures AND analytics.experiment_exposures_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-61.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-61.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-61.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.experiment_exposures_rebuild; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-62.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-62.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-62.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_experiment_exposures TO analytics.experiment_exposures AS +SELECT account_id, toDate(occurred_at) AS day, CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM analytics.events_raw WHERE event_name = 'experiment_viewed' AND (user_id != '' OR anonymous_id != '') AND match(experiment, '^[a-z0-9_.:-]{1,128}$') AND match(variant, '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-63.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-63.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-63.sql @@ -0,0 +1,35 @@ + + +-- Exact context snapshots. The connection refresh deadline is the earliest +-- future fingerprint expiry in its evidence. Parameters defer temporary binding. +CREATE OR REPLACE VIEW analytics.identity_context_resolution AS +WITH + (a,k,v) -> arrayDistinct(arrayMap(p -> p.1, + arrayFilter(p -> k!='fingerprint' OR p.2>=now64(3)-INTERVAL 180 DAY, + joinGet('kansoku_signal_cache','observations',a,k,v)))) AS global_users, + (a,svc,sid,v) -> joinGet('kansoku_session_cache','users',a,svc,sid,v) AS session_users, + k -> multiIf(k IN ('email_hmac','email_sha256'),0.95,k='cookie',0.90,k='device',0.80,k='fingerprint',0.50,0.0) AS confidence, + (a,svc,sid,k,v) -> if(k='cookie' AND sid!='' AND notEmpty(session_users(a,svc,sid,v)), + session_users(a,svc,sid,v),global_users(a,k,v)) AS bound_users +SELECT * EXCEPT(targets,best_confidence,winners),if(length(winners)=1,winners[1],'') AS stitched_user +FROM +( + SELECT *,arrayMax(arrayMap(h -> h.2,targets)) AS best_confidence, + arrayDistinct(arrayMap(h -> h.1[1],arrayFilter(h -> h.2=best_confidence,targets))) AS winners + FROM + ( + SELECT *,arrayFilter(h -> length(h.1)=1 AND h.2>=0.50, + arrayMap(sig -> tuple(bound_users(account_id,source_service,session_id,sig.1,sig.2),confidence(sig.1)), + arrayDistinct(arrayFilter(sig -> sig.2!='',arrayConcat([('cookie',anonymous_id)], + JSONExtractKeysAndValues(identifiers,'String')))))) AS targets + FROM + ( + SELECT DISTINCT account_id,context_key.1 AS source_service,context_key.2 AS session_id, + context_key.3 AS anonymous_id,context_key.4 AS identifiers + FROM analytics.identity_context_signals FINAL + WHERE ({scope_all_accounts:UInt8}=1 OR account_id={scope_account:String}) + AND day>={scope_from:Date} AND day<{scope_to:Date} + + ) + ) +); diff --git a/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-64.sql b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-64.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0018_conservative_identity_resolution/statement-64.sql @@ -0,0 +1,14 @@ + + +-- HTTP-only relation. The physical query connection refreshes this exact +-- snapshot on index revisions and before its earliest fingerprint expiry. +CREATE OR REPLACE VIEW analytics.events_resolved_http AS +SELECT *,canonical_user, + if(startsWith(canonical_user,'user:'),'user',if(startsWith(canonical_user,'anonymous:'),'anonymous','')) AS subject_kind, + substring(canonical_user,length(subject_kind)+2) AS subject_id +FROM analytics.events_canonical +ARRAY JOIN [if(user_id!='',concat('user:',user_id), + joinGet('kansoku_context_cache','canonical_user',identity_context_key))] AS canonical_user +WHERE ({scope_all_accounts:UInt8}=1 OR account_id={scope_account:String}) + AND throwIf(joinGet('kansoku_identity_clock','deadline',toUInt8(0))<=toUnixTimestamp64Milli(now64(3)), 'identity snapshot expired')=0 + AND occurred_at>={scope_from:DateTime64(3,'UTC')} AND occurred_at<{scope_to:DateTime64(3,'UTC')}; diff --git a/test/fixtures/kansoku/steps/0019_web_enrichment/statement-01.sql b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-01.sql @@ -0,0 +1,13 @@ +-- EP-33: additive private enrichment; no producer-chosen categorical columns. +-- Existing development facts retain unknown/version-zero enrichment. Reclassification +-- is an explicit rebuild operation, never an implicit change to accepted events. +ALTER TABLE analytics.events_raw + ADD COLUMN IF NOT EXISTS browser_family LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS os_family LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS device_class LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS is_bot Bool DEFAULT false, + ADD COLUMN IF NOT EXISTS channel LowCardinality(String) DEFAULT 'unknown', + ADD COLUMN IF NOT EXISTS country_code LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS enrichment_version UInt16 DEFAULT 0, + ADD COLUMN IF NOT EXISTS context_json JSON(max_dynamic_paths=256) + MATERIALIZED CAST(if(isValidJSON(context),context,'{}') AS JSON(max_dynamic_paths=256)); diff --git a/test/fixtures/kansoku/steps/0019_web_enrichment/statement-02.sql b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-02.sql @@ -0,0 +1,4 @@ + + +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *,properties_json,identity_context_key,context_json FROM analytics.events_raw FINAL; diff --git a/test/fixtures/kansoku/steps/0019_web_enrichment/statement-03.sql b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-03.sql @@ -0,0 +1,73 @@ + + +-- Refresh view headers after adding raw columns; keep the public seven-column shape. +CREATE OR REPLACE VIEW analytics.events_resolved_internal AS +WITH +candidates AS +( + SELECT * FROM analytics.events_canonical + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND occurred_at >= {scope_from:DateTime64(3, 'UTC')} + AND occurred_at < {scope_to:DateTime64(3, 'UTC')} +), +candidate_signals AS MATERIALIZED +( + SELECT DISTINCT account_id, context_key, context_key.1 AS source_service, context_key.2 AS session_id, + signal_kind, signal_value + FROM analytics.identity_context_signals FINAL + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND day >= toDate({scope_from:DateTime64(3, 'UTC')}) + AND toDateTime64(day,3,'UTC') < {scope_to:DateTime64(3, 'UTC')} +), +bindings AS +( + SELECT account_id, signal_kind, signal_value, uniqExact(user_id) AS users, min(user_id) AS unique_user, + multiIf(signal_kind IN ('email_hmac','email_sha256'), 0.95, + signal_kind = 'cookie', 0.90, signal_kind = 'device', 0.80, + signal_kind = 'fingerprint', 0.50, 0.0) AS confidence + FROM analytics.identity_signal_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,signal_kind,signal_value) IN (SELECT account_id,signal_kind,signal_value FROM candidate_signals) + AND (signal_kind != 'fingerprint' OR observed_at >= now64(3) - INTERVAL 180 DAY) + GROUP BY account_id, signal_kind, signal_value +), +session_users AS +( + SELECT account_id, source_service, session_id, signal_value, uniqExact(user_id) AS users, + min(user_id) AS unique_user + FROM analytics.identity_session_users + WHERE ({scope_all_accounts:UInt8} = 1 OR account_id = {scope_account:String}) + AND (account_id,source_service,session_id,signal_value) IN + (SELECT account_id,source_service,session_id,signal_value FROM candidate_signals WHERE signal_kind='cookie' AND session_id!='') + GROUP BY account_id, source_service, session_id, signal_value +), +matches AS +( + SELECT m.account_id AS account_id, m.context_key AS context_key, + if(m.signal_kind = 'cookie' AND s.users > 0, s.unique_user, b.unique_user) AS candidate_user, + b.confidence AS confidence + FROM candidate_signals AS m + LEFT JOIN bindings AS b ON b.account_id=m.account_id AND b.signal_kind=m.signal_kind AND b.signal_value=m.signal_value + LEFT JOIN session_users AS s ON m.signal_kind='cookie' AND s.account_id=m.account_id + AND s.source_service=m.source_service AND s.session_id=m.session_id AND s.signal_value=m.signal_value + WHERE if(m.signal_kind='cookie' AND s.users > 0, s.users, b.users) = 1 AND b.confidence >= 0.50 +), +best AS +( + SELECT account_id, context_key, groupArray(tuple(candidate_user,confidence)) AS hits, + arrayMax(arrayMap(h -> h.2,hits)) AS winning_confidence, + arrayDistinct(arrayMap(h -> h.1,arrayFilter(h -> h.2=winning_confidence,hits))) AS winners, + if(length(winners)=1,winners[1],'') AS stitched_user + FROM matches GROUP BY account_id, context_key +), +subjects AS +( + SELECT e.* EXCEPT(account_id), e.account_id AS account_id, + multiIf(e.user_id!='','user',m.stitched_user!='','user',e.anonymous_id!='','anonymous','') AS subject_kind, + multiIf(e.user_id!='',e.user_id,m.stitched_user!='',m.stitched_user,e.anonymous_id) AS subject_id + FROM candidates AS e + LEFT JOIN best AS m ON m.account_id=e.account_id AND m.context_key=tuple(e.source_service,e.session_id,e.anonymous_id,e.identifiers) +) +SELECT *, if(subject_kind='', '', concat(subject_kind, ':', subject_id)) AS canonical_user +FROM subjects +SETTINGS enable_materialized_cte=1; diff --git a/test/fixtures/kansoku/steps/0019_web_enrichment/statement-04.sql b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-04.sql @@ -0,0 +1,6 @@ + + +CREATE OR REPLACE VIEW analytics.events_resolved_v3 AS +SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json +FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='', + scope_from=toDateTime64('1970-01-01 00:00:00',3,'UTC'),scope_to=toDateTime64('2100-01-01 00:00:00',3,'UTC')); diff --git a/test/fixtures/kansoku/steps/0019_web_enrichment/statement-05.sql b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-05.sql @@ -0,0 +1,2 @@ + +CREATE OR REPLACE VIEW analytics.events_resolved AS SELECT * FROM analytics.events_resolved_v3; diff --git a/test/fixtures/kansoku/steps/0019_web_enrichment/statement-06.sql b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0019_web_enrichment/statement-06.sql @@ -0,0 +1,13 @@ + + + +CREATE OR REPLACE VIEW analytics.events_resolved_http AS +SELECT *,canonical_user, + if(startsWith(canonical_user,'user:'),'user',if(startsWith(canonical_user,'anonymous:'),'anonymous','')) AS subject_kind, + substring(canonical_user,length(subject_kind)+2) AS subject_id +FROM analytics.events_canonical +ARRAY JOIN [if(user_id!='',concat('user:',user_id), + joinGet('kansoku_context_cache','canonical_user',identity_context_key))] AS canonical_user +WHERE ({scope_all_accounts:UInt8}=1 OR account_id={scope_account:String}) + AND throwIf(joinGet('kansoku_identity_clock','deadline',toUInt8(0))<=toUnixTimestamp64Milli(now64(3)), 'identity snapshot expired')=0 + AND occurred_at>={scope_from:DateTime64(3,'UTC')} AND occurred_at<{scope_to:DateTime64(3,'UTC')}; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-01.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-01.sql @@ -0,0 +1,4 @@ +-- EP-34: merge whole retained sessions, never per-block start-day groups. +-- The operator keeps writers stopped during migrations/rebuilds. Every state +-- is idempotent for immutable event IDs: sets, min/max and deterministic argmin/max. +DROP TABLE IF EXISTS analytics.mv_sessions; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-02.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-02.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS analytics.sessions_rebuild; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-03.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-03.sql @@ -0,0 +1,17 @@ + +CREATE TABLE analytics.sessions_rebuild +ENGINE = AggregatingMergeTree +ORDER BY (account_id,source_service,session_id) +AS +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM analytics.events_canonical +WHERE source='web' AND session_id!='' AND 0 +GROUP BY account_id,source_service,session_id; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-04.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-04.sql @@ -0,0 +1,15 @@ + + +INSERT INTO analytics.sessions_rebuild +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM analytics.events_canonical +WHERE source='web' AND session_id!='' +GROUP BY account_id,source_service,session_id; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-05.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-05.sql @@ -0,0 +1,3 @@ + + +CREATE TABLE IF NOT EXISTS analytics.sessions AS analytics.sessions_rebuild; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-06.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-06.sql @@ -0,0 +1,2 @@ + +EXCHANGE TABLES analytics.sessions AND analytics.sessions_rebuild; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-07.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-07.sql @@ -0,0 +1,2 @@ + +DROP TABLE analytics.sessions_rebuild; diff --git a/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-08.sql b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0020_replay_safe_sessions/statement-08.sql @@ -0,0 +1,15 @@ + + +CREATE MATERIALIZED VIEW analytics.mv_sessions TO analytics.sessions AS +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM analytics.events_raw +WHERE source='web' AND session_id!='' +GROUP BY account_id,source_service,session_id; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-01.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-01.sql @@ -0,0 +1,6 @@ +-- EP-40: governed current traits. Keep ingest/writers fenced during DDL/rebuilds. +-- Identify v1 is identity-only. Eligibility mirrors the shared registry and +-- checks actual stored schema_version and the immutable server validation marker. +-- Exact decimal integrality is validated before Float64/native JSON conversion. +-- Strings are stored decoded; numeric values use the metrics Float64 contract. +ALTER TABLE analytics.events_raw ADD COLUMN IF NOT EXISTS governed_traits_valid Bool DEFAULT false; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-02.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-02.sql @@ -0,0 +1,4 @@ + + +CREATE OR REPLACE VIEW analytics.events_canonical +AS SELECT *,properties_json,identity_context_key,context_json FROM analytics.events_raw FINAL; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-03.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-03.sql @@ -0,0 +1,5 @@ + + +CREATE TABLE IF NOT EXISTS analytics.trait_event_erasures +(account_id String,event_id String) +ENGINE=ReplacingMergeTree ORDER BY (account_id,event_id); diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-04.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-04.sql @@ -0,0 +1,6 @@ + + +CREATE TABLE IF NOT EXISTS analytics.user_properties +(account_id String,user_id String,name String, + value AggregateFunction(argMax,Tuple(String,String,Float64),Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,user_id,name); diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-05.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-05.sql @@ -0,0 +1,6 @@ + + +CREATE TABLE IF NOT EXISTS analytics.user_properties_once +(account_id String,user_id String,name String, + value AggregateFunction(argMin,Tuple(String,String,Float64),Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,user_id,name); diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-06.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-06.sql @@ -0,0 +1,19 @@ + + +CREATE OR REPLACE VIEW analytics.user_trait_events AS +WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_canonical +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1); diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-07.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-07.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_user_properties; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-08.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-08.sql @@ -0,0 +1,19 @@ + +CREATE MATERIALIZED VIEW analytics.mv_user_properties TO analytics.user_properties AS +SELECT account_id,user_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_raw +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-09.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-09.sql @@ -0,0 +1,5 @@ + + +INSERT INTO analytics.user_properties +SELECT account_id,user_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (SELECT * FROM analytics.user_trait_events) WHERE event_name='identify' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-10.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-10.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_user_properties_once; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-11.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-11.sql @@ -0,0 +1,19 @@ + +CREATE MATERIALIZED VIEW analytics.mv_user_properties_once TO analytics.user_properties_once AS +SELECT account_id,user_id,name,argMinState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_raw +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify_once' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-12.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-12.sql @@ -0,0 +1,5 @@ + + +INSERT INTO analytics.user_properties_once +SELECT account_id,user_id,name,argMinState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (SELECT * FROM analytics.user_trait_events) WHERE event_name='identify_once' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-13.sql b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0021_governed_current_user_traits/statement-13.sql @@ -0,0 +1,14 @@ + + +CREATE OR REPLACE VIEW analytics.user_traits AS +SELECT account_id,user_id,name,selected.1 AS trait_type,selected.2 AS text_value,selected.3 AS number_value +FROM ( + SELECT account_id,user_id,name,argMax(candidate,priority) AS selected + FROM ( + SELECT account_id,user_id,name,argMaxMerge(value) AS candidate,toUInt8(1) AS priority + FROM analytics.user_properties GROUP BY account_id,user_id,name + UNION ALL + SELECT account_id,user_id,name,argMinMerge(value) AS candidate,toUInt8(0) AS priority + FROM analytics.user_properties_once GROUP BY account_id,user_id,name + ) GROUP BY account_id,user_id,name +); diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-01.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-01.sql @@ -0,0 +1,9 @@ +-- Applied cutoff advances only through fenced kansoku-admin apply-retention. +-- Physical TTL cleanup is separate from the immediate canonical predicate. +CREATE TABLE IF NOT EXISTS analytics.retention_policy +( + singleton UInt8, + revision UInt64, + cutoff DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(revision) ORDER BY singleton; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-02.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-02.sql @@ -0,0 +1,2 @@ + +INSERT INTO analytics.retention_policy VALUES (1,0,'1970-01-01 00:00:00'); diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-03.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-03.sql @@ -0,0 +1,5 @@ + +CREATE OR REPLACE VIEW analytics.events_canonical AS +SELECT *, properties_json, identity_context_key, context_json +FROM analytics.events_raw FINAL +WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy); diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-04.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-04.sql @@ -0,0 +1,6 @@ + + +-- Every raw-triggered projection filters the applied cutoff before updating state. +-- Keep the raw table inside the subquery so native insert-block triggers still fire. + +DROP TABLE IF EXISTS analytics.mv_daily_event_counts; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-05.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-05.sql @@ -0,0 +1,6 @@ + +CREATE MATERIALIZED VIEW analytics.mv_daily_event_counts TO analytics.daily_event_counts AS +SELECT account_id, toDate(occurred_at) AS day, event_name, + uniqExactState(event_id) AS events +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +GROUP BY account_id, day, event_name; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-06.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-06.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_page_views_by_route; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-07.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-07.sql @@ -0,0 +1,7 @@ + +CREATE MATERIALIZED VIEW analytics.mv_page_views_by_route TO analytics.page_views_by_route AS +SELECT account_id, toDate(occurred_at) AS day, path(url) AS route, + uniqExactState(event_id) AS views +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +WHERE event_name = 'page_view' +GROUP BY account_id, day, route; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-08.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-08.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_identity_signals; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-09.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-09.sql @@ -0,0 +1,9 @@ + +CREATE MATERIALIZED VIEW analytics.mv_identity_signals TO analytics.identity_signals AS +SELECT account_id, sig.1 AS signal_kind, sig.2 AS signal_value, user_id, + occurred_at AS observed_at, event_id, received_at, source_service, session_id +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.1 != '' AND kv.2 != '', + arrayConcat([(if(event_name IN ('identify','identify_once','alias'), 'cookie', ''), anonymous_id)], + JSONExtractKeysAndValues(identifiers, 'String')))) AS sig +WHERE user_id != ''; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-10.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-10.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_identity_context_signals; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-11.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-11.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-11.sql @@ -0,0 +1,8 @@ + +CREATE MATERIALIZED VIEW analytics.mv_identity_context_signals TO analytics.identity_context_signals AS +SELECT account_id,toDate(occurred_at) AS day,tuple(source_service,session_id,anonymous_id,identifiers) AS context_key, + sig.1 AS signal_kind,sig.2 AS signal_value +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN arrayDistinct(arrayFilter(kv -> kv.2!='', + arrayConcat([('cookie',anonymous_id)],JSONExtractKeysAndValues(identifiers,'String')))) AS sig +WHERE user_id=''; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-12.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-12.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-12.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_dau; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-13.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-13.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-13.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_dau TO analytics.dau AS +SELECT account_id, toDate(occurred_at) AS day, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, day; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-14.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-14.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-14.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_wau; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-15.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-15.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-15.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_wau TO analytics.wau AS +SELECT account_id, toMonday(occurred_at) AS week, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) WHERE user_id != '' OR anonymous_id != '' +GROUP BY account_id, week; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-16.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-16.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-16.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_experiment_exposures; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-17.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-17.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-17.sql @@ -0,0 +1,5 @@ + +CREATE MATERIALIZED VIEW analytics.mv_experiment_exposures TO analytics.experiment_exposures AS +SELECT account_id, toDate(occurred_at) AS day, CAST(toString(getSubcolumn(properties_json, 'experiment')), 'String') AS experiment, CAST(toString(getSubcolumn(properties_json, 'variant')), 'String') AS variant, uniqCombinedState(if(user_id!='',concat('user:',user_id),concat('anonymous:',anonymous_id))) AS users +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) WHERE event_name = 'experiment_viewed' AND (user_id != '' OR anonymous_id != '') AND match(experiment, '^[a-z0-9_.:-]{1,128}$') AND match(variant, '^[a-z0-9_.:-]{1,128}$') +GROUP BY account_id, experiment, variant, day; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-18.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-18.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-18.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_sessions; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-19.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-19.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-19.sql @@ -0,0 +1,14 @@ + +CREATE MATERIALIZED VIEW analytics.mv_sessions TO analytics.sessions AS +SELECT account_id,source_service,session_id, + minState(occurred_at) AS started, + maxState(occurred_at) AS ended, + uniqExactIfState(event_id,event_name='page_view') AS page_ids, + uniqExactIfState(event_id,event_name NOT IN ('page_view','session_start','page_leave','identify','identify_once','alias','group','experiment_viewed')) AS engagement_ids, + argMinState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id)) AS first_event, + argMinIfState(tuple(toString(event_name),toString(source),toString(url),toString(referrer),toString(utm_source),toString(utm_medium),toString(utm_campaign),toString(utm_content),toString(utm_term),toString(browser_family),toString(os_family),toString(device_class),toString(channel),toString(country_code),toString(properties)),tuple(occurred_at,event_id),event_name='page_view') AS first_page, + argMaxIfState(tuple(url),tuple(occurred_at,event_id),event_name='page_view') AS last_page, + maxState(is_bot) AS bot +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +WHERE source='web' AND session_id!='' +GROUP BY account_id,source_service,session_id; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-20.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-20.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-20.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_user_properties; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-21.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-21.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-21.sql @@ -0,0 +1,19 @@ + +CREATE MATERIALIZED VIEW analytics.mv_user_properties TO analytics.user_properties AS +SELECT account_id,user_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-22.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-22.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-22.sql @@ -0,0 +1,3 @@ + + +DROP TABLE IF EXISTS analytics.mv_user_properties_once; diff --git a/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-23.sql b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-23.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0022_applied_retention_policy/statement-23.sql @@ -0,0 +1,19 @@ + +CREATE MATERIALIZED VIEW analytics.mv_user_properties_once TO analytics.user_properties_once AS +SELECT account_id,user_id,name,argMinState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH + toFloat64OrNull(toString(getSubcolumn(properties_json,'company_size'))) AS company_number +SELECT account_id,user_id,event_id,event_name,occurred_at,received_at, + trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM (SELECT *, properties_json, identity_context_key, context_json FROM analytics.events_raw WHERE occurred_at >= (SELECT argMax(cutoff,revision) FROM analytics.retention_policy)) +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('signup_source','string',ifNull(dynamicElement(getSubcolumn(properties_json,'signup_source'),'String'),''),toFloat64(0)),('initial_plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'initial_plan'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('role','string',ifNull(dynamicElement(getSubcolumn(properties_json,'role'),'String'),''),toFloat64(0)),('company_size','number','',ifNull(company_number,toFloat64(0)))] AS trait +WHERE governed_traits_valid AND user_id!='' + AND ((event_name='identify' AND schema_version=2) OR (event_name='identify_once' AND schema_version=1)) + AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) + AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') + AND (NOT JSONHas(properties,'signup_source') OR dynamicType(getSubcolumn(properties_json,'signup_source'))='String') + AND (NOT JSONHas(properties,'initial_plan') OR dynamicType(getSubcolumn(properties_json,'initial_plan'))='String') + AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') + AND (NOT JSONHas(properties,'role') OR dynamicType(getSubcolumn(properties_json,'role'))='String') + AND (NOT JSONHas(properties,'company_size') OR (dynamicType(getSubcolumn(properties_json,'company_size')) IN ('Int64','UInt64','Float64') AND isFinite(company_number) AND company_number>=0 AND floor(company_number)=company_number)) + AND JSONHas(properties,trait.1)) WHERE event_name='identify_once' GROUP BY account_id,user_id,name; diff --git a/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-01.sql b/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-01.sql @@ -0,0 +1,3 @@ +-- The seven-column contract remains frozen; SELECT rights on these deliberate +-- projections do not require rights on the changing identity/storage relations. +ALTER TABLE analytics.events_resolved_v3 MODIFY SQL SECURITY DEFINER DEFINER = CURRENT_USER; diff --git a/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-02.sql b/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-02.sql @@ -0,0 +1,2 @@ + +ALTER TABLE analytics.events_resolved MODIFY SQL SECURITY DEFINER DEFINER = CURRENT_USER; diff --git a/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-03.sql b/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0023_stable_reader_surface/statement-03.sql @@ -0,0 +1,13 @@ + + +-- Explicit bounded candidate parameters preserve conservative current identity +-- semantics without the HTTP connection's private Join-table cache. +CREATE OR REPLACE VIEW analytics.events_analytics_v1 +DEFINER = CURRENT_USER SQL SECURITY DEFINER +AS SELECT + event_id, event_name, occurred_at, url, account_id, canonical_user, properties_json, + source, source_service, session_id, is_bot, browser_family, os_family, + device_class, channel, country_code +FROM analytics.events_resolved_internal( + scope_all_accounts=0, scope_account={account:String}, + scope_from={from_time:DateTime64(3, 'UTC')}, scope_to={to_time:DateTime64(3, 'UTC')}); diff --git a/test/fixtures/kansoku/steps/0024_cohort_members/statement-01.sql b/test/fixtures/kansoku/steps/0024_cohort_members/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0024_cohort_members/statement-01.sql @@ -0,0 +1,10 @@ +CREATE TABLE analytics.cohort_members +( + account_id String, + cohort_id String, + user_id String, + version DateTime64(3, 'UTC'), + present UInt8 +) +ENGINE = ReplacingMergeTree(version) +ORDER BY (account_id, cohort_id, user_id); diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-01.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-01.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-01.sql @@ -0,0 +1,10 @@ +-- EP-45: append the organization without changing the first seven columns. +-- Rollback (after reverting consumers): +-- CREATE OR REPLACE VIEW analytics.events_resolved_v3 DEFINER = CURRENT_USER SQL SECURITY DEFINER AS +-- SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json +-- FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='',scope_from=toDateTime64('1970-01-01',3,'UTC'),scope_to=toDateTime64('2100-01-01',3,'UTC')); +-- CREATE OR REPLACE VIEW analytics.events_resolved DEFINER = CURRENT_USER SQL SECURITY DEFINER AS SELECT * FROM analytics.events_resolved_v3; +CREATE OR REPLACE VIEW analytics.events_resolved_v3 +DEFINER = CURRENT_USER SQL SECURITY DEFINER AS +SELECT event_id,event_name,occurred_at,url,account_id,canonical_user,properties_json,organization_id +FROM analytics.events_resolved_internal(scope_all_accounts=1,scope_account='',scope_from=toDateTime64('1970-01-01',3,'UTC'),scope_to=toDateTime64('2100-01-01',3,'UTC')); diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-02.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-02.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-02.sql @@ -0,0 +1,3 @@ + +CREATE OR REPLACE VIEW analytics.events_resolved +DEFINER = CURRENT_USER SQL SECURITY DEFINER AS SELECT * FROM analytics.events_resolved_v3; diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-03.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-03.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-03.sql @@ -0,0 +1,6 @@ + + +CREATE TABLE IF NOT EXISTS analytics.organization_properties +(account_id String,organization_id String,name String, + value AggregateFunction(argMax,Tuple(String,String,Float64),Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,organization_id,name); diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-04.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-04.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-04.sql @@ -0,0 +1,14 @@ + +CREATE OR REPLACE VIEW analytics.organization_trait_events AS +WITH toFloat64OrNull(toString(getSubcolumn(properties_json,'seats'))) AS seat_number +SELECT account_id,organization_id,event_id,occurred_at,received_at, +trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_canonical +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('industry','string',ifNull(dynamicElement(getSubcolumn(properties_json,'industry'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('seats','number','',ifNull(seat_number,toFloat64(0)))] AS trait +WHERE event_name='group' AND schema_version=2 AND organization_id!='' AND governed_traits_valid +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') +AND (NOT JSONHas(properties,'industry') OR dynamicType(getSubcolumn(properties_json,'industry'))='String') +AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') +AND (NOT JSONHas(properties,'seats') OR (dynamicType(getSubcolumn(properties_json,'seats')) IN ('Int64','UInt64','Float64') AND isFinite(seat_number) AND seat_number>=0 AND floor(seat_number)=seat_number)) +AND JSONHas(properties,trait.1); diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-05.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-05.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-05.sql @@ -0,0 +1,15 @@ + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_organization_properties TO analytics.organization_properties AS +SELECT account_id,organization_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (WITH toFloat64OrNull(toString(getSubcolumn(properties_json,'seats'))) AS seat_number +SELECT account_id,organization_id,event_id,occurred_at,received_at, +trait.1 AS name,trait.2 AS trait_type,trait.3 AS text_value,trait.4 AS number_value +FROM analytics.events_raw +ARRAY JOIN [('plan','string',ifNull(dynamicElement(getSubcolumn(properties_json,'plan'),'String'),''),toFloat64(0)),('industry','string',ifNull(dynamicElement(getSubcolumn(properties_json,'industry'),'String'),''),toFloat64(0)),('country','string',ifNull(dynamicElement(getSubcolumn(properties_json,'country'),'String'),''),toFloat64(0)),('seats','number','',ifNull(seat_number,toFloat64(0)))] AS trait +WHERE event_name='group' AND schema_version=2 AND organization_id!='' AND governed_traits_valid +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +AND (NOT JSONHas(properties,'plan') OR dynamicType(getSubcolumn(properties_json,'plan'))='String') +AND (NOT JSONHas(properties,'industry') OR dynamicType(getSubcolumn(properties_json,'industry'))='String') +AND (NOT JSONHas(properties,'country') OR dynamicType(getSubcolumn(properties_json,'country'))='String') +AND (NOT JSONHas(properties,'seats') OR (dynamicType(getSubcolumn(properties_json,'seats')) IN ('Int64','UInt64','Float64') AND isFinite(seat_number) AND seat_number>=0 AND floor(seat_number)=seat_number)) +AND JSONHas(properties,trait.1)) GROUP BY account_id,organization_id,name; diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-06.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-06.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-06.sql @@ -0,0 +1,4 @@ + +INSERT INTO analytics.organization_properties +SELECT account_id,organization_id,name,argMaxState(tuple(trait_type,text_value,number_value),tuple(occurred_at,received_at,event_id)) AS value +FROM (SELECT * FROM analytics.organization_trait_events) GROUP BY account_id,organization_id,name; diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-07.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-07.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-07.sql @@ -0,0 +1,4 @@ + +CREATE OR REPLACE VIEW analytics.organization_traits AS +SELECT account_id,organization_id,name,selected.1 AS trait_type,selected.2 AS text_value,selected.3 AS number_value +FROM (SELECT account_id,organization_id,name,argMaxMerge(value) AS selected FROM analytics.organization_properties GROUP BY account_id,organization_id,name); diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-08.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-08.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-08.sql @@ -0,0 +1,6 @@ + + +CREATE TABLE IF NOT EXISTS analytics.user_organizations +(account_id String,user_id String, + organization_id AggregateFunction(argMax,String,Tuple(DateTime64(3,'UTC'),DateTime64(3,'UTC'),String))) +ENGINE=AggregatingMergeTree ORDER BY (account_id,user_id); diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-09.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-09.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-09.sql @@ -0,0 +1,7 @@ + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.mv_user_organizations TO analytics.user_organizations AS +SELECT account_id,user_id,argMaxState(r.organization_id,tuple(occurred_at,received_at,event_id)) AS organization_id +FROM analytics.events_raw AS r +WHERE event_name='group' AND r.organization_id!='' AND user_id!='' +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +GROUP BY account_id,user_id; diff --git a/test/fixtures/kansoku/steps/0025_organization_analytics/statement-10.sql b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-10.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/kansoku/steps/0025_organization_analytics/statement-10.sql @@ -0,0 +1,7 @@ + +INSERT INTO analytics.user_organizations +SELECT account_id,user_id,argMaxState(r.organization_id,tuple(occurred_at,received_at,event_id)) AS organization_id +FROM analytics.events_canonical AS r +WHERE event_name='group' AND r.organization_id!='' AND user_id!='' +AND (account_id,event_id) NOT IN (SELECT account_id,event_id FROM analytics.trait_event_erasures) +GROUP BY account_id,user_id; diff --git a/test/scripts/native_fixture.py b/test/scripts/native_fixture.py new file mode 100644 --- /dev/null +++ b/test/scripts/native_fixture.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""Owned native server and fault proxies, controlled over private stdin/stdout RPC.""" +import argparse +import base64 +import hashlib +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import shutil +import signal +import socket +import ssl +import subprocess +import sys +import tempfile +import threading +import time +import urllib.request +import urllib.parse +from xml.sax.saxutils import escape + +PASSWORD = 'test-password' +AUTH = 'Basic ' + base64.b64encode(('default:' + PASSWORD).encode()).decode() + + +def port(): + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + return listener.getsockname()[1] + + +def binary_for(entry): + if sys.platform != 'darwin' or os.uname().machine != 'arm64': + raise RuntimeError('Candidate matrix requires native Apple Silicon; Linux execution is deferred') + cache = Path.home() / 'Library/Caches/clickhouse-migrate' / entry['version'] + cache.mkdir(parents=True, exist_ok=True) + binary = cache / 'clickhouse' + def digest(path): + with path.open('rb') as source: + return hashlib.file_digest(source, 'sha256').hexdigest() + if binary.exists() and digest(binary) == entry['sha256']: + return binary + with tempfile.NamedTemporaryFile(prefix='download-', dir=cache, delete=False) as destination: + candidate = Path(destination.name) + try: + with urllib.request.urlopen(entry['url'], timeout=60) as response: + shutil.copyfileobj(response, destination) + destination.flush() + if digest(candidate) != entry['sha256']: + raise RuntimeError('Server binary digest mismatch') + candidate.chmod(0o755) + candidate.replace(binary) + finally: + candidate.unlink(missing_ok=True) + return binary + + +class NativeServer: + def __init__(self, entry): + self.entry = entry + self.root = Path(tempfile.mkdtemp(prefix='clickhouse-migrate-server-', dir='/tmp')) + self.port = port() + self.tcp_port = port() + self.binary = binary_for(entry) + self.process = None + self.log = (self.root / 'console.log').open('w') + self.proxies = [] + self.keep = True + for directory in ['data', 'tmp', 'user_files', 'format_schemas']: + (self.root / directory).mkdir() + users = self.root / 'users.xml' + users.write_text('''210100000001break1 +test-password127.0.0.1 +defaultdefault1 +360000 +000''') + root = escape(str(self.root)) + (self.root / 'config.xml').write_text(f''' +warning{root}/server.log{root}/error.log +{root}/data/{root}/tmp/{root}/user_files/ +{root}/format_schemas/{root}/users.xml +127.0.0.1{self.port}{self.tcp_port} +64128 +82 +2 +2 +162 +22 +00 +''') + try: + self.process = subprocess.Popen([str(self.binary), 'server', '--config-file', str(self.root / 'config.xml')], stdout=self.log, stderr=subprocess.STDOUT, start_new_session=True) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if self.process.poll() is not None: + raise RuntimeError(f'Owned server exited before readiness: {self.process.returncode}') + try: + with urllib.request.urlopen(self.endpoint + '/ping', timeout=.2) as response: + if response.read() == b'Ok.\n': + break + except OSError: + pass + time.sleep(.05) + else: + raise RuntimeError('Owned server readiness deadline expired') + identity = self.query('SELECT version() AS version, toString(serverUUID()) AS server_id')[0] + version = identity['version'] + if identity['server_id'] != (self.root / 'data/uuid').read_text().strip(): + raise RuntimeError('Endpoint is not the owned server data directory') + if version != entry['version']: + raise RuntimeError('Candidate server version mismatch') + except BaseException: + print(f"Owned server failure evidence: {self.root}", file=sys.stderr, flush=True) + self.close(False) + raise + + @property + def endpoint(self): + return f'http://127.0.0.1:{self.port}' + + def query(self, sql): + request = urllib.request.Request(self.endpoint + '/?default_format=JSONEachRow&wait_end_of_query=1&async_insert=0&mutations_sync=1&alter_sync=1&fsync_metadata=1&max_result_rows=0&max_result_bytes=0&result_overflow_mode=throw&read_overflow_mode=throw', data=sql.encode(), headers={'Authorization': AUTH}, method='POST') + with urllib.request.urlopen(request, timeout=10) as response: + return [json.loads(line) for line in response.read().splitlines() if line.strip()] + + def restart(self): + """Kill/restart only this owned process, preserving its data/config/UUID.""" + if self.process is not None and self.process.poll() is None: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=5) + self.process = subprocess.Popen([str(self.binary), 'server', '--config-file', str(self.root / 'config.xml')], stdout=self.log, stderr=subprocess.STDOUT, start_new_session=True) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if self.process.poll() is not None: + raise RuntimeError('Owned restarted server exited') + try: + identity = self.query('SELECT version() AS version, toString(serverUUID()) AS server_id')[0] + if identity['version'] == self.entry['version'] and identity['server_id'] == (self.root / 'data/uuid').read_text().strip(): + return + except OSError: + pass + time.sleep(.05) + raise RuntimeError('Owned server restart deadline expired') + + def close(self, success): + for proxy in self.proxies: + proxy.close() + if self.process is not None and self.process.poll() is None: + os.killpg(self.process.pid, signal.SIGTERM) + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=5) + self.log.close() + if success: + shutil.rmtree(self.root) + + +def leading_sql(body): + """Classify without recording bodies; account for legal leading comments.""" + while True: + body = body.lstrip() + if body.startswith((b'--', b'//', b'#!', b'# ')): + ends = [index for mark in (b'\r', b'\n') if (index := body.find(mark)) >= 0] + if not ends: + return b'' + body = body[min(ends) + 1:] + elif body.startswith(b'/*'): + depth, cursor = 1, 2 + while depth and cursor < len(body): + if body[cursor:cursor + 2] == b'/*': + depth += 1 + cursor += 2 + elif body[cursor:cursor + 2] == b'*/': + depth -= 1 + cursor += 2 + else: + cursor += 1 + if depth: + return b'' + body = body[cursor:] + else: + return body.upper() + + +class FaultProxy: + def __init__(self, target, tls=False): + self.target = target + self.tls = tls + self.mode = 'pass' + self.query_id = None + self.stage = None + self.step = None + self.rules = [] + self.release = threading.Event() + self.identity_spoof = False + self.closed = False + self.events = [] + self.open_connections = 0 + self.lock = threading.Lock() + proxy = self + class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + def setup(self): + super().setup() + with proxy.lock: + proxy.open_connections += 1 + def finish(self): + try: + super().finish() + finally: + with proxy.lock: + proxy.open_connections -= 1 + def log_message(self, *args): + pass + def do_POST(self): + body = self.rfile.read(int(self.headers.get('Content-Length', '0'))) + with proxy.lock: + stage = None + step = None + header = leading_sql(body) + if header.startswith(b'CREATE DATABASE IF NOT EXISTS ') and header.endswith(b' ENGINE=ATOMIC'): + stage = 'ddl-database' + elif header.startswith(b'CREATE TABLE IF NOT EXISTS ') and b'(SEQUENCE UINT64, EVENT_ID UUID, LEDGER_ID UUID, EVENT STRING)' in header: + stage = 'ddl-table' + elif b'FORMAT JSONEachRow\n' in body: + try: + payload = json.loads(body.split(b'FORMAT JSONEachRow\n', 1)[1]) + event = json.loads(payload['event']) + stage = event['kind'] + context = event.get('context') or {} + step = (context.get('step') or {}).get('name') + except (ValueError, KeyError, TypeError): + pass + if header.startswith(b'SELECT TOSTRING(SEQUENCE) AS SEQUENCE, TOSTRING(EVENT_ID)'): + stage = 'history-read' + if stage is None and header.startswith((b'CREATE ', b'ALTER ', b'INSERT ', b'DROP ', b'TRUNCATE ', b'RENAME ', b'EXCHANGE ')): + stage = 'application-sql' + mode = proxy.mode + if proxy.stage is not None and stage != proxy.stage: + mode = 'pass' + query_id = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).get('query_id', [''])[0] + if proxy.query_id is not None and query_id != proxy.query_id: + mode = 'pass' + if proxy.step is not None and step != proxy.step: + mode = 'pass' + for rule in proxy.rules: + if rule['stage'] == stage and (rule.get('step') is None or rule['step'] == step): + mode = rule['mode'] + if mode == 'delay-first': + mode = 'delay' if not proxy.events else 'pass' + if mode == 'close-second': + mode = 'pass' if not proxy.events else 'close' + event_index = len(proxy.events) + proxy.events.append({'path': self.path, 'mode': mode, 'stage': stage, 'connection': self.client_address[1], 'forwarded': mode not in ['redirect', 'close', 'hold-before'], 'bodyPrefixSent': False}) + if mode == 'close': + self.close_connection = True + return + if mode == 'redirect': + self.send_response(307) + self.send_header('Location', proxy.redirect_url) + self.send_header('Content-Length', '0') + self.end_headers() + return + if mode == 'hold-before': + if not proxy.release.wait(timeout=15): + self.close_connection = True + return + with proxy.lock: + proxy.events[event_index]['forwarded'] = True + upstream = http.client.HTTPConnection('127.0.0.1', target.port, timeout=20) + try: + upstream.request('POST', self.path, body=body, headers={'Authorization': self.headers.get('Authorization', ''), 'Content-Type': 'text/plain'}) + response = upstream.getresponse() + data = response.read() + headers = dict(response.getheaders()) + status = response.status + finally: + upstream.close() + with proxy.lock: + proxy.events[event_index]['responseBytes'] = len(data) + proxy.events[event_index]['maxRowBytes'] = max(map(len, data.splitlines()), default=0) + if mode == 'drop': + self.close_connection = True + return + if mode == 'delay': + time.sleep(2) + if proxy.identity_spoof and data.strip(): + rows = [json.loads(line) for line in data.splitlines() if line.strip()] + if rows and 'server_id' in rows[0]: + rows[0]['server_id'] = '00000000-0000-0000-0000-000000000000' + data = b''.join(json.dumps(row).encode() + b'\n' for row in rows) + if mode in ['unsupported-version', 'prerelease-version', 'weakened-policy']: + rows = [json.loads(line) for line in data.splitlines() if line.strip()] + if mode == 'unsupported-version': + rows[0]['version'] = '26.8.28.15' + elif mode == 'prerelease-version': + rows[0]['version'] = '26.8.28.14-rc1' + else: + rows[0]['mutations_sync'] = 0 + data = b''.join(json.dumps(row).encode() + b'\n' for row in rows) + if mode == 'weakened-storage' and data.strip(): + rows = [json.loads(line) for line in data.splitlines() if line.strip()] + for row in rows: + if 'is_remote' in row: + row['is_remote'], row['type'] = 1, 'ObjectStorage' + data = b''.join(json.dumps(row).encode() + b'\n' for row in rows) + if mode == 'append-late-error': + data += b'Code: 999. DB::Exception: SQL_SECRET_MARKER (SYNTHETIC_ERROR)\n' + if mode == 'oversized-history-row': + data += b'{"oversized":"' + b'x' * 1048577 + b'"}\n' + if mode in ['history-gap', 'history-identity-conflict', 'history-duplicate-conflict']: + rows = [json.loads(line) for line in data.splitlines() if line.strip()] + extra = dict(max(rows, key=lambda row: int(row['sequence']))) + event = json.loads(extra['event']) + if mode == 'history-gap': + event['sequence'] = int(event['sequence']) + 2 + event['eventId'] = 'ffffffff-ffff-4fff-8fff-ffffffffffff' + elif mode == 'history-identity-conflict': + event['sequence'] = int(event['sequence']) + 1 + else: + event['audit']['actor'] = 'conflicting-operator' + extra['sequence'], extra['event_id'] = str(event['sequence']), event['eventId'] + extra['event'] = json.dumps(event, sort_keys=True, separators=(',', ':'), ensure_ascii=False) + rows.append(extra) + data = b''.join(json.dumps(row, ensure_ascii=False).encode() + b'\n' for row in rows) + if mode == 'late-error': + status, data = 200, b'Code: 999. DB::Exception: SQL_SECRET_MARKER (SYNTHETIC_ERROR)\n' + self.send_response(status) + for name in ['X-ClickHouse-Exception-Code', 'Content-Type']: + if name in headers: + self.send_header(name, headers[name]) + if mode == 'switch-identity': + proxy.identity_spoof = True + self.send_header('Connection', 'close') + self.close_connection = True + self.send_header('Content-Length', str(len(data) + (20 if mode == 'truncate' else 0))) + self.end_headers() + try: + if mode == 'delay-history-body': + pivot = len(data) // 2 + self.wfile.write(data[:pivot]) + self.wfile.flush() + with proxy.lock: + proxy.events[event_index]['bodyPrefixSent'] = True + time.sleep(2) + self.wfile.write(data[pivot:]) + else: + self.wfile.write(data) + self.wfile.flush() + except OSError: + pass + if mode == 'truncate': + self.close_connection = True + self.http = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + if tls: + cert = target.root / ('proxy-cert-' + str(self.http.server_port) + '.pem') + key = target.root / ('proxy-key-' + str(self.http.server_port) + '.pem') + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-keyout', str(key), '-out', str(cert), '-subj', '/CN=localhost', + '-addext', 'subjectAltName=DNS:localhost,IP:127.0.0.1'], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + self.http.socket = context.wrap_socket(self.http.socket, server_side=True) + self.http.daemon_threads = True + self.http.block_on_close = False + self.redirect_url = self.endpoint + self.thread = threading.Thread(target=self.http.serve_forever, daemon=True) + self.thread.start() + + @property + def endpoint(self): + return f'{"https" if self.tls else "http"}://127.0.0.1:{self.http.server_port}' + + def close(self): + if self.closed: + return + self.closed = True + self.release.set() + self.http.shutdown() + self.http.server_close() + self.thread.join(timeout=2) + with self.lock: + evidence = {'receivedCount': len(self.events), 'events': list(self.events)} + (self.target.root / f'proxy-{self.http.server_port}-events.json').write_text(json.dumps(evidence, indent=2) + '\n') + + +def serve(matrix, entry_id): + entries = json.loads(Path(matrix).read_text())['entries'] + entry = next(value for value in entries if value['id'] == entry_id) + server = NativeServer(entry) + def stop_signal(*_): + raise SystemExit(1) + signal.signal(signal.SIGTERM, stop_signal) + print(json.dumps({'endpoint': server.endpoint, 'root': str(server.root), 'version': entry['version']}), flush=True) + success = False + try: + for line in sys.stdin: + request = json.loads(line) + operation = request['operation'] + try: + if operation == 'restart': + server.restart() + result = None + elif operation == 'query': + result = server.query(request['sql']) + elif operation == 'proxy-start': + proxy = FaultProxy(server, bool(request.get("tls", False))) + server.proxies.append(proxy) + result = {'id': len(server.proxies)-1, 'endpoint': proxy.endpoint} + elif operation == 'proxy-stop': + server.proxies[request['id']].close() + result = None + elif operation == 'proxy-mode': + proxy = server.proxies[request['id']] + with proxy.lock: + proxy.mode = request['mode'] + proxy.query_id = request.get('queryId') + proxy.stage = request.get('stage') + proxy.step = request.get('step') + proxy.rules.clear() + proxy.release.clear() + result = None + elif operation == 'proxy-rule': + proxy = server.proxies[request['id']] + with proxy.lock: + proxy.rules.append(request) + result = None + elif operation == 'proxy-release': + server.proxies[request['id']].release.set() + result = None + elif operation == 'proxy-open-connections': + proxy = server.proxies[request['id']] + with proxy.lock: + result = proxy.open_connections + elif operation == 'proxy-events': + proxy = server.proxies[request['id']] + with proxy.lock: + result = list(proxy.events) + elif operation == 'proxy-redirect': + server.proxies[request['id']].redirect_url = request['endpoint'] + result = None + elif operation == 'stop': + success = bool(request['success']) + print(json.dumps({'result': None}), flush=True) + break + else: + raise RuntimeError('Unknown fixture operation') + print(json.dumps({'result': result}), flush=True) + except Exception: + print(json.dumps({'fixtureError': 'owned-fixture-operation-failed'}), flush=True) + finally: + server.close(success) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--matrix', required=True) + parser.add_argument('--entry', default='lts-native') + options = parser.parse_args() + serve(options.matrix, options.entry) diff --git a/test/scripts/transport_smoke.py b/test/scripts/transport_smoke.py new file mode 100644 --- /dev/null +++ b/test/scripts/transport_smoke.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Early real-server producer/consumer acceptance before expanding fault variants.""" +import argparse +import json +from pathlib import Path +import select +import subprocess +import sys +import tempfile + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--driver', required=True) + args = parser.parse_args() + script = Path(__file__).resolve().parent / 'native_fixture.py' + matrix = script.parent.parent / 'data/clickhouse-matrix.json' + evidence = Path(tempfile.mkdtemp(prefix='clickhouse-migrate-smoke-', dir='/tmp')) + with (evidence / 'fixture.log').open('w') as log: + service = subprocess.Popen([sys.executable, str(script), '--matrix', str(matrix)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log, text=True) + success = False + try: + if not select.select([service.stdout], [], [], 40)[0]: + raise RuntimeError('Owned fixture startup watchdog expired') + ready = json.loads(service.stdout.readline()) + def rpc(operation, **fields): + service.stdin.write(json.dumps({'operation': operation, **fields}) + '\n') + service.stdin.flush() + if not select.select([service.stdout], [], [], 20)[0]: + raise RuntimeError('Owned fixture RPC watchdog expired') + response = json.loads(service.stdout.readline()) + if 'result' not in response: + raise RuntimeError('Owned fixture operation failed') + return response['result'] + proxy = rpc('proxy-start') + rpc('proxy-mode', id=proxy['id'], mode='drop') + sql = evidence / 'create.sql' + sql.write_text('CREATE TABLE default.response_lost (id UInt8) ENGINE=Memory') + result = subprocess.run([args.driver, 'command', proxy['endpoint'], 'lost-create-response', str(sql), '3000000', '1048576'], text=True, capture_output=True, timeout=10, check=True) + report = json.loads(result.stdout) + assert report['scopeError'] is None and report['cleanupErrors'] == [], report + assert report['outcomes'][0]['outcome'] == 'submission-uncertain', report + assert rpc('query', sql='EXISTS TABLE default.response_lost')[0]['result'] == 1 + events = rpc('proxy-events', id=proxy['id']) + assert len(events) == 1 and events[0]['forwarded'], events + assert 'test-password' not in json.dumps(events) + result.stdout + print(f"ClickHouse {ready['version']}: committed effect + lost response => uncertainty; exactly one submission") + success = True + finally: + if service.poll() is None: + try: + service.stdin.write(json.dumps({'operation': 'stop', 'success': success}) + '\n') + service.stdin.flush() + service.communicate(timeout=20) + except (OSError, subprocess.TimeoutExpired): + service.terminate() + service.wait(timeout=20) + if not success: + print(f'Early transport evidence retained at {evidence}', file=sys.stderr) + if success: + import shutil + shutil.rmtree(evidence) + + +if __name__ == '__main__': + main()