diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog for clickhouse-migrate-cli +## 0.2.0.0 — 2026-11-08 + +### Breaking Changes + +- Extend the JSON v1 contract with both checksum formats in offline plans, + the recorded checksum format in history/status, and replicated initialization + fields and error codes. + +### New Features + +- Populate offline plans with v1 and v2 migration and step digests. +- Render replicated history and failures with structured evidence. + +### Other Changes + +- Retain legacy `PlanStep` and `PlanMigration` construction and matching arities; + legacy manually constructed reports render unknown v2 digests as null. +- Require `clickhouse-migrate ==0.2.*` and update JSON golden fixtures. + ## 0.1.0.0 — 2026-10-16 Initial release. diff --git a/clickhouse-migrate-cli.cabal b/clickhouse-migrate-cli.cabal --- a/clickhouse-migrate-cli.cabal +++ b/clickhouse-migrate-cli.cabal @@ -1,6 +1,6 @@ cabal-version: 3.12 name: clickhouse-migrate-cli -version: 0.1.0.0 +version: 0.2.0.0 synopsis: Reusable administrative CLI for ClickHouse migrations description: Reusable administrative CLI for ClickHouse migrations. @@ -40,7 +40,7 @@ aeson >=2.2.4.1 && <2.4, base >=4.21 && <4.22, bytestring >=0.12.2 && <0.13, - clickhouse-migrate ==0.1.*, + clickhouse-migrate ==0.2.*, containers >=0.7 && <0.9, optparse-applicative >=0.18.1 && <0.20, text >=2.1 && <2.2, diff --git a/src/ClickHouseMigrate/CLI/Handler.hs b/src/ClickHouseMigrate/CLI/Handler.hs --- a/src/ClickHouseMigrate/CLI/Handler.hs +++ b/src/ClickHouseMigrate/CLI/Handler.hs @@ -25,11 +25,12 @@ [ PlanComponent (componentNameText (componentViewName component)) (map componentNameText (Set.toAscList (componentDependencies component))) - [ PlanMigration + [ PlanMigrationChecksums (migrationNameText (migrationIdName (migrationId migration))) (migrationDescription migration) (migrationChecksumHex (migrationChecksum migration)) - [PlanStep (stepNameText (stepViewName step)) (stepChecksumHex (stepChecksum step)) | step <- NE.toList (migrationSteps migration)] + (Just (migrationChecksumHex (migrationChecksumFor ChecksumV2 migration))) + [PlanStepChecksums (stepNameText (stepViewName step)) (stepChecksumHex (stepChecksum step)) (Just (stepChecksumHex (stepChecksumFor ChecksumV2 step))) | step <- NE.toList (migrationSteps migration)] | migration <- NE.toList (componentMigrations component) ] | component <- NE.toList (inspectPlan plan) diff --git a/src/ClickHouseMigrate/CLI/Render/Json.hs b/src/ClickHouseMigrate/CLI/Render/Json.hs --- a/src/ClickHouseMigrate/CLI/Render/Json.hs +++ b/src/ClickHouseMigrate/CLI/Render/Json.hs @@ -33,7 +33,7 @@ reportValue :: CommandReport -> Value reportValue = \case - PlanReport components schedule -> object ["kind" .= ("plan" :: Text), "checksumFormat" .= Plan.checksumFormatName Plan.ChecksumV1, "completionPolicy" .= Encoding.decodeUtf8 Plan.completionPolicy, "components" .= componentsValue schedule 0 components, "schedule" .= [object ["ordinal" .= ordinal, "component" .= component, "migration" .= migration] | (ordinal, (component, migration)) <- zip [0 :: Int ..] schedule]] + PlanReport components schedule -> object ["kind" .= ("plan" :: Text), "checksumFormat" .= Plan.checksumFormatName Plan.ChecksumV1, "checksumFormats" .= (["sha256-v1", "sha256-v2"] :: [Text]), "completionPolicy" .= Encoding.decodeUtf8 Plan.completionPolicy, "components" .= componentsValue schedule 0 components, "schedule" .= [object ["ordinal" .= ordinal, "component" .= component, "migration" .= migration] | (ordinal, (component, migration)) <- zip [0 :: Int ..] schedule]] StatusReport report -> object ["kind" .= ("status" :: Text), "report" .= inspectionValue report] HistoryReport report -> object ["kind" .= ("history" :: Text), "report" .= historyValue report] VerificationReport (R.VerificationReport report current) -> object ["kind" .= ("verification" :: Text), "compatible" .= True, "current" .= current, "report" .= inspectionValue report] @@ -43,7 +43,7 @@ componentsValue :: [(Text, Text)] -> Int -> [PlanComponent] -> [Value] componentsValue _ _ [] = [] componentsValue schedule componentIndex (PlanComponent name dependencies migrations : rest) = - object ["index" .= componentIndex, "name" .= name, "dependencies" .= dependencies, "migrations" .= [object ["ordinal" .= lookup (name, migration) (zip schedule [0 :: Int ..]), "name" .= migration, "description" .= description, "checksum" .= checksum, "steps" .= [object ["index" .= stepIndex, "name" .= step, "checksum" .= digest] | (stepIndex, PlanStep step digest) <- zip [0 :: Int ..] steps]] | PlanMigration migration description checksum steps <- migrations]] : componentsValue schedule (componentIndex + 1) rest + object ["index" .= componentIndex, "name" .= name, "dependencies" .= dependencies, "migrations" .= [object ["ordinal" .= lookup (name, migration) (zip schedule [0 :: Int ..]), "name" .= migration, "description" .= description, "checksum" .= checksum, "checksums" .= object ["sha256-v1" .= checksum, "sha256-v2" .= checksumV2], "steps" .= [object ["index" .= stepIndex, "name" .= step, "checksum" .= digest, "checksums" .= object ["sha256-v1" .= digest, "sha256-v2" .= digestV2]] | (stepIndex, PlanStepChecksums step digest digestV2) <- zip [0 :: Int ..] steps]] | PlanMigrationChecksums migration description checksum checksumV2 steps <- migrations]] : componentsValue schedule (componentIndex + 1) rest definitionValue :: R.DefinitionSummary -> Value definitionValue (R.DefinitionSummary ordinal componentIndex component dependencies migration checksum steps) = object ["ordinal" .= ordinal, "componentIndex" .= componentIndex, "component" .= component, "dependencies" .= dependencies, "migration" .= migration, "checksum" .= checksum, "steps" .= [object ["name" .= name, "checksum" .= digest] | (name, digest) <- steps]] @@ -87,7 +87,7 @@ serverValue identity = object ["serverId" .= T.serverIdText (T.identityServerId identity), "version" .= T.identityVersion identity, "hostName" .= T.identityHostName identity] historyValue :: R.HistoryReport -> Value -historyValue (R.HistoryReport target server presence position access events migrations unresolved) = object ["target" .= T.targetKeyText target, "server" .= nullable serverValue server, "ledger" .= presenceValue presence, "position" .= fmap L.ledgerPositionText position, "access" .= observationAccess access, "events" .= map eventValue events, "migrations" .= map migrationValue migrations, "unresolved" .= unresolved] +historyValue (R.HistoryReport target server presence position access events migrations unresolved) = object ["target" .= T.targetKeyText target, "checksumFormat" .= recordedFormat events, "server" .= nullable serverValue server, "ledger" .= presenceValue presence, "position" .= fmap L.ledgerPositionText position, "access" .= observationAccess access, "events" .= map eventValue events, "migrations" .= map migrationValue migrations, "unresolved" .= unresolved] inspectionValue :: R.InspectionReport -> Value inspectionValue (R.InspectionReport history migrations issues) = object ["observation" .= historyValue history, "migrations" .= map migrationValue migrations, "issues" .= map issueValue issues] @@ -118,6 +118,7 @@ bodyValue :: R.HistoryBody -> Value bodyValue = \case R.LedgerInitialized target -> object ["target" .= T.targetKeyText target] + R.LedgerInitializedReplicated target -> object ["target" .= T.targetKeyText target, "checksumFormat" .= ("sha256-v2" :: Text)] R.MigrationDefinition definition -> object ["definition" .= definitionValue definition] R.StepIntent index query -> object ["step" .= index, "queryId" .= T.queryIdText query] R.StepCompletion index query -> object ["step" .= index, "queryId" .= T.queryIdText query] @@ -159,13 +160,13 @@ transportValue (L.TransportError code status server) = object ["code" .= transportErrorCode code, "httpStatus" .= status, "serverCode" .= server] transportErrorCode :: L.TransportErrorCode -> Text -transportErrorCode = \case L.AdministrativeAccessLost -> "administrative-access-lost"; L.InvalidRequest -> "invalid-request"; L.ClientScopeFailed -> "client-scope-failed"; L.ConnectionFailed -> "connection-failed"; L.DeadlineExpired -> "deadline-expired"; L.ResponseInterrupted -> "response-interrupted"; L.ResponseTooLarge -> "response-too-large"; L.UnexpectedResponse -> "unexpected-response"; L.ServerRejected -> "server-rejected"; L.RedirectRejected -> "redirect-rejected"; L.ServerIdentityMismatch -> "server-identity-mismatch"; L.UnsupportedServer -> "unsupported-server"; L.CompletionPolicyMismatch -> "completion-policy-mismatch"; L.CleanupFailed -> "cleanup-failed"; L.CleanupTimedOut -> "cleanup-timed-out" +transportErrorCode = \case L.AdministrativeAccessLost -> "administrative-access-lost"; L.InvalidRequest -> "invalid-request"; L.ClientScopeFailed -> "client-scope-failed"; L.ConnectionFailed -> "connection-failed"; L.DeadlineExpired -> "deadline-expired"; L.ResponseInterrupted -> "response-interrupted"; L.ResponseTooLarge -> "response-too-large"; L.UnexpectedResponse -> "unexpected-response"; L.ServerRejected -> "server-rejected"; L.RedirectRejected -> "redirect-rejected"; L.ServerIdentityMismatch -> "server-identity-mismatch"; L.UnsupportedServer -> "unsupported-server"; L.CompletionPolicyMismatch -> "completion-policy-mismatch"; L.ReplicaSetMismatch -> "replica-set-mismatch"; L.CleanupFailed -> "cleanup-failed"; L.CleanupTimedOut -> "cleanup-timed-out" accessErrorCode :: Access.AccessError -> Text accessErrorCode = \case Access.InvalidAccessStrategy -> "invalid-access-strategy"; Access.AcquisitionTimedOut -> "acquisition-timed-out"; Access.AcquisitionFailed -> "acquisition-failed"; Access.AccessLost -> "access-lost"; Access.AccessScopeClosed -> "access-scope-closed"; Access.AccessActionFailed -> "access-action-failed"; Access.PredecessorQuiescenceRequired -> "predecessor-quiescence-required"; Access.AccessReleaseFailed -> "access-release-failed"; Access.AccessReleaseTimedOut -> "access-release-timed-out" ledgerErrorCode :: L.LedgerError -> Text -ledgerErrorCode = \case L.InvalidPosition -> "invalid-position"; L.InvalidAudit -> "invalid-audit"; L.InvalidEvidence -> "invalid-evidence"; L.InvalidEvent -> "invalid-event"; L.UnsupportedLedgerVersion -> "unsupported-ledger-version"; L.SequenceGap -> "sequence-gap"; L.SequenceConflict -> "sequence-conflict"; L.EventIdentityConflict -> "event-identity-conflict"; L.LedgerIdentityMismatch -> "ledger-identity-mismatch"; L.MissingInitialization -> "missing-initialization"; L.ImpossibleTransition -> "impossible-transition"; L.DefinitionMismatch -> "definition-mismatch"; L.InvalidLedgerSchema -> "invalid-ledger-schema"; L.UnsupportedLedgerStorage -> "unsupported-ledger-storage"; L.LedgerAbsent -> "ledger-absent"; L.LedgerUninitialized -> "ledger-uninitialized"; L.InitializationNotAuthorized -> "initialization-not-authorized"; L.StaleLedgerState -> "stale-ledger-state"; L.UnresolvedHistory -> "unresolved-history"; L.OutstandingRequests -> "outstanding-requests"; L.LedgerLimitExceeded -> "ledger-limit-exceeded" +ledgerErrorCode = \case L.InvalidPosition -> "invalid-position"; L.InvalidAudit -> "invalid-audit"; L.InvalidEvidence -> "invalid-evidence"; L.InvalidEvent -> "invalid-event"; L.UnsupportedLedgerVersion -> "unsupported-ledger-version"; L.SequenceGap -> "sequence-gap"; L.SequenceConflict -> "sequence-conflict"; L.EventIdentityConflict -> "event-identity-conflict"; L.LedgerIdentityMismatch -> "ledger-identity-mismatch"; L.MissingInitialization -> "missing-initialization"; L.ImpossibleTransition -> "impossible-transition"; L.DefinitionMismatch -> "definition-mismatch"; L.InvalidLedgerSchema -> "invalid-ledger-schema"; L.LedgerDatabaseMissing -> "ledger-database-missing"; L.UnsupportedLedgerStorage -> "unsupported-ledger-storage"; L.LedgerAbsent -> "ledger-absent"; L.LedgerUninitialized -> "ledger-uninitialized"; L.InitializationNotAuthorized -> "initialization-not-authorized"; L.StaleLedgerState -> "stale-ledger-state"; L.UnresolvedHistory -> "unresolved-history"; L.OutstandingRequests -> "outstanding-requests"; L.LedgerLimitExceeded -> "ledger-limit-exceeded" adoptionErrorCode :: A.AdoptionError -> Text adoptionErrorCode = \case A.InvalidSourceEvidence -> "invalid-source-evidence"; A.UnsupportedSourceChecksum -> "unsupported-source-checksum"; A.InvalidSourceChecksum -> "invalid-source-checksum"; A.DuplicateSourceRow -> "duplicate-source-row"; A.DuplicateMapping -> "duplicate-mapping"; A.IncompleteMapping -> "incomplete-mapping"; A.UnknownSourceRow -> "unknown-source-row"; A.MappingOrderMismatch -> "mapping-order-mismatch"; A.MappingDefinitionMismatch -> "mapping-definition-mismatch"; A.HistoricalChecksumMismatch -> "historical-checksum-mismatch"; A.HistoricalChecksumRequired -> "historical-checksum-required"; A.AdoptionTargetMismatch -> "adoption-target-mismatch"; A.AdoptionInitializationRequired -> "adoption-initialization-required"; A.AdoptionExpectedStateMismatch -> "adoption-expected-state-mismatch"; A.AdoptionNativeConflict -> "adoption-native-conflict"; A.AdoptionBatchConflict -> "adoption-batch-conflict"; A.AdoptionResumeRequired -> "adoption-resume-required"; A.AdoptionBatchTooLarge -> "adoption-batch-too-large" @@ -180,6 +181,7 @@ runErrorCode = \case R.InvalidRuntimePlan _ -> "invalid-runtime-plan" R.LedgerNamespaceInSql _ -> "ledger-namespace-in-sql" + R.DatabaseCreationOnReplicatedTarget _ -> "database-creation-on-replicated-target" R.DefinitionTooLarge _ -> "definition-too-large" R.AccessFailed problem -> accessErrorCode problem R.TargetFailed _ -> "target-failed" @@ -209,6 +211,7 @@ fields = \case R.InvalidRuntimePlan step -> ["step" .= stepReferenceValue step] R.LedgerNamespaceInSql step -> ["step" .= stepReferenceValue step] + R.DatabaseCreationOnReplicatedTarget step -> ["step" .= stepReferenceValue step] R.DefinitionTooLarge definition -> ["definition" .= definitionValue definition] R.AccessFailed _ -> [] R.TargetFailed problem -> ["transport" .= transportValue problem] @@ -240,3 +243,9 @@ R.RecoveryStepStartEvent -> "step-start" R.RecoveryStepSuccessEvent -> "step-success" R.RecoveryStepFailureEvent -> "step-failure" + +recordedFormat :: [R.HistoryEvent] -> Maybe Text +recordedFormat events = case [kind | R.HistoryEvent _ _ _ _ _ _ _ _ _ kind <- events] of + R.LedgerInitialized _ : _ -> Just "sha256-v1" + R.LedgerInitializedReplicated _ : _ -> Just "sha256-v2" + _ -> Nothing diff --git a/src/ClickHouseMigrate/CLI/Types.hs b/src/ClickHouseMigrate/CLI/Types.hs --- a/src/ClickHouseMigrate/CLI/Types.hs +++ b/src/ClickHouseMigrate/CLI/Types.hs @@ -1,5 +1,29 @@ +{-# LANGUAGE PatternSynonyms #-} + -- | Command values contain input, never an acquired administrative capability. -module ClickHouseMigrate.CLI.Types where +module ClickHouseMigrate.CLI.Types + ( OutputFormat (..), + Invocation (..), + AuditInput (..), + UpInput (..), + RepairOperation (..), + RepairInput (..), + AdoptionOperation (..), + ExpectedInput (..), + AdoptionInput (..), + Command (..), + CommandPolicy (..), + CommandEnvironment (..), + CommandError (..), + CommandFailure (..), + PlanStep (PlanStep, PlanStepChecksums), + PlanMigration (PlanMigration, PlanMigrationChecksums), + PlanComponent (..), + CommandReport (..), + CommandResult (..), + ParseIssue (..), + ) +where import ClickHouseMigrate.Adoption (AdoptionError) import ClickHouseMigrate.Event (DiagnosticPolicy, EventSink) @@ -69,9 +93,25 @@ | AdoptionRejected !AdoptionError !R.HistoryReport deriving stock (Show) -data PlanStep = PlanStep !Text !Text deriving stock (Eq, Show) +data PlanStep = PlanStepChecksums !Text !Text !(Maybe Text) deriving stock (Eq, Show) -data PlanMigration = PlanMigration !Text !(Maybe Text) !Text ![PlanStep] deriving stock (Eq, Show) +data PlanMigration = PlanMigrationChecksums !Text !(Maybe Text) !Text !(Maybe Text) ![PlanStep] deriving stock (Eq, Show) + +-- Preserve existing construction and matching arities while retaining additive +-- v2 data. Manually constructed legacy reports have no known v2 digest. +pattern PlanStep :: Text -> Text -> PlanStep +pattern PlanStep name checksum <- PlanStepChecksums name checksum _ + where + PlanStep name checksum = PlanStepChecksums name checksum Nothing + +pattern PlanMigration :: Text -> Maybe Text -> Text -> [PlanStep] -> PlanMigration +pattern PlanMigration name description checksum steps <- PlanMigrationChecksums name description checksum _ steps + where + PlanMigration name description checksum steps = PlanMigrationChecksums name description checksum Nothing steps + +{-# COMPLETE PlanStep #-} + +{-# COMPLETE PlanMigration #-} data PlanComponent = PlanComponent !Text ![Text] ![PlanMigration] deriving stock (Eq, Show) diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -11,6 +11,7 @@ import Data.Aeson qualified as JSON import Data.ByteString qualified as BS import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE import Data.Set qualified as Set import Data.Text qualified as Text import Data.Text.Encoding qualified as Encoding @@ -45,6 +46,16 @@ decode = decodeAdoptionInput key (L.ExpectAbsent key) authorization quiet audit offline <- handleCommand (error "Offline plan forced configuration") plan PlanCommand assert "offline plan missing" (case offline of CommandResult "plan" (Right (PlanReport [_] [("app", "one")])) -> True; _ -> False) + let boundMigration = NE.head (componentMigrations (NE.head (inspectPlan plan))) + boundStep = NE.head (migrationSteps boundMigration) + v1 = migrationChecksumHex (migrationChecksum boundMigration) + v2 = migrationChecksumHex (migrationChecksumFor ChecksumV2 boundMigration) + rendered = Encoding.decodeUtf8 (renderJson offline) + contains key value = ("\"" <> key <> "\":\"" <> value <> "\"") `Text.isInfixOf` rendered + assert "v1/v2 migration hashes must differ" (v1 /= v2) + assert "offline existing checksum moved from v1" (contains "checksum" v1) + assert "offline migration checksum formats missing" (contains "sha256-v1" v1 && contains "sha256-v2" v2) + assert "offline step checksum formats missing" (contains "sha256-v1" (stepChecksumHex (stepChecksum boundStep)) && contains "sha256-v2" (stepChecksumHex (stepChecksumFor ChecksumV2 boundStep))) let two = right (sqlMigration "two" (step :| [])) a = right (migrationComponent "a" Set.empty (migration :| [two])) b = right (migrationComponent "b" (Set.singleton "a") (migration :| [])) diff --git a/test/golden/accepted.json b/test/golden/accepted.json --- a/test/golden/accepted.json +++ b/test/golden/accepted.json @@ -69,7 +69,8 @@ ] } ], - "unresolved": true + "unresolved": true, + "checksumFormat": null }, "migrations": [ { diff --git a/test/golden/delayed-ledger.json b/test/golden/delayed-ledger.json --- a/test/golden/delayed-ledger.json +++ b/test/golden/delayed-ledger.json @@ -95,7 +95,8 @@ ] } ], - "unresolved": true + "unresolved": true, + "checksumFormat": null }, "migrations": [ { diff --git a/test/golden/partial.json b/test/golden/partial.json --- a/test/golden/partial.json +++ b/test/golden/partial.json @@ -100,7 +100,8 @@ ] } ], - "unresolved": true + "unresolved": true, + "checksumFormat": null }, "migrations": [ { diff --git a/test/golden/recovery-too-large.json b/test/golden/recovery-too-large.json --- a/test/golden/recovery-too-large.json +++ b/test/golden/recovery-too-large.json @@ -82,7 +82,8 @@ ] } ], - "unresolved": true + "unresolved": true, + "checksumFormat": null }, "migrations": [ { diff --git a/test/golden/schedule.json b/test/golden/schedule.json --- a/test/golden/schedule.json +++ b/test/golden/schedule.json @@ -17,14 +17,22 @@ "name": "one", "description": null, "checksum": "a1-hash", - "steps": [] + "steps": [], + "checksums": { + "sha256-v1": "a1-hash", + "sha256-v2": null + } }, { "ordinal": 2, "name": "two", "description": "upgrade", "checksum": "a2-hash", - "steps": [] + "steps": [], + "checksums": { + "sha256-v1": "a2-hash", + "sha256-v2": null + } } ] }, @@ -40,7 +48,11 @@ "name": "one", "description": null, "checksum": "b1-hash", - "steps": [] + "steps": [], + "checksums": { + "sha256-v1": "b1-hash", + "sha256-v2": null + } } ] } @@ -61,6 +73,10 @@ "component": "a", "migration": "two" } + ], + "checksumFormats": [ + "sha256-v1", + "sha256-v2" ] }, "error": null