diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog for clickhouse-migrate-embed + +## 0.1.0.0 — 2026-10-16 + +Initial release. + +### New Features + +- Compile-time embedding of ordered migration manifests + (`ClickHouseMigrate.Embed`), reconstructed through the public + `clickhouse-migrate` constructors. +- Manifest validation: duplicate-key rejection and path containment. +- Content-qualified rebuilds: declared inputs are fingerprinted so Cabal cannot + skip recompilation when embedded SQL changes, including for incremental and + source-free consumers built from source distributions. diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2026, shinzui +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# clickhouse-migrate-embed + +Compile an ordered package-local JSON manifest and exact SQL bytes into a core +`MigrationComponent` with `$(embedComponent "migrations/manifest.json")`. +Enable TemplateHaskell in the caller. Manifest and path validation fail compilation; +generated code uses public core constructors and never reopens source at runtime. + +Use the shipped content-qualified build entry point for timestamp-independent +Cabal builds, declaring every manifest containing splices: + +```bash +clickhouse-migrate-embed-build --input path/to/package migrations/manifest.json -- build all +``` + +The command also accepts `test` and `haddock`, with ordinary Cabal arguments after +the command. This preflight is needed because raw Cabal can skip GHC for preserved +modification times. The public splice still registers all compiler dependencies. + +The workspace's `docs/user/contracts/manifest-v1.md` defines fields, containment and +build guarantees. The source distribution includes fixtures and qualification +scripts in `test/fixtures/` and `test/scripts/`; these expect sibling core/embed +sources with a cabal.project naming both. No compiler plugin is required. diff --git a/build-driver/Main.hs b/build-driver/Main.hs new file mode 100644 --- /dev/null +++ b/build-driver/Main.hs @@ -0,0 +1,50 @@ +-- | Content-qualified Cabal entry point. Never executes application SQL. +module Main (main) where + +import ClickHouseMigrate.Embed.Manifest (constructComponent, loadComponentFiles) +import ClickHouseMigrate.Prelude +import Crypto.Hash.SHA256 qualified as SHA256 +import Data.ByteString qualified as BS +import Data.ByteString.Builder (Builder, byteString, toLazyByteString, word64BE) +import Data.ByteString.Lazy qualified as Lazy +import Data.Set qualified as Set +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Encoding +import Numeric (showHex) +import System.Environment (getArgs) +import System.Process (callProcess) + +main :: IO () +main = do + args <- getArgs + (inputs, command, options) <- either fail pure (parseArguments [] args) + chunks <- traverse fingerprintInput inputs + let hash = SHA256.hash (Lazy.toStrict (toLazyByteString (byteString "clickhouse-migrate/embed-build-v1\0" <> mconcat chunks))) + hex byte = case showHex byte "" of [c] -> ['0', c]; chars -> chars + token = concatMap hex (BS.unpack hash) + -- The content token changes Cabal's build configuration before its file + -- monitor can incorrectly skip GHC. Force-recomp makes the resulting compiler + -- invocation independent of GHC's own source timestamp shortcuts. + callProcess "cabal" ([command, "--ghc-options=-fforce-recomp -DCHM_EMBED_INPUT_" ++ token] ++ options) + +parseArguments :: [(FilePath, FilePath)] -> [String] -> Either String ([(FilePath, FilePath)], String, [String]) +parseArguments inputs ("--input" : root : manifest : rest) = parseArguments (inputs ++ [(root, manifest)]) rest +parseArguments inputs ("--" : command : options) + | not (null inputs) && command `elem` ["build", "test", "haddock"] = Right (inputs, command, options) +parseArguments _ _ = Left "Usage: clickhouse-migrate-embed-build --input PACKAGE_ROOT MANIFEST [--input ROOT MANIFEST ...] -- build|test|haddock [CABAL_ARGS...]" + +fingerprintInput :: (FilePath, FilePath) -> IO Builder +fingerprintInput (root, manifest) = do + result <- loadComponentFiles root manifest + (parsed, sources, paths) <- either (fail . show) pure result + void (either (fail . show) pure (constructComponent parsed sources)) + fields <- + traverse + ( \path -> do + bytes <- BS.readFile path + pure (field (Encoding.encodeUtf8 (Text.pack path)) <> field bytes) + ) + (Set.toAscList (Set.fromList paths)) + pure (mconcat fields) + where + field bytes = word64BE (fromIntegral (BS.length bytes)) <> byteString bytes diff --git a/clickhouse-migrate-embed.cabal b/clickhouse-migrate-embed.cabal new file mode 100644 --- /dev/null +++ b/clickhouse-migrate-embed.cabal @@ -0,0 +1,81 @@ +cabal-version: 3.12 +name: clickhouse-migrate-embed +version: 0.1.0.0 +synopsis: Compile-time embedding for ClickHouse migration definitions +description: + Compile-time embedding for ClickHouse migration definitions. + Validated public APIs with explicit application-owned configuration and scope. + +license: BSD-3-Clause +license-file: LICENSE +author: shinzui +maintainer: shinzui@users.noreply.github.com +category: Database +build-type: Simple +extra-doc-files: + CHANGELOG.md + README.md + +extra-source-files: + test/fixtures/*.cabal + test/fixtures/*.hs + test/fixtures/component/*.json + test/fixtures/component/sql/*.sql + test/scripts/*.py + +tested-with: ghc ==9.12.4 + +source-repository head + type: git + location: https://github.com/shinzui/clickhouse-migrate.git + subdir: clickhouse-migrate-embed + +common common + default-language: GHC2024 + default-extensions: + DeriveAnyClass + DuplicateRecordFields + OverloadedLabels + OverloadedStrings + + ghc-options: -Wall + build-depends: + aeson >=2.2.4.1 && <2.4, + base >=4.21 && <4.22, + bytestring >=0.12.2 && <0.13, + clickhouse-migrate ==0.1.*, + containers >=0.7 && <0.9, + directory >=1.3.10 && <1.4, + filepath >=1.5.4 && <1.6, + template-haskell >=2.23 && <2.24, + text >=2.1 && <2.2, + +library + import: common + hs-source-dirs: src + exposed-modules: ClickHouseMigrate.Embed + other-modules: ClickHouseMigrate.Embed.Manifest + +test-suite clickhouse-migrate-embed-test + import: common + type: exitcode-stdio-1.0 + hs-source-dirs: + test + src + + main-is: Main.hs + other-modules: + ClickHouseMigrate.Embed + ClickHouseMigrate.Embed.Manifest + +executable clickhouse-migrate-embed-build + import: common + hs-source-dirs: + build-driver + src + + main-is: Main.hs + other-modules: ClickHouseMigrate.Embed.Manifest + build-depends: + cryptohash-sha256 >=0.11.102.1 && <0.12, + process >=1.6.25 && <1.7, diff --git a/src/ClickHouseMigrate/Embed.hs b/src/ClickHouseMigrate/Embed.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/Embed.hs @@ -0,0 +1,56 @@ +{-# LANGUAGE TemplateHaskell #-} + +-- | Compile an ordered package-relative manifest into validated definitions. +module ClickHouseMigrate.Embed (embedComponent, ManifestError (..), ManifestErrorCode (..)) where + +import ClickHouseMigrate +import ClickHouseMigrate.Embed.Manifest +import ClickHouseMigrate.Prelude +import Data.ByteString qualified as BS +import Data.Set qualified as Set +import Data.Text qualified as Text +import Language.Haskell.TH (Exp, Q, listE) +import Language.Haskell.TH.Syntax (addDependentFile, getPackageRoot, lift, runIO) + +-- | Paths resolve from the compiler's package root. The manifest directory is +-- the containment boundary for every SQL file. No source files are read at runtime. +embedComponent :: FilePath -> Q Exp +embedComponent file = do + root <- getPackageRoot + result <- runIO (loadComponentFiles root file) + case result of + Left failure -> fail ("clickhouse-migrate-embed: " ++ show failure) + Right (manifest, sources, paths) -> case constructComponent manifest sources of + Left failure -> fail ("clickhouse-migrate-embed: " ++ show failure) + Right _ -> do + mapM_ addDependentFile (Set.toAscList (Set.fromList paths)) + body <- componentExpression manifest sources + [| + case $(pure body) of + Right component -> component + Left _ -> error "clickhouse-migrate-embed: generated definition failed validation" + |] + +textExpression :: Text -> Q Exp +textExpression value = [|Text.pack $(lift (Text.unpack value))|] + +componentExpression :: Manifest -> [(FilePath, ByteString)] -> Q Exp +componentExpression (Manifest name deps migrations) sources = + [| + do + definitions <- sequence $(listE (map migrationExpression migrations)) + migrationComponentFromList $(textExpression name) (Set.fromList $(listE (map textExpression deps))) definitions + |] + where + migrationExpression (ManifestMigration migration description steps) = + [| + do + definitions <- sequence $(listE (map stepExpression steps)) + value <- sqlMigrationFromList $(textExpression migration) definitions + pure (withMigrationDescription $(descriptionExpression description) value) + |] + stepExpression (ManifestStep step path) = case lookup path sources of + Nothing -> fail "clickhouse-migrate-embed: validated source missing" + Just bytes -> [|sqlStep $(textExpression step) (BS.pack $(lift (BS.unpack bytes)))|] + descriptionExpression Nothing = [|Nothing|] + descriptionExpression (Just description) = [|Just $(textExpression description)|] diff --git a/src/ClickHouseMigrate/Embed/Manifest.hs b/src/ClickHouseMigrate/Embed/Manifest.hs new file mode 100644 --- /dev/null +++ b/src/ClickHouseMigrate/Embed/Manifest.hs @@ -0,0 +1,175 @@ +-- | Private ordered manifest parser and package/manifest-tree containment. +module ClickHouseMigrate.Embed.Manifest + ( ManifestError (..), + ManifestErrorCode (..), + Manifest (..), + ManifestMigration (..), + ManifestStep (..), + decodeManifest, + loadComponentFiles, + constructComponent, + ) +where + +import ClickHouseMigrate +import ClickHouseMigrate.Prelude +import Control.Exception (Exception, IOException, throwIO, try) +import Data.Aeson (Value (..), eitherDecodeStrict') +import Data.Aeson.Decoding.ByteString (bsToTokens) +import Data.Aeson.Decoding.Tokens +import Data.Aeson.Key qualified as Key +import Data.Aeson.KeyMap qualified as KM +import Data.ByteString qualified as BS +import Data.Foldable (toList) +import Data.List (inits) +import Data.Set qualified as Set +import Data.Text qualified as Text +import System.Directory (canonicalizePath, doesFileExist) +import System.FilePath (isAbsolute, joinPath, makeRelative, splitDirectories, takeDirectory, ()) + +data ManifestErrorCode = InvalidJson | DuplicateKey | UnknownField | MissingField | WrongType | UnsupportedManifestVersion | DuplicateDependency | UnsafePath | MissingFile | FileAccessFailure | InvalidDefinition + deriving stock (Eq, Ord, Show) + +data ManifestError = ManifestError + { code :: !ManifestErrorCode, + path :: ![Text], + definitionError :: !(Maybe DefinitionError) + } + deriving stock (Eq, Show) + +instance Exception ManifestError + +data Manifest = Manifest !Text ![Text] ![ManifestMigration] deriving stock (Eq, Show) + +data ManifestMigration = ManifestMigration !Text !(Maybe Text) ![ManifestStep] deriving stock (Eq, Show) + +data ManifestStep = ManifestStep !Text !FilePath deriving stock (Eq, Show) + +err :: ManifestErrorCode -> [Text] -> Either ManifestError a +err c p = Left (ManifestError c p Nothing) + +-- Walk aeson's public token representation before it discards duplicate keys. +checkTokens :: [Text] -> Tokens k String -> Either ManifestError k +checkTokens _ (TkLit _ next) = Right next +checkTokens _ (TkText _ next) = Right next +checkTokens _ (TkNumber _ next) = Right next +checkTokens p (TkArrayOpen values) = checkArray p 0 values +checkTokens p (TkRecordOpen pairs) = checkRecord p Set.empty pairs +checkTokens p (TkErr _) = err InvalidJson p + +checkArray :: [Text] -> Int -> TkArray k String -> Either ManifestError k +checkArray _ _ (TkArrayEnd next) = Right next +checkArray p _ (TkArrayErr _) = err InvalidJson p +checkArray p i (TkItem value) = checkTokens (p ++ [Text.pack (show i)]) value >>= checkArray p (i + 1) + +checkRecord :: [Text] -> Set Key.Key -> TkRecord k String -> Either ManifestError k +checkRecord _ _ (TkRecordEnd next) = Right next +checkRecord p _ (TkRecordErr _) = err InvalidJson p +checkRecord p seen (TkPair key value) + | Set.member key seen = err DuplicateKey (p ++ [Text.take 128 (Key.toText key)]) + | otherwise = checkTokens (p ++ [Text.take 128 (Key.toText key)]) value >>= checkRecord p (Set.insert key seen) + +decodeManifest :: ByteString -> Either ManifestError Manifest +decodeManifest bytes = do + _ <- checkTokens [] (bsToTokens bytes) + value <- either (const (err InvalidJson [])) Right (eitherDecodeStrict' bytes) + object <- objectAt [] ["manifestVersion", "component", "dependencies", "migrations"] value + version <- required [] "manifestVersion" object + case version of Number 1 -> Right (); _ -> err UnsupportedManifestVersion ["manifestVersion"] + name <- required [] "component" object >>= textAt ["component"] + deps <- required [] "dependencies" object >>= arrayAt ["dependencies"] >>= traverse (textAt ["dependencies"]) + if Set.size (Set.fromList deps) /= length deps then err DuplicateDependency ["dependencies"] else pure () + migrations <- required [] "migrations" object >>= arrayAt ["migrations"] >>= indexed parseMigration ["migrations"] + pure (Manifest name deps migrations) + where + parseMigration p value = do + object <- objectAt p ["name", "description", "steps"] value + name <- required p "name" object >>= textAt (p ++ ["name"]) + description <- case KM.lookup "description" object of + Nothing -> Right Nothing + Just Null -> Right Nothing + Just v -> Just <$> textAt (p ++ ["description"]) v + steps <- required p "steps" object >>= arrayAt (p ++ ["steps"]) >>= indexed parseStep (p ++ ["steps"]) + pure (ManifestMigration name description steps) + parseStep p value = do + object <- objectAt p ["name", "file"] value + name <- required p "name" object >>= textAt (p ++ ["name"]) + file <- required p "file" object >>= textAt (p ++ ["file"]) + pure (ManifestStep name (Text.unpack file)) + +indexed :: ([Text] -> a -> Either ManifestError b) -> [Text] -> [a] -> Either ManifestError [b] +indexed f p = traverse (\(i, value) -> f (p ++ [Text.pack (show i)]) value) . zip [0 :: Int ..] + +objectAt :: [Text] -> [Key.Key] -> Value -> Either ManifestError (KM.KeyMap Value) +objectAt p allowed (Object object) = case filter (`notElem` allowed) (KM.keys object) of + [] -> Right object + key : _ -> err UnknownField (p ++ [Text.take 128 (Key.toText key)]) +objectAt p _ _ = err WrongType p + +required :: [Text] -> Key.Key -> KM.KeyMap Value -> Either ManifestError Value +required p key = maybe (err MissingField (p ++ [Key.toText key])) Right . KM.lookup key + +textAt :: [Text] -> Value -> Either ManifestError Text +textAt _ (String text) = Right text +textAt p _ = err WrongType p + +arrayAt :: [Text] -> Value -> Either ManifestError [Value] +arrayAt _ (Array values) = Right (toList values) +arrayAt p _ = err WrongType p + +-- All IO is compile-time. Track both lexical and canonical paths for symlinks. +loadComponentFiles :: FilePath -> FilePath -> IO (Either ManifestError (Manifest, [(FilePath, ByteString)], [FilePath])) +loadComponentFiles packageRoot manifestFile = do + outcome <- try @IOException (try @ManifestError load) + pure $ case outcome of + Left _ -> Left (ManifestError FileAccessFailure [Text.pack (take 256 manifestFile)] Nothing) + Right result -> result + where + load = do + root <- canonicalizePath packageRoot + manifest <- containedFile root manifestFile + bytes <- BS.readFile manifest + parsed@(Manifest _ _ migrations) <- either throwIO pure (decodeManifest bytes) + let directory = takeDirectory manifest + files = [file | ManifestMigration _ _ steps <- migrations, ManifestStep _ file <- steps] + loaded <- + traverse + ( \file -> do + canonical <- containedFile directory file + sql <- BS.readFile canonical + pure (file, sql, [directory file, canonical]) + ) + files + pure (parsed, [(file, sql) | (file, sql, _) <- loaded], [root manifestFile, manifest] ++ concat [paths | (_, _, paths) <- loaded]) + +containedFile :: FilePath -> FilePath -> IO FilePath +containedFile root relative = do + let parts = splitDirectories relative + unsafe = null relative || isAbsolute relative || any (== "..") parts || '\\' `elem` relative || '\0' `elem` relative + reject = throwIO (ManifestError UnsafePath [Text.pack (take 256 relative)] Nothing) + inside path = not (isAbsolute (makeRelative root path)) && not (".." `elem` splitDirectories (makeRelative root path)) + if unsafe then reject else pure () + -- Inspect every ancestor: a symlink may escape and later return inside. + mapM_ + ( \prefix -> do + canonical <- canonicalizePath (root joinPath prefix) + unless (inside canonical) reject + ) + (drop 1 (inits parts)) + canonical <- canonicalizePath (root relative) + exists <- doesFileExist canonical + unless exists (throwIO (ManifestError MissingFile [Text.pack (take 256 relative)] Nothing)) + pure canonical + +constructComponent :: Manifest -> [(FilePath, ByteString)] -> Either ManifestError MigrationComponent +constructComponent (Manifest name deps migrations) sources = do + definitions <- traverse buildMigration migrations + core ["component"] (migrationComponentFromList name (Set.fromList deps) definitions) + where + core p = either (\e -> Left (ManifestError InvalidDefinition p (Just e))) Right + buildMigration (ManifestMigration migration description steps) = do + definitions <- traverse (buildStep migration) steps + withMigrationDescription description <$> core ["migrations", Text.take 128 migration] (sqlMigrationFromList migration definitions) + buildStep migration (ManifestStep step file) = case lookup file sources of + Nothing -> err MissingFile [Text.pack file] + Just bytes -> core ["migrations", Text.take 128 migration, "steps", Text.take 128 step, Text.pack (take 256 file)] (sqlStep step bytes) diff --git a/test/Main.hs b/test/Main.hs new file mode 100644 --- /dev/null +++ b/test/Main.hs @@ -0,0 +1,48 @@ +{-# LANGUAGE TemplateHaskell #-} + +module Main (main) where + +import ClickHouseMigrate +import ClickHouseMigrate.Embed (embedComponent) +import ClickHouseMigrate.Embed.Manifest qualified as Manifest +import ClickHouseMigrate.Prelude +import Data.ByteString qualified as BS +import Data.ByteString.Char8 qualified as Char8 +import Data.List (isInfixOf) + +embedded :: MigrationComponent +embedded = $(embedComponent "test/fixtures/component/manifest.json") + +main :: IO () +main = do + bytes <- BS.readFile "test/fixtures/component/manifest.json" + manifest <- either (fail . show) pure (Manifest.decodeManifest bytes) + create <- BS.readFile "test/fixtures/component/sql/create.sql" + seed <- BS.readFile "test/fixtures/component/sql/seed.sql" + ordinary <- either (fail . show) pure (Manifest.constructComponent manifest [("sql/create.sql", create), ("sql/seed.sql", seed)]) + unless (embedded == ordinary) (fail "Embedded bytes differ from public construction") + _ <- either (fail . show) pure (migrationPlan (embedded :| [])) + mapM_ + ( \(expected, invalid) -> case Manifest.decodeManifest invalid of + Left (Manifest.ManifestError actual _ _) | actual == expected -> pure () + _ -> fail ("Manifest error mismatch: " ++ show expected) + ) + [ (Manifest.DuplicateKey, "{\"component\":\"a\",\"component\":\"b\"}"), + (Manifest.DuplicateKey, "{\"name\":1,\"\\u006eame\":2}"), + (Manifest.DuplicateKey, "{\"migrations\":[{\"name\":1,\"name\":2}]}"), + (Manifest.UnknownField, "{\"unknown\":1}"), + (Manifest.MissingField, "{}"), + (Manifest.UnsupportedManifestVersion, "{\"manifestVersion\":2}"), + (Manifest.WrongType, "[]"), + (Manifest.InvalidJson, "{} trailing"), + (Manifest.InvalidJson, "{broken"), + (Manifest.DuplicateDependency, "{\"manifestVersion\":1,\"component\":\"a\",\"dependencies\":[\"a\",\"a\"],\"migrations\":[]}") + ] + case Manifest.constructComponent (Manifest.Manifest "a" [] []) [] of + Left (Manifest.ManifestError Manifest.InvalidDefinition _ _) -> pure () + _ -> fail "Empty manifest migrations accepted" + let secret = "SQL_SECRET_MARKER" + case Manifest.constructComponent (Manifest.Manifest "a" [] [Manifest.ManifestMigration "m" Nothing [Manifest.ManifestStep "s" "query.sql"]]) [("query.sql", Char8.pack secret)] of + Left failure | not (secret `isInfixOf` show failure) -> pure () + _ -> fail "Invalid SQL accepted or leaked in diagnostics" + putStrLn "Embedded bytes match public construction; manifest ambiguity and redaction checks pass." diff --git a/test/fixtures/Main.hs b/test/fixtures/Main.hs new file mode 100644 --- /dev/null +++ b/test/fixtures/Main.hs @@ -0,0 +1,26 @@ +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Main (main) where + +import ClickHouseMigrate +import ClickHouseMigrate.Embed (embedComponent) +import Data.ByteString qualified as BS +import Data.List.NonEmpty (NonEmpty (..)) +import Data.List.NonEmpty qualified as NE +import Data.Text qualified as Text +import Numeric (showHex) + +component :: MigrationComponent +component = $(embedComponent "component/manifest.json") + +main :: IO () +main = case migrationPlan (component :| []) of + Left _ -> fail "Embedded plan failed" + Right plan -> mapM_ render (concatMap (NE.toList . componentMigrations) (NE.toList (inspectPlan plan))) + where + render migration = do + putStrLn (Text.unpack (migrationChecksumHex (migrationChecksum migration))) + mapM_ (putStrLn . concatMap hex . BS.unpack . stepSql) (NE.toList (migrationSteps migration)) + hex byte = case showHex byte "" of [c] -> ['0', c]; chars -> chars diff --git a/test/fixtures/component/manifest.json b/test/fixtures/component/manifest.json new file mode 100644 --- /dev/null +++ b/test/fixtures/component/manifest.json @@ -0,0 +1,9 @@ +{ + "manifestVersion": 1, + "component": "analytics", + "dependencies": [], + "migrations": [{"name": "0001", "description": "Create storage", "steps": [ + {"name": "create", "file": "sql/create.sql"}, + {"name": "seed", "file": "sql/seed.sql"} + ]}] +} diff --git a/test/fixtures/component/sql/create.sql b/test/fixtures/component/sql/create.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/component/sql/create.sql @@ -0,0 +1,1 @@ +CREATE TABLE app.events (id UInt64) ENGINE=MergeTree ORDER BY id diff --git a/test/fixtures/component/sql/seed.sql b/test/fixtures/component/sql/seed.sql new file mode 100644 --- /dev/null +++ b/test/fixtures/component/sql/seed.sql @@ -0,0 +1,1 @@ +INSERT INTO app.events VALUES (1) diff --git a/test/fixtures/embedding-consumer.cabal b/test/fixtures/embedding-consumer.cabal new file mode 100644 --- /dev/null +++ b/test/fixtures/embedding-consumer.cabal @@ -0,0 +1,25 @@ +cabal-version: 3.12 +name: embedding-consumer +version: 0.1.0.0 +build-type: Simple +extra-source-files: + component/manifest.json + component/sql/*.sql + +common common + default-language: GHC2024 + default-extensions: + DeriveAnyClass + DuplicateRecordFields + OverloadedLabels + OverloadedStrings + +executable embedding-consumer + import: common + main-is: Main.hs + build-depends: + base >=4.21 && <4.22, + bytestring, + clickhouse-migrate, + clickhouse-migrate-embed, + text, diff --git a/test/scripts/check-embedding-failures.py b/test/scripts/check-embedding-failures.py new file mode 100644 --- /dev/null +++ b/test/scripts/check-embedding-failures.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Compile invalid manifests through the public splice; isolate all file edits.""" +import json +from pathlib import Path +import shutil +import subprocess +import tempfile + +EMBED_ROOT = Path(__file__).resolve().parents[2] +ROOT = EMBED_ROOT.parent +FIXTURES = EMBED_ROOT / 'test/fixtures' + + +def main(): + evidence = Path(tempfile.mkdtemp(prefix='clickhouse-migrate-embed-errors-', dir='/tmp')) + outside = evidence / 'outside' + shutil.copytree(FIXTURES / 'component', outside) + cases = ['duplicate-key', 'nested-duplicate', 'unknown-field', 'bad-version', 'empty-steps', + 'missing-file', 'parent-path', 'absolute-path', 'manifest-parent', + 'manifest-symlink', 'file-symlink', 'directory-symlink', 'escaping-ancestor-return'] + try: + for case in cases: + fixture = evidence / case + shutil.copytree(FIXTURES, fixture) + manifest = fixture / 'component/manifest.json' + data = json.loads(manifest.read_text()) + expected = 'UnsafePath' + if case == 'duplicate-key': + manifest.write_text('{"component":"a","component":"b"}') + expected = 'DuplicateKey' + elif case == 'nested-duplicate': + manifest.write_text('{"migrations":[{"name":"a","\\u006eame":"b"}]}') + expected = 'DuplicateKey' + elif case == 'unknown-field': + data['migrations'][0]['steps'][0]['extra'] = 1 + expected = 'UnknownField' + elif case == 'bad-version': + data['manifestVersion'] = 2 + expected = 'UnsupportedManifestVersion' + elif case == 'empty-steps': + data['migrations'][0]['steps'] = [] + expected = 'InvalidDefinition' + elif case == 'missing-file': + data['migrations'][0]['steps'][0]['file'] = 'sql/missing.sql' + expected = 'MissingFile' + elif case == 'parent-path': + data['migrations'][0]['steps'][0]['file'] = '../outside.sql' + elif case == 'absolute-path': + data['migrations'][0]['steps'][0]['file'] = str(fixture / 'component/sql/create.sql') + elif case == 'manifest-parent': + source = fixture / 'Main.hs' + source.write_text(source.read_text().replace('component/manifest.json', '../outside/manifest.json')) + elif case == 'manifest-symlink': + manifest.unlink() + manifest.symlink_to(outside / 'manifest.json') + elif case == 'file-symlink': + file = fixture / 'component/sql/create.sql' + file.unlink() + file.symlink_to(outside / 'sql/create.sql') + elif case == 'directory-symlink': + # Still in the package but outside the canonical manifest tree. + (fixture / 'component/sql').rename(fixture / 'elsewhere') + (fixture / 'component/sql').symlink_to(fixture / 'elsewhere', target_is_directory=True) + elif case == 'escaping-ancestor-return': + bridge = evidence / 'bridge' + bridge.mkdir() + (bridge / 'back').symlink_to(fixture / 'component', target_is_directory=True) + (fixture / 'component/bridge').symlink_to(bridge, target_is_directory=True) + data['migrations'][0]['steps'][0]['file'] = 'bridge/back/sql/create.sql' + if case in ['unknown-field', 'bad-version', 'empty-steps', 'missing-file', 'parent-path', 'absolute-path', 'escaping-ancestor-return']: + manifest.write_text(json.dumps(data)) + output = fixture / 'build' + output.mkdir() + result = subprocess.run(['cabal', 'exec', '--', 'ghc', '--make', '-fforce-recomp', '-XGHC2024', + '-package', 'clickhouse-migrate', '-package', 'clickhouse-migrate-embed', + '-working-dir', str(fixture), 'Main.hs', '-outputdir', str(output), '-o', str(fixture / 'consumer')], + cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (fixture / 'compile.log').write_text(result.stdout) + if result.returncode == 0 or expected not in result.stdout: + raise RuntimeError(f'{case}: expected {expected}; see {fixture / "compile.log"}\n{result.stdout[-1500:]}') + print(f'{case}: compilation rejects {expected}', flush=True) + except BaseException: + print(f'Compile-failure evidence retained at {evidence}', flush=True) + raise + else: + shutil.rmtree(evidence) + print('All 13 public manifest/path compile-failure cases passed', flush=True) + + +if __name__ == '__main__': + main() diff --git a/test/scripts/test-embedding-rebuild.py b/test/scripts/test-embedding-rebuild.py new file mode 100644 --- /dev/null +++ b/test/scripts/test-embedding-rebuild.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Incremental Cabal consumer builds against independent fresh GHC splice builds.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + +EMBED_ROOT = Path(__file__).resolve().parents[2] +ROOT = EMBED_ROOT.parent +FIXTURES = EMBED_ROOT / 'test/fixtures' + + +def command(args, cwd, log, success=True): + result = subprocess.run(args, cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + log.write_text(result.stdout) + if success and result.returncode: + raise RuntimeError(f'Command failed; see {log}\n{result.stdout[-2500:]}') + return result + + +def main(): + subprocess.run(['cabal', 'build', 'clickhouse-migrate', 'clickhouse-migrate-embed', 'clickhouse-migrate-embed:exe:clickhouse-migrate-embed-build'], cwd=ROOT, check=True) + driver = subprocess.check_output(['cabal', 'list-bin', 'clickhouse-migrate-embed:exe:clickhouse-migrate-embed-build'], cwd=ROOT, text=True).strip() + evidence = Path(tempfile.mkdtemp(prefix='clickhouse-migrate-embedding-', dir='/tmp')) + fixture = evidence / 'consumer' + shutil.copytree(FIXTURES, fixture) + project = fixture / 'cabal.project' + project.write_text('packages:\n .\n ' + str(ROOT / 'clickhouse-migrate') + '\n ' + str(ROOT / 'clickhouse-migrate-embed') + '\nindex-state: 2026-10-15T00:00:00Z\n') + manifest = fixture / 'component/manifest.json' + seed = fixture / 'component/sql/seed.sql' + previous = None + count = 0 + + def check(label, changed=None): + nonlocal previous, count + count += 1 + log = evidence / f'{count:02d}-{label}' + build = [driver, '--input', str(fixture), 'component/manifest.json', '--', 'build', '--project-dir=' + str(fixture), 'exe:embedding-consumer'] + command(build, fixture, log.with_suffix('.cabal.log')) + binary = command(['cabal', 'list-bin', '--project-dir=' + str(fixture), 'exe:embedding-consumer'], fixture, log.with_suffix('.bin.log')).stdout.strip() + observed = command([binary], fixture, log.with_suffix('.run.log')).stdout + oracle_dir = evidence / f'oracle-{count}' + oracle_dir.mkdir() + oracle = oracle_dir / 'consumer' + command(['cabal', 'exec', '--', 'ghc', '--make', '-fforce-recomp', '-XGHC2024', + '-package', 'clickhouse-migrate', '-package', 'clickhouse-migrate-embed', + '-working-dir', str(fixture), 'Main.hs', '-outputdir', str(oracle_dir), '-o', str(oracle)], + ROOT, log.with_suffix('.oracle-build.log')) + expected = command([str(oracle)], fixture, log.with_suffix('.oracle-run.log')).stdout + if observed != expected: + raise RuntimeError(f'{label}: missed rebuild; evidence at {evidence}') + if changed is not None and (observed != previous) != changed: + raise RuntimeError(f'{label}: unexpected definition change; evidence at {evidence}') + previous = observed + print(f'{label}: incremental checksum/bytes match fresh build', flush=True) + return binary + + try: + check('baseline') + check('unchanged-input', False) + # Same-size, timestamp-preserving SQL edit is the hardest early rebuild case. + timestamp = seed.stat() + seed.write_bytes(seed.read_bytes().replace(b'(1)', b'(2)')) + os.utime(seed, ns=(timestamp.st_atime_ns, timestamp.st_mtime_ns)) + check('sql-preserved-time-and-size', True) + seed.write_bytes(seed.read_bytes() + b'\n') + check('sql-whitespace', True) + data = json.loads(manifest.read_text()) + data['migrations'][0]['steps'].reverse() + manifest.write_text(json.dumps(data)) + check('manifest-step-reorder', True) + extra = fixture / 'component/sql/extra.sql' + extra.write_bytes(b'INSERT INTO app.events VALUES (3)\n') + data['migrations'][0]['steps'].append({'name': 'extra', 'file': 'sql/extra.sql'}) + manifest.write_text(json.dumps(data)) + check('manifest-add-reference', True) + extra.write_bytes(extra.read_bytes().replace(b'(3)', b'(4)')) + check('new-reference-edit', True) + data['migrations'][0]['steps'].pop() + manifest.write_text(json.dumps(data)) + extra.unlink() + check('manifest-remove-reference', True) + manifest.write_text(json.dumps(data, indent=2)) + check('manifest-whitespace', False) + relocated = fixture / 'component/sql/relocated.sql' + seed.rename(relocated) + for step in data['migrations'][0]['steps']: + if step['file'] == 'sql/seed.sql': + step['file'] = 'sql/relocated.sql' + manifest.write_text(json.dumps(data, indent=2)) + check('sql-file-relocation', False) + timestamp = manifest.stat() + manifest.write_bytes(manifest.read_bytes().replace(b'analytics', b'analyticz')) + os.utime(manifest, ns=(timestamp.st_atime_ns, timestamp.st_mtime_ns)) + binary = check('manifest-preserved-time-and-size', True) + # Deployed artifact must execute without source and without env SQL replacement. + shutil.rmtree(fixture / 'component') + env = os.environ.copy() + env['CLICKHOUSE_MIGRATE_SQL'] = 'different SQL' + result = subprocess.run([binary], cwd=fixture, env=env, text=True, capture_output=True, check=True) + if result.stdout != previous: + raise RuntimeError('Artifact changed after removing sources') + print('source-free execution: exact compiled definitions retained', flush=True) + except BaseException: + print(f'Embedding evidence retained at {evidence}', flush=True) + raise + else: + shutil.rmtree(evidence) + print('All embedding rebuild cases passed', flush=True) + + +if __name__ == '__main__': + main()