diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,20 @@ # Changelog for settei-yaml -## 0.1.0.0 — 2026-07-18 +## 0.2.0.0 — 2026-07-19 +- Add `renderYamlErrorText` and `renderYamlErrorsText` for stable, operator-readable, + secret-safe adapter diagnostics. + +## 0.1.0.0 — 2026-07-19 + - Initial experimental release. - Add a strict YAML mapping with exact node locations, explicit unsupported-feature errors, and mounted-file Kubernetes annotations. +- Reject numeric scalars whose base-10 exponent magnitude exceeds 4096 instead of + attempting an unbounded exact conversion at load time. +- Restrict boolean scalars to the YAML 1.2 core schema: only case-insensitive `true` and + `false` are booleans, whether plain or tagged `!!bool`. The YAML 1.1 spellings `y`, + `yes`, `on`, `n`, `no`, and `off` are now plain text, eliminating the Norway problem. +- Contain all synchronous failures at the pure decode boundary; unexpected exceptions + become `YamlSyntaxError` with a fixed secret-safe message, while asynchronous exceptions + continue to propagate. diff --git a/settei-yaml.cabal b/settei-yaml.cabal --- a/settei-yaml.cabal +++ b/settei-yaml.cabal @@ -1,6 +1,6 @@ cabal-version: 3.8 name: settei-yaml -version: 0.1.0.0 +version: 0.2.0.0 synopsis: YAML sources for Settei description: Translate one strict YAML document into a provenance-aware Settei source @@ -46,7 +46,7 @@ , generic-lens >=2.2 && <2.4 , libyaml >=0.1.4 && <0.2 , scientific >=0.3.7 && <0.4 - , settei ==0.1.0.0 + , settei ==0.2.0.0 , text >=2.1 && <2.2 test-suite settei-yaml-tests @@ -65,8 +65,8 @@ , bytestring >=0.12 && <0.13 , containers >=0.6.8 && <0.8 , generic-lens >=2.2 && <2.4 - , settei ==0.1.0.0 - , settei-yaml ==0.1.0.0 + , settei ==0.2.0.0 + , settei-yaml ==0.2.0.0 , tasty >=1.5 && <1.6 , tasty-hunit >=0.10.2 && <0.11 , text >=2.1 && <2.2 diff --git a/src/Settei/Yaml.hs b/src/Settei/Yaml.hs --- a/src/Settei/Yaml.hs +++ b/src/Settei/Yaml.hs @@ -11,6 +11,8 @@ decodeYamlSource, fromKubernetesMountedFile, readYamlSource, + renderYamlErrorText, + renderYamlErrorsText, withYamlSourcePath, yamlErrorCategory, yamlErrorColumn, @@ -27,7 +29,16 @@ where import Control.Applicative ((<|>)) -import Control.Exception (IOException, displayException, try) +import Control.Exception + ( AsyncException, + IOException, + SomeAsyncException, + SomeException, + displayException, + fromException, + throwIO, + try, + ) import Control.Monad (when) import Data.Attoparsec.Text qualified as Attoparsec import Data.Bifunctor (first) @@ -40,7 +51,7 @@ import Data.List.NonEmpty qualified as NonEmpty import Data.Map.Strict qualified as Map import Data.Ratio qualified as Ratio -import Data.Scientific (Scientific) +import Data.Scientific (Scientific, base10Exponent) import Data.Text qualified as Text import Data.Text.Encoding qualified as TextEncoding import Settei @@ -145,6 +156,35 @@ yamlErrorMessage :: YamlSourceError -> Text yamlErrorMessage problem = problem ^. #message +-- | Render one YAML input failure as a single operator-readable line. +-- +-- The line leads with the stable source name, then a parenthetical built from the +-- colon-joined available location pieces (path, one-based line, one-based column), +-- then the structural context, then the fixed secret-safe message. Absent pieces +-- are omitted; when no piece is known the parenthetical disappears. No raw +-- configuration value can appear because 'YamlSourceError' retains none. +renderYamlErrorText :: YamlSourceError -> Text +renderYamlErrorText problem = + problem ^. #name + <> locationText + <> " at " + <> problem ^. #context + <> ": " + <> problem ^. #message + where + locationText = case locationPieces of + [] -> "" + pieces -> " (" <> Text.intercalate ":" pieces <> ")" + locationPieces = + maybe [] (pure . Text.pack) (problem ^. #path) + <> maybe [] (pure . Text.pack . show) (problem ^. #line) + <> maybe [] (pure . Text.pack . show) (problem ^. #column) + +-- | Render every YAML input failure, one line per problem, matching +-- 'Settei.Render.renderErrorsText' in shape and trailing newline. +renderYamlErrorsText :: NonEmpty YamlSourceError -> Text +renderYamlErrorsText = Text.unlines . fmap renderYamlErrorText . NonEmpty.toList + -- | Decode exactly one YAML mapping into a Settei file source. -- -- Duplicate keys, non-string keys, dotted keys, aliases, anchors, merge keys, custom @@ -187,9 +227,26 @@ decodeMarkedEvents :: YamlSourceOptions -> ByteString -> Either YamlSourceError [Libyaml.MarkedEvent] decodeMarkedEvents options bytes = unsafePerformIO $ do decoded <- - try @Libyaml.YamlException + try @SomeException (runConduitRes (Libyaml.decodeMarked bytes .| ConduitList.consume)) - pure $ first (syntaxError options) decoded + case decoded of + Right events -> pure (Right events) + Left exception + | Just (_ :: SomeAsyncException) <- fromException exception -> throwIO exception + | Just asynchronous <- fromException @AsyncException exception -> throwIO asynchronous + | Just yamlException <- fromException @Libyaml.YamlException exception -> + pure (Left (syntaxError options yamlException)) + | otherwise -> + pure + ( Left + ( yamlError + options + YamlSyntaxError + Nothing + [] + "YAML decoding failed with an unexpected error" + ) + ) {-# NOINLINE decodeMarkedEvents #-} syntaxError :: YamlSourceOptions -> Libyaml.YamlException -> YamlSourceError @@ -345,13 +402,14 @@ Libyaml.NullTag -> Right RawNull Libyaml.BoolTag -> parseBoolean options context marked value Libyaml.IntTag -> parseTaggedInteger options context marked value - Libyaml.FloatTag -> RawNumber . toRational <$> parseNumber options context marked value + Libyaml.FloatTag -> + RawNumber <$> (parseNumber options context marked value >>= scalarRational options context marked) Libyaml.NoTag | style `elem` [Libyaml.SingleQuoted, Libyaml.DoubleQuoted, Libyaml.Literal, Libyaml.Folded] -> Right (RawText value) | isNull value -> Right RawNull | Just boolean <- yamlBoolean value -> Right (RawBool boolean) - | Right number <- parseYamlNumber value -> Right (RawNumber (toRational number)) + | Right number <- parseYamlNumber value -> RawNumber <$> scalarRational options context marked number | otherwise -> Right (RawText value) _ -> Left (yamlError options YamlUnsupportedFeature (Just (startMark marked)) context "this YAML scalar tag is not supported") @@ -365,7 +423,7 @@ parseTaggedInteger :: YamlSourceOptions -> [PathPiece] -> Libyaml.MarkedEvent -> Text -> Either YamlSourceError RawValue parseTaggedInteger options context marked value = do number <- parseNumber options context marked value - let rational = toRational number + rational <- scalarRational options context marked number if Ratio.denominator rational == 1 then Right (RawNumber rational) else Left (yamlError options YamlInvalidScalar (Just (startMark marked)) context "integer tag requires a whole number") @@ -376,6 +434,34 @@ (const (yamlError options YamlInvalidScalar (Just (startMark marked)) context "invalid or non-finite numeric scalar")) (parseYamlNumber value) +-- | Largest absolute base-10 exponent accepted for a numeric scalar. +-- +-- 'toRational' materializes @10 ^ exponent@ as an exact 'Integer', so cost +-- grows with the exponent's magnitude rather than the input's length. This +-- adapter-local bound leaves core 'RawNumber' values unbounded. +maximumScalarExponent :: Int +maximumScalarExponent = 4096 + +scalarRational :: + YamlSourceOptions -> + [PathPiece] -> + Libyaml.MarkedEvent -> + Scientific -> + Either YamlSourceError Rational +scalarRational options context marked number + | scalarExponent < negate maximumScalarExponent || scalarExponent > maximumScalarExponent = + Left + ( yamlError + options + YamlInvalidScalar + (Just (startMark marked)) + context + "numeric scalar exponent is out of the supported range" + ) + | otherwise = Right (toRational number) + where + scalarExponent = base10Exponent number + parseYamlNumber :: Text -> Either String Scientific parseYamlNumber = Attoparsec.parseOnly (yamlNumberParser <* Attoparsec.endOfInput) @@ -391,13 +477,7 @@ yamlBoolean :: Text -> Maybe Bool yamlBoolean value = case Text.toCaseFold value of - "y" -> Just True - "yes" -> Just True - "on" -> Just True "true" -> Just True - "n" -> Just False - "no" -> Just False - "off" -> Just False "false" -> Just False _ -> Nothing diff --git a/test/Settei/YamlCharacterizationTest.hs b/test/Settei/YamlCharacterizationTest.hs --- a/test/Settei/YamlCharacterizationTest.hs +++ b/test/Settei/YamlCharacterizationTest.hs @@ -3,8 +3,11 @@ import Data.ByteString.Char8 qualified as ByteString8 import Data.Generics.Labels () import Data.List.NonEmpty qualified as NonEmpty +import Data.Ratio ((%)) +import Data.Text (Text) +import Settei import Settei.Yaml -import Test.Tasty (TestTree, testGroup) +import Test.Tasty (TestTree, localOption, mkTimeout, testGroup) import Test.Tasty.HUnit (assertBool, testCase, (@?=)) tests :: TestTree @@ -43,11 +46,74 @@ problem <- expectError "service: [one, two\n" yamlErrorCategory problem @?= YamlSyntaxError assertBool "syntax error omitted its line" (maybe False (> 0) (yamlErrorLine problem)) - assertBool "syntax error omitted its column" (maybe False (> 0) (yamlErrorColumn problem)) + assertBool "syntax error omitted its column" (maybe False (> 0) (yamlErrorColumn problem)), + testCase "tagged !!bool with a YAML 1.1 spelling fails as an invalid scalar" $ do + problem <- expectError "feature:\n enabled: !!bool yes\n" + yamlErrorCategory problem @?= YamlInvalidScalar + yamlErrorContext problem @?= "$.feature.enabled" + yamlErrorMessage problem @?= "invalid boolean scalar", + testCase "tagged !!bool true still parses" $ do + input <- expectSource "feature:\n enabled: !!bool true\n" + expectValue "feature.enabled" input (RawBool True), + localOption (mkTimeout 10000000) $ + testGroup + "bounded numeric scalar conversion" + [ testCase "huge positive exponents are rejected quickly" $ do + problem <- expectError "huge: 1e1000000000\n" + expectExponentError "$.huge" problem, + testCase "huge negative exponents are rejected quickly" $ do + problem <- expectError "tiny: 1e-1000000000\n" + expectExponentError "$.tiny" problem, + testCase "float-tagged numbers apply the exponent bound" $ do + problem <- expectError "huge: !!float 1e1000000000\n" + expectExponentError "$.huge" problem, + testCase "integer-tagged numbers apply the exponent bound" $ do + problem <- expectError "huge: !!int 1e1000000000\n" + expectExponentError "$.huge" problem, + testCase "positive boundary exponents convert exactly" $ do + input <- expectSource "edge: 1e4096\n" + expectValue "edge" input (RawNumber (10 ^ (4096 :: Integer) % 1)), + testCase "negative boundary exponents convert exactly" $ do + input <- expectSource "edge: 1e-4096\n" + expectValue "edge" input (RawNumber (1 % 10 ^ (4096 :: Integer))), + testCase "exponents immediately above the bound are rejected" $ do + problem <- expectError "over: 1e4097\n" + expectExponentError "$.over" problem, + testCase "ordinary decimals, hex, octal, and integers remain exact" $ do + input <- expectSource "rate: 1.5e-3\nmask: 0x1A\nmode: 0o17\nport: 8080\n" + expectValue "rate" input (RawNumber (3 % 2000)) + expectValue "mask" input (RawNumber 26) + expectValue "mode" input (RawNumber 15) + expectValue "port" input (RawNumber 8080) + ] ] +sourceOptions :: YamlSourceOptions +sourceOptions = withYamlSourcePath "characterization.yaml" (yamlSourceOptions "characterization") + +expectSource :: String -> IO Source +expectSource input = + case decodeYamlSource sourceOptions (ByteString8.pack input) of + Left errors -> fail (show errors) + Right sourceValue -> pure sourceValue + expectError :: String -> IO YamlSourceError expectError input = - case decodeYamlSource (withYamlSourcePath "characterization.yaml" (yamlSourceOptions "characterization")) (ByteString8.pack input) of + case decodeYamlSource sourceOptions (ByteString8.pack input) of Left errors -> pure (NonEmpty.head errors) Right _ -> fail "expected strict YAML characterization to fail" + +expectValue :: Text -> Source -> RawValue -> IO () +expectValue keyText input expected = case lookupSource (validKey keyText) input of + Right (Just found) -> assertBool "unexpected YAML raw value" (candidateValue found == expected) + _ -> fail "expected YAML candidate" + +expectExponentError :: Text -> YamlSourceError -> IO () +expectExponentError expectedContext problem = do + yamlErrorCategory problem @?= YamlInvalidScalar + yamlErrorLine problem @?= Just 1 + yamlErrorContext problem @?= expectedContext + yamlErrorMessage problem @?= "numeric scalar exponent is out of the supported range" + +validKey :: Text -> Key +validKey value = either (error . show) id (parseKey value) diff --git a/test/Settei/YamlTest.hs b/test/Settei/YamlTest.hs --- a/test/Settei/YamlTest.hs +++ b/test/Settei/YamlTest.hs @@ -17,7 +17,8 @@ tests = testGroup "Settei.Yaml" - [ testCase "nested mappings become segmented keys with exact locations" $ do + [ errorRenderingTests, + testCase "nested mappings become segmented keys with exact locations" $ do input <- expectSource "service:\n http:\n host: api.internal\n port: 8080\n" host <- expectCandidate serviceHttpHost input assertBool "host did not remain text" (candidateValue host == RawText "api.internal") @@ -47,17 +48,42 @@ assertBool "array shape changed" (candidateValue names == RawArray [RawText "one", RawText "two"]), + testCase "Norway regression: YAML 1.1 boolean spellings remain text" $ do + input <- + expectSource + "country: no\nfeature:\n legacy: yes\n toggle: on\n kill: off\n short: y\n tiny: n\n" + country <- expectCandidate countryKey input + assertCandidateValue "country did not remain text" country (RawText "no") + legacy <- expectCandidate legacyKey input + assertCandidateValue "legacy did not remain text" legacy (RawText "yes") + toggle <- expectCandidate toggleKey input + assertCandidateValue "toggle did not remain text" toggle (RawText "on") + kill <- expectCandidate killKey input + assertCandidateValue "kill did not remain text" kill (RawText "off") + short <- expectCandidate shortKey input + assertCandidateValue "short did not remain text" short (RawText "y") + tiny <- expectCandidate tinyKey input + assertCandidateValue "tiny did not remain text" tiny (RawText "n"), + testCase "core-schema booleans parse case-insensitively and quoted true stays text" $ do + input <- expectSource "plain: true\nupper: TRUE\nnegative: false\nquoted: \"true\"\n" + plain <- expectCandidate plainKey input + assertCandidateValue "plain true was not boolean" plain (RawBool True) + upper <- expectCandidate upperKey input + assertCandidateValue "uppercase true was not boolean" upper (RawBool True) + negative <- expectCandidate negativeKey input + assertCandidateValue "plain false was not boolean" negative (RawBool False) + quoted <- expectCandidate quotedKey input + assertCandidateValue "quoted true did not remain text" quoted (RawText "true"), testCase "higher YAML overrides leaves and replaces arrays wholesale" $ do low <- expectNamedSource "low" "service:\n host: old.internal\n port: 7000\n names: [one, two]\n" high <- expectNamedSource "high" "service:\n port: 9000\n names: [three]\n" - result <- - expectResolution - ( resolve + let result = + resolve defaultResolveOptions [low, high] ((,,) <$> required hostSetting <*> required portSetting <*> required namesSetting) - ) - result ^. #value @?= ("old.internal", 9000, ["three"]) + value <- expectResolution result + value @?= ("old.internal", 9000, ["three"]) case result ^. #report . #nodes . at servicePort of Just node -> do node ^. #origin . _Just . #name @?= "high" @@ -79,11 +105,17 @@ "unexpected invalid UTF-8 category" (yamlErrorCategory problem `elem` [YamlSyntaxError, YamlInvalidScalar]) Right _ -> fail "expected invalid UTF-8 to fail", + testCase "malformed input remains a positioned syntax error" $ do + problem <- expectError "key: [unclosed" + yamlErrorCategory problem @?= YamlSyntaxError + assertBool "expected a syntax-error line" (yamlErrorLine problem /= Nothing) + assertBool "expected a syntax-error column" (yamlErrorColumn problem /= Nothing), testCase "mounted Secret metadata remains visible while its setting is redacted" $ do let reference = kubernetesRef SecretObject (Just "production") "database-config" (Just "config.yaml") options = fromKubernetesMountedFile reference sourceOptions input <- expectSourceWith options ("database:\n password: " <> Text.unpack secretSentinel <> "\n") - result <- expectResolution (resolve defaultResolveOptions [input] (required passwordSetting)) + let result = resolve defaultResolveOptions [input] (required passwordSetting) + _ <- expectResolution result let textOutput = renderResolutionText (result ^. #report) jsonOutput = renderResolutionJson (result ^. #report) assertBool "secret reached text output" (not (secretSentinel `Text.isInfixOf` textOutput)) @@ -121,6 +153,85 @@ yamlErrorCategory problem @?= YamlTopLevelType ] +errorRenderingTests :: TestTree +errorRenderingTests = + testGroup + "error rendering" + [ rendererCase + "syntax" + YamlSyntaxError + "key: [unclosed" + "application (application.yaml:2:1) at $: while parsing a flow sequence: did not find expected ',' or ']'", + testCase "IO errors render a stable located prefix" $ do + result <- readYamlSource (yamlSourceOptions "missing") "test/fixtures/does-not-exist.yaml" + case result of + Left errors -> do + let problem = NonEmpty.head errors + yamlErrorCategory problem @?= YamlIoError + assertBool + "IO renderer omitted its stable prefix" + ( "missing (test/fixtures/does-not-exist.yaml) at $: " + `Text.isPrefixOf` renderYamlErrorText problem + ) + Right _ -> fail "expected missing YAML file to fail", + rendererCase + "multiple documents" + YamlMultipleDocuments + "---\nservice: one\n---\nservice: two\n" + "application (application.yaml:3:1) at $: multiple YAML documents are not supported", + rendererCase + "duplicate key" + YamlDuplicateKey + "service:\n port: 8000\n port: 9000\n" + "application (application.yaml:3:3) at $.service.port: duplicate mapping key", + rendererCase + "non-string key" + YamlNonStringKey + "1: value\n" + "application (application.yaml:1:1) at $: mapping keys must be strings", + rendererCase + "dotted key" + YamlDottedKey + "service.port: 8080\n" + "application (application.yaml:1:1) at $.service.port: mapping keys containing dots are not supported; use nested mappings", + rendererCase + "unsupported feature" + YamlUnsupportedFeature + "service: *shared\n" + "application (application.yaml:1:10) at $.service: YAML aliases are not supported", + rendererCase + "invalid scalar" + YamlInvalidScalar + "service:\n ratio: !!float .inf\n" + "application (application.yaml:2:10) at $.service.ratio: invalid or non-finite numeric scalar", + rendererCase + "top-level type" + YamlTopLevelType + "[one, two]\n" + "application (application.yaml) at $: the top-level YAML value must be a mapping", + testCase "a missing path omits it but retains a known position" $ do + let options = yamlSourceOptions "memory" + problem <- expectErrorWith options "key: [unclosed" + assertBool + "position-only YAML renderer omitted its location" + ("memory (2:1)" `Text.isPrefixOf` renderYamlErrorText problem), + testCase "a missing path and position omit the parenthetical" $ do + problem <- expectErrorWith (yamlSourceOptions "memory") "[one, two]\n" + renderYamlErrorText problem + @?= "memory at $: the top-level YAML value must be a mapping", + testCase "plural rendering is singular rendering plus a newline" $ do + problem <- expectError "service.port: 8080\n" + renderYamlErrorsText (NonEmpty.singleton problem) + @?= renderYamlErrorText problem <> "\n" + ] + +rendererCase :: String -> YamlErrorCategory -> String -> Text -> TestTree +rendererCase label expectedCategory input expected = + testCase label $ do + problem <- expectError input + yamlErrorCategory problem @?= expectedCategory + renderYamlErrorText problem @?= expected + sourceOptions :: YamlSourceOptions sourceOptions = withYamlSourcePath "application.yaml" (yamlSourceOptions "application") @@ -138,7 +249,10 @@ Right value -> pure value expectError :: String -> IO YamlSourceError -expectError input = case decodeYamlSource sourceOptions (ByteString8.pack input) of +expectError = expectErrorWith sourceOptions + +expectErrorWith :: YamlSourceOptions -> String -> IO YamlSourceError +expectErrorWith options input = case decodeYamlSource options (ByteString8.pack input) of Left errors -> pure (NonEmpty.head errors) Right _ -> fail "expected YAML decoding to fail" @@ -147,8 +261,12 @@ Right (Just found) -> pure found _ -> fail "expected YAML candidate" -expectResolution :: Either (NonEmpty ConfigError) a -> IO a -expectResolution = \case +assertCandidateValue :: String -> Candidate -> RawValue -> IO () +assertCandidateValue message found expected = + assertBool message (candidateValue found == expected) + +expectResolution :: ResolveResult a -> IO a +expectResolution result = case result ^. #answer of Left _ -> fail "expected YAML resolution to succeed" Right value -> pure value @@ -174,7 +292,7 @@ RawText value -> Right value _ -> Left (decodeFailure key "an array of text") -serviceHttpHost, serviceHttpPort, serviceHost, servicePort, serviceNames, serviceOptional, hugeNumber, ratioNumber, featureEnabled, featureNames, databasePassword :: Key +serviceHttpHost, serviceHttpPort, serviceHost, servicePort, serviceNames, serviceOptional, hugeNumber, ratioNumber, featureEnabled, featureNames, databasePassword, countryKey, legacyKey, toggleKey, killKey, shortKey, tinyKey, plainKey, upperKey, negativeKey, quotedKey :: Key serviceHttpHost = validKey "service.http.host" serviceHttpPort = validKey "service.http.port" serviceHost = validKey "service.host" @@ -186,6 +304,16 @@ featureEnabled = validKey "feature.enabled" featureNames = validKey "feature.names" databasePassword = validKey "database.password" +countryKey = validKey "country" +legacyKey = validKey "feature.legacy" +toggleKey = validKey "feature.toggle" +killKey = validKey "feature.kill" +shortKey = validKey "feature.short" +tinyKey = validKey "feature.tiny" +plainKey = validKey "plain" +upperKey = validKey "upper" +negativeKey = validKey "negative" +quotedKey = validKey "quoted" validKey :: Text -> Key validKey value = either (error . show) id (parseKey value) diff --git a/test/fixtures/characterization/README.md b/test/fixtures/characterization/README.md --- a/test/fixtures/characterization/README.md +++ b/test/fixtures/characterization/README.md @@ -9,7 +9,8 @@ duplicates fail at the second key. Syntax errors and successful values retain marks, which the public adapter reports as one-based positions. Multiple documents, anchors, aliases, merge keys, custom tags, and non-string keys fail explicitly. Scalars support null, -booleans, exact finite numbers, strings, arrays, and ordinary string-keyed mappings. +YAML 1.2 core-schema booleans (`true`/`false` only, case-insensitive), exact finite +numbers, strings, arrays, and ordinary string-keyed mappings. These fixtures are small review artifacts matching the inline executable cases: