diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ ## [Unreleased] +## [0.6.0.1] - 2026-08-16 + +### Fixed + +- Profile diagnostics quote a non-ASCII frontmatter value as written. The six + messages that echo the offending value decoded Aeson's UTF-8 output as Latin-1, + so `東京` printed as `æ±äº¬`. `--json` output was unaffected. + +### Changed + +- Requires `okf-core ^>=0.6.0.1`, released alongside this version. The library + itself is unchanged from `0.6.0.0`. + ## [0.6.0.0] - 2026-08-11 ### Added diff --git a/okf-cli.cabal b/okf-cli.cabal --- a/okf-cli.cabal +++ b/okf-cli.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: okf-cli -version: 0.6.0.0 +version: 0.6.0.1 synopsis: Command-line interface for Open Knowledge Format bundles description: okf-cli provides the @okf@ executable for working with Open Knowledge Format @@ -69,7 +69,7 @@ , generic-lens >=2.2 && <2.4 , githash ^>=0.1 , lens ^>=5.3 - , okf-core ^>=0.6.0.0 + , okf-core ^>=0.6.0.1 , optparse-applicative >=0.18 && <0.20 , process >=1.6 && <1.7 , text ^>=2.1 @@ -81,11 +81,12 @@ main-is: Main.hs hs-source-dirs: test build-depends: + , aeson >=2.2 && <2.4 , base >=4.20 && <5 , directory , filepath , okf-cli - , okf-core ^>=0.6.0.0 + , okf-core ^>=0.6.0.1 , optparse-applicative >=0.18 , temporary , text ^>=2.1 diff --git a/src/Okf/Cli.hs b/src/Okf/Cli.hs --- a/src/Okf/Cli.hs +++ b/src/Okf/Cli.hs @@ -24,6 +24,7 @@ parserInfo, profileRegistryEnvVar, renderProfileDetail, + renderProfileViolation, renderRegistryTable, runCli, runCommand, @@ -35,6 +36,7 @@ import Data.Aeson qualified as Aeson import Data.Aeson.Key qualified as AesonKey import Data.Aeson.KeyMap qualified as KeyMap +import Data.ByteString.Lazy qualified as LazyBytes import Data.ByteString.Lazy.Char8 qualified as LazyByteString import Data.Foldable (toList, traverse_) import Data.List qualified as List @@ -43,6 +45,7 @@ import Data.Maybe (mapMaybe) import Data.Set qualified as Set import Data.Text qualified as Text +import Data.Text.Encoding qualified as Text.Encoding import Data.Text.IO qualified as Text.IO import Data.Time (defaultTimeLocale, formatTime, getCurrentTime, utctDay) import Okf.Actor (parseActor, renderActor) @@ -2131,9 +2134,27 @@ bundleValidationErrorIsAdvisory :: BundleValidationError -> Bool bundleValidationErrorIsAdvisory = not . bundleValidationErrorIsFailure +-- | A frontmatter value as JSON text, for a diagnostic that must show the author +-- exactly what it found. +-- +-- The decode step is the point. 'Aeson.encode' produces UTF-8 bytes, and turning +-- those into 'Text' with a @Char8@ unpack — which is a Latin-1 decode — renders +-- every non-ASCII value as mojibake: 東京 comes back as @æ±äº¬@. The lenient +-- decoder is used rather than the strict one because this is the renderer that +-- reports what went wrong with a document, and a partial function is the wrong +-- thing to put there even when its input is UTF-8 by construction. +renderJsonValue :: Aeson.Value -> Text +renderJsonValue = Text.Encoding.decodeUtf8Lenient . LazyBytes.toStrict . Aeson.encode + -- | One deviation as one line. The 'ProfileSpec' is here only so a missing -- required field can carry the profile's own explanation of what that field is -- for; every other case ignores it. +-- +-- Exported so a test can assert a whole diagnostic line rather than only the +-- accessors behind it, as 'computationReport' and 'renderProfileDetail' already +-- are. The constructors that quote a frontmatter value back to the author ignore +-- both the 'CompiledProfile' and the concept list, so such a test can pass any +-- compiled profile and an empty list. renderProfileViolation :: CompiledProfile -> [Concept] -> ProfileViolation -> Text renderProfileViolation compiled concepts = \case TypeNotInProfile cid ctype -> @@ -2169,7 +2190,7 @@ <> " must be one of [" <> Text.intercalate ", " allowed <> "], found: " - <> Text.pack (LazyByteString.unpack (Aeson.encode actual)) + <> renderJsonValue actual CardinalityMismatch cid fieldPath expected actual -> renderConceptId cid <> ": frontmatter cardinality at " @@ -2179,7 +2200,7 @@ <> ", found " <> valueCardinalityName actual <> ": " - <> Text.pack (LazyByteString.unpack (Aeson.encode actual)) + <> renderJsonValue actual ValueFormatMismatch cid fieldPath expected actual -> renderConceptId cid <> ": frontmatter value at " @@ -2187,7 +2208,7 @@ <> " must match format " <> renderFieldFormat expected <> ", found: " - <> Text.pack (LazyByteString.unpack (Aeson.encode actual)) + <> renderJsonValue actual DanglingHandleReference cid fieldPath handle -> renderConceptId cid <> ": " @@ -2208,7 +2229,7 @@ <> ": malformed document reference at " <> renderFieldPath fieldPath <> ": " - <> Text.pack (LazyByteString.unpack (Aeson.encode actual)) + <> renderJsonValue actual ExternalReferenceSchemeNotAllowed cid fieldPath actualScheme allowedSchemes -> renderConceptId cid <> ": external reference at " @@ -2239,7 +2260,7 @@ <> ": malformed path at " <> renderFieldPath fieldPath <> ": " - <> Text.pack (LazyByteString.unpack (Aeson.encode actual)) + <> renderJsonValue actual PathEscapesBundle cid fieldPath rawPath -> renderConceptId cid <> ": path at " @@ -2253,7 +2274,7 @@ <> ": frontmatter element at " <> renderFieldPath fieldPath <> " must be a record, found: " - <> Text.pack (LazyByteString.unpack (Aeson.encode actual)) + <> renderJsonValue actual PathPatternMismatch cid ctype patternText -> renderConceptId cid <> ": " <> ctype <> " must match path pattern: " <> patternText MissingResource cid ctype scheme -> diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -2,7 +2,10 @@ import Control.Exception (bracket) import Control.Monad (unless) +import Data.Aeson (Value (..), toJSON) +import Data.Foldable (traverse_) import Data.List qualified as List +import Data.List.NonEmpty (NonEmpty (..)) import Data.Text qualified as Text import Data.Text.IO qualified as Text.IO import Data.Time.Calendar (fromGregorian) @@ -15,10 +18,10 @@ import Okf.Cli.Fzf (Candidate (..), FzfOpts (..), optsToArgs, parseSelectionIndex, renderCandidateLines, shellQuote, withAnsi, withHeight, withNoSort, withPrompt) import Okf.Cli.Fzf.Selector (ConceptOrder (..), conceptCandidates, conceptPreviewCommand, orderConcepts, parseBundleSearchRoots) import Okf.Cli.Help (HelpTopic (..), helpTopics) -import Okf.ConceptId (parseConceptId, renderConceptId) +import Okf.ConceptId (ConceptId, parseConceptId, renderConceptId) import Okf.Document (Attester (..), Executor (..), Parameter (..), parseDocument) import Okf.Index (OkfVersion (..), VersionDeclaration (..), parseOkfVersion, readBundleVersion) -import Okf.Profile (Cardinality (..), FieldCondition (..), FieldFormat (..), FieldRule (..), FrontmatterRules (..), HandleReferenceRule (..), NestedFieldRule (..), NestedRules (..), PathReferenceRule (..), ProfileSpec (..), TypeRule (..), compileProfile, loadProfileFile, validateProfile, validateProfileVersion) +import Okf.Profile (Cardinality (..), CompiledProfile, FieldCondition (..), FieldFormat (..), FieldPath (..), FieldPathSegment (..), FieldRule (..), FrontmatterRules (..), HandleReferenceRule (..), NestedFieldRule (..), NestedRules (..), PathReferenceRule (..), ProfileSpec (..), ProfileViolation (..), TypeRule (..), compileProfile, loadProfileFile, validateProfile, validateProfileVersion) import Okf.Profile.Registry (RegistryEntry (..), defaultRegistryReference) import Okf.Query (ConceptFilter (..), FieldSelector (..), filterConcepts) import Okf.Validation (ValidationProfile (..), validateBundle) @@ -58,6 +61,7 @@ conceptsKeepsStatusDefaultOut <- testConceptsDoesNotApplyStatusDefault profileDocStrictWithTimestamp <- testProfileDocumentationStrictWithTimestamp conceptMenuOrdering <- testConceptMenuOrdering + nonAsciiDiagnostics <- testNonAsciiValuesSurviveDiagnostics let results = [ parseSucceeds ["validate", "bundle"], parseSucceeds ["validate", "bundle", "--strict"], @@ -440,6 +444,7 @@ conceptsKeepsStatusDefaultOut, profileDocStrictWithTimestamp, conceptMenuOrdering, + nonAsciiDiagnostics, configDefaults, configProjectPrecedence, configEnvPrecedence, @@ -498,6 +503,73 @@ case execParserPure defaultPrefs parserInfo args of Success (Options (Validate opts)) -> opts == expected _ -> False + +-- | Every profile diagnostic that quotes the offending frontmatter value must +-- quote it as the author wrote it. Six 'ProfileViolation' constructors carry a +-- raw 'Value' and print it; all six once turned Aeson's UTF-8 output into 'Text' +-- with a @Data.ByteString.Lazy.Char8@ unpack, which is a Latin-1 decode, so +-- @東京@ was reported as @æ±äº¬@ while the allowed values on the very same line +-- rendered correctly. +-- +-- The check is on the rendered line rather than on the helper behind it, so a +-- future constructor that reintroduces the unpack is caught rather than only a +-- helper nobody calls. The exact-line assertion on 'ValueNotInVocabulary' pins +-- the wording and the placement of the value as well as its encoding. +-- +-- 'samplePostgresqlProfile' is used for no reason beyond needing a +-- 'CompiledProfile' to pass: none of these six constructors consults it. +testNonAsciiValuesSurviveDiagnostics :: IO Bool +testNonAsciiValuesSurviveDiagnostics = + case (compileProfile samplePostgresqlProfile, parseConceptId "places/tokyo") of + (Left definitionErrors, _) -> + reportFailure ("the sample profile does not compile: " <> show definitionErrors) + (_, Left err) -> + reportFailure ("places/tokyo is not a concept id: " <> show err) + (Right compiled, Right cid) -> + case nonAsciiDiagnosticLines compiled cid of + [] -> reportFailure "no diagnostics were rendered at all" + rendered@(vocabularyLine : _) -> do + let mangled = filter (not . ("東京" `Text.isInfixOf`)) rendered + vocabularyMatches = vocabularyLine == expectedVocabularyLine + unless (null mangled) $ do + putStrLn "profile diagnostics mangled a non-ASCII value:" + traverse_ (Text.IO.putStrLn . (" " <>)) mangled + unless vocabularyMatches $ + putStrLn + ( "the vocabulary diagnostic did not render as expected:\n wanted: " + <> Text.unpack expectedVocabularyLine + <> "\n got: " + <> Text.unpack vocabularyLine + ) + pure (null mangled && vocabularyMatches) + where + reportFailure message = putStrLn message >> pure False + +-- | One rendered line per 'ProfileViolation' constructor that echoes a raw +-- frontmatter value, all carrying the same Japanese value. The vocabulary case +-- comes first because 'testNonAsciiValuesSurviveDiagnostics' also asserts it +-- whole. +nonAsciiDiagnosticLines :: CompiledProfile -> ConceptId -> [Text.Text] +nonAsciiDiagnosticLines compiled cid = + [ render (ValueNotInVocabulary cid prefecture ["東京都", "京都府"] japanese), + render (CardinalityMismatch cid prefecture List japanese), + render (ValueFormatMismatch cid prefecture Uri japanese), + render (MalformedDocumentReference cid prefecture japanese), + render (MalformedPathReference cid prefecture japanese), + render (NestedElementNotRecord cid nestedElement (toJSON ["東京" :: Text.Text])) + ] + where + render = renderProfileViolation compiled [] + japanese = String "東京" + prefecture = FieldPath (FieldName "prefecture" :| []) + nestedElement = FieldPath (FieldName "reviews" :| [ArrayIndex 0]) + +-- | The whole vocabulary diagnostic, exactly. Both halves of this line carry the +-- same characters, which is the point: before the fix the allowed values on the +-- left rendered correctly and the found value on the right did not. +expectedVocabularyLine :: Text.Text +expectedVocabularyLine = + "places/tokyo: frontmatter value at prefecture must be one of [東京都, 京都府], found: \"東京\"" -- | One root-level entry and one nested entry whose columns differ in width, so -- the padding in 'renderRegistryTable' is actually exercised, and the @(root)@