diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.1.1.0 + +- Accept probabilities rounded to two decimals, as returned by OpenRouter. + Distribution sums now tolerate `0.005` per returned probability, the selected + Choice may trail another option by a rounded tie (`0.01`), and Score checks + allow for rounded probabilities. Values are still not renormalized; clearly + invalid distributions are still rejected. +- Add `renderJevError`, a log-friendly rendering of `JevError` with the + constructor, status code, request ID, and message or truncated body. Unlike + `show`, it never includes response headers such as `Set-Cookie`. + ## 0.1.0.0 - Enforce `timeoutMicros` across the complete HTTP operation, including reading diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -10,7 +10,10 @@ bounds currently target that tested environment; other compiler versions have not been verified. You can use Cabal directly; Nix is optional for consumers. -To consume a local checkout, place it beside your application and use this +Jev is published on [Hackage](https://hackage.haskell.org/package/jev): add +`jev` to your `build-depends`, as in the example below, and run `cabal update`. + +To instead consume a local checkout, place it beside your application and use this `cabal.project` in the application directory: ```cabal @@ -31,7 +34,7 @@ executable jev-demo main-is: Main.hs - build-depends: base >= 4.20 && < 4.21, text >= 2.1 && < 2.2, jev == 0.1.0.0 + build-depends: base >= 4.20 && < 4.21, text >= 2.1 && < 2.2, jev ^>= 0.1.1.0 default-language: GHC2021 default-extensions: OverloadedStrings, OverloadedRecordDot ``` @@ -158,7 +161,7 @@ Question construction is pure; validation happens before any HTTP request. An entirely `pure` question or empty traversal contains no questions and is rejected. Choice accepts 1–255 options with unique wire labels; Score accepts 2–10 levels. No typeclass instances are required for Choice domain values. -Results expose constructors for pattern matching. Choice distributions pair domain values with probabilities in the supplied option order. Score distributions and legends use `IntMap`; supplied sparse distributions are preserved. The decoder checks finite probabilities in [0,1], distribution sums within `0.001` of one, a maximal selected Choice probability within `0.001`, and a Score within `0.001 * numberOfLevels` of its probability-weighted value. Legends must cover every returned probability index. Values are never renormalized or filled in. Confidence is retained as a separate provider measure in [0,1], not recomputed from the distribution. Public constructors allow manually created values that bypass these checks. +Results expose constructors for pattern matching. Choice distributions pair domain values with probabilities in the supplied option order. Score distributions and legends use `IntMap`; supplied sparse distributions are preserved. The decoder checks finite probabilities in [0,1] and tolerates providers that round them to two decimals, as OpenRouter does: distribution sums within `0.001 + 0.005 * numberOfReturnedProbabilities` of one, a maximal selected Choice probability within `0.011` (a rounded tie), and a Score within `0.001 * numberOfLevels + 0.005 * (1 + sumOfReturnedIndices)` of its probability-weighted value. Legends must cover every returned probability index. Values are never renormalized or filled in. Confidence is retained as a separate provider measure in [0,1], not recomputed from the distribution. Public constructors allow manually created values that bypass these checks. `Response`, `Choice`, `Option`, `JsonOption`, and `NoulCriteria` have `Functor` instances. Mapping a `Choice` transforms both the selection and the values in its @@ -202,6 +205,10 @@ - `ResponseDecodeError metadata message`: a 2xx response could not be decoded; the same metadata is retained, including the body ID when readable or header ID. - `DecodeError message`: decoding a fixture through `decodeResponse` failed. + +`renderJevError err` produces log-friendly `Text` with the constructor, status code, +request ID, and message or body (truncated to 500 bytes). It never includes +response headers such as `Set-Cookie`, unlike `show err`, which prints them in full. Asynchronous cancellation propagates normally. Exceptions from custom manager hooks or user functions mapped over questions are not generally converted to diff --git a/jev.cabal b/jev.cabal --- a/jev.cabal +++ b/jev.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: jev -version: 0.1.0.0 +version: 0.1.1.0 synopsis: Typed decisions with Jev through TypeSafe and OpenRouter description: Typed, composable Jev questions with reusable HTTPS clients. homepage: https://github.com/realbogart/jev diff --git a/lib/Jev/Internal/Protocol.hs b/lib/Jev/Internal/Protocol.hs --- a/lib/Jev/Internal/Protocol.hs +++ b/lib/Jev/Internal/Protocol.hs @@ -81,8 +81,12 @@ probabilityTolerance :: Double probabilityTolerance = 1e-3 +-- Maximum error of one probability rounded to two decimals, as OpenRouter returns them. +roundingTolerance :: Double +roundingTolerance = 5e-3 + distribution :: [Double] -> Parser () -distribution values = unless (abs (sum values - 1) <= probabilityTolerance) (fail "Probabilities must sum to approximately one") +distribution values = unless (abs (sum values - 1) <= probabilityTolerance + roundingTolerance * fromIntegral (length values)) (fail "Probabilities must sum to approximately one") choiceQuestion :: Value -> [JsonOption a] -> Question (Choice a) choiceQuestion instructions options = primitive "choice" instructions (Just criteria) check $ \o -> do @@ -100,7 +104,7 @@ options distribution (map snd probabilities) selectedProbability <- maybe (fail "Missing selected probability") pure (Map.lookup label values) - unless (all (\p -> p <= selectedProbability + probabilityTolerance) (Map.elems values)) (fail "Selected choice is not a highest-probability option") + unless (all (\p -> p <= selectedProbability + probabilityTolerance + 2 * roundingTolerance) (Map.elems values)) (fail "Selected choice is not a highest-probability option") pure (Choice selected confidence probabilities) where mapping = Map.fromList [(key, a) | JsonOption a key _ <- options] @@ -122,7 +126,9 @@ distribution (IM.elems probabilities) unless (all (`IM.member` legend) (IM.keys probabilities)) (fail "Score legend is missing probability indices") let expected = sum [fromIntegral i * p | (i, p) <- IM.toList probabilities] - unless (abs (score - expected) <= probabilityTolerance * fromIntegral (length levels)) (fail "Score does not match its probability-weighted rubric") + -- The score and each index-weighted probability may carry a rounding error. + roundingError = roundingTolerance * (1 + fromIntegral (sum (IM.keys probabilities))) + unless (abs (score - expected) <= probabilityTolerance * fromIntegral (length levels) + roundingError) (fail "Score does not match its probability-weighted rubric") pure (Score score confidence probabilities legend) where check _ = do diff --git a/lib/Jev/Types.hs b/lib/Jev/Types.hs --- a/lib/Jev/Types.hs +++ b/lib/Jev/Types.hs @@ -18,13 +18,17 @@ ResponseMetadata (..), TransportFailure (..), JevError (..), + renderJevError, ) where import Data.Aeson (Value) import Data.ByteString.Lazy qualified as LBS import Data.IntMap.Strict (IntMap) +import Data.Maybe (fromMaybe) import Data.Text (Text) +import Data.Text qualified as T +import Data.Text.Encoding (decodeUtf8Lenient) import Network.HTTP.Types.Header (ResponseHeaders) -- | The gateway used for validation and the default endpoint. @@ -205,3 +209,19 @@ | -- | A successful HTTP response contained an invalid answer. ResponseDecodeError ResponseMetadata Text deriving (Eq, Show) + +-- | Log-friendly description with the constructor, status code, request ID, and +-- message or body (truncated to 500 bytes). Never includes response headers. +-- Error bodies are server-controlled and may still contain sensitive data. +renderJevError :: JevError -> Text +renderJevError err = case err of + ValidationError message -> "ValidationError: " <> message + TransportError failure -> "TransportError: " <> T.pack (show failure) + HttpError metadata body -> "HttpError" <> context metadata <> ": " <> truncated body + DecodeError message -> "DecodeError: " <> message + ResponseDecodeError metadata message -> "ResponseDecodeError" <> context metadata <> ": " <> message + where + context metadata = " (status " <> T.pack (show metadata.statusCode) <> ", request ID " <> fromMaybe "unknown" metadata.requestId <> ")" + truncated body + | LBS.length body > 500 = decodeUtf8Lenient (LBS.toStrict (LBS.take 500 body)) <> "... (truncated)" + | otherwise = decodeUtf8Lenient (LBS.toStrict body) diff --git a/test/JevSpec.hs b/test/JevSpec.hs --- a/test/JevSpec.hs +++ b/test/JevSpec.hs @@ -212,6 +212,27 @@ decodeFixture rubric (replace "score" (Number 0) scoreAnswer) `shouldSatisfy` isDecodeError decodeFixture rubric (replace "legend" (object []) scoreAnswer) `shouldSatisfy` isDecodeError fmap (\r -> r.answers.probabilities) (decodeFixture rubric scoreAnswer) `shouldBe` Right (IM.fromList [(1, 0.6), (2, 0.4)]) + it "accepts distributions rounded to two decimals and rejects clearly invalid ones" $ do + let letters n = choice "Pick" [Option c (T.singleton c) Nothing | c <- take n ['a' ..]] + rounded selected values = object ["type" .= ("choice" :: Text), "choice" .= selected, "confidence" .= (0.5 :: Double), "probabilities" .= object values] + decodeFixture question value = do + prepared <- prepareRequest OpenRouter "model" (String "State") question + fmap (\r -> r.answers.choice) (decodeResponse prepared (encode (envelope (object ["q0" .= value])))) + decodeFixture (letters 3) (rounded ("a" :: Text) ["a" .= (0.33 :: Double), "b" .= (0.33 :: Double), "c" .= (0.33 :: Double)]) `shouldBe` Right 'a' + decodeFixture (letters 4) (rounded ("a" :: Text) ["a" .= (0.26 :: Double), "b" .= (0.25 :: Double), "c" .= (0.25 :: Double), "d" .= (0.25 :: Double)]) `shouldBe` Right 'a' + decodeFixture (letters 3) (rounded ("a" :: Text) ["a" .= (0.36 :: Double), "b" .= (0.37 :: Double), "c" .= (0.27 :: Double)]) `shouldBe` Right 'a' + decodeFixture route (rounded ("billing" :: Text) ["billing" .= (0.85 :: Double), "technical" .= (0.05 :: Double)]) `shouldSatisfy` isDecodeError + decodeFixture route (rounded ("billing" :: Text) ["billing" .= (0.9 :: Double), "technical" .= (0.3 :: Double)]) `shouldSatisfy` isDecodeError + decodeFixture (letters 3) (rounded ("a" :: Text) ["a" .= (0.34 :: Double), "b" .= (0.36 :: Double), "c" .= (0.3 :: Double)]) `shouldSatisfy` isDecodeError + describe "error rendering" $ do + it "includes status, request ID, and message but never headers" $ do + let metadata = ResponseMetadata 503 [("Set-Cookie", "session=secret-cookie"), ("x-typesafe-request-id", "req-1")] (Just "req-1") + httpText = renderJevError (HttpError metadata "upstream unavailable") + decodeText = renderJevError (ResponseDecodeError metadata "Error in $.q0: Probabilities must sum to approximately one") + httpText `shouldBe` "HttpError (status 503, request ID req-1): upstream unavailable" + decodeText `shouldBe` "ResponseDecodeError (status 503, request ID req-1): Error in $.q0: Probabilities must sum to approximately one" + mapM_ (\rendered -> rendered `shouldSatisfy` (not . T.isInfixOf "secret-cookie")) [httpText, decodeText] + T.length (renderJevError (HttpError metadata (LBS.replicate 10000 120))) `shouldSatisfy` (< 600) it "retains retry headers and request IDs on HTTP and decoding failures" $ do let app status body _ respond = respond (responseLBS status [("Retry-After", "7"), ("x-typesafe-request-id", "header-id")] body) failed <- withServer (app status429 "limited") $ \client -> decide client "State" (noul "Yes?")