packages feed

heni-hspec (empty) → 0.1.0.0

raw patch · 7 files changed

+295/−0 lines, 7 filesdep +QuickCheckdep +basedep +containers

Dependencies added: QuickCheck, base, containers, directory, filepath, heni, heni-hspec, hspec, hspec-api, hspec-core, process, text, unix

Files

+ CHANGELOG.md view
@@ -0,0 +1,5 @@+# 0.1.0.0 — 2026-10-24++## New Features++Initial optional framework reporter with stable test IDs and complete attempt-bound evidence. Framework dependencies remain outside the core package.
+ LICENSE view
@@ -0,0 +1,27 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+   list of conditions and the following disclaimer.+2. Redistributions in binary form must reproduce the above copyright notice,+   this list of conditions and the following disclaimer in the documentation+   and/or other materials provided with the distribution.+3. Neither the name of the copyright holder nor the names of its contributors+   may be used to endorse or promote products derived from this software+   without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,7 @@+# heni-hspec++Replace Hspec.hspec with heniHspec. Use heniProperty for native QuickCheck counterexample replay; its ID must match the enclosing full example ID. Outside Heni, normal framework behavior is preserved.++Add `heni-hspec ==0.1.0.0` to the test component and select `"adapter": "heni-v1"` in the Heni case. Complete attempt-bound records prove selected/executed IDs and named failures. Missing, stale or partial evidence cannot count as detection.++See the [adapter guide](https://github.com/shinzui/heni/blob/master/docs/design/adapters.md) and packaged test fixture for runnable examples.
+ heni-hspec.cabal view
@@ -0,0 +1,77 @@+cabal-version: 3.4+name: heni-hspec+version: 0.1.0.0+synopsis: Attempt-bound Hspec evidence for Heni+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+copyright: 2026 Nadeem Bitar+category: Testing+build-type: Simple+tested-with: ghc ==9.12.4+homepage: https://github.com/shinzui/heni+bug-reports: https://github.com/shinzui/heni/issues+description: Optional framework reporter for trustworthy named mutation evidence.+extra-doc-files:+  CHANGELOG.md+  README.md++common common+  default-language: GHC2024+  default-extensions:+    DuplicateRecordFields+    OverloadedStrings++  ghc-options: -Wall+  build-depends:+    base >=4.21.2 && <4.22,+    containers >=0.7 && <0.9,+    heni ==0.1.0.0,+    text >=2.1.4 && <2.2,++library+  import: common+  hs-source-dirs: src+  exposed-modules: Test.Heni.Hspec+  build-depends:+    QuickCheck >=2.19 && <2.20,+    hspec >=2.11.18 && <2.12,+    hspec-api >=2.11.18 && <2.12,+    hspec-core >=2.11.18 && <2.12,++executable heni-hspec-fixture+  import: common+  hs-source-dirs: test/fixture+  main-is: Main.hs+  build-depends:+    QuickCheck >=2.19 && <2.20,+    heni-hspec ==0.1.0.0,+    hspec >=2.11.18 && <2.12,+    hspec-api >=2.11.18 && <2.12,+    hspec-core >=2.11.18 && <2.12,++  ghc-options: -threaded++test-suite heni-hspec-test+  import: common+  type: exitcode-stdio-1.0+  hs-source-dirs: test+  main-is: Main.hs+  build-depends:+    directory >=1.3.10.1 && <1.4,+    filepath >=1.5.5 && <1.6,+    heni-hspec ==0.1.0.0,+    process >=1.6.26.1 && <1.7,+    unix >=2.8.8 && <2.9,++  build-tool-depends:+    heni-cli:heni,+    heni-hspec:heni-hspec-fixture,++  ghc-options: -threaded++source-repository head+  type: git+  location: https://github.com/shinzui/heni.git+  subdir: adapters/heni-hspec
+ src/Test/Heni/Hspec.hs view
@@ -0,0 +1,83 @@+{-# LANGUAGE CPP #-}++-- | Hspec formatter and an explicit QuickCheck replay helper. Replace @hspec@+-- with 'heniHspec'; framework dependencies stay outside Heni's core.+module Test.Heni.Hspec (heniHspec, heniProperty) where++import Control.Exception (throwIO)+import Control.Monad (unless, when)+import Data.Maybe (fromMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import System.Environment (getArgs, lookupEnv)+import System.IO (hPutStrLn, stderr)+import Test.Heni.Adapter+import Test.Heni.Expectation (ReplayEvidence (..))+import Test.Hspec qualified as H+import Test.Hspec.Api.Format.V2 qualified as F+import Test.Hspec.Core.Runner qualified as R+import Test.QuickCheck qualified as QC+import Test.QuickCheck.Random (QCGen)+import Text.Read (readMaybe)++-- | Native Hspec options are parsed first; the Heni selector is a stable-ID+-- substring intersected with that selection. History-based reruns are rejected+-- inside an attempt because they can execute a different test set.+heniHspec :: H.Spec -> IO ()+heniHspec spec =+  currentEvidenceAttempt >>= \case+    Nothing -> H.hspec spec+    Just attempt -> guardEvidenceAttempt attempt $ do+      (initial, forest) <- R.evalSpec R.defaultConfig spec+      args <- getArgs+      config <- R.readConfig initial args+      when (R.configRerun config || R.configRerunAllOnSuccess config) (throwIO (EvidenceError "Hspec history reruns are unsupported inside a Heni attempt"))+      when (R.configDryRun config) (throwIO (EvidenceError "Hspec dry-run does not prove execution inside a Heni attempt"))+      selector <- fmap Text.pack <$> lookupEnv "HENI_SELECTOR"+      let include path = fromMaybe (const True) (R.configFilterPredicate config) path && maybe True (`Text.isInfixOf` pathId path) selector+          finalConfig = F.useFormatter ("heni", formatter attempt) config {R.configFilterPredicate = Just include}+      R.runSpecForest forest finalConfig >>= R.evaluateResult++pathId :: F.Path -> Text+pathId (groups, name) = stableTestId (map Text.pack (groups ++ [name]))++formatter :: EvidenceAttempt -> F.FormatConfig -> IO F.Format+formatter attempt config = pure $ \case+  F.Done items -> do+    property <- readPropertyReplay attempt+    let ids = map (pathId . fst) items+        errors = ["Hspec completed a partial selection" | length items /= F.formatConfigExpectedTotalCount config]+        observation (path, item) = case F.itemResult item of+          F.Success -> TestPassed (pathId path)+          F.Pending {} -> TestSkipped (pathId path)+          F.Failure _ (F.Error _ e) -> TestSetupFailed (pathId path) (Text.pack (show e))+          F.Failure {} -> TestFailed (pathId path)+    emitSuite attempt (SuiteEvidence ("hspec/" <> Text.pack VERSION_hspec <> ";heni-hspec/0.1.0.0") Nothing ids (map observation items) errors (fst <$> property) (maybe [] snd property))+  _ -> pure ()++-- | The ID must equal the enclosing Hspec example's stable full ID. Records+-- QuickCheck's native seed/size, generator arguments and failing input. Exactly+-- one replay property is supported per selected attempt. Invalid replay data+-- makes the example pending, which can never satisfy a required reproduction.+heniProperty :: (QC.Testable property) => Text -> QC.Args -> property -> H.Expectation+heniProperty ident args property = do+  token <- lookupEnv "HENI_REPLAY_TOKEN"+  expectedGenerator <- lookupEnv "HENI_REPLAY_GENERATOR"+  let generator = Text.pack ("QuickCheck/" ++ VERSION_QuickCheck ++ ";" ++ show (QC.maxSuccess args, QC.maxDiscardRatio args, QC.maxSize args, QC.maxShrinks args))+      reject why = hPutStrLn stderr why >> H.pendingWith why+  replay <- case token of+    Nothing -> pure (QC.replay args)+    Just value -> do+      unless (expectedGenerator == Just (Text.unpack generator)) (reject "QuickCheck generator configuration mismatch")+      case readMaybe value :: Maybe (QCGen, Int) of+        Nothing -> reject "invalid native QuickCheck replay token" >> pure Nothing+        Just pair -> pure (Just pair)+  result <- QC.quickCheckWithResult (args {QC.chatty = False, QC.replay = replay, QC.maxSuccess = if token == Nothing then QC.maxSuccess args else 1}) property+  case result of+    QC.Success {} -> pure ()+    QC.Failure {QC.usedSeed, QC.usedSize, QC.theException, QC.failingTestCase, QC.output} -> do+      case theException of Just e -> throwIO e; Nothing -> pure ()+      recordPropertyReplay (ReplayEvidence (Text.pack (show (usedSeed, usedSize))) generator (Set.singleton ident)) (map Text.pack failingTestCase)+      H.expectationFailure output+    other -> H.pendingWith (QC.output other)
+ test/Main.hs view
@@ -0,0 +1,77 @@+module Main (main) where++import Control.Exception+import Control.Monad (forM_, unless)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text qualified as Text+import System.Directory qualified as Dir+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Posix.Temp (mkdtemp)+import System.Process (readProcessWithExitCode)+import System.Timeout (timeout)+import Test.Heni+import Test.Heni.Replay qualified as Replay++checked :: (Show e) => Either e a -> a+checked = either (error . show) id++assert' :: String -> Bool -> IO ()+assert' why good = unless good (error why)++main :: IO ()+main = timeout 120000000 checks >>= \case Nothing -> error "adapter integration watchdog"; Just () -> pure ()++checks :: IO ()+checks = bracket (Dir.getTemporaryDirectory >>= \p -> mkdtemp (p </> "heni-hspec-")) Dir.removePathForcibly $ \root -> do+  let source = root </> "source"+      exe = "heni-hspec-fixture"+      bid = checked (mkCommandId "build")+      tid = checked (mkCommandId "test")+      cid = checked (mkCaseId "named")+      command args = Command exe args Set.empty Nothing [] (checked (mkStageBudget 5000000)) (OutputPolicy 4096)+      definition mode selector intended replay =+        CaseDefinition+          cid+          "framework evidence"+          Set.empty+          [Replace (checked (mkRelativeTarget "input")) "anchor" mode 1]+          bid+          (Just tid)+          selector+          (NamedTests (Set.singleton intended) replay)+          StructuredTestsV1+      run name def hooks args =+        runCampaign+          (defaultRunOptions source (root </> name))+          (checked (mkCampaign (Map.fromList [(bid, command ["--heni-build"]), (tid, command args)]) [(def, hooks)]))+      is verdict RunResult {results = [CaseResult {verdict = actual}]} = actual == verdict+      is _ _ = False+  Dir.createDirectory source+  writeFile (source </> "input") "anchor"+  old <- Dir.getCurrentDirectory+  direct <- bracket_ (Dir.setCurrentDirectory source) (Dir.setCurrentDirectory old) (readProcessWithExitCode exe [] "")+  assert' "ordinary direct execution" (case direct of (ExitSuccess, _, _) -> True; _ -> False)+  forM_ [("intended", Caught), ("wrong", WrongFailure), ("setup", InfrastructureError)] $ \(mode, expected) -> do+    result <- run mode (definition (Text.pack mode) Nothing "suite/intended" NoReplay) noHooks []+    assert' (mode ++ ": " ++ show result) (is expected result)+  empty <- run "empty" (definition "intended" (Just "absent") "suite/intended" NoReplay) noHooks []+  assert' "empty selection baseline prevents candidate" (is NotRun empty)+  filtered <- run "filtered" (definition "intended" (Just "intended") "suite/intended" NoReplay) noHooks ["--match", "intended"]+  assert' ("native and Heni filters: " ++ show filtered) (is Caught filtered)+  dry <- run "dry-run" (definition "intended" Nothing "suite/intended" NoReplay) noHooks ["--dry-run"]+  assert' "dry-run cannot prove execution" (is InfrastructureError dry)+  skipped <- run "skipped" (definition "skipped" Nothing "suite/intended" NoReplay) noHooks []+  assert' "pending intended test is uncovered" (is Uncovered skipped)+  property <- run "property" (definition "property" (Just "property") "suite/property" RequireReplay) noHooks []+  assert' ("real QuickCheck reproduction: " ++ show property) (is Caught property)+  let badHooks = CaseHooks [] Nothing (Just (\e c -> pure (Right (corrupt (Replay.replayCommand e c)))))+      corrupt (Command e as s w env b o) = Command e as s w (("HENI_REPLAY_TOKEN", "invalid") : filter ((/= "HENI_REPLAY_TOKEN") . fst) env) b o+  wrongToken <- run "property-invalid" (definition "property" (Just "property") "suite/property" RequireReplay) badHooks []+  assert' ("invalid native QuickCheck token: " ++ show wrongToken) (is WrongFailure wrongToken)++  writeFile (root </> "manifest.json") "{\"schemaVersion\": 1, \"sourceRoot\": \"source\", \"commands\": {\"build\": {\"executable\": \"heni-hspec-fixture\", \"arguments\": [\"--heni-build\"]}, \"test\": {\"executable\": \"heni-hspec-fixture\"}}, \"cases\": [{\"id\": \"named\", \"description\": \"framework evidence\", \"build\": \"build\", \"test\": \"test\", \"adapter\": \"heni-v1\", \"edits\": [{\"kind\": \"replace\", \"path\": \"input\", \"old\": \"anchor\", \"new\": \"intended\"}], \"expect\": {\"kind\": \"named-tests\", \"failures\": [\"suite/intended\"]}}]}"+  cli <- readProcessWithExitCode "heni" ["run", root </> "manifest.json", "--output", root </> "cli", "--json"] ""+  assert' ("CLI adapter parity: " ++ show cli) (case cli of (ExitSuccess, _, _) -> True; _ -> False)+  putStrLn "hspec direct execution and named subprocess evidence passed"
+ test/fixture/Main.hs view
@@ -0,0 +1,19 @@+module Main (main) where++import System.Environment (getArgs)+import Test.Heni.Hspec+import Test.Hspec+import Test.QuickCheck qualified as QC++main :: IO ()+main =+  getArgs >>= \case+    ["--heni-build"] -> pure ()+    _ -> do+      mode <- readFile "input"+      heniHspec $ describe "suite" $ do+        it "intended" $ if mode == "skipped" then pending else mode `shouldNotBe` "intended"+        it "other" $ if mode == "setup" then ioError (userError "fixture setup failed") else mode `shouldNotBe` "wrong"+        it "property" $+          heniProperty "suite/property" (QC.stdArgs {QC.maxSuccess = 1, QC.maxSize = 30}) $+            QC.forAll (QC.chooseInt (1, 100)) (\n -> mode /= "property" || n < 0)