diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ Changes ======= +Version 2.3.6 +------------- + +* Option `--no-create-file` now available internally as `NoCreateFile` + ([Issue #50](https://github.com/UnkindPartition/tasty-golden/issues/50)) +* Drop support for GHC 7, remove obsolete `deriving Typeable` +* Tested with GHC 8.0 - 9.14.1 + +_Andreas Abel, 2026-02-01_ + +Version 2.3.5 +------------- + +* Fixes for launching external processes (like `diff`) on Windows +* Update the golden file on `--accept` if decoding the golden file failed with an exception +* Do not depend on `unix-compat` + Version 2.3.4 ------------- @@ -64,12 +81,12 @@ Version 2.3.0.2 --------------- -Switch from temporary-rc to temporary +Switch from `temporary-rc` to `temporary` Version 2.3.0.1 --------------- -Impose a lower bound version constraint on bytestring. +Impose a lower bound version constraint on `bytestring`. Version 2.3 ----------- diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +This package provides support for «golden testing». + +A golden test is an IO action that writes its result to a file. +To pass the test, this output file should be identical to the corresponding +«golden» file, which contains the correct result for the test. + +To **get started** with golden testing and this library, see +[Introduction to golden testing](https://ro-che.info/articles/2017-12-04-golden-tests). + +Command-line options +-------------------- + +To see the command-line options, run your test suite with `--help`. Here's an +example output: + +``` +Mmm... tasty test suite + +Usage: test [-p|--pattern PATTERN] [-t|--timeout DURATION] [-l|--list-tests] + [-j|--num-threads NUMBER] [-q|--quiet] [--hide-successes] + [--color never|always|auto] [--ansi-tricks ARG] [--accept] + [--no-create] [--size-cutoff n] + [--delete-output never|onpass|always] + +Available options: + -h,--help Show this help text + -p,--pattern PATTERN Select only tests which satisfy a pattern or awk + expression + -t,--timeout DURATION Timeout for individual tests (suffixes: ms,s,m,h; + default: s) + -l,--list-tests Do not run the tests; just print their names + -j,--num-threads NUMBER Number of threads to use for tests + execution (default: # of cores/capabilities) + -q,--quiet Do not produce any output; indicate success only by + the exit code + --hide-successes Do not print tests that passed successfully + --color never|always|auto + When to use colored output (default: auto) + --ansi-tricks ARG Enable various ANSI terminal tricks. Can be set to + 'true' or 'false'. (default: true) + --accept Accept current results of golden tests + --no-create Error when golden file does not exist + --size-cutoff n hide golden test output if it's larger than n + bytes (default: 1000) + --delete-output never|onpass|always + If there is a golden file, when to delete output + files (default: never) +``` + +See also [tasty's README](https://github.com/UnkindPartition/tasty/blob/master/README.md#runtime). + +Maintainers +----------- + +[Roman Cheplyaka](https://github.com/UnkindPartition) is the primary maintainer. + +[Oliver Charles](https://github.com/ocharles) is the backup maintainer. Please +get in touch with him if the primary maintainer cannot be reached. diff --git a/Test/Tasty/Golden.hs b/Test/Tasty/Golden.hs --- a/Test/Tasty/Golden.hs +++ b/Test/Tasty/Golden.hs @@ -1,18 +1,23 @@ {- | + +== Getting Started To get started with golden testing and this library, see . This module provides a simplified interface. If you want more, see "Test.Tasty.Golden.Advanced". -Note about filenames. They are looked up in the usual way, thus relative +== Filenames +Filenames are looked up in the usual way, Thus relative names are relative to the processes current working directory. It is common to run tests from the package's root directory (via @cabal test@ or @cabal install --enable-tests@), so if your test files are under the @tests\/@ subdirectory, your relative file names should start with @tests\/@ (even if your @test.hs@ is itself under @tests\/@, too). -Note about line endings. The best way to avoid headaches with line endings +== Line endings + +The best way to avoid headaches with line endings (when running tests both on UNIX and Windows) is to treat your golden files as binary, even when they are actually textual. @@ -43,8 +48,28 @@ As a last resort, you can strip all @\\r@s from both arguments in your comparison function when necessary. But most of the time treating the files as binary does the job. + +== Linking +The test suite should be compiled with @-threaded@ if you want to avoid +blocking any other threads while 'goldenVsFileDiff' and similar functions +wait for the result of the diff command. + +== Windows limitations +When using 'goldenVsFileDiff' or 'goldenVsStringDiff' under Windows the exit +code from the diff program that you specify will not be captured correctly +if that program uses @exec@. + +More specifically, you will get the exit code of the /original child/ +(which always exits with code 0, since it called @exec@), not the exit +code of the process which carried on with execution after @exec@. +This is different from the behavior prescribed by POSIX but is the best +approximation that can be realised under the restrictions of the +Windows process model. See 'System.Process' for further details or + for even more. + -} +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} module Test.Tasty.Golden ( @@ -56,6 +81,7 @@ -- * Options , SizeCutoff(..) , DeleteOutputFile(..) + , NoCreateFile(..) -- * Various utilities , writeBinaryFile , findByExtension @@ -68,19 +94,21 @@ import Test.Tasty.Golden.Internal import Text.Printf import qualified Data.ByteString.Lazy as LBS -import Data.Monoid import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.Encoding as LT import System.IO import System.IO.Temp -import System.Process +import qualified System.Process.Typed as PT import System.Exit import System.FilePath import System.Directory -import System.PosixCompat.Files import Control.Exception import Control.Monad import qualified Data.Set as Set +import Foreign.C.Error +#if !MIN_VERSION_base(4,11,0) +import Data.Monoid +#endif -- | Compare the output file's contents against the golden file's contents -- after the given action has created the output file. @@ -129,6 +157,9 @@ return $ if x == y then Nothing else Just e -- | Same as 'goldenVsFile', but invokes an external diff command. +-- +-- See the notes at the top of this module regarding linking with +-- @-threaded@ and Windows-specific issues. goldenVsFileDiff :: TestName -- ^ test name -> (FilePath -> FilePath -> [String]) @@ -146,33 +177,28 @@ askOption $ \sizeCutoff -> goldenTest2 name - (getFileStatus ref >> return ()) - -- Use getFileStatus to check if the golden file exists. If the file - -- doesn't exist, getFileStatus will throw an isDoesNotExistError that - -- runGolden will handle by creating the golden file before proceeding. - -- See #32. + (throwIfDoesNotExist ref) act - (cmp sizeCutoff) + (\_ _ -> runDiff (cmdf ref new) sizeCutoff) upd del where - cmd = cmdf ref new - cmp sizeCutoff _ _ - | null cmd = error "goldenVsFileDiff: empty command line" - | otherwise = do - (_, Just sout, _, pid) <- createProcess (proc (head cmd) (tail cmd)) { std_out = CreatePipe } - -- strictly read the whole output, so that the process can terminate - out <- hGetContentsStrict sout - - r <- waitForProcess pid - return $ case r of - ExitSuccess -> Nothing - _ -> Just . unpackUtf8 . truncateLargeOutput sizeCutoff $ out - upd _ = readFileStrict new >>= createDirectoriesAndWriteFile ref del = removeFile new +-- If the golden file doesn't exist, throw an isDoesNotExistError that +-- runGolden will handle by creating the golden file before proceeding. +-- See #32. +throwIfDoesNotExist :: FilePath -> IO () +throwIfDoesNotExist ref = do + exists <- doesFileExist ref + unless exists $ ioError $ + errnoToIOError "goldenVsFileDiff" eNOENT Nothing Nothing + -- | Same as 'goldenVsString', but invokes an external diff command. +-- +-- See the notes at the top of this module regarding linking with +-- @-threaded@ and Windows-specific issues. goldenVsStringDiff :: TestName -- ^ test name -> (FilePath -> FilePath -> [String]) @@ -201,17 +227,10 @@ LBS.hPut tmpHandle actBS >> hFlush tmpHandle let cmd = cmdf ref tmpFile - - when (null cmd) $ error "goldenVsFileDiff: empty command line" - - (_, Just sout, _, pid) <- createProcess (proc (head cmd) (tail cmd)) { std_out = CreatePipe } - -- strictly read the whole output, so that the process can terminate - out <- hGetContentsStrict sout + diff_result :: Maybe String <- runDiff cmd sizeCutoff - r <- waitForProcess pid - return $ case r of - ExitSuccess -> Nothing - _ -> Just (printf "Test output was different from '%s'. Output of %s:\n" ref (show cmd) <> unpackUtf8 (truncateLargeOutput sizeCutoff out)) + return $ flip fmap diff_result $ \diff -> + printf "Test output was different from '%s'. Output of %s:\n" ref (show cmd) <> diff upd = createDirectoriesAndWriteFile ref @@ -299,12 +318,24 @@ evaluate $ forceLbs s return s -hGetContentsStrict :: Handle -> IO LBS.ByteString -hGetContentsStrict h = do - hSetBinaryMode h True - s <- LBS.hGetContents h - evaluate $ forceLbs s - return s - unpackUtf8 :: LBS.ByteString -> String unpackUtf8 = LT.unpack . LT.decodeUtf8 + +runDiff + :: [String] -- ^ the diff command + -> SizeCutoff + -> IO (Maybe String) +runDiff cmd sizeCutoff = + case cmd of + [] -> throwIO $ ErrorCall "tasty-golden: empty diff command" + prog : args -> do + let + procConf = + PT.setStdin PT.closed + . PT.setStderr PT.inherit + $ PT.proc prog args + + (exitCode, out) <- PT.readProcessStdout procConf + return $ case exitCode of + ExitSuccess -> Nothing + _ -> Just . unpackUtf8 . truncateLargeOutput sizeCutoff $ out diff --git a/Test/Tasty/Golden/Internal.hs b/Test/Tasty/Golden/Internal.hs --- a/Test/Tasty/Golden/Internal.hs +++ b/Test/Tasty/Golden/Internal.hs @@ -1,11 +1,10 @@ -{-# LANGUAGE RankNTypes, ExistentialQuantification, DeriveDataTypeable, +{-# LANGUAGE RankNTypes, ExistentialQuantification, MultiParamTypeClasses, GeneralizedNewtypeDeriving, CPP #-} module Test.Tasty.Golden.Internal where import Control.DeepSeq import Control.Exception import Control.Monad (when) -import Data.Typeable (Typeable) import Data.Proxy import Data.Int import Data.Char (toLower) @@ -26,12 +25,11 @@ (a -> a -> IO (Maybe String)) (a -> IO ()) (IO ()) - deriving Typeable -- | This option, when set to 'True', specifies that we should run in the -- «accept tests» mode newtype AcceptTests = AcceptTests Bool - deriving (Eq, Ord, Typeable) + deriving (Eq, Ord) instance IsOption AcceptTests where defaultValue = AcceptTests False parseValue = fmap AcceptTests . safeReadBool @@ -41,8 +39,10 @@ -- | This option, when set to 'True', specifies to error when a file does -- not exist, instead of creating a new file. +-- +-- @since 2.3.6 newtype NoCreateFile = NoCreateFile Bool - deriving (Eq, Ord, Typeable) + deriving (Eq, Ord) instance IsOption NoCreateFile where defaultValue = NoCreateFile False parseValue = fmap NoCreateFile . safeReadBool @@ -55,8 +55,10 @@ -- for readability. -- -- The default value is 1000 (i.e. 1Kb). +-- +-- @since 2.3.3 newtype SizeCutoff = SizeCutoff { getSizeCutoff :: Int64 } - deriving (Eq, Ord, Typeable, Num, Real, Enum, Integral) + deriving (Eq, Ord, Num, Real, Enum, Integral) instance IsOption SizeCutoff where defaultValue = 1000 showDefaultValue = Just . show . getSizeCutoff @@ -74,7 +76,7 @@ | OnPass -- ^ Delete the output file if the test passes | Always -- ^ Always delete the output file. (May not be commonly used, -- but provided for completeness.) - deriving (Eq, Ord, Typeable, Show) + deriving (Eq, Ord, Show) -- | This option controls when / whether the test output file is deleted -- For example, it may be convenient to delete the output file when a test @@ -127,18 +129,29 @@ mbRef <- try getGolden case mbRef of - Left e | isDoesNotExistError e -> - if noCreate - then - -- Don't ever delete the output file in this case, as there is - -- no duplicate golden file - return $ testFailed "Golden file does not exist; --no-create flag specified" - else do + Left e + | Just e' <- fromException e, isDoesNotExistError e' -> + if noCreate + then + -- Don't ever delete the output file in this case, as there is + -- no duplicate golden file + return $ testFailed "Golden file does not exist; --no-create flag specified" + else do + update new + when (delOut `elem` [Always, OnPass]) delete + return $ testPassed "Golden file did not exist; created" + + | Just (_ :: AsyncException) <- fromException e -> throwIO e + | Just (_ :: IOError) <- fromException e -> throwIO e + + + | otherwise -> do + -- Other types of exceptions may be due to failing to decode the + -- golden file. In that case, it makes sense to replace a broken + -- golden file with the current version. update new when (delOut `elem` [Always, OnPass]) delete - return $ testPassed "Golden file did not exist; created" - - | otherwise -> throwIO e + return $ testPassed $ "Accepted the new version. Was failing with exception:\n" ++ show e Right ref -> do diff --git a/Test/Tasty/Golden/Manage.hs b/Test/Tasty/Golden/Manage.hs --- a/Test/Tasty/Golden/Manage.hs +++ b/Test/Tasty/Golden/Manage.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE GeneralizedNewtypeDeriving, PatternGuards, DeriveDataTypeable #-} +{-# LANGUAGE GeneralizedNewtypeDeriving, PatternGuards #-} -- | Previously, accepting tests (by the @--accept@ flag) was done by this -- module, and there was an ingredient to handle that mode. -- diff --git a/tasty-golden.cabal b/tasty-golden.cabal --- a/tasty-golden.cabal +++ b/tasty-golden.cabal @@ -1,5 +1,6 @@ +cabal-version: 2.0 name: tasty-golden -version: 2.3.4 +version: 2.3.6 synopsis: Golden tests support for tasty description: This package provides support for «golden testing». @@ -13,35 +14,48 @@ license: MIT license-file: LICENSE -Homepage: https://github.com/feuerbach/tasty-golden -Bug-reports: https://github.com/feuerbach/tasty-golden/issues +Homepage: https://github.com/UnkindPartition/tasty-golden +Bug-reports: https://github.com/UnkindPartition/tasty-golden/issues author: Roman Cheplyaka maintainer: Roman Cheplyaka -- copyright: category: Testing build-type: Simple -cabal-version: >=1.14 -extra-source-files: + +extra-doc-files: CHANGELOG.md + README.md + +extra-source-files: example/golden/fail/*.golden example/golden/success/*.golden tests/golden/*.golden -Tested-With: - GHC ==7.8.4 || - ==7.10.3 || - ==8.0.2 || - ==8.2.2 || - ==8.4.4 || - ==8.6.5 || - ==8.8.2 +tested-with: + GHC == 9.14.1 + GHC == 9.12.2 + GHC == 9.10.3 + GHC == 9.8.4 + GHC == 9.6.7 + GHC == 9.4.8 + GHC == 9.2.8 + GHC == 9.0.2 + GHC == 8.10.7 + GHC == 8.8.4 + GHC == 8.6.5 + GHC == 8.4.4 + GHC == 8.2.2 + GHC == 8.0.2 + Source-repository head type: git - location: git://github.com/feuerbach/tasty-golden.git + location: https://github.com/UnkindPartition/tasty-golden.git library Default-language: Haskell2010 + default-extensions: + ScopedTypeVariables exposed-modules: Test.Tasty.Golden Test.Tasty.Golden.Advanced Test.Tasty.Golden.Manage @@ -51,21 +65,17 @@ ghc-options: -Wall build-depends: - base >= 4.7, + base >= 4.9 && < 5, tasty >= 1.3, bytestring >= 0.9.2.1, - process, - mtl, + typed-process, optparse-applicative >= 0.3.1, filepath, temporary, - tagged, deepseq, containers, directory, - async, - text, - unix-compat + text Test-suite test Default-language: @@ -77,16 +87,18 @@ Main-is: test.hs Build-depends: - base >= 4 && < 5 + base >= 4.9 && < 5 , tasty >= 1.2 , tasty-hunit , tasty-golden , filepath , directory - , process + , typed-process , temporary if (flag(build-example)) cpp-options: -DBUILD_EXAMPLE + build-tool-depends: tasty-golden:example + Ghc-options: -threaded flag build-example default: False @@ -112,3 +124,4 @@ , bytestring , tasty , tasty-golden + Ghc-options: -threaded diff --git a/tests/golden/before-accept.golden b/tests/golden/before-accept.golden --- a/tests/golden/before-accept.golden +++ b/tests/golden/before-accept.golden @@ -7,6 +7,7 @@ Failing tests goldenVsFile: FAIL Files 'example/golden/fail/goldenVsFile.golden' and 'example/golden/fail/goldenVsFile.actual' differ + Use -p '$0=="Tests.Failing tests.goldenVsFile"' to rerun this test only. goldenVsFileDiff: FAIL 1d0 < 1 @@ -35,6 +36,7 @@ 169d156 < Use --accept or increase --size-cutoff to see full output. + Use -p '/Failing tests.goldenVsFileDiff/' to rerun this test only. goldenVsString: FAIL Test output was different from 'example/golden/fail/goldenVsString.golden'. It was: 2 @@ -87,8 +89,9 @@ 55 56 Use --accept or increase --size-cutoff to see full output. + Use -p '$0=="Tests.Failing tests.goldenVsString"' to rerun this test only. goldenVsStringDiff: FAIL - Test output was different from 'example/golden/fail/goldenVsStringDiff.golden'. Output of ["diff","example/golden/fail/goldenVsStringDiff.golden","/tmp/goldenVsStringDiff.actual"]: + Test output was different from 'example/golden/fail/goldenVsStringDiff.golden'. Output of ["diff","example/golden/fail/goldenVsStringDiff.golden","/private/var/folders/19/d9jtc4c5365g3c_5jjk7m_980000gn/T/goldenVsStringDiff.actual"]: 1d0 < 1 4d2 @@ -116,5 +119,6 @@ 169d156 < Use --accept or increase --size-cutoff to see full output. + Use -p '/Failing tests.goldenVsStringDiff/' to rerun this test only. 4 out of 8 tests failed diff --git a/tests/test.hs b/tests/test.hs --- a/tests/test.hs +++ b/tests/test.hs @@ -6,7 +6,7 @@ import System.IO.Temp import System.FilePath import System.Directory -import System.Process +import System.Process.Typed import Data.List (sort) touch f = writeFile f "" @@ -71,7 +71,7 @@ (do tmp0 <- getCanonicalTemporaryDirectory tmp <- createTempDirectory tmp0 "golden-test" - callProcess "cp" ["-r", "example", tmp] + runProcess_ $ shell $ "cp -r example " ++ tmp return tmp ) ({-removeDirectoryRecursive-}const $ return ()) $ \tmpIO -> @@ -89,9 +89,10 @@ -- timings. -- -- NB: cannot use multiline literals because of CPP. - callCommand ("cd " ++ tmp ++ " && example | " ++ - "sed -Ee 's/[[:digit:]-]+\\.actual/.actual/g; s/ \\([[:digit:].]+s\\)//' > " ++ - our"tests/golden/before-accept.actual || true") + let cmd = shell ("cd " ++ tmp ++ " && example | " ++ + "sed -Ee 's/[[:digit:]-]+\\.actual/.actual/g; s/ \\([[:digit:].]+s\\)//' > " ++ + our"tests/golden/before-accept.actual || true") + runProcess_ cmd ) , after AllFinish "/before --accept/" $ goldenVsFileDiff "with --accept" @@ -101,7 +102,9 @@ (do tmp <- tmpIO our <- getCurrentDirectory - callCommand ("cd " ++ tmp ++ " && example --accept | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ our "tests/golden/with-accept.actual") + let cmd = shell ("cd " ++ tmp ++ " && example --accept | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ + our "tests/golden/with-accept.actual") + runProcess_ cmd ) , after AllFinish "/with --accept/" $ goldenVsFileDiff "after --accept" @@ -111,7 +114,9 @@ (do tmp <- tmpIO our <- getCurrentDirectory - callCommand ("cd " ++ tmp ++ " && example | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ our"tests/golden/after-accept.actual") + let cmd = shell ("cd " ++ tmp ++ " && example | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ + our"tests/golden/after-accept.actual") + runProcess_ cmd ) ] #endif