diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ Changes ======= +Version 2.3.5 +------------- + +* Fixes for launching external processes (like diff) on Windows +* Update the golden file on --accept if decoding the golden file failed with an exception +* Do not depend on unix-compat + Version 2.3.4 ------------- diff --git a/Test/Tasty/Golden.hs b/Test/Tasty/Golden.hs --- a/Test/Tasty/Golden.hs +++ b/Test/Tasty/Golden.hs @@ -1,18 +1,23 @@ {- | + +== Getting Started To get started with golden testing and this library, see . This module provides a simplified interface. If you want more, see "Test.Tasty.Golden.Advanced". -Note about filenames. They are looked up in the usual way, thus relative +== Filenames +Filenames are looked up in the usual way, Thus relative names are relative to the processes current working directory. It is common to run tests from the package's root directory (via @cabal test@ or @cabal install --enable-tests@), so if your test files are under the @tests\/@ subdirectory, your relative file names should start with @tests\/@ (even if your @test.hs@ is itself under @tests\/@, too). -Note about line endings. The best way to avoid headaches with line endings +== Line endings + +The best way to avoid headaches with line endings (when running tests both on UNIX and Windows) is to treat your golden files as binary, even when they are actually textual. @@ -43,8 +48,28 @@ As a last resort, you can strip all @\\r@s from both arguments in your comparison function when necessary. But most of the time treating the files as binary does the job. + +== Linking +The test suite should be compiled with @-threaded@ if you want to avoid +blocking any other threads while 'goldenVsFileDiff' and similar functions +wait for the result of the diff command. + +== Windows limitations +When using 'goldenVsFileDiff' or 'goldenVsStringDiff' under Windows the exit +code from the diff program that you specify will not be captured correctly +if that program uses @exec@. + +More specifically, you will get the exit code of the /original child/ +(which always exits with code 0, since it called @exec@), not the exit +code of the process which carried on with execution after @exec@. +This is different from the behavior prescribed by POSIX but is the best +approximation that can be realised under the restrictions of the +Windows process model. See 'System.Process' for further details or + for even more. + -} +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} module Test.Tasty.Golden ( @@ -68,19 +93,21 @@ import Test.Tasty.Golden.Internal import Text.Printf import qualified Data.ByteString.Lazy as LBS -import Data.Monoid import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.Encoding as LT import System.IO import System.IO.Temp -import System.Process +import qualified System.Process.Typed as PT import System.Exit import System.FilePath import System.Directory -import System.PosixCompat.Files import Control.Exception import Control.Monad import qualified Data.Set as Set +import Foreign.C.Error +#if !MIN_VERSION_base(4,11,0) +import Data.Monoid +#endif -- | Compare the output file's contents against the golden file's contents -- after the given action has created the output file. @@ -129,6 +156,9 @@ return $ if x == y then Nothing else Just e -- | Same as 'goldenVsFile', but invokes an external diff command. +-- +-- See the notes at the top of this module regarding linking with +-- @-threaded@ and Windows-specific issues. goldenVsFileDiff :: TestName -- ^ test name -> (FilePath -> FilePath -> [String]) @@ -146,33 +176,28 @@ askOption $ \sizeCutoff -> goldenTest2 name - (getFileStatus ref >> return ()) - -- Use getFileStatus to check if the golden file exists. If the file - -- doesn't exist, getFileStatus will throw an isDoesNotExistError that - -- runGolden will handle by creating the golden file before proceeding. - -- See #32. + (throwIfDoesNotExist ref) act - (cmp sizeCutoff) + (\_ _ -> runDiff (cmdf ref new) sizeCutoff) upd del where - cmd = cmdf ref new - cmp sizeCutoff _ _ - | null cmd = error "goldenVsFileDiff: empty command line" - | otherwise = do - (_, Just sout, _, pid) <- createProcess (proc (head cmd) (tail cmd)) { std_out = CreatePipe } - -- strictly read the whole output, so that the process can terminate - out <- hGetContentsStrict sout - - r <- waitForProcess pid - return $ case r of - ExitSuccess -> Nothing - _ -> Just . unpackUtf8 . truncateLargeOutput sizeCutoff $ out - upd _ = readFileStrict new >>= createDirectoriesAndWriteFile ref del = removeFile new +-- If the golden file doesn't exist, throw an isDoesNotExistError that +-- runGolden will handle by creating the golden file before proceeding. +-- See #32. +throwIfDoesNotExist :: FilePath -> IO () +throwIfDoesNotExist ref = do + exists <- doesFileExist ref + unless exists $ ioError $ + errnoToIOError "goldenVsFileDiff" eNOENT Nothing Nothing + -- | Same as 'goldenVsString', but invokes an external diff command. +-- +-- See the notes at the top of this module regarding linking with +-- @-threaded@ and Windows-specific issues. goldenVsStringDiff :: TestName -- ^ test name -> (FilePath -> FilePath -> [String]) @@ -201,17 +226,10 @@ LBS.hPut tmpHandle actBS >> hFlush tmpHandle let cmd = cmdf ref tmpFile - - when (null cmd) $ error "goldenVsFileDiff: empty command line" - - (_, Just sout, _, pid) <- createProcess (proc (head cmd) (tail cmd)) { std_out = CreatePipe } - -- strictly read the whole output, so that the process can terminate - out <- hGetContentsStrict sout + diff_result :: Maybe String <- runDiff cmd sizeCutoff - r <- waitForProcess pid - return $ case r of - ExitSuccess -> Nothing - _ -> Just (printf "Test output was different from '%s'. Output of %s:\n" ref (show cmd) <> unpackUtf8 (truncateLargeOutput sizeCutoff out)) + return $ flip fmap diff_result $ \diff -> + printf "Test output was different from '%s'. Output of %s:\n" ref (show cmd) <> diff upd = createDirectoriesAndWriteFile ref @@ -299,12 +317,24 @@ evaluate $ forceLbs s return s -hGetContentsStrict :: Handle -> IO LBS.ByteString -hGetContentsStrict h = do - hSetBinaryMode h True - s <- LBS.hGetContents h - evaluate $ forceLbs s - return s - unpackUtf8 :: LBS.ByteString -> String unpackUtf8 = LT.unpack . LT.decodeUtf8 + +runDiff + :: [String] -- ^ the diff command + -> SizeCutoff + -> IO (Maybe String) +runDiff cmd sizeCutoff = + case cmd of + [] -> throwIO $ ErrorCall "tasty-golden: empty diff command" + prog : args -> do + let + procConf = + PT.setStdin PT.closed + . PT.setStderr PT.inherit + $ PT.proc prog args + + (exitCode, out) <- PT.readProcessStdout procConf + return $ case exitCode of + ExitSuccess -> Nothing + _ -> Just . unpackUtf8 . truncateLargeOutput sizeCutoff $ out diff --git a/Test/Tasty/Golden/Internal.hs b/Test/Tasty/Golden/Internal.hs --- a/Test/Tasty/Golden/Internal.hs +++ b/Test/Tasty/Golden/Internal.hs @@ -55,6 +55,8 @@ -- for readability. -- -- The default value is 1000 (i.e. 1Kb). +-- +-- @since 2.3.3 newtype SizeCutoff = SizeCutoff { getSizeCutoff :: Int64 } deriving (Eq, Ord, Typeable, Num, Real, Enum, Integral) instance IsOption SizeCutoff where @@ -127,18 +129,29 @@ mbRef <- try getGolden case mbRef of - Left e | isDoesNotExistError e -> - if noCreate - then - -- Don't ever delete the output file in this case, as there is - -- no duplicate golden file - return $ testFailed "Golden file does not exist; --no-create flag specified" - else do + Left e + | Just e' <- fromException e, isDoesNotExistError e' -> + if noCreate + then + -- Don't ever delete the output file in this case, as there is + -- no duplicate golden file + return $ testFailed "Golden file does not exist; --no-create flag specified" + else do + update new + when (delOut `elem` [Always, OnPass]) delete + return $ testPassed "Golden file did not exist; created" + + | Just (_ :: AsyncException) <- fromException e -> throwIO e + | Just (_ :: IOError) <- fromException e -> throwIO e + + + | otherwise -> do + -- Other types of exceptions may be due to failing to decode the + -- golden file. In that case, it makes sense to replace a broken + -- golden file with the current version. update new when (delOut `elem` [Always, OnPass]) delete - return $ testPassed "Golden file did not exist; created" - - | otherwise -> throwIO e + return $ testPassed $ "Accepted the new version. Was failing with exception:\n" ++ show e Right ref -> do diff --git a/tasty-golden.cabal b/tasty-golden.cabal --- a/tasty-golden.cabal +++ b/tasty-golden.cabal @@ -1,5 +1,5 @@ name: tasty-golden -version: 2.3.4 +version: 2.3.5 synopsis: Golden tests support for tasty description: This package provides support for «golden testing». @@ -13,14 +13,14 @@ license: MIT license-file: LICENSE -Homepage: https://github.com/feuerbach/tasty-golden -Bug-reports: https://github.com/feuerbach/tasty-golden/issues +Homepage: https://github.com/UnkindPartition/tasty-golden +Bug-reports: https://github.com/UnkindPartition/tasty-golden/issues author: Roman Cheplyaka maintainer: Roman Cheplyaka -- copyright: category: Testing build-type: Simple -cabal-version: >=1.14 +cabal-version: 1.14 extra-source-files: CHANGELOG.md example/golden/fail/*.golden @@ -37,11 +37,13 @@ Source-repository head type: git - location: git://github.com/feuerbach/tasty-golden.git + location: git://github.com/UnkindPartition/tasty-golden.git library Default-language: Haskell2010 + default-extensions: + ScopedTypeVariables exposed-modules: Test.Tasty.Golden Test.Tasty.Golden.Advanced Test.Tasty.Golden.Manage @@ -54,7 +56,7 @@ base >= 4.7, tasty >= 1.3, bytestring >= 0.9.2.1, - process, + typed-process, mtl, optparse-applicative >= 0.3.1, filepath, @@ -64,8 +66,7 @@ containers, directory, async, - text, - unix-compat + text Test-suite test Default-language: @@ -83,10 +84,11 @@ , tasty-golden , filepath , directory - , process + , typed-process , temporary if (flag(build-example)) cpp-options: -DBUILD_EXAMPLE + Ghc-options: -threaded flag build-example default: False @@ -112,3 +114,4 @@ , bytestring , tasty , tasty-golden + Ghc-options: -threaded diff --git a/tests/test.hs b/tests/test.hs --- a/tests/test.hs +++ b/tests/test.hs @@ -6,7 +6,7 @@ import System.IO.Temp import System.FilePath import System.Directory -import System.Process +import System.Process.Typed import Data.List (sort) touch f = writeFile f "" @@ -71,7 +71,7 @@ (do tmp0 <- getCanonicalTemporaryDirectory tmp <- createTempDirectory tmp0 "golden-test" - callProcess "cp" ["-r", "example", tmp] + runProcess_ $ shell $ "cp -r example " ++ tmp return tmp ) ({-removeDirectoryRecursive-}const $ return ()) $ \tmpIO -> @@ -89,9 +89,10 @@ -- timings. -- -- NB: cannot use multiline literals because of CPP. - callCommand ("cd " ++ tmp ++ " && example | " ++ - "sed -Ee 's/[[:digit:]-]+\\.actual/.actual/g; s/ \\([[:digit:].]+s\\)//' > " ++ - our"tests/golden/before-accept.actual || true") + let cmd = shell ("cd " ++ tmp ++ " && example | " ++ + "sed -Ee 's/[[:digit:]-]+\\.actual/.actual/g; s/ \\([[:digit:].]+s\\)//' > " ++ + our"tests/golden/before-accept.actual || true") + runProcess_ cmd ) , after AllFinish "/before --accept/" $ goldenVsFileDiff "with --accept" @@ -101,7 +102,9 @@ (do tmp <- tmpIO our <- getCurrentDirectory - callCommand ("cd " ++ tmp ++ " && example --accept | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ our "tests/golden/with-accept.actual") + let cmd = shell ("cd " ++ tmp ++ " && example --accept | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ + our "tests/golden/with-accept.actual") + runProcess_ cmd ) , after AllFinish "/with --accept/" $ goldenVsFileDiff "after --accept" @@ -111,7 +114,9 @@ (do tmp <- tmpIO our <- getCurrentDirectory - callCommand ("cd " ++ tmp ++ " && example | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ our"tests/golden/after-accept.actual") + let cmd = shell ("cd " ++ tmp ++ " && example | sed -Ee 's/ \\([[:digit:].]+s\\)//' > " ++ + our"tests/golden/after-accept.actual") + runProcess_ cmd ) ] #endif