hnix-0.17.0: tests/NixLanguageTests.hs
module NixLanguageTests (genTests) where
import Nix.Prelude
import Control.Exception
import GHC.Err ( errorWithoutStackTrace )
import Control.Monad.ST
import Data.List ( delete )
import Data.List.Split ( splitOn )
import qualified Data.Map as Map
import qualified Data.Set as Set
import qualified Data.String as String
import qualified Data.Text as Text
import Data.Time
import GHC.Exts
import Nix.Lint
import Nix.Options
import Nix.Options.Parser
import Nix.Parser
import Nix.Pretty
import Nix.String
import Nix.XML
import qualified Options.Applicative as Opts
import System.Environment ( setEnv )
import System.FilePath.Glob ( compile
, globDir1
)
import Test.Tasty
import Test.Tasty.HUnit
import TestCommon
{-
From (git://nix)/tests/lang.sh we see that
lang/parse-fail-*.nix -> parsing should fail
lang/parse-okay-*.nix -> parsing should succeed
lang/eval-fail-*.nix -> eval should fail
lang/eval-okay-*.{nix,xml} -> eval should succeed,
xml dump should be the same as the .xml
lang/eval-okay-*.{nix,exp} -> eval should succeed,
plain text output should be the same as the .exp
lang/eval-okay-*.{nix,exp,flags} -> eval should succeed,
plain text output should be the same as the .exp,
pass the extra flags to nix-instantiate
NIX_PATH=lang/dir3:lang/dir4 should be in the environment of all
eval-okay-*.nix evaluations
TEST_VAR=foo should be in all the environments # for eval-okay-getenv.nix
-}
groupBy :: Ord k => (v -> k) -> [v] -> Map k [v]
groupBy key = Map.fromListWith (<>) . fmap (key &&& pure)
-- | New tests, which have never yet passed. Once any of these is passing,
-- please remove it from this list. Do not add tests to this list if they have
-- previously passed.
newFailingTests :: Set String
newFailingTests = Set.fromList
[ "eval-okay-fromTOML"
, "eval-okay-zipAttrsWith"
, "eval-okay-tojson"
, "eval-okay-search-path"
, "eval-okay-sort"
, "eval-okay-path-antiquotation"
, "eval-okay-getattrpos-functionargs"
, "eval-okay-attrs6"
]
-- | Upstream tests that test cases that HNix disaded as a misfeature that is used so rarely
-- that it more effective to fix it & lint it out of existance.
deprecatedRareNixQuirkTests :: Set String
deprecatedRareNixQuirkTests = Set.fromList
[ -- A rare quirk of Nix that is proper to fix&enforce then to support (see git commit history)
"eval-okay-strings-as-attrs-names"
-- Nix upstream removed this test altogether
, "eval-okay-hash"
]
genTests :: IO TestTree
genTests =
do
testFiles <- getTestFiles
let
testsGroupedByName :: Map Path [Path]
testsGroupedByName = groupBy (takeFileName . dropExtensions) testFiles
testsGroupedByTypeThenName :: Map [String] [(Path, [Path])]
testsGroupedByTypeThenName = groupBy testType $ Map.toList testsGroupedByName
testTree :: [TestTree]
testTree = mkTestGroup <$> Map.toList testsGroupedByTypeThenName
pure $
localOption
(mkTimeout 2000000)
$ testGroup
"Nix (upstream) language tests"
testTree
where
getTestFiles :: IO [Path]
getTestFiles = sortTestFiles <$> collectTestFiles
where
collectTestFiles :: IO [Path]
collectTestFiles = coerce (globDir1 (compile "*-*-*.*") nixTestDir)
sortTestFiles :: [Path] -> [Path]
sortTestFiles =
sort
-- Disabling the not yet done tests cases.
. filter withoutDisabledTests
. filter withoutXml
where
withoutDisabledTests :: Path -> Bool
withoutDisabledTests = (`Set.notMember` (newFailingTests `Set.union` deprecatedRareNixQuirkTests)) . takeBaseName
withoutXml :: Path -> Bool
withoutXml = (/= ".xml") . takeExtension
testType :: (Path, b) -> [String]
testType (fullpath, _files) = coerce (take 2 . splitOn "-") $ takeFileName fullpath
mkTestGroup :: ([String], [(Path, [Path])]) -> TestTree
mkTestGroup (tType, tests) =
testGroup (String.unwords tType) $ mkTestCase <$> tests
where
mkTestCase :: (Path, [Path]) -> TestTree
mkTestCase (basename, files) =
testCase
(coerce $ takeFileName basename)
$ do
time <- liftIO getCurrentTime
let opts = defaultOptions time
case tType of
["parse", "okay"] -> assertParse opts $ the files
["parse", "fail"] -> assertParseFail opts $ the files
["eval" , "okay"] -> assertEval opts files
["eval" , "fail"] -> assertEvalFail $ the files
_ -> fail $ "Unexpected: " <> show tType
assertParse :: Options -> Path -> Assertion
assertParse _opts file =
either
(\ err -> assertFailure $ "Failed to parse " <> coerce file <> ":\n" <> show err)
(const stub) -- pure $! runST $ void $ lint opts expr
=<< parseNixFileLoc file
assertParseFail :: Options -> Path -> Assertion
assertParseFail opts file =
(`catch` \(_ :: SomeException) -> stub) $
either
(const stub)
(\ expr ->
do
_ <- pure $! runST $ void $ lint opts expr
assertFailure $ "Unexpected success parsing `" <> coerce file <> ":\nParsed value: " <> show expr
)
=<< parseNixFileLoc file
assertLangOk :: Options -> Path -> Assertion
assertLangOk opts fileBaseName =
do
actual <- printNix <$> hnixEvalFile opts (addNixExt fileBaseName)
expected <- read fileBaseName ".exp"
assertEqual mempty expected (actual <> "\n")
assertLangOkXml :: Options -> Path -> Assertion
assertLangOkXml opts fileBaseName =
do
actual <- ignoreContext . toXML <$> hnixEvalFile opts (addNixExt fileBaseName)
expected <- read fileBaseName ".exp.xml"
assertEqual mempty expected actual
assertEval :: Options -> [Path] -> Assertion
assertEval _opts files =
do
time <- liftIO getCurrentTime
let opts = defaultOptions time
case delete ".nix" $ sort $ fromString @Text . takeExtensions <$> files of
[] -> void $ hnixEvalFile opts $ addNixExt name
[".exp" ] -> assertLangOk opts name
[".exp.xml" ] -> assertLangOkXml opts name
[".exp.disabled" ] -> stub
[".exp-disabled" ] -> stub
[".exp", ".flags"] ->
do
liftIO $ setEnv "NIX_PATH" "lang/dir4:lang/dir5"
flags <- read name ".flags"
let
flags' :: Text
flags' =
bool
id
Text.init
(Text.last flags == '\n')
flags
case runParserGetResult time flags' of
Opts.Failure err -> errorWithoutStackTrace $ "Error parsing flags from " <> coerce name <> ".flags: " <> show err
Opts.CompletionInvoked _ -> fail "unused"
Opts.Success opts' -> assertLangOk opts' name
_ -> assertFailure $ "Unknown test type " <> show files
where
runParserGetResult :: UTCTime -> Text -> Opts.ParserResult Options
runParserGetResult time flags' =
Opts.execParserPure
Opts.defaultPrefs
(nixOptionsInfo time)
(fmap toString $ fixup $ Text.splitOn " " flags')
name :: Path
name = coerce nixTestDir <> the (takeFileName . dropExtensions <$> files)
fixup :: [Text] -> [Text]
fixup ("--arg" : x : y : rest) = "--arg" : (x <> "=" <> y) : fixup rest
fixup ("--argstr" : x : y : rest) = "--argstr" : (x <> "=" <> y) : fixup rest
fixup (x : rest) = x : fixup rest
fixup [] = mempty
assertEvalFail :: Path -> Assertion
assertEvalFail file =
(`catch` (\(_ :: SomeException) -> stub)) $
do
time <- liftIO getCurrentTime
evalResult <- printNix <$> hnixEvalFile (defaultOptions time) file
evalResult `seq` assertFailure $ "File: ''" <> coerce file <> "'' should not evaluate.\nThe evaluation result was `" <> toString evalResult <> "`."
nixTestDir :: FilePath
nixTestDir = "data/nix/tests/lang/"
addNixExt :: Path -> Path
addNixExt path = addExtension path ".nix"
read :: Path -> String -> IO Text
read path ext = readFile $ addExtension path ext