import Test.Lattest.Adapter.StandardAdapters
import Test.Lattest.Exec.StandardTestControllers
import Test.Lattest.Exec.NComplete
import Test.Lattest.Exec.Testing
import Test.Lattest.Model.BoundedMonad
import Test.Lattest.Model.Internal.MCDCGeneration
import Test.Lattest.Model.StandardAutomata
import Test.Lattest.Model.STSTest
import Test.Lattest.Model.Symbolic.Expr
import Test.Lattest.Model.Symbolic.MCDCSTS
import Test.Lattest.Util.ModelParsingUtils
import Test.Lattest.Util.STSJSONParserTest
import Test.Lattest.Util.STSJSONWriterTest
import Test.System.IO.Streams.Synchronized(prop_consumeBufferedWith, testConsumeBufferedWith,testConsumeBufferedWith_short, prop_jsonStream)
import Test.Tasty
import Test.Tasty.Runners as Tasty
import Test.Tasty.Providers as Tasty
import Test.HUnit as HUnit
import Test.Tasty.QuickCheck
import System.Environment (getArgs, withArgs)
import Data.List (partition)
-- | Command-line flag that switches the golden tests from comparing to (re)generating their expected-output files.
-- Pass it via @stack test --test-arguments=--regenerate-golden-files@ (or the shorthand @--ta=...@). Without it, the
-- test suite has no side-effects; with it, the golden files under test/expected-test-output are overwritten.
regenerateFlag :: String
regenerateFlag = "--regenerate-golden-files"
durationSeconds :: Int
durationSeconds = 3
main :: IO ()
main = do
args <- getArgs
-- Consume our own flag before handing the remaining arguments to tasty, which would reject it as unknown.
let (ours, tastyArgs) = partition (== regenerateFlag) args
regenerate = not (null ours)
hunitTests <- makeHUnitTests regenerate
withArgs tastyArgs $ defaultMain $
localOption (NumThreads 1) $ -- some of these tests open concrete sockets, and thus can't be run multiple times in parallel
testGroup "Lattest-tests"
[ hunitTests
, quickCheckTests
]
quickCheckTests :: TestTree
quickCheckTests = testGroup "Quickcheck"
[ quickCheckWithTimeout (prop_jsonStream :: [(Int,Bool,Bool)] -> Property) "jsonStream"
-- Disable symbolic expression tests for now as they are too flaky
-- quickCheckWithTimeoutWithNum (prop_evalSymbolic :: PropEvalSymbolic Bool) 10000
-- quickCheckWithTimeoutWithNumWithSize prop_solveSymbolic 100 2
, quickCheckWithTimeoutWithNum prop_consumeBufferedWith (15 :: Int) "consumeBufferedWith"
, quickCheckWithTimeoutWithNum (prop_latticeIsCNF :: LatticeOp Int -> Bool) (10000 :: Int) "latticeIsCNF"
-- concrete-trace specified/allowed correspondence, with traces generated from the model alphabet
-- , testProperty "specifiedAllowedCorrespondence" $
-- within (durationSeconds * 1000000) $ withMaxSuccess 200 $
-- prop_specifiedAllowedCorrespondence composedCoffeeMachineIntrpr
, testProperty "show and read types" propShowReadType
]
where
quickCheckWithTimeout prop = quickCheckWithTimeoutWithNum prop (100 :: Int)
quickCheckWithTimeoutWithNum prop _ name = testProperty name $ \testparam ->
within (durationSeconds * 1000000) $ withMaxSize 20 $
-- Shrinking interacts really badly with the timeout: QuickCheck ends up on a search for the smallest input that exceeds the timelimit.
noShrinking $ prop testparam
makeHUnitTests :: Bool -> IO TestTree
makeHUnitTests regenerate = do
return $
localOption (NumThreads 1) $ -- some of these tests open concrete sockets, and thus can't be run in parallel
singleTest "unit tests" $
TestList $
[
testAccSeq,
testADG,
testConsumeBufferedWith,
testConsumeBufferedWith_short,
testJSONSocketAdapterByte,
testAdapterAcceptingInput,
testJSONSocketAdapterInt,
testJSONSocketAdapterObject,
testQuiscence,
testInputDelay,
testTraceHappy,
testTraceFailsAtLastOutput,
testTraceFailsBeforeLastOutput,
testTraceIncompleteAtLastOutput,
testTraceIncompleteBeforeLastOutput,
testTraceFailsWithQuiescence,
testOutputOutsideAlphabet,
testSpecF,
testPrintSpecF,
testSpecG,
testSpecGQuiescent,
testExponentialNonDeterminism,
testSTSTestSelection,
testSTSDataSelectionGuardedInput,
testRandomFCorrect,
testRandomFIncorrectOutput,
testRandomFIncorrectInput,
testSTSHappyFlow,
testSTSHappyFlowFloat,
testSTSHappyFlowLists,
testErrorThrowingGates,
testSTSUnHappyFlow,
testComposedSeTreeStructure regenerate,
testComposedPathCondition regenerate,
testConcreteTraceSpecifiedAllowedCorrespondence,
testPrintSTS,
testPrintSeqCompSTS,
testSeqComposedSTS,
testSeqComposedAtSTS,
testSequentiallyAtNonSinkLocation,
testSequentiallyAtSameAction,
testPrintSelfSeqComposedSTS,
testSelfSeqComposed,
testSelfSeqComposedAt,
testPrintSeqCompPrunedSTS,
testPrintSeqCompPrunedSTSInit1,
testSeqCompPrunedRules,
testSelfSeqCompPrunedRules,
testSelfSeqComposedAtOne,
testConjunctionGuardedSTS,
testDisjunctionGuardedSTS,
testConjunctionAllGuardedSTS,
testDisjunctionAllGuardedSTS,
testPrintTriDisj,
testErrorDisjOfSTSWithMultpInitStates,
testPrintPrependOutputChecksDisj,
testPrintPrependOutputChecksConj,
testPrependOutputChecksDisj,
testPrependOutputChecksConj,
testReadAutFile,
testSTSJSONParserNominal,
testSTSJSONParserNominalFloat,
testSTSJSONParserUnknownType,
testSTSJSONParserUnsupportedGuardOperand,
testSTSJSONParserUnsupportedAssignmentOperand,
testSTSJSONParserMissingSwitches,
testSTSJSONParserMissingGates,
testSTSJSONParserAssignmentTypeMismatch,
testSTSJSONParserGuardTypeMismatch,
testSTSJSONParserGateIdDup,
testCoverageCheckerIsExhaustive,
testSTSJSONWriterComposedConjunction,
testLatticeCoffeeSTS
]
++ testLatticeSTS
++ testLatticeSTSQuiescence
++ evalTests
++ solveTests
++ mcdcGenerationTests
++ mcdcSTSTests
++ testQuantifiers
-- TODO: This wraps HUnit tests into a Tasty test.
-- The reason we need this wrapper, is that plain HUnit
-- does not set the exit code, making CI runs pass even
-- when tests fail.
-- Instead, we should migrate the HUnit tests to
-- tasty-hunit, which exposes a very similar interface.
instance Tasty.IsTest HUnit.Test where
run _ t _ = runTestTT t >>= \c ->
if errors c + failures c > 0
then return $ testFailed (show c)
else return $ testPassed (show c)
testOptions = pure []