packages feed

lattest-lib-0.2: test/Test/Lattest/Exec/Testing.hs

module Test.Lattest.Exec.Testing (
testTraceHappy,
testTraceFailsAtLastOutput,
testTraceFailsBeforeLastOutput,
testTraceIncompleteAtLastOutput,
testTraceIncompleteBeforeLastOutput,
testTraceFailsWithQuiescence,
testOutputOutsideAlphabet
)
where

import Test.HUnit hiding (Path, path)

import Lattest.Exec.Testing(TestController(..), Verdict(..), runTester, Verdict(Pass))
import Lattest.Model.Automaton(AutSyntax, automaton)
import Lattest.Model.Alphabet(IOAct(..), IOSuspAct, Suspended(..), SuspendedIF)
import Lattest.Model.BoundedMonad
import qualified Data.Map as Map (Map, insert, fromList)
import Data.Maybe (fromJust)
import Lattest.Adapter.StandardAdapters(Adapter,pureMealyAdapter, pureAdapter)
import System.Random(StdGen, mkStdGen)
import Lattest.Model.StandardAutomata(interpretQuiescentConcrete, interpretQuiescentInputAttemptConcrete, detConcTransFromRel)
import Lattest.Exec.StandardTestControllers


testTraceHappy :: Test
testTraceHappy = TestCase $ do
    let t = [In 1, In 2, Out 3, In 4, Out 5, Out 6] :: [IOAct Integer Integer]
    adap <- traceAdapter t
    (verdict, finished) <- runTester (interpretQuiescentConcrete $ traceSpecification t) (ioTraceTestController t) adap
    assertEqual "testTraceHappy should pass" verdict Pass
    assertEqual "testTraceHappy should be complete" finished True

testTraceFailsAtLastOutput :: Test
testTraceFailsAtLastOutput = TestCase $ do
    let t = [Out 1, Out 1, Out 2] :: [IOAct Integer Integer]
    let tspec = [Out 1, Out 2] :: [IOAct Integer Integer]
    adap <- traceAdapter t
    (verdict, finished) <- runTester (interpretQuiescentConcrete $ traceSpecification tspec) (ioTraceTestController tspec) adap
    assertEqual "testTraceFailsAtLastOutput should fail" Fail verdict
    assertEqual "testTraceFailsAtLastOutput should be complete" True finished

testTraceFailsBeforeLastOutput :: Test
testTraceFailsBeforeLastOutput = TestCase $ do
    let t = [Out 1, Out 2] :: [IOAct Integer Integer]
    let tspec = [Out 2, Out 1] :: [IOAct Integer Integer]
    adap <- traceAdapter t
    (verdict, finished) <- runTester (interpretQuiescentConcrete $ traceSpecification tspec) (ioTraceTestController tspec) adap
    assertEqual "testTraceFailsBeforeLastOutput should fail" Fail verdict
    assertEqual "testTraceFailsBeforeLastOutput should be incomplete" False finished

testTraceIncompleteAtLastOutput :: Test
testTraceIncompleteAtLastOutput = TestCase $ do
    let t = [Out 1, Out 2] :: [IOAct Integer Integer]
    let tController = [Out 1, Out 1] :: [IOAct Integer Integer]
    adap <- traceAdapter t
    (verdict, finished) <- runTester (interpretQuiescentConcrete $ traceSpecification t) (ioTraceTestController tController) adap
    assertEqual "testTraceIncompleteAtLastOutput should pass" Pass verdict
    assertEqual "testTraceIncompleteAtLastOutput should be complete" True finished

testTraceIncompleteBeforeLastOutput :: Test
testTraceIncompleteBeforeLastOutput = TestCase $ do
    let t = [Out 1, Out 2] :: [IOAct Integer Integer]
    let tController = [Out 2, Out 2] :: [IOAct Integer Integer]
    adap <- traceAdapter t
    (verdict, finished) <- runTester (interpretQuiescentConcrete $ traceSpecification t) (ioTraceTestController tController) adap
    assertEqual "testTraceIncompleteBeforeLastOutput should pass" Pass verdict
    assertEqual "testTraceIncompleteBeforeLastOutput should be incomplete" False finished

testTraceFailsWithQuiescence :: Test
testTraceFailsWithQuiescence = TestCase $ do
    let t = [Out 1, In 2] :: [IOAct Integer Integer]
    let tspec = [Out 1, Out 2] :: [IOAct Integer Integer]
    adap <- traceAdapter t
    (verdict, finished) <- runTester (interpretQuiescentConcrete $ traceSpecification tspec) (ioTraceTestController tspec) adap
    assertEqual "testTraceFailsWithQuiescence should fail" Fail verdict
    assertEqual "testTraceFailsWithQuiescence should be incomplete" True finished

ioTraceTestController :: (Eq i, Eq o) => [IOAct i o] -> TestController m loc q t tdest (IOSuspAct i o) [Either (Maybe i) (IOSuspAct i o)] (Maybe i) Bool
ioTraceTestController ioActs = traceTestController $ toCommandsAndActs ioActs
    where
    toCommandsAndActs [] = []
    toCommandsAndActs (In i:rest) = Left (Just i) : Right (In i) : toCommandsAndActs rest
    toCommandsAndActs (Out o:rest) = Right (Out $ OutSusp o) : toCommandsAndActs rest

-- a hardcoded test controller just follows the input commands and observations in the given list. Returns whether it finish the list
traceTestController :: (Eq act) => [Either (Maybe i) act] -> TestController m loc q t tdest act [Either (Maybe i) act] (Maybe i) Bool
traceTestController steps = TestController {
    -- testControllerState :: (TestChoice i act) => [Either i act]
    testControllerState = steps,
    --selectTest :: (TestChoice i act) => [Either i act] -> AutIntrpr m loc q t tdest act -> m q -> IO (Either (i, [Either i act]) Boolean),
    selectTest = traceSelectTest,
    --updateTestController :: [Either i act] -> AutIntrpr m loc q t tdest act -> act -> m q -> IO (Either [Either i act] Boolean),
    updateTestController = traceUpdateTestController,
    --handleTestClose :: [Either i act] -> IO Boolean -- When testing finishes, return a result
    handleTestClose = return . null
    }
    where
    traceSelectTest [] _ _ = return $ Right True
    traceSelectTest (Left (Just i):steps') _ _ = return $ Left (Just i, steps')
    traceSelectTest (Right act:steps') _ _ = return $ Left (Nothing, Right act:steps') -- the test controller must choose input but it wanted to make an observation.
    traceSelectTest _ _ _ = error "should not happen"
    traceUpdateTestController [] _ _ _ = return $ Right True
    traceUpdateTestController (Left _:_) _ _ _ = return $ Right False -- test controller makes an observation but wanted to choose an input
    traceUpdateTestController (Right expectedAct:steps') _ act _ = return $ if expectedAct == act then Left steps' else Right (null steps')

traceSpecification :: (Ord i, Ord o) => [IOAct i o] -> AutSyntax Det [IOAct i o] (IOAct i o) () -- TODO automata are inconsistent: no explicit alphabet, but an explicit transition map
traceSpecification steps = automaton (if null steps then UnderspecDet else Det steps) steps traceTransRel
    where
    traceTransRel (step:steps') = Map.insert step (Det ((), steps')) baseTransRel
    traceTransRel [] = baseTransRel
    baseTransRel = Map.fromList [(act, detStateFromAct act) | act <- steps]
    detStateFromAct (In _) = UnderspecDet
    detStateFromAct (Out _) = ForbiddenDet
{-
traceAdapter :: (Show i, Show o) => [IOAct i o] -> IO (Adapter (IOSuspAct i o) (Maybe i))
traceAdapter steps = pureAdapter traceTrans traceOutput $ if (null $ takeOutputs steps) then steps else error "traceAdapter cannot start with output"
    where
    -- invariant: before and in between transitions, the first step is always an input
    traceTrans steps' _ = snd (span isOutput steps') -- after the first input, drop all next outputs
    traceOutput steps' _ = takeOutputs steps' -- after the first input, take all next outputs
    takeOutputs steps' = fst (span isOutput steps')
-}
traceAdapter :: (Eq i) => [IOAct i o] -> IO (Adapter (IOSuspAct i o) (Maybe i))
traceAdapter = pureMealyAdapter traceTrans traceOutput
    where
    traceTrans [] _ = []
    traceTrans (In is:steps') (Just i) = if i == is then steps' else []
    traceTrans (In is:steps') Nothing = In is:steps'
    traceTrans (Out _:steps') _ = steps' -- potential race condition between the (Just i) and the output. Let the adapter win to pester the tester
    traceOutput [] _ = [Out Quiescence] -- if the adapter is not processing any more actions, show timeouts. Even if an input is attempted, because it will not be processed
    traceOutput (In _:_) (Just i) = [In i]
    traceOutput (In _:_) Nothing = [Out Quiescence] -- the adapter is waiting for an input but doesn't receive one, so show a timeout.
    traceOutput (Out os:_) _ = [Out $ OutSusp os] -- potential race condition between the (Just i) and the output. Let the adapter win to pester the tester

testSelector :: TestController Det StateFDet StateFDet (IOAct IF OF) () (SuspendedIF IF OF) ((((StdGen, Int), [SuspendedIF IF OF]), Maybe (Det StateFDet)), Maybe (Det StateFDet)) (Maybe IF) (([SuspendedIF IF OF], Maybe (Det StateFDet)), Maybe (Det StateFDet))
testSelector = randomTestSelectorFromSeed 456 `untilCondition` stopAfterSteps 50
                `observingOnly` traceObserver `andObserving` stateObserver `andObserving` inconclusiveStateObserver

data IF = A deriving (Show, Eq, Ord)
data OF = X | Y deriving (Show, Eq, Ord)
data StateFDet = Q0fd deriving (Show, Eq, Ord)
xf :: IOAct i OF
xf = Out X
yf :: IOAct i OF
yf = Out Y
af :: IOAct IF o
af = In A
q0f :: Det StateFDet
q0f = pure Q0fd
tWithX :: StateFDet -> Map.Map (IOAct IF OF) StateFDet
tWithX Q0fd = Map.fromList [(af, Q0fd), (xf, Q0fd)]
impWithX :: IO (Adapter (SuspendedIF IF OF) (Maybe IF))
impWithX = pureAdapter (mkStdGen 123) 0.0 tWithX Q0fd

menuWithY :: [IOAct IF OF]
menuWithY = [af, yf]
tWithY :: StateFDet -> Map.Map (IOAct IF OF) (Det ((), StateFDet))
tWithY = fromJust $ detConcTransFromRel [(Q0fd, af, Q0fd), (Q0fd, yf, Q0fd)]
specWithY :: AutSyntax Det StateFDet (IOAct IF OF) ()
specWithY = automaton q0f menuWithY tWithY

testOutputOutsideAlphabet :: Test
testOutputOutsideAlphabet = TestCase $ do
    imp <- impWithX
    let model = interpretQuiescentInputAttemptConcrete specWithY
    (verdict, ((_, _), _)) <- runTester model testSelector imp
    case verdict of
        Inconclusive _ -> return ()
        _ -> assertFailure $ "Output outside alphabet used, expected inconclusive verdict instead of " ++ show verdict