crackNum-4.7: src/CrackNum/TestSuite.hs
---------------------------------------------------------------------------
-- |
-- Module : TestSuite
-- Copyright : (c) Levent Erkok
-- License : BSD3
-- Maintainer : erkokl@gmail.com
-- Stability : experimental
--
-- Test-suite for crackNum
-----------------------------------------------------------------------------
{-# LANGUAGE ScopedTypeVariables #-}
{-# OPTIONS_GHC -Wall -Werror #-}
module CrackNum.TestSuite(runTests, runTestsWith) where
import Control.Exception as C
import Test.Tasty
import Test.Tasty.Golden (goldenVsFileDiff)
import Test.Tasty.HUnit (assertBool, assertEqual, testCase)
import System.FilePath
import System.Directory (removeFile)
import System.Environment (getExecutablePath)
import System.Exit (ExitCode(ExitSuccess))
import System.IO (IOMode(WriteMode), hPutStr, hSetNewlineMode, noNewlineTranslation, withFile)
import System.Process (readProcessWithExitCode)
import Data.List (intercalate, isInfixOf)
golden :: FilePath -> TestName -> String -> TestTree
golden executable n as = goldenVsFileDiff n diff gf gfTmp (rm gfTmp >> run)
where gf = "Golds" </> n <.> "gold"
gfTmp = gf ++ "_temp"
rm f = removeFile f `C.catch` (\(_ :: C.SomeException) -> return ())
args = words as
-- NB. Written with newline translation off, so the temp file uses LF on
-- every platform. The golds are LF; plain writeFile would emit CRLF on
-- Windows, and --accept there would rewrite every gold in the process.
write f s = withFile f WriteMode $ \h -> hSetNewlineMode h noNewlineTranslation >> hPutStr h s
run = do (ec, so, se) <- readProcessWithExitCode executable args ""
write gfTmp $ intercalate "\n" $ [ "Arguments: " ++ as
, "Exit code: " ++ show ec
, so
]
++ concat [["STDERR:", se] | not (null se)]
diff ref new = ["diff", "-w", "-u", ref, new]
-- | run the test suite
runTests :: IO ()
runTests = getExecutablePath >>= runTestsWith
-- | Run against an explicitly selected executable. The Cabal test component
-- receives the just-built crackNum as a build tool and passes that path here;
-- the executable's --runTests mode uses getExecutablePath above. Neither path
-- can silently pick up a stale installation from PATH.
runTestsWith :: FilePath -> IO ()
runTestsWith executable = defaultMain (tests executable)
encodingHas :: FilePath -> TestName -> [String] -> String -> TestTree
encodingHas executable name args expected = testCase name $ do
(ec, so, se) <- readProcessWithExitCode executable args ""
assertEqual ("stderr:\n" ++ se) ExitSuccess ec
assertBool ("Expected output to contain " ++ show expected ++ ", got:\n" ++ so)
(expected `isInfixOf` so)
tests :: FilePath -> TestTree
tests executable = let gold = golden executable
in testGroup "CrackNum" [
testGroup "Encode" [
gold "encode0" "-i4 -- -2"
, gold "encode1" "-w4 2"
, gold "encode2" "-f3+4 2.5"
, gold "encode3" "-f3+4 2.5 -rRTZ"
, gold "encode4" "-fbp 2.5"
, gold "encode5" "-fdp 2.5"
, gold "encode6" "-f3+3 -- -inf"
, gold "encode7" "-f3+3 -- -infinity"
, gold "encode8" "-f3+3 inf"
, gold "encode9" "-f3+3 infinity"
, gold "encode10" "-f3+3 nan"
, gold "encode11" "-fsp -- -inf"
, gold "encode12" "-fsp -- -infinity"
, gold "encode13" "-fsp inf"
, gold "encode14" "-fsp infinity"
, gold "encode15" "-fsp nan"
, gold "encode16" "-fe5m2 2.5"
, gold "encode17" "-fsp -- -0x2p3"
, gold "encode18" "-fdp -- 0x1.3"
, gold "encode19" "-fhp -- 0x1.3p4"
, gold "encode20" "-ftf32 2.5"
, gold "encode21" "-ftf32 -- -inf"
, gold "encode22" "-ftf32 nan"
]
, testGroup "EncodeE4M3" [
gold "encodeE4M3_nan" "-fe4m3 nan"
, gold "encodeE4M3_+inf" "-fe4m3 inf"
, gold "encodeE4M3_-inf" "-fe4m3 -- -inf"
, gold "encodeE4M3_in1" "-fe4m3 -- -448.0001"
, gold "encodeE4M3_in2" "-fe4m3 -- 448.0001"
, gold "encodeE4M3_bnd1" "-fe4m3 -- -239.9999"
, gold "encodeE4M3_bnd2" "-fe4m3 -- 239.9999"
, gold "encodeE4M3_zero1" "-fe4m3 -- 0"
, gold "encodeE4M3_zero2" "-fe4m3 -- -0"
, gold "encodeE4M3_mr1" "-fe4m3 -- 240"
, gold "encodeE4M3_mr2" "-fe4m3 -- 240.00"
, gold "encodeE4M3_mr3" "-fe4m3 -- -240"
, gold "encodeE4M3_mr4" "-fe4m3 -- -240.00"
]
, testGroup "EncodeE4M3Special" $ concat [
[ gold ("encodeE4M3_special_" ++ rm ++ "_+" ++ show i) ("-fe4m3 -r" ++ rm ++ " -- " ++ show i)
, gold ("encodeE4M3_special_" ++ rm ++ "_-" ++ show i) ("-fe4m3 -r" ++ rm ++ " -- -" ++ show i)
]
| rm <- ["RNE", "RNA", "RTP", "RTN", "RTZ"]
, i :: Double <- [240.01, 248, 419, 432]
]
, testGroup "EncodeFP4" [
gold "encodeFP4_nan" "-ffp4 nan"
, gold "encodeFP4_+inf" "-ffp4 inf"
, gold "encodeFP4_-inf" "-ffp4 -- -inf"
, gold "encodeFP4_zero1" "-ffp4 -- 0"
, gold "encodeFP4_zero2" "-ffp4 -- -0"
, gold "encodeFP4_sub" "-ffp4 -- 0.5" -- Subnormal
, gold "encodeFP4_norm" "-ffp4 -- 1.5"
, gold "encodeFP4_dev1" "-ffp4 -- 4" -- Deviates from IEEE: would be +Inf
, gold "encodeFP4_dev2" "-ffp4 -- 6" -- Deviates from IEEE: would be NaN
, gold "encodeFP4_dev3" "-ffp4 -- -6"
, gold "encodeFP4_oob1" "-ffp4 -- 100" -- Saturates
, gold "encodeFP4_oob2" "-ffp4 -- -100"
, gold "encodeFP4_hex" "-ffp4 -- 0x1.8p2"
]
-- Every value that sits exactly half-way between two representable magnitudes,
-- plus one out-of-range value, over all rounding modes.
, testGroup "EncodeFP4Ties" $ concat [
[ gold ("encodeFP4_tie_" ++ rm ++ "_+" ++ show i) ("-ffp4 -r" ++ rm ++ " -- " ++ show i)
, gold ("encodeFP4_tie_" ++ rm ++ "_-" ++ show i) ("-ffp4 -r" ++ rm ++ " -- -" ++ show i)
]
| rm <- ["RNE", "RNA", "RTP", "RTN", "RTZ"]
, i :: Double <- [0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5, 7]
]
, testGroup "EncodeFP4E0M3" [
gold "encodeFP4E0M3_nan" "-ffp4e0m3 nan"
, gold "encodeFP4E0M3_+inf" "-ffp4e0m3 inf"
, gold "encodeFP4E0M3_-inf" "-ffp4e0m3 -- -inf"
, gold "encodeFP4E0M3_zero1" "-ffp4e0m3 -- 0"
, gold "encodeFP4E0M3_zero2" "-ffp4e0m3 -- -0"
, gold "encodeFP4E0M3_exact" "-ffp4e0m3 -- 5"
, gold "encodeFP4E0M3_max" "-ffp4e0m3 -- 7"
, gold "encodeFP4E0M3_min" "-ffp4e0m3 -- -7"
, gold "encodeFP4E0M3_rnd1" "-ffp4e0m3 -- 0.4" -- Rounds down to a zero
, gold "encodeFP4E0M3_rnd2" "-ffp4e0m3 -- -0.4" -- Rounds down to a negative zero
, gold "encodeFP4E0M3_oob1" "-ffp4e0m3 -- 100" -- Saturates
, gold "encodeFP4E0M3_oob2" "-ffp4e0m3 -- -100"
, gold "encodeFP4E0M3_hex" "-ffp4e0m3 -- 0x1.8p1"
]
-- Every value that sits exactly half-way between two representable magnitudes,
-- plus one out-of-range value, over all rounding modes.
, testGroup "EncodeFP4E0M3Ties" $ concat [
[ gold ("encodeFP4E0M3_tie_" ++ rm ++ "_+" ++ show i) ("-ffp4e0m3 -r" ++ rm ++ " -- " ++ show i)
, gold ("encodeFP4E0M3_tie_" ++ rm ++ "_-" ++ show i) ("-ffp4e0m3 -r" ++ rm ++ " -- -" ++ show i)
]
| rm <- ["RNE", "RNA", "RTP", "RTN", "RTZ"]
, i :: Double <- [0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 8]
]
, testGroup "EncodeE8M0" [
gold "encodeE8M0_nan" "-fe8m0 nan" -- Representable, and uniquely so
, gold "encodeE8M0_+inf" "-fe8m0 inf" -- The limiting overflow; saturates
, gold "encodeE8M0_-inf" "-fe8m0 -- -inf" -- Negative: rejected, not saturated
, gold "encodeE8M0_neg" "-fe8m0 -- -5"
, gold "encodeE8M0_zero1" "-fe8m0 -- 0" -- No zero in the format; saturates up to 2^-127
, gold "encodeE8M0_zero2" "-fe8m0 -- -0" -- But a negative zero is still negative
, gold "encodeE8M0_one" "-fe8m0 -- 1"
, gold "encodeE8M0_exact" "-fe8m0 -- 0.25"
, gold "encodeE8M0_min" "-fe8m0 -- 0x1p-127"
, gold "encodeE8M0_max" "-fe8m0 -- 0x1p+127"
, gold "encodeE8M0_oob1" "-fe8m0 -- 1e40" -- Saturates
, gold "encodeE8M0_oob2" "-fe8m0 -- 1e-40"
, gold "encodeE8M0_hex" "-fe8m0 -- 0x1.8p1"
]
-- Every value that sits exactly half-way between two representable powers of
-- two, over all rounding modes. Note that 1.5 and 3 are what pin the RNE tie
-- rule down: they straddle stored exponents of opposite parity, so reading
-- "ties to even" as the unbiased exponent rather than the stored one would
-- send them the other way. All values are positive; negatives are rejected.
, testGroup "EncodeE8M0Ties" [
gold ("encodeE8M0_tie_" ++ rm ++ "_+" ++ show i) ("-fe8m0 -r" ++ rm ++ " -- " ++ show i)
| rm <- ["RNE", "RNA", "RTP", "RTN", "RTZ"]
, i :: Double <- [0.75, 1.5, 3, 6]
]
, testGroup "EncodeUE5M3" [
gold "encodeUE5M3_nan" "-fue5m3 nan" -- Representable, and uniquely so
, gold "encodeUE5M3_+inf" "-fue5m3 inf" -- No Inf to saturate to, so it becomes NaN
, gold "encodeUE5M3_-inf" "-fue5m3 -- -inf" -- Negative: rejected before the range check
, gold "encodeUE5M3_neg" "-fue5m3 -- -5"
, gold "encodeUE5M3_zero1" "-fue5m3 -- 0" -- Unlike E8M0, this format does have a zero
, gold "encodeUE5M3_zero2" "-fue5m3 -- -0" -- But a negative zero is still negative
, gold "encodeUE5M3_one" "-fue5m3 -- 1"
, gold "encodeUE5M3_exact" "-fue5m3 -- 2.5"
, gold "encodeUE5M3_sub" "-fue5m3 -- 0x1p-17" -- The smallest non-zero value there is
, gold "encodeUE5M3_subrnd" "-fue5m3 -- 1e-10" -- Under it, so rounds away to zero
, gold "encodeUE5M3_ieeemax" "-fue5m3 -- 61440" -- The largest value IEEE would have stopped at
, gold "encodeUE5M3_ieeernd" "-fue5m3 -- 61441" -- IEEE would overflow here; UE5M3 does not
, gold "encodeUE5M3_dev" "-fue5m3 -- 65536" -- First of the seven that deviate
, gold "encodeUE5M3_max" "-fue5m3 -- 114688" -- Last of them, and the largest value
, gold "encodeUE5M3_oob" "-fue5m3 -- 114689" -- Over the top: NaN, following E4M3
, gold "encodeUE5M3_hex" "-fue5m3 -- 0x1.8p1"
]
-- Every value that sits exactly half-way between two representable magnitudes, over
-- all rounding modes. These pin down the RNE tie rule where it is easiest to get
-- wrong: at the zero/subnormal boundary, across a binade, and -- for the last two --
-- across the point where UE5M3 parts company with IEEE and its top seven encodings
-- become ordinary numbers rather than infinity and NaN. All values are positive;
-- negatives are rejected outright.
, testGroup "EncodeUE5M3Ties" [
gold ("encodeUE5M3_tie_" ++ rm ++ "_" ++ nm) ("-fue5m3 -r" ++ rm ++ " -- " ++ v)
| rm <- ["RNE", "RNA", "RTP", "RTN", "RTZ"]
, (nm, v) <- [ ("sub0", "0x1p-18") -- Between zero and the smallest subnormal
, ("sub1", "0x3p-18") -- Between the two smallest subnormals
, ("norm", "1.0625")
, ("binade","1.9375") -- Straddles a binade boundary
, ("dev0", "63488") -- Straddles the last IEEE-shaped value
, ("dev1", "69632")
]
]
, testGroup "RoundingRegressions" [
encodingHas executable "SP RNE rounds upward"
["-fsp", "-rRNE", "1.00000006"] "Hex layout: 3F80 0001"
, encodingHas executable "SP RTZ rounds downward"
["-fsp", "-rRTZ", "1.00000006"] "Hex layout: 3F80 0000"
, encodingHas executable "DP RNE rounds upward"
["-fdp", "-rRNE", "1.0000000000000002"] "Hex layout: 3FF0 0000 0000 0001"
, encodingHas executable "DP RTZ rounds downward"
["-fdp", "-rRTZ", "1.0000000000000002"] "Hex layout: 3FF0 0000 0000 0000"
, encodingHas executable "E4M3 RTZ rounds positive downward"
["-fe4m3", "-rRTZ", "1.1"] "Hex layout: 38"
, encodingHas executable "E4M3 RTN rounds negative downward"
["-fe4m3", "-rRTN", "--", "-1.1"] "Hex layout: B9"
, encodingHas executable "E4M3 RTP uses upper extra value"
["-fe4m3", "-rRTP", "241"] "Hex layout: 78"
, encodingHas executable "E4M3 RTZ uses lower extra value"
["-fe4m3", "-rRTZ", "255"] "Hex layout: 77"
, encodingHas executable "E4M3 preserves digits above a midpoint"
["-fe4m3", "-rRNE", "1.06250000000000000001"] "Hex layout: 39"
, encodingHas executable "E4M3 preserves digits below a midpoint"
["-fe4m3", "-rRNE", "1.06249999999999999999"] "Hex layout: 38"
, encodingHas executable "FP4 preserves digits above a midpoint"
["-ffp4", "-rRNE", "1.25000000000000000001"] "Hex layout: 3"
, encodingHas executable "FP4E0M3 preserves digits above a midpoint"
["-ffp4e0m3", "-rRNE", "2.50000000000000000001"] "Hex layout: 3"
, encodingHas executable "E8M0 preserves digits below a midpoint"
["-fe8m0", "-rRNE", "1.49999999999999999999"] "Hex layout: 7F"
, encodingHas executable "UE5M3 preserves digits above a midpoint"
["-fue5m3", "-rRNE", "1.06250000000000000001"] "Hex layout: 79"
, encodingHas executable "SP accepts uppercase hexadecimal prefix"
["-fsp", "0X10"] "Hex layout: 4180 0000"
]
, testGroup "Decode" [
gold "decode0" "-i4 0b0110"
, gold "decode1" "-w4 0xE"
, gold "decode2" "-f3+4 0b0111001"
, gold "decode3" "-fbp 0x000F"
, gold "decode4" "-fdp 0x8000000000000000"
, gold "decode5" "-fhp 0x7c01"
, gold "decode6" "-fhp -l8 128'hffffffffffffffffbdffaaffdc71fc60"
, gold "decode7" "-fe5m2 0b01111011"
, gold "decode8" "-w2 -rRNE 2\'h1"
, gold "decode9" "-ftf32 19\'h0000F"
, gold "decode10" "-ftf32 0b0_10000000_0100000000"
]
, testGroup "DecodeE4M3" [
gold ("decodeE4M3_" ++ show (if sign then (-val :: Int) else val))
$ "-fe4m3 0b" ++ (if sign then "1" else "0") ++ "1111" ++ frac
| (val, frac) <- [ (256, "000")
, (288, "001")
, (320, "010")
, (352, "011")
, (384, "100")
, (416, "101")
, (448, "110")
]
, sign <- [False, True]
]
-- The patterns that are ordinary FP 4 4 values. These go through the look-alike
-- untouched, so they are the ones that used to leak its type name.
, testGroup "DecodeE4M3Regular" [
gold ("decodeE4M3_regular_" ++ bits) ("-fe4m3 0b" ++ bits)
| bits <- [ "00000000" -- +0
, "10000000" -- -0
, "00000001" -- Smallest subnormal
, "00111000" -- 1
, "10111100" -- -1.5
, "01110111" -- 240, the largest non-deviating magnitude
]
]
, testGroup "DecodeE4M3_NaN" [
gold "decodeE4M3_+NaN" "-fe4m3 0b_0111_1111"
, gold "decodeE4M3_-NaN" "-fe4m3 0b_1111_1111"
]
-- FP4 is small enough that we can simply decode every last one of its 16 patterns.
, testGroup "DecodeFP4" [
gold ("decodeFP4_" ++ bits) ("-ffp4 0b" ++ bits)
| s <- ["0", "1"]
, e <- ["00", "01", "10", "11"]
, m <- ["0", "1"]
, let bits = s ++ e ++ m
]
-- FP4E0M3 is small enough that we can simply decode every last one of its 16 patterns.
, testGroup "DecodeFP4E0M3" [
gold ("decodeFP4E0M3_" ++ bits) ("-ffp4e0m3 0b" ++ bits)
| s <- ["0", "1"]
, m <- ["000", "001", "010", "011", "100", "101", "110", "111"]
, let bits = s ++ m
]
-- E8M0 is all exponent, so decoding is a table lookup; a spread of patterns
-- covering both ends, the unit value, and the sole NaN is enough.
, testGroup "DecodeE8M0" [
gold ("decodeE8M0_" ++ bits) ("-fe8m0 0x" ++ bits)
| bits <- [ "00" -- Smallest: 2^-127. Not zero: the format has none
, "01"
, "7F" -- 1.0
, "80"
, "FD"
, "FE" -- Largest: 2^127
, "FF" -- NaN, and the only one
]
]
-- UE5M3 has 256 patterns, so we take a spread the way DecodeE8M0 does, covering
-- each structural case: the zero, both ends of the subnormals, the first normal,
-- the unit value, the last IEEE-shaped value, the deviants either side, and the
-- sole NaN.
, testGroup "DecodeUE5M3" [
gold ("decodeUE5M3_" ++ bits) ("-fue5m3 0x" ++ bits)
| bits <- [ "00" -- Zero
, "01" -- Smallest subnormal: 2^-17
, "07" -- Largest subnormal
, "08" -- Smallest normal
, "78" -- 1.0
, "F7" -- 61440: the largest an IEEE format of this shape would reach
, "F8" -- 65536: IEEE would call this infinity
, "FB" -- 90112: IEEE would call this NaN
, "FE" -- 114688, the largest value
, "FF" -- NaN, and the only one
]
]
, testGroup "Bad" [
gold "badInvocation0" "-f3+4 0b01"
, gold "badInvocation1" "-f3+4 0xFFFF"
-- We accept 1-bit exponents/significands, but the solver needs 2 of
-- each; make sure that surfaces as an error, not a raw exception.
, gold "badInvocation2" "-f3+1 0.5"
, gold "badInvocation3" "-f1+3 0b0000"
]
]