tahoe-chk-0.1.0.2: test/SpecCHK.hs

{-# LANGUAGE OverloadedStrings #-}

module SpecCHK (
    tests,
    testsFromVectors,
) where

import Control.Arrow (
    (&&&),
 )
import Crypto.Cipher.AES128 (
    AESKey128,
 )
import Crypto.Classes (
    encode,
 )
import qualified Data.Binary as Binary
import qualified Data.ByteString as B
import qualified Data.ByteString.Base64 as Base64
import qualified Data.ByteString.Lazy as BL
import Data.Coerce (coerce)
import Data.Text (
    Text,
    concat,
    unpack,
 )
import Data.TreeDiff.Class (ToExpr, ediff)
import Data.TreeDiff.Pretty (prettyEditExpr)
import GHC.Generics (Generic)
import Tahoe.CHK.URIExtension (uriExtCodecParams)

import Control.Monad.IO.Class (MonadIO (liftIO))
import Data.ByteString.Base32 (decodeBase32Unpadded)
import Data.Word (Word32)
import Generators (genParameters, shares)
import Hedgehog (
    Property,
    assert,
    diff,
    forAll,
    property,
    tripping,
 )
import qualified Hedgehog.Gen as Gen
import qualified Hedgehog.Range as Range
import Tahoe.CHK (padCiphertext)
import qualified Tahoe.CHK (decode, encode, segmentCiphertext)
import Tahoe.CHK.Capability (CHK (CHKReader), dangerRealShow, pCapability, pReader)
import Tahoe.CHK.Crypto (convergenceSecretLength)
import Tahoe.CHK.Encrypt (encrypt)
import Tahoe.CHK.Share (
    Share (
        shareBlockSize,
        shareURIExtension
    ),
 )
import Tahoe.CHK.Types (
    Parameters (..),
 )
import Tahoe.CHK.Upload (
    UploadResult (..),
    Uploadable (..),
    adjustSegmentSize,
    encryptAndEncode,
    getConvergentKey,
    memoryUploadableWithConvergence,
    store,
 )
import Tahoe.Server (
    nullStorageServer,
 )
import Test.Tasty (
    TestTree,
    testGroup,
 )
import Test.Tasty.HUnit (
    Assertion,
    assertBool,
    assertEqual,
    assertFailure,
    testCase,
 )
import Test.Tasty.Hedgehog (testProperty)
import Text.Megaparsec (parse)
import Vectors (
    Format (..),
    JSONByteString (..),
    Sample (..),
    TestCase (..),
    VectorSpec (..),
 )

assertEqual' :: (Generic a, ToExpr a, Eq a) => a -> a -> Assertion
assertEqual' a b = assertBool (show . prettyEditExpr $ ediff a b) (a == b)

-- | Create tests for each case in the test vector specification.
testsFromVectors :: VectorSpec -> TestTree
testsFromVectors vectorSpec =
    testGroup
        "Vectors"
        [ testCap vectorSpec
        , testCapabilityParser vectorSpec
        ]

tests :: TestTree
tests =
    testGroup
        "CHK"
        [ testEncrypt
        , testProperty "expand returns the correct number of bytes" prop_expand_length
        , testProperty "expand returns bytes containing the template repeated" prop_expand_template
        , testProperty "Share round-trips through put / get" prop_share_roundtrip
        , testWellKnownShare1of2
        , testWellKnownShare2of3
        , testWellKnownShare3of10
        , testProperty "segmentCiphertext preserves all of the ciphertext" prop_segmentCiphertext_identity
        , testProperty "padCiphertext returns a string with a length that is a multiple of the given requiredShares value" prop_paddedCiphertext_boundary
        , testProperty "ciphertext round-trips through decode . encode" prop_share_encoding_roundtrip
        , testSizes
        , testOutOfBoundsShareNumbers
        ]

testSizes :: TestTree
testSizes =
    testCase "the maximum segment size encoded in the UEB equals the actual segment size" $ do
        uploadable <- memoryUploadableWithConvergence (B.replicate 32 0x00) (fromIntegral $ BL.length ciphertext) ciphertext params
        (shares', _cap) <- Tahoe.CHK.encode (uploadableKey uploadable) params ciphertext
        mapM_ (assertEqual "The shareBlockSize reflects the parameters and real ciphertext size" (fromIntegral $ BL.length ciphertext `div` 2) . shareBlockSize) shares'
        mapM_ (assertEqual "The segment size is reduced to the ciphertext size" (fromIntegral $ BL.length ciphertext) . getSegmentSize) shares'
  where
    getSegmentSize = paramSegmentSize . uriExtCodecParams . shareURIExtension
    params =
        Parameters
            { paramSegmentSize = 100000
            , paramTotalShares = 3
            , paramHappyShares = 1
            , paramRequiredShares = 2
            }
    ciphertext = BL.pack [1 .. 56]

{- | segmentCiphertext may split up ciphertext but it may not change its
 content in any way.
-}
prop_segmentCiphertext_identity :: Property
prop_segmentCiphertext_identity = property $ do
    ciphertext <- forAll $ BL.fromStrict <$> Gen.bytes (Range.linear 1 1024)
    params <- forAll genParameters

    let segments = Tahoe.CHK.segmentCiphertext params ciphertext
        recovered = BL.concat segments

    diff ciphertext (==) recovered

prop_paddedCiphertext_boundary :: Property
prop_paddedCiphertext_boundary = property $ do
    ciphertext <- forAll $ BL.fromStrict <$> Gen.bytes (Range.linear 1 1024)
    Parameters{paramRequiredShares} <- forAll genParameters

    let padded = padCiphertext paramRequiredShares ciphertext

    diff (BL.length padded `mod` fromIntegral paramRequiredShares) (==) 0

wellKnownCase :: Parameters -> (Int -> FilePath) -> Text -> Assertion
wellKnownCase params pathToExpected expectedCap =
    let -- Must be at least 56 bytes or we cannot get shares for
        -- comparison out of Tahoe-LAFS (instead, it emits a LIT cap).
        plaintext =
            "abcdefghijklmnopqrstuvwxyz\
            \ZYXWVUTSRQPONMLKJIJHGRFCBA\
            \1357"

        -- Hard-code the particular convergence secret used to generated
        -- the expected value.
        Right convergenceSecret = decodeBase32Unpadded "lcngfrvgaksfwrelc6ae5kucb3zufssoe6cj74rozcqibnl6uy2a"

        extractShareData :: BL.ByteString -> BL.ByteString
        extractShareData container = shareData
          where
            shareData = BL.take (fromIntegral shareDataLength) . BL.drop 0x0c $ container
            shareDataLength = Binary.decode . BL.take 4 . BL.drop 4 $ container :: Word32
     in do
            uploadable <- memoryUploadableWithConvergence convergenceSecret (fromIntegral $ BL.length plaintext) plaintext params
            let ciphertext = encrypt (uploadableKey uploadable) plaintext
            (shares', cap) <- Tahoe.CHK.encode (uploadableKey uploadable) params ciphertext

            -- Encoded by Tahoe-LAFS itself, hacked to use an 8 byte
            -- maximum segment size.
            shareContainers <- mapM BL.readFile (pathToExpected <$> [0 .. length shares' - 1])

            -- Find the real share data admist the storage server framing
            -- and metadata.  encode only produces the share data so we
            -- must scrape it out of the server storage format files we
            -- have as test data.  what encryptAndEncode is expected to
            -- produce.
            let expectedEncoded = extractShareData <$> shareContainers
                expectedShares = Binary.decode <$> expectedEncoded
                encodedShares = Binary.encode <$> shares'

            assertEqual' expectedShares shares'
            assertEqual' expectedEncoded encodedShares
            assertEqual "The cap matches" (dangerRealShow $ CHKReader cap) expectedCap

testWellKnownShare3of10 :: TestTree
testWellKnownShare3of10 =
    testCase
        "a known 3-of-10 case encodes as expected"
        ( wellKnownCase
            Parameters
                { paramSegmentSize = 8
                , paramTotalShares = 10
                , paramHappyShares = 1
                , paramRequiredShares = 3
                }
            (("test/3of10." <>) . show)
            "URI:CHK:o4lpfdvt7ib5xei2qhz6ovkz34:uvhgccbgigj4gfqfeyh5g5uogyt7etmlmqnvswqxumm7q3rqh7uq:3:10:56"
        )

testWellKnownShare2of3 :: TestTree
testWellKnownShare2of3 =
    testCase
        "a known 2-of-3 case encodes as expected"
        -- Just match the parameters given to Tahoe-LAFS when the test value
        -- was generated.
        ( wellKnownCase
            Parameters
                { paramSegmentSize = 8
                , paramTotalShares = 3
                , paramHappyShares = 1
                , paramRequiredShares = 2
                }
            (("test/2of3." <>) . show)
            "URI:CHK:co4s2wzrwos726nu24ervz2ffu:orrq3znudwnwgcazuc7qbm3prf4a46c3gmboecbror4l2k62jtkq:2:3:56"
        )

testWellKnownShare1of2 :: TestTree
testWellKnownShare1of2 =
    testCase
        "a known 1-of-2 case encodes as expected"
        ( wellKnownCase
            Parameters
                { paramSegmentSize = 8
                , paramTotalShares = 2
                , paramHappyShares = 1
                , paramRequiredShares = 1
                }
            (("test/1of2." <>) . show)
            "URI:CHK:pyv3qypbpk6knq5ozeibenuubq:jh3twlgmxtytwqtzn6jtbsfy2w574ybkcnalurlnlq2snuu3j5da:1:2:56"
        )

prop_share_encoding_roundtrip :: Property
prop_share_encoding_roundtrip = property $ do
    convergenceSecret <- forAll $ Gen.bytes (Range.singleton 32)
    ciphertext <- forAll $ BL.fromStrict <$> Gen.bytes (Range.linear 1 2048)
    params <- forAll $ fixParams <$> genParameters
    let key = getConvergentKey convergenceSecret (adjustSegmentSize params (fromIntegral $ BL.length ciphertext)) ciphertext
    (shares', cap) <- liftIO $ Tahoe.CHK.encode key params ciphertext
    recovered <- liftIO $ Tahoe.CHK.decode cap (zip [0 ..] shares')

    diff (Just ciphertext) (==) recovered
  where
    -- XXX Our ZFEC bindings are unhappy with k == n.  genParameters will
    -- happily give us that so adjust k or n if we happen to hit such a case.
    fixParams p@Parameters{paramRequiredShares = 256, paramTotalShares = 256} = p{paramRequiredShares = 255}
    fixParams p@Parameters{paramRequiredShares, paramTotalShares}
        | paramRequiredShares == paramRequiredShares = p{paramTotalShares = paramTotalShares + 1}
        | otherwise = p

prop_share_roundtrip :: Property
prop_share_roundtrip =
    let decode' = ((\(_, _, sh) -> sh) <$>) . Binary.decodeOrFail
     in property $ do
            share <- forAll shares
            tripping share Binary.encode decode'

testEncrypt :: TestTree
testEncrypt =
    testGroup
        "chkEncrypt"
        [ testCase "ciphertext" $ do
            assertEqual
                "expected convergence key"
                "oBcuR/wKdCgCV2GKKXqiNg=="
                (Base64.encode $ encode convergenceKey)
            let b64ciphertext = Base64.encode (BL.toStrict ciphertext)
            assertEqual "known result" knownCorrect b64ciphertext
        ]
  where
    -- For all the magic values see
    -- allmydata.test.test_upload.FileHandleTests.test_get_encryption_key_convergent
    knownCorrect :: B.ByteString
    knownCorrect = "Jd2LHCRXozwrEJc="

    plaintext :: BL.ByteString
    plaintext = "hello world"

    ciphertext :: BL.ByteString
    ciphertext = encrypt convergenceKey plaintext

    convergenceKey :: AESKey128
    convergenceKey = getConvergentKey convergenceSecret params plaintext

    convergenceSecret = B.replicate convergenceSecretLength 0x42
    params =
        adjustSegmentSize
            Parameters
                { paramSegmentSize = 128 * 1024
                , paramTotalShares = 10
                , paramHappyShares = 5
                , paramRequiredShares = 3
                }
            (fromIntegral $ BL.length plaintext)

{- | Build a test tree that applies a test function to every CHK case in a
 test vector.
-}
chkTests ::
    -- | A name to give the group of tests.
    String ->
    -- | A function to call with a CHK test case to get back a test.
    (TestCase -> Assertion) ->
    -- | The test vector containing CHK test cases.
    VectorSpec ->
    -- | A test tree with one test per CHK case in the test vector.
    TestTree
chkTests name makeOneTest =
    testGroup name . map (uncurry ($) . (testCase . unpack . expected &&& makeOneTest)) . filter pickCase . vector
  where
    pickCase TestCase{format, zfec} = format == CHK && (paramTotalShares zfec > paramRequiredShares zfec && paramTotalShares zfec < 256)

{- | Every CHK case in the test vector can be reproduced by this
 implementation.
-}
testCap :: VectorSpec -> TestTree
testCap = chkTests "chkCap" testOneCase

{- | Every CHK capability in the test vector can be parsed and then serialized
 back to the same byte string.
-}
testCapabilityParser :: VectorSpec -> TestTree
testCapabilityParser = chkTests "testCapabilityParser" testParseOneCapability

{- | Assert that a specific CHK capability can be parsed and serialized back
 to the same byte string.
-}
testParseOneCapability :: TestCase -> Assertion
testParseOneCapability TestCase{expected} = do
    serialized <- case parse pCapability "" expected of
        Left err -> assertFailure $ show err
        Right cap -> pure $ dangerRealShow cap
    assertEqual "expected /= serialized" expected serialized

{- | Assert that verify and read capability strings with n/k/size below the
 minimum legal or above the maximum legal value are rejected by the parser.
-}
testOutOfBoundsShareNumbers :: TestTree
testOutOfBoundsShareNumbers =
    testCase
        "out-of-bounds share numbers cause capability string parse errors"
        $ mapM_ assertParseFail cases
  where
    cases =
        [ -- Verify caps with n/k/size too small
          "URI:CHK-Verifier:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:0:1:56"
        , "URI:CHK-Verifier:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:0:56"
        , "URI:CHK-Verifier:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:1:0"
        , -- Read caps with n/k/size too small
          "URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:0:1:56"
        , "URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:0:56"
        , "URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:1:0"
        , -- Verify caps with n/k/size too large
          "URI:CHK-Verifier:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:257:256:1000"
        , "URI:CHK-Verifier:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:256:257:1000"
        , "URI:CHK-Verifier:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:256:256:18446744073709551616"
        , -- Read caps with n/k/size too large
          "URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:257:256:1000"
        , "URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:256:257:1000"
        , "URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:256:256:18446744073709551616"
        ]

    assertParseFail s =
        case parse pCapability "" s of
            Left _err -> pure ()
            Right cap ->
                assertFailure . unpack . Data.Text.concat $
                    [ "Expected parse failure of "
                    , s
                    , " instead got "
                    , dangerRealShow cap
                    ]

{- | Assert that a specific CHK case can be reproduced by this implementation.
 This means we can encode the same plaintext using the same secrets to the
 same ciphertext and share layout and that the resulting capability string
 is the same byte sequence as given by the test vector.
-}
testOneCase :: TestCase -> Assertion
testOneCase
    TestCase
        { convergence
        , format = CHK
        , sample
        , zfec
        , expected
        } =
        do
            uploadable <- memoryUploadableWithConvergence (coerce convergence) (fromIntegral $ sampleLength sample) (BL.fromStrict $ expand sample) zfec
            upresult <- store [nullStorageServer] uploadable
            assertEqual "yes" (parse pReader "" expected) (Right $ uploadResultReadCap upresult)
testOneCase x = error $ "testOneCase got bad input" <> show x

expand :: Sample -> B.ByteString
expand (Sample sampleTemplate sampleLength) =
    B.take sampleLength . B.concat $ take sampleLength (replicate n bs)
  where
    n = (sampleLength `div` B.length bs) + 1
    bs = coerce sampleTemplate -- yuck

prop_expand_length :: Property
prop_expand_length =
    property $ do
        sample <- forAll $ Sample <$> (JSONByteString <$> Gen.bytes (Range.linear 1 16)) <*> Gen.int (Range.linear 1 1000)
        diff (sampleLength sample) (==) (B.length $ expand sample)

prop_expand_template :: Property
prop_expand_template =
    property $ do
        template <- forAll $ Gen.bytes (Range.linear 1 16)
        sample <- forAll $ Sample (JSONByteString template) <$> Gen.int (Range.linear 1 1000)
        assert $ checkTemplate template (expand sample)
  where
    checkTemplate :: B.ByteString -> B.ByteString -> Bool
    checkTemplate _ "" = True
    checkTemplate template expanded =
        all (uncurry (==)) (B.zip template expanded)
            && checkTemplate template (B.drop (B.length template) expanded)