Alteryx.hs

{-# LANGUAGE OverloadedStrings #-}

module Tests.Database.Alteryx (yxdbTests) where

import qualified Database.Alteryx.CLI.Csv2Yxdb as C2Y
import qualified Database.Alteryx.CLI.Yxdb2Csv as Y2C
import Database.Alteryx
import Tests.Database.Alteryx.Arbitrary
import Tests.Database.Utils

import Prelude hiding (readFile)

import Control.Applicative
import Control.Lens hiding (elements)
import Control.Monad (replicateM, when)
import Control.Monad.State (evalStateT)
import Control.Monad.Trans.Resource
import Data.Array.IArray (listArray)
import Data.Binary
import Data.Binary.Get (runGet)
import Data.Binary.Put (runPut)
import Data.ByteString as BS
import Data.ByteString.Char8 as BSC
import Data.ByteString.Lazy as BSL
import Data.Conduit
import Data.Conduit.Combinators
import Data.Conduit.Lift
import Data.Conduit.List as CL
import qualified Data.Conduit.Text as CT
import qualified Data.CSV.Conduit as CSVT
import Data.Text as T
import Data.Text.Encoding as T
import Data.Text.IO as T

import Test.Framework
import Test.Framework.Providers.HUnit (testCase)
import Test.Framework.Providers.QuickCheck2 (testProperty)
import Test.HUnit
import Test.QuickCheck

assertEq :: (Eq a, Show a) => a -> a -> Property
assertEq a b = let
    aStr = show a
    bStr = show b
    in printTestCase ("\nA: " ++ aStr ++ "\nB: " ++ bStr) (a == b)

numBytes :: Binary a => a -> Int
numBytes x = fromIntegral $ BSL.length $ runPut $ put x

prop_HeaderLength :: Header -> Property
prop_HeaderLength header =
    assertEq headerPageSize (numBytes header)

prop_MetadataLength :: YxdbFile -> Property
prop_MetadataLength yxdb =
    assertEq (numMetadataBytesHeader $ yxdb ^. yxdbFileHeader)
             (numMetadataBytesActual $ yxdb ^. yxdbFileMetadata)

-- prop_BlocksLength :: YxdbFile -> Property
-- prop_BlocksLength yxdb =
--     assertEq (numBlocksBytesHeader $ header yxdb) (numBlocksBytesActual $ blocks yxdb)

prop_ValueGetAndPutAreInverses :: PairedValue -> Property
prop_ValueGetAndPutAreInverses (PairedValue field value) =
    assertEq (runGet (getValue field) (runPut $ putValue field value)) value

prop_BlockIndexGetAndPutAreInverses :: BlockIndex -> Property
prop_BlockIndexGetAndPutAreInverses x = assertEq (decode $ encode x) x

prop_MetadataGetAndPutAreInverses :: RecordInfo -> Property
prop_MetadataGetAndPutAreInverses x = assertEq (decode $ encode x) x

prop_HeaderGetAndPutAreInverses :: Header -> Property
prop_HeaderGetAndPutAreInverses x = assertEq (decode $ encode x) x

prop_BlocksGetAndPutAreInverses :: Block -> Property
prop_BlocksGetAndPutAreInverses x = assertEq (decode $ encode x) x

prop_YxdbFileGetAndPutAreInverses :: YxdbFile -> Property
prop_YxdbFileGetAndPutAreInverses x = assertEq (decode $ encode x) x

test_LoadingSmallModule :: Assertion
test_LoadingSmallModule = do
  parsed <- decodeFileOrFail "test-data/samples_of_various_field_types.yxdb"
  case parsed of
    Left (bytes, msg) -> assertFailure (msg ++ " at " ++ show bytes ++ " bytes")
    Right yxdbFile -> const (return ()) (yxdbFile :: YxdbFile)

csv2yxdb2csv :: C2Y.Settings -> FilePath -> FilePath -> IO T.Text
csv2yxdb2csv settings inputFilename outputFilename = do
  let c2ySettings = settings & C2Y.settingFilename .~ inputFilename
                             & C2Y.settingOutput .~ outputFilename
  let y2cSettings = Y2C.defaultSettings & Y2C.settingFilename .~ outputFilename
  evalStateT C2Y.runCsv2Yxdb c2ySettings
  newCsv <- T.decodeUtf8 <$> (captureStdout $ evalStateT Y2C.runYxdb2Csv y2cSettings)

  return newCsv

readCsvRecords :: Maybe Text -> FilePath -> IO [Record]
readCsvRecords header filename =
    runResourceT $
      sourceCsvRecords filename header alteryxCsvSettings $$
      sinkList

test_csv2yxdb2csvIsIdentity :: Assertion
test_csv2yxdb2csvIsIdentity = do
  let inputFilename = "test-data/samples.csv"
  let outputFilename = "test-data/samples.yxdb"
  newCsv <- csv2yxdb2csv C2Y.defaultSettings inputFilename outputFilename
  originalCsv <- T.readFile inputFilename
  assertEqual "CSV to YXDB to CSV" originalCsv newCsv

test_csv2yxdbQuoteParsing :: Assertion
test_csv2yxdbQuoteParsing = do
  let inputFilename = "test-data/quote-parsing.csv"
  let outputFilename = "test-data/quote-parsing.yxdb"
  oldCsv <- T.readFile inputFilename
  newCsv <- csv2yxdb2csv C2Y.defaultSettings inputFilename outputFilename
  assertEqual "" oldCsv newCsv

test_csv2yxdbDateFormat :: Assertion
test_csv2yxdbDateFormat = do
  let inputFilename = "test-data/date-format.csv"
      outputFilename = "test-data/date-format.yxdb"
  oldCsv <- T.readFile inputFilename
  newCsv <- csv2yxdb2csv C2Y.defaultSettings inputFilename outputFilename
  assertEqual "" oldCsv newCsv

nonAnsiCodepointFieldValue :: Maybe FieldValue
nonAnsiCodepointFieldValue = Just $ FVString "plátano"

nonAnsiCodepointField :: Field
nonAnsiCodepointField = Field {
                          _fieldName = "derp",
                          _fieldType = FTString,
                          _fieldSize = Just 10,
                          _fieldScale = Nothing
                        }

test_renderNonAnsiCodepoint :: Assertion
test_renderNonAnsiCodepoint = do
  let fieldValue = nonAnsiCodepointFieldValue
      field = nonAnsiCodepointField
      bs = runPut (putValue field fieldValue)
      parsedValue = runGet (getValue field) bs
  assertEqual "Converting a non-ANSI character lost information" fieldValue parsedValue

test_renderNonAnsiCodepointRecord :: Assertion
test_renderNonAnsiCodepointRecord = do
    let fieldValue = nonAnsiCodepointFieldValue
        field = nonAnsiCodepointField

        record = Record [ fieldValue ]
        recordInfo = RecordInfo [ field ]

        record2recordConduit = record2csv recordInfo =$=
                         csv2bytes =$=
                         CT.decode CT.utf8 =$=
                         csv2records alteryxCsvSettings
    newRecords <- runResourceT $ CL.sourceList [ record ] =$= record2recordConduit $$ sinkList

    let result = Prelude.head newRecords
    assertEqual "Converting a non-ANSI character lost information" record result

test_renderNonAnsiCodepointCSVRecord :: Assertion
test_renderNonAnsiCodepointCSVRecord =
  let csvText = "derp:string(10)\nplátano\n"
      recordInfo = RecordInfo [ nonAnsiCodepointField ]
      csv2csvConduit =
          csv2records alteryxCsvSettings =$=
          record2csv recordInfo =$=
          csv2bytes =$=
          CT.decode CT.utf8

  in do
    newCsv <- runResourceT $ CL.sourceList [ csvText ] =$= csv2csvConduit $$ sinkList
    assertEqual "" csvText $ T.concat newCsv

test_renderNonAnsiCodepointCSV2Record2Block2Record2CSV :: Assertion
test_renderNonAnsiCodepointCSV2Record2Block2Record2CSV =
  let csvText = "derp:string(10)\nplátano\n"
      recordInfo = RecordInfo [ nonAnsiCodepointField ]
      csv2csvConduit =
          csv2records alteryxCsvSettings =$=
          evalStateLC defaultStatistics (recordsToBlocks recordInfo) =$=
          blocksToRecords recordInfo =$=
          record2csv recordInfo =$=
          csv2bytes =$=
          CT.decode CT.utf8
  in do
    newCsv <- runResourceT $ CL.sourceList [ csvText ] =$= csv2csvConduit $$ sinkList
    assertEqual "" csvText $ T.concat newCsv


test_recordParsing :: Assertion
test_recordParsing = do
  let inputFilename = "test-data/samples.csv"
      outputFilename = "test-data/samples.yxdb"
      secondOutputFilename = "test-data/samples.yxdb.csv"
      expectedRecord =
          [
           Record [
            Just (FVByte 1),
            Just (FVInt16 256),
            Just (FVInt32 32768),
            Just (FVInt64 4294967296),
            Just (FVString "1.10000"),
            Just (FVFloat 1.1),
            Just (FVDouble 1.1),
            Just (FVString "plátano"),
            Just (FVWString "香蕉"),
            Just (FVString "plátano"),
            Just (FVWString "香蕉"),
            Just (FVString "2013-12-31"),
            Just (FVString "23:34:56"),
            Just (FVString "2013-12-31 23:34:56")
           ]
          ]
  records <- readCsvRecords Nothing inputFilename
  assertEqual "Parse from CSV" expectedRecord records

  newCsv <- csv2yxdb2csv C2Y.defaultSettings inputFilename outputFilename
  yxdbFile <- decodeFile outputFilename :: IO YxdbFile

  T.writeFile secondOutputFilename newCsv
  newRecords <- readCsvRecords Nothing secondOutputFilename

  assertEqual "Parse from YXDB" expectedRecord (yxdbFile ^. yxdbFileRecords)
  assertEqual "New CSV doesn't match old" expectedRecord newRecords

test_signedRecordParsing :: Assertion
test_signedRecordParsing = do
  let inputFilename = "test-data/signed.csv"
      outputFilename = "test-data/signed.yxdb"
      secondOutputFilename = "test-data/signed.yxdb.csv"
      expectedRecord =
          [
           Record [
            Just (FVByte $ -1),
            Just (FVInt16 $ -256),
            Just (FVInt32 $ -32768),
            Just (FVInt64 $ -4294967296),
            Just (FVString "1.10000"),
            Just (FVFloat $ -1.1),
            Just (FVDouble $ -1.1)
           ]
          ]
  records <- readCsvRecords Nothing inputFilename
  assertEqual "Parse from CSV" expectedRecord records

  newCsv <- csv2yxdb2csv C2Y.defaultSettings inputFilename outputFilename
  yxdbFile <- decodeFile outputFilename :: IO YxdbFile

  T.writeFile secondOutputFilename newCsv
  newRecords <- readCsvRecords Nothing secondOutputFilename

  assertEqual "Parse from YXDB" expectedRecord (yxdbFile ^. yxdbFileRecords)
  assertEqual "New CSV doesn't match old" expectedRecord newRecords

yxdbTests :: Test.Framework.Test
yxdbTests =
    testGroup "YXDB" [
        testProperty "Header length" prop_HeaderLength,
        testProperty "Metadata length" prop_MetadataLength,
--        testProperty "Blocks length" prop_BlocksLength, --
        testProperty "Block Index get & put inverses" prop_BlockIndexGetAndPutAreInverses,
        testProperty "Metadata get and put inverses" prop_MetadataGetAndPutAreInverses,
        testProperty "Header get & put inverses" prop_HeaderGetAndPutAreInverses,
        testProperty "Value get & put inverses" prop_ValueGetAndPutAreInverses,
        testProperty "Blocks get & put inverses" prop_BlocksGetAndPutAreInverses,
--        testProperty "Yxdb get & put inverses" prop_YxdbFileGetAndPutAreInverses,
        testCase "Loading small module" test_LoadingSmallModule,
        testCase "CSV to YXDB to CSV is the identity" test_csv2yxdb2csvIsIdentity,
        testCase "Quotes are not used to escape" test_csv2yxdbQuoteParsing,
        testCase "Can parse ISO style dates" test_csv2yxdbDateFormat,
        testCase "Example CSV parses into correct structures" test_recordParsing,
        testCase "Example CSV with signed values" test_signedRecordParsing,
        testCase "Rendering a non-ANSI codepoint" test_renderNonAnsiCodepoint,
        testCase "Rendering a non-ANSI codepoint at record level" test_renderNonAnsiCodepointRecord,
        testCase "Rendering a non-ANSI codepoint from CSV to CSV" test_renderNonAnsiCodepointCSVRecord,
        testCase "Rendering a non-ANSI codepoint from CSV to YXDB blocks to CSV" test_renderNonAnsiCodepointCSV2Record2Block2Record2CSV
    ]