dataframe-1.0.0.0: tests/Parquet.hs
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE TypeApplications #-}
module Parquet where
import Assertions (assertExpectException)
import qualified DataFrame as D
import qualified DataFrame.Functions as F
import qualified DataFrame.IO.Parquet as DP
import qualified Data.ByteString as BS
import Data.Int
import qualified Data.Set as S
import Data.Text (Text)
import qualified Data.Text as T
import Data.Time
import Data.Word
import DataFrame.IO.Parquet.Thrift (
columnMetaData,
columnPathInSchema,
columnStatistics,
rowGroupColumns,
rowGroups,
schema,
)
import DataFrame.IO.Parquet.Types (columnNullCount)
import DataFrame.Internal.Binary (
littleEndianWord32,
littleEndianWord64,
word32ToLittleEndian,
word64ToLittleEndian,
)
import DataFrame.Internal.Column (hasMissing)
import DataFrame.Internal.DataFrame (unsafeGetColumn)
import GHC.IO (unsafePerformIO)
import Test.HUnit
allTypes :: D.DataFrame
allTypes =
D.fromNamedColumns
[ ("id", D.fromList [4 :: Int32, 5, 6, 7, 2, 3, 0, 1])
, ("bool_col", D.fromList [True, False, True, False, True, False, True, False])
, ("tinyint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1])
, ("smallint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1])
, ("int_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1])
, ("bigint_col", D.fromList [0 :: Int64, 10, 0, 10, 0, 10, 0, 10])
, ("float_col", D.fromList [0 :: Float, 1.1, 0, 1.1, 0, 1.1, 0, 1.1])
, ("double_col", D.fromList [0 :: Double, 10.1, 0, 10.1, 0, 10.1, 0, 10.1])
,
( "date_string_col"
, D.fromList
[ "03/01/09" :: Text
, "03/01/09"
, "04/01/09"
, "04/01/09"
, "02/01/09"
, "02/01/09"
, "01/01/09"
, "01/01/09"
]
)
, ("string_col", D.fromList (take 8 (cycle ["0" :: Text, "1"])))
,
( "timestamp_col"
, D.fromList
[ UTCTime{utctDay = fromGregorian 2009 3 1, utctDayTime = secondsToDiffTime 0}
, UTCTime{utctDay = fromGregorian 2009 3 1, utctDayTime = secondsToDiffTime 60}
, UTCTime{utctDay = fromGregorian 2009 4 1, utctDayTime = secondsToDiffTime 0}
, UTCTime{utctDay = fromGregorian 2009 4 1, utctDayTime = secondsToDiffTime 60}
, UTCTime{utctDay = fromGregorian 2009 2 1, utctDayTime = secondsToDiffTime 0}
, UTCTime{utctDay = fromGregorian 2009 2 1, utctDayTime = secondsToDiffTime 60}
, UTCTime{utctDay = fromGregorian 2009 1 1, utctDayTime = secondsToDiffTime 0}
, UTCTime{utctDay = fromGregorian 2009 1 1, utctDayTime = secondsToDiffTime 60}
]
)
]
testBothReadParquetPaths :: ((FilePath -> IO D.DataFrame) -> Test) -> Test
testBothReadParquetPaths test =
TestList
[ test D.readParquet
, test (DP._readParquetWithOpts (Just True) D.defaultParquetReadOptions)
]
allTypesPlain :: Test
allTypesPlain = testBothReadParquetPaths $ \readParquet ->
TestCase
( assertEqual
"allTypesPlain"
allTypes
(unsafePerformIO (readParquet "./tests/data/alltypes_plain.parquet"))
)
allTypesTinyPagesDimensions :: Test
allTypesTinyPagesDimensions =
TestCase
( assertEqual
"allTypesTinyPages last few"
(7300, 13)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/alltypes_tiny_pages.parquet"))
)
)
tinyPagesLast10 :: D.DataFrame
tinyPagesLast10 =
D.fromNamedColumns
[ ("id", D.fromList @Int32 (reverse [6174 .. 6183]))
, ("bool_col", D.fromList @Bool (Prelude.take 10 (cycle [False, True])))
, ("tinyint_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4])
, ("smallint_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4])
, ("int_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4])
, ("bigint_col", D.fromList @Int64 [30, 20, 10, 0, 90, 80, 70, 60, 50, 40])
,
( "float_col"
, D.fromList @Float [3.3, 2.2, 1.1, 0, 9.9, 8.8, 7.7, 6.6, 5.5, 4.4]
)
,
( "date_string_col"
, D.fromList @Text
[ "09/11/10"
, "09/11/10"
, "09/11/10"
, "09/11/10"
, "09/10/10"
, "09/10/10"
, "09/10/10"
, "09/10/10"
, "09/10/10"
, "09/10/10"
]
)
,
( "string_col"
, D.fromList @Text ["3", "2", "1", "0", "9", "8", "7", "6", "5", "4"]
)
,
( "timestamp_col"
, D.fromList @UTCTime
[ UTCTime
{ utctDay = fromGregorian 2010 9 10
, utctDayTime = secondsToDiffTime 85384
}
, UTCTime
{ utctDay = fromGregorian 2010 9 10
, utctDayTime = secondsToDiffTime 85324
}
, UTCTime
{ utctDay = fromGregorian 2010 9 10
, utctDayTime = secondsToDiffTime 85264
}
, UTCTime
{ utctDay = fromGregorian 2010 9 10
, utctDayTime = secondsToDiffTime 85204
}
, UTCTime
{ utctDay = fromGregorian 2010 9 9
, utctDayTime = secondsToDiffTime 85144
}
, UTCTime
{ utctDay = fromGregorian 2010 9 9
, utctDayTime = secondsToDiffTime 85084
}
, UTCTime
{ utctDay = fromGregorian 2010 9 9
, utctDayTime = secondsToDiffTime 85024
}
, UTCTime
{ utctDay = fromGregorian 2010 9 9
, utctDayTime = secondsToDiffTime 84964
}
, UTCTime
{ utctDay = fromGregorian 2010 9 9
, utctDayTime = secondsToDiffTime 84904
}
, UTCTime
{ utctDay = fromGregorian 2010 9 9
, utctDayTime = secondsToDiffTime 84844
}
]
)
, ("year", D.fromList @Int32 (replicate 10 2010))
, ("month", D.fromList @Int32 (replicate 10 9))
]
allTypesTinyPagesLastFew :: Test
allTypesTinyPagesLastFew = testBothReadParquetPaths $ \readParquet ->
TestCase
( assertEqual
"allTypesTinyPages dimensions"
tinyPagesLast10
( unsafePerformIO
-- Excluding doubles because they are weird to compare.
( fmap
(D.takeLast 10 . D.exclude ["double_col"])
(readParquet "./tests/data/alltypes_tiny_pages.parquet")
)
)
)
allTypesPlainSnappy :: Test
allTypesPlainSnappy = testBothReadParquetPaths $ \readParquet ->
TestCase
( assertEqual
"allTypesPlainSnappy"
(D.filter (F.col @Int32 "id") (`elem` [6, 7]) allTypes)
(unsafePerformIO (readParquet "./tests/data/alltypes_plain.snappy.parquet"))
)
allTypesDictionary :: Test
allTypesDictionary = testBothReadParquetPaths $ \readParquet ->
TestCase
( assertEqual
"allTypesPlainSnappy"
(D.filter (F.col @Int32 "id") (`elem` [0, 1]) allTypes)
(unsafePerformIO (readParquet "./tests/data/alltypes_dictionary.parquet"))
)
selectedColumnsWithOpts :: Test
selectedColumnsWithOpts =
TestCase
( assertEqual
"selectedColumnsWithOpts"
(D.select ["id", "bool_col"] allTypes)
( unsafePerformIO
( D.readParquetWithOpts
(D.defaultParquetReadOptions{D.selectedColumns = Just ["id", "bool_col"]})
"./tests/data/alltypes_plain.parquet"
)
)
)
rowRangeWithOpts :: Test
rowRangeWithOpts =
TestCase
( assertEqual
"rowRangeWithOpts"
(3, 11)
( unsafePerformIO
( D.dimensions
<$> D.readParquetWithOpts
(D.defaultParquetReadOptions{D.rowRange = Just (2, 5)})
"./tests/data/alltypes_plain.parquet"
)
)
)
predicateWithOpts :: Test
predicateWithOpts =
TestCase
( assertEqual
"predicateWithOpts"
(D.fromNamedColumns [("id", D.fromList [6 :: Int32, 7])])
( unsafePerformIO
( D.readParquetWithOpts
( D.defaultParquetReadOptions
{ D.selectedColumns = Just ["id"]
, D.predicate =
Just
( F.geq
(F.col @Int32 "id")
(F.lit (6 :: Int32))
)
}
)
"./tests/data/alltypes_plain.parquet"
)
)
)
predicateUsesNonSelectedColumnWithOpts :: Test
predicateUsesNonSelectedColumnWithOpts =
TestCase
( assertEqual
"predicateUsesNonSelectedColumnWithOpts"
(D.fromNamedColumns [("bool_col", D.fromList [True, False])])
( unsafePerformIO
( D.readParquetWithOpts
( D.defaultParquetReadOptions
{ D.selectedColumns = Just ["bool_col"]
, D.predicate =
Just
( F.geq
(F.col @Int32 "id")
(F.lit (6 :: Int32))
)
}
)
"./tests/data/alltypes_plain.parquet"
)
)
)
predicateWithOptsAcrossFiles :: Test
predicateWithOptsAcrossFiles =
TestCase
( assertEqual
"predicateWithOptsAcrossFiles"
(4, 1)
( unsafePerformIO
( D.dimensions
<$> D.readParquetFilesWithOpts
( D.defaultParquetReadOptions
{ D.selectedColumns = Just ["id"]
, D.predicate =
Just
( F.geq
(F.col @Int32 "id")
(F.lit (6 :: Int32))
)
}
)
"./tests/data/alltypes_plain*.parquet"
)
)
)
missingSelectedColumnWithOpts :: Test
missingSelectedColumnWithOpts =
TestCase
( assertExpectException
"missingSelectedColumnWithOpts"
"Column not found"
( D.readParquetWithOpts
(D.defaultParquetReadOptions{D.selectedColumns = Just ["does_not_exist"]})
"./tests/data/alltypes_plain.parquet"
)
)
transactions :: D.DataFrame
transactions =
D.fromNamedColumns
[ ("transaction_id", D.fromList [1 :: Int32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
,
( "event_time"
, D.fromList
[ UTCTime
{ utctDay = fromGregorian 2024 1 3
, utctDayTime = secondsToDiffTime 29564 + picosecondsToDiffTime 2311000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 3
, utctDayTime = secondsToDiffTime 35101 + picosecondsToDiffTime 118900000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 4
, utctDayTime = secondsToDiffTime 39802 + picosecondsToDiffTime 774512000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 5
, utctDayTime = secondsToDiffTime 53739 + picosecondsToDiffTime 1000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 6
, utctDayTime = secondsToDiffTime 8278 + picosecondsToDiffTime 543210000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 6
, utctDayTime = secondsToDiffTime 8284 + picosecondsToDiffTime 211000000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 7
, utctDayTime = secondsToDiffTime 63000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 8
, utctDayTime = secondsToDiffTime 24259 + picosecondsToDiffTime 390000000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 9
, utctDayTime = secondsToDiffTime 48067 + picosecondsToDiffTime 812345000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 10
, utctDayTime = secondsToDiffTime 82799 + picosecondsToDiffTime 999999000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 11
, utctDayTime = secondsToDiffTime 36000 + picosecondsToDiffTime 100000000000
}
, UTCTime
{ utctDay = fromGregorian 2024 1 12
, utctDayTime = secondsToDiffTime 56028 + picosecondsToDiffTime 667891000000
}
]
)
,
( "user_email"
, D.fromList
[ "alice@example.com" :: Text
, "bob@example.com"
, "carol@example.com"
, "alice@example.com"
, "dave@example.com"
, "dave@example.com"
, "eve@example.com"
, "frank@example.com"
, "grace@example.com"
, "dave@example.com"
, "alice@example.com"
, "heidi@example.com"
]
)
,
( "transaction_type"
, D.fromList
[ "purchase" :: Text
, "purchase"
, "refund"
, "purchase"
, "purchase"
, "purchase"
, "purchase"
, "withdrawal"
, "purchase"
, "purchase"
, "purchase"
, "refund"
]
)
,
( "amount"
, D.fromList
[ 142.50 :: Double
, 29.99
, 89.00
, 2399.00
, 15.00
, 15.00
, 450.75
, 200.00
, 55.20
, 3200.00
, 74.99
, 120.00
]
)
,
( "currency"
, D.fromList
[ "USD" :: Text
, "USD"
, "EUR"
, "USD"
, "GBP"
, "GBP"
, "USD"
, "EUR"
, "CAD"
, "USD"
, "USD"
, "GBP"
]
)
,
( "status"
, D.fromList
[ "approved" :: Text
, "approved"
, "approved"
, "declined"
, "approved"
, "declined"
, "approved"
, "approved"
, "approved"
, "flagged"
, "approved"
, "approved"
]
)
,
( "location"
, D.fromList
[ "New York, US" :: Text
, "London, GB"
, "Berlin, DE"
, "New York, US"
, "Manchester, GB"
, "Lagos, NG"
, "San Francisco, US"
, "Paris, FR"
, "Toronto, CA"
, "New York, US"
, "New York, US"
, "Edinburgh, GB"
]
)
]
transactionsTest :: Test
transactionsTest = testBothReadParquetPaths $ \readParquet ->
TestCase
( assertEqual
"transactions"
transactions
(unsafePerformIO (readParquet "./tests/data/transactions.parquet"))
)
mtCarsDataset :: D.DataFrame
mtCarsDataset =
D.fromNamedColumns
[
( "model"
, D.fromList
[ "Mazda RX4" :: Text
, "Mazda RX4 Wag"
, "Datsun 710"
, "Hornet 4 Drive"
, "Hornet Sportabout"
, "Valiant"
, "Duster 360"
, "Merc 240D"
, "Merc 230"
, "Merc 280"
, "Merc 280C"
, "Merc 450SE"
, "Merc 450SL"
, "Merc 450SLC"
, "Cadillac Fleetwood"
, "Lincoln Continental"
, "Chrysler Imperial"
, "Fiat 128"
, "Honda Civic"
, "Toyota Corolla"
, "Toyota Corona"
, "Dodge Challenger"
, "AMC Javelin"
, "Camaro Z28"
, "Pontiac Firebird"
, "Fiat X1-9"
, "Porsche 914-2"
, "Lotus Europa"
, "Ford Pantera L"
, "Ferrari Dino"
, "Maserati Bora"
, "Volvo 142E"
]
)
,
( "mpg"
, D.fromList
[ 21.0 :: Double
, 21.0
, 22.8
, 21.4
, 18.7
, 18.1
, 14.3
, 24.4
, 22.8
, 19.2
, 17.8
, 16.4
, 17.3
, 15.2
, 10.4
, 10.4
, 14.7
, 32.4
, 30.4
, 33.9
, 21.5
, 15.5
, 15.2
, 13.3
, 19.2
, 27.3
, 26.0
, 30.4
, 15.8
, 19.7
, 15.0
, 21.4
]
)
,
( "cyl"
, D.fromList
[ 6 :: Int32
, 6
, 4
, 6
, 8
, 6
, 8
, 4
, 4
, 6
, 6
, 8
, 8
, 8
, 8
, 8
, 8
, 4
, 4
, 4
, 4
, 8
, 8
, 8
, 8
, 4
, 4
, 4
, 8
, 6
, 8
, 4
]
)
,
( "disp"
, D.fromList
[ 160.0 :: Double
, 160.0
, 108.0
, 258.0
, 360.0
, 225.0
, 360.0
, 146.7
, 140.8
, 167.6
, 167.6
, 275.8
, 275.8
, 275.8
, 472.0
, 460.0
, 440.0
, 78.7
, 75.7
, 71.1
, 120.1
, 318.0
, 304.0
, 350.0
, 400.0
, 79.0
, 120.3
, 95.1
, 351.0
, 145.0
, 301.0
, 121.0
]
)
,
( "hp"
, D.fromList
[ 110 :: Int32
, 110
, 93
, 110
, 175
, 105
, 245
, 62
, 95
, 123
, 123
, 180
, 180
, 180
, 205
, 215
, 230
, 66
, 52
, 65
, 97
, 150
, 150
, 245
, 175
, 66
, 91
, 113
, 264
, 175
, 335
, 109
]
)
,
( "drat"
, D.fromList
[ 3.9 :: Double
, 3.9
, 3.85
, 3.08
, 3.15
, 2.76
, 3.21
, 3.69
, 3.92
, 3.92
, 3.92
, 3.07
, 3.07
, 3.07
, 2.93
, 3.0
, 3.23
, 4.08
, 4.93
, 4.22
, 3.7
, 2.76
, 3.15
, 3.73
, 3.08
, 4.08
, 4.43
, 3.77
, 4.22
, 3.62
, 3.54
, 4.11
]
)
,
( "wt"
, D.fromList
[ 2.62 :: Double
, 2.875
, 2.32
, 3.215
, 3.44
, 3.46
, 3.57
, 3.19
, 3.15
, 3.44
, 3.44
, 4.07
, 3.73
, 3.78
, 5.25
, 5.424
, 5.345
, 2.2
, 1.615
, 1.835
, 2.465
, 3.52
, 3.435
, 3.84
, 3.845
, 1.935
, 2.14
, 1.513
, 3.17
, 2.77
, 3.57
, 2.78
]
)
,
( "qsec"
, D.fromList
[ 16.46 :: Double
, 17.02
, 18.61
, 19.44
, 17.02
, 20.22
, 15.84
, 20.0
, 22.9
, 18.3
, 18.9
, 17.4
, 17.6
, 18.0
, 17.98
, 17.82
, 17.42
, 19.47
, 18.52
, 19.9
, 20.01
, 16.87
, 17.3
, 15.41
, 17.05
, 18.9
, 16.7
, 16.9
, 14.5
, 15.5
, 14.6
, 18.6
]
)
,
( "vs"
, D.fromList
[ 0 :: Int32
, 0
, 1
, 1
, 0
, 1
, 0
, 1
, 1
, 1
, 1
, 0
, 0
, 0
, 0
, 0
, 0
, 1
, 1
, 1
, 1
, 0
, 0
, 0
, 0
, 1
, 0
, 1
, 0
, 0
, 0
, 1
]
)
,
( "am"
, D.fromList
[ 1 :: Int32
, 1
, 1
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 0
, 1
, 1
, 1
, 0
, 0
, 0
, 0
, 0
, 1
, 1
, 1
, 1
, 1
, 1
, 1
]
)
,
( "gear"
, D.fromList
[ 4 :: Int32
, 4
, 4
, 3
, 3
, 3
, 3
, 4
, 4
, 4
, 4
, 3
, 3
, 3
, 3
, 3
, 3
, 4
, 4
, 4
, 3
, 3
, 3
, 3
, 3
, 4
, 5
, 5
, 5
, 5
, 5
, 4
]
)
,
( "carb"
, D.fromList
[ 4 :: Int32
, 4
, 1
, 1
, 2
, 1
, 4
, 2
, 2
, 4
, 4
, 3
, 3
, 3
, 4
, 4
, 4
, 1
, 2
, 1
, 1
, 2
, 2
, 4
, 2
, 1
, 2
, 2
, 4
, 6
, 8
, 2
]
)
]
mtCars :: Test
mtCars =
TestCase
( assertEqual
"mt_cars"
mtCarsDataset
(unsafePerformIO (D.readParquet "./tests/data/mtcars.parquet"))
)
littleEndianWord64KnownPattern :: Test
littleEndianWord64KnownPattern =
TestCase
( assertEqual
"littleEndianWord64KnownPattern"
(0x1122334455667788 :: Word64)
( littleEndianWord64
(BS.pack [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11])
)
)
littleEndianWord32KnownPattern :: Test
littleEndianWord32KnownPattern =
TestCase
( assertEqual
"littleEndianWord32KnownPattern"
(0x11223344 :: Word32)
(littleEndianWord32 (BS.pack [0x44, 0x33, 0x22, 0x11]))
)
littleEndianWord64ShortInputPadsZeroes :: Test
littleEndianWord64ShortInputPadsZeroes =
TestCase
( assertEqual
"littleEndianWord64ShortInputPadsZeroes"
(0x00CCBBAA :: Word64)
(littleEndianWord64 (BS.pack [0xAA, 0xBB, 0xCC]))
)
littleEndianWord32ShortInputPadsZeroes :: Test
littleEndianWord32ShortInputPadsZeroes =
TestCase
( assertEqual
"littleEndianWord32ShortInputPadsZeroes"
(0x0000BEEF :: Word32)
(littleEndianWord32 (BS.pack [0xEF, 0xBE]))
)
littleEndianWord64RoundTrip :: Test
littleEndianWord64RoundTrip =
TestCase
( assertEqual
"littleEndianWord64RoundTrip"
value
(littleEndianWord64 (word64ToLittleEndian value))
)
where
value = 0x1122334455667788 :: Word64
littleEndianWord32RoundTrip :: Test
littleEndianWord32RoundTrip =
TestCase
( assertEqual
"littleEndianWord32RoundTrip"
value
(littleEndianWord32 (word32ToLittleEndian value))
)
where
value = 0xA1B2C3D4 :: Word32
-- ---------------------------------------------------------------------------
-- Group 1: Plain variant
-- ---------------------------------------------------------------------------
allTypesTinyPagesPlain :: Test
allTypesTinyPagesPlain =
TestCase
( assertEqual
"alltypes_tiny_pages_plain dimensions"
(7300, 13)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/alltypes_tiny_pages_plain.parquet")
)
)
)
-- ---------------------------------------------------------------------------
-- Group 2: Compression codecs (unsupported → error tests)
-- ---------------------------------------------------------------------------
hadoopLz4Compressed :: Test
hadoopLz4Compressed =
TestCase
( assertExpectException
"hadoopLz4Compressed"
"LZ4"
(D.readParquet "./tests/data/hadoop_lz4_compressed.parquet")
)
hadoopLz4CompressedLarger :: Test
hadoopLz4CompressedLarger =
TestCase
( assertExpectException
"hadoopLz4CompressedLarger"
"LZ4"
(D.readParquet "./tests/data/hadoop_lz4_compressed_larger.parquet")
)
nonHadoopLz4Compressed :: Test
nonHadoopLz4Compressed =
TestCase
( assertExpectException
"nonHadoopLz4Compressed"
"LZ4"
(D.readParquet "./tests/data/non_hadoop_lz4_compressed.parquet")
)
lz4RawCompressed :: Test
lz4RawCompressed =
TestCase
( assertExpectException
"lz4RawCompressed"
"LZ4_RAW"
(D.readParquet "./tests/data/lz4_raw_compressed.parquet")
)
lz4RawCompressedLarger :: Test
lz4RawCompressedLarger =
TestCase
( assertExpectException
"lz4RawCompressedLarger"
"LZ4_RAW"
(D.readParquet "./tests/data/lz4_raw_compressed_larger.parquet")
)
concatenatedGzipMembers :: Test
concatenatedGzipMembers =
TestCase
( assertExpectException
"concatenatedGzipMembers"
"12"
(D.readParquet "./tests/data/concatenated_gzip_members.parquet")
)
largeBrotliMap :: Test
largeBrotliMap =
TestCase
( assertExpectException
"largeBrotliMap"
"BROTLI"
(D.readParquet "./tests/data/large_string_map.brotli.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 3: Delta / RLE encodings (unsupported → error tests)
-- ---------------------------------------------------------------------------
deltaBinaryPacked :: Test
deltaBinaryPacked =
TestCase
( assertExpectException
"deltaBinaryPacked"
"EDELTA_BINARY_PACKED"
(D.readParquet "./tests/data/delta_binary_packed.parquet")
)
deltaByteArray :: Test
deltaByteArray =
TestCase
( assertExpectException
"deltaByteArray"
"EDELTA_BYTE_ARRAY"
(D.readParquet "./tests/data/delta_byte_array.parquet")
)
deltaEncodingOptionalColumn :: Test
deltaEncodingOptionalColumn =
TestCase
( assertExpectException
"deltaEncodingOptionalColumn"
"EDELTA_BINARY_PACKED"
(D.readParquet "./tests/data/delta_encoding_optional_column.parquet")
)
deltaEncodingRequiredColumn :: Test
deltaEncodingRequiredColumn =
TestCase
( assertExpectException
"deltaEncodingRequiredColumn"
"EDELTA_BINARY_PACKED"
(D.readParquet "./tests/data/delta_encoding_required_column.parquet")
)
deltaLengthByteArray :: Test
deltaLengthByteArray =
TestCase
( assertExpectException
"deltaLengthByteArray"
"ZSTD"
(D.readParquet "./tests/data/delta_length_byte_array.parquet")
)
rleBooleanEncoding :: Test
rleBooleanEncoding =
TestCase
( assertExpectException
"rleBooleanEncoding"
"Zlib"
(D.readParquet "./tests/data/rle_boolean_encoding.parquet")
)
dictPageOffsetZero :: Test
dictPageOffsetZero =
TestCase
( assertExpectException
"dictPageOffsetZero"
"Unknown kv"
(D.readParquet "./tests/data/dict-page-offset-zero.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 4: Data Page V2 (unsupported → error tests)
-- ---------------------------------------------------------------------------
datapageV2Snappy :: Test
datapageV2Snappy =
TestCase
( assertExpectException
"datapageV2Snappy"
"InvalidOffset"
(D.readParquet "./tests/data/datapage_v2.snappy.parquet")
)
datapageV2EmptyDatapage :: Test
datapageV2EmptyDatapage =
TestCase
( assertExpectException
"datapageV2EmptyDatapage"
"UnexpectedEOF"
(D.readParquet "./tests/data/datapage_v2_empty_datapage.snappy.parquet")
)
pageV2EmptyCompressed :: Test
pageV2EmptyCompressed =
TestCase
( assertExpectException
"pageV2EmptyCompressed"
"10"
(D.readParquet "./tests/data/page_v2_empty_compressed.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 5: Checksum files (all read successfully)
-- ---------------------------------------------------------------------------
datapageV1UncompressedChecksum :: Test
datapageV1UncompressedChecksum =
TestCase
( assertEqual
"datapageV1UncompressedChecksum"
(5120, 2)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/datapage_v1-uncompressed-checksum.parquet")
)
)
)
datapageV1SnappyChecksum :: Test
datapageV1SnappyChecksum =
TestCase
( assertEqual
"datapageV1SnappyChecksum"
(5120, 2)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/datapage_v1-snappy-compressed-checksum.parquet")
)
)
)
plainDictUncompressedChecksum :: Test
plainDictUncompressedChecksum =
TestCase
( assertEqual
"plainDictUncompressedChecksum"
(1000, 2)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/plain-dict-uncompressed-checksum.parquet")
)
)
)
rleDictSnappyChecksum :: Test
rleDictSnappyChecksum =
TestCase
( assertEqual
"rleDictSnappyChecksum"
(1000, 2)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/rle-dict-snappy-checksum.parquet")
)
)
)
datapageV1CorruptChecksum :: Test
datapageV1CorruptChecksum =
TestCase
( assertEqual
"datapageV1CorruptChecksum"
(5120, 2)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/datapage_v1-corrupt-checksum.parquet")
)
)
)
rleDictUncompressedCorruptChecksum :: Test
rleDictUncompressedCorruptChecksum =
TestCase
( assertEqual
"rleDictUncompressedCorruptChecksum"
(1000, 2)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/rle-dict-uncompressed-corrupt-checksum.parquet")
)
)
)
-- ---------------------------------------------------------------------------
-- Group 6: NULL handling
-- ---------------------------------------------------------------------------
nullsSnappy :: Test
nullsSnappy =
TestCase
( assertEqual
"nullsSnappy"
(8, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nulls.snappy.parquet"))
)
)
int32WithNullPages :: Test
int32WithNullPages =
TestCase
( assertEqual
"int32WithNullPages"
(1000, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/int32_with_null_pages.parquet"))
)
)
nullableImpala :: Test
nullableImpala =
TestCase
( assertEqual
"nullableImpala"
(7, 13)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nullable.impala.parquet"))
)
)
nonnullableImpala :: Test
nonnullableImpala =
TestCase
( assertEqual
"nonnullableImpala"
(1, 13)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nonnullable.impala.parquet"))
)
)
singleNan :: Test
singleNan =
TestCase
( assertEqual
"singleNan"
(1, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/single_nan.parquet"))
)
)
nanInStats :: Test
nanInStats =
TestCase
( assertEqual
"nanInStats"
(2, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nan_in_stats.parquet"))
)
)
-- ---------------------------------------------------------------------------
-- Group 7: Decimal types
-- ---------------------------------------------------------------------------
int32Decimal :: Test
int32Decimal =
TestCase
( assertEqual
"int32Decimal"
(24, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/int32_decimal.parquet"))
)
)
int64Decimal :: Test
int64Decimal =
TestCase
( assertEqual
"int64Decimal"
(24, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/int64_decimal.parquet"))
)
)
byteArrayDecimal :: Test
byteArrayDecimal =
TestCase
( assertEqual
"byteArrayDecimal"
(24, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/byte_array_decimal.parquet"))
)
)
fixedLengthDecimal :: Test
fixedLengthDecimal =
TestCase
( assertExpectException
"fixedLengthDecimal"
"FIXED_LEN_BYTE_ARRAY"
(D.readParquet "./tests/data/fixed_length_decimal.parquet")
)
fixedLengthDecimalLegacy :: Test
fixedLengthDecimalLegacy =
TestCase
( assertExpectException
"fixedLengthDecimalLegacy"
"FIXED_LEN_BYTE_ARRAY"
(D.readParquet "./tests/data/fixed_length_decimal_legacy.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 8: Binary / fixed-length bytes
-- ---------------------------------------------------------------------------
binaryFile :: Test
binaryFile =
TestCase
( assertEqual
"binaryFile"
(12, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/binary.parquet"))
)
)
binaryTruncatedMinMax :: Test
binaryTruncatedMinMax =
TestCase
( assertEqual
"binaryTruncatedMinMax"
(12, 6)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/binary_truncated_min_max.parquet")
)
)
)
fixedLengthByteArray :: Test
fixedLengthByteArray =
TestCase
( assertExpectException
"fixedLengthByteArray"
"FIXED_LEN_BYTE_ARRAY"
(D.readParquet "./tests/data/fixed_length_byte_array.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 9: INT96 timestamps
-- ---------------------------------------------------------------------------
int96FromSpark :: Test
int96FromSpark =
TestCase
( assertEqual
"int96FromSpark"
(6, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/int96_from_spark.parquet"))
)
)
-- ---------------------------------------------------------------------------
-- Group 10: Metadata / index / bloom filters
-- ---------------------------------------------------------------------------
columnChunkKeyValueMetadata :: Test
columnChunkKeyValueMetadata =
TestCase
( assertExpectException
"columnChunkKeyValueMetadata"
"Unknown page header field"
(D.readParquet "./tests/data/column_chunk_key_value_metadata.parquet")
)
dataIndexBloomEncodingStats :: Test
dataIndexBloomEncodingStats =
TestCase
( assertEqual
"dataIndexBloomEncodingStats"
(14, 1)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/data_index_bloom_encoding_stats.parquet")
)
)
)
dataIndexBloomEncodingWithLength :: Test
dataIndexBloomEncodingWithLength =
TestCase
( assertEqual
"dataIndexBloomEncodingWithLength"
(14, 1)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/data_index_bloom_encoding_with_length.parquet")
)
)
)
sortColumns :: Test
sortColumns =
TestCase
( assertEqual
"sortColumns"
(3, 2)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/sort_columns.parquet"))
)
)
overflowI16PageCnt :: Test
overflowI16PageCnt =
TestCase
( assertExpectException
"overflowI16PageCnt"
"UNIMPLEMENTED"
(D.readParquet "./tests/data/overflow_i16_page_cnt.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 11: Nested / complex types and byte-stream-split
-- ---------------------------------------------------------------------------
byteStreamSplitZstd :: Test
byteStreamSplitZstd =
TestCase
( assertExpectException
"byteStreamSplitZstd"
"EBYTE_STREAM_SPLIT"
(D.readParquet "./tests/data/byte_stream_split.zstd.parquet")
)
byteStreamSplitExtendedGzip :: Test
byteStreamSplitExtendedGzip =
TestCase
( assertExpectException
"byteStreamSplitExtendedGzip"
"FIXED_LEN_BYTE_ARRAY"
(D.readParquet "./tests/data/byte_stream_split_extended.gzip.parquet")
)
float16NonzerosAndNans :: Test
float16NonzerosAndNans =
TestCase
( assertExpectException
"float16NonzerosAndNans"
"PFIXED_LEN_BYTE_ARRAY"
(D.readParquet "./tests/data/float16_nonzeros_and_nans.parquet")
)
float16ZerosAndNans :: Test
float16ZerosAndNans =
TestCase
( assertExpectException
"float16ZerosAndNans"
"PFIXED_LEN_BYTE_ARRAY"
(D.readParquet "./tests/data/float16_zeros_and_nans.parquet")
)
nestedListsSnappy :: Test
nestedListsSnappy =
TestCase
( assertEqual
"nestedListsSnappy"
(3, 2)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nested_lists.snappy.parquet"))
)
)
nestedMapsSnappy :: Test
nestedMapsSnappy =
TestCase
( assertEqual
"nestedMapsSnappy"
(6, 5)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nested_maps.snappy.parquet"))
)
)
nestedStructsRust :: Test
nestedStructsRust =
TestCase
( assertEqual
"nestedStructsRust"
(1, 216)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/nested_structs.rust.parquet"))
)
)
listColumns :: Test
listColumns =
TestCase
( assertEqual
"listColumns"
(3, 2)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/list_columns.parquet"))
)
)
oldListStructure :: Test
oldListStructure =
TestCase
( assertEqual
"oldListStructure"
(1, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/old_list_structure.parquet"))
)
)
nullList :: Test
nullList =
TestCase
( assertEqual
"nullList"
(1, 1)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/null_list.parquet"))
)
)
mapNoValue :: Test
mapNoValue =
TestCase
( assertEqual
"mapNoValue"
(3, 4)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/map_no_value.parquet"))
)
)
incorrectMapSchema :: Test
incorrectMapSchema =
TestCase
( assertEqual
"incorrectMapSchema"
(1, 2)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/incorrect_map_schema.parquet"))
)
)
repeatedNoAnnotation :: Test
repeatedNoAnnotation =
TestCase
( assertEqual
"repeatedNoAnnotation"
(6, 3)
( unsafePerformIO
(fmap D.dimensions (D.readParquet "./tests/data/repeated_no_annotation.parquet"))
)
)
repeatedPrimitiveNoList :: Test
repeatedPrimitiveNoList =
TestCase
( assertEqual
"repeatedPrimitiveNoList"
(4, 4)
( unsafePerformIO
( fmap
D.dimensions
(D.readParquet "./tests/data/repeated_primitive_no_list.parquet")
)
)
)
unknownLogicalType :: Test
unknownLogicalType =
TestCase
( assertExpectException
"unknownLogicalType"
"Unknown logical type"
(D.readParquet "./tests/data/unknown-logical-type.parquet")
)
-- ---------------------------------------------------------------------------
-- Group 12: Malformed files
-- ---------------------------------------------------------------------------
nationDictMalformed :: Test
nationDictMalformed =
TestCase
( assertExpectException
"nationDictMalformed"
"dict index count mismatch"
(D.readParquet "./tests/data/nation.dict-malformed.parquet")
)
shardedNullableSchema :: Test
shardedNullableSchema =
TestCase $ do
metas <-
mapM
(fmap fst . DP.readMetadataFromPath)
["data/sharded/part-0.parquet", "data/sharded/part-1.parquet"]
let nullableCols =
S.fromList
[ last (map T.pack colPath)
| meta <- metas
, rg <- rowGroups meta
, cc <- rowGroupColumns rg
, let cm = columnMetaData cc
colPath = columnPathInSchema cm
, not (null colPath)
, columnNullCount (columnStatistics cm) > 0
]
df =
foldl
(\acc meta -> acc <> F.schemaToEmptyDataFrame nullableCols (schema meta))
D.empty
metas
assertBool "id should be nullable" (hasMissing (unsafeGetColumn "id" df))
assertBool "name should be nullable" (hasMissing (unsafeGetColumn "name" df))
assertBool "score should be nullable" (hasMissing (unsafeGetColumn "score" df))
singleShardNoNulls :: Test
singleShardNoNulls =
TestCase $ do
(meta, _) <- DP.readMetadataFromPath "data/sharded/part-0.parquet"
let nullableCols =
S.fromList
[ last (map T.pack colPath)
| rg <- rowGroups meta
, cc <- rowGroupColumns rg
, let cm = columnMetaData cc
colPath = columnPathInSchema cm
, not (null colPath)
, columnNullCount (columnStatistics cm) > 0
]
df = F.schemaToEmptyDataFrame nullableCols (schema meta)
assertBool
"id should NOT be nullable"
(not (hasMissing (unsafeGetColumn "id" df)))
assertBool
"name should NOT be nullable"
(not (hasMissing (unsafeGetColumn "name" df)))
assertBool
"score should NOT be nullable"
(not (hasMissing (unsafeGetColumn "score" df)))
tests :: [Test]
tests =
[ allTypesPlain
, allTypesPlainSnappy
, allTypesDictionary
, selectedColumnsWithOpts
, rowRangeWithOpts
, predicateWithOpts
, predicateUsesNonSelectedColumnWithOpts
, predicateWithOptsAcrossFiles
, missingSelectedColumnWithOpts
, mtCars
, allTypesTinyPagesLastFew
, allTypesTinyPagesDimensions
, transactionsTest
, littleEndianWord64KnownPattern
, littleEndianWord32KnownPattern
, littleEndianWord64ShortInputPadsZeroes
, littleEndianWord32ShortInputPadsZeroes
, littleEndianWord64RoundTrip
, littleEndianWord32RoundTrip
, -- Group 1
allTypesTinyPagesPlain
, -- Group 2: compression codecs
hadoopLz4Compressed
, hadoopLz4CompressedLarger
, nonHadoopLz4Compressed
, lz4RawCompressed
, lz4RawCompressedLarger
, concatenatedGzipMembers
, largeBrotliMap
, -- Group 3: delta / rle encodings
deltaBinaryPacked
, deltaByteArray
, deltaEncodingOptionalColumn
, deltaEncodingRequiredColumn
, deltaLengthByteArray
, rleBooleanEncoding
, dictPageOffsetZero
, -- Group 4: Data Page V2
datapageV2Snappy
, datapageV2EmptyDatapage
, pageV2EmptyCompressed
, -- Group 5: checksum files
datapageV1UncompressedChecksum
, datapageV1SnappyChecksum
, plainDictUncompressedChecksum
, rleDictSnappyChecksum
, datapageV1CorruptChecksum
, rleDictUncompressedCorruptChecksum
, -- Group 6: NULL handling
nullsSnappy
, int32WithNullPages
, nullableImpala
, nonnullableImpala
, singleNan
, nanInStats
, -- Group 7: decimal types
int32Decimal
, int64Decimal
, byteArrayDecimal
, fixedLengthDecimal
, fixedLengthDecimalLegacy
, -- Group 8: binary / fixed-length bytes
binaryFile
, binaryTruncatedMinMax
, fixedLengthByteArray
, -- Group 9: INT96 timestamps
int96FromSpark
, -- Group 10: metadata / bloom filters
columnChunkKeyValueMetadata
, dataIndexBloomEncodingStats
, dataIndexBloomEncodingWithLength
, sortColumns
, overflowI16PageCnt
, -- Group 11: nested / complex types
byteStreamSplitZstd
, byteStreamSplitExtendedGzip
, float16NonzerosAndNans
, float16ZerosAndNans
, nestedListsSnappy
, nestedMapsSnappy
, nestedStructsRust
, listColumns
, oldListStructure
, nullList
, mapNoValue
, incorrectMapSchema
, repeatedNoAnnotation
, repeatedPrimitiveNoList
, unknownLogicalType
, -- Group 12: malformed files
nationDictMalformed
, -- Group 13: metadata-based null detection
shardedNullableSchema
, singleShardNoNulls
]