dataframe 0.3.0.3 → 0.3.0.4
raw patch · 9 files changed
+160/−69 lines, 9 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- CHANGELOG.md +18/−1
- README.md +0/−16
- dataframe.cabal +1/−1
- src/DataFrame/IO/Parquet/Binary.hs +16/−14
- src/DataFrame/IO/Parquet/Encoding.hs +11/−14
- src/DataFrame/Operations/Statistics.hs +29/−5
- tests/Operations/Filter.hs +8/−6
- tests/Operations/Statistics.hs +39/−0
- tests/Parquet.hs +38/−12
CHANGELOG.md view
@@ -1,7 +1,24 @@ # Revision history for dataframe +## 0.3.0.4+* Fix bug with parquet reader.+ ## 0.3.0.3-* Improved parquet reader. The reader now supports most parquet files found in the wild.+* Improved parquet reader. The reader now supports most parquet files downloaded from internet sources+ * Supports all primitive parquet types plain and uncompressed.+ * Can decode both v1 and v2 data pages.+ * Supports Snappy and ZSTD compression.+ * Supports RLE/bitpacking encoding for primitive types+ * Backward compatible with INT96 type.+ * From the parquet-testing repo we can successfully read the following:+ * alltypes_dictionary.parquet+ * alltypes_plain.parquet+ * alltypes_plain.snappy.parquet+ * alltypes_tiny_pages_plain.parquet+ * binary_truncated_min_max.parquet+ * datapage_v1-corrupt-checksum.parquet+ * datapage_v1-snappy-compressed-checksum.parquet+ * datapage_v1-uncompressed-checksum.parquet * Improve CSV parsing: Parse bytestring and convert to text only at the end. Remove some redundancies in parsing with suggestions from @Jhingon. * Faster correlation computation. * Update version of granite that ships with dataframe and add new scatterBy plot.
README.md view
@@ -118,22 +118,6 @@ ## Supported input formats * CSV * Apache Parquet- * Supports all primitive parquet types plain and uncompressed.- * Can decode both v1 and v2 data pages.- * Supports Snappy and ZSTD compression.- * Supports RLE/bitpacking encoding for primitive types- * Backward compatible with INT96 type.- * From the parquet-testing repo we can successfully read the following:- * alltypes_dictionary.parquet- * alltypes_plain.parquet- * alltypes_plain.snappy.parquet- * alltypes_tiny_pages_plain.parquet- * binary_truncated_min_max.parquet- * datapage_v1-corrupt-checksum.parquet- * datapage_v1-snappy-compressed-checksum.parquet- * datapage_v1-uncompressed-checksum.parquet- * We will continue adding functionality/coverage as needed.- ## Supported output formats * CSV
dataframe.cabal view
@@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe-version: 0.3.0.3+version: 0.3.0.4 synopsis: A fast, safe, and intuitive DataFrame library.
src/DataFrame/IO/Parquet/Binary.hs view
@@ -2,13 +2,14 @@ module DataFrame.IO.Parquet.Binary where -import Control.Monad-import Data.Bits-import Data.Char-import Data.IORef-import Data.Word-import Foreign-import System.IO+import Control.Monad+import Data.Bits+import qualified Data.ByteString as BS+import Data.Char+import Data.IORef+import Data.Word+import Foreign+import System.IO littleEndianWord32 :: [Word8] -> Word32 littleEndianWord32 bytes@@ -93,13 +94,14 @@ map (chr . fromIntegral) <$> replicateM nameSize (readAndAdvance pos buf) readBytes :: Handle -> Int64 -> Int64 -> IO [Word8]-readBytes handle colStart colLen = do- buf <- mallocBytes (fromIntegral colLen) :: IO (Ptr Word8)- hSeek handle AbsoluteSeek (fromIntegral colStart)- _ <- hGetBuf handle buf (fromIntegral colLen)- columnBytes <- mapM (peekByteOff buf) [0 .. fromIntegral colLen - 1]- free buf- pure columnBytes+readBytes h colStart colLen = do+ hSeek h AbsoluteSeek (fromIntegral colStart)+ bs <- BS.hGet h (fromIntegral colLen)+ if BS.length bs /= (fromIntegral colLen)+ then ioError (userError ("short read: wanted "+ ++ show colLen ++ " bytes, got "+ ++ show (BS.length bs)))+ else pure (BS.unpack bs) numBytesInFile :: Handle -> IO Integer numBytesInFile handle = do
src/DataFrame/IO/Parquet/Encoding.hs view
@@ -26,22 +26,19 @@ totalBytes = (totalBits + 7) `div` 8 chunk = take totalBytes bs rest = drop totalBytes bs- in if length chunk < totalBytes- then error $ "Not enough bytes: need " ++ show totalBytes ++ ", got " ++ show (length chunk)- else- let bits = concatMap (\b -> map (\i -> (b `shiftR` i) .&. 1) [0 .. 7]) chunk- toN xs = foldl' (\a (i, b) -> a .|. (b `shiftL` i)) 0 (zip [0 ..] xs)+ bits = concatMap (\b -> map (\i -> (b `shiftR` i) .&. 1) [0 .. 7]) chunk+ toN xs = foldl' (\a (i, b) -> a .|. (b `shiftL` i)) 0 (zip [0 ..] xs) - extractValues _ [] = []- extractValues n bitsLeft- | n <= 0 = []- | length bitsLeft < bw = []- | otherwise =- let (this, bitsLeft') = splitAt bw bitsLeft- in toN this : extractValues (n - 1) bitsLeft'+ extractValues _ [] = []+ extractValues n bitsLeft+ | n <= 0 = []+ | length bitsLeft < bw = []+ | otherwise =+ let (this, bitsLeft') = splitAt bw bitsLeft+ in toN this : extractValues (n - 1) bitsLeft' - vals = extractValues count bits- in (map fromIntegral vals, rest)+ vals = extractValues count bits+ in (map fromIntegral vals, rest) decodeRLEBitPackedHybrid :: Int -> Int -> [Word8] -> ([Word32], [Word8]) decodeRLEBitPackedHybrid bw need bs
src/DataFrame/Operations/Statistics.hs view
@@ -88,7 +88,7 @@ -- | Calculates the skewness of a given column as a standalone value. skewness :: T.Text -> DataFrame -> Maybe Double-skewness = applyStatistic SS.skewness+skewness = applyStatistic skewness' -- | Calculates the variance of a given column as a standalone value. variance :: T.Text -> DataFrame -> Maybe Double@@ -212,14 +212,14 @@ -- accumulator: count, mean, m2 data VarAcc = VarAcc !Int !Double !Double deriving (Show) -step :: VarAcc -> Double -> VarAcc-step (VarAcc !n !mean !m2) !x =+varianceStep :: VarAcc -> Double -> VarAcc+varianceStep (VarAcc !n !mean !m2) !x = let !n' = n + 1 !delta = x - mean !mean' = mean + delta / fromIntegral n' !m2' = m2 + delta * (x - mean') in VarAcc n' mean' m2'-{-# INLINE step #-}+{-# INLINE varianceStep #-} computeVariance :: VarAcc -> Double computeVariance (VarAcc !n _ !m2)@@ -228,8 +228,32 @@ {-# INLINE computeVariance #-} variance' :: VU.Vector Double -> Double-variance' = computeVariance . VU.foldl' step (VarAcc 0 0 0)+variance' = computeVariance . VU.foldl' varianceStep (VarAcc 0 0 0) {-# INLINE variance' #-}++-- accumulator: count, mean, m2, m3+data SkewAcc = SkewAcc !Int !Double !Double !Double deriving (Show)++skewnessStep :: SkewAcc -> Double -> SkewAcc+skewnessStep (SkewAcc !n !mean !m2 !m3) !x =+ let !n' = n + 1+ !k = fromIntegral n'+ !delta = x - mean+ !mean' = mean + delta / k+ !m2' = m2 + (delta ^ 2 * (k - 1)) / k+ !m3' = m3 + (delta ^ 3 * (k - 1) * (k - 2)) / k ^ 2 - (3 * delta * m2) / k+ in SkewAcc n' mean' m2' m3'+{-# INLINE skewnessStep #-}++computeSkewness :: SkewAcc -> Double+computeSkewness (SkewAcc n _ m2 m3)+ | n < 3 = 0 -- or error "skewness of <3 samples"+ | otherwise = (sqrt (fromIntegral n - 1) * m3) / sqrt (m2 ^ 3)+{-# INLINE computeSkewness #-}++skewness' :: VU.Vector Double -> Double+skewness' = computeSkewness . VU.foldl' skewnessStep (SkewAcc 0 0 0 0)+{-# INLINE skewness' #-} correlation' :: VU.Vector Double -> VU.Vector Double -> Maybe Double correlation' xs ys
tests/Operations/Filter.hs view
@@ -7,8 +7,10 @@ import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D-import qualified DataFrame as DE-import qualified DataFrame as DI+import qualified DataFrame.Internal.Column as D+import qualified DataFrame.Internal.DataFrame as D+import qualified DataFrame.Internal.Column as DI+import qualified DataFrame.Internal.DataFrame as DI import Assertions import Test.HUnit@@ -34,7 +36,7 @@ TestCase ( assertExpectException "[Error Case]"- (DE.columnNotFound "test0" "filter" (D.columnNames testData))+ (D.columnNotFound "test0" "filter" (D.columnNames testData)) (print $ D.filter @Int "test0" even testData) ) @@ -43,7 +45,7 @@ TestCase ( assertExpectException "[Error Case]"- (DE.typeMismatchError (show $ typeRep @Integer) (show $ typeRep @Int))+ (D.typeMismatchError (show $ typeRep @Integer) (show $ typeRep @Int)) (print $ D.filter @Integer "test1" even testData) ) @@ -52,7 +54,7 @@ TestCase ( assertExpectException "[Error Case]"- (DE.columnNotFound "test0" "filter" (D.columnNames testData))+ (D.columnNotFound "test0" "filter" (D.columnNames testData)) (print $ D.filterBy @Int even "test0" testData) ) @@ -61,7 +63,7 @@ TestCase ( assertExpectException "[Error Case]"- (DE.typeMismatchError (show $ typeRep @Integer) (show $ typeRep @Int))+ (D.typeMismatchError (show $ typeRep @Integer) (show $ typeRep @Int)) (print $ D.filterBy @Integer even "test1" testData) )
tests/Operations/Statistics.hs view
@@ -1,5 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-}+{-# LANGUAGE NumericUnderscores #-} module Operations.Statistics where @@ -39,9 +40,47 @@ (print $ D.median' (VU.fromList [])) ) +skewnessOfDataSetWithSameElements :: Test+skewnessOfDataSetWithSameElements =+ TestCase+ ( assertBool+ "Skewness of a data set with the same elements"+ (isNaN (D.skewness' (VU.fromList $ replicate 10 42.0)))+ )++skewnessOfSymmetricDataSet :: Test+skewnessOfSymmetricDataSet =+ TestCase+ ( assertEqual+ "Skewness of a symmetric data set"+ (D.skewness' (VU.fromList [-3.0, -2.0, -1.5, 0, 1.5, 2.0, 3.0]))+ 0+ )++skewnessOfSimpleDataSet :: Test+skewnessOfSimpleDataSet =+ TestCase+ ( assertBool+ "Skewness of a simple data set"+ (abs (D.skewness' (VU.fromList [25, 28, 26, 30, 40, 50, 40]) - 0.566_731_633_676) < 1e-12)+ )++skewnessOfEmptyDataSet :: Test+skewnessOfEmptyDataSet =+ TestCase+ ( assertEqual+ "Skewness of an empty data set"+ (D.skewness' (VU.fromList []))+ 0+ )+ tests :: [Test] tests = [ TestLabel "medianOfOddLengthDataSet" medianOfOddLengthDataSet , TestLabel "medianOfEvenLengthDataSet" medianOfEvenLengthDataSet , TestLabel "medianOfEmptyDataSet" medianOfEmptyDataSet+ , TestLabel "skewnessOfDataSetWithSameElements" skewnessOfDataSetWithSameElements+ , TestLabel "skewnessOfSymmetricDataSet" skewnessOfSymmetricDataSet+ , TestLabel "skewnessOfSimpleDataSet" skewnessOfSimpleDataSet+ , TestLabel "skewnessOfEmptyDataSet" skewnessOfEmptyDataSet ]
tests/Parquet.hs view
@@ -15,16 +15,16 @@ allTypes :: D.DataFrame allTypes = D.fromNamedColumns- [ ("id", D.fromList [(4 :: Int32), 5, 6, 7, 2, 3, 0, 1])+ [ ("id", D.fromList [4 :: Int32, 5, 6, 7, 2, 3, 0, 1]) , ("bool_col", D.fromList [True, False, True, False, True, False, True, False])- , ("tinyint_col", D.fromList [(0 :: Int32), 1, 0, 1, 0, 1, 0, 1])- , ("smallint_col", D.fromList [(0 :: Int32), 1, 0, 1, 0, 1, 0, 1])- , ("int_col", D.fromList [(0 :: Int32), 1, 0, 1, 0, 1, 0, 1])- , ("bigint_col", D.fromList [(0 :: Int64), 10, 0, 10, 0, 10, 0, 10])- , ("float_col", D.fromList [(0 :: Float), 1.1, 0, 1.1, 0, 1.1, 0, 1.1])- , ("double_col", D.fromList [(0 :: Double), 10.1, 0, 10.1, 0, 10.1, 0, 10.1])- , ("date_string_col", D.fromList [("03/01/09" :: Text), "03/01/09", "04/01/09", "04/01/09", "02/01/09", "02/01/09", "01/01/09", "01/01/09"])- , ("string_col", D.fromList (take 8 (cycle [("0" :: Text), "1"])))+ , ("tinyint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1])+ , ("smallint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1])+ , ("int_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1])+ , ("bigint_col", D.fromList [0 :: Int64, 10, 0, 10, 0, 10, 0, 10])+ , ("float_col", D.fromList [0 :: Float, 1.1, 0, 1.1, 0, 1.1, 0, 1.1])+ , ("double_col", D.fromList [0 :: Double, 10.1, 0, 10.1, 0, 10.1, 0, 10.1])+ , ("date_string_col", D.fromList ["03/01/09" :: Text, "03/01/09", "04/01/09", "04/01/09", "02/01/09", "02/01/09", "01/01/09", "01/01/09"])+ , ("string_col", D.fromList (take 8 (cycle ["0" :: Text, "1"]))) , ( "timestamp_col" , D.fromList@@ -54,7 +54,7 @@ TestCase ( assertEqual "allTypesPlainSnappy"- (D.filter "id" (`elem` [(6 :: Int32), 7]) allTypes)+ (D.filter "id" (`elem` [6 :: Int32, 7]) allTypes) (unsafePerformIO (D.readParquet "./tests/data/alltypes_plain.snappy.parquet")) ) @@ -63,11 +63,37 @@ TestCase ( assertEqual "allTypesPlainSnappy"- (D.filter "id" (`elem` [(0 :: Int32), 1]) allTypes)+ (D.filter "id" (`elem` [0 :: Int32, 1]) allTypes) (unsafePerformIO (D.readParquet "./tests/data/alltypes_dictionary.parquet")) ) +mtCarsDataset :: D.DataFrame+mtCarsDataset =+ D.fromNamedColumns+ [ ("model", D.fromList ["Mazda RX4" :: Text, "Mazda RX4 Wag", "Datsun 710", "Hornet 4 Drive", "Hornet Sportabout", "Valiant", "Duster 360", "Merc 240D", "Merc 230", "Merc 280", "Merc 280C", "Merc 450SE", "Merc 450SL", "Merc 450SLC", "Cadillac Fleetwood", "Lincoln Continental", "Chrysler Imperial", "Fiat 128", "Honda Civic", "Toyota Corolla", "Toyota Corona", "Dodge Challenger", "AMC Javelin", "Camaro Z28", "Pontiac Firebird", "Fiat X1-9", "Porsche 914-2", "Lotus Europa", "Ford Pantera L", "Ferrari Dino", "Maserati Bora", "Volvo 142E"])+ , ("mpg", D.fromList [21.0 :: Double, 21.0, 22.8, 21.4, 18.7, 18.1, 14.3, 24.4, 22.8, 19.2, 17.8, 16.4, 17.3, 15.2, 10.4, 10.4, 14.7, 32.4, 30.4, 33.9, 21.5, 15.5, 15.2, 13.3, 19.2, 27.3, 26.0, 30.4, 15.8, 19.7, 15.0, 21.4])+ , ("cyl", D.fromList [6 :: Int32, 6, 4, 6, 8, 6, 8, 4, 4, 6, 6, 8, 8, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8, 4, 4, 4, 8, 6, 8, 4])+ , ("disp", D.fromList [160.0 :: Double, 160.0, 108.0, 258.0, 360.0, 225.0, 360.0, 146.7, 140.8, 167.6, 167.6, 275.8, 275.8, 275.8, 472.0, 460.0, 440.0, 78.7, 75.7, 71.1, 120.1, 318.0, 304.0, 350.0, 400.0, 79.0, 120.3, 95.1, 351.0, 145.0, 301.0, 121.0])+ , ("hp", D.fromList [110 :: Int32, 110, 93, 110, 175, 105, 245, 62, 95, 123, 123, 180, 180, 180, 205, 215, 230, 66, 52, 65, 97, 150, 150, 245, 175, 66, 91, 113, 264, 175, 335, 109])+ , ("drat", D.fromList [3.9 :: Double, 3.9, 3.85, 3.08, 3.15, 2.76, 3.21, 3.69, 3.92, 3.92, 3.92, 3.07, 3.07, 3.07, 2.93, 3.0, 3.23, 4.08, 4.93, 4.22, 3.7, 2.76, 3.15, 3.73, 3.08, 4.08, 4.43, 3.77, 4.22, 3.62, 3.54, 4.11])+ , ("wt", D.fromList [2.62 :: Double, 2.875, 2.32, 3.215, 3.44, 3.46, 3.57, 3.19, 3.15, 3.44, 3.44, 4.07, 3.73, 3.78, 5.25, 5.424, 5.345, 2.2, 1.615, 1.835, 2.465, 3.52, 3.435, 3.84, 3.845, 1.935, 2.14, 1.513, 3.17, 2.77, 3.57, 2.78])+ , ("qsec", D.fromList [16.46 :: Double, 17.02, 18.61, 19.44, 17.02, 20.22, 15.84, 20.0, 22.9, 18.3, 18.9, 17.4, 17.6, 18.0, 17.98, 17.82, 17.42, 19.47, 18.52, 19.9, 20.01, 16.87, 17.3, 15.41, 17.05, 18.9, 16.7, 16.9, 14.5, 15.5, 14.6, 18.6])+ , ("vs", D.fromList [0 :: Int32, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1])+ , ("am", D.fromList [1 :: Int32, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1])+ , ("gear", D.fromList [4 :: Int32, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 4, 4, 4, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 4])+ , ("carb", D.fromList [4 :: Int32, 4, 1, 1, 2, 1, 4, 2, 2, 4, 4, 3, 3, 3, 4, 4, 4, 1, 2, 1, 1, 2, 2, 4, 2, 1, 2, 2, 4, 6, 8, 2])+ ]++mtCars :: Test+mtCars =+ TestCase+ ( assertEqual+ "mt_cars"+ mtCarsDataset+ (unsafePerformIO (D.readParquet "./data/mtcars.parquet"))+ )+ -- Uncomment to run parquet tests. -- Currently commented because they don't run with github CI tests :: [Test]-tests = [] -- [allTypesPlain, allTypesPlainSnappy, allTypesDictionary]+tests = [] -- [allTypesPlain, allTypesPlainSnappy, allTypesDictionary, mtCars]