dataframe 0.3.5.0 → 0.4.0.0

raw patch · 12 files changed

+526/−308 lines, 12 filesdep +zlibdep ~dataframePVP ok

version bump matches the API change (PVP)

Dependencies added: zlib

Dependency ranges changed: dataframe

API changes (from Hackage documentation)

Files

• CHANGELOG.md +32/−0
• dataframe.cabal +5/−4
• src/DataFrame.hs +1/−8
• src/DataFrame/Functions.hs +23/−3
• src/DataFrame/IO/CSV.hs +10/−6
• src/DataFrame/IO/Parquet/Page.hs +3/−0
• src/DataFrame/IO/Parquet/Thrift.hs +166/−235
• src/DataFrame/Monad.hs +36/−6
• src/DataFrame/Operations/Core.hs +176/−1
• src/DataFrame/Operations/Statistics.hs +7/−9
• src/DataFrame/Synthesis.hs +25/−36
• tests/Functions.hs +42/−0

@@ -1,5 +1,37 @@ # Revision history for dataframe +## 0.4.0.0+* `readSeparated` no longer takes the separator as an argument. This is not placed into readOptions.+* Some improvements to the synthesis demo+* Add a `declareColumnsParquetFile` function. +* Column conversion functions now take expressions instead of strings.+* Add more monadic functions to make previously tricky transformations easier to write:+    ```haskell+    {-# LANGUAGE OverloadedStrings #-}+    {-# LANGUAGE TemplateHaskell #-}++    module Main where++    import qualified DataFrame as D+    import qualified DataFrame.Functions as F++    import DataFrame.Monad++    import Data.Text (Text)+    import DataFrame.Functions ((.&&), (.>=))++    $(F.declareColumnsFromCsvFile "./data/housing.csv")++    main :: IO ()+    main = do+        df <- D.readCsv "./data/housing.csv"+        print $ execFrameM df $ do+            is_expensive <- deriveM "is_expensive" (median_house_value .>= 500000)+            meanBedrooms <- inspectM (D.meanMaybe total_bedrooms)+            totalBedrooms <- imputeM total_bedrooms meanBedrooms+            filterWhereM (totalBedrooms .>= 200 .&& is_expensive)+    ```+ ## 0.3.5.0 * Add a `deriveWithExpr` that returns an expression that you can use in a subsequent expressions. * Add `declareColumnsFromCsvFile` which can create the expressions up front for use in scripts.

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.5.0+version:            0.4.0.0  synopsis: A fast, safe, and intuitive DataFrame library. @@ -107,6 +107,7 @@                       unordered-containers >= 0.1 && < 1,                       vector ^>= 0.13,                       vector-algorithms ^>= 0.9,+                      zlib >= 0.5 && < 1,                       zstd >= 0.1.2.0 && < 0.2,                       mmap >= 0.5.8 && <= 0.5.9,                       parallel >= 3.2.2.0 && < 5@@ -121,7 +122,7 @@     import: warnings     main-is: Benchmark.hs     build-depends:    base >= 4 && < 5,-                      dataframe ^>= 0.3,+                      dataframe ^>= 0.4,                       random >= 1 && < 2,                       time >= 1.12 && < 2,                       vector ^>= 0.13,@@ -150,7 +151,7 @@     build-depends: base >= 4 && < 5,                    criterion >= 1 && < 2,                    process >= 1.6 && < 2,-                   dataframe ^>= 0.3+                   dataframe ^>= 0.4     default-language: Haskell2010     ghc-options:       -threaded@@ -178,7 +179,7 @@                    Operations.Take,                    Parquet     build-depends:  base >= 4 && < 5,-                    dataframe ^>= 0.3,+                    dataframe ^>= 0.4,                     HUnit ^>= 1.6,                     random >= 1 && < 2,                     random-shuffle >= 0.0.4 && < 1,

@@ -270,16 +270,8 @@ import DataFrame.Internal.DataFrame as Dataframe (     DataFrame,     GroupedDataFrame,-    columnAsDoubleVector,-    columnAsFloatVector,-    columnAsIntVector,-    columnAsList,-    columnAsVector,     empty,     null,-    toDoubleMatrix,-    toFloatMatrix,-    toIntMatrix,     toMarkdownTable,  ) import DataFrame.Internal.Expression as Expression (Expr)@@ -317,6 +309,7 @@     imputeWith,     interQuartileRange,     mean,+    meanMaybe,     median,     skewness,     standardDeviation,

@@ -39,6 +39,7 @@ import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import qualified DataFrame.IO.CSV as CSV+import qualified DataFrame.IO.Parquet as Parquet import Debug.Trace (trace) import Language.Haskell.TH import qualified Language.Haskell.TH.Syntax as TH@@ -46,7 +47,7 @@ import Prelude hiding (maximum, minimum) import Prelude as P -infix 4 .==, .<, .<=, .>=, .>+infix 4 .==, .<, .<=, .>=, .>, ./= infixr 3 .&& infixr 2 .|| @@ -92,6 +93,9 @@ (.==) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool (.==) = BinaryOp "eq" (==) +(./=) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool+(./=) = BinaryOp "eq" (/=)+ eq :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool eq = BinaryOp "eq" (==) @@ -101,11 +105,15 @@ lt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool lt = BinaryOp "lt" (<) +-- TODO: Generalize this pattern for other equality functions. (.>) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.>) = BinaryOp "gt" (>)+(.>) (Lit l) (Lit r) = Lit (l > r)+(.>) (Lit l) expr = UnaryOp ("gt " <> T.pack (show l)) (l >) expr+(.>) expr (Lit r) = UnaryOp ("leq " <> T.pack (show r)) (r <=) expr+(.>) l r = BinaryOp "gt" (>) l r  gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-gt = BinaryOp "gt" (>)+gt = (.>)  (.<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool (.<=) = BinaryOp "leq" (<=)@@ -203,6 +211,7 @@  pow :: (Columnable a, Num a) => Expr a -> Int -> Expr a pow _ 0 = Lit 1+pow (Lit n) i = Lit (n ^ i) pow expr 1 = expr pow expr i = UnaryOp ("pow " <> T.pack (show i)) (^ i) expr @@ -363,6 +372,17 @@ declareColumnsFromCsvFile :: String -> DecsQ declareColumnsFromCsvFile path = do     df <- liftIO (CSV.readCsv path)+    declareColumns df++-- TODO: We don't have to read the whole file, we can just read the schema.+declareColumnsFromParquetFile :: String -> DecsQ+declareColumnsFromParquetFile path = do+    df <- liftIO (Parquet.readParquet path)+    declareColumns df++declareColumnsFromCsvWithOpts :: CSV.ReadOptions -> String -> DecsQ+declareColumnsFromCsvWithOpts opts path = do+    df <- liftIO (CSV.readSeparated opts path)     declareColumns df  declareColumns :: DataFrame -> DecsQ

@@ -163,6 +163,8 @@     Just 2010-03-04     @     -}+    , columnSeparator :: Char+    -- ^ Character that separates column values.     }  shouldInferFromSample :: TypeSpec -> Bool@@ -184,6 +186,7 @@         , typeSpec = InferFromSample 100         , safeRead = True         , dateFormat = "%Y-%m-%d"+        , columnSeparator = ','         }  {- | Read CSV file from path and load it into a dataframe.@@ -195,7 +198,7 @@ @ -} readCsv :: FilePath -> IO DataFrame-readCsv = readSeparated ',' defaultReadOptions+readCsv = readSeparated defaultReadOptions  {- | Read CSV file from path and load it into a dataframe. @@ -206,7 +209,7 @@ @ -} readCsvWithOpts :: ReadOptions -> FilePath -> IO DataFrame-readCsvWithOpts = readSeparated ','+readCsvWithOpts = readSeparated  {- | Read TSV (tab separated) file from path and load it into a dataframe. @@ -217,18 +220,19 @@ @ -} readTsv :: FilePath -> IO DataFrame-readTsv = readSeparated '\t' defaultReadOptions+readTsv = readSeparated (defaultReadOptions{columnSeparator = '\t'})  {- | Read text file with specified delimiter into a dataframe.  ==== __Example__ @-ghci> D.readSeparated ';' D.defaultReadOptions ".\/data\/taxi.txt"+ghci> D.readSeparated (D.defaultReadOptions { columnSeparator = ';' }) ".\/data\/taxi.txt"  @ -}-readSeparated :: Char -> ReadOptions -> FilePath -> IO DataFrame-readSeparated !sep !opts !path = do+readSeparated :: ReadOptions -> FilePath -> IO DataFrame+readSeparated !opts !path = do+    let sep = columnSeparator opts     csvData <- BL.readFile path     let decodeOpts = Csv.defaultDecodeOptions{Csv.decDelimiter = fromIntegral (ord sep)}     let stream = CsvStream.decodeWith decodeOpts Csv.NoHeader csvData

@@ -3,9 +3,11 @@  module DataFrame.IO.Parquet.Page where +import qualified Codec.Compression.GZip as GZip import Codec.Compression.Zstd.Streaming import Data.Bits import qualified Data.ByteString as BSO+import qualified Data.ByteString.Lazy as LB import Data.Int import Data.Maybe (fromMaybe, listToMaybe) import Data.Word@@ -42,6 +44,7 @@             Left e -> error (show e)             Right res -> pure res         UNCOMPRESSED -> pure (BSO.pack compressed)+        GZIP -> pure (LB.toStrict (GZip.decompress (LB.pack compressed)))         other -> error ("Unsupported compression type: " ++ show other)     pure         ( Just $ Page hdr (BSO.unpack fullData)

@@ -1,4 +1,5 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE StrictData #-} {-# LANGUAGE TypeApplications #-}  module DataFrame.IO.Parquet.Thrift where@@ -202,8 +203,8 @@                 ]  readField ::-    BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO (Maybe (TType, Int16))-readField buf pos lastFieldId fieldStack = do+    BS.ByteString -> IORef Int -> Int16 -> IO (Maybe (TType, Int16))+readField buf pos lastFieldId = do     t <- readAndAdvance pos buf     if t .&. 0x0f == 0         then return Nothing@@ -256,19 +257,17 @@             BS.pack $                 map (BS.index contents) [metadataStartPos .. (metadataStartPos + size - 1)]     let lastFieldId = 0-    let fieldStack = []     bufferPos <- newIORef (0 :: Int)-    readFileMetaData defaultMetadata metadataBytes bufferPos lastFieldId fieldStack+    readFileMetaData defaultMetadata metadataBytes bufferPos lastFieldId  readFileMetaData ::     FileMetadata ->     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO FileMetadata-readFileMetaData metadata metaDataBuf bufferPos lastFieldId fieldStack = do-    fieldContents <- readField metaDataBuf bufferPos lastFieldId fieldStack+readFileMetaData metadata metaDataBuf bufferPos lastFieldId = do+    fieldContents <- readField metaDataBuf bufferPos lastFieldId     case fieldContents of         Nothing -> return metadata         Just (elemType, identifier) -> case identifier of@@ -279,7 +278,6 @@                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             2 -> do                 sizeAndType <- readAndAdvance bufferPos metaDataBuf                 listSize <-@@ -290,13 +288,12 @@                 schemaElements <-                     replicateM                         listSize-                        (readSchemaElement defaultSchemaElement metaDataBuf bufferPos 0 [])+                        (readSchemaElement defaultSchemaElement metaDataBuf bufferPos 0)                 readFileMetaData                     (metadata{schema = schemaElements})                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             3 -> do                 numRows <- readIntFromBuffer @Int64 metaDataBuf bufferPos                 readFileMetaData@@ -304,7 +301,6 @@                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             4 -> do                 sizeAndType <- readAndAdvance bufferPos metaDataBuf                 listSize <-@@ -315,13 +311,12 @@                 -- TODO actually check elemType agrees (also for all the other underscored _elemType in this module)                 let _elemType = toTType sizeAndType                 rowGroups <--                    replicateM listSize (readRowGroup emptyRowGroup metaDataBuf bufferPos 0 [])+                    replicateM listSize (readRowGroup emptyRowGroup metaDataBuf bufferPos 0)                 readFileMetaData                     (metadata{rowGroups = rowGroups})                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             5 -> do                 sizeAndType <- readAndAdvance bufferPos metaDataBuf                 listSize <-@@ -331,13 +326,12 @@                  let _elemType = toTType sizeAndType                 keyValueMetadata <--                    replicateM listSize (readKeyValue emptyKeyValue metaDataBuf bufferPos 0 [])+                    replicateM listSize (readKeyValue emptyKeyValue metaDataBuf bufferPos 0)                 readFileMetaData                     (metadata{keyValueMetadata = keyValueMetadata})                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             6 -> do                 createdBy <- readString metaDataBuf bufferPos                 readFileMetaData@@ -345,7 +339,6 @@                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             7 -> do                 sizeAndType <- readAndAdvance bufferPos metaDataBuf                 listSize <-@@ -354,21 +347,19 @@                         else return $ fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f)                  let _elemType = toTType sizeAndType-                columnOrders <- replicateM listSize (readColumnOrder metaDataBuf bufferPos 0 [])+                columnOrders <- replicateM listSize (readColumnOrder metaDataBuf bufferPos 0)                 readFileMetaData                     (metadata{columnOrders = columnOrders})                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             8 -> do-                encryptionAlgorithm <- readEncryptionAlgorithm metaDataBuf bufferPos 0 []+                encryptionAlgorithm <- readEncryptionAlgorithm metaDataBuf bufferPos 0                 readFileMetaData                     (metadata{encryptionAlgorithm = encryptionAlgorithm})                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             9 -> do                 footerSigningKeyMetadata <- readByteString metaDataBuf bufferPos                 readFileMetaData@@ -376,7 +367,6 @@                     metaDataBuf                     bufferPos                     identifier-                    fieldStack             n -> return $ error $ "UNIMPLEMENTED " ++ show n  readSchemaElement ::@@ -384,13 +374,11 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO SchemaElement-readSchemaElement schemaElement buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readSchemaElement schemaElement buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return schemaElement-        Just (STOP, _) -> return schemaElement         Just (elemType, identifier) -> case identifier of             1 -> do                 schemaElemType <- toIntegralType <$> readInt32FromBuffer buf pos@@ -399,7 +387,6 @@                     buf                     pos                     identifier-                    fieldStack             2 -> do                 typeLength <- readInt32FromBuffer buf pos                 readSchemaElement@@ -407,7 +394,6 @@                     buf                     pos                     identifier-                    fieldStack             3 -> do                 fieldRepetitionType <- readInt32FromBuffer buf pos                 readSchemaElement@@ -415,11 +401,10 @@                     buf                     pos                     identifier-                    fieldStack             4 -> do                 nameSize <- readVarIntFromBuffer @Int buf pos                 if nameSize <= 0-                    then readSchemaElement schemaElement buf pos identifier fieldStack+                    then readSchemaElement schemaElement buf pos identifier                     else do                         contents <- replicateM nameSize (readAndAdvance pos buf)                         readSchemaElement@@ -427,7 +412,6 @@                             buf                             pos                             identifier-                            fieldStack             5 -> do                 numChildren <- readInt32FromBuffer buf pos                 readSchemaElement@@ -435,7 +419,6 @@                     buf                     pos                     identifier-                    fieldStack             6 -> do                 convertedType <- readInt32FromBuffer buf pos                 readSchemaElement@@ -443,10 +426,9 @@                     buf                     pos                     identifier-                    fieldStack             7 -> do                 scale <- readInt32FromBuffer buf pos-                readSchemaElement (schemaElement{scale = scale}) buf pos identifier fieldStack+                readSchemaElement (schemaElement{scale = scale}) buf pos identifier             8 -> do                 precision <- readInt32FromBuffer buf pos                 readSchemaElement@@ -454,7 +436,6 @@                     buf                     pos                     identifier-                    fieldStack             9 -> do                 fieldId <- readInt32FromBuffer buf pos                 readSchemaElement@@ -462,21 +443,19 @@                     buf                     pos                     identifier-                    fieldStack             10 -> do-                logicalType <- readLogicalType buf pos 0 []+                logicalType <- readLogicalType LOGICAL_TYPE_UNKNOWN buf pos 0                 readSchemaElement                     (schemaElement{logicalType = logicalType})                     buf                     pos                     identifier-                    fieldStack-            _ -> error "Uknown schema element"+            n -> error ("Uknown schema element: " ++ show n)  readRowGroup ::-    RowGroup -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO RowGroup-readRowGroup r buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+    RowGroup -> BS.ByteString -> IORef Int -> Int16 -> IO RowGroup+readRowGroup r buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return r         Just (elemType, identifier) -> case identifier of@@ -488,18 +467,18 @@                         else return $ fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f)                 let _elemType = toTType sizeAndType                 columnChunks <--                    replicateM listSize (readColumnChunk emptyColumnChunk buf pos 0 [])-                readRowGroup (r{rowGroupColumns = columnChunks}) buf pos identifier fieldStack+                    replicateM listSize (readColumnChunk emptyColumnChunk buf pos 0)+                readRowGroup (r{rowGroupColumns = columnChunks}) buf pos identifier             2 -> do                 totalBytes <- readIntFromBuffer @Int64 buf pos-                readRowGroup (r{totalByteSize = totalBytes}) buf pos identifier fieldStack+                readRowGroup (r{totalByteSize = totalBytes}) buf pos identifier             3 -> do                 nRows <- readIntFromBuffer @Int64 buf pos-                readRowGroup (r{rowGroupNumRows = nRows}) buf pos identifier fieldStack+                readRowGroup (r{rowGroupNumRows = nRows}) buf pos identifier             4 -> return r             5 -> do                 offset <- readIntFromBuffer @Int64 buf pos-                readRowGroup (r{fileOffset = offset}) buf pos identifier fieldStack+                readRowGroup (r{fileOffset = offset}) buf pos identifier             6 -> do                 compressedSize <- readIntFromBuffer @Int64 buf pos                 readRowGroup@@ -507,16 +486,15 @@                     buf                     pos                     identifier-                    fieldStack             7 -> do                 ordinal <- readIntFromBuffer @Int16 buf pos-                readRowGroup (r{ordinal = ordinal}) buf pos identifier fieldStack+                readRowGroup (r{ordinal = ordinal}) buf pos identifier             _ -> error $ "Unknown row group field: " ++ show identifier  readColumnChunk ::-    ColumnChunk -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnChunk-readColumnChunk c buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+    ColumnChunk -> BS.ByteString -> IORef Int -> Int16 -> IO ColumnChunk+readColumnChunk c buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return c         Just (elemType, identifier) -> case identifier of@@ -529,7 +507,6 @@                     buf                     pos                     identifier-                    fieldStack             2 -> do                 columnChunkMetadataFileOffset <- readIntFromBuffer @Int64 buf pos                 readColumnChunk@@ -537,15 +514,13 @@                     buf                     pos                     identifier-                    fieldStack             3 -> do-                columnMetadata <- readColumnMetadata emptyColumnMetadata buf pos 0 []+                columnMetadata <- readColumnMetadata emptyColumnMetadata buf pos 0                 readColumnChunk                     (c{columnMetaData = columnMetadata})                     buf                     pos                     identifier-                    fieldStack             4 -> do                 columnOffsetIndexOffset <- readIntFromBuffer @Int64 buf pos                 readColumnChunk@@ -553,7 +528,6 @@                     buf                     pos                     identifier-                    fieldStack             5 -> do                 columnOffsetIndexLength <- readInt32FromBuffer buf pos                 readColumnChunk@@ -561,7 +535,6 @@                     buf                     pos                     identifier-                    fieldStack             6 -> do                 columnChunkColumnIndexOffset <- readIntFromBuffer @Int64 buf pos                 readColumnChunk@@ -569,7 +542,6 @@                     buf                     pos                     identifier-                    fieldStack             7 -> do                 columnChunkColumnIndexLength <- readInt32FromBuffer buf pos                 readColumnChunk@@ -577,7 +549,6 @@                     buf                     pos                     identifier-                    fieldStack             _ -> error "Unknown column chunk"  readColumnMetadata ::@@ -585,27 +556,25 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO ColumnMetaData-readColumnMetadata cm buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readColumnMetadata cm buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return cm         Just (elemType, identifier) -> case identifier of             1 -> do                 cType <- parquetTypeFromInt <$> readInt32FromBuffer buf pos-                readColumnMetadata (cm{columnType = cType}) buf pos identifier []+                readColumnMetadata (cm{columnType = cType}) buf pos identifier             2 -> do                 sizeAndType <- readAndAdvance pos buf                 let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int                 let _elemType = toTType sizeAndType-                encodings <- replicateM sizeOnly (readParquetEncoding buf pos 0 [])+                encodings <- replicateM sizeOnly (readParquetEncoding buf pos 0)                 readColumnMetadata                     (cm{columnEncodings = encodings})                     buf                     pos                     identifier-                    fieldStack             3 -> do                 sizeAndType <- readAndAdvance pos buf                 let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int@@ -616,13 +585,12 @@                     buf                     pos                     identifier-                    fieldStack             4 -> do                 cType <- compressionCodecFromInt <$> readInt32FromBuffer buf pos-                readColumnMetadata (cm{columnCodec = cType}) buf pos identifier []+                readColumnMetadata (cm{columnCodec = cType}) buf pos identifier             5 -> do                 numValues <- readIntFromBuffer @Int64 buf pos-                readColumnMetadata (cm{columnNumValues = numValues}) buf pos identifier []+                readColumnMetadata (cm{columnNumValues = numValues}) buf pos identifier             6 -> do                 columnTotalUncompressedSize <- readIntFromBuffer @Int64 buf pos                 readColumnMetadata@@ -630,7 +598,6 @@                     buf                     pos                     identifier-                    []             7 -> do                 columnTotalCompressedSize <- readIntFromBuffer @Int64 buf pos                 readColumnMetadata@@ -638,19 +605,17 @@                     buf                     pos                     identifier-                    []             8 -> do                 sizeAndType <- readAndAdvance pos buf                 let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int                 let _elemType = toTType sizeAndType                 columnKeyValueMetadata <--                    replicateM sizeOnly (readKeyValue emptyKeyValue buf pos 0 [])+                    replicateM sizeOnly (readKeyValue emptyKeyValue buf pos 0)                 readColumnMetadata                     (cm{columnKeyValueMetadata = columnKeyValueMetadata})                     buf                     pos                     identifier-                    fieldStack             9 -> do                 columnDataPageOffset <- readIntFromBuffer @Int64 buf pos                 readColumnMetadata@@ -658,7 +623,6 @@                     buf                     pos                     identifier-                    []             10 -> do                 columnIndexPageOffset <- readIntFromBuffer @Int64 buf pos                 readColumnMetadata@@ -666,7 +630,6 @@                     buf                     pos                     identifier-                    []             11 -> do                 columnDictionaryPageOffset <- readIntFromBuffer @Int64 buf pos                 readColumnMetadata@@ -674,22 +637,20 @@                     buf                     pos                     identifier-                    []             12 -> do-                stats <- readStatistics emptyColumnStatistics buf pos 0 []-                readColumnMetadata (cm{columnStatistics = stats}) buf pos identifier fieldStack+                stats <- readStatistics emptyColumnStatistics buf pos 0+                readColumnMetadata (cm{columnStatistics = stats}) buf pos identifier             13 -> do                 sizeAndType <- readAndAdvance pos buf                 let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int                 let _elemType = toTType sizeAndType                 pageEncodingStats <--                    replicateM sizeOnly (readPageEncodingStats emptyPageEncodingStats buf pos 0 [])+                    replicateM sizeOnly (readPageEncodingStats emptyPageEncodingStats buf pos 0)                 readColumnMetadata                     (cm{columnEncodingStats = pageEncodingStats})                     buf                     pos                     identifier-                    fieldStack             14 -> do                 bloomFilterOffset <- readIntFromBuffer @Int64 buf pos                 readColumnMetadata@@ -697,7 +658,6 @@                     buf                     pos                     identifier-                    []             15 -> do                 bloomFilterLength <- readInt32FromBuffer buf pos                 readColumnMetadata@@ -705,22 +665,20 @@                     buf                     pos                     identifier-                    []             16 -> do-                stats <- readSizeStatistics emptySizeStatistics buf pos 0 []+                stats <- readSizeStatistics emptySizeStatistics buf pos 0                 readColumnMetadata                     (cm{columnSizeStatistics = stats})                     buf                     pos                     identifier-                    fieldStack             17 -> return $ error "UNIMPLEMENTED"             _ -> error $ "Unknown column metadata " ++ show identifier  readEncryptionAlgorithm ::-    BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm-readEncryptionAlgorithm buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+    BS.ByteString -> IORef Int -> Int16 -> IO EncryptionAlgorithm+readEncryptionAlgorithm buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return ENCRYPTION_ALGORITHM_UNKNOWN         Just (elemType, identifier) -> case identifier of@@ -730,25 +688,24 @@                     buf                     pos                     0-                    []             2 -> do                 readAesGcmCtrV1                     (AesGcmCtrV1{aadPrefix = [], aadFileUnique = [], supplyAadPrefix = False})                     buf                     pos                     0-                    []             n -> return ENCRYPTION_ALGORITHM_UNKNOWN  readColumnOrder ::-    BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnOrder-readColumnOrder buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+    BS.ByteString -> IORef Int -> Int16 -> IO ColumnOrder+readColumnOrder buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return COLUMN_ORDER_UNKNOWN         Just (elemType, identifier) -> case identifier of             1 -> do-                replicateM_ 2 (readTypeOrder buf pos 0 [])+                -- Read begin struct and stop since this an empty struct.+                replicateM_ 2 (readTypeOrder buf pos 0)                 return TYPE_ORDER             _ -> return COLUMN_ORDER_UNKNOWN @@ -757,16 +714,15 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO EncryptionAlgorithm-readAesGcmCtrV1 v@(AesGcmCtrV1 aadPrefix aadFileUnique supplyAadPrefix) buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readAesGcmCtrV1 v@(AesGcmCtrV1 aadPrefix aadFileUnique supplyAadPrefix) buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return v         Just (elemType, identifier) -> case identifier of             1 -> do                 aadPrefix <- readByteString buf pos-                readAesGcmCtrV1 (v{aadPrefix = aadPrefix}) buf pos lastFieldId fieldStack+                readAesGcmCtrV1 (v{aadPrefix = aadPrefix}) buf pos lastFieldId             2 -> do                 aadFileUnique <- readByteString buf pos                 readAesGcmCtrV1@@ -774,7 +730,6 @@                     buf                     pos                     lastFieldId-                    fieldStack             3 -> do                 supplyAadPrefix <- readAndAdvance pos buf                 readAesGcmCtrV1@@ -782,9 +737,8 @@                     buf                     pos                     lastFieldId-                    fieldStack             _ -> return ENCRYPTION_ALGORITHM_UNKNOWN-readAesGcmCtrV1 _ _ _ _ _ =+readAesGcmCtrV1 _ _ _ _ =     error "readAesGcmCtrV1 called with non AesGcmCtrV1"  readAesGcmV1 ::@@ -792,19 +746,18 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO EncryptionAlgorithm-readAesGcmV1 v@(AesGcmV1 aadPrefix aadFileUnique supplyAadPrefix) buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readAesGcmV1 v@(AesGcmV1 aadPrefix aadFileUnique supplyAadPrefix) buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return v         Just (elemType, identifier) -> case identifier of             1 -> do                 aadPrefix <- readByteString buf pos-                readAesGcmV1 (v{aadPrefix = aadPrefix}) buf pos lastFieldId fieldStack+                readAesGcmV1 (v{aadPrefix = aadPrefix}) buf pos lastFieldId             2 -> do                 aadFileUnique <- readByteString buf pos-                readAesGcmV1 (v{aadFileUnique = aadFileUnique}) buf pos lastFieldId fieldStack+                readAesGcmV1 (v{aadFileUnique = aadFileUnique}) buf pos lastFieldId             3 -> do                 supplyAadPrefix <- readAndAdvance pos buf                 readAesGcmV1@@ -812,35 +765,34 @@                     buf                     pos                     lastFieldId-                    fieldStack             _ -> return ENCRYPTION_ALGORITHM_UNKNOWN-readAesGcmV1 _ _ _ _ _ =+readAesGcmV1 _ _ _ _ =     error "readAesGcmV1 called with non AesGcmV1"  readTypeOrder ::-    BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnOrder-readTypeOrder buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+    BS.ByteString -> IORef Int -> Int16 -> IO ColumnOrder+readTypeOrder buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return TYPE_ORDER         Just (elemType, identifier) ->             if elemType == STOP                 then return TYPE_ORDER-                else readTypeOrder buf pos identifier fieldStack+                else readTypeOrder buf pos identifier  readKeyValue ::-    KeyValue -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO KeyValue-readKeyValue kv buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+    KeyValue -> BS.ByteString -> IORef Int -> Int16 -> IO KeyValue+readKeyValue kv buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return kv         Just (elemType, identifier) -> case identifier of             1 -> do                 k <- readString buf pos-                readKeyValue (kv{key = k}) buf pos identifier fieldStack+                readKeyValue (kv{key = k}) buf pos identifier             2 -> do                 v <- readString buf pos-                readKeyValue (kv{value = v}) buf pos identifier fieldStack+                readKeyValue (kv{value = v}) buf pos identifier             _ -> error "Unknown kv"  readPageEncodingStats ::@@ -848,19 +800,18 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO PageEncodingStats-readPageEncodingStats pes buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readPageEncodingStats pes buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return pes         Just (elemType, identifier) -> case identifier of             1 -> do                 pType <- pageTypeFromInt <$> readInt32FromBuffer buf pos-                readPageEncodingStats (pes{pageEncodingPageType = pType}) buf pos identifier []+                readPageEncodingStats (pes{pageEncodingPageType = pType}) buf pos identifier             2 -> do                 pEnc <- parquetEncodingFromInt <$> readInt32FromBuffer buf pos-                readPageEncodingStats (pes{pageEncoding = pEnc}) buf pos identifier []+                readPageEncodingStats (pes{pageEncoding = pEnc}) buf pos identifier             3 -> do                 encodedCount <- readInt32FromBuffer buf pos                 readPageEncodingStats@@ -868,34 +819,32 @@                     buf                     pos                     identifier-                    []             _ -> error "Unknown page encoding stats"  readParquetEncoding ::-    BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ParquetEncoding-readParquetEncoding buf pos lastFieldId fieldStack = parquetEncodingFromInt <$> readInt32FromBuffer buf pos+    BS.ByteString -> IORef Int -> Int16 -> IO ParquetEncoding+readParquetEncoding buf pos lastFieldId = parquetEncodingFromInt <$> readInt32FromBuffer buf pos  readStatistics ::     ColumnStatistics ->     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO ColumnStatistics-readStatistics cs buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readStatistics cs buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return cs         Just (elemType, identifier) -> case identifier of             1 -> do                 maxInBytes <- readByteString buf pos-                readStatistics (cs{columnMax = maxInBytes}) buf pos identifier fieldStack+                readStatistics (cs{columnMax = maxInBytes}) buf pos identifier             2 -> do                 minInBytes <- readByteString buf pos-                readStatistics (cs{columnMin = minInBytes}) buf pos identifier fieldStack+                readStatistics (cs{columnMin = minInBytes}) buf pos identifier             3 -> do                 nullCount <- readIntFromBuffer @Int64 buf pos-                readStatistics (cs{columnNullCount = nullCount}) buf pos identifier fieldStack+                readStatistics (cs{columnNullCount = nullCount}) buf pos identifier             4 -> do                 distinctCount <- readIntFromBuffer @Int64 buf pos                 readStatistics@@ -903,13 +852,12 @@                     buf                     pos                     identifier-                    fieldStack             5 -> do                 maxInBytes <- readByteString buf pos-                readStatistics (cs{columnMaxValue = maxInBytes}) buf pos identifier fieldStack+                readStatistics (cs{columnMaxValue = maxInBytes}) buf pos identifier             6 -> do                 minInBytes <- readByteString buf pos-                readStatistics (cs{columnMinValue = minInBytes}) buf pos identifier fieldStack+                readStatistics (cs{columnMinValue = minInBytes}) buf pos identifier             7 -> do                 isMaxValueExact <- readAndAdvance pos buf                 readStatistics@@ -917,7 +865,6 @@                     buf                     pos                     identifier-                    fieldStack             8 -> do                 isMinValueExact <- readAndAdvance pos buf                 readStatistics@@ -925,7 +872,6 @@                     buf                     pos                     identifier-                    fieldStack             _ -> error "Unknown statistics"  readSizeStatistics ::@@ -933,10 +879,9 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO SizeStatistics-readSizeStatistics ss buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readSizeStatistics ss buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return ss         Just (elemType, identifier) -> case identifier of@@ -947,7 +892,6 @@                     buf                     pos                     identifier-                    fieldStack             2 -> do                 sizeAndType <- readAndAdvance pos buf                 let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int@@ -959,7 +903,6 @@                     buf                     pos                     identifier-                    fieldStack             3 -> do                 sizeAndType <- readAndAdvance pos buf                 let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int@@ -971,7 +914,6 @@                     buf                     pos                     identifier-                    fieldStack             _ -> error "Unknown size statistics"  footerSize :: Int@@ -990,68 +932,63 @@     | otherwise = error ("Unknown type in schema: " ++ show n)  readLogicalType ::-    BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO LogicalType-readLogicalType buf pos lastFieldId fieldStack = do-    t <- readAndAdvance pos buf-    if t .&. 0x0f == 0-        then return LOGICAL_TYPE_UNKNOWN-        else do-            let modifier = fromIntegral ((t .&. 0xf0) `shiftR` 4) :: Int16-            identifier <--                if modifier == 0-                    then readIntFromBuffer @Int16 buf pos-                    else return (lastFieldId + modifier)-            let _elemType = toTType (t .&. 0x0f)-            case identifier of-                1 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return STRING_TYPE-                2 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return MAP_TYPE-                3 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return LIST_TYPE-                4 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return ENUM_TYPE-                5 -> do-                    readDecimalType 0 0 buf pos 0 []-                6 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return DATE_TYPE-                7 -> do-                    readTimeType False MILLISECONDS buf pos 0 []-                8 -> do-                    readTimestampType False MILLISECONDS buf pos 0 []-                -- Apparently reserved for interval types-                9 -> return LOGICAL_TYPE_UNKNOWN-                10 -> do-                    intType <- readIntType 0 False buf pos 0 []-                    _ <- readField buf pos 0 []-                    pure intType-                11 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return LOGICAL_TYPE_UNKNOWN-                12 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return JSON_TYPE-                13 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return BSON_TYPE-                14 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return UUID_TYPE-                15 -> do-                    replicateM_ 2 (readField buf pos 0 [])-                    return FLOAT16_TYPE-                16 -> do-                    return VariantType{specificationVersion = 1}-                17 -> do-                    return GeometryType{crs = ""}-                18 -> do-                    return GeographyType{crs = "", algorithm = SPHERICAL}-                _ -> return LOGICAL_TYPE_UNKNOWN+    LogicalType -> BS.ByteString -> IORef Int -> Int16 -> IO LogicalType+readLogicalType logicalType buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId+    case fieldContents of+        Nothing -> pure logicalType+        Just (elemType, identifier) -> case identifier of+            1 -> do+                -- This is an empty enum and is read as a field.+                _ <- readField buf pos 0+                readLogicalType STRING_TYPE buf pos identifier+            2 -> do+                _ <- readField buf pos 0+                readLogicalType MAP_TYPE buf pos identifier+            3 -> do+                _ <- readField buf pos 0+                readLogicalType LIST_TYPE buf pos identifier+            4 -> do+                _ <- readField buf pos 0+                readLogicalType ENUM_TYPE buf pos identifier+            5 -> do+                decimal <- readDecimalType 0 0 buf pos 0+                readLogicalType decimal buf pos identifier+            6 -> do+                _ <- readField buf pos 0+                readLogicalType DATE_TYPE buf pos identifier+            7 -> do+                time <- readTimeType False MILLISECONDS buf pos 0+                readLogicalType time buf pos identifier+            8 -> do+                timestamp <- readTimestampType False MILLISECONDS buf pos 0+                readLogicalType timestamp buf pos identifier+            -- Apparently reserved for interval types+            9 -> do+                _ <- readField buf pos 0+                readLogicalType LOGICAL_TYPE_UNKNOWN buf pos identifier+            10 -> do+                intType <- readIntType 0 False buf pos 0+                readLogicalType intType buf pos identifier+            11 -> do+                _ <- readField buf pos 0+                readLogicalType LOGICAL_TYPE_UNKNOWN buf pos identifier+            12 -> do+                _ <- readField buf pos 0+                readLogicalType JSON_TYPE buf pos identifier+            13 -> do+                _ <- readField buf pos 0+                readLogicalType BSON_TYPE buf pos identifier+            14 -> do+                _ <- readField buf pos 0+                readLogicalType UUID_TYPE buf pos identifier+            15 -> do+                _ <- readField buf pos 0+                readLogicalType FLOAT16_TYPE buf pos identifier+            16 -> error "Variant fields are unsupported"+            17 -> error "Geometry fields are unsupported"+            18 -> error "Geography fields are unsupported"+            n -> error $ "Unknown logical type field: " ++ show n  readIntType ::     Int8 ->@@ -1059,9 +996,8 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO LogicalType-readIntType bitWidth intIsSigned buf pos lastFieldId fieldStack = do+readIntType bitWidth intIsSigned buf pos lastFieldId = do     t <- readAndAdvance pos buf     if t .&. 0x0f == 0         then return (IntType bitWidth intIsSigned)@@ -1075,10 +1011,10 @@             case identifier of                 1 -> do                     bitWidth' <- readAndAdvance pos buf-                    readIntType (fromIntegral bitWidth') intIsSigned buf pos identifier fieldStack+                    readIntType (fromIntegral bitWidth') intIsSigned buf pos identifier                 2 -> do                     let intIsSigned' = (t .&. 0x0f) == compactBooleanTrue-                    readIntType bitWidth intIsSigned' buf pos identifier fieldStack+                    readIntType bitWidth intIsSigned' buf pos identifier                 _ -> error $ "UNKNOWN field ID for IntType: " ++ show identifier  readDecimalType ::@@ -1087,19 +1023,18 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO LogicalType-readDecimalType precision scale buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readDecimalType precision scale buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return (DecimalType precision scale)         Just (elemType, identifier) -> case identifier of             1 -> do                 scale' <- readInt32FromBuffer buf pos-                readDecimalType precision scale' buf pos lastFieldId fieldStack+                readDecimalType precision scale' buf pos lastFieldId             2 -> do                 precision' <- readInt32FromBuffer buf pos-                readDecimalType precision' scale buf pos lastFieldId fieldStack+                readDecimalType precision' scale buf pos lastFieldId             _ -> error $ "UNKNOWN field ID for DecimalType" ++ show identifier  readTimeType ::@@ -1108,20 +1043,19 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO LogicalType-readTimeType isAdjustedToUTC unit buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readTimeType isAdjustedToUTC unit buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return (TimeType isAdjustedToUTC unit)         Just (elemType, identifier) -> case identifier of             1 -> do                 -- TODO: Check for empty                 isAdjustedToUTC' <- (== compactBooleanTrue) <$> readAndAdvance pos buf-                readTimeType isAdjustedToUTC' unit buf pos lastFieldId fieldStack+                readTimeType isAdjustedToUTC' unit buf pos lastFieldId             2 -> do-                unit' <- readUnit buf pos 0 []-                readTimeType isAdjustedToUTC unit' buf pos lastFieldId fieldStack+                unit' <- readUnit TIME_UNIT_UNKNOWN buf pos 0+                readTimeType isAdjustedToUTC unit' buf pos lastFieldId             _ -> error $ "UNKNOWN field ID for TimeType" ++ show identifier  readTimestampType ::@@ -1130,35 +1064,32 @@     BS.ByteString ->     IORef Int ->     Int16 ->-    [Int16] ->     IO LogicalType-readTimestampType isAdjustedToUTC unit buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readTimestampType isAdjustedToUTC unit buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of         Nothing -> return (TimestampType isAdjustedToUTC unit)         Just (elemType, identifier) -> case identifier of             1 -> do                 -- TODO: Check for empty-                isAdjustedToUTC' <- (== compactBooleanTrue) <$> readAndAdvance pos buf-                readTimestampType isAdjustedToUTC' unit buf pos lastFieldId fieldStack+                isAdjustedToUTC' <- (== compactBooleanTrue) <$> readNoAdvance pos buf+                readTimestampType False unit buf pos lastFieldId             2 -> do-                unit' <- readUnit buf pos 0 []-                readTimestampType isAdjustedToUTC unit' buf pos lastFieldId fieldStack+                _ <- readField buf pos identifier+                unit' <- readUnit TIME_UNIT_UNKNOWN buf pos 0+                readTimestampType isAdjustedToUTC unit' buf pos lastFieldId             _ -> error $ "UNKNOWN field ID for TimestampType" ++ show identifier -readUnit :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO TimeUnit-readUnit buf pos lastFieldId fieldStack = do-    fieldContents <- readField buf pos lastFieldId fieldStack+readUnit :: TimeUnit -> BS.ByteString -> IORef Int -> Int16 -> IO TimeUnit+readUnit unit buf pos lastFieldId = do+    fieldContents <- readField buf pos lastFieldId     case fieldContents of-        Nothing -> return TIME_UNIT_UNKNOWN+        Nothing -> return unit         Just (elemType, identifier) -> case identifier of             1 -> do-                _ <- readField buf pos 0 []-                return MILLISECONDS+                readUnit MILLISECONDS buf pos identifier             2 -> do-                _ <- readField buf pos 0 []-                return MICROSECONDS+                readUnit MICROSECONDS buf pos identifier             3 -> do-                _ <- readField buf pos 0 []-                return NANOSECONDS-            _ -> return TIME_UNIT_UNKNOWN+                readUnit NANOSECONDS buf pos identifier+            n -> error $ "Unknown time unit: " ++ show n

@@ -1,6 +1,7 @@ {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-}+{-# LANGUAGE InstanceSigs #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TupleSections #-}@@ -10,7 +11,7 @@ import DataFrame (DataFrame) import qualified DataFrame as D import DataFrame.Internal.Column (Columnable)-import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Expression (Expr (..))  import qualified Data.Text as T @@ -20,31 +21,60 @@ newtype FrameM a = FrameM {runFrameM_ :: DataFrame -> (DataFrame, a)}  instance Functor FrameM where+    fmap :: (a -> b) -> FrameM a -> FrameM b     fmap f (FrameM g) = FrameM $ \df ->         let (df', x) = g df          in (df', f x)  instance Applicative FrameM where     pure x = FrameM (,x)+    (<*>) :: FrameM (a -> b) -> FrameM a -> FrameM b     FrameM ff <*> FrameM fx = FrameM $ \df ->         let (df1, f) = ff df             (df2, x) = fx df1          in (df2, f x)  instance Monad FrameM where+    (>>=) :: FrameM a -> (a -> FrameM b) -> FrameM b     FrameM g >>= f = FrameM $ \df ->         let (df1, x) = g df             FrameM h = f x          in h df1 +modifyM :: (DataFrame -> DataFrame) -> FrameM ()+modifyM f = FrameM $ \df -> (f df, ())++inspectM :: (DataFrame -> b) -> FrameM b+inspectM f = FrameM $ \df -> (df, f df)+ deriveM :: (Columnable a) => T.Text -> Expr a -> FrameM (Expr a) deriveM name expr = FrameM $ \df ->     let df' = D.derive name expr df-     in (df', expr) -- or (df', F.col @a name)+     in (df', Col name)  filterWhereM :: Expr Bool -> FrameM ()-filterWhereM p = FrameM $ \df ->-    (D.filterWhere p df, ())+filterWhereM p = modifyM (D.filterWhere p) -runFrameM :: DataFrame -> FrameM a -> DataFrame-runFrameM df action = fst (runFrameM_ action df)+filterJustM :: (Columnable a) => Expr (Maybe a) -> FrameM (Expr a)+filterJustM (Col name) = FrameM $ \df ->+    let df' = D.filterJust name df+     in (df', Col name)+filterJustM expr =+    error $ "Cannot filter on compound expression: " ++ show expr++imputeM :: (Columnable a) => Expr (Maybe a) -> a -> FrameM (Expr a)+imputeM expr@(Col name) value = FrameM $ \df ->+    let df' = D.impute expr value df+     in (df', Col name)+imputeM expr _ = error $ "Cannot impute on compound expression: " ++ show expr++runFrameM :: DataFrame -> FrameM a -> (a, DataFrame)+runFrameM df (FrameM action) =+    let (df', a) = action df+     in (a, df')++evalFrameM :: DataFrame -> FrameM a -> a+evalFrameM df m = fst (runFrameM df m)++execFrameM :: DataFrame -> FrameM a -> DataFrame+execFrameM df m = snd (runFrameM df m)

@@ -31,8 +31,17 @@     expandColumn,     fromList,     fromVector,+    toDoubleVector,+    toFloatVector,+    toIntVector,+    toUnboxedVector,  )-import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn)+import DataFrame.Internal.DataFrame (+    DataFrame (..),+    empty,+    getColumn,+    unsafeGetColumn,+ ) import DataFrame.Internal.Expression import DataFrame.Internal.Parsing (isNullish) import DataFrame.Internal.Row (Any, mkColumnFromRow)@@ -749,3 +758,169 @@ -} fold :: (a -> DataFrame -> DataFrame) -> [a] -> DataFrame -> DataFrame fold f xs acc = L.foldl' (flip f) acc xs++{- | Returns a dataframe as a two dimensional vector of floats.++Converts all columns in the dataframe to float vectors and transposes them+into a row-major matrix representation.++This is useful for handing data over into ML systems.++Returns 'Left' with an error if any column cannot be converted to floats.+-}+toFloatMatrix ::+    DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float))+toFloatMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> toFloatVector c)+    (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Float)))+    (columns df) of+    Left e -> Left e+    Right m ->+        pure $+            V.generate+                (fst (dataframeDimensions df))+                ( \i ->+                    foldl+                        (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))+                        VU.empty+                        [0 .. (V.length m - 1)]+                )++{- | Returns a dataframe as a two dimensional vector of doubles.++Converts all columns in the dataframe to double vectors and transposes them+into a row-major matrix representation.++This is useful for handing data over into ML systems.++Returns 'Left' with an error if any column cannot be converted to doubles.+-}+toDoubleMatrix ::+    DataFrame -> Either DataFrameException (V.Vector (VU.Vector Double))+toDoubleMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> toDoubleVector c)+    (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Double)))+    (columns df) of+    Left e -> Left e+    Right m ->+        pure $+            V.generate+                (fst (dataframeDimensions df))+                ( \i ->+                    foldl+                        (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))+                        VU.empty+                        [0 .. (V.length m - 1)]+                )++{- | Returns a dataframe as a two dimensional vector of ints.++Converts all columns in the dataframe to int vectors and transposes them+into a row-major matrix representation.++This is useful for handing data over into ML systems.++Returns 'Left' with an error if any column cannot be converted to ints.+-}+toIntMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Int))+toIntMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> toIntVector c)+    (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Int)))+    (columns df) of+    Left e -> Left e+    Right m ->+        pure $+            V.generate+                (fst (dataframeDimensions df))+                ( \i ->+                    foldl+                        (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))+                        VU.empty+                        [0 .. (V.length m - 1)]+                )++{- | Get a specific column as a vector.++You must specify the type via type applications.++==== __Examples__++>>> columnAsVector @Int "age" df+[25, 30, 35, ...]++>>> columnAsVector @Text "name" df+["Alice", "Bob", "Charlie", ...]++==== __Throws__++* 'error' - if the column type doesn't match the requested type+-}+columnAsVector :: forall a. (Columnable a) => Expr a -> DataFrame -> V.Vector a+columnAsVector (Col name) df = case unsafeGetColumn name df of+    (BoxedColumn (col :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of+        Nothing -> error "Type error"+        Just Refl -> col+    (OptionalColumn (col :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of+        Nothing -> error "Type error"+        Just Refl -> col+    (UnboxedColumn (col :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of+        Nothing -> error "Type error"+        Just Refl -> VG.convert col+columnAsVector _ _ = error "UNIMPLEMENTED"++{- | Retrieves a column as an unboxed vector of 'Int' values.++Returns 'Left' with a 'DataFrameException' if the column cannot be converted to ints.+This may occur if the column contains non-numeric data or values outside the 'Int' range.+-}+columnAsIntVector ::+    Expr Int -> DataFrame -> Either DataFrameException (VU.Vector Int)+columnAsIntVector (Col name) df = toIntVector (unsafeGetColumn name df)+columnAsIntVector _ _ = error "UNIMPLEMENTED"++{- | Retrieves a column as an unboxed vector of 'Double' values.++Returns 'Left' with a 'DataFrameException' if the column cannot be converted to doubles.+This may occur if the column contains non-numeric data.+-}+columnAsDoubleVector ::+    Expr Double -> DataFrame -> Either DataFrameException (VU.Vector Double)+columnAsDoubleVector (Col name) df = toDoubleVector (unsafeGetColumn name df)+columnAsDoubleVector _ _ = error "UNIMPLEMENTED"++{- | Retrieves a column as an unboxed vector of 'Float' values.++Returns 'Left' with a 'DataFrameException' if the column cannot be converted to floats.+This may occur if the column contains non-numeric data.+-}+columnAsFloatVector ::+    Expr Float -> DataFrame -> Either DataFrameException (VU.Vector Float)+columnAsFloatVector (Col name) df = toFloatVector (unsafeGetColumn name df)+columnAsFloatVector _ _ = error "UNIMPLEMENTED"++columnAsUnboxedVector ::+    forall a.+    (Columnable a, VU.Unbox a) =>+    Expr a -> DataFrame -> Either DataFrameException (VU.Vector a)+columnAsUnboxedVector (Col name) df = toUnboxedVector @a (unsafeGetColumn name df)+columnAsUnboxedVector _ _ = error "UNIMPLEMENTED"++{- | Get a specific column as a list.++You must specify the type via type applications.++==== __Examples__++>>> columnAsList @Int "age" df+[25, 30, 35, ...]++>>> columnAsList @Text "name" df+["Alice", "Bob", "Charlie", ...]++==== __Throws__++* 'error' - if the column type doesn't match the requested type+-}+columnAsList :: forall a. (Columnable a) => Expr a -> DataFrame -> [a]+columnAsList expr@(Col name) df = V.toList (columnAsVector expr df)+columnAsList _ _ = error "UNIMPLEMENTED"

@@ -25,8 +25,6 @@ import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (     DataFrame (..),-    columnAsUnboxedVector,-    columnAsVector,     empty,     getColumn,  )@@ -89,7 +87,7 @@ -- | Calculates the mean of a given column as a standalone value. mean ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-mean (Col name) df = case columnAsUnboxedVector @a name df of+mean (Col name) df = case columnAsUnboxedVector (Col @a name) df of     Right xs -> mean' xs     Left e -> throw e mean expr df = case interpret df expr of@@ -100,7 +98,7 @@  meanMaybe ::     forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double-meanMaybe (Col name) df = (mean' . optionalToDoubleVector) (columnAsVector @(Maybe a) name df)+meanMaybe (Col name) df = (mean' . optionalToDoubleVector) (columnAsVector (Col @(Maybe a) name) df) meanMaybe expr df = case interpret @(Maybe a) df expr of     Left e -> throw e     Right (TColumn col) -> case toVector @(Maybe a) col of@@ -110,7 +108,7 @@ -- | Calculates the median of a given column as a standalone value. median ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-median (Col name) df = case columnAsUnboxedVector @a name df of+median (Col name) df = case columnAsUnboxedVector (Col @a name) df of     Right xs -> median' xs     Left e -> throw e median expr df = case interpret df expr of@@ -122,7 +120,7 @@ -- | Calculates the standard deviation of a given column as a standalone value. standardDeviation ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-standardDeviation (Col name) df = case columnAsUnboxedVector @a name df of+standardDeviation (Col name) df = case columnAsUnboxedVector (Col @a name) df of     Right xs -> (sqrt . variance') xs     Left e -> throw e standardDeviation expr df = case interpret df expr of@@ -134,7 +132,7 @@ -- | Calculates the skewness of a given column as a standalone value. skewness ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-skewness (Col name) df = case columnAsUnboxedVector @a name df of+skewness (Col name) df = case columnAsUnboxedVector (Col @a name) df of     Right xs -> skewness' xs     Left e -> throw e skewness expr df = case interpret df expr of@@ -146,7 +144,7 @@ -- | Calculates the variance of a given column as a standalone value. variance ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-variance (Col name) df = case columnAsUnboxedVector @a name df of+variance (Col name) df = case columnAsUnboxedVector (Col @a name) df of     Right xs -> variance' xs     Left e -> throw e variance expr df = case interpret df expr of@@ -158,7 +156,7 @@ -- | Calculates the inter-quartile range of a given column as a standalone value. interQuartileRange ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-interQuartileRange (Col name) df = case columnAsUnboxedVector @a name df of+interQuartileRange (Col name) df = case columnAsUnboxedVector (Col @a name) df of     Right xs -> interQuartileRange' xs     Left e -> throw e interQuartileRange expr df = case interpret df expr of

@@ -22,11 +22,10 @@     Expr (..),     eSize,     interpret,-    replaceExpr,  ) import DataFrame.Internal.Statistics import qualified DataFrame.Operations.Statistics as Stats-import DataFrame.Operations.Subset (exclude, select)+import DataFrame.Operations.Subset (exclude)  import Control.Exception (throw) import Data.Containers.ListUtils@@ -40,7 +39,6 @@ import qualified Data.Vector.Unboxed as VU import DataFrame.Functions ((.&&), (.<=), (.>), (.||)) import qualified DataFrame.Operations.Core as D-import qualified DataFrame.Operations.Transformations as D import Debug.Trace (trace) import Type.Reflection (typeRep) @@ -80,6 +78,7 @@         existingPrograms             ++ [ transform p                | p <- ps ++ vars+               , Prelude.not (isConditional p)                , transform <-                     [ sqrt                     , abs@@ -93,20 +92,30 @@                ]             ++ [ F.pow p i                | p <- existingPrograms+               , Prelude.not (isConditional p)                , i <- [2 .. 6]                ]             ++ [ p + q                | (i, p) <- zip [0 ..] existingPrograms                , (j, q) <- zip [0 ..] existingPrograms                , Prelude.not (isLiteral p && isLiteral q)+               , Prelude.not (isConditional p || isConditional q)                , i >= j                ]+            ++ [ p - q+               | (i, p) <- zip [0 ..] existingPrograms+               , (j, q) <- zip [0 ..] existingPrograms+               , Prelude.not (isLiteral p && isLiteral q)+               , Prelude.not (isConditional p || isConditional q)+               , i /= j+               ]             ++ ( if includeConds                     then                         [ F.min p q                         | (i, p) <- zip [0 ..] existingPrograms                         , (j, q) <- zip [0 ..] existingPrograms                         , Prelude.not (isLiteral p && isLiteral q)+                        , Prelude.not (isConditional p || isConditional q)                         , p /= q                         , i > j                         ]@@ -114,6 +123,7 @@                                | (i, p) <- zip [0 ..] existingPrograms                                , (j, q) <- zip [0 ..] existingPrograms                                , Prelude.not (isLiteral p && isLiteral q)+                               , Prelude.not (isConditional p || isConditional q)                                , p /= q                                , i > j                                ]@@ -121,26 +131,23 @@                                | cond <- conds                                , r <- existingPrograms                                , s <- existingPrograms+                               , Prelude.not (isConditional r || isConditional s)                                , r /= s                                ]                     else []                )-            ++ [ p - q-               | (i, p) <- zip [0 ..] existingPrograms-               , (j, q) <- zip [0 ..] existingPrograms-               , Prelude.not (isLiteral p && isLiteral q)-               , i /= j-               ]             ++ [ p * q                | (i, p) <- zip [0 ..] existingPrograms                , (j, q) <- zip [0 ..] existingPrograms                , Prelude.not (isLiteral p && isLiteral q)+               , Prelude.not (isConditional p || isConditional q)                , i >= j                ]             ++ [ p / q                | p <- existingPrograms                , q <- existingPrograms                , Prelude.not (isLiteral p && isLiteral q)+               , Prelude.not (isConditional p || isConditional q)                , p /= q                ] @@ -148,6 +155,10 @@ isLiteral (Lit _) = True isLiteral _ = False +isConditional :: Expr a -> Bool+isConditional (If{}) = True+isConditional _ = False+ deduplicate ::     forall a.     (Columnable a) =>@@ -289,34 +300,12 @@         t = case interpret df (Col target) of             Left e -> throw e             Right v -> v+        cfg = BeamConfig d b MeanSquaredError True+        constants = percentiles df' ++ [Lit 10, Lit 1, Lit 0.1, Lit targetMean]      in-        case beamSearch-            df'-            ( BeamConfig-                d-                b-                MutualInformation-                False-            )-            t-            (percentiles df')-            []-            [] of+        case beamSearch df' cfg t constants [] [] of             Nothing -> Left "No programs found"-            Just p ->-                trace (show p) $-                    let-                     in case beamSearch-                            ( D.derive "_generated_regression_feature_" p df-                                & select ["_generated_regression_feature_"]-                            )-                            (BeamConfig d b MeanSquaredError False)-                            t-                            (percentiles df' ++ [Lit targetMean, Lit 10])-                            []-                            [Col "_generated_regression_feature_"] of-                            Nothing -> Left "Could not find coefficients"-                            Just p' -> Right (replaceExpr p (Col @Double "_generated_regression_feature_") p')+            Just p -> Right p  data LossFunction     = PearsonCorrelation@@ -375,7 +364,7 @@             (generatePrograms (includeConditionals cfg) conditions vars constants ps)   where     vars = map Col names-    conditions = generateConditions outputs conds (vars ++ constants ++ ps) df+    conditions = generateConditions outputs conds vars df     ps = pickTopN df outputs cfg $ deduplicate df programs     names = (map fst . L.sortBy (compare `on` snd) . M.toList . columnIndices) df 

@@ -6,9 +6,11 @@ import qualified DataFrame as D import DataFrame.Functions (     sanitize,+    (.>),  ) import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression import Test.HUnit  -- Test cases for the sanitize function@@ -79,8 +81,48 @@             (D.derive "sum" (F.sum (F.col @Int "A")) df)         ) +testGtTwoLitsRewriteAsLit :: Test+testGtTwoLitsRewriteAsLit =+    TestCase+        ( assertEqual+            "Two lits rewrite as one"+            (Lit True)+            (Lit (5 :: Int) .> Lit 4)+        )++testGtLeftLitRewriteAsUnary :: Test+testGtLeftLitRewriteAsUnary =+    TestCase+        ( assertEqual+            "Two lits rewrite as single literal"+            (UnaryOp "gt 5" ((5 :: Int) >) (Col "x"))+            (Lit (5 :: Int) .> Col "x")+        )++testGtRightLitRewriteAsUnary :: Test+testGtRightLitRewriteAsUnary =+    TestCase+        ( assertEqual+            "Two lits rewrite as unary op"+            (UnaryOp "gt 5" ((5 :: Int) >) (Col "x"))+            (Lit (5 :: Int) .> Col "x")+        )++testGtNoLitWriteAsBinary :: Test+testGtNoLitWriteAsBinary =+    TestCase+        ( assertEqual+            "Two lits rewrite as unary op"+            (BinaryOp "gt" (>) (Col @Int "x") (Col "y"))+            (Col @Int "x" .> Col "y")+        )+ tests :: [Test] tests =     [ TestLabel "sanitizeIdentifiers" sanitizeIdentifiers     , TestLabel "testSum" testSum+    , TestLabel "testGtTwoLitRewrite" testGtTwoLitsRewriteAsLit+    , TestLabel "testGtLeftLit" testGtLeftLitRewriteAsUnary+    , TestLabel "testGtRightLit" testGtRightLitRewriteAsUnary+    , TestLabel "testGtNoLit" testGtNoLitWriteAsBinary     ]