dataframe-parquet-th 1.3.0.0 → 1.4.0.0

raw patch · 3 files changed

+32/−46 lines, 3 filesdep ~dataframe-coredep ~dataframe-parquetdep ~dataframe-th

Dependency ranges changed: dataframe-core, dataframe-parquet, dataframe-th

Files

• dataframe-parquet-th.cabal +4/−4
• src/DataFrame/TH/Parquet.hs +27/−28
• src/DataFrame/Typed/TH/Parquet.hs +1/−14

@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-parquet-th-version:            1.3.0.0+version:            1.4.0.0 synopsis:           Parquet-file-based Template Haskell splices for the dataframe ecosystem. description:     Splices that read Parquet file metadata at compile time and emit@@ -30,9 +30,9 @@                         DataFrame.Typed.TH.Parquet     build-depends:      base >= 4 && < 5,                         containers >= 0.6.7 && < 0.10,-                        dataframe-core >= 2.2 && < 2.3,-                        dataframe-parquet >= 1.5 && < 1.6,-                        dataframe-th >= 2.2 && < 2.3,+                        dataframe-core >= 2.6 && < 2.7,+                        dataframe-parquet >= 1.6 && < 1.7,+                        dataframe-th >= 2.3 && < 2.4,                         directory >= 1.3.0.0 && < 2,                         filepath >= 1.4 && < 2,                         Glob >= 0.10 && < 1,

@@ -1,14 +1,9 @@ {-# LANGUAGE FlexibleContexts #-} -{- |-Module      : DataFrame.TH.Parquet-License     : MIT--Parquet-file-based 'DataFrame.TH' splices. Splits out the Parquet ingest-path so @dataframe-th@ stays IO-agnostic.--}+-- | Splices that derive column bindings from Parquet files. module DataFrame.TH.Parquet (     declareColumnsFromParquetFile,+    deriveSchemaValuesFromParquetFile, ) where  import Control.Monad (filterM)@@ -36,19 +31,25 @@     unField,  ) import qualified DataFrame.Internal.DataFrame as DI-import DataFrame.TH.Records (declareColumns)+import DataFrame.TH.Records (declareColumns, declareSchemaValues) import Prelude as P -{- | Splice a binding for every column of a parquet file (or directory of-parquet files). The schema is read from each file's metadata and merged.--}+-- | Splice a binding for every column of a parquet file or directory. declareColumnsFromParquetFile :: String -> DecsQ-declareColumnsFromParquetFile path = do-    isDir <- liftIO $ doesDirectoryExist path+declareColumnsFromParquetFile path =+    liftIO (emptyFrameFromParquet path) >>= declareColumns++deriveSchemaValuesFromParquetFile :: String -> String -> DecsQ+deriveSchemaValuesFromParquetFile prefix path =+    liftIO (emptyFrameFromParquet path) >>= declareSchemaValues prefix++emptyFrameFromParquet :: String -> IO DI.DataFrame+emptyFrameFromParquet path = do+    isDir <- doesDirectoryExist path     let pat = if isDir then path </> "*.parquet" else path-    matches <- liftIO $ glob pat-    files <- liftIO $ filterM (fmap P.not . doesDirectoryExist) matches-    metas <- liftIO $ mapM Parquet.readMetadataFromPath files+    matches <- glob pat+    files <- filterM (fmap P.not . doesDirectoryExist) matches+    metas <- mapM Parquet.readMetadataFromPath files     let nullableCols :: S.Set T.Text         nullableCols =             S.fromList@@ -66,15 +67,13 @@                             Maybe.fromMaybe 0 (unField $ stats_null_count stats)                 , nc > 0                 ]-    let df =-            foldl-                ( \acc meta ->-                    acc-                        <> schemaToEmptyDataFrame-                            nullableCols-                            (unField (schema meta))-                )-                DI.empty-                metas--    declareColumns df+    pure $+        foldl+            ( \acc meta ->+                acc+                    <> schemaToEmptyDataFrame+                        nullableCols+                        (unField (schema meta))+            )+            DI.empty+            metas

@@ -1,17 +1,4 @@-{- |-Module      : DataFrame.Typed.TH.Parquet-License     : MIT--Parquet-file-based typed-schema derivation. The typed counterpart of-'DataFrame.TH.Parquet.declareColumnsFromParquetFile': instead of binding one-@Expr@ per column, it generates a type synonym describing the file's schema,-suitable for 'DataFrame.Typed.freeze' / @readParquet \@Schema@.--@-deriveSchemaFromParquetFile \"Trips\" \"trips.parquet\"--- generates:  type Trips = '[ '(\"id\", Int), '(\"fare\", Double), ...]-@--}+-- | Derives a typed schema from a Parquet file. module DataFrame.Typed.TH.Parquet (     deriveSchemaFromParquetFile, ) where