packages feed

dataframe-parquet-th (empty) → 1.0.0.0

raw patch · 3 files changed

+141/−0 lines, 3 filesdep +Globdep +basedep +containers

Dependencies added: Glob, base, containers, dataframe-core, dataframe-parquet, dataframe-th, directory, filepath, template-haskell, text

Files

+ LICENSE view
@@ -0,0 +1,20 @@+Copyright (c) 2026 Michael Chavinda++Permission is hereby granted, free of charge, to any person obtaining+a copy of this software and associated documentation files (the+"Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish,+distribute, sublicense, and/or sell copies of the Software, and to+permit persons to whom the Software is furnished to do so, subject to+the following conditions:++The above copyright notice and this permission notice shall be included+in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ dataframe-parquet-th.cabal view
@@ -0,0 +1,41 @@+cabal-version:      2.4+name:               dataframe-parquet-th+version:            1.0.0.0++synopsis:           Parquet-file-based Template Haskell splices for the dataframe ecosystem.+description:+    Splices that read Parquet file metadata at compile time and emit+    per-column bindings. Record-based splices live in @dataframe-th@.++bug-reports:        https://github.com/mchav/dataframe/issues+license:            MIT+license-file:       LICENSE+author:             Michael Chavinda+maintainer:         mschavinda@gmail.com+copyright:          (c) 2024-2025 Michael Chavinda+category:           Data+tested-with:        GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2++common warnings+    ghc-options:+        -Wincomplete-patterns+        -Wincomplete-uni-patterns+        -Wunused-imports+        -Wunused-local-binds++library+    import:             warnings+    exposed-modules:+                        DataFrame.TH.Parquet+    build-depends:      base >= 4 && < 5,+                        containers >= 0.6.7 && < 0.9,+                        dataframe-core ^>= 1.0,+                        dataframe-parquet ^>= 1.0,+                        dataframe-th ^>= 1.0,+                        directory >= 1.3.0.0 && < 2,+                        filepath >= 1.4 && < 2,+                        Glob >= 0.10 && < 1,+                        template-haskell >= 2.0 && < 3,+                        text >= 2.0 && < 3+    hs-source-dirs:     src+    default-language:   Haskell2010
+ src/DataFrame/TH/Parquet.hs view
@@ -0,0 +1,80 @@+{-# LANGUAGE FlexibleContexts #-}++{- |+Module      : DataFrame.TH.Parquet+License     : MIT++Parquet-file-based 'DataFrame.TH' splices. Splits out the Parquet ingest+path so @dataframe-th@ stays IO-agnostic.+-}+module DataFrame.TH.Parquet (+    declareColumnsFromParquetFile,+) where++import Control.Monad (filterM)+import Control.Monad.IO.Class (liftIO)+import Data.Int (Int64)+import qualified Data.Maybe as Maybe+import qualified Data.Set as S+import qualified Data.Text as T++import Language.Haskell.TH+import System.Directory (doesDirectoryExist)+import System.FilePath ((</>))+import System.FilePath.Glob (glob)++import qualified DataFrame.IO.Parquet as Parquet+import DataFrame.IO.Parquet.Schema (schemaToEmptyDataFrame)+import DataFrame.IO.Parquet.Thrift (+    cc_meta_data,+    cmd_path_in_schema,+    cmd_statistics,+    rg_columns,+    row_groups,+    schema,+    stats_null_count,+    unField,+ )+import qualified DataFrame.Internal.DataFrame as DI+import DataFrame.TH.Records (declareColumns)+import Prelude as P++{- | Splice a binding for every column of a parquet file (or directory of+parquet files). The schema is read from each file's metadata and merged.+-}+declareColumnsFromParquetFile :: String -> DecsQ+declareColumnsFromParquetFile path = do+    isDir <- liftIO $ doesDirectoryExist path+    let pat = if isDir then path </> "*.parquet" else path+    matches <- liftIO $ glob pat+    files <- liftIO $ filterM (fmap P.not . doesDirectoryExist) matches+    metas <- liftIO $ mapM Parquet.readMetadataFromPath files+    let nullableCols :: S.Set T.Text+        nullableCols =+            S.fromList+                [ T.pack (last colPath)+                | meta <- metas+                , rg <- unField (row_groups meta)+                , cc <- unField (rg_columns rg)+                , Just cm <- [unField (cc_meta_data cc)]+                , let colPath = map T.unpack (unField (cmd_path_in_schema cm))+                , P.not (null colPath)+                , let nc :: Int64+                      nc = case unField (cmd_statistics cm) of+                        Nothing -> 0+                        Just stats ->+                            Maybe.fromMaybe 0 (unField $ stats_null_count stats)+                , nc > 0+                ]+    let df =+            foldl+                ( \acc meta ->+                    acc+                        <> schemaToEmptyDataFrame+                            nullableCols+                            (unField (schema meta))+                )+                DI.empty+                metas++    declareColumns df