diff --git a/Frames.cabal b/Frames.cabal --- a/Frames.cabal +++ b/Frames.cabal @@ -1,5 +1,5 @@ name: Frames -version: 0.7.2 +version: 0.7.3 synopsis: Data frames For working with tabular data files description: User-friendly, type safe, runtime efficient tooling for working with tabular data deserialized from @@ -30,7 +30,7 @@ data/left1.csv data/right1.csv data/left_summary.csv data/FL2.csv cabal-version: >=1.10 -tested-with: GHC == 8.6.5 || == 8.8.4 || == 8.10.4 || == 9.0.1 +tested-with: GHC == 8.6.5 || == 8.8.4 || == 8.10.7 || == 9.0.1 || == 9.2.1 source-repository head type: git @@ -64,8 +64,8 @@ TypeOperators, ConstraintKinds, StandaloneDeriving, UndecidableInstances, ScopedTypeVariables, OverloadedStrings, TypeApplications - build-depends: base >=4.10 && <4.16, - ghc-prim >=0.3 && <0.8, + build-depends: base >=4.10 && <4.17, + ghc-prim >=0.3 && <0.9, primitive >= 0.6 && < 0.8, text >= 1.1.1.0, template-haskell, @@ -78,7 +78,7 @@ pipes-parse >= 3.0 && < 3.1, pipes-safe >= 2.2.6 && < 2.4, bytestring, - vinyl >= 0.13.0 && < 0.14, + vinyl >= 0.13.0 && < 0.15, discrimination, contravariant, hashable, @@ -95,7 +95,11 @@ buildable: False main-is: GetData.hs if flag(demos) - build-depends: base, bytestring, http-client >= 0.4.3, zip-archive, directory + build-depends: + base, bytestring, + http-client >= 0.4.3, + http-client-tls, + zip-archive, directory hs-source-dirs: data default-language: Haskell2010 ghc-options: -Wall @@ -146,7 +150,7 @@ pipes hs-source-dirs: demo/framestack/app default-language: Haskell2010 - ghc-options: -O2 + ghc-options: -O2 -fsimpl-tick-factor=200 -- ghc-options: -O2 -fllvm executable tutorial @@ -172,7 +176,7 @@ hs-source-dirs: benchmarks default-language: Haskell2010 -- ghc-options: -O2 - ghc-options: -O2 -fllvm + ghc-options: -O2 -fllvm -fsimpl-tick-factor=200 -- A demonstration of dealing with missing data. Provided for source -- code and experimentation rather than a useful executable. @@ -234,7 +238,7 @@ temporary, directory, htoml, regex-applicative, pretty, unordered-containers, pipes, HUnit, vinyl, foldl >= 1.3 && < 1.5, - attoparsec, lens + attoparsec, lens, bytestring ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall default-language: Haskell2010 diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you have a CSV data where the values of each column may be classified by a single type, and ideally you have a header row giving each column a name, you may simply want to avoid writing out the Haskell type corresponding to each row. `Frames` provides `TemplateHaskell` machinery to infer a Haskell type for each row of your data set, thus preventing the situation where your code quietly diverges from your data. -We generate a collection of definitions generated by inspecting the data file at compile time (using `tableTypes`), then, at runtime, load that data into column-oriented storage in memory (an **in-core** array of structures (AoS)). We're going to compute the average ratio of two columns, so we'll use the `foldl` library. Our fold will project the columns we want, and apply a function that divides one by the other after appropriate numeric type conversions. Here is the entirety of that [program](https://github.com/acowley/Frames/tree/master/test/UncurryFold.hs). +We generate a collection of definitions generated by inspecting the data file at compile time (using `tableTypes`), then, at runtime, load that data into column-oriented storage in memory with a row-oriented interface (an **in-core** array of structures (AoS)). We're going to compute the average ratio of two columns, so we'll use the `foldl` library. Our fold will project the columns we want, and apply a function that divides one by the other after appropriate numeric type conversions. Here is the entirety of that [program](https://github.com/acowley/Frames/tree/master/test/UncurryFold.hs). ```haskell {-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell, TypeApplications #-} diff --git a/data/GetData.hs b/data/GetData.hs --- a/data/GetData.hs +++ b/data/GetData.hs @@ -4,13 +4,14 @@ import Data.Maybe (fromJust) import Data.Monoid (First(..)) import Network.HTTP.Client +import Network.HTTP.Client.TLS (tlsManagerSettings) import System.Directory (createDirectoryIfMissing) getPrestige :: IO () -getPrestige = do m <- newManager defaultManagerSettings +getPrestige = do m <- newManager tlsManagerSettings httpLbs req m >>= B.writeFile "data/prestige.csv" . responseBody - where Just req = parseUrlThrow "http://vincentarelbundock.github.io/Rdatasets/csv/car/Prestige.csv" + where Just req = parseUrlThrow "https://vincentarelbundock.github.io/Rdatasets/csv/carData/Prestige.csv" getFLinsurance :: IO () getFLinsurance = do m <- newManager defaultManagerSettings diff --git a/demo/Plot2.hs b/demo/Plot2.hs --- a/demo/Plot2.hs +++ b/demo/Plot2.hs @@ -44,6 +44,7 @@ putStrLn $ "The average farmer/fisher is "++ show (fromIntegral age_ / n) ++ " and made " ++ show (fromIntegral inc / n) + mkPlot where aux !(!sumAge, !sumIncome, n) f = (sumAge + f^.age, sumIncome + f^.capitalGain, n+1) -- Independent folds diff --git a/src/Frames/CSV.hs b/src/Frames/CSV.hs --- a/src/Frames/CSV.hs +++ b/src/Frames/CSV.hs @@ -244,7 +244,6 @@ -> P.Producer (Rec (Maybe :. ElField) rs) m () readTableMaybeOpt opts csvFile = produceTokens csvFile (columnSeparator opts) >-> pipeTableMaybeOpt opts -{-# INLINE readTableMaybeOpt #-} -- | Stream lines of CSV data into rows of ’Rec’ values values where -- any given entry can fail to parse. @@ -256,7 +255,6 @@ P.map (rmap (either (const (Compose Nothing)) (Compose . Just) . getCompose) . readRec) -{-# INLINE pipeTableMaybeOpt #-} -- | Stream lines of CSV data into rows of ’Rec’ values values where -- any given entry can fail to parse. In the case of a parse failure, the @@ -267,20 +265,17 @@ pipeTableEitherOpt opts = do when (isNothing (headerOverride opts)) (() <$ P.await) P.map (readRow opts) -{-# INLINE pipeTableEitherOpt #-} -- | Produce rows where any given entry can fail to parse. readTableMaybe :: (P.MonadSafe m, ReadRec rs, RMap rs) => FilePath -> P.Producer (Rec (Maybe :. ElField) rs) m () readTableMaybe = readTableMaybeOpt defaultParser -{-# INLINE readTableMaybe #-} -- | Stream lines of CSV data into rows of ’Rec’ values where any -- given entry can fail to parse. pipeTableMaybe :: (Monad m, ReadRec rs, RMap rs) => P.Pipe [T.Text] (Rec (Maybe :. ElField) rs) m () pipeTableMaybe = pipeTableMaybeOpt defaultParser -{-# INLINE pipeTableMaybe #-} -- | Stream lines of CSV data into rows of ’Rec’ values where any -- given entry can fail to parse. In the case of a parse failure, the @@ -288,7 +283,6 @@ pipeTableEither :: (Monad m, ReadRec rs) => P.Pipe T.Text (Rec (Either T.Text :. ElField) rs) m () pipeTableEither = pipeTableEitherOpt defaultParser -{-# INLINE pipeTableEither #-} -- -- | Returns a `MonadPlus` producer of rows for which each column was -- -- successfully parsed. This is typically slower than 'readTableOpt'. @@ -305,14 +299,12 @@ -- False -> let r = recMaybe . readRow opts <$> T.hGetLine h -- in liftIO r >>= maybe go (flip mplus go . return) -- go --- {-# INLINE readTableOpt' #-} -- -- | Returns a `MonadPlus` producer of rows for which each column was -- -- successfully parsed. This is typically slower than 'readTable'. -- readTable' :: forall m rs. (P.MonadSafe m, ReadRec rs) -- => FilePath -> m (Record rs) -- readTable' = readTableOpt' defaultParser --- {-# INLINE readTable' #-} -- | Returns a producer of rows for which each column was successfully -- parsed. @@ -320,21 +312,18 @@ => ParserOptions -> FilePath -> P.Producer (Record rs) m () readTableOpt opts csvFile = readTableMaybeOpt opts csvFile P.>-> go where go = P.await >>= maybe go (\x -> P.yield x >> go) . recMaybe -{-# INLINE readTableOpt #-} -- | Pipe lines of CSV text into rows for which each column was -- successfully parsed. pipeTableOpt :: (ReadRec rs, RMap rs, Monad m) => ParserOptions -> P.Pipe [T.Text] (Record rs) m () pipeTableOpt opts = pipeTableMaybeOpt opts >-> P.map recMaybe >-> P.concat -{-# INLINE pipeTableOpt #-} -- | Returns a producer of rows for which each column was successfully -- parsed. readTable :: (P.MonadSafe m, ReadRec rs, RMap rs) => FilePath -> P.Producer (Record rs) m () readTable = readTableOpt defaultParser -{-# INLINE readTable #-} readRecEither :: (ReadRec rs, RMap rs) => [T.Text] -> Either (Rec (Either T.Text :. ElField) rs) (Record rs) @@ -372,7 +361,6 @@ pipeTable :: (ReadRec rs, RMap rs, Monad m) => P.Pipe [T.Text] (Record rs) m () pipeTable = pipeTableOpt defaultParser -{-# INLINE pipeTable #-} -- * Writing CSV Data diff --git a/src/Frames/Categorical.hs b/src/Frames/Categorical.hs --- a/src/Frames/Categorical.hs +++ b/src/Frames/Categorical.hs @@ -76,7 +76,11 @@ fromStringClause variant variantCon = Clause [LitP (StringL variant)] (NormalB (ConE variantCon)) [] showCSVClause variant variantCon = +#if MIN_VERSION_template_haskell(2,18,0) + Clause [ConP variantCon [] []] +#else Clause [ConP variantCon []] +#endif (NormalB (AppE (VarE 'T.pack) (LitE (StringL variant)))) [] readableGuarded :: Name -> String -> Name -> (Guard, Exp) diff --git a/src/Frames/ColumnUniverse.hs b/src/Frames/ColumnUniverse.hs --- a/src/Frames/ColumnUniverse.hs +++ b/src/Frames/ColumnUniverse.hs @@ -98,7 +98,6 @@ instance (T.Text ∈ ts, RPureConstrained Parseable ts) => Monoid (CoRec ColInfo ts) where mempty = CoRec (ColInfo ( Right (ConT (mkName "Text")), Possibly T.empty)) - mappend x y = x <> y -- | A helper For the 'Semigroup' instance below. mergeEqTypeParses :: forall ts. (RPureConstrained Parseable ts, T.Text ∈ ts) diff --git a/src/Frames/Exploration.hs b/src/Frames/Exploration.hs --- a/src/Frames/Exploration.hs +++ b/src/Frames/Exploration.hs @@ -14,6 +14,7 @@ takeRows, dropRows) where import Data.Char (isSpace, isUpper) import qualified Data.Foldable as F +import Data.Kind (Type) import Data.List (intercalate) import Data.Proxy import qualified Data.Vinyl as V @@ -21,7 +22,7 @@ import Data.Vinyl.Functor (ElField(Field), Const(..)) import Frames.Rec import GHC.TypeLits (Symbol) -import Language.Haskell.TH +import Language.Haskell.TH hiding (Type) import Language.Haskell.TH.Quote import Pipes hiding (Proxy) import qualified Pipes as P @@ -92,7 +93,7 @@ -- * ToList -recToList :: forall a (rs :: [(Symbol,*)]). +recToList :: forall a (rs :: [(Symbol, Type)]). (V.RecMapMethod ((~) a) ElField rs, V.RecordToList rs) => Record rs -> [a] recToList = V.recordToList . V.rmapMethod @((~) a) aux diff --git a/src/Frames/Frame.hs b/src/Frames/Frame.hs --- a/src/Frames/Frame.hs +++ b/src/Frames/Frame.hs @@ -36,7 +36,6 @@ -- a new 'Frame' with the rows of @f1@ followed by the rows of @f2@. instance Monoid (Frame r) where mempty = Frame 0 (const $ error "index out of bounds (empty frame)") - f1 `mappend` f2 = f1 <> f2 instance Semigroup (Frame r) where Frame l1 f1 <> Frame l2 f2 = diff --git a/src/Frames/InCore.hs b/src/Frames/InCore.hs --- a/src/Frames/InCore.hs +++ b/src/Frames/InCore.hs @@ -1,11 +1,12 @@ {-# LANGUAGE BangPatterns, CPP, DataKinds, EmptyCase, - FlexibleInstances, PolyKinds, ScopedTypeVariables, - TupleSections, TypeFamilies, TypeOperators, - UndecidableInstances #-} + FlexibleContexts, FlexibleInstances, PolyKinds, + ScopedTypeVariables, TupleSections, TypeFamilies, + TypeOperators, UndecidableInstances #-} -- | Efficient in-memory (in-core) storage of tabular data. module Frames.InCore where import Control.Monad.Primitive import Control.Monad.ST (runST) +import Data.Kind (Type) import Data.Proxy import Data.Text (Text) import qualified Data.Vector as VB @@ -23,12 +24,12 @@ import GHC.Prim (RealWorld) #endif import GHC.TypeLits (KnownSymbol) -import GHC.Types (Symbol, Type) +import GHC.Types (Symbol) import qualified Pipes as P import qualified Pipes.Prelude as P -- | The most efficient vector type for each column data type. -type family VectorFor t :: * -> * +type family VectorFor t :: Type -> Type type instance VectorFor Bool = VU.Vector type instance VectorFor Int = VU.Vector type instance VectorFor Float = VU.Vector diff --git a/src/Frames/RecF.hs b/src/Frames/RecF.hs --- a/src/Frames/RecF.hs +++ b/src/Frames/RecF.hs @@ -29,6 +29,7 @@ ColFun, ColumnHeaders, columnHeaders) where -- import Data.List (intercalate) +import Data.Kind (Type) import Data.Proxy import qualified Data.Vinyl as V import Data.Vinyl (Rec(..)) @@ -70,7 +71,7 @@ -- pattern x :& xs <- (frameUncons -> (x, xs)) where -- x :& xs = frameCons x xs -class ColumnHeaders (cs::[(Symbol,*)]) where +class ColumnHeaders (cs::[(Symbol, Type)]) where -- | Return the column names for a record. columnHeaders :: proxy (Rec f cs) -> [String] @@ -88,7 +89,7 @@ -- | Strip the column information from each element of a list of -- types. -type family UnColumn (ts :: [(Symbol,*)]) where +type family UnColumn (ts :: [(Symbol, Type)]) where UnColumn '[] = '[] UnColumn ((s :-> t) ': ts) = t ': UnColumn ts diff --git a/test/DataCSV.hs b/test/DataCSV.hs --- a/test/DataCSV.hs +++ b/test/DataCSV.hs @@ -2,10 +2,12 @@ module DataCSV where import Control.Monad ((>=>)) import Data.Bifunctor (first) +import qualified Data.ByteString as BS import qualified Data.HashMap.Lazy as H import Data.Maybe (catMaybes) import qualified Data.Text as T import qualified Data.Text.IO as T +import Data.Text.Encoding (decodeUtf8) import Language.Haskell.TH.Syntax (Lift(..)) import Text.Toml import Text.Toml.Types (Node (VTable, VString), Table) @@ -20,8 +22,11 @@ -- lift (CsvExample n c g) = [e| CsvExample n c g |] examplesFrom :: FilePath -> IO [CsvExample] -examplesFrom fp = (either error id . ((first show . parseTomlDoc "examples") >=> go)) - <$> T.readFile fp +examplesFrom fp = + (either error id + . ((first show . parseTomlDoc "examples") >=> go) + . decodeUtf8) + <$> BS.readFile fp where go :: Table -> Either String [CsvExample] go = fmap catMaybes . mapM (uncurry ex . first T.unpack) . H.toList ex :: String -> Node -> Either String (Maybe CsvExample)