diff --git a/Frames.cabal b/Frames.cabal --- a/Frames.cabal +++ b/Frames.cabal @@ -1,5 +1,5 @@ name: Frames -version: 0.6.1 +version: 0.6.2 synopsis: Data frames For working with tabular data files description: User-friendly, type safe, runtime efficient tooling for working with tabular data deserialized from @@ -29,7 +29,7 @@ data/left1.csv data/right1.csv data/left_summary.csv data/FL2.csv cabal-version: >=1.10 -tested-with: GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.3 +tested-with: GHC == 8.4.4 || == 8.6.5 || == 8.8.2 source-repository head type: git @@ -62,9 +62,9 @@ TypeOperators, ConstraintKinds, StandaloneDeriving, UndecidableInstances, ScopedTypeVariables, OverloadedStrings, TypeApplications - build-depends: base >=4.8 && <4.13, + build-depends: base >=4.8 && <4.14, ghc-prim >=0.3 && <0.6, - primitive >= 0.6 && < 0.7, + primitive >= 0.6 && < 0.8, text >= 1.1.1.0, template-haskell, transformers, @@ -76,7 +76,7 @@ pipes-parse >= 3.0 && < 3.1, pipes-safe >= 2.2.6 && < 2.4, bytestring, - vinyl >= 0.10.0 && < 0.12, + vinyl >= 0.10.0 && < 0.13, discrimination, contravariant, hashable, @@ -209,6 +209,15 @@ main-is: Kata04.hs if flag(demos) build-depends: base, Frames, vinyl, text, readable + hs-source-dirs: demo + default-language: Haskell2010 + +executable modcsv + if !flag(demos) + buildable: False + main-is: ModifyCSV.hs + if flag(demos) + build-depends: base, Frames, microlens, pipes hs-source-dirs: demo default-language: Haskell2010 diff --git a/benchmarks/BenchDemo.hs b/benchmarks/BenchDemo.hs --- a/benchmarks/BenchDemo.hs +++ b/benchmarks/BenchDemo.hs @@ -1,9 +1,13 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell #-} +{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-} -- | Demonstration of streaming data processing. Try building with -- cabal (@cabal build benchdemo@), then running in bash with -- something like, -- -- @$ /usr/bin/time -l dist/build/benchdemo/benchdemo 2>&1 | head -n 4@ +-- Or, for cabal-install 3 (using Linux and GHC-8.8.2 as an example), +-- @$ time dist-newstyle/build/x86_64-linux/ghc-8.8.2/Frames-0.6.2/x/benchdemo/build/benchdemo/benchdemo@ +-- Or, for all resource usage on linux (avoid the bash builtin time), +-- @$ $(which time) -v dist-newstyle/build/x86_64-linux/ghc-8.8.2/Frames-0.6.2/x/benchdemo/build/benchdemo/benchdemo@ import qualified Control.Foldl as F import Frames import Pipes.Prelude (fold) diff --git a/demo/ModifyCSV.hs b/demo/ModifyCSV.hs new file mode 100644 --- /dev/null +++ b/demo/ModifyCSV.hs @@ -0,0 +1,23 @@ +{-# language DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-} +-- | A demonstration of ingesting a CSV file, modifying the data (in +-- this case multiplying a column by 2), then writing it back out to a +-- new CSV file. +import Frames +import Frames.CSV (pipeToCSV, consumeTextLines) +import Lens.Micro +import Pipes ((>->), Effect) +import qualified Pipes.Prelude as P + +tableTypes "Row" "data/data1.csv" + +myFun :: Row -> Row +myFun = age %~ (*2) + +myPipeline :: MonadSafe m => Effect m () +myPipeline = readTable "data/data1.csv" + >-> P.map myFun + >-> pipeToCSV + >-> consumeTextLines "data/dataMod.csv" + +main :: IO () +main = runSafeEffect myPipeline diff --git a/demo/Plot.hs b/demo/Plot.hs --- a/demo/Plot.hs +++ b/demo/Plot.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell #-} +{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-} module Main where import qualified Data.Vector.Unboxed as V import Diagrams.Backend.Rasterific diff --git a/demo/Plot2.hs b/demo/Plot2.hs --- a/demo/Plot2.hs +++ b/demo/Plot2.hs @@ -1,5 +1,5 @@ {-# LANGUAGE BangPatterns, DataKinds, FlexibleContexts, OverloadedStrings, - TemplateHaskell #-} + TemplateHaskell, TypeApplications #-} import Diagrams.Backend.Rasterific import Diagrams (dims2D, width, height) import Frames diff --git a/demo/framestack/app/Main.hs b/demo/framestack/app/Main.hs --- a/demo/framestack/app/Main.hs +++ b/demo/framestack/app/Main.hs @@ -7,31 +7,31 @@ import qualified Pipes as P import qualified Pipes.Prelude as P -tableTypes "Row" "../../data/data1.csv" +tableTypes "Row" "data/data1.csv" tbl :: IO [Row] -tbl = runSafeT . P.toListM $ readTable "../../data/data1.csv" +tbl = runSafeT . P.toListM $ readTable "data/data1.csv" ageDoubler :: (Age ∈ rs) => Record rs -> Record rs ageDoubler = age %~ (* 2) tbl2 :: IO [Row] -tbl2 = runSafeT . P.toListM $ readTable "../../data/data2.csv" +tbl2 = runSafeT . P.toListM $ readTable "data/data2.csv" tbl2a :: IO [ColFun Maybe Row] -tbl2a = runSafeT . P.toListM $ readTableMaybe "../../data/data2.csv" +tbl2a = runSafeT . P.toListM $ readTableMaybe "data/data2.csv" -- Sample data from http://support.spatialkey.com/spatialkey-sample-csv-data/ -- Note: We have to replace carriage returns (\r) with line feed -- characters (\n) for the text library's line parsing to work. -tableTypes "Ins" "../../data/FL2.csv" +tableTypes "Ins" "data/FL2.csv" insuranceTbl :: MonadSafe m => P.Producer Ins m () -insuranceTbl = readTable "../../data/FL2.csv" +insuranceTbl = readTable "data/FL2.csv" insMaybe :: MonadSafe m => P.Producer (ColFun Maybe Ins) m () -insMaybe = readTableMaybe "../../data/FL2.csv" +insMaybe = readTableMaybe "data/FL2.csv" type TinyIns = Record [PolicyID, PointLatitude, PointLongitude] diff --git a/src/Frames/CSV.hs b/src/Frames/CSV.hs --- a/src/Frames/CSV.hs +++ b/src/Frames/CSV.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, FlexibleInstances, GADTs, +{-# LANGUAGE CPP, DataKinds, FlexibleContexts, FlexibleInstances, GADTs, LambdaCase, OverloadedStrings, RankNTypes, ScopedTypeVariables, TemplateHaskell, TypeApplications, TypeOperators #-} @@ -13,7 +13,9 @@ import qualified Data.Foldable as F import Data.List (intercalate) import Data.Maybe (isNothing, fromMaybe) +#if __GLASGOW_HASKELL__ < 808 import Data.Monoid ((<>)) +#endif import Data.Proxy import qualified Data.Text as T import qualified Data.Text.Encoding as T @@ -364,12 +366,12 @@ -- streaming input that you wish to use to produce streaming output. pipeToCSV :: forall ts m. (Monad m, ColumnHeaders ts, RecordToList ts, - RecMapMethod Show ElField ts) + RecMapMethod ShowCSV ElField ts) => P.Pipe (Record ts) T.Text m () pipeToCSV = P.yield (T.intercalate "," (map T.pack header)) >> go where header = columnHeaders (Proxy :: Proxy (Record ts)) go :: P.Pipe (Record ts) T.Text m () - go = P.map (T.intercalate "," . map T.pack . showFields) + go = P.map (T.intercalate "," . showFieldsCSV) -- | Write a header row with column names followed by a line of text -- for each 'Record' to the given file. diff --git a/src/Frames/Categorical.hs b/src/Frames/Categorical.hs --- a/src/Frames/Categorical.hs +++ b/src/Frames/Categorical.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE DataKinds, KindSignatures, MagicHash, +{-# LANGUAGE CPP, DataKinds, KindSignatures, MagicHash, ScopedTypeVariables, TemplateHaskell, TypeFamilies, ViewPatterns #-} -- | Support for representing so-called categorical variables: a @@ -108,7 +108,11 @@ InstanceD Nothing [] (AppT (ConT ''ShowCSV) (ConT nameName)) [FunD 'showCSV (onVariants showCSVClause)] iVectorFor = +#if __GLASGOW_HASKELL__ >= 808 + TySynInstD (TySynEqn Nothing (AppT (ConT ''VectorFor) (ConT nameName)) (ConT ''VU.Vector)) +#else TySynInstD ''VectorFor (TySynEqn [ConT nameName] (ConT ''VU.Vector)) +#endif iNFData = let argName = mkName "x" in InstanceD Nothing [] (AppT (ConT ''NFData) (ConT nameName)) diff --git a/src/Frames/ColumnUniverse.hs b/src/Frames/ColumnUniverse.hs --- a/src/Frames/ColumnUniverse.hs +++ b/src/Frames/ColumnUniverse.hs @@ -10,7 +10,9 @@ CommonColumns, CommonColumnsCat, parsedTypeRep ) where import Data.Maybe (fromMaybe) +#if __GLASGOW_HASKELL__ < 808 import Data.Semigroup (Semigroup((<>))) +#endif import qualified Data.Text as T import Data.Vinyl import Data.Vinyl.CoRec diff --git a/src/Frames/Frame.hs b/src/Frames/Frame.hs --- a/src/Frames/Frame.hs +++ b/src/Frames/Frame.hs @@ -29,7 +29,7 @@ instance Eq r => Eq (Frame r) where Frame l1 r1 == Frame l2 r2 = - l1 == l2 && and (map (\i -> r1 i == r2 i) [0 .. l1 - 1]) + l1 == l2 && all (\i -> r1 i == r2 i) [0 .. l1 - 1] -- | The 'Monoid' instance for 'Frame' provides a mechanism for -- vertical concatenation of 'Frame's. That is, @f1 <> f2@ will return diff --git a/src/Frames/ShowCSV.hs b/src/Frames/ShowCSV.hs --- a/src/Frames/ShowCSV.hs +++ b/src/Frames/ShowCSV.hs @@ -14,3 +14,4 @@ instance ShowCSV Int where instance ShowCSV Double where instance ShowCSV Text where + showCSV = id diff --git a/test/Categorical.hs b/test/Categorical.hs --- a/test/Categorical.hs +++ b/test/Categorical.hs @@ -1,14 +1,19 @@ {-# LANGUAGE DataKinds, FlexibleContexts, MultiParamTypeClasses, - OverloadedLabels, OverloadedStrings, TemplateHaskell, - TypeFamilies, TypeOperators #-} + OverloadedLabels, OverloadedStrings, TemplateHaskell, + TypeApplications, TypeFamilies, TypeOperators #-} module Categorical where -import Data.Vinyl.Derived -import Data.Vinyl.XRec (toHKD) -import Frames -import Frames.Categorical (declareCategorical) -import Frames.TH (rowGenCat, RowGen(..), declarePrefixedColumn) -import Pipes (Producer, (>->)) -import qualified Pipes.Prelude as P +import Data.Vinyl.Derived +import Data.Vinyl.XRec ( toHKD ) +import Frames +import Frames.Categorical ( declareCategorical ) +import Frames.TH ( rowGenCat + , RowGen(..) + , declarePrefixedColumn + ) +import Pipes ( Producer + , (>->) + ) +import qualified Pipes.Prelude as P -- * Automatically inferred categorical data types @@ -19,17 +24,19 @@ -- no more than 8 variants. In our small data file, five distinct -- months appear, so a data type is generated and used. fifthMonthSmall :: IO (Maybe SmallMonth) -fifthMonthSmall = fmap (fmap (rvalf #month)) . runSafeEffect . P.head - $ load >-> P.drop 4 - where load :: MonadSafe m => Producer Small m () - load = readTableOpt smallParser "test/data/catSmall.csv" +fifthMonthSmall = + fmap (fmap (rvalf #month)) . runSafeEffect . P.head $ load >-> P.drop 4 + where + load :: MonadSafe m => Producer Small m () + load = readTableOpt smallParser "test/data/catSmall.csv" -- When every month appears, Frames leaves that column's type as 'Text'. fifthMonthLarge :: IO (Maybe Text) -fifthMonthLarge = fmap (fmap (rvalf #month)) . runSafeEffect . P.head - $ load >-> P.drop 4 - where load :: MonadSafe m => Producer Large m () - load = readTableOpt largeParser "test/data/catLarge.csv" +fifthMonthLarge = + fmap (fmap (rvalf #month)) . runSafeEffect . P.head $ load >-> P.drop 4 + where + load :: MonadSafe m => Producer Large m () + load = readTableOpt largeParser "test/data/catLarge.csv" -- * Custom categorical type @@ -51,7 +58,8 @@ -- parses. We use 'toHKD' to cut through the noise of the @(Maybe -- :. ElField)@ interpretation we parsed into. fifthMonthCustom :: IO (Maybe MyMonthData) -fifthMonthCustom = fmap (>>= toHKD . rgetf #month) . runSafeEffect . P.head - $ load >-> P.drop 4 - where load :: MonadSafe m => Producer (ColFun Maybe MyRow) m () - load = readTableMaybe "test/data/catSmall.csv" +fifthMonthCustom = + fmap (>>= toHKD . rgetf #month) . runSafeEffect . P.head $ load >-> P.drop 4 + where + load :: MonadSafe m => Producer (ColFun Maybe MyRow) m () + load = readTableMaybe "test/data/catSmall.csv" diff --git a/test/Issue114.hs b/test/Issue114.hs --- a/test/Issue114.hs +++ b/test/Issue114.hs @@ -5,23 +5,27 @@ {-# LANGUAGE OverloadedLabels #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeApplications #-} module Issue114 where import Frames -import qualified Control.Foldl as Foldl +import qualified Control.Foldl as Foldl import Control.Lens -import Pipes.Prelude (fold) +import Pipes.Prelude ( fold ) import Pipes -import qualified Pipes.Prelude as P -import qualified Pipes.Core as PC +import qualified Pipes.Prelude as P +import qualified Pipes.Core as PC -import qualified Data.Vinyl as V -import Data.Vinyl (Rec(..)) +import qualified Data.Vinyl as V +import Data.Vinyl ( Rec(..) ) -import qualified Data.Text as T +import qualified Data.Text as T import Data.Attoparsec.Text import Control.Applicative -import Frames.TH (tableTypesText', rowGen, RowGen(..)) +import Frames.TH ( tableTypesText' + , rowGen + , RowGen(..) + ) tableTypesText' (rowGen "test/data/issue114.csv") { rowTypeName = "ProductionData" } @@ -29,13 +33,12 @@ getOilVol :: ProductionData -> Record '[OilVolDouble] getOilVol x = V.Field (f (parseOnly parseDouble (x ^. oilVol))) :& RNil - where - f (Left _) = 0.0 / 0.0 - f (Right y) = y + where + f (Left _) = 0.0 / 0.0 + f (Right y) = y readOilVol :: MonadSafe m => Producer (Record '[OilVolDouble]) m () -readOilVol = (readTable "test/data/issue114.csv") >-> - (P.map getOilVol) +readOilVol = (readTable "test/data/issue114.csv") >-> (P.map getOilVol) oilVolLength :: Foldl.Fold (Record '[OilVolDouble]) Int oilVolLength = Foldl.length @@ -48,16 +51,16 @@ parseDouble :: Parser Double parseDouble = - do d <- double - return d - <|> - do _ <- string "" - return 0.0 + do + d <- double + return d + <|> do + _ <- string "" + return 0.0 test :: IO () test = do - (t, l) <- runSafeT $ - Foldl.purely fold oilVolTotalAndLength readOilVol + (t, l) <- runSafeT $ Foldl.purely fold oilVolTotalAndLength readOilVol putStrLn $ show l ++ " records totalling " ++ show t loadTable :: MonadSafe m => PC.Producer ProductionData m () diff --git a/test/LatinTest.hs b/test/LatinTest.hs --- a/test/LatinTest.hs +++ b/test/LatinTest.hs @@ -10,9 +10,8 @@ module LatinTest where -import Data.Vinyl (Rec) import Frames -import Frames.CSV (pipeTableMaybe, readFileLatin1Ln) +import Frames.CSV (readFileLatin1Ln) import Pipes (Producer, (>->)) import qualified Pipes.Prelude as P diff --git a/test/NoHeader.hs b/test/NoHeader.hs --- a/test/NoHeader.hs +++ b/test/NoHeader.hs @@ -1,11 +1,15 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, OverloadedLabels, TemplateHaskell #-} +{-# LANGUAGE DataKinds, FlexibleContexts, OverloadedLabels, TemplateHaskell, TypeApplications #-} module NoHeader where -import Control.Arrow ((&&&)) -import Data.Vinyl.Derived -import Frames -import Frames.TH (rowGen, RowGen(..)) -import Pipes (Producer, (>->)) -import qualified Pipes.Prelude as P +import Control.Arrow ( (&&&) ) +import Data.Vinyl.Derived +import Frames +import Frames.TH ( rowGen + , RowGen(..) + ) +import Pipes ( Producer + , (>->) + ) +import qualified Pipes.Prelude as P tableTypes' (rowGen "test/data/prestigeNoHeader.csv") { rowTypeName = "Row" @@ -22,7 +26,8 @@ -- | Extract the @Job@ and @Schooling@ columns of the indicated row. getJobAndSchooling :: Int -> IO (Maybe (Text, Double)) -getJobAndSchooling n = fmap (fmap aux) . runSafeEffect . P.head - $ loadData >-> P.drop (max n 0) - where aux :: Row -> (Text, Double) - aux = rvalf #job &&& rvalf #schooling +getJobAndSchooling n = + fmap (fmap aux) . runSafeEffect . P.head $ loadData >-> P.drop (max n 0) + where + aux :: Row -> (Text, Double) + aux = rvalf #job &&& rvalf #schooling diff --git a/test/Overlap.hs b/test/Overlap.hs --- a/test/Overlap.hs +++ b/test/Overlap.hs @@ -1,6 +1,9 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell #-} -module Main (main) where -import Frames +{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-} +module Main + ( main + ) +where +import Frames -- These data files have overlapping column definitions. Frames should -- not try to re-define an existing identifier. diff --git a/test/Spec.hs b/test/Spec.hs --- a/test/Spec.hs +++ b/test/Spec.hs @@ -132,10 +132,11 @@ -- The test data isn't formatted quite how we'd do it: text -- fields aren't quoted, and salaries represented as Doubles -- do not have decimal points. - let csvInput' = T.replace "Joe" "\"Joe\"" - . T.replace "Sarah" "\"Sarah\"" - . T.replace "\"80,000\"" "80000.0" - $ T.pack csvInput + -- let csvInput' = T.replace "Joe" "\"Joe\"" + -- . T.replace "Sarah" "\"Sarah\"" + -- . T.replace "\"80,000\"" "80000.0" + -- $ T.pack csvInput + let csvInput' = T.replace "\"80,000\"" "80000.0" (T.pack csvInput) it "Produces expected output" $ T.unlines (map T.pack csvOutput) `shouldBe` csvInput' it "Produces parseable output" $ diff --git a/test/UncurryFold.hs b/test/UncurryFold.hs --- a/test/UncurryFold.hs +++ b/test/UncurryFold.hs @@ -1,9 +1,8 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell #-} +{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell, TypeApplications #-} module UncurryFold where -import qualified Control.Foldl as L -import Data.Vinyl (rcast) -import Data.Vinyl.Curry (runcurryX) -import Frames +import qualified Control.Foldl as L +import Data.Vinyl.Curry ( runcurryX ) +import Frames -- Data set from http://vincentarelbundock.github.io/Rdatasets/datasets.html tableTypes "Row" "test/data/prestige.csv" diff --git a/test/UncurryFoldNoHeader.hs b/test/UncurryFoldNoHeader.hs --- a/test/UncurryFoldNoHeader.hs +++ b/test/UncurryFoldNoHeader.hs @@ -1,10 +1,12 @@ -{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell #-} +{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell, TypeApplications #-} module UncurryFoldNoHeader where -import qualified Control.Foldl as L -import Data.Vinyl (rcast) -import Data.Vinyl.Curry (runcurryX) -import Frames -import Frames.TH (rowGen, RowGen(..)) +import qualified Control.Foldl as L +import Data.Vinyl ( rcast ) +import Data.Vinyl.Curry ( runcurryX ) +import Frames +import Frames.TH ( rowGen + , RowGen(..) + ) -- Data set from http://vincentarelbundock.github.io/Rdatasets/datasets.html tableTypes' (rowGen "test/data/prestigeNoHeader.csv")