Frames 0.6.1 → 0.6.2

raw patch · 19 files changed

+157/−92 lines, 19 filesdep ~basedep ~pipesdep ~primitivenew-component:exe:modcsvPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: base, pipes, primitive, vinyl

API changes (from Hackage documentation)

Files

• Frames.cabal +14/−5
• benchmarks/BenchDemo.hs +5/−1
• demo/ModifyCSV.hs +23/−0
• demo/Plot.hs +1/−1
• demo/Plot2.hs +1/−1
• demo/framestack/app/Main.hs +7/−7
• src/Frames/CSV.hs +5/−3
• src/Frames/Categorical.hs +5/−1
• src/Frames/ColumnUniverse.hs +2/−0
• src/Frames/Frame.hs +1/−1
• src/Frames/ShowCSV.hs +1/−0
• test/Categorical.hs +29/−21
• test/Issue114.hs +23/−20
• test/LatinTest.hs +1/−2
• test/NoHeader.hs +16/−11
• test/Overlap.hs +6/−3
• test/Spec.hs +5/−4
• test/UncurryFold.hs +4/−5
• test/UncurryFoldNoHeader.hs +8/−6

@@ -1,5 +1,5 @@ name:                Frames-version:             0.6.1+version:             0.6.2 synopsis:            Data frames For working with tabular data files description:         User-friendly, type safe, runtime efficient tooling for                      working with tabular data deserialized from@@ -29,7 +29,7 @@                      data/left1.csv data/right1.csv data/left_summary.csv                      data/FL2.csv cabal-version:       >=1.10-tested-with:         GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.3+tested-with:         GHC == 8.4.4 || == 8.6.5 || == 8.8.2  source-repository head   type:     git@@ -62,9 +62,9 @@                        TypeOperators, ConstraintKinds, StandaloneDeriving,                        UndecidableInstances, ScopedTypeVariables,                        OverloadedStrings, TypeApplications-  build-depends:       base >=4.8 && <4.13,+  build-depends:       base >=4.8 && <4.14,                        ghc-prim >=0.3 && <0.6,-                       primitive >= 0.6 && < 0.7,+                       primitive >= 0.6 && < 0.8,                        text >= 1.1.1.0,                        template-haskell,                        transformers,@@ -76,7 +76,7 @@                        pipes-parse >= 3.0 && < 3.1,                        pipes-safe >= 2.2.6 && < 2.4,                        bytestring,-                       vinyl >= 0.10.0 && < 0.12,+                       vinyl >= 0.10.0 && < 0.13,                        discrimination,                        contravariant,                        hashable,@@ -209,6 +209,15 @@   main-is: Kata04.hs   if flag(demos)     build-depends: base, Frames, vinyl, text, readable+  hs-source-dirs: demo+  default-language: Haskell2010++executable modcsv+  if !flag(demos)+    buildable: False+  main-is: ModifyCSV.hs+  if flag(demos)+    build-depends: base, Frames, microlens, pipes   hs-source-dirs: demo   default-language: Haskell2010 

@@ -1,9 +1,13 @@-{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell #-}+{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-} -- | Demonstration of streaming data processing. Try building with -- cabal (@cabal build benchdemo@), then running in bash with -- something like, -- -- @$ /usr/bin/time -l dist/build/benchdemo/benchdemo 2>&1 | head -n 4@+-- Or, for cabal-install 3 (using Linux and GHC-8.8.2 as an example),+-- @$ time dist-newstyle/build/x86_64-linux/ghc-8.8.2/Frames-0.6.2/x/benchdemo/build/benchdemo/benchdemo@+-- Or, for all resource usage on linux (avoid the bash builtin time),+-- @$ $(which time) -v dist-newstyle/build/x86_64-linux/ghc-8.8.2/Frames-0.6.2/x/benchdemo/build/benchdemo/benchdemo@ import qualified Control.Foldl as F import Frames import Pipes.Prelude (fold)

@@ -0,0 +1,23 @@+{-# language DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-}+-- | A demonstration of ingesting a CSV file, modifying the data (in+-- this case multiplying a column by 2), then writing it back out to a+-- new CSV file.+import Frames+import Frames.CSV (pipeToCSV, consumeTextLines)+import Lens.Micro+import Pipes ((>->), Effect)+import qualified Pipes.Prelude as P++tableTypes "Row" "data/data1.csv"++myFun :: Row -> Row+myFun = age %~ (*2)++myPipeline :: MonadSafe m => Effect m ()+myPipeline = readTable "data/data1.csv"+             >-> P.map myFun+             >-> pipeToCSV+             >-> consumeTextLines "data/dataMod.csv"++main :: IO ()+main = runSafeEffect myPipeline

@@ -1,4 +1,4 @@-{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell #-}+{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-} module Main where import qualified Data.Vector.Unboxed as V import Diagrams.Backend.Rasterific

@@ -1,5 +1,5 @@ {-# LANGUAGE BangPatterns, DataKinds, FlexibleContexts, OverloadedStrings,-             TemplateHaskell #-}+             TemplateHaskell, TypeApplications #-} import Diagrams.Backend.Rasterific import Diagrams (dims2D, width, height) import Frames

@@ -7,31 +7,31 @@ import qualified Pipes as P import qualified Pipes.Prelude as P -tableTypes "Row" "../../data/data1.csv"+tableTypes "Row" "data/data1.csv"  tbl :: IO [Row]-tbl = runSafeT . P.toListM $ readTable "../../data/data1.csv"+tbl = runSafeT . P.toListM $ readTable "data/data1.csv"   ageDoubler :: (Age ∈ rs) => Record rs -> Record rs ageDoubler = age %~ (* 2)  tbl2 :: IO [Row]-tbl2 = runSafeT . P.toListM $ readTable "../../data/data2.csv"+tbl2 = runSafeT . P.toListM $ readTable "data/data2.csv"  tbl2a :: IO [ColFun Maybe Row]-tbl2a = runSafeT . P.toListM $ readTableMaybe "../../data/data2.csv"+tbl2a = runSafeT . P.toListM $ readTableMaybe "data/data2.csv"  -- Sample data from http://support.spatialkey.com/spatialkey-sample-csv-data/ -- Note: We have to replace carriage returns (\r) with line feed -- characters (\n) for the text library's line parsing to work.-tableTypes "Ins" "../../data/FL2.csv"+tableTypes "Ins" "data/FL2.csv"  insuranceTbl :: MonadSafe m => P.Producer Ins m ()-insuranceTbl = readTable "../../data/FL2.csv"+insuranceTbl = readTable "data/FL2.csv"  insMaybe :: MonadSafe m => P.Producer (ColFun Maybe Ins) m ()-insMaybe = readTableMaybe "../../data/FL2.csv"+insMaybe = readTableMaybe "data/FL2.csv"  type TinyIns = Record [PolicyID, PointLatitude, PointLongitude] 

@@ -1,4 +1,4 @@-{-# LANGUAGE DataKinds, FlexibleContexts, FlexibleInstances, GADTs,+{-# LANGUAGE CPP, DataKinds, FlexibleContexts, FlexibleInstances, GADTs,              LambdaCase, OverloadedStrings, RankNTypes,              ScopedTypeVariables, TemplateHaskell, TypeApplications,              TypeOperators #-}@@ -13,7 +13,9 @@ import qualified Data.Foldable as F import Data.List (intercalate) import Data.Maybe (isNothing, fromMaybe)+#if __GLASGOW_HASKELL__ < 808 import Data.Monoid ((<>))+#endif import Data.Proxy import qualified Data.Text as T import qualified Data.Text.Encoding as T@@ -364,12 +366,12 @@ -- streaming input that you wish to use to produce streaming output. pipeToCSV :: forall ts m.              (Monad m, ColumnHeaders ts, RecordToList ts,-              RecMapMethod Show ElField ts)+              RecMapMethod ShowCSV ElField ts)           => P.Pipe (Record ts) T.Text m () pipeToCSV = P.yield (T.intercalate "," (map T.pack header)) >> go   where header = columnHeaders (Proxy :: Proxy (Record ts))         go :: P.Pipe (Record ts) T.Text m ()-        go = P.map (T.intercalate "," . map T.pack . showFields)+        go = P.map (T.intercalate "," . showFieldsCSV)  -- | Write a header row with column names followed by a line of text -- for each 'Record' to the given file.

@@ -1,4 +1,4 @@-{-# LANGUAGE DataKinds, KindSignatures, MagicHash,+{-# LANGUAGE CPP, DataKinds, KindSignatures, MagicHash,              ScopedTypeVariables, TemplateHaskell, TypeFamilies,              ViewPatterns #-} -- | Support for representing so-called categorical variables: a@@ -108,7 +108,11 @@           InstanceD Nothing [] (AppT (ConT ''ShowCSV) (ConT nameName))                     [FunD 'showCSV (onVariants showCSVClause)]         iVectorFor =+#if __GLASGOW_HASKELL__ >= 808+          TySynInstD (TySynEqn Nothing (AppT (ConT ''VectorFor) (ConT nameName)) (ConT ''VU.Vector))+#else           TySynInstD ''VectorFor (TySynEqn [ConT nameName] (ConT ''VU.Vector))+#endif         iNFData =           let argName = mkName "x"           in InstanceD Nothing [] (AppT (ConT ''NFData) (ConT nameName))

@@ -10,7 +10,9 @@   CommonColumns, CommonColumnsCat, parsedTypeRep ) where import Data.Maybe (fromMaybe)+#if __GLASGOW_HASKELL__ < 808 import Data.Semigroup (Semigroup((<>)))+#endif import qualified Data.Text as T import Data.Vinyl import Data.Vinyl.CoRec

@@ -29,7 +29,7 @@  instance Eq r => Eq (Frame r) where   Frame l1 r1 == Frame l2 r2 =-    l1 == l2 && and (map (\i -> r1 i == r2 i) [0 .. l1 - 1])+    l1 == l2 && all (\i -> r1 i == r2 i) [0 .. l1 - 1]  -- | The 'Monoid' instance for 'Frame' provides a mechanism for -- vertical concatenation of 'Frame's. That is, @f1 <> f2@ will return

@@ -14,3 +14,4 @@ instance ShowCSV Int where instance ShowCSV Double where instance ShowCSV Text where+  showCSV = id

@@ -1,14 +1,19 @@ {-# LANGUAGE DataKinds, FlexibleContexts, MultiParamTypeClasses,-             OverloadedLabels, OverloadedStrings, TemplateHaskell,-             TypeFamilies, TypeOperators #-}+             OverloadedLabels, OverloadedStrings, TemplateHaskell, +             TypeApplications, TypeFamilies, TypeOperators #-} module Categorical where-import Data.Vinyl.Derived-import Data.Vinyl.XRec (toHKD)-import Frames-import Frames.Categorical (declareCategorical)-import Frames.TH (rowGenCat, RowGen(..), declarePrefixedColumn)-import Pipes (Producer, (>->))-import qualified Pipes.Prelude as P+import           Data.Vinyl.Derived+import           Data.Vinyl.XRec                ( toHKD )+import           Frames+import           Frames.Categorical             ( declareCategorical )+import           Frames.TH                      ( rowGenCat+                                                , RowGen(..)+                                                , declarePrefixedColumn+                                                )+import           Pipes                          ( Producer+                                                , (>->)+                                                )+import qualified Pipes.Prelude                 as P  -- * Automatically inferred categorical data types @@ -19,17 +24,19 @@ -- no more than 8 variants. In our small data file, five distinct -- months appear, so a data type is generated and used. fifthMonthSmall :: IO (Maybe SmallMonth)-fifthMonthSmall = fmap (fmap (rvalf #month)) . runSafeEffect . P.head-                $ load >-> P.drop 4-  where load :: MonadSafe m => Producer Small m ()-        load = readTableOpt smallParser "test/data/catSmall.csv"+fifthMonthSmall =+  fmap (fmap (rvalf #month)) . runSafeEffect . P.head $ load >-> P.drop 4+ where+  load :: MonadSafe m => Producer Small m ()+  load = readTableOpt smallParser "test/data/catSmall.csv"  -- When every month appears, Frames leaves that column's type as 'Text'. fifthMonthLarge :: IO (Maybe Text)-fifthMonthLarge = fmap (fmap (rvalf #month)) . runSafeEffect . P.head-                $ load >-> P.drop 4-  where load :: MonadSafe m => Producer Large m ()-        load = readTableOpt largeParser "test/data/catLarge.csv"+fifthMonthLarge =+  fmap (fmap (rvalf #month)) . runSafeEffect . P.head $ load >-> P.drop 4+ where+  load :: MonadSafe m => Producer Large m ()+  load = readTableOpt largeParser "test/data/catLarge.csv"  -- * Custom categorical type @@ -51,7 +58,8 @@ -- parses. We use 'toHKD' to cut through the noise of the @(Maybe -- :. ElField)@ interpretation we parsed into. fifthMonthCustom :: IO (Maybe MyMonthData)-fifthMonthCustom = fmap (>>= toHKD . rgetf #month) . runSafeEffect . P.head-                 $ load >-> P.drop 4-  where load :: MonadSafe m => Producer (ColFun Maybe MyRow) m ()-        load = readTableMaybe "test/data/catSmall.csv"+fifthMonthCustom =+  fmap (>>= toHKD . rgetf #month) . runSafeEffect . P.head $ load >-> P.drop 4+ where+  load :: MonadSafe m => Producer (ColFun Maybe MyRow) m ()+  load = readTableMaybe "test/data/catSmall.csv"

@@ -5,23 +5,27 @@ {-# LANGUAGE OverloadedLabels #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes        #-}+{-# LANGUAGE TypeApplications  #-} module Issue114 where import           Frames-import qualified Control.Foldl as Foldl+import qualified Control.Foldl                 as Foldl import           Control.Lens-import           Pipes.Prelude (fold)+import           Pipes.Prelude                  ( fold ) import           Pipes-import qualified Pipes.Prelude as P-import qualified Pipes.Core as PC+import qualified Pipes.Prelude                 as P+import qualified Pipes.Core                    as PC -import qualified Data.Vinyl as V-import           Data.Vinyl (Rec(..))+import qualified Data.Vinyl                    as V+import           Data.Vinyl                     ( Rec(..) ) -import qualified Data.Text as T+import qualified Data.Text                     as T import           Data.Attoparsec.Text import           Control.Applicative -import Frames.TH (tableTypesText', rowGen, RowGen(..))+import           Frames.TH                      ( tableTypesText'+                                                , rowGen+                                                , RowGen(..)+                                                )  tableTypesText' (rowGen "test/data/issue114.csv") { rowTypeName = "ProductionData" } @@ -29,13 +33,12 @@  getOilVol :: ProductionData -> Record '[OilVolDouble] getOilVol x = V.Field (f (parseOnly parseDouble (x ^. oilVol))) :& RNil-  where-    f (Left _)  = 0.0 / 0.0-    f (Right y) = y+ where+  f (Left  _) = 0.0 / 0.0+  f (Right y) = y  readOilVol :: MonadSafe m => Producer (Record '[OilVolDouble]) m ()-readOilVol = (readTable "test/data/issue114.csv") >->-             (P.map getOilVol)+readOilVol = (readTable "test/data/issue114.csv") >-> (P.map getOilVol)  oilVolLength :: Foldl.Fold (Record '[OilVolDouble]) Int oilVolLength = Foldl.length@@ -48,16 +51,16 @@  parseDouble :: Parser Double parseDouble =-  do d <- double-     return d-  <|>-  do _ <- string ""-     return 0.0+  do+      d <- double+      return d+    <|> do+          _ <- string ""+          return 0.0  test :: IO () test = do-  (t, l) <- runSafeT $-            Foldl.purely fold oilVolTotalAndLength readOilVol+  (t, l) <- runSafeT $ Foldl.purely fold oilVolTotalAndLength readOilVol   putStrLn $ show l ++ " records totalling " ++ show t  loadTable :: MonadSafe m => PC.Producer ProductionData m ()

@@ -10,9 +10,8 @@  module LatinTest where -import           Data.Vinyl    (Rec) import           Frames-import           Frames.CSV    (pipeTableMaybe, readFileLatin1Ln)+import           Frames.CSV    (readFileLatin1Ln) import           Pipes         (Producer, (>->)) import qualified Pipes.Prelude as P 

@@ -1,11 +1,15 @@-{-# LANGUAGE DataKinds, FlexibleContexts, OverloadedLabels, TemplateHaskell   #-}+{-# LANGUAGE DataKinds, FlexibleContexts, OverloadedLabels, TemplateHaskell, TypeApplications   #-} module NoHeader where-import Control.Arrow ((&&&))-import Data.Vinyl.Derived-import Frames-import Frames.TH (rowGen, RowGen(..))-import Pipes (Producer, (>->))-import qualified Pipes.Prelude as P+import           Control.Arrow                  ( (&&&) )+import           Data.Vinyl.Derived+import           Frames+import           Frames.TH                      ( rowGen+                                                , RowGen(..)+                                                )+import           Pipes                          ( Producer+                                                , (>->)+                                                )+import qualified Pipes.Prelude                 as P  tableTypes' (rowGen "test/data/prestigeNoHeader.csv")             { rowTypeName = "Row"@@ -22,7 +26,8 @@  -- | Extract the @Job@ and @Schooling@ columns of the indicated row. getJobAndSchooling :: Int -> IO (Maybe (Text, Double))-getJobAndSchooling n = fmap (fmap aux) . runSafeEffect . P.head-                     $ loadData >-> P.drop (max n 0)-  where aux :: Row -> (Text, Double)-        aux = rvalf #job &&& rvalf #schooling+getJobAndSchooling n =+  fmap (fmap aux) . runSafeEffect . P.head $ loadData >-> P.drop (max n 0)+ where+  aux :: Row -> (Text, Double)+  aux = rvalf #job &&& rvalf #schooling

@@ -1,6 +1,9 @@-{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell #-}-module Main (main) where-import Frames+{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell, TypeApplications #-}+module Main+  ( main+  )+where+import           Frames  -- These data files have overlapping column definitions. Frames should -- not try to re-define an existing identifier.

@@ -132,10 +132,11 @@          -- The test data isn't formatted quite how we'd do it: text          -- fields aren't quoted, and salaries represented as Doubles          -- do not have decimal points.-         let csvInput' = T.replace "Joe" "\"Joe\""-                       . T.replace "Sarah" "\"Sarah\""-                       . T.replace "\"80,000\"" "80000.0"-                       $ T.pack csvInput+         -- let csvInput' = T.replace "Joe" "\"Joe\""+         --               . T.replace "Sarah" "\"Sarah\""+         --               . T.replace "\"80,000\"" "80000.0"+         --               $ T.pack csvInput+         let csvInput' = T.replace "\"80,000\"" "80000.0" (T.pack csvInput)          it "Produces expected output" $            T.unlines (map T.pack csvOutput) `shouldBe` csvInput'          it "Produces parseable output" $

@@ -1,9 +1,8 @@-{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell #-}+{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell, TypeApplications #-} module UncurryFold where-import qualified Control.Foldl as L-import Data.Vinyl (rcast)-import Data.Vinyl.Curry (runcurryX)-import Frames+import qualified Control.Foldl                 as L+import           Data.Vinyl.Curry               ( runcurryX )+import           Frames  -- Data set from http://vincentarelbundock.github.io/Rdatasets/datasets.html tableTypes "Row" "test/data/prestige.csv"

@@ -1,10 +1,12 @@-{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell #-}+{-# LANGUAGE DataKinds, FlexibleContexts, QuasiQuotes, TemplateHaskell, TypeApplications #-} module UncurryFoldNoHeader where-import qualified Control.Foldl as L-import Data.Vinyl (rcast)-import Data.Vinyl.Curry (runcurryX)-import Frames-import Frames.TH (rowGen, RowGen(..))+import qualified Control.Foldl                 as L+import           Data.Vinyl                     ( rcast )+import           Data.Vinyl.Curry               ( runcurryX )+import           Frames+import           Frames.TH                      ( rowGen+                                                , RowGen(..)+                                                )  -- Data set from http://vincentarelbundock.github.io/Rdatasets/datasets.html tableTypes' (rowGen "test/data/prestigeNoHeader.csv")