diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,10 @@ # Revision history for dataframe -## 0.1.0.0 -- 2024-02-24 - -* Initial commit with small example - -## 0.1.0.1 -- 2024-11-24 +## 0.1.0.0 -* Added a few more dataframe operations. Notably sum and filtering logic. -* Slightly better error messaging for wrong types -* Another example dataset + example program. +* Initial release -## 0.1.0.2 -- 2024-11-30 +## 0.1.0.1 -* Changed default string type to bytestring -* Add a single-column groupBy implementation -* Improved CSV IO to use mutable vectors and reduce allocations. -* Multi-column groupBy operation +* Fixed parse failure on nested, escaped quotation. +* Fixed column info when field name isn't found. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -4,17 +4,21 @@ A tool for exploratory data analysis. +## Installing +* Install Haskell (ghc + cabal) via [ghcup](https://www.haskell.org/ghcup/install/) selecting all the default options. +* To install dataframe run `cabal update && cabal install dataframe` +* Open a Haskell repl with dataframe loaded by running `cabal repl --build-depends dataframe`. +* Follow along any one of the tutorials below. + + ## What is exploratory data analysis? -We provide a primer [here](./docs/exploratory_data_analysis_primer.md) and show how to do some common analyses. +We provide a primer [here](https://github.com/mchav/dataframe/blob/main/docs/exploratory_data_analysis_primer.md) and show how to do some common analyses. ## Coming from other dataframe libraries Familiar with another dataframe library? Get started: -* [Coming from Pandas](./docs/coming_from_pandas.md) -* [Coming from Polars](./docs/coming_from_polars.md) -* [Coming from dplyr](./docs/coming_from_dplyr.md) - -## Current status -The library is still in active development with a v0.1 launch planned for March 2025. +* [Coming from Pandas](https://github.com/mchav/dataframe/blob/main/docs/coming_from_pandas.md) +* [Coming from Polars](https://github.com/mchav/dataframe/blob/main/docs/coming_from_polars.md) +* [Coming from dplyr](https://github.com/mchav/dataframe/blob/main/docs/coming_from_dplyr.md) ## Example usage diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 0.1.0.0 +version: 0.1.0.1 synopsis: An intuitive, dynamically-typed DataFrame library. diff --git a/src/Data/DataFrame/IO/CSV.hs b/src/Data/DataFrame/IO/CSV.hs --- a/src/Data/DataFrame/IO/CSV.hs +++ b/src/Data/DataFrame/IO/CSV.hs @@ -30,6 +30,7 @@ import Data.DataFrame.Internal.DataFrame (DataFrame(..)) import Data.DataFrame.Internal.Parsing import Data.DataFrame.Operations.Typing +import Data.Foldable (fold) import Data.Function (on) import Data.IORef import Data.Maybe @@ -184,27 +185,24 @@ "field" {-# INLINE field #-} +unquotedTerminators :: Char -> S.Set Char +unquotedTerminators sep = S.fromList [sep, '\n', '\r', '"'] + unquotedField :: Char -> Parser T.Text unquotedField sep = - takeWhile nonTerminal "unquoted field" - where nonTerminal = (`S.notMember` S.fromList [sep, '\n', '\r', '"']) + takeWhile (not . (`S.member` terminators)) "unquoted field" + where terminators = unquotedTerminators sep {-# INLINE unquotedField #-} -insideQuotes :: Parser T.Text -insideQuotes = - T.append <$> takeWhile (/= '"') - <*> (T.concat <$> many (T.cons <$> dquotes <*> insideQuotes)) - "inside of double quotes" - where - dquotes = - string "\"\"" >> return '"' - "paired double quotes" -{-# INLINE insideQuotes #-} - quotedField :: Parser T.Text -quotedField = - char '"' *> insideQuotes <* char '"' - "quoted field" +quotedField = char '"' *> contents <* char '"' "quoted field" + where + contents = fold <$> many (unquote <|> unescape) + where + unquote = takeWhile1 (notInClass "\"\\") + unescape = char '\\' *> do + T.singleton <$> do + char '\\' <|> char '"' {-# INLINE quotedField #-} lineEnd :: Parser () @@ -226,7 +224,7 @@ Fail unconsumed ctx er -> do erpos <- hTell h fail $ "Failed to parse CSV file around " <> show erpos <> " byte; due: " - <> show er <> "; context: " <> show ctx + <> show er <> "; context: " <> show ctx <> " " <> show unconsumed Partial c -> do fail $ "Partial handler is called; n = " <> show n Done (unconsumed :: T.Text) _ -> diff --git a/src/Data/DataFrame/Operations/Core.hs b/src/Data/DataFrame/Operations/Core.hs --- a/src/Data/DataFrame/Operations/Core.hs +++ b/src/Data/DataFrame/Operations/Core.hs @@ -169,7 +169,7 @@ where infos = L.sortBy (compare `on` nonNullValues) (V.ifoldl' go [] (columns df)) :: [ColumnInfo] indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) - columnName i = indexMap M.! i + columnName i = M.lookup i indexMap go acc i Nothing = acc go acc i (Just col@(OptionalColumn (c :: V.Vector a))) = let cname = columnName i @@ -177,20 +177,20 @@ countPartial = partiallyParsed col columnType = T.pack $ show $ typeRep @a unique = S.size $ VG.foldr S.insert S.empty c - in ColumnInfo cname (columnLength col - countNulls) countNulls countPartial unique columnType : acc + in if cname == Nothing then acc else ColumnInfo (fromMaybe "" cname) (columnLength col - countNulls) countNulls countPartial unique columnType : acc go acc i (Just col@(BoxedColumn (c :: V.Vector a))) = let cname = columnName i countPartial = partiallyParsed col columnType = T.pack $ show $ typeRep @a unique = S.size $ VG.foldr S.insert S.empty c - in ColumnInfo cname (columnLength col) 0 countPartial unique columnType : acc + in if cname == Nothing then acc else ColumnInfo (fromMaybe "" cname) (columnLength col) 0 countPartial unique columnType : acc go acc i (Just col@(UnboxedColumn c)) = let cname = columnName i columnType = T.pack $ columnTypeString col unique = S.size $ VG.foldr S.insert S.empty c -- Unboxed columns cannot have nulls since Maybe -- is not an instance of Unbox a - in ColumnInfo cname (columnLength col) 0 0 unique columnType : acc + in if cname == Nothing then acc else ColumnInfo (fromMaybe "" cname) (columnLength col) 0 0 unique columnType : acc nulls :: Column -> Int nulls (OptionalColumn xs) = VG.length $ VG.filter isNothing xs