dataframe 0.3.3.5 → 0.3.3.6

raw patch · 11 files changed

+372/−279 lines, 11 filesdep ~parallelPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: parallel

API changes (from Hackage documentation)

Files

• CHANGELOG.md +5/−0
• README.md +1/−1
• dataframe.cabal +2/−4
• src/DataFrame.hs +28/−27
• src/DataFrame/Functions.hs +43/−34
• src/DataFrame/Internal/DataFrame.hs +10/−8
• src/DataFrame/Internal/Parsing.hs +3/−1
• src/DataFrame/Operations/Core.hs +150/−164
• src/DataFrame/Operations/Statistics.hs +7/−7
• src/DataFrame/Operations/Typing.hs +49/−32
• tests/Main.hs +74/−1

@@ -1,5 +1,10 @@ # Revision history for dataframe +## 0.3.3.6+* Fix bug where doubles were parsing as ints+* Fix bugs where optionals were left in boxed column (instead of optionals)+* Change syntax for conditional operations so it doesn't clash with regular operations.+ ## 0.3.3.5 * Fix parsing logic for doubles. Entire parsing logic is still a work in progress. * Speed up index selection by using backPermute.

@@ -31,4 +31,4 @@ * Designed for interactivity: expressive syntax, helpful error messages, and sensible defaults. * Works seamlessly in both command-line and notebook environments—great for exploration and scripting alike. -For an installation guide and tutorials checkout the [project documentation](https://dataframe.readthedocs.io/) and for an API reference checkout the [hackage documentation](https://hackage-content.haskell.org/package/dataframe-0.3.3.5/docs/DataFrame.html).+For an installation guide and tutorials checkout the [project documentation](https://dataframe.readthedocs.io/) and for an API reference checkout the [hackage documentation](https://hackage-content.haskell.org/package/dataframe-0.3.3.6/docs/DataFrame.html).

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.3.5+version:            0.3.3.6  synopsis: A fast, safe, and intuitive DataFrame library. @@ -101,13 +101,12 @@                       vector-algorithms ^>= 0.9,                       zstd >= 0.1.2.0 && < 0.2,                       mmap >= 0.5.8 && <= 0.5.9,-                      parallel >= 3.2.2.0+                      parallel >= 3.2.2.0 && < 5     hs-source-dirs:   src     c-sources:        cbits/process_csv.c     include-dirs:     cbits     includes:         process_csv.h     install-includes: process_csv.h-    cc-options:       -O3     default-language: Haskell2010  executable dataframe-benchmark-example@@ -146,7 +145,6 @@                    dataframe ^>= 0.3     default-language: Haskell2010     ghc-options:-      -O2       -threaded       -rtsopts       -with-rtsopts=-N

@@ -53,25 +53,25 @@ -- 1) Load data ghci> df0 <- D.readCsv "data/housing.csv" ghci> D.describeColumns df0----------------------------------------------------------------------------------------------------------------------index |    Column Name     | # Non-null Values | # Null Values | # Partially parsed | # Unique Values |     Type-------|--------------------|-------------------|---------------|--------------------|-----------------|-------------- Int  |        Text        |        Int        |      Int      |        Int         |       Int       |     Text-------|--------------------|-------------------|---------------|--------------------|-----------------|--------------0     | ocean_proximity    | 20640             | 0             | 0                  | 5               | Text-1     | median_house_value | 20640             | 0             | 0                  | 3842            | Double-2     | median_income      | 20640             | 0             | 0                  | 12928           | Double-3     | households         | 20640             | 0             | 0                  | 1815            | Double-4     | population         | 20640             | 0             | 0                  | 3888            | Double-5     | total_bedrooms     | 20640             | 0             | 0                  | 1924            | Maybe Double-6     | total_rooms        | 20640             | 0             | 0                  | 5926            | Double-7     | housing_median_age | 20640             | 0             | 0                  | 52              | Double-8     | latitude           | 20640             | 0             | 0                  | 862             | Double-9     | longitude          | 20640             | 0             | 0                  | 844             | Double+-------------------------------------------------------------------------------------------------------------+    Column Name     | # Non-null Values | # Null Values | # Partially parsed | # Unique Values |     Type+--------------------|-------------------|---------------|--------------------|-----------------|-------------+        Text        |        Int        |      Int      |        Int         |       Int       |     Text+--------------------|-------------------|---------------|--------------------|-----------------|-------------+ ocean_proximity    | 20640             | 0             | 0                  | 5               | Text+ median_house_value | 20640             | 0             | 0                  | 3842            | Double+ median_income      | 20640             | 0             | 0                  | 12928           | Double+ households         | 20640             | 0             | 0                  | 1815            | Double+ population         | 20640             | 0             | 0                  | 3888            | Double+ total_bedrooms     | 20640             | 0             | 0                  | 1924            | Maybe Double+ total_rooms        | 20640             | 0             | 0                  | 5926            | Double+ housing_median_age | 20640             | 0             | 0                  | 52              | Double+ latitude           | 20640             | 0             | 0                  | 862             | Double+ longitude          | 20640             | 0             | 0                  | 844             | Double  -- 2) Project & filter ghci> :exposeColumn df-ghci> df1 = D.filterWhere (ocean_proximity F.== F.lit \"ISLAND\") df0 D.|> D.select [F.name median_house_value, F.name median_income, F.name ocean_proximity]+ghci> df1 = D.filterWhere (ocean_proximity .== \"ISLAND\") df0 D.|> D.select [F.name median_house_value, F.name median_income, F.name ocean_proximity]  -- 3) Add a derived column using the expression DSL --    (col types are explicit via TypeApplications)@@ -84,16 +84,16 @@              [ F.maximum median_house_value \`F.as\` "max_house_value"]              grouped ghci> D.take 5 summary-------------------------------------------index | ocean_proximity | max_house_value-------|-----------------|----------------- Int  |      Text       |     Double-------|-----------------|-----------------0     | <1H OCEAN       | 500001.0-1     | INLAND          | 500001.0-2     | ISLAND          | 450000.0-3     | NEAR BAY        | 500001.0-4     | NEAR OCEAN      | 500001.0+----------------------------------+ ocean_proximity | max_house_value+-----------------|----------------+      Text       |     Double+-----------------|----------------+ <1H OCEAN       | 500001.0+ INLAND          | 500001.0+ ISLAND          | 450000.0+ NEAR BAY        | 500001.0+ NEAR OCEAN      | 500001.0 @  == Simple operations (cheat sheet)@@ -148,6 +148,7 @@ @ (+), (-), (*), (/), abs, log, exp, round (F.eq), (F.gt), (F.geq), (F.lt), (F.leq)+(.==), (.>), (.>=), (.<), (.<=) @  Aggregations (for D.'aggregate'):@@ -182,7 +183,7 @@ payment :: Expr Text  -- You can use them directly:-ghci> D.derive "fare_with_tip" (fare * F.lit 1.2)+ghci> D.derive "fare_with_tip" (fare * 1.2) @  Notes:

@@ -88,48 +88,57 @@ mod :: (Integral a, Columnable a) => Expr a -> Expr a -> Expr a mod = BinaryOp "mod" Prelude.mod -(==) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool-(==) = BinaryOp "eq" (Prelude.==)+(.==) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool+(.==) = BinaryOp "eq" (==)  eq :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool-eq = BinaryOp "eq" (Prelude.==)+eq = BinaryOp "eq" (==) -(<) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(<) = BinaryOp "lt" (Prelude.<)+(.<) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool+(.<) = BinaryOp "lt" (<)  lt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-lt = BinaryOp "lt" (Prelude.<)+lt = BinaryOp "lt" (<) -(>) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(>) = BinaryOp "gt" (Prelude.>)+(.>) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool+(.>) = BinaryOp "gt" (>)  gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-gt = BinaryOp "gt" (Prelude.>)+gt = BinaryOp "gt" (>) -(<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-(<=) = BinaryOp "leq" (Prelude.<=)+(.<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool+(.<=) = BinaryOp "leq" (<=)  leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-leq = BinaryOp "leq" (Prelude.<=)+leq = BinaryOp "leq" (<=) -(>=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-(>=) = BinaryOp "geq" (Prelude.>=)+(.>=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool+(.>=) = BinaryOp "geq" (>=)  geq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-geq = BinaryOp "geq" (Prelude.>=)+geq = BinaryOp "geq" (>=)  and :: Expr Bool -> Expr Bool -> Expr Bool and = BinaryOp "and" (&&) +(.&&) :: Expr Bool -> Expr Bool -> Expr Bool+(.&&) = BinaryOp "and" (&&)+ or :: Expr Bool -> Expr Bool -> Expr Bool or = BinaryOp "or" (||) +(.||) :: Expr Bool -> Expr Bool -> Expr Bool+(.||) = BinaryOp "or" (||)+ not :: Expr Bool -> Expr Bool not = UnaryOp "not" Prelude.not  count :: (Columnable a) => Expr a -> Expr Int count expr = AggFold expr "count" 0 (\acc _ -> acc + 1) +collect :: (Columnable a) => Expr a -> Expr [a]+collect expr = AggFold expr "collect" [] (flip (:))+ mode :: (Columnable a, Eq a) => Expr a -> Expr a mode expr =     AggVector@@ -196,7 +205,7 @@ generateConditions labels conds ps df =     let         newConds =-            [ p DataFrame.Functions.<= q+            [ p .<= q             | p <- ps             , q <- ps             , p /= q@@ -207,8 +216,8 @@         expandedConds =             conds                 ++ newConds-                ++ [p `DataFrame.Functions.and` q | p <- newConds, q <- conds, p /= q]-                ++ [p `DataFrame.Functions.or` q | p <- newConds, q <- conds, p /= q]+                ++ [p .&& q | p <- newConds, q <- conds, p /= q]+                ++ [p .|| q | p <- newConds, q <- conds, p /= q]      in         pickTopNBool df labels (deduplicate df expandedConds) @@ -241,21 +250,21 @@                    | (i, p) <- zip [0 ..] vars                    , (j, q) <- zip [0 ..] vars                    , Prelude.not (isLiteral p && isLiteral q)-                   , i Prelude.> j+                   , i > j                    ]                 ++ [ DataFrame.Functions.min p q                    | (i, p) <- zip [0 ..] vars                    , (j, q) <- zip [0 ..] vars                    , p /= q                    , Prelude.not (isLiteral p && isLiteral q)-                   , i Prelude.> j+                   , i > j                    ]                 ++ [ DataFrame.Functions.max p q                    | (i, p) <- zip [0 ..] vars                    , (j, q) <- zip [0 ..] vars                    , p /= q                    , Prelude.not (isLiteral p && isLiteral q)-                   , i Prelude.> j+                   , i > j                    ]                 ++ [ ifThenElse cond r s                    | cond <- conds@@ -273,7 +282,7 @@                    | (i, p) <- zip [0 ..] vars                    , (j, q) <- zip [0 ..] vars                    , Prelude.not (isLiteral p && isLiteral q)-                   , i Prelude.>= j+                   , i >= j                    ]                 ++ [ p / q                    | (i, p) <- zip [0 ..] vars@@ -307,21 +316,21 @@                | (i, p) <- zip [0 ..] existingPrograms                , (j, q) <- zip [0 ..] existingPrograms                , Prelude.not (isLiteral p && isLiteral q)-               , i Prelude.>= j+               , i >= j                ]             ++ [ DataFrame.Functions.min p q                | (i, p) <- zip [0 ..] existingPrograms                , (j, q) <- zip [0 ..] existingPrograms                , Prelude.not (isLiteral p && isLiteral q)                , p /= q-               , i Prelude.> j+               , i > j                ]             ++ [ DataFrame.Functions.max p q                | (i, p) <- zip [0 ..] existingPrograms                , (j, q) <- zip [0 ..] existingPrograms                , Prelude.not (isLiteral p && isLiteral q)                , p /= q-               , i Prelude.> j+               , i > j                ]             ++ [ ifThenElse cond r s                | cond <- conds@@ -339,7 +348,7 @@                | (i, p) <- zip [0 ..] existingPrograms                , (j, q) <- zip [0 ..] existingPrograms                , Prelude.not (isLiteral p && isLiteral q)-               , i Prelude.>= j+               , i >= j                ]             ++ [ p / q                | p <- existingPrograms@@ -377,7 +386,7 @@  -- | Checks if two programs generate the same outputs given all the same inputs. equivalent :: DataFrame -> Expr Double -> Expr Double -> Bool-equivalent df p1 p2 = case (Prelude.==) <$> interpret df p1 <*> interpret df p2 of+equivalent df p1 p2 = case (==) <$> interpret df p1 <*> interpret df p2 of     Left e -> throw e     Right v -> v @@ -408,7 +417,7 @@ f1FromBinary trues preds =     let (!tp, !fp, !fn) =             VU.foldl' step (0 :: Int, 0 :: Int, 0 :: Int) $-                VU.zip (VU.map (Prelude.> 0) preds) (VU.map (Prelude.> 0) trues)+                VU.zip (VU.map (> 0) preds) (VU.map (> 0) trues)      in f1FromCounts tp fp fn   where     step (!tp, !fp, !fn) (!p, !t) =@@ -423,9 +432,9 @@     let tp' = fromIntegral tp         fp' = fromIntegral fp         fn' = fromIntegral fn-        precision = if tp' + fp' Prelude.== 0 then 0 else tp' / (tp' + fp')-        recall = if tp' + fn' Prelude.== 0 then 0 else tp' / (tp' + fn')-     in if precision + recall Prelude.== 0+        precision = if tp' + fp' == 0 then 0 else tp' / (tp' + fp')+        recall = if tp' + fn' == 0 then 0 else tp' / (tp' + fn')+     in if precision + recall == 0             then Nothing             else Just (2 * precision * recall / (precision + recall)) @@ -453,7 +462,7 @@             []             [] of             Nothing -> Left "No programs found"-            Just p -> Right (ifThenElse (p DataFrame.Functions.> 0) 1 0)+            Just p -> Right (ifThenElse (p .> 0) 1 0)  percentiles :: DataFrame -> [Expr Double] percentiles df =@@ -551,7 +560,7 @@     [Expr Double] ->     Maybe (Expr Double) beamSearch df cfg outputs constants conds programs-    | searchDepth cfg Prelude.== 0 = case ps of+    | searchDepth cfg == 0 = case ps of         [] -> Nothing         (x : _) -> Just x     | otherwise =@@ -672,7 +681,7 @@             Left e -> throw e             Right v -> v      in-        result Prelude.== col+        result == col  -- See Section 2.4 of the Haskell Report https://www.haskell.org/definition/haskell2010.pdf isReservedId :: T.Text -> Bool

@@ -70,19 +70,21 @@     show :: DataFrame -> String     show d =         let+            rows = 20             (r, c) = dataframeDimensions d             d' =                 d-                    { columns = V.map (takeColumn 10) (columns d)-                    , dataframeDimensions = (min 10 r, c)+                    { columns = V.map (takeColumn rows) (columns d)+                    , dataframeDimensions = (min rows r, c)                     }+            truncationInfo =+                "\n"+                    ++ "Showing "+                    ++ show (min rows r)+                    ++ " rows out of "+                    ++ show r          in-            T.unpack (asText d' False)-                ++ "\n"-                ++ "Showing "-                ++ show (min 10 r)-                ++ " rows out of "-                ++ show r+            T.unpack (asText d' False) ++ (if r > rows then truncationInfo else "")  -- | For showing the dataframe as markdown in notebooks. toMarkdownTable :: DataFrame -> T.Text

@@ -13,7 +13,9 @@ import Text.Read (readMaybe)  isNullish :: T.Text -> Bool-isNullish s = s `S.member` S.fromList ["Nothing", "NULL", "", " ", "nan", "null"]+isNullish s =+    s+        `S.member` S.fromList ["Nothing", "NULL", "", " ", "nan", "null", "N/A", "NaN", "NAN"]  readValue :: (HasCallStack, Read a) => T.Text -> a readValue s = case readMaybe (T.unpack s) of

@@ -11,7 +11,6 @@ import qualified Data.List as L import qualified Data.Map as M import qualified Data.Map.Strict as MS-import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Generic as VG@@ -75,21 +74,21 @@  ghci> D.insertVector "numbers" (V.fromList [1..10]) D.empty -----------------index | numbers-------|--------- Int  |   Int-------|---------0     | 1-1     | 2-2     | 3-3     | 4-4     | 5-5     | 6-6     | 7-7     | 8-8     | 9-9     | 10+--------+ numbers+--------+   Int+--------+ 1+ 2+ 3+ 4+ 5+ 6+ 7+ 8+ 9+ 10  @ -}@@ -151,21 +150,21 @@ @ ghci> D.insertColumn "numbers" (D.fromList [1..10]) D.empty -----------------index | numbers-------|--------- Int  |   Int-------|---------0     | 1-1     | 2-2     | 3-3     | 4-4     | 5-5     | 6-6     | 7-7     | 8-8     | 9-9     | 10+--------+ numbers+--------+   Int+--------+ 1+ 2+ 3+ 4+ 5+ 6+ 7+ 8+ 9+ 10  @ -}@@ -204,21 +203,21 @@  ghci> D.cloneColumn "numbers" "others" df --------------------------index | numbers | others-------|---------|-------- Int  |   Int   |  Int-------|---------|--------0     | 1       | 1-1     | 2       | 2-2     | 3       | 3-3     | 4       | 4-4     | 5       | 5-5     | 6       | 6-6     | 7       | 7-7     | 8       | 8-8     | 9       | 9-9     | 10      | 10+-----------------+ numbers | others+---------|-------+   Int   |  Int+---------|-------+ 1       | 1+ 2       | 2+ 3       | 3+ 4       | 4+ 5       | 5+ 6       | 6+ 7       | 7+ 8       | 8+ 9       | 9+ 10      | 10  @ -}@@ -241,21 +240,21 @@  ghci> D.rename "numbers" "others" df ----------------index | others-------|-------- Int  |  Int-------|--------0     | 1-1     | 2-2     | 3-3     | 4-4     | 5-5     | 6-6     | 7-7     | 8-8     | 9-9     | 10+-------+ others+-------+  Int+-------+ 1+ 2+ 3+ 4+ 5+ 6+ 7+ 8+ 9+ 10  @ -}@@ -272,39 +271,39 @@  ghci> df --------------------------index | numbers | others-------|---------|-------- Int  |   Int   |  Int-------|---------|--------0     | 1       | 11-1     | 2       | 12-2     | 3       | 13-3     | 4       | 14-4     | 5       | 15-5     | 6       | 16-6     | 7       | 17-7     | 8       | 18-8     | 9       | 19-9     | 10      | 20+-----------------+ numbers | others+---------|-------+   Int   |  Int+---------|-------+ 1       | 11+ 2       | 12+ 3       | 13+ 4       | 14+ 5       | 15+ 6       | 16+ 7       | 17+ 8       | 18+ 9       | 19+ 10      | 20  ghci> D.renameMany [("numbers", "first_10"), ("others", "next_10")] df ----------------------------index | first_10 | next_10-------|----------|--------- Int  |   Int    |   Int-------|----------|---------0     | 1        | 11-1     | 2        | 12-2     | 3        | 13-3     | 4        | 14-4     | 5        | 15-5     | 6        | 16-6     | 7        | 17-7     | 8        | 18-8     | 9        | 19-9     | 10       | 20+-------------------+ first_10 | next_10+----------|--------+   Int    |   Int+----------|--------+ 1        | 11+ 2        | 12+ 3        | 13+ 4        | 14+ 5        | 15+ 6        | 16+ 7        | 17+ 8        | 18+ 9        | 19+ 10       | 20  @ -}@@ -325,8 +324,6 @@     { nameOfColumn :: !T.Text     , nonNullValues :: !Int     , nullValues :: !Int-    , partiallyParsedValues :: !Int-    , uniqueValues :: !Int     , typeOfColumn :: !T.Text     } @@ -340,13 +337,13 @@  ghci> D.describeColumns df -------------------------------------------------------------------------------------------------------index | Column Name | # Non-null Values | # Null Values | # Partially parsed | # Unique Values | Type-------|-------------|-------------------|---------------|--------------------|-----------------|------ Int  |    Text     |        Int        |      Int      |        Int         |       Int       | Text-------|-------------|-------------------|---------------|--------------------|-----------------|------0     | others      | 10                | 0             | 0                  | 10              | Int-1     | numbers     | 10                | 0             | 0                  | 10              | Int+--------------------------------------------------------+ Column Name | # Non-null Values | # Null Values | Type+-------------|-------------------|---------------|-----+    Text     |        Int        |      Int      | Text+-------------|-------------------|---------------|-----+ others      | 10                | 0             | Int+ numbers     | 10                | 0             | Int  @ -}@@ -356,8 +353,6 @@         & insertColumn "Column Name" (fromList (map nameOfColumn infos))         & insertColumn "# Non-null Values" (fromList (map nonNullValues infos))         & insertColumn "# Null Values" (fromList (map nullValues infos))-        & insertColumn "# Partially parsed" (fromList (map partiallyParsedValues infos))-        & insertColumn "# Unique Values" (fromList (map uniqueValues infos))         & insertColumn "Type" (fromList (map typeOfColumn infos))   where     infos =@@ -369,9 +364,7 @@         let             cname = columnName i             countNulls = nulls col-            countPartial = partiallyParsed col             columnType = T.pack $ show $ typeRep @a-            unique = S.size $ VG.foldr S.insert S.empty c          in             if isNothing cname                 then acc@@ -380,16 +373,12 @@                         (fromMaybe "" cname)                         (columnLength col - countNulls)                         countNulls-                        countPartial-                        unique                         columnType                         : acc     go acc i col@(BoxedColumn (c :: V.Vector a)) =         let             cname = columnName i-            countPartial = partiallyParsed col             columnType = T.pack $ show $ typeRep @a-            unique = S.size $ VG.foldr S.insert S.empty c          in             if isNothing cname                 then acc@@ -398,22 +387,19 @@                         (fromMaybe "" cname)                         (columnLength col)                         0-                        countPartial-                        unique                         columnType                         : acc     go acc i col@(UnboxedColumn c) =         let             cname = columnName i             columnType = T.pack $ columnTypeString col-            unique = S.size $ VG.foldr S.insert S.empty c          in             -- Unboxed columns cannot have nulls since Maybe             -- is not an instance of Unbox a             if isNothing cname                 then acc                 else-                    ColumnInfo (fromMaybe "" cname) (columnLength col) 0 0 unique columnType : acc+                    ColumnInfo (fromMaybe "" cname) (columnLength col) 0 columnType : acc  nulls :: Column -> Int nulls (OptionalColumn xs) = VG.length $ VG.filter isNothing xs@@ -445,21 +431,21 @@  ghci> df --------------------------index | numbers | others-------|---------|-------- Int  |   Int   |  Int-------|---------|--------0     | 1       | 11-1     | 2       | 12-2     | 3       | 13-3     | 4       | 14-4     | 5       | 15-5     | 6       | 16-6     | 7       | 17-7     | 8       | 18-8     | 9       | 19-9     | 10      | 20+-----------------+ numbers | others+---------|-------+   Int   |  Int+---------|-------+ 1       | 11+ 2       | 12+ 3       | 13+ 4       | 14+ 5       | 15+ 6       | 16+ 7       | 17+ 8       | 18+ 9       | 19+ 10      | 20  @ -}@@ -477,20 +463,20 @@ ghci> df  ------------------index |  0  |  1-------|-----|----- Int  | Int | Int-------|-----|-----0     | 1   | 11-1     | 2   | 12-2     | 3   | 13-3     | 4   | 14-4     | 5   | 15-5     | 6   | 16-6     | 7   | 17-7     | 8   | 18-8     | 9   | 19-9     | 10  | 20+  0  |  1+-----|----+ Int | Int+-----|----+ 1   | 11+ 2   | 12+ 3   | 13+ 4   | 14+ 5   | 15+ 6   | 16+ 7   | 17+ 8   | 18+ 9   | 19+ 10  | 20  @ -}@@ -505,14 +491,14 @@  ghci> df -------------------index |  A  |  B-------|-----|----- Int  | Int | Int-------|-----|-----0     | 1   | 11-1     | 2   | 12-2     | 3   | 13+----------+  A  |  B+-----|----+ Int | Int+-----|----+ 1   | 11+ 2   | 12+ 3   | 13  @ -}@@ -597,21 +583,21 @@ @ ghci> D.fold (const id) [1..5] df -------------------index |  0  |  1-------|-----|----- Int  | Int | Int-------|-----|-----0     | 1   | 11-1     | 2   | 12-2     | 3   | 13-3     | 4   | 14-4     | 5   | 15-5     | 6   | 16-6     | 7   | 17-7     | 8   | 18-8     | 9   | 19-9     | 10  | 20+----------+  0  |  1+-----|----+ Int | Int+-----|----+ 1   | 11+ 2   | 12+ 3   | 13+ 4   | 14+ 5   | 15+ 6   | 16+ 7   | 17+ 8   | 18+ 9   | 19+ 10  | 20  @ -}

@@ -41,13 +41,13 @@  ghci> D.frequencies "ocean_proximity" df ------------------------------------------------------------------------------index |   Statistic    | <1H OCEAN | INLAND | ISLAND | NEAR BAY | NEAR OCEAN-------|----------------|-----------|--------|--------|----------|------------ Int  |      Text      |    Any    |  Any   |  Any   |   Any    |    Any-------|----------------|-----------|--------|--------|----------|------------0     | Count          | 9136      | 6551   | 5      | 2290     | 2658-1     | Percentage (%) | 44.26%    | 31.74% | 0.02%  | 11.09%   | 12.88%+---------------------------------------------------------------------+   Statistic    | <1H OCEAN | INLAND | ISLAND | NEAR BAY | NEAR OCEAN+----------------|-----------|--------|--------|----------|-----------+      Text      |    Any    |  Any   |  Any   |   Any    |    Any+----------------|-----------|--------|--------|----------|-----------+ Count          | 9136      | 6551   | 5      | 2290     | 2658+ Percentage (%) | 44.26%    | 31.74% | 0.02%  | 11.09%   | 12.88% @ -} frequencies :: T.Text -> DataFrame -> DataFrame

@@ -37,39 +37,31 @@         Just Refl -> parseFromExamples n dateFormat safeRead (V.map (fromMaybe "") c) parseDefault _ _ _ column = column -emptyToNothing :: T.Text -> Maybe T.Text-emptyToNothing v = if isNullish v then Nothing else Just v--parseTimeOpt :: String -> T.Text -> Maybe Day-parseTimeOpt dateFormat s =-    parseTimeM {- Accept leading/trailing whitespace -}-        True-        defaultTimeLocale-        dateFormat-        (T.unpack s)--unsafeParseTime :: String -> T.Text -> Day-unsafeParseTime dateFormat s =-    parseTimeOrError {- Accept leading/trailing whitespace -}-        True-        defaultTimeLocale-        dateFormat-        (T.unpack s)- parseFromExamples :: Int -> String -> Bool -> V.Vector T.Text -> Column parseFromExamples n dateFormat safeRead c-    | V.all isJust (V.map readInt examples) =-        let safeVector = V.map ((=<<) readInt . emptyToNothing) c-            hasNulls = V.elem Nothing safeVector-         in if safeRead && hasNulls-                then BoxedColumn safeVector-                else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))+    | V.any isJust (V.map readInt examples)+        && equalNonEmpty (V.map readInt examples) (V.map readDouble examples) =+        let safeVector = V.map readIntEither c+            notAllParsed = V.any isLeft safeVector+         in if safeRead && notAllParsed+                then+                    if V.all (either isNullish (const False)) (V.filter isLeft safeVector)+                        then OptionalColumn (V.map (either (const Nothing) Just) safeVector)+                        else BoxedColumn safeVector+                else+                    UnboxedColumn+                        (VU.generate (V.length c) (either undefined id . (safeVector V.!)))     | V.any isJust (V.map readDouble examples) =-        let safeVector = V.map ((=<<) readDouble . emptyToNothing) c-            hasNulls = V.elem Nothing safeVector-         in if safeRead && hasNulls-                then BoxedColumn safeVector-                else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))+        let safeVector = V.map readDoubleEither c+            notAllParsed = V.any isLeft safeVector+         in if safeRead && notAllParsed+                then+                    if V.all (either isNullish (const False)) (V.filter isLeft safeVector)+                        then OptionalColumn (V.map (either (const Nothing) Just) safeVector)+                        else BoxedColumn safeVector+                else+                    UnboxedColumn+                        (VU.generate (V.length c) (either undefined id . (safeVector V.!)))     | V.any isJust (V.map (parseTimeOpt dateFormat) examples) =         let             -- failed parse should be Either, nullish should be Maybe@@ -86,7 +78,7 @@             if safeRead                 then                     if onlyNulls-                        then BoxedColumn (V.map toMaybe safeVector)+                        then OptionalColumn (V.map toMaybe safeVector)                         else                             if V.any isLeft safeVector                                 then BoxedColumn safeVector@@ -97,6 +89,31 @@             safeVector = V.map emptyToNothing c             hasNulls = V.any isNullish c          in-            if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c+            if safeRead && hasNulls then OptionalColumn safeVector else BoxedColumn c   where     examples = V.take n c++emptyToNothing :: T.Text -> Maybe T.Text+emptyToNothing v = if isNullish v then Nothing else Just v++parseTimeOpt :: String -> T.Text -> Maybe Day+parseTimeOpt dateFormat s =+    parseTimeM {- Accept leading/trailing whitespace -}+        True+        defaultTimeLocale+        dateFormat+        (T.unpack s)++unsafeParseTime :: String -> T.Text -> Day+unsafeParseTime dateFormat s =+    parseTimeOrError {- Accept leading/trailing whitespace -}+        True+        defaultTimeLocale+        dateFormat+        (T.unpack s)++countNonEmpty :: V.Vector (Maybe a) -> Int+countNonEmpty xs = V.length (V.filter isJust xs)++equalNonEmpty :: V.Vector (Maybe a) -> V.Vector (Maybe b) -> Bool+equalNonEmpty xs ys = countNonEmpty xs == countNonEmpty ys

@@ -5,6 +5,7 @@  import qualified Data.Text as T import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D import qualified DataFrame.Internal.Column as DI import qualified DataFrame.Operations.Typing as D@@ -66,6 +67,74 @@      in         TestCase (assertEqual "Correctly parses gregorian date" expected actual) +parseInt :: Test+parseInt =+    let+        expected =+            DI.UnboxedColumn+                ( VU.fromList+                    [1 :: Int .. 5]+                )+        actual =+            D.parseDefault+                10+                True+                "%Y-%m-%d"+                (DI.fromVector (V.fromList ["1" :: T.Text, "2", "3", "4", "5"]))+     in+        TestCase (assertEqual "Correctly parses integers" expected actual)++parseMaybeInt :: Test+parseMaybeInt =+    let+        expected =+            DI.OptionalColumn+                ( V.fromList+                    [Just (1 :: Int), Nothing, Just 3, Just 4, Just 5]+                )+        actual =+            D.parseDefault+                10+                True+                "%Y-%m-%d"+                (DI.fromVector (V.fromList ["1" :: T.Text, "N/A", "3", "4", "5"]))+     in+        TestCase (assertEqual "Correctly parses optional integers" expected actual)++parseDouble :: Test+parseDouble =+    let+        expected =+            DI.UnboxedColumn+                ( VU.fromList+                    [1 :: Double, 2, 3, 4, 5]+                )+        actual =+            D.parseDefault+                10+                True+                "%Y-%m-%d"+                (DI.fromVector (V.fromList ["1" :: T.Text, "2.0", "3", "4", "5"]))+     in+        TestCase (assertEqual "Correctly parses doubles" expected actual)++parseMaybeDouble :: Test+parseMaybeDouble =+    let+        expected =+            DI.OptionalColumn+                ( V.fromList+                    [Just 1 :: Maybe Double, Nothing, Just 3, Just 4, Just 5]+                )+        actual =+            D.parseDefault+                10+                True+                "%Y-%m-%d"+                (DI.fromVector (V.fromList ["1" :: T.Text, "N/A", "3.0", "4", "5"]))+     in+        TestCase (assertEqual "Correctly parses optional doubles" expected actual)+ incompleteDataParseEither :: Test incompleteDataParseEither =     let@@ -90,7 +159,7 @@ incompleteDataParseMaybe =     let         expected =-            DI.BoxedColumn+            DI.OptionalColumn                 ( V.fromList                     [Just $ fromGregorian 2020 02 14, Nothing, Just $ fromGregorian 2022 02 14]                 )@@ -109,6 +178,10 @@     [ TestLabel "parseDate" parseDate     , TestLabel "incompleteDataParseMaybe" incompleteDataParseMaybe     , TestLabel "incompleteDataParseEither" incompleteDataParseEither+    , TestLabel "parseInt" parseInt+    , TestLabel "parseMaybeInt" parseMaybeInt+    , TestLabel "parseDouble" parseDouble+    , TestLabel "parseMaybeDouble" parseMaybeDouble     ]  tests :: Test