dataframe 0.3.3.7 → 0.3.3.8

raw patch · 18 files changed

+996/−370 lines, 18 filesdep +aesondep +cassavadep +regex-tdfaPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: aeson, cassava, regex-tdfa, scientific, unordered-containers

API changes (from Hackage documentation)

Files

• CHANGELOG.md +11/−0
• README.md +1/−1
• dataframe.cabal +7/−1
• src/DataFrame.hs +2/−0
• src/DataFrame/Display/Terminal/PrettyPrint.hs +4/−4
• src/DataFrame/Functions.hs +36/−1
• src/DataFrame/IO/CSV.hs +210/−257
• src/DataFrame/IO/JSON.hs +133/−0
• src/DataFrame/Internal/Column.hs +5/−0
• src/DataFrame/Internal/DataFrame.hs +19/−0
• src/DataFrame/Internal/Parsing.hs +12/−0
• src/DataFrame/Operations/Core.hs +189/−45
• src/DataFrame/Operations/Join.hs +75/−48
• src/DataFrame/Operations/Statistics.hs +59/−3
• src/DataFrame/Operations/Typing.hs +37/−1
• tests/Main.hs +139/−5
• tests/Operations/Join.hs +39/−4
• tests/Operations/Statistics.hs +18/−0

@@ -1,5 +1,16 @@ # Revision history for dataframe +## 0.3.3.8+* More efficient inner joins using hashmaps.+* Initial JSON lines implementation+* More robust logic when specifying CSV types.+* Strip spaces from titles and rows in CSV reading.+* Auto parsing bools in CSV.+* Add `imputeWith`, `bind`, `nRows`, `nColumns`, `recodeWitDefault` function that takes +* Better support for proper markdown+* Fix bug with full outer join.+* Unify `insertVector` and `insertList` functions into insert.+ ## 0.3.3.7 * Many functions how rely on expressions (not strings). * full, left, and right join now implemented.

@@ -16,7 +16,7 @@ <p align="center">   <a href="https://dataframe.readthedocs.io/en/latest/">User guide</a>   |-  <a href="https://discord.gg/XJE5wKT2kb">Discord</a>+  <a href="https://discord.gg/8u8SCWfrNC">Discord</a> </p>  # DataFrame

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.3.7+version:            0.3.3.8  synopsis: A fast, safe, and intuitive DataFrame library. @@ -68,6 +68,7 @@                     DataFrame.Display,                     DataFrame.Display.Terminal.Plot,                     DataFrame.IO.CSV,+                    DataFrame.IO.JSON,                     DataFrame.IO.Unstable.CSV,                     DataFrame.IO.Parquet,                     DataFrame.IO.Parquet.Binary@@ -83,10 +84,12 @@                     DataFrame.Lazy.IO.CSV,                     DataFrame.Lazy.Internal.DataFrame     build-depends:    base >= 4 && <5,+                      aeson >= 0.11.0.0 && < 3,                       array >= 0.5.4.0 && < 0.6,                       attoparsec >= 0.12 && < 0.15,                       bytestring >= 0.11 && < 0.13,                       bytestring-lexing >= 0.5 && < 0.6,+                      cassava >= 0.1 && < 1,                       containers >= 0.6.7 && < 0.9,                       directory >= 1.3.0.0 && < 2,                       granite ^>= 0.3,@@ -94,9 +97,12 @@                       process ^>= 1.6,                       snappy-hs ^>= 0.1,                       random >= 1 && < 2,+                      regex-tdfa >= 1.3.0 && < 2,+                      scientific >=0.3.1 && <0.4,                       template-haskell >= 2.0 && < 3,                       text >= 2.0 && < 3,                       time >= 1.12 && < 2,+                      unordered-containers >= 0.1 && < 1,                       vector ^>= 0.13,                       vector-algorithms ^>= 0.9,                       zstd >= 0.1.2.0 && < 0.2,

@@ -273,6 +273,7 @@     columnAsDoubleVector,     columnAsFloatVector,     columnAsIntVector,+    columnAsList,     columnAsVector,     empty,     null,@@ -313,6 +314,7 @@ import DataFrame.Operations.Statistics as Statistics (     correlation,     frequencies,+    imputeWith,     interQuartileRange,     mean,     median,

@@ -65,10 +65,10 @@         lines =             if properMarkdown                 then-                    border-                        : fillCols colTitleFill consolidatedHeader-                        : separator-                        : map (fillCols colValueFill) rows+                    T.concat ["  ", border, "  "]+                        : T.concat ["| ", fillCols colTitleFill consolidatedHeader, " |"]+                        : T.concat ["| ", separator, " |"]+                        : map ((\t -> T.concat ["| ", t, " |"]) . fillCols colValueFill) rows                 else                     border                         : fillCols colTitleFill consolidatedHeader

@@ -39,9 +39,10 @@ import Data.Function import qualified Data.List as L import qualified Data.Map as M-import Data.Maybe (listToMaybe)+import Data.Maybe (fromMaybe, listToMaybe) import qualified Data.Set as S import qualified Data.Text as T+import Data.Time import Data.Type.Equality import qualified Data.Vector as V import qualified Data.Vector.Generic as VG@@ -51,6 +52,7 @@ import Debug.Trace (trace, traceShow) import Language.Haskell.TH import qualified Language.Haskell.TH.Syntax as TH+import Text.Regex.TDFA import Type.Reflection (typeRep) import Prelude hiding (maximum, minimum, sum) @@ -220,6 +222,39 @@     forall a b.     (Columnable a, Columnable b) => [(a, b)] -> Expr a -> Expr (Maybe b) recode mapping = UnaryOp (T.pack ("recode " ++ show mapping)) (`lookup` mapping)++recodeWithDefault ::+    forall a b.+    (Columnable a, Columnable b) => b -> [(a, b)] -> Expr a -> Expr b+recodeWithDefault d mapping =+    UnaryOp (T.pack ("recode " ++ show mapping)) (fromMaybe d . (`lookup` mapping))++firstOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)+firstOrNothing = lift listToMaybe++lastOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)+lastOrNothing = lift (listToMaybe . reverse)++splitOn :: T.Text -> Expr T.Text -> Expr [T.Text]+splitOn delim = lift (T.splitOn delim)++match :: T.Text -> Expr T.Text -> Expr (Maybe T.Text)+match regex = lift ((\r -> if T.null r then Nothing else Just r) . (=~ regex))++matchAll :: T.Text -> Expr T.Text -> Expr [T.Text]+matchAll regex = lift (getAllTextMatches . (=~ regex))++parseDate :: T.Text -> Expr T.Text -> Expr (Maybe Day)+parseDate format = lift (parseTimeM True defaultTimeLocale (T.unpack format) . T.unpack)++daysBetween :: Expr Day -> Expr Day -> Expr Int+daysBetween d1 d2 = lift fromIntegral (lift2 diffDays d1 d2)++bind ::+    forall a b m.+    (Columnable a, Columnable (m a), Monad m, Columnable b, Columnable (m b)) =>+    (a -> m b) -> Expr (m a) -> Expr (m b)+bind f = lift (>>= f)  generateConditions ::     TypedColumn Double -> [Expr Bool] -> [Expr Double] -> DataFrame -> [Expr Bool]

@@ -1,7 +1,7 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-}+{-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-}-{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-}@@ -10,9 +10,7 @@  module DataFrame.IO.CSV where -import qualified Data.ByteString as BS-import qualified Data.ByteString.Builder as Builder-import qualified Data.ByteString.Char8 as C8+import qualified Data.ByteString.Lazy as BL import qualified Data.List as L import qualified Data.Map.Strict as M import qualified Data.Proxy as P@@ -24,18 +22,20 @@ import qualified Data.Vector.Unboxed as VU import qualified Data.Vector.Unboxed.Mutable as VUM -import Control.Applicative (many, (<|>))-import Control.Monad (forM_, unless, zipWithM, zipWithM_)-import Data.Attoparsec.ByteString.Char8 hiding (endOfLine)-import Data.Bits (shiftL)+import Data.Csv.Streaming (Records (..))+import qualified Data.Csv.Streaming as CsvStream++import Control.Monad import Data.Char+import qualified Data.Csv as Csv import Data.Either import Data.Function (on) import Data.Functor import Data.IORef import Data.Maybe import Data.Type.Equality (TestEquality (testEquality))-import DataFrame.Internal.Column (Column (..), columnLength)+import Data.Word (Word8)+import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing import DataFrame.Internal.Schema@@ -44,45 +44,105 @@ import Type.Reflection import Prelude hiding (concat, takeWhile) -data GrowingVector a = GrowingVector-    { gvData :: !(IORef (VM.IOVector a))-    , gvSize :: !(IORef Int)-    , gvCapacity :: !(IORef Int)+chunkSize :: Int+chunkSize = 16_384++data PagedVector a = PagedVector+    { pvChunks :: !(IORef [V.Vector a])+    -- ^ Finished chunks (reverse order)+    , pvActive :: !(IORef (VM.IOVector a))+    -- ^ Current mutable chunk+    , pvCount :: !(IORef Int)+    -- ^ Items written in current chunk     } -data GrowingUnboxedVector a = GrowingUnboxedVector-    { guvData :: !(IORef (VUM.IOVector a))-    , guvSize :: !(IORef Int)-    , guvCapacity :: !(IORef Int)+data PagedUnboxedVector a = PagedUnboxedVector+    { puvChunks :: !(IORef [VU.Vector a])+    , puvActive :: !(IORef (VUM.IOVector a))+    , puvCount :: !(IORef Int)     } -data GrowingColumn-    = GrowingInt !(GrowingUnboxedVector Int) !(IORef [Int])-    | GrowingDouble !(GrowingUnboxedVector Double) !(IORef [Int])-    | GrowingText !(GrowingVector T.Text) !(IORef [Int])+data BuilderColumn+    = BuilderInt !(PagedUnboxedVector Int) !(PagedUnboxedVector Word8)+    | BuilderDouble !(PagedUnboxedVector Double) !(PagedUnboxedVector Word8)+    | BuilderText !(PagedVector T.Text) !(PagedUnboxedVector Word8) -data HeaderSpec-    = -- | File has no header row-      NoHeader-    | -- | Use first row as column names-      UseFirstRow-    | -- | Supply names for a no-header file-      ProvideNames [T.Text]-    deriving (Eq, Show)+newPagedVector :: IO (PagedVector a)+newPagedVector = do+    active <- VM.unsafeNew chunkSize+    PagedVector <$> newIORef [] <*> newIORef active <*> newIORef 0 -data TypeSpec-    = InferFromSample Int-    | SpecifyTypes [SchemaType]-    | NoInference+newPagedUnboxedVector :: (VUM.Unbox a) => IO (PagedUnboxedVector a)+newPagedUnboxedVector = do+    active <- VUM.unsafeNew chunkSize+    PagedUnboxedVector <$> newIORef [] <*> newIORef active <*> newIORef 0 -shouldInferFromSample :: TypeSpec -> Bool-shouldInferFromSample (InferFromSample _) = True-shouldInferFromSample _ = False+appendPagedVector :: PagedVector a -> a -> IO ()+appendPagedVector (PagedVector chunksRef activeRef countRef) !val = do+    count <- readIORef countRef+    active <- readIORef activeRef -typeInferenceSampleSize :: TypeSpec -> Int-typeInferenceSampleSize (InferFromSample n) = n-typeInferenceSampleSize _ = 0+    if count < chunkSize+        then do+            VM.unsafeWrite active count val+            writeIORef countRef $! count + 1+        else do+            frozen <- V.freeze active+            modifyIORef' chunksRef (frozen :) +            newActive <- VM.unsafeNew chunkSize+            VM.unsafeWrite newActive 0 val++            writeIORef activeRef newActive+            writeIORef countRef 1+{-# INLINE appendPagedVector #-}++appendPagedUnboxedVector :: (VUM.Unbox a) => PagedUnboxedVector a -> a -> IO ()+appendPagedUnboxedVector (PagedUnboxedVector chunksRef activeRef countRef) !val = do+    count <- readIORef countRef+    active <- readIORef activeRef++    if count < chunkSize+        then do+            VUM.unsafeWrite active count val+            writeIORef countRef $! count + 1+        else do+            frozen <- VU.freeze active+            modifyIORef' chunksRef (frozen :)++            newActive <- VUM.unsafeNew chunkSize+            VUM.unsafeWrite newActive 0 val++            writeIORef activeRef newActive+            writeIORef countRef 1+{-# INLINE appendPagedUnboxedVector #-}++freezePagedVector :: PagedVector a -> IO (V.Vector a)+freezePagedVector (PagedVector chunksRef activeRef countRef) = do+    count <- readIORef countRef+    active <- readIORef activeRef+    chunks <- readIORef chunksRef++    lastChunk <- V.freeze (VM.slice 0 count active)++    return $! V.concat (reverse (lastChunk : chunks))++freezePagedUnboxedVector ::+    (VUM.Unbox a) => PagedUnboxedVector a -> IO (VU.Vector a)+freezePagedUnboxedVector (PagedUnboxedVector chunksRef activeRef countRef) = do+    count <- readIORef countRef+    active <- readIORef activeRef+    chunks <- readIORef chunksRef++    lastChunk <- VU.freeze (VUM.slice 0 count active)+    return $! VU.concat (reverse (lastChunk : chunks))++-- | STANDARD CONFIG TYPES+data HeaderSpec = NoHeader | UseFirstRow | ProvideNames [T.Text]+    deriving (Eq, Show)++data TypeSpec = InferFromSample Int | SpecifyTypes [SchemaType] | NoInference+ -- | CSV read parameters. data ReadOptions = ReadOptions     { headerSpec :: HeaderSpec@@ -91,8 +151,6 @@     -- ^ Whether/how to infer types. (default: InferFromSample 100)     , safeRead :: Bool     -- ^ Whether to partially parse values into `Maybe`/`Either`. (default: True)-    , chunkSize :: Int-    -- ^ Default chunk size (in bytes) for csv reader. (default: 512'000)     , dateFormat :: String     {- ^ Format of date fields as recognized by the Data.Time.Format module. @@ -107,78 +165,27 @@     -}     } +shouldInferFromSample :: TypeSpec -> Bool+shouldInferFromSample (InferFromSample _) = True+shouldInferFromSample _ = False++schemaTypes :: TypeSpec -> [SchemaType]+schemaTypes (SpecifyTypes xs) = xs+schemaTypes _ = []++typeInferenceSampleSize :: TypeSpec -> Int+typeInferenceSampleSize (InferFromSample n) = n+typeInferenceSampleSize _ = 0+ defaultReadOptions :: ReadOptions defaultReadOptions =     ReadOptions         { headerSpec = UseFirstRow         , typeSpec = InferFromSample 100         , safeRead = True-        , chunkSize = 512_000         , dateFormat = "%Y-%m-%d"         } -newGrowingVector :: Int -> IO (GrowingVector a)-newGrowingVector !initCap = do-    vec <- VM.unsafeNew initCap-    GrowingVector <$> newIORef vec <*> newIORef 0 <*> newIORef initCap--newGrowingUnboxedVector :: (VUM.Unbox a) => Int -> IO (GrowingUnboxedVector a)-newGrowingUnboxedVector !initCap = do-    vec <- VUM.unsafeNew initCap-    GrowingUnboxedVector <$> newIORef vec <*> newIORef 0 <*> newIORef initCap--appendGrowingVector :: GrowingVector a -> a -> IO ()-appendGrowingVector (GrowingVector vecRef sizeRef capRef) !val = do-    size <- readIORef sizeRef-    cap <- readIORef capRef-    vec <- readIORef vecRef--    vec' <--        if size >= cap-            then do-                let !newCap = cap `shiftL` 1-                newVec <- VM.unsafeGrow vec newCap-                writeIORef vecRef newVec-                writeIORef capRef newCap-                return newVec-            else return vec--    VM.unsafeWrite vec' size val-    writeIORef sizeRef $! size + 1--appendGrowingUnboxedVector ::-    (VUM.Unbox a) => GrowingUnboxedVector a -> a -> IO ()-appendGrowingUnboxedVector (GrowingUnboxedVector vecRef sizeRef capRef) !val = do-    size <- readIORef sizeRef-    cap <- readIORef capRef-    vec <- readIORef vecRef--    vec' <--        if size >= cap-            then do-                let !newCap = cap `shiftL` 1-                newVec <- VUM.unsafeGrow vec newCap-                writeIORef vecRef newVec-                writeIORef capRef newCap-                return newVec-            else return vec--    VUM.unsafeWrite vec' size val-    writeIORef sizeRef $! size + 1--freezeGrowingVector :: GrowingVector a -> IO (V.Vector a)-freezeGrowingVector (GrowingVector vecRef sizeRef _) = do-    vec <- readIORef vecRef-    size <- readIORef sizeRef-    V.freeze (VM.slice 0 size vec)--freezeGrowingUnboxedVector ::-    (VUM.Unbox a) => GrowingUnboxedVector a -> IO (VU.Vector a)-freezeGrowingUnboxedVector (GrowingUnboxedVector vecRef sizeRef _) = do-    vec <- readIORef vecRef-    size <- readIORef sizeRef-    VU.freeze (VUM.slice 0 size vec)- {- | Read CSV file from path and load it into a dataframe.  ==== __Example__@@ -221,35 +228,45 @@ @ -} readSeparated :: Char -> ReadOptions -> FilePath -> IO DataFrame-readSeparated !sep !opts !path = withFile path ReadMode $ \handle -> do-    hSetBuffering handle (BlockBuffering (Just (chunkSize opts)))--    firstLine <- C8.hGetLine handle-    let firstRow = parseLine sep firstLine-        columnNames = case headerSpec opts of-            NoHeader -> map (T.pack . show) [0 .. length firstRow - 1]-            UseFirstRow -> map (stripQuotes . TE.decodeUtf8Lenient) firstRow-            ProvideNames ns -> ns--    unless (headerSpec opts == UseFirstRow) $ hSeek handle AbsoluteSeek 0+readSeparated !sep !opts !path = do+    csvData <- BL.readFile path+    let decodeOpts = Csv.defaultDecodeOptions{Csv.decDelimiter = fromIntegral (ord sep)}+    let stream = CsvStream.decodeWith decodeOpts Csv.NoHeader csvData -    dataLine <- C8.hGetLine handle-    let dataRow = parseLine sep dataLine-    growingCols <- initializeColumns dataRow opts+    let peekStream (Cons (Right row) rest) = return (row, rest)+        peekStream (Cons (Left err) _) = error $ "Error parsing CSV header: " ++ err+        peekStream (Nil Nothing _) = error "Empty CSV file"+        peekStream (Nil (Just err) _) = error err -    processRow 0 dataRow growingCols+    (firstRowRaw, dataStream) <- peekStream stream -    processFile handle sep growingCols (chunkSize opts) 1+    let (columnNames, rowsToProcess) = case headerSpec opts of+            NoHeader ->+                ( map (T.pack . show) [0 .. V.length firstRowRaw - 1]+                , Cons (Right firstRowRaw) dataStream+                )+            UseFirstRow ->+                ( map (T.strip . TE.decodeUtf8Lenient . BL.toStrict) (V.toList firstRowRaw)+                , dataStream+                )+            ProvideNames ns ->+                ( ns ++ drop (length ns) (map (T.pack . show) [0 .. V.length firstRowRaw - 1])+                , Cons (Right firstRowRaw) dataStream+                ) -    frozenCols <- V.fromList <$> mapM freezeGrowingColumn growingCols+    (sampleRow, _) <- peekStream rowsToProcess+    builderCols <- initializeColumns (V.toList sampleRow) opts+    processStream rowsToProcess builderCols +    frozenCols <- V.fromList <$> mapM freezeBuilderColumn builderCols     let numRows = maybe 0 columnLength (frozenCols V.!? 0)-        df =++    let df =             DataFrame-                { columns = frozenCols-                , columnIndices = M.fromList (zip columnNames [0 ..])-                , dataframeDimensions = (numRows, V.length frozenCols)-                }+                frozenCols+                (M.fromList (zip columnNames [0 ..]))+                (numRows, V.length frozenCols)+     return $         if shouldInferFromSample (typeSpec opts)             then@@ -258,159 +275,95 @@                     (safeRead opts)                     (dateFormat opts)                     df-            else df+            else+                if not (null (schemaTypes (typeSpec opts)))+                    then parseWithTypes (schemaTypes (typeSpec opts)) df+                    else df -initializeColumns :: [BS.ByteString] -> ReadOptions -> IO [GrowingColumn]+initializeColumns :: [BL.ByteString] -> ReadOptions -> IO [BuilderColumn] initializeColumns row opts = case typeSpec opts of     NoInference -> zipWithM initColumn row (expandTypes [])     InferFromSample _ -> zipWithM initColumn row (expandTypes [])     SpecifyTypes ts -> zipWithM initColumn row (expandTypes ts)   where     expandTypes xs = xs ++ replicate (length row - length xs) (schemaType @T.Text)-    initColumn :: BS.ByteString -> SchemaType -> IO GrowingColumn-    initColumn bs t = do-        nullsRef <- newIORef []+    initColumn _ t = do+        validityRef <- newPagedUnboxedVector         case t of             SType (_ :: P.Proxy a) -> case testEquality (typeRep @a) (typeRep @Int) of-                Just Refl -> GrowingInt <$> newGrowingUnboxedVector 1_024 <*> pure nullsRef+                Just Refl -> BuilderInt <$> newPagedUnboxedVector <*> pure validityRef                 Nothing -> case testEquality (typeRep @a) (typeRep @Double) of-                    Just Refl -> GrowingDouble <$> newGrowingUnboxedVector 1_024 <*> pure nullsRef-                    Nothing -> GrowingText <$> newGrowingVector 1_024 <*> pure nullsRef--data InferredType = IntType | DoubleType | TextType+                    Just Refl -> BuilderDouble <$> newPagedUnboxedVector <*> pure validityRef+                    Nothing -> BuilderText <$> newPagedVector <*> pure validityRef -inferType :: T.Text -> InferredType-inferType !t-    | T.null t = TextType-    | isJust (readInt t) = IntType-    | isJust (readDouble t) = DoubleType-    | otherwise = TextType+processStream ::+    CsvStream.Records (V.Vector BL.ByteString) -> [BuilderColumn] -> IO ()+processStream (Cons (Right row) rest) cols = processRow row cols >> processStream rest cols+processStream (Cons (Left err) _) _ = error ("CSV Parse Error: " ++ err)+processStream (Nil _ _) _ = return () -processRow :: Int -> [BS.ByteString] -> [GrowingColumn] -> IO ()-processRow !rowIdx !vals !cols = zipWithM_ (processValue rowIdx) vals cols+processRow :: V.Vector BL.ByteString -> [BuilderColumn] -> IO ()+processRow !vals !cols = V.zipWithM_ processValue vals (V.fromList cols)   where-    processValue :: Int -> BS.ByteString -> GrowingColumn -> IO ()-    processValue !idx !bs !col = do-        let !val = (stripQuotes . TE.decodeUtf8Lenient) bs+    processValue !bs !col = do+        let bs' = BL.toStrict bs         case col of-            GrowingInt gv nulls ->-                case readByteStringInt bs of-                    Just !i -> appendGrowingUnboxedVector gv i-                    Nothing -> do-                        appendGrowingUnboxedVector gv 0-                        modifyIORef' nulls (idx :)-            GrowingDouble gv nulls ->-                case readByteStringDouble bs of-                    Just !d -> appendGrowingUnboxedVector gv d-                    Nothing -> do-                        appendGrowingUnboxedVector gv 0.0-                        modifyIORef' nulls (idx :)-            GrowingText gv nulls ->+            BuilderInt gv valid -> case readByteStringInt bs' of+                Just !i -> appendPagedUnboxedVector gv i >> appendPagedUnboxedVector valid 1+                Nothing -> appendPagedUnboxedVector gv 0 >> appendPagedUnboxedVector valid 0+            BuilderDouble gv valid -> case readByteStringDouble bs' of+                Just !d -> appendPagedUnboxedVector gv d >> appendPagedUnboxedVector valid 1+                Nothing -> appendPagedUnboxedVector gv 0.0 >> appendPagedUnboxedVector valid 0+            BuilderText gv valid -> do+                let !val = T.strip (TE.decodeUtf8Lenient bs')                 if isNull val-                    then do-                        appendGrowingVector gv T.empty-                        modifyIORef' nulls (idx :)-                    else appendGrowingVector gv val+                    then appendPagedVector gv T.empty >> appendPagedUnboxedVector valid 0+                    else appendPagedVector gv val >> appendPagedUnboxedVector valid 1  isNull :: T.Text -> Bool isNull t = T.null t || t == "NA" || t == "NULL" || t == "null" -processFile :: Handle -> Char -> [GrowingColumn] -> Int -> Int -> IO ()-processFile !handle !sep !cols !chunk r = do-    let go remain !rowIdx =-            parseWith (C8.hGetNonBlocking handle chunk) (parseRow sep) remain >>= \case-                Fail unconsumed ctx er -> do-                    erpos <- hTell handle-                    fail $-                        "Failed to parse CSV file around "-                            <> show erpos-                            <> " byte; due: "-                            <> show er-                            <> "; context: "-                            <> show ctx-                Partial c -> do-                    fail "Partial handler is called"-                Done (unconsumed :: C8.ByteString) (row :: [C8.ByteString]) -> do-                    processRow rowIdx row cols-                    unless (null row || unconsumed == mempty) $ go unconsumed $! rowIdx + 1-    go "" r--parseLine :: Char -> BS.ByteString -> [BS.ByteString]-parseLine !sep = fromRight [] . parseOnly (record sep)--parseRow :: Char -> Parser [C8.ByteString]-parseRow sep = record sep <* (endOfLine <|> endOfInput) <?> "CSV row"--record :: Char -> Parser [BS.ByteString]-record sep = field sep `sepBy` char sep <?> "CSV record"--field :: Char -> Parser BS.ByteString-field sep = quotedField <|> unquotedField sep <?> "CSV field"--unquotedField :: Char -> Parser BS.ByteString-unquotedField sep = takeWhile (\c -> c /= sep && c /= '\n' && c /= '\r') <?> "unquoted field"--quotedField :: Parser BS.ByteString-quotedField =-    do-        char '"'-        contents <- Builder.toLazyByteString <$> parseQuotedContents-        char '"'-        return $ BS.toStrict contents-        <?> "quoted field"-  where-    parseQuotedContents = mconcat <$> many quotedChar-    quotedChar =-        Builder.byteString <$> takeWhile1 (/= '"')-            <|> ((char '"' *> char '"') Data.Functor.$> Builder.char8 '"')-            <?> "quoted field content"--endOfLine :: Parser ()-endOfLine =-    (void (string "\r\n") <|> void (char '\n') <|> void (char '\r'))-        <?> "line ending"--freezeGrowingColumn :: GrowingColumn -> IO Column-freezeGrowingColumn (GrowingInt gv nullsRef) = do-    vec <- freezeGrowingUnboxedVector gv-    nulls <- readIORef nullsRef-    if null nulls+freezeBuilderColumn :: BuilderColumn -> IO Column+freezeBuilderColumn (BuilderInt gv validRef) = do+    vec <- freezePagedUnboxedVector gv+    valid <- freezePagedUnboxedVector validRef+    if VU.all (== 1) valid         then return $ UnboxedColumn vec-        else do-            let size = VU.length vec-            mvec <- VM.new size-            forM_ [0 .. size - 1] $ \i -> do-                if i `elem` nulls-                    then VM.write mvec i Nothing-                    else VM.write mvec i (Just (vec VU.! i))-            OptionalColumn <$> V.freeze mvec-freezeGrowingColumn (GrowingDouble gv nullsRef) = do-    vec <- freezeGrowingUnboxedVector gv-    nulls <- readIORef nullsRef-    if null nulls+        else constructOptional vec valid+freezeBuilderColumn (BuilderDouble gv validRef) = do+    vec <- freezePagedUnboxedVector gv+    valid <- freezePagedUnboxedVector validRef+    if VU.all (== 1) valid         then return $ UnboxedColumn vec-        else do-            let size = VU.length vec-            mvec <- VM.new size-            forM_ [0 .. size - 1] $ \i -> do-                if i `elem` nulls-                    then VM.write mvec i Nothing-                    else VM.write mvec i (Just (vec VU.! i))-            OptionalColumn <$> V.freeze mvec-freezeGrowingColumn (GrowingText gv nullsRef) = do-    vec <- freezeGrowingVector gv-    nulls <- readIORef nullsRef-    if null nulls+        else constructOptional vec valid+freezeBuilderColumn (BuilderText gv validRef) = do+    vec <- freezePagedVector gv+    valid <- freezePagedUnboxedVector validRef+    if VU.all (== 1) valid         then return $ BoxedColumn vec-        else do-            let size = V.length vec-            mvec <- VM.new size-            forM_ [0 .. size - 1] $ \i -> do-                if i `elem` nulls-                    then VM.write mvec i Nothing-                    else VM.write mvec i (Just (vec V.! i))-            OptionalColumn <$> V.freeze mvec+        else constructOptionalBoxed vec valid +constructOptional ::+    (VU.Unbox a, Columnable a) => VU.Vector a -> VU.Vector Word8 -> IO Column+constructOptional vec valid = do+    let size = VU.length vec+    mvec <- VM.new size+    forM_ [0 .. size - 1] $ \i ->+        if (valid VU.! i) == 0+            then VM.write mvec i Nothing+            else VM.write mvec i (Just (vec VU.! i))+    OptionalColumn <$> V.freeze mvec++constructOptionalBoxed :: V.Vector T.Text -> VU.Vector Word8 -> IO Column+constructOptionalBoxed vec valid = do+    let size = V.length vec+    mvec <- VM.new size+    forM_ [0 .. size - 1] $ \i ->+        if (valid VU.! i) == 0+            then VM.write mvec i Nothing+            else VM.write mvec i (Just (vec V.! i))+    OptionalColumn <$> V.freeze mvec+ writeCsv :: FilePath -> DataFrame -> IO () writeCsv = writeSeparated ',' @@ -427,7 +380,7 @@     TIO.hPutStrLn handle (T.intercalate ", " headers)     forM_ [0 .. (rows - 1)] $ \i -> do         let row = getRowAsText df i-        TIO.hPutStrLn handle (T.intercalate ", " row)+        TIO.hPutStrLn handle (T.intercalate "," row)  getRowAsText :: DataFrame -> Int -> [T.Text] getRowAsText df i = V.ifoldr go [] (columns df)

@@ -0,0 +1,133 @@+{-# LANGUAGE TypeApplications #-}++module DataFrame.IO.JSON (+    readJSON,+    readJSONEither,+) where++import Control.Monad (forM)+import Data.Aeson+import qualified Data.Aeson.Key as K+import qualified Data.Aeson.KeyMap as KM+import qualified Data.ByteString.Lazy as LBS+import Data.Maybe (catMaybes)+import Data.Scientific (toRealFloat)+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Vector as V++import qualified DataFrame.Internal.Column as D+import qualified DataFrame.Internal.DataFrame as D+import qualified DataFrame.Operations.Core as D++readJSONEither :: LBS.ByteString -> Either String D.DataFrame+readJSONEither bs = do+    v <- note "Could not decode JSON" (decode @Value bs)+    rows <- toArrayOfObjects v+    let cols :: [Text]+        cols =+            uniq+                . concatMap (map K.toText . KM.keys)+                . V.toList+                $ rows++    columns <- forM cols $ \c -> do+        let col = buildColumn rows c+        pure (c, col)++    pure $ D.fromNamedColumns columns++readJSON :: FilePath -> IO D.DataFrame+readJSON path = do+    contents <- LBS.readFile path+    case readJSONEither contents of+        Left err -> fail $ "readJSON: " <> err+        Right df -> pure df++toArrayOfObjects :: Value -> Either String (V.Vector Object)+toArrayOfObjects (Array xs)+    | V.null xs = Left "Top-level JSON array is empty"+    | otherwise = traverse asObject xs+toArrayOfObjects _ =+    Left "Top-level JSON value must be a JSON array of objects"++asObject :: Value -> Either String Object+asObject (Object o) = Right o+asObject _ = Left "Expected each element of the array to be an object"++uniq :: (Ord a) => [a] -> [a]+uniq = go mempty+  where+    go _ [] = []+    go seen (x : xs)+        | x `elem` seen = go seen xs+        | otherwise = x : go (x : seen) xs++note :: e -> Maybe a -> Either e a+note e = maybe (Left e) Right++data ColType+    = CTString+    | CTNumber+    | CTBool+    | CTArray+    | CTMixed++buildColumn :: V.Vector Object -> Text -> D.Column+buildColumn rows colName =+    let key = K.fromText colName+        values :: V.Vector (Maybe Value)+        values = V.map (KM.lookup key) rows+        colType = detectColType values+     in case colType of+            CTString ->+                D.fromVector (fmap (fmap asText) values)+            CTNumber ->+                D.fromVector (fmap (fmap asDouble) values)+            CTBool ->+                D.fromVector (fmap (fmap asBool) values)+            CTArray ->+                D.fromVector (fmap (fmap asArray) values)+            CTMixed ->+                D.fromVector values++detectColType :: V.Vector (Maybe Value) -> ColType+detectColType vals =+    case nonMissing of+        [] -> CTMixed+        vs+            | all isString vs -> CTString+            | all isNumber vs -> CTNumber+            | all isBool vs -> CTBool+            | all isArray vs -> CTArray+            | otherwise -> CTMixed+  where+    nonMissing = catMaybes (V.toList vals)++    isString (String _) = True+    isString _ = False++    isNumber (Number _) = True+    isNumber _ = False++    isBool (Bool _) = True+    isBool _ = False++    isArray (Array _) = True+    isArray _ = False++asText :: Value -> Text+asText (String s) = s+asText v = T.pack (show v)++asDouble :: Value -> Double+asDouble (Number s) = toRealFloat @Double s+asDouble v = error $ "asDouble: non-number value: " <> show v++asBool :: Value -> Bool+asBool (Bool b) = b+asBool v = error $ "asBool: non-bool value: " <> show v++asArray :: Value -> V.Vector Value+asArray (Array a) = a+asArray v = error $ "asArray: non-array value: " <> show v

@@ -70,6 +70,11 @@ hasMissing (OptionalColumn column) = True hasMissing _ = False +-- | Checks if a column contains only missing values.+allMissing :: Column -> Bool+allMissing (OptionalColumn column) = VB.length (VB.filter isNothing column) == VB.length column+allMissing _ = False+ -- | Checks if a column contains numeric values. isNumeric :: Column -> Bool isNumeric (UnboxedColumn (vec :: VU.Vector a)) = case sNumeric @a of

@@ -303,3 +303,22 @@     (Columnable a, VU.Unbox a) =>     T.Text -> DataFrame -> Either DataFrameException (VU.Vector a) columnAsUnboxedVector name df = toUnboxedVector @a (unsafeGetColumn name df)++{- | Get a specific column as a list.++You must specify the type via type applications.++==== __Examples__++>>> columnAsList @Int "age" df+[25, 30, 35, ...]++>>> columnAsList @Text "name" df+["Alice", "Bob", "Charlie", ...]++==== __Throws__++* 'error' - if the column type doesn't match the requested type+-}+columnAsList :: forall a. (Columnable a) => T.Text -> DataFrame -> [a]+columnAsList name df = V.toList (columnAsVector name df)

@@ -19,10 +19,22 @@             ["Nothing", "NULL", "", " ", "nan", "null", "N/A", "NaN", "NAN", "NA"]     ) +isTrueish :: T.Text -> Bool+isTrueish t = t `elem` ["True", "true", "TRUE"]++isFalseish :: T.Text -> Bool+isFalseish t = t `elem` ["False", "false", "FALSE"]+ readValue :: (HasCallStack, Read a) => T.Text -> a readValue s = case readMaybe (T.unpack s) of     Nothing -> error $ "Could not read value: " ++ T.unpack s     Just value -> value++readBool :: (HasCallStack) => T.Text -> Maybe Bool+readBool s+    | isTrueish s = Just True+    | isFalseish s = Just False+    | otherwise = Nothing  readInteger :: (HasCallStack) => T.Text -> Maybe Integer readInteger s = case signed decimal (T.strip s) of

@@ -18,6 +18,7 @@  import Control.Exception (throw) import Data.Either+import qualified Data.Foldable as Fold import Data.Function (on, (&)) import Data.Maybe import Data.Type.Equality (TestEquality (..))@@ -42,7 +43,10 @@  ==== __Example__ @-ghci> D.dimensions df+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> df = D.fromNamedColumns [("a", D.fromList [1..100]), ("b", D.fromList [1..100]), ("c", D.fromList [1..100])]+>>> D.dimensions df  (100, 3) @@@ -51,13 +55,44 @@ dimensions = dataframeDimensions {-# INLINE dimensions #-} +{- | O(1) Get number of rows in a dataframe.++==== __Example__+@+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> df = D.fromNamedColumns [("a", D.fromList [1..100]), ("b", D.fromList [1..100]), ("c", D.fromList [1..100])]+>>> D.nRows df+100+@+-}+nRows :: DataFrame -> Int+nRows = fst . dataframeDimensions++{- | O(1) Get number of columns in a dataframe.++==== __Example__+@+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> df = D.fromNamedColumns [("a", D.fromList [1..100]), ("b", D.fromList [1..100]), ("c", D.fromList [1..100])]+>>> D.nColumns df+3+@+-}+nColumns :: DataFrame -> Int+nColumns = snd . dataframeDimensions+ {- | O(k) Get column names of the DataFrame in order of insertion.  ==== __Example__ @-ghci> D.columnNames df+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> df = D.fromNamedColumns [("a", D.fromList [1..100]), ("b", D.fromList [1..100]), ("c", D.fromList [1..100])]+>>> D.columnNames df -["col_a", "col_b", "col_c"]+["a", "b", "c"] @ -} columnNames :: DataFrame -> [T.Text]@@ -71,9 +106,10 @@  ==== __Example__ @-ghci> import qualified Data.Vector as V--ghci> D.insertVector "numbers" (V.fromList [1..10]) D.empty+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> import qualified Data.Vector as V+>>> D.insertVector "numbers" (V.fromList [(1 :: Int)..10]) D.empty  --------  numbers@@ -106,11 +142,79 @@ insertVector name xs = insertColumn name (fromVector xs) {-# INLINE insertVector #-} -{- | /O(k)/ Add a column to the dataframe providing a default.-This constructs a new vector and also may convert it-to an unboxed vector if necessary. Since columns are usually-large the runtime is dominated by the length of the list, k.+{- | Adds a foldable collection to the dataframe. If the collection has less elements than the+dataframe and the dataframe is not empty+the collection is converted to type `Maybe a` filled with `Nothing` to match the size of the dataframe. Similarly,+if the collection has more elements than what's currently in the dataframe, the other columns in the dataframe are+change to `Maybe <Type>` and filled with `Nothing`.++Be careful not to insert infinite collections with this function as that will crash the program.++==== __Example__+@+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> D.insert "numbers" [(1 :: Int)..10] D.empty++--------+ numbers+--------+   Int+--------+ 1+ 2+ 3+ 4+ 5+ 6+ 7+ 8+ 9+ 10++@ -}+insert ::+    forall a t.+    (Columnable a, Foldable t) =>+    -- | Column Name+    T.Text ->+    -- | Sequence to add to dataframe+    t a ->+    -- | DataFrame to add column to+    DataFrame ->+    DataFrame+insert name xs = insertColumn name (fromList (Fold.foldr' (:) [] xs)) -- TODO: Do reflection on container type so we can sometimes avoid the list construction.+{-# INLINE insert #-}++{- | Adds a vector to the dataframe and pads it with a default value if it has less elements than the number of rows.++==== __Example__+@+>>> :set -XOverloadedStrings+>>> import qualified Data.Vector as V+>>> import qualified DataFrame as D+>>> df = D.fromNamedColumns [("x", D.fromList [(1 :: Int)..10])]+>>> D.insertVectorWithDefault 0 "numbers" (V.fromList [(1 :: Int),2,3]) df++-------------+ x  | numbers+----|--------+Int |   Int+----|--------+1   | 1+2   | 2+3   | 3+4   | 0+5   | 0+6   | 0+7   | 0+8   | 0+9   | 0+10  | 0++@+-} insertVectorWithDefault ::     forall a.     (Columnable a) =>@@ -128,6 +232,51 @@         values = xs V.++ V.replicate (rows - V.length xs) defaultValue      in insertColumn name (fromVector values) d +{- | Adds a list to the dataframe and pads it with a default value if it has less elements than the number of rows.++==== __Example__+@+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> df = D.fromNamedColumns [("x", D.fromList [(1 :: Int)..10])]+>>> D.insertWithDefault 0 "numbers" [(1 :: Int),2,3] df++-------------+ x  | numbers+----|--------+Int |   Int+----|--------+1   | 1+2   | 2+3   | 3+4   | 0+5   | 0+6   | 0+7   | 0+8   | 0+9   | 0+10  | 0++@+-}+insertWithDefault ::+    forall a t.+    (Columnable a, Foldable t) =>+    -- | Default Value+    a ->+    -- | Column name+    T.Text ->+    -- | Data to add to column+    t a ->+    -- | DataFrame to add the column to+    DataFrame ->+    DataFrame+insertWithDefault defaultValue name xs d =+    let (rows, _) = dataframeDimensions d+        xs' = Fold.foldr' (:) [] xs+        values = xs' ++ replicate (rows - length xs') defaultValue+     in insertColumn name (fromList values) d+ {- | /O(n)/ Adds an unboxed vector to the dataframe.  Same as insertVector but takes an unboxed vector. If you insert a vector of numbers through insertVector it will either way be converted@@ -149,7 +298,9 @@  ==== __Example__ @-ghci> D.insertColumn "numbers" (D.fromList [1..10]) D.empty+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> D.insertColumn "numbers" (D.fromList [(1 :: Int)..10]) D.empty  --------  numbers@@ -198,11 +349,10 @@  ==== __Example__ @-ghci> import qualified Data.Vector as V--ghci> df = insertVector "numbers" (V.fromList [1..10]) D.empty--ghci> D.cloneColumn "numbers" "others" df+>>> :set -XOverloadedStrings+>>> import qualified Data.Vector as V+>>> df = insertVector "numbers" (V.fromList [1..10]) D.empty+>>> D.cloneColumn "numbers" "others" df  -----------------  numbers | others@@ -235,11 +385,11 @@  ==== __Example__ @-ghci> import qualified Data.Vector as V--ghci> df = insertVector "numbers" (V.fromList [1..10]) D.empty--ghci> D.rename "numbers" "others" df+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> import qualified Data.Vector as V+>>> df = insertVector "numbers" (V.fromList [1..10]) D.empty+>>> D.rename "numbers" "others" df  -------  others@@ -266,11 +416,11 @@  ==== __Example__ @-ghci> import qualified Data.Vector as V--ghci> df = D.insertVector "others" (V.fromList [11..20]) (D.insertVector "numbers" (V.fromList [1..10]) D.empty)--ghci> df+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> import qualified Data.Vector as V+>>> df = D.insertVector "others" (V.fromList [11..20]) (D.insertVector "numbers" (V.fromList [1..10]) D.empty)+>>> df  -----------------  numbers | others@@ -288,7 +438,7 @@  9       | 19  10      | 20 -ghci> D.renameMany [("numbers", "first_10"), ("others", "next_10")] df+>>> D.renameMany [("numbers", "first_10"), ("others", "next_10")] df  -------------------  first_10 | next_10@@ -332,11 +482,9 @@  ==== __Example__ @-ghci> import qualified Data.Vector as V--ghci> df = D.insertVector "others" (V.fromList [11..20]) (D.insertVector "numbers" (V.fromList [1..10]) D.empty)--ghci> D.describeColumns df+>>> import qualified Data.Vector as V+>>> df = D.insertVector "others" (V.fromList [11..20]) (D.insertVector "numbers" (V.fromList [1..10]) D.empty)+>>> D.describeColumns df  --------------------------------------------------------  Column Name | # Non-null Values | # Null Values | Type@@ -428,10 +576,8 @@  ==== __Example__ @-ghci> df = D.fromNamedColumns [("numbers", D.fromList [1..10]), ("others", D.fromList [11..20])]--ghci> df-+>>> df = D.fromNamedColumns [("numbers", D.fromList [1..10]), ("others", D.fromList [11..20])]+>>> df -----------------  numbers | others ---------|-------@@ -459,10 +605,8 @@  ==== __Example__ @-ghci> df = D.fromUnnamedColumns [D.fromList [1..10], D.fromList [11..20]]--ghci> df-+>>> df = D.fromUnnamedColumns [D.fromList [1..10], D.fromList [11..20]]+>>> df -----------------   0  |  1 -----|----@@ -488,9 +632,9 @@  ==== __Example__ @-ghci> df = D.fromRows ["A", "B"] [[D.toAny 1, D.toAny 11], [D.toAny 2, D.toAny 12], [D.toAny 3, D.toAny 13]]+>>> df = D.fromRows ["A", "B"] [[D.toAny 1, D.toAny 11], [D.toAny 2, D.toAny 12], [D.toAny 3, D.toAny 13]] -ghci> df+>>> df  ----------   A  |  B@@ -514,9 +658,9 @@  ==== __Example__ @-ghci> df = D.fromUnnamedColumns [D.fromList [1..10], D.fromList [11..20]]+>>> df = D.fromUnnamedColumns [D.fromList [1..10], D.fromList [11..20]] -ghci> D.valueCounts @Int "0" df+>>> D.valueCounts @Int "0" df  [(1,1),(2,1),(3,1),(4,1),(5,1),(6,1),(7,1),(8,1),(9,1),(10,1)] @@ -583,7 +727,7 @@  ==== __Example__ @-ghci> D.fold (const id) [1..5] df+>>> D.fold (const id) [1..5] df  ----------   0  |  1

@@ -1,17 +1,25 @@ {-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE GADTs #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}  module DataFrame.Operations.Join where +import Control.Applicative (asum)+import qualified Data.HashMap.Strict as HM import qualified Data.Map.Strict as M+import Data.Maybe (fromMaybe) import qualified Data.Text as T+import Data.Type.Equality (TestEquality (..)) import qualified Data.Vector as VB import qualified Data.Vector.Unboxed as VU import DataFrame.Internal.Column as D import DataFrame.Internal.DataFrame as D import DataFrame.Operations.Aggregation as D import DataFrame.Operations.Core as D+import Type.Reflection  -- | Equivalent to SQL join types. data JoinType@@ -55,64 +63,73 @@  @ -}-innerJoin ::-    [T.Text] -> DataFrame -> DataFrame -> DataFrame+innerJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame innerJoin cs right left =     let-        leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left)-        leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left)-        -- key -> [index0, index1]-        leftKeyCountsAndIndices =-            VU.foldr-                (\(i, v) acc -> M.insertWith (++) v [i] acc)-                M.empty-                (VU.indexed leftRowRepresentations)-        -- key -> [index0, index1]-        rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right)-        rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right)-        rightKeyCountsAndIndices =-            VU.foldr-                (\(i, v) acc -> M.insertWith (++) v [i] acc)-                M.empty-                (VU.indexed rightRowRepresentations)-        -- key -> [(left_indexes0, right_indexes1)]-        mergedKeyCountsAndIndices =-            M.foldrWithKey-                ( \k v m ->-                    if k `M.member` rightKeyCountsAndIndices-                        then M.insert k (VU.fromList v, VU.fromList (rightKeyCountsAndIndices M.! k)) m-                        else m-                )-                M.empty-                leftKeyCountsAndIndices-        -- [(ints, ints)]-        leftAndRightIndicies = M.elems mergedKeyCountsAndIndices-        -- [(ints, ints)] (expanded to n * m)-        expandedIndices =-            map-                ( \(l, r) -> (mconcat (replicate (VU.length r) l), mconcat (replicate (VU.length l) r))+        -- Prepare Keys for the Right DataFrame+        rightIndicesToGroup =+            [c | (k, c) <- M.toList (D.columnIndices right), k `elem` cs]++        rightRowRepresentations :: VU.Vector Int+        rightRowRepresentations =+            VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right)++        -- Build the Hash Map: Int -> Vector of Indices+        -- We use ifoldr to efficiently insert (index, key) without intermediate allocations.+        rightKeyMap :: HM.HashMap Int (VU.Vector Int)+        rightKeyMap =+            let accumulator =+                    VU.ifoldr+                        (\i key acc -> HM.insertWith (++) key [i] acc)+                        HM.empty+                        rightRowRepresentations+             in HM.map (VU.fromList . reverse) accumulator++        -- Prepare Keys for Left DataFrame+        leftIndicesToGroup =+            [c | (k, c) <- M.toList (D.columnIndices left), k `elem` cs]++        leftRowRepresentations :: VU.Vector Int+        leftRowRepresentations =+            VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left)++        -- Perform the Join+        (leftIndexChunks, rightIndexChunks) =+            VU.ifoldr+                ( \lIdx key (lAcc, rAcc) ->+                    case HM.lookup key rightKeyMap of+                        Nothing -> (lAcc, rAcc)+                        Just rIndices ->+                            let len = VU.length rIndices+                                -- Replicate the Left Index to match the number of Right matches+                                lChunk = VU.replicate len lIdx+                             in (lChunk : lAcc, rIndices : rAcc)                 )-                leftAndRightIndicies-        expandedLeftIndicies = mconcat (map fst expandedIndices)-        expandedRightIndicies = mconcat (map snd expandedIndices)-        -- df+                ([], [])+                leftRowRepresentations++        -- Flatten chunks+        expandedLeftIndicies = VU.concat leftIndexChunks+        expandedRightIndicies = VU.concat rightIndexChunks++        resultLen = VU.length expandedLeftIndicies++        -- Construct Result DataFrames         expandedLeft =             left                 { columns = VB.map (D.atIndicesStable expandedLeftIndicies) (D.columns left)-                , dataframeDimensions =-                    (VU.length expandedLeftIndicies, snd (D.dataframeDimensions left))+                , dataframeDimensions = (resultLen, snd (D.dataframeDimensions left))                 }-        -- df+         expandedRight =             right                 { columns = VB.map (D.atIndicesStable expandedRightIndicies) (D.columns right)-                , dataframeDimensions =-                    (VU.length expandedRightIndicies, snd (D.dataframeDimensions right))+                , dataframeDimensions = (resultLen, snd (D.dataframeDimensions right))                 }-        -- [string]+         leftColumns = D.columnNames left         rightColumns = D.columnNames right-        initDf = expandedLeft+         insertIfPresent _ Nothing df = df         insertIfPresent name (Just c) df = D.insertColumn name c df      in@@ -127,7 +144,7 @@                         )             )             rightColumns-            initDf+            expandedLeft  {- | Performs a left join on two dataframes using the specified key columns. Returns all rows from the left dataframe, with matching rows from the right dataframe.@@ -346,12 +363,22 @@         D.fold             ( \name df ->                 if name `elem` cs-                    then df+                    then case (D.unsafeGetColumn name expandedRight, D.unsafeGetColumn name expandedLeft) of+                        ( OptionalColumn (left :: VB.Vector (Maybe a))+                            , OptionalColumn (right :: VB.Vector (Maybe b))+                            ) -> case testEquality (typeRep @a) (typeRep @b) of+                                Nothing -> error "Cannot join columns of different types"+                                Just Refl ->+                                    D.insert+                                        name+                                        (VB.map (fromMaybe undefined) (VB.zipWith (\l r -> asum [l, r]) left right))+                                        df+                        _ -> error "Join should have optional keys."                     else                         ( if name `elem` leftColumns                             then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df                             else insertIfPresent name (D.getColumn name expandedRight) df                         )-            )+            ) -- ???             rightColumns             initDf

@@ -35,6 +35,7 @@ import DataFrame.Internal.Types import DataFrame.Operations.Core import DataFrame.Operations.Subset (filterJust)+import DataFrame.Operations.Transformations (impute) import Text.Printf (printf) import Type.Reflection (typeRep) @@ -198,6 +199,60 @@         Left e -> throw e         Right xs -> VG.sum xs +{- | /O(n)/ Impute missing values in a column using a derived scalar.++Given++* an expression @f :: 'Expr' b -> 'Expr' b@ that, when interpreted over a+  non-nullable column, produces the same value in every row (for example a+  mean, median, or other aggregate), and+* a nullable column @'Expr' ('Maybe' b)@++this function:++1. Drops all @Nothing@ values from the target column.+2. Interprets @f@ on the remaining non-null values.+3. Checks that the resulting column contains a single repeated value.+4. Uses that value to impute all @Nothing@s in the original column.++==== __Throws__++* 'DataFrameException' - if the column does not exist, is empty,++==== __Example__+@+>>> :set -XOverloadedStrings+>>> import qualified DataFrame as D+>>> let df =+...       D.fromNamedColumns+...         [ ("age", D.fromList [Just 10, Nothing, Just 20 :: Maybe Int]) ]+>>>+>>> -- Impute missing ages with the mean of the observed ages+>>> D.imputeWith F.mean "age" df+-- age+-- ----+-- 10+-- 15+-- 20+@+-}+imputeWith ::+    forall b.+    (Columnable b) =>+    (Expr b -> Expr b) ->+    Expr (Maybe b) ->+    DataFrame ->+    DataFrame+imputeWith f col@(Col columnName) df = case interpret @b (filterJust columnName df) (f (Col @b columnName)) of+    Left e -> throw e+    Right (TColumn value) -> case headColumn @b value of+        Left e -> throw e+        Right h ->+            if all (== h) (toList @b value)+                then impute col h df+                else error "Impute expression returned more than one value"+imputeWith _ _ df = df+ applyStatistic ::     (VU.Vector Double -> Double) -> T.Text -> DataFrame -> Maybe Double applyStatistic f name df = apply =<< _getColumnAsDouble name (filterJust name df)@@ -259,17 +314,18 @@             quartile3 = flip (VG.!) 3 <$> quantiles             max' = flip (VG.!) 4 <$> quantiles             iqr = (-) <$> quartile3 <*> quartile1+            doubleColumn col = _getColumnAsDouble col (filterJust col df)          in             [ count-            , mean' <$> _getColumnAsDouble name df+            , mean' <$> doubleColumn name             , min'             , quartile1             , median'             , quartile3             , max'-            , sqrt . variance' <$> _getColumnAsDouble name df+            , sqrt . variance' <$> doubleColumn name             , iqr-            , skewness' <$> _getColumnAsDouble name df+            , skewness' <$> doubleColumn name             ]  -- | Round a @Double@ to Specified Precision

@@ -10,11 +10,14 @@ import qualified Data.Vector as V  import Data.Maybe (fromMaybe)+import qualified Data.Proxy as P import Data.Time import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import DataFrame.Internal.Column (Column (..), fromVector) import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing+import DataFrame.Internal.Schema+import Text.Read import Type.Reflection (typeRep)  type DateFormat = String@@ -45,12 +48,22 @@         asMaybeText = V.map converter cols      in         case makeParsingAssumption dateFormat examples of+            BoolAssumption -> handleBoolAssumption asMaybeText             IntAssumption -> handleIntAssumption asMaybeText             DoubleAssumption -> handleDoubleAssumption asMaybeText             TextAssumption -> handleTextAssumption asMaybeText             DateAssumption -> handleDateAssumption dateFormat asMaybeText             NoAssumption -> handleNoAssumption dateFormat asMaybeText +handleBoolAssumption :: V.Vector (Maybe T.Text) -> Column+handleBoolAssumption asMaybeText+    | parsableAsBool =+        maybe (fromVector asMaybeBool) fromVector (sequenceA asMaybeBool)+    | otherwise = maybe (fromVector asMaybeText) fromVector (sequenceA asMaybeText)+  where+    asMaybeBool = V.map (>>= readBool) asMaybeText+    parsableAsBool = vecSameConstructor asMaybeText asMaybeBool+ handleIntAssumption :: V.Vector (Maybe T.Text) -> Column handleIntAssumption asMaybeText     | parsableAsInt =@@ -93,14 +106,17 @@     -- means that the examples consisted only of null values, so we can     -- confidently know that this column must be an OptionalColumn     | V.all (== Nothing) asMaybeText = fromVector asMaybeText+    | parsableAsBool = fromVector asMaybeBool     | parsableAsInt = fromVector asMaybeInt     | parsableAsDouble = fromVector asMaybeDouble     | parsableAsDate = fromVector asMaybeDate     | otherwise = fromVector asMaybeText   where+    asMaybeBool = V.map (>>= readBool) asMaybeText     asMaybeInt = V.map (>>= readInt) asMaybeText     asMaybeDouble = V.map (>>= readDouble) asMaybeText     asMaybeDate = V.map (>>= parseTimeOpt dateFormat) asMaybeText+    parsableAsBool = vecSameConstructor asMaybeText asMaybeBool     parsableAsInt =         vecSameConstructor asMaybeText asMaybeInt             && vecSameConstructor asMaybeText asMaybeDouble@@ -149,6 +165,7 @@     -- After accounting for nulls, parsing for Ints and Doubles results in the     -- same corresponding positions of Justs and Nothings, so we assume     -- that the best way to parse is Int+    | vecSameConstructor asMaybeText asMaybeBool = BoolAssumption     | vecSameConstructor asMaybeText asMaybeInt         && vecSameConstructor asMaybeText asMaybeDouble =         IntAssumption@@ -160,13 +177,32 @@     | vecSameConstructor asMaybeText asMaybeDate = DateAssumption     | otherwise = TextAssumption   where+    asMaybeBool = V.map (>>= readBool) asMaybeText     asMaybeInt = V.map (>>= readInt) asMaybeText     asMaybeDouble = V.map (>>= readDouble) asMaybeText     asMaybeDate = V.map (>>= parseTimeOpt dateFormat) asMaybeText  data ParsingAssumption-    = IntAssumption+    = BoolAssumption+    | IntAssumption     | DoubleAssumption     | DateAssumption     | NoAssumption     | TextAssumption++parseWithTypes :: [SchemaType] -> DataFrame -> DataFrame+parseWithTypes ts df = df{columns = go 0 ts (columns df)}+  where+    go :: Int -> [SchemaType] -> V.Vector Column -> V.Vector Column+    go n [] xs = xs+    go n (t : rest) xs+        | n >= V.length xs = xs+        | otherwise =+            go (n + 1) rest (V.update xs (V.fromList [(n, asType t (xs V.! n))]))+    asType :: SchemaType -> Column -> Column+    asType (SType (_ :: P.Proxy a)) c@(BoxedColumn (col :: V.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> c+        Nothing -> case testEquality (typeRep @T.Text) (typeRep @b) of+            Just Refl -> fromVector (V.map ((readMaybe @a) . T.unpack) col)+            Nothing -> fromVector (V.map ((readMaybe @a) . show) col)+    asType _ c = c

@@ -52,6 +52,21 @@  -- PARSING TESTS ------- 1. SIMPLE CASES+parseBools :: Test+parseBools =+    let afterParse :: [Bool]+        afterParse = [True, True, True] ++ [False, False, False]+        beforeParse :: [T.Text]+        beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"]+        expected = DI.UnboxedColumn $ VU.fromList afterParse+        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools without missing values as UnboxedColumn of Bools"+                expected+                actual+            )+ parseInts :: Test parseInts =     let afterParse :: [Int]@@ -217,6 +232,21 @@             )  --- 2. COMBINATION CASES+parseBoolsAndIntsAsTexts :: Test+parseBoolsAndIntsAsTexts =+    let afterParse :: [T.Text]+        afterParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"]+        beforeParse :: [T.Text]+        beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"]+        expected = DI.BoxedColumn $ V.fromList afterParse+        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses mixture of Bools and Ints as Text"+                expected+                actual+            )+ parseIntsAndDoublesAsDoubles :: Test parseIntsAndDoublesAsDoubles =     let afterParse :: [Double]@@ -733,6 +763,21 @@  -- 3A. PARSING WITH SAFEREAD OFF +parseBoolsWithoutSafeRead :: Test+parseBoolsWithoutSafeRead =+    let afterParse :: [Bool]+        afterParse = replicate 10 True ++ replicate 10 False+        beforeParse :: [T.Text]+        beforeParse = replicate 10 "true" ++ replicate 10 "false"+        expected = DI.UnboxedColumn $ VU.fromList afterParse+        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools without missing values as UnboxedColumn of Bools, when safeRead is off"+                expected+                actual+            )+ parseIntsWithoutSafeRead :: Test parseIntsWithoutSafeRead =     let afterParse :: [Int]@@ -1109,6 +1154,21 @@                 actual             ) +parseBoolsAndEmptyStringsWithoutSafeRead :: Test+parseBoolsAndEmptyStringsWithoutSafeRead =+    let afterParse :: [Maybe Bool]+        afterParse = replicate 10 Nothing ++ replicate 10 (Just True) ++ replicate 10 (Just False)+        beforeParse :: [T.Text]+        beforeParse = replicate 10 "" ++ replicate 10 "true" ++ replicate 10 "false"+        expected = DI.OptionalColumn $ V.fromList afterParse+        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools and empty Strings as OptionalColumn of Bools, when safeRead is off"+                expected+                actual+            )+ parseIntsAndEmptyStringsWithoutSafeRead :: Test parseIntsAndEmptyStringsWithoutSafeRead =     let afterParse :: [Maybe Int]@@ -1529,6 +1589,21 @@                 actual             ) +parseBoolsAndNullishStringsWithoutSafeRead :: Test+parseBoolsAndNullishStringsWithoutSafeRead =+    let afterParse :: [T.Text]+        afterParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False"+        beforeParse :: [T.Text]+        beforeParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False"+        expected = DI.BoxedColumn $ V.fromList afterParse+        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools with nullish values as BoxedColumn of Texts, when safeRead is off"+                expected+                actual+            )+ parseIntsAndNullishStringsWithoutSafeRead :: Test parseIntsAndNullishStringsWithoutSafeRead =     let afterParse :: [T.Text]@@ -2038,6 +2113,21 @@             )  -- 3B. PARSING WITH SAFEREAD ON+parseBoolsAndEmptyStringsWithSafeRead :: Test+parseBoolsAndEmptyStringsWithSafeRead =+    let afterParse :: [Maybe Bool]+        afterParse = replicate 10 Nothing ++ replicate 10 (Just True)+        beforeParse :: [T.Text]+        beforeParse = replicate 10 "" ++ replicate 10 "true"+        expected = DI.OptionalColumn $ V.fromList afterParse+        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools and empty strings as OptionalColumn of Bools, when safeRead is on"+                expected+                actual+            )+ parseIntsAndEmptyStringsWithSafeRead :: Test parseIntsAndEmptyStringsWithSafeRead =     let afterParse :: [Maybe Int]@@ -3064,6 +3154,36 @@             )  -- 4. PARSING SHOULD NOT DEPEND ON THE NUMBER OF EXAMPLES.+parseBoolsWithOneExample :: Test+parseBoolsWithOneExample =+    let afterParse :: [Bool]+        afterParse = False : replicate 50 True+        beforeParse :: [T.Text]+        beforeParse = "false" : replicate 50 "true"+        expected = DI.UnboxedColumn $ VU.fromList afterParse+        actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example"+                expected+                actual+            )++parseBoolsWithManyExamples :: Test+parseBoolsWithManyExamples =+    let afterParse :: [Bool]+        afterParse = False : replicate 50 True+        beforeParse :: [T.Text]+        beforeParse = "false" : replicate 50 "true"+        expected = DI.UnboxedColumn $ VU.fromList afterParse+        actual = D.parseDefault 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+     in TestCase+            ( assertEqual+                "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example"+                expected+                actual+            )+ parseIntsWithOneExample :: Test parseIntsWithOneExample =     let afterParse :: [Int]@@ -4854,21 +4974,27 @@ parseTests :: [Test] parseTests =     [ -- 1. SIMPLE CASES-      TestLabel "parseInts" parseInts+      TestLabel "parseBools" parseBools+    , TestLabel "parseInts" parseInts     , TestLabel "parseDoubles" parseDoubles     , TestLabel "parseDates" parseDates     , TestLabel "parseTexts" parseTexts     , -- 2. COMBINATION CASES-      TestLabel "parseIntsAndDoublesAsDoubles" parseIntsAndDoublesAsDoubles+      TestLabel "parseBoolsAndIntsAsTexts" parseBoolsAndIntsAsTexts+    , TestLabel "parseIntsAndDoublesAsDoubles" parseIntsAndDoublesAsDoubles     , TestLabel "parseIntsAndDatesAsTexts" parseIntsAndDatesAsTexts     , TestLabel "parseTextsAndDoublesAsTexts" parseTextsAndDoublesAsTexts     , TestLabel "parseDatesAndTextsAsTexts" parseDatesAndTextsAsTexts     , -- 3A. PARSING WITH SAFEREAD OFF-      TestLabel "parseIntsWithoutSafeRead" parseIntsWithoutSafeRead+      TestLabel "parseBoolsWithoutSafeRead" parseBoolsWithoutSafeRead+    , TestLabel "parseIntsWithoutSafeRead" parseIntsWithoutSafeRead     , TestLabel "parseDoublesWithoutSafeRead" parseDoublesWithoutSafeRead     , TestLabel "parseDatesWithoutSafeRead" parseDatesWithoutSafeRead     , TestLabel "parseTextsWithoutSafeRead" parseTextsWithoutSafeRead     , TestLabel+        "parseBoolsAndEmptyStringsWithoutSafeRead"+        parseBoolsAndEmptyStringsWithoutSafeRead+    , TestLabel         "parseIntsAndEmptyStringsWithoutSafeRead"         parseIntsAndEmptyStringsWithoutSafeRead     , TestLabel@@ -4881,6 +5007,9 @@         "parseTextsAndEmptyStringsWithoutSafeRead"         parseTextsAndEmptyStringsWithoutSafeRead     , TestLabel+        "parseBoolsAndNullishStringsWithoutSafeRead"+        parseBoolsAndNullishStringsWithoutSafeRead+    , TestLabel         "parseIntsAndNullishStringsWithoutSafeRead"         parseIntsAndNullishStringsWithoutSafeRead     , TestLabel@@ -4894,6 +5023,9 @@         parseTextsAndEmptyAndNullishStringsWithoutSafeRead     , -- 3B. PARSING WITH SAFEREAD ON       TestLabel+        "parseBoolsAndEmptyStringsWithSafeRead"+        parseBoolsAndEmptyStringsWithSafeRead+    , TestLabel         "parseIntsAndEmptyStringsWithSafeRead"         parseIntsAndEmptyStringsWithSafeRead     , TestLabel@@ -4920,8 +5052,10 @@     , TestLabel         "parseTextsAndEmptyAndNullishStringsWithSafeRead"         parseTextsAndEmptyAndNullishStringsWithSafeRead-    , -- 4A. PARSING MUST NOT DEPEND ON THE NUMBER OF EXAMPLES-      TestLabel "parseIntsWithOneExample" parseIntsWithOneExample+    , -- 4. PARSING MUST NOT DEPEND ON THE NUMBER OF EXAMPLES+      TestLabel "parseBoolsWithOneExample" parseBoolsWithOneExample+    , TestLabel "parseBoolsWithManyExamples" parseBoolsWithManyExamples+    , TestLabel "parseIntsWithOneExample" parseIntsWithOneExample     , TestLabel "parseIntsWithTwentyFiveExamples" parseIntsWithTwentyFiveExamples     , TestLabel "parseIntsWithFortyNineExamples" parseIntsWithFortyNineExamples     , TestLabel "parseDatesWithOneExample" parseDatesWithOneExample

@@ -1,4 +1,5 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}  module Operations.Join where @@ -63,18 +64,51 @@             (D.sortBy D.Ascending ["key"] (rightJoin ["key"] df2 df1))         ) +staffDf :: D.DataFrame+staffDf =+    D.fromRows+        ["Name", "Role"]+        [ [D.toAny @Text "Kelly", D.toAny @Text "Director of HR"]+        , [D.toAny @Text "Sally", D.toAny @Text "Course liasion"]+        , [D.toAny @Text "James", D.toAny @Text "Grader"]+        ]++studentDf :: D.DataFrame+studentDf =+    D.fromRows+        ["Name", "School"]+        [ [D.toAny @Text "James", D.toAny @Text "Business"]+        , [D.toAny @Text "Mike", D.toAny @Text "Law"]+        , [D.toAny @Text "Sally", D.toAny @Text "Engineering"]+        ]+ testFullOuterJoin :: Test testFullOuterJoin =     TestCase         ( assertEqual             "Test full outer join with single key"             ( D.fromNamedColumns-                [ ("key", D.fromList ["K0" :: Text, "K1", "K2", "K3", "K4", "K5"])-                , ("A", D.fromList ["A0" :: Text, "A1", "A2", "A3", "A4", "A5"])-                , ("B", D.fromList [Just "B0", Just "B1" :: Maybe Text, Just "B2"])+                [+                    ( "Name"+                    , D.fromList ["James" :: Text, "Kelly", "Mike", "Sally"]+                    )+                ,+                    ( "Role"+                    , D.fromList+                        [ Just "Grader" :: Maybe Text+                        , Just "Director of HR"+                        , Nothing+                        , Just "Course liasion"+                        ]+                    )+                ,+                    ( "School"+                    , D.fromList+                        [Just "Business" :: Maybe Text, Nothing, Just "Law", Just "Engineering"]+                    )                 ]             )-            (D.sortBy D.Ascending ["key"] (fullOuterJoin ["key"] df2 df1))+            (D.sortBy D.Ascending ["Name"] (fullOuterJoin ["Name"] studentDf staffDf))         )  tests :: [Test]@@ -82,4 +116,5 @@     [ TestLabel "innerJoin" testInnerJoin     , TestLabel "leftJoin" testLeftJoin     , TestLabel "rightJoin" testRightJoin+    , TestLabel "fullOuterJoin" testFullOuterJoin     ]

@@ -180,6 +180,23 @@             (print $ D.quantiles' (VU.fromList [5]) 4 (VU.fromList [1 :: Int, 2, 3, 4, 5]))         ) +summarizeOptional :: Test+summarizeOptional =+    TestCase+        ( assertEqual+            "Summarizes `Num a => Maybe a` column"+            3 -- The three columns should be Statistics, A, and B+            ( D.nColumns+                ( D.summarize+                    ( D.fromNamedColumns+                        [ ("A", D.fromList [1 :: Int, 2])+                        , ("B", D.fromList [Just (1 :: Int), Nothing])+                        ]+                    )+                )+            )+        )+ tests :: [Test] tests =     [ TestLabel "medianOfOddLengthDataSet" medianOfOddLengthDataSet@@ -202,4 +219,5 @@         interQuartileRangeOfEvenLengthDataSet     , TestLabel "wrongQuantileNumber" wrongQuantileNumber     , TestLabel "wrongQuantileIndex" wrongQuantileIndex+    , TestLabel "summarizeOptional" summarizeOptional     ]