packages feed

hw-dsv 0.3.6 → 0.3.7

raw patch · 7 files changed

+303/−186 lines, 7 files

Files

app/App/Commands/QueryLazy.hs view
@@ -20,15 +20,17 @@ import Options.Applicative          hiding (columns) import Text.Read                    (readEither) -import qualified App.Commands.Options.Type         as Z-import qualified App.IO                            as IO-import qualified Data.ByteString                   as BS-import qualified Data.ByteString.Builder           as B-import qualified Data.ByteString.Lazy              as LBS-import qualified Data.Vector                       as DV-import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL-import qualified System.Exit                       as IO-import qualified System.IO                         as IO+import qualified App.Commands.Options.Type                as Z+import qualified App.IO                                   as IO+import qualified Data.ByteString                          as BS+import qualified Data.ByteString.Builder                  as B+import qualified Data.ByteString.Lazy                     as LBS+import qualified Data.Vector                              as DV+import qualified HaskellWorks.Data.Dsv.Lazy.Cursor        as SVL+import qualified HaskellWorks.Data.Dsv.Lazy.Cursor.Lazy   as SVLL+import qualified HaskellWorks.Data.Dsv.Lazy.Cursor.Strict as SVLS+import qualified System.Exit                              as IO+import qualified System.IO                                as IO  defaultMethod :: String defaultMethod = "lazy-traverse"@@ -47,7 +49,7 @@   !bs <- IO.readInputFile (opts ^. the @"filePath")    let !c = SVL.makeCursor (opts ^. the @"delimiter") bs-  let !rows = SVL.toListVector c+  let !rows = SVLL.toListVector c   let !outDelimiterBuilder = B.word8 (opts ^. the @"outDelimiter")    runResourceT $ do@@ -69,7 +71,7 @@   !bs <- IO.readInputFile (opts ^. the @"filePath")    let !c = SVL.makeCursor (opts ^. the @"delimiter") bs-  let !rows = SVL.toListVectorStrict c+  let !rows = SVLS.toListVector c   let !outDelimiterBuilder = B.word8 (opts ^. the @"outDelimiter")    runResourceT $ do@@ -92,7 +94,7 @@    let !c = SVL.makeCursor (opts ^. the @"delimiter") bs   let !sel = opts ^. the @"columns"-  let !rows = SVL.selectListVector sel c+  let !rows = SVLL.selectListVector sel c   let !outDelimiterBuilder = B.word8 (opts ^. the @"outDelimiter")    runResourceT $ do
hw-dsv.cabal view
@@ -1,7 +1,7 @@ cabal-version: 2.2  name:                   hw-dsv-version:                0.3.6+version:                0.3.7 synopsis:               Unbelievably fast streaming DSV file parser description:            Please see the README on Github at <https://github.com/haskell-works/hw-dsv#readme> category:               Text, CSV, SIMD, Succinct Data Structures, Data Structures@@ -106,6 +106,8 @@                         HaskellWorks.Data.Dsv.Internal.Vector                         HaskellWorks.Data.Dsv.Lazy.Cursor                         HaskellWorks.Data.Dsv.Lazy.Cursor.Internal+                        HaskellWorks.Data.Dsv.Lazy.Cursor.Lazy+                        HaskellWorks.Data.Dsv.Lazy.Cursor.Strict                         HaskellWorks.Data.Dsv.Lazy.Cursor.Type                         HaskellWorks.Data.Dsv.Strict.Cursor                         HaskellWorks.Data.Dsv.Strict.Cursor.Internal
src/HaskellWorks/Data/Dsv/Internal/Vector.hs view
@@ -2,7 +2,13 @@ {-# LANGUAGE RankNTypes          #-} {-# LANGUAGE ScopedTypeVariables #-} -module HaskellWorks.Data.Dsv.Internal.Vector where+module HaskellWorks.Data.Dsv.Internal.Vector+  ( empty64+  , constructNS+  , ltWord+  , indexCsvChunk+  , oddsMask+  ) where  import Control.Monad.ST import Data.Bits.Pdep@@ -17,6 +23,10 @@  import qualified Data.Vector.Storable         as DVS import qualified Data.Vector.Storable.Mutable as DVSM++empty64 :: DVS.Vector Word64+empty64 = DVS.replicate 64 0+{-# NOINLINE empty64 #-}  constructNS :: forall a s. Storable a => Int -> s -> (s -> DVS.Vector a -> (s, a)) -> (s, DVS.Vector a) constructNS n s f = DVS.createT (go 0 s)
src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs view
@@ -1,47 +1,46 @@ module HaskellWorks.Data.Dsv.Lazy.Cursor   ( DsvCursor (..)   , makeCursor-  , snippet   , trim   , atEnd   , nextField   , advanceField   , nextRow   , nextPosition+  -- Functions returning lazy bytestrings+  , snippet   , toListVector   , toVectorVector   , selectListVector+  -- Functions returning strict bytestrings   , getRowBetweenStrict   , toListVectorStrict   ) where  import Data.Function import Data.Word-import GHC.Word                                  (Word8)+import GHC.Word                                   (Word8)+import HaskellWorks.Data.Dsv.Lazy.Cursor.Internal import HaskellWorks.Data.Dsv.Lazy.Cursor.Type-import HaskellWorks.Data.Positioning-import HaskellWorks.Data.RankSelect.Base.Rank1-import HaskellWorks.Data.RankSelect.Base.Select1 import HaskellWorks.Data.Vector.AsVector64s import Prelude -import qualified Data.ByteString                       as BS-import qualified Data.ByteString.Lazy                  as LBS-import qualified Data.Vector                           as DV-import qualified Data.Vector.Storable                  as DVS-import qualified HaskellWorks.Data.Dsv.Internal.Char   as C-import qualified HaskellWorks.Data.Dsv.Internal.Vector as DVS-import qualified HaskellWorks.Data.Simd.Comparison     as DVS--empty64 :: DVS.Vector Word64-empty64 = DVS.replicate 64 0+import qualified Data.ByteString                          as BS+import qualified Data.ByteString.Lazy                     as LBS+import qualified Data.Vector                              as DV+import qualified Data.Vector.Storable                     as DVS+import qualified HaskellWorks.Data.Dsv.Internal.Char      as C+import qualified HaskellWorks.Data.Dsv.Internal.Vector    as DVS+import qualified HaskellWorks.Data.Dsv.Lazy.Cursor.Lazy   as LCL+import qualified HaskellWorks.Data.Dsv.Lazy.Cursor.Strict as LCS+import qualified HaskellWorks.Data.Simd.Comparison        as DVS  makeIndexes :: [DVS.Vector Word64] -> [DVS.Vector Word64] -> [DVS.Vector Word64] -> ([DVS.Vector Word64], [DVS.Vector Word64]) makeIndexes ds ns qs = unzip $ go 0 0 ds ns qs   where go pc carry (dv:dvs) (nv:nvs) (qv:qvs) =           let (dv', nv', pc', carry') = DVS.indexCsvChunk pc carry dv nv qv in           (dv', nv'):go pc' carry' dvs nvs qvs-        go _ _ [] [] [] = [(empty64, empty64)]+        go _ _ [] [] [] = [(DVS.empty64, DVS.empty64)]         go _ _ _ _ _ = error "Unbalanced inputs"  makeCursor :: Word8 -> LBS.ByteString -> DsvCursor@@ -59,139 +58,31 @@ {-# INLINE makeCursor #-}  snippet :: DsvCursor -> LBS.ByteString-snippet c = LBS.take (len `max` 0) $ LBS.drop posC $ dsvCursorText c-  where d = nextField c-        posC = fromIntegral $ dsvCursorPosition c-        posD = fromIntegral $ dsvCursorPosition d-        len  = posD - posC+snippet = LCL.snippet+{-# DEPRECATED snippet "Use HaskellWorks.Data.Dsv.Lazy.Cursor.Strict.snippet instead" #-} {-# INLINE snippet #-} -trim :: DsvCursor -> DsvCursor-trim c = if dsvCursorPosition c >= 512-  then trim c-    { dsvCursorText     = LBS.drop 512 (dsvCursorText c)-    , dsvCursorMarkers  = drop 1 (dsvCursorMarkers c)-    , dsvCursorNewlines = drop 1 (dsvCursorNewlines c)-    , dsvCursorPosition = dsvCursorPosition c - 512-    }-  else c-{-# INLINE trim #-}--atEnd :: DsvCursor -> Bool-atEnd c = LBS.null (LBS.drop (fromIntegral (dsvCursorPosition c)) (dsvCursorText c))-{-# INLINE atEnd #-}--nextField :: DsvCursor -> DsvCursor-nextField cursor = cursor-  { dsvCursorPosition = newPos-  }-  where currentRank = rank1   (dsvCursorMarkers cursor) (dsvCursorPosition cursor)-        newPos      = select1 (dsvCursorMarkers cursor) (currentRank + 1) - 1-{-# INLINE nextField #-}--advanceField :: Count -> DsvCursor -> DsvCursor-advanceField n cursor = cursor-  { dsvCursorPosition = newPos-  }-  where currentRank = rank1   (dsvCursorMarkers cursor) (dsvCursorPosition cursor)-        newPos      = select1 (dsvCursorMarkers cursor) (currentRank + n) - 1-{-# INLINE advanceField #-}--nextRow :: DsvCursor -> DsvCursor-nextRow cursor = cursor-  { dsvCursorPosition = if newPos > dsvCursorPosition cursor-                          then newPos-                          else fromIntegral (LBS.length (dsvCursorText cursor))--  }-  where currentRank = rank1   (dsvCursorNewlines cursor) (dsvCursorPosition cursor)-        newPos      = select1 (dsvCursorNewlines cursor) (currentRank + 1) - 1-{-# INLINE nextRow #-}--nextPosition :: DsvCursor -> DsvCursor-nextPosition cursor = cursor-    { dsvCursorPosition = if LBS.null (LBS.drop (fromIntegral newPos) (dsvCursorText cursor))-                            then fromIntegral (LBS.length (dsvCursorText cursor))-                            else newPos-    }-  where newPos  = dsvCursorPosition cursor + 1-{-# INLINE nextPosition #-}--getRowBetween :: DsvCursor -> DsvCursor -> Bool -> DV.Vector LBS.ByteString-getRowBetween c d dEnd = DV.unfoldrN fields go c-  where cr  = rank1 (dsvCursorMarkers c) (dsvCursorPosition c)-        dr  = rank1 (dsvCursorMarkers d) (dsvCursorPosition d)-        c2d = fromIntegral (dr - cr)-        fields = if dEnd then c2d +1 else c2d-        go :: DsvCursor -> Maybe (LBS.ByteString, DsvCursor)-        go e = case nextField e of-          f -> case nextPosition f of-            g -> case snippet e of-              s -> Just (s, g)-        {-# INLINE go #-}-{-# INLINE getRowBetween #-}- toListVector :: DsvCursor -> [DV.Vector LBS.ByteString]-toListVector c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)-  then getRowBetween c d dEnd:toListVector (trim d)-  else []-  where nr = nextRow c-        d = nextPosition nr-        dEnd = atEnd nr+toListVector = LCL.toListVector+{-# DEPRECATED toListVector "Use HaskellWorks.Data.Dsv.Lazy.Cursor.toListVector instead" #-} {-# INLINE toListVector #-}  toVectorVector :: DsvCursor -> DV.Vector (DV.Vector LBS.ByteString)-toVectorVector = DV.fromList . toListVector+toVectorVector = LCL.toVectorVector+{-# DEPRECATED toVectorVector "Use HaskellWorks.Data.Dsv.Lazy.Cursor.toVectorVector instead" #-} {-# INLINE toVectorVector #-} -selectRowFrom :: [Int] -> DsvCursor -> [LBS.ByteString]-selectRowFrom sel c = go <$> sel-  where go :: Int -> LBS.ByteString-        go n = snippet nc-          where nc = nextPosition (advanceField (fromIntegral n) c)-        {-# INLINE go #-}-{-# INLINE selectRowFrom #-}- selectListVector :: [Int] -> DsvCursor -> [[LBS.ByteString]]-selectListVector sel c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)-  then selectRowFrom sel c:selectListVector sel (trim d)-  else []-  where nr = nextRow c-        d = nextPosition nr+selectListVector = LCL.selectListVector+{-# DEPRECATED selectListVector "Use HaskellWorks.Data.Dsv.Lazy.Cursor.selectListVector instead" #-} {-# INLINE selectListVector #-} -getRowBetweenStrict :: DsvCursor -> DsvCursor -> Bool -> DV.Vector BS.ByteString-getRowBetweenStrict c d dEnd = DV.unfoldrN fields go c-  where bsA = fromIntegral $ dsvCursorPosition c-        bsZ = fromIntegral $ dsvCursorPosition d-        bsT = dsvCursorText c-        bs  = LBS.toStrict $ LBS.take (bsZ - bsA) (LBS.drop bsA bsT)--        cr  = rank1 (dsvCursorMarkers c) (dsvCursorPosition c)-        dr  = rank1 (dsvCursorMarkers d) (dsvCursorPosition d)-        c2d = fromIntegral (dr - cr)-        fields = if dEnd then c2d +1 else c2d-        go :: DsvCursor -> Maybe (BS.ByteString, DsvCursor)-        go e = case nextField e of-          f -> case nextPosition f of-            g -> case snippetStrict e (fromIntegral bsA) bs of-              s -> Just (s, g)-        {-# INLINE go #-}-{-# INLINE getRowBetweenStrict #-}--snippetStrict :: DsvCursor -> Int -> BS.ByteString -> BS.ByteString-snippetStrict c offset bs = BS.take (len `max` 0) $ BS.drop posC $ bs-  where d = nextField c-        posC = fromIntegral (dsvCursorPosition c) - offset-        posD = fromIntegral (dsvCursorPosition d) - offset-        len  = posD - posC-{-# INLINE snippetStrict #-}- toListVectorStrict :: DsvCursor -> [DV.Vector BS.ByteString]-toListVectorStrict c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)-  then getRowBetweenStrict c d dEnd:toListVectorStrict (trim d)-  else []-  where nr = nextRow c-        d = nextPosition nr-        dEnd = atEnd nr+toListVectorStrict = LCS.toListVector+{-# DEPRECATED toListVectorStrict "Use HaskellWorks.Data.Dsv.Lazy.Cursor.Strict.toListVector instead" #-} {-# INLINE toListVectorStrict #-}++getRowBetweenStrict :: DsvCursor -> DsvCursor -> Bool -> DV.Vector BS.ByteString+getRowBetweenStrict = LCS.getRowVectorBetween+{-# INLINE getRowBetweenStrict #-}+{-# DEPRECATED getRowBetweenStrict "Use HaskellWorks.Data.Dsv.Lazy.Cursor.Strict.getRowVectorBetween instead" #-}
src/HaskellWorks/Data/Dsv/Lazy/Cursor/Internal.hs view
@@ -2,45 +2,69 @@ {-# LANGUAGE FlexibleContexts #-}  module HaskellWorks.Data.Dsv.Lazy.Cursor.Internal-  ( makeCummulativePopCount-  , makeQuoteMask+  ( nextField+  , nextPosition+  , atEnd+  , nextRow+  , trim+  , advanceField   ) where -import Data.Word-import HaskellWorks.Data.AtIndex-import HaskellWorks.Data.Bits.PopCount.PopCount1-import HaskellWorks.Data.Dsv.Internal.Broadword+import HaskellWorks.Data.Dsv.Lazy.Cursor.Type+import HaskellWorks.Data.Positioning+import HaskellWorks.Data.RankSelect.Base.Rank1+import HaskellWorks.Data.RankSelect.Base.Select1 import Prelude -import qualified Data.Vector.Storable as DVS+import qualified Data.ByteString.Lazy as LBS -{-# ANN module ("HLint: ignore Reduce duplication"  :: String) #-}+advanceField :: Count -> DsvCursor -> DsvCursor+advanceField n cursor = cursor+  { dsvCursorPosition = newPos+  }+  where currentRank = rank1   (dsvCursorMarkers cursor) (dsvCursorPosition cursor)+        newPos      = select1 (dsvCursorMarkers cursor) (currentRank + n) - 1+{-# INLINE advanceField #-} -makeQuoteMask1 :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-makeQuoteMask1 ibv pcv = DVS.constructN (DVS.length ibv) go-  where go :: DVS.Vector Word64 -> Word64-        go u =-          let ui = end u in-          toggle64 (pcv !!! ui) (ibv !!! ui)-{-# INLINE makeQuoteMask1 #-}+nextField :: DsvCursor -> DsvCursor+nextField cursor = cursor+  { dsvCursorPosition = newPos+  }+  where currentRank = rank1   (dsvCursorMarkers cursor) (dsvCursorPosition cursor)+        newPos      = select1 (dsvCursorMarkers cursor) (currentRank + 1) - 1+{-# INLINE nextField #-} -makeQuoteMask :: [DVS.Vector Word64] -> [DVS.Vector Word64] -> [DVS.Vector Word64]-makeQuoteMask (ibv:ibvs) (pcv:pcvs) = makeQuoteMask1 ibv pcv:makeQuoteMask ibvs pcvs-makeQuoteMask _ _                   = []-{-# INLINE makeQuoteMask #-}+nextPosition :: DsvCursor -> DsvCursor+nextPosition cursor = cursor+    { dsvCursorPosition = if LBS.null (LBS.drop (fromIntegral newPos) (dsvCursorText cursor))+                            then fromIntegral (LBS.length (dsvCursorText cursor))+                            else newPos+    }+  where newPos  = dsvCursorPosition cursor + 1+{-# INLINE nextPosition #-} -makeCummulativePopCount2 :: Word64 -> DVS.Vector Word64 -> (DVS.Vector Word64, Word64)-makeCummulativePopCount2 c v = let r = DVS.constructN (DVS.length v + 1) go in (DVS.unsafeInit r, DVS.unsafeLast r)-  where go :: DVS.Vector Word64 -> Word64-        go u = let ui = end u in-          if ui > 0-            then popCount1 (v !!! (ui - 1)) + (u !!! (ui - 1))-            else c-{-# INLINE makeCummulativePopCount2 #-}+atEnd :: DsvCursor -> Bool+atEnd c = LBS.null (LBS.drop (fromIntegral (dsvCursorPosition c)) (dsvCursorText c))+{-# INLINE atEnd #-} -makeCummulativePopCount :: [DVS.Vector Word64] -> [DVS.Vector Word64]-makeCummulativePopCount = go 0-  where go :: Word64 -> [DVS.Vector Word64] -> [DVS.Vector Word64]-        go c (v:vs) = let (u, c') = makeCummulativePopCount2 c v in u:go c' vs-        go _ []     = []-{-# INLINE makeCummulativePopCount #-}+nextRow :: DsvCursor -> DsvCursor+nextRow cursor = cursor+  { dsvCursorPosition = if newPos > dsvCursorPosition cursor+                          then newPos+                          else fromIntegral (LBS.length (dsvCursorText cursor))++  }+  where currentRank = rank1   (dsvCursorNewlines cursor) (dsvCursorPosition cursor)+        newPos      = select1 (dsvCursorNewlines cursor) (currentRank + 1) - 1+{-# INLINE nextRow #-}++trim :: DsvCursor -> DsvCursor+trim c = if dsvCursorPosition c >= 512+  then trim c+    { dsvCursorText     = LBS.drop 512 (dsvCursorText c)+    , dsvCursorMarkers  = drop 1 (dsvCursorMarkers c)+    , dsvCursorNewlines = drop 1 (dsvCursorNewlines c)+    , dsvCursorPosition = dsvCursorPosition c - 512+    }+  else c+{-# INLINE trim #-}
+ src/HaskellWorks/Data/Dsv/Lazy/Cursor/Lazy.hs view
@@ -0,0 +1,97 @@+{-|+Module      : HaskellWorks.Data.Dsv.Lazy.Cursor.Strict+Description : Extraction functions that yields lazy bytestrings+-}+module HaskellWorks.Data.Dsv.Lazy.Cursor.Lazy+  ( snippet+  , getRowListBetween+  , getRowVectorBetween+  , toListList+  , toListVector+  , toVectorVector+  , selectListVector+  ) where++import Data.Function+import HaskellWorks.Data.Dsv.Lazy.Cursor.Internal+import HaskellWorks.Data.Dsv.Lazy.Cursor.Type+import HaskellWorks.Data.RankSelect.Base.Rank1+import Prelude++import qualified Data.ByteString.Lazy as LBS+import qualified Data.List            as L+import qualified Data.Vector          as DV++snippet :: DsvCursor -> LBS.ByteString+snippet c = LBS.take (len `max` 0) $ LBS.drop posC $ dsvCursorText c+  where d = nextField c+        posC = fromIntegral $ dsvCursorPosition c+        posD = fromIntegral $ dsvCursorPosition d+        len  = posD - posC+{-# INLINE snippet #-}++getRowVectorBetween :: DsvCursor -> DsvCursor -> Bool -> DV.Vector LBS.ByteString+getRowVectorBetween c d dEnd = DV.unfoldrN fields go c+  where cr  = rank1 (dsvCursorMarkers c) (dsvCursorPosition c)+        dr  = rank1 (dsvCursorMarkers d) (dsvCursorPosition d)+        c2d = fromIntegral (dr - cr)+        fields = if dEnd then c2d +1 else c2d+        go :: DsvCursor -> Maybe (LBS.ByteString, DsvCursor)+        go e = case nextField e of+          f -> case nextPosition f of+            g -> case snippet e of+              s -> Just (s, g)+        {-# INLINE go #-}+{-# INLINE getRowVectorBetween #-}++getRowListBetween :: DsvCursor -> DsvCursor -> Bool -> [LBS.ByteString]+getRowListBetween c d dEnd = take fields (L.unfoldr go c)+  where cr  = rank1 (dsvCursorMarkers c) (dsvCursorPosition c)+        dr  = rank1 (dsvCursorMarkers d) (dsvCursorPosition d)+        c2d = fromIntegral (dr - cr)+        fields = if dEnd then c2d +1 else c2d+        go :: DsvCursor -> Maybe (LBS.ByteString, DsvCursor)+        go e = case nextField e of+          f -> case nextPosition f of+            g -> case snippet e of+              s -> Just (s, g)+        {-# INLINE go #-}+{-# INLINE getRowListBetween #-}++toListList :: DsvCursor -> [[LBS.ByteString]]+toListList c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)+  then getRowListBetween c d dEnd:toListList (trim d)+  else []+  where nr = nextRow c+        d = nextPosition nr+        dEnd = atEnd nr+{-# INLINE toListList #-}++toListVector :: DsvCursor -> [DV.Vector LBS.ByteString]+toListVector c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)+  then getRowVectorBetween c d dEnd:toListVector (trim d)+  else []+  where nr = nextRow c+        d = nextPosition nr+        dEnd = atEnd nr+{-# INLINE toListVector #-}++toVectorVector :: DsvCursor -> DV.Vector (DV.Vector LBS.ByteString)+toVectorVector = DV.fromList . toListVector+{-# INLINE toVectorVector #-}++selectRowFrom :: [Int] -> DsvCursor -> [LBS.ByteString]+selectRowFrom sel c = go <$> sel+  where go :: Int -> LBS.ByteString+        go n = snippet nc+          where nc = nextPosition (advanceField (fromIntegral n) c)+        {-# INLINE go #-}+{-# INLINE selectRowFrom #-}++selectListVector :: [Int] -> DsvCursor -> [[LBS.ByteString]]+selectListVector sel c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)+  then selectRowFrom sel c:selectListVector sel (trim d)+  else []+  where nr = nextRow c+        d = nextPosition nr+{-# INLINE selectListVector #-}
+ src/HaskellWorks/Data/Dsv/Lazy/Cursor/Strict.hs view
@@ -0,0 +1,91 @@+{-|+Module      : HaskellWorks.Data.Dsv.Lazy.Cursor.Strict+Description : Extraction functions that yields strict bytestrings+-}+module HaskellWorks.Data.Dsv.Lazy.Cursor.Strict+  ( snippet+  , getRowListBetween+  , getRowVectorBetween+  , toListList+  , toListVector+  , toVectorVector+  ) where++import Data.Function+import HaskellWorks.Data.Dsv.Lazy.Cursor.Internal+import HaskellWorks.Data.Dsv.Lazy.Cursor.Type+import HaskellWorks.Data.RankSelect.Base.Rank1+import Prelude++import qualified Data.ByteString      as BS+import qualified Data.ByteString.Lazy as LBS+import qualified Data.List            as L+import qualified Data.Vector          as DV++snippet :: DsvCursor -> Int -> BS.ByteString -> BS.ByteString+snippet c offset bs = BS.take (len `max` 0) $ BS.drop posC $ bs+  where d = nextField c+        posC = fromIntegral (dsvCursorPosition c) - offset+        posD = fromIntegral (dsvCursorPosition d) - offset+        len  = posD - posC+{-# INLINE snippet #-}++getRowListBetween :: DsvCursor -> DsvCursor -> Bool -> [BS.ByteString]+getRowListBetween c d dEnd = take fields (L.unfoldr go c)+  where bsA = fromIntegral $ dsvCursorPosition c+        bsZ = fromIntegral $ dsvCursorPosition d+        bsT = dsvCursorText c+        bs  = LBS.toStrict $ LBS.take (bsZ - bsA) (LBS.drop bsA bsT)++        cr  = rank1 (dsvCursorMarkers c) (dsvCursorPosition c)+        dr  = rank1 (dsvCursorMarkers d) (dsvCursorPosition d)+        c2d = fromIntegral (dr - cr)+        fields = if dEnd then c2d +1 else c2d+        go :: DsvCursor -> Maybe (BS.ByteString, DsvCursor)+        go e = case nextField e of+          f -> case nextPosition f of+            g -> case snippet e (fromIntegral bsA) bs of+              s -> Just (s, g)+        {-# INLINE go #-}+{-# INLINE getRowListBetween #-}++getRowVectorBetween :: DsvCursor -> DsvCursor -> Bool -> DV.Vector BS.ByteString+getRowVectorBetween c d dEnd = DV.unfoldrN fields go c+  where bsA = fromIntegral $ dsvCursorPosition c+        bsZ = fromIntegral $ dsvCursorPosition d+        bsT = dsvCursorText c+        bs  = LBS.toStrict $ LBS.take (bsZ - bsA) (LBS.drop bsA bsT)++        cr  = rank1 (dsvCursorMarkers c) (dsvCursorPosition c)+        dr  = rank1 (dsvCursorMarkers d) (dsvCursorPosition d)+        c2d = fromIntegral (dr - cr)+        fields = if dEnd then c2d +1 else c2d+        go :: DsvCursor -> Maybe (BS.ByteString, DsvCursor)+        go e = case nextField e of+          f -> case nextPosition f of+            g -> case snippet e (fromIntegral bsA) bs of+              s -> Just (s, g)+        {-# INLINE go #-}+{-# INLINE getRowVectorBetween #-}++toListList :: DsvCursor -> [[BS.ByteString]]+toListList c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)+  then getRowListBetween c d dEnd:toListList (trim d)+  else []+  where nr = nextRow c+        d = nextPosition nr+        dEnd = atEnd nr+{-# INLINE toListList #-}++toListVector :: DsvCursor -> [DV.Vector BS.ByteString]+toListVector c = if dsvCursorPosition d > dsvCursorPosition c && not (atEnd c)+  then getRowVectorBetween c d dEnd:toListVector (trim d)+  else []+  where nr = nextRow c+        d = nextPosition nr+        dEnd = atEnd nr+{-# INLINE toListVector #-}++toVectorVector :: DsvCursor -> DV.Vector (DV.Vector BS.ByteString)+toVectorVector = DV.fromList . toListVector+{-# INLINE toVectorVector #-}