jsonl-conduit (empty) → 0.1.0.0
raw patch · 7 files changed
+199/−0 lines, 7 filesdep +aesondep +basedep +bytestringsetup-changed
Dependencies added: aeson, base, bytestring, conduit, hspec, jsonl, jsonl-conduit
Files
- LICENSE +30/−0
- README.md +5/−0
- Setup.hs +2/−0
- jsonl-conduit.cabal +47/−0
- records +4/−0
- src/JSONL/Conduit.hs +110/−0
- test/Spec.hs +1/−0
+ LICENSE view
@@ -0,0 +1,30 @@+Copyright Author name here (c) 2022++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.++ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following+ disclaimer in the documentation and/or other materials provided+ with the distribution.++ * Neither the name of Author name here nor the names of other+ contributors may be used to endorse or promote products derived+ from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,5 @@+# jsonl-conduit++[](https://travis-ci.org/unfoldml/jsonl-conduit)++TODO Description.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ jsonl-conduit.cabal view
@@ -0,0 +1,47 @@+name: jsonl-conduit+version: 0.1.0.0+synopsis: Conduit interface to JSONL-encoded data+description: Streaming interface to JSONL-encoded files and bytestrings+homepage: https://github.com/unfoldml/jsonl-conduit+license: BSD3+license-file: LICENSE+author: Marco Zocca+maintainer: UnfoldML AB+copyright: (c) 2022 UnfoldML AB, Marco Zocca+category: JSON, Web, Text+build-type: Simple+extra-source-files: README.md+ records+cabal-version: >=1.10+tested-with: GHC == 9.0.2++library+ default-language: Haskell2010+ ghc-options: -Wall+ hs-source-dirs: src+ exposed-modules: JSONL.Conduit+ build-depends: base >= 4.7 && < 5+ , aeson+ , bytestring+ , conduit+ , jsonl >= 0.2+ -- debug+ , hspec++test-suite spec+ default-language: Haskell2010+ ghc-options: -Wall+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Spec.hs+ build-depends: base+ , aeson+ , bytestring+ , conduit+ , jsonl-conduit+ , hspec+++source-repository head+ type: git+ location: https://github.com/unfoldml/jsonl-conduit
+ records view
@@ -0,0 +1,4 @@+["Gilbert","2013",24,true]+["Alexa","2013",29,true]+["May","2012B",14,false]+["Deloise","2012A",19,true]
+ src/JSONL/Conduit.hs view
@@ -0,0 +1,110 @@+{-# LANGUAGE TupleSections #-}+{-# options_ghc -Wno-unused-imports #-}+module JSONL.Conduit (+ -- * Encode+ jsonToLBSC+ -- ** I/O+ , sinkFileC+ -- * Decode+ , jsonFromLBSC+ -- ** I/O+ , sourceFileC+ ) where++import Data.Void (Void)++import Control.Monad.IO.Class (MonadIO(..))++ -- aeson+import Data.Aeson (ToJSON(..), FromJSON(..), eitherDecode' )+-- bytestring+import qualified Data.ByteString as BS (ByteString, null)+import qualified Data.ByteString.Builder as BBS (toLazyByteString, Builder)+import qualified Data.ByteString.Internal as BS (c2w)+import qualified Data.ByteString.Char8 as BS8 (span, drop, putStrLn, putStr)+import qualified Data.ByteString.Lazy as LBS (ByteString, drop, span, toStrict, fromStrict)+-- conduit+import qualified Conduit as C (ConduitT, runConduit, sourceFile, sinkFile, await, yield, mapC, unfoldC, foldMapC, foldlC, printC)+import Conduit ( (.|) , MonadResource)+-- jsonl+import JSONL (jsonLine)++-- | Render a stream of JSON-encodable objects into a `LBS.ByteString`+jsonToLBSC :: (ToJSON a, Monad m) => C.ConduitT a o m LBS.ByteString+jsonToLBSC = BBS.toLazyByteString <$> jsonToBuilderC++-- | Render a stream of JSON-encodable objects into a `BSB.Builder`+jsonToBuilderC :: (ToJSON a, Monad m) => C.ConduitT a o m BBS.Builder+jsonToBuilderC = C.foldMapC jsonLine++-- | Render a stream of JSON-encodable objects into a JSONL file+sinkFileC :: (ToJSON a, MonadResource m) =>+ FilePath -- ^ path of JSONL file to be created+ -> C.ConduitT a o m ()+sinkFileC fpath = C.mapC (LBS.toStrict . BBS.toLazyByteString . jsonLine) .|+ C.sinkFile fpath++-- | Read a JSONL file and stream the decoded records+--+-- NB : ignores any parsing errors and returns +sourceFileC :: (MonadResource m, FromJSON a) =>+ FilePath -- ^ path of JSONL file to be read+ -> C.ConduitT () a m ()+sourceFileC fpath = C.sourceFile fpath .|+ parseChunk++parseChunk :: (Monad m, FromJSON a) => C.ConduitT BS.ByteString a m ()+parseChunk = go mempty+ where+ go acc =+ if not (BS.null acc) -- buffer is non empty+ then+ case chopDecode acc of+ Left _ -> pure ()+ Right (y, srest) -> do+ C.yield y+ go srest+ else do+ mc <- C.await -- get data from upstream+ case mc of+ Nothing -> pure ()+ Just x -> do+ let+ acc' = acc <> x+ case chopDecode acc' of+ Left _ -> pure ()+ Right (y, srest) -> do+ C.yield y+ go srest++chopDecode :: FromJSON a =>+ BS.ByteString -> Either String (a, BS.ByteString)+chopDecode acc = (, srest) <$> eitherDecode' (LBS.fromStrict s)+ where+ (s, srest) = chopBS8 acc++++-- | Source a `LBS.ByteString` for JSONL records+jsonFromLBSC :: (FromJSON a, Monad m) => LBS.ByteString -> C.ConduitT Void a m ()+jsonFromLBSC = C.unfoldC mk+ where+ mk lbs = case eitherDecode' s of+ Right x -> Just (x, srest)+ Left _ -> Nothing+ where+ (s, srest) = chopLBS lbs -- LBS.span (== BS.c2w '\n') lbs+++-- * utilities++-- | 'span' for a strict bytestring encoding a JSONL record+chopBS8 :: BS.ByteString -> (BS.ByteString, BS.ByteString)+chopBS8 lbs = (s, BS8.drop 1 srest)+ where (s, srest) = BS8.span (/= '\n') lbs++-- | 'span' for a lazy bytestring encoding a JSONL record+chopLBS :: LBS.ByteString -> (LBS.ByteString, LBS.ByteString)+chopLBS lbs = (s, LBS.drop 1 srest)+ where (s, srest) = LBS.span (/= BS.c2w '\n') lbs+
+ test/Spec.hs view
@@ -0,0 +1,1 @@+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}