warc 0.2.0 → 0.3.0
raw patch · 3 files changed
+45/−21 lines, 3 filesdep ~lensdep ~optparse-applicativedep ~pipes
Dependency ranges changed: lens, optparse-applicative, pipes
Files
- Data/Warc.hs +30/−9
- Data/Warc/Header.hs +12/−9
- warc.cabal +3/−3
Data/Warc.hs view
@@ -2,9 +2,17 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE GADTs #-} +-- | WARC (or Web ARCive) is a archival file format widely used to distribute+-- corpora of crawled web content (see, for instance the Common Crawl corpus). A+-- WARC file consists of a set of records, each of which describes a web request+-- or response.+--+-- This module provides a streaming parser and encoder for WARC archives for use+-- with the @pipes@ package.+-- module Data.Warc- ( Record(..)- , Warc(..)+ ( Warc(..)+ , Record(..) -- * Parsing , parseWarc , iterRecords@@ -31,20 +39,32 @@ -- | A WARC record+--+-- This represents a single record of a WARC file, consisting of a set of+-- headers and a means of producing the record's body. data Record m r = Record { recHeader :: RecordHeader+ -- ^ the WARC headers , recContent :: Producer BS.ByteString m r+ -- ^ the body of the record } instance Monad m => Functor (Record m) where fmap f (Record hdr r) = Record hdr (fmap f r) --- | A WARC archive+-- | A WARC archive.+--+-- This represents a sequence of records followed by whatever data+-- was leftover from the parse. type Warc m a = FreeT (Record m) m (Producer BS.ByteString m a) -- | Parse a WARC archive.+--+-- Note that this function does not actually do any parsing itself;+-- it merely returns a 'Warc' value which can then be run to parse+-- individual records. parseWarc :: (Functor m, Monad m)- => Producer ByteString m a- -> Warc m a+ => Producer ByteString m a -- ^ a producer of a stream of WARC content+ -> Warc m a -- ^ the parsed WARC archive parseWarc = loop where loop upstream = FreeT $ do@@ -64,9 +84,9 @@ -- | Iterate over the 'Record's in a WARC archive iterRecords :: forall m a. Monad m- => (forall b. Record m b -> m b)- -> Warc m a- -> m (Producer BS.ByteString m a)+ => (forall b. Record m b -> m b) -- ^ the action to run on each 'Record'+ -> Warc m a -- ^ the 'Warc' file+ -> m (Producer BS.ByteString m a) -- ^ returns any leftover data iterRecords f warc = iterT iter warc where iter :: Record m (m (Producer BS.ByteString m a))@@ -80,13 +100,14 @@ -> Warc m a -- ^ a WARC archive (see 'parseWarc') -> Producer o m (Producer BS.ByteString m a)- -- ^ returns any leftovers+ -- ^ returns any leftover data produceRecords f warc = iterTM iter warc where iter :: Record m (Producer o m (Producer BS.ByteString m a)) -> Producer o m (Producer BS.ByteString m a) iter (Record hdr body) = join $ f hdr body +-- | Encode a 'Record' in WARC format. encodeRecord :: Monad m => Record m a -> Producer BS.ByteString m a encodeRecord (Record hdr content) = do PBS.fromLazy $ BB.toLazyByteString $ encodeHeader hdr
Data/Warc/Header.hs view
@@ -2,17 +2,20 @@ {-# LANGUAGE TemplateHaskell #-} module Data.Warc.Header- ( RecordHeader(..)- , Version(..)- , WarcType (..)- , RecordId (..)- , TruncationReason (..)- , Digest (..)- , Uri (..)- , header+ ( -- * Parsing+ header+ -- * Encoding , encodeHeader+ -- * Types+ , RecordHeader(..)+ , Version(..)+ , WarcType(..)+ , RecordId(..)+ , TruncationReason(..)+ , Digest(..)+ , Uri(..) -- * Header field types- , Field (..)+ , Field(..) -- ** Prisms , _WarcRecordId , _ContentLength
warc.cabal view
@@ -1,5 +1,5 @@ name: warc-version: 0.2.0+version: 0.3.0 synopsis: A parser for the Web Archive (WARC) format description: A streaming parser for the Web Archive (WARC) format. homepage: http://github.com/bgamari/warc@@ -20,12 +20,12 @@ exposed-modules: Data.Warc, Data.Warc.Header other-extensions: RankNTypes, OverloadedStrings, TemplateHaskell build-depends: base >=4.8 && <4.10,- pipes >=4.1 && <4.2,+ pipes >=4.1 && <4.3, attoparsec >=0.12 && <0.14, bytestring >=0.10 && <0.11, pipes-bytestring >=2.1 && <2.2, transformers >=0.4 && <0.6,- lens >=4.7 && <4.14,+ lens >=4.7 && <4.15, pipes-attoparsec >=0.5 && <0.6, free >=4.10 && <4.13, errors >=1.4 && <3.0,