scalpel 0.6.1 → 0.6.2

raw patch · 5 files changed

+76/−13 lines, 5 filesdep ~scalpel-corePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: scalpel-core

API changes (from Hackage documentation)

- Text.HTML.Scalpel: data Scraper str a

- Text.HTML.Scalpel: data SerialScraper str a

+ Text.HTML.Scalpel: data ScraperT str (m :: Type -> Type) a

+ Text.HTML.Scalpel: data SerialScraperT str (m :: Type -> Type) a

+ Text.HTML.Scalpel: fetchTags :: StringLike str => URL -> IO [Tag str]

+ Text.HTML.Scalpel: fetchTagsWithConfig :: StringLike str => Config str -> URL -> IO [Tag str]

+ Text.HTML.Scalpel: scrapeStringLikeT :: (StringLike str, Monad m) => str -> ScraperT str m a -> m (Maybe a)

+ Text.HTML.Scalpel: scrapeT :: StringLike str => ScraperT str m a -> [Tag str] -> m (Maybe a)

+ Text.HTML.Scalpel: type Scraper str = ScraperT str Identity

+ Text.HTML.Scalpel: type SerialScraper str a = SerialScraperT str Identity a

- Text.HTML.Scalpel: attr :: (Show str, StringLike str) => String -> Selector -> Scraper str str

+ Text.HTML.Scalpel: attr :: (Show str, StringLike str, Monad m) => String -> Selector -> ScraperT str m str

- Text.HTML.Scalpel: attrs :: (Show str, StringLike str) => String -> Selector -> Scraper str [str]

+ Text.HTML.Scalpel: attrs :: (Show str, StringLike str, Monad m) => String -> Selector -> ScraperT str m [str]

- Text.HTML.Scalpel: chroot :: StringLike str => Selector -> Scraper str a -> Scraper str a

+ Text.HTML.Scalpel: chroot :: (StringLike str, Monad m) => Selector -> ScraperT str m a -> ScraperT str m a

- Text.HTML.Scalpel: chroots :: StringLike str => Selector -> Scraper str a -> Scraper str [a]

+ Text.HTML.Scalpel: chroots :: (StringLike str, Monad m) => Selector -> ScraperT str m a -> ScraperT str m [a]

- Text.HTML.Scalpel: html :: StringLike str => Selector -> Scraper str str

+ Text.HTML.Scalpel: html :: (StringLike str, Monad m) => Selector -> ScraperT str m str

- Text.HTML.Scalpel: htmls :: StringLike str => Selector -> Scraper str [str]

+ Text.HTML.Scalpel: htmls :: (StringLike str, Monad m) => Selector -> ScraperT str m [str]

- Text.HTML.Scalpel: inSerial :: StringLike str => SerialScraper str a -> Scraper str a

+ Text.HTML.Scalpel: inSerial :: (StringLike str, Monad m) => SerialScraperT str m a -> ScraperT str m a

- Text.HTML.Scalpel: innerHTML :: StringLike str => Selector -> Scraper str str

+ Text.HTML.Scalpel: innerHTML :: (StringLike str, Monad m) => Selector -> ScraperT str m str

- Text.HTML.Scalpel: innerHTMLs :: StringLike str => Selector -> Scraper str [str]

+ Text.HTML.Scalpel: innerHTMLs :: (StringLike str, Monad m) => Selector -> ScraperT str m [str]

- Text.HTML.Scalpel: matches :: StringLike str => Selector -> Scraper str ()

+ Text.HTML.Scalpel: matches :: (StringLike str, Monad m) => Selector -> ScraperT str m ()

- Text.HTML.Scalpel: position :: StringLike str => Scraper str Int

+ Text.HTML.Scalpel: position :: (StringLike str, Monad m) => ScraperT str m Int

- Text.HTML.Scalpel: seekBack :: StringLike str => Scraper str a -> SerialScraper str a

+ Text.HTML.Scalpel: seekBack :: (StringLike str, Monad m) => ScraperT str m a -> SerialScraperT str m a

- Text.HTML.Scalpel: seekNext :: StringLike str => Scraper str a -> SerialScraper str a

+ Text.HTML.Scalpel: seekNext :: (StringLike str, Monad m) => ScraperT str m a -> SerialScraperT str m a

- Text.HTML.Scalpel: stepBack :: StringLike str => Scraper str a -> SerialScraper str a

+ Text.HTML.Scalpel: stepBack :: (StringLike str, Monad m) => ScraperT str m a -> SerialScraperT str m a

- Text.HTML.Scalpel: stepNext :: StringLike str => Scraper str a -> SerialScraper str a

+ Text.HTML.Scalpel: stepNext :: (StringLike str, Monad m) => ScraperT str m a -> SerialScraperT str m a

- Text.HTML.Scalpel: text :: StringLike str => Selector -> Scraper str str

+ Text.HTML.Scalpel: text :: (StringLike str, Monad m) => Selector -> ScraperT str m str

- Text.HTML.Scalpel: texts :: StringLike str => Selector -> Scraper str [str]

+ Text.HTML.Scalpel: texts :: (StringLike str, Monad m) => Selector -> ScraperT str m [str]

- Text.HTML.Scalpel: untilBack :: StringLike str => Scraper str a -> SerialScraper str b -> SerialScraper str b

+ Text.HTML.Scalpel: untilBack :: (StringLike str, Monad m) => ScraperT str m a -> SerialScraperT str m b -> SerialScraperT str m b

- Text.HTML.Scalpel: untilNext :: StringLike str => Scraper str a -> SerialScraper str b -> SerialScraper str b

+ Text.HTML.Scalpel: untilNext :: (StringLike str, Monad m) => ScraperT str m a -> SerialScraperT str m b -> SerialScraperT str m b

Files

CHANGELOG.md +2/−0
README.md +47/−0
scalpel.cabal +3/−3
src/Text/HTML/Scalpel.hs +6/−0
src/Text/HTML/Scalpel/Internal/Scrape/URL.hs +18/−10

CHANGELOG.md view

@@ -2,6 +2,8 @@  ## HEAD +## 0.6.2+ ## 0.6.1  - Support GHC 8.8.

README.md view

@@ -232,6 +232,53 @@ [generalized-repetition](https://github.com/fimad/scalpel/tree/master/examples/generalized-repetition/) in the examples directory. +### Operating with other monads inside the Scraper+`ScraperT` is a monad transformer scraper: it allows lifting `m a` operations+inside a `ScraperT str m a` with functions like:++```haskell+-- Particularizes to 'm a -> ScraperT str m a'+lift :: (MonadTrans t, Monad m) => m a -> t m a++-- Particularizes to things like `IO a -> ScraperT str IO a'+liftIO :: MonadIO m => IO a -> m a+```++Example: Perform HTTP requests on page images as you scrape:++1. Isolate images using `chroots`.++2. Within that context of an `img` tag, obtain the `src` attribute containing+   the location of the file.++3. Perform an IO operation to request metadata headers from the source.++4. Use the data to build and return more complex data++```haskell+-- Holds original link and data if it could be fetched+data Image = Image String (Maybe Metadata)+  deriving Show++-- Holds mime type and file size+data Metadata = Metadata String Int+  deriving Show++-- Scrape the page for images: get their metadata+scrapeImages :: URL -> ScraperT String IO [Image]+scrapeImages topUrl = do+    chroots "img" $ do+        source <- attr "src" "img"+        guard . not . null $ source+        -- getImageMeta is called via liftIO because ScrapeT transforms over IO+        liftM (Image source) $ liftIO (getImageMeta topUrl source)+```++For the full source of this example, see+[downloading data](https://github.com/fimad/scalpel/tree/master/examples/image-sizes/)++For more documentation on monad transformers, see the [hackage page](https://hackage.haskell.org/package/transformers)+ ### scalpel-core  The `scalpel` package depends on 'http-client' and 'http-client-tls' to provide

scalpel.cabal view

@@ -1,5 +1,5 @@ name:                scalpel-version:             0.6.1+version:             0.6.2 synopsis:            A high level web scraping library for Haskell. description:     Scalpel is a web scraping library inspired by libraries like Parsec and@@ -24,7 +24,7 @@ source-repository this   type:     git   location: https://github.com/fimad/scalpel.git-  tag:      v0.6.1+  tag:      v0.6.2  library   other-extensions:@@ -38,7 +38,7 @@   default-language: Haskell2010   build-depends:           base            >= 4.6 && < 5-      ,   scalpel-core    == 0.6.1+      ,   scalpel-core    == 0.6.2       ,   bytestring       ,   case-insensitive       ,   data-default

src/Text/HTML/Scalpel.hs view

@@ -122,6 +122,7 @@  -- * Scrapers ,   Scraper+,   ScraperT -- ** Primitives ,   attr ,   attrs@@ -138,7 +139,11 @@ -- ** Executing scrapers ,   scrape ,   scrapeStringLike+,   scrapeT+,   scrapeStringLikeT ,   URL+,   fetchTags+,   fetchTagsWithConfig ,   scrapeURL ,   scrapeURLWithConfig ,   Config (..)@@ -149,6 +154,7 @@  -- * Serial Scraping ,   SerialScraper+,   SerialScraperT ,   inSerial -- ** Primitives ,   stepNext

src/Text/HTML/Scalpel/Internal/Scrape/URL.hs view

@@ -9,16 +9,18 @@ ,   utf8Decoder ,   iso88591Decoder +,   fetchTags+,   fetchTagsWithConfig ,   scrapeURL ,   scrapeURLWithConfig ) where  import Text.HTML.Scalpel.Core -import Control.Applicative ((<$>))+import Control.Monad import Data.CaseInsensitive () import Data.Default (def)-import Data.Maybe (fromMaybe, listToMaybe)+import Data.Maybe (listToMaybe)  import qualified Data.ByteString.Lazy as LBS import qualified Data.Default as Default@@ -64,14 +66,20 @@ scrapeURLWithConfig :: (TagSoup.StringLike str)                   => Config str -> URL -> Scraper str a -> IO (Maybe a) scrapeURLWithConfig config url scraper = do-    manager <- fromMaybe HTTP.getGlobalManager (return <$> manager config)-    tags <- downloadAsTags (decoder config) manager url-    return (scrape scraper tags)-    where-        downloadAsTags decoder manager url = do-            request <- HTTP.parseRequest url-            response <- HTTP.httpLbs request manager-            return $ TagSoup.parseTags $ decoder response+    scrape scraper `liftM` fetchTagsWithConfig config url++-- | Download and parse the contents of the given URL.+fetchTags :: TagSoup.StringLike str+                => URL -> IO [TagSoup.Tag str]+fetchTags = fetchTagsWithConfig def++-- | Download and parse the contents of the given URL with the given 'Config'.+fetchTagsWithConfig :: TagSoup.StringLike str+                  => Config str -> URL -> IO [TagSoup.Tag str]+fetchTagsWithConfig config url = do+    manager <- maybe HTTP.getGlobalManager return (manager config)+    response <- flip HTTP.httpLbs manager =<< HTTP.parseRequest url+    return $ TagSoup.parseTags $ decoder config $ response  -- | The default response decoder. This decoder attempts to infer the character -- set of the HTTP response body from the `Content-Type` header. If this header