scalpel 0.4.0 → 0.4.1

raw patch · 8 files changed

+69/−10 lines, 8 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

+ Text.HTML.Scalpel: notP :: AttributePredicate -> AttributePredicate

Files

CHANGELOG.md +4/−0
README.md +32/−0
scalpel.cabal +2/−2
src/Text/HTML/Scalpel.hs +1/−0
src/Text/HTML/Scalpel/Internal/Select.hs +1/−1
src/Text/HTML/Scalpel/Internal/Select/Combinators.hs +10/−5
src/Text/HTML/Scalpel/Internal/Select/Types.hs +9/−2
tests/TestMain.hs +10/−0

CHANGELOG.md view

@@ -2,6 +2,10 @@  ## HEAD +## 0.4.1++- Added `notP` attribute predicate.+ ## 0.4.0  - Add the `chroot` tricks (#23 and #25) to README.md and added examples.

README.md view

@@ -231,3 +231,35 @@ For the full source of this example, see [generalized-repetition](https://github.com/fimad/scalpel/tree/master/examples/generalized-repetition/) in the examples directory.++Troubleshooting+---------------++### My Scraping Target Doesn't Return The Markup I Expected++Some websites return different markup depending on the user agent sent along+with the request. In some cases, this even means returning no markup at all in+an effort to prevent scraping.++To work around this, you can add your own user agent string with a curl option.++```haskell+#!/usr/local/bin/stack+-- stack runghc --resolver lts-6.24 --install-ghc --package scalpel-0.4.0++import Network.Curl+import Text.HTML.Scalpel++main = do+    html <- scrapeURLWithOpts opts url $ htmls anySelector+    maybe printError printHtml html+  where+    url = "https://www.google.com"+    opts = [ CurlUserAgent "some user agent string" ]+    printError = putStrLn "Failed"+    printHtml = mapM_ putStrLn+```++A list of user agent strings can be found+[here](http://www.useragentstring.com/pages/useragentstring.php).+

scalpel.cabal view

@@ -1,5 +1,5 @@ name:                scalpel-version:             0.4.0+version:             0.4.1 synopsis:            A high level web scraping library for Haskell. description:     Scalpel is a web scraping library inspired by libraries like Parsec and@@ -24,7 +24,7 @@ source-repository this   type:     git   location: https://github.com/fimad/scalpel.git-  tag:      v0.4.0+  tag:      v0.4.1  library   other-extensions:

src/Text/HTML/Scalpel.hs view

@@ -113,6 +113,7 @@ ,   (@=) ,   (@=~) ,   hasClass+,   notP ,   match  -- * Scrapers

src/Text/HTML/Scalpel/Internal/Select.hs view

@@ -263,5 +263,5 @@            => [AttributePredicate] -> TagSoup.Tag str -> Bool checkPreds preds tag     =  TagSoup.isTagOpen tag-    && and [or [checkPred p attr | attr <- attrs] | p <- preds]+    && all (flip checkPred attrs) preds     where (TagSoup.TagOpen _ attrs) = tag

src/Text/HTML/Scalpel/Internal/Select/Combinators.hs view

@@ -8,6 +8,7 @@ ,   (@=) ,   (@=~) ,   hasClass+,   notP ,   match ) where @@ -30,8 +31,8 @@ -- If you are attempting to match a specific class of a tag with potentially -- multiple classes, you should use the 'hasClass' utility function. (@=) :: AttributeName -> String -> AttributePredicate-(@=) key value = MkAttributePredicate $ \(attrKey, attrValue) ->-                                         matchKey key attrKey+(@=) key value = anyAttrPredicate $ \(attrKey, attrValue) ->+                                      matchKey key attrKey                                       && TagSoup.fromString value == attrValue infixl 6 @= @@ -40,7 +41,7 @@ -- expression. (@=~) :: RE.RegexLike re String       => AttributeName -> re -> AttributePredicate-(@=~) key re = MkAttributePredicate $ \(attrKey, attrValue) ->+(@=~) key re = anyAttrPredicate $ \(attrKey, attrValue) ->        matchKey key attrKey     && RE.matchTest re (TagSoup.toString attrValue) infixl 6 @=~@@ -58,7 +59,7 @@ -- the @class@ attribute. The 'hasClass' function will match a @class@ attribute -- if the given class appears anywhere in the space separated list of classes. hasClass :: String -> AttributePredicate-hasClass clazz = MkAttributePredicate hasClass'+hasClass clazz = anyAttrPredicate hasClass'     where         hasClass' (attrName, classes)             | "class" == TagSoup.toString attrName = textClass `elem` classList@@ -67,10 +68,14 @@                   textClasses = TagSoup.castString classes                   classList   = T.split (== ' ') textClasses +-- | Negates an 'AttributePredicate'. +notP :: AttributePredicate -> AttributePredicate+notP (MkAttributePredicate p) = MkAttributePredicate $ not . p+ -- | The 'match' function allows for the creation of arbitrary -- 'AttributePredicate's. The argument is a function that takes the attribute -- key followed by the attribute value and returns a boolean indicating if the -- attribute satisfies the predicate. match :: (String -> String -> Bool) -> AttributePredicate-match f = MkAttributePredicate $ \(attrKey, attrValue) ->+match f = anyAttrPredicate $ \(attrKey, attrValue) ->               f (TagSoup.toString attrKey) (TagSoup.toString attrValue)

src/Text/HTML/Scalpel/Internal/Select/Types.hs view

@@ -9,6 +9,7 @@ ,   checkPred ,   AttributeName (..) ,   matchKey+,   anyAttrPredicate ,   TagName (..) ,   SelectNode (..) ,   tagSelector@@ -39,12 +40,18 @@ -- returns a 'Bool' indicating if the given attribute matches a predicate. data AttributePredicate         = MkAttributePredicate-                (forall str. TagSoup.StringLike str => TagSoup.Attribute str+                (forall str. TagSoup.StringLike str => [TagSoup.Attribute str]                                                     -> Bool)  checkPred :: TagSoup.StringLike str-          => AttributePredicate -> TagSoup.Attribute str -> Bool+          => AttributePredicate -> [TagSoup.Attribute str] -> Bool checkPred (MkAttributePredicate p) = p++-- | Creates an 'AttributePredicate' from a predicate function of a single+-- attribute that matches if any one of the attributes matches the predicate.+anyAttrPredicate :: (forall str. TagSoup.StringLike str => (str, str) -> Bool)+                 -> AttributePredicate+anyAttrPredicate p = MkAttributePredicate $ any p  -- | 'Selector' defines a selection of an HTML DOM tree to be operated on by -- a web scraper. The selection includes the opening tag that matches the

tests/TestMain.hs view

@@ -80,6 +80,11 @@             (Just [])             (htmls ("a" @: [hasClass "c"])) +    , scrapeTest+            "<a>foo</a><a class=\"a b\">bar</a><a class=\"b\">baz</a>"+            (Just ["foo", "baz"])+            (texts ("a" @: [notP $ hasClass "a"]))+     ,   scrapeTest             "<a key=\"value\">foo</a>"             (Just ["<a key=\"value\">foo</a>"])@@ -189,6 +194,11 @@             "<a B=C>foo</a>"             (Just [])             (texts $ "A" @: ["b" @= "c"])++    , scrapeTest+            "<a>foo</a><a B=C>bar</a><a B=D>baz</a>"+            (Just ["foo", "baz"])+            (texts ("a" @: [notP $ "b" @= "C"]))      ,   scrapeTest             "<a>foo</a>"