scalpel 0.3.0.1 → 0.3.1

raw patch · 7 files changed

+81/−24 lines, 7 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• CHANGELOG.md +7/−0
• scalpel.cabal +2/−2
• src/Text/HTML/Scalpel.hs +3/−0
• src/Text/HTML/Scalpel/Internal/Scrape.hs +27/−4
• src/Text/HTML/Scalpel/Internal/Select.hs +0/−1
• src/Text/HTML/Scalpel/Internal/Select/Combinators.hs +9/−0
• tests/TestMain.hs +33/−17

@@ -2,6 +2,13 @@  ## HEAD +## 0.3.1++- Added the `innerHTML` and `innerHTMLs` scraper.+- Added the `match` function which allows for the creation of arbitrary+  attribute predicates.+- Fixed build breakage with GHC 8.0.1.+ ## 0.3.0.1  - Make tag and attribute matching case-insensitive.

@@ -1,5 +1,5 @@ name:                scalpel-version:             0.3.0.1+version:             0.3.1 synopsis:            A high level web scraping library for Haskell. description:     Scalpel is a web scraping library inspired by libraries like Parsec and@@ -24,7 +24,7 @@ source-repository this   type:     git   location: https://github.com/fimad/scalpel.git-  tag:      v0.3.0.1+  tag:      v0.3.1  library   other-extensions:

@@ -113,6 +113,7 @@ ,   (@=) ,   (@=~) ,   hasClass+,   match  -- * Scrapers ,   Scraper@@ -121,6 +122,8 @@ ,   attrs ,   html ,   htmls+,   innerHTML+,   innerHTMLs ,   text ,   texts ,   chroot

@@ -6,6 +6,8 @@ ,   attrs ,   html ,   htmls+,   innerHTML+,   innerHTMLs ,   text ,   texts ,   chroot@@ -92,7 +94,8 @@  -- | The 'texts' function takes a selector and returns the inner text from every -- set of tags matching the given selector.-texts :: (Ord str, TagSoup.StringLike str, Selectable s) => s -> Scraper str [str]+texts :: (Ord str, TagSoup.StringLike str, Selectable s)+      => s -> Scraper str [str] texts s = MkScraper $ withAll tagsToText . select_ s  -- | The 'html' function takes a selector and returns the html string from the@@ -103,11 +106,28 @@ html :: (Ord str, TagSoup.StringLike str, Selectable s) => s -> Scraper str str html s = MkScraper $ withHead tagsToHTML . select_ s --- | The 'htmls' function takes a selector and returns the html string from every--- set of tags matching the given selector.-htmls :: (Ord str, TagSoup.StringLike str, Selectable s) => s -> Scraper str [str]+-- | The 'htmls' function takes a selector and returns the html string from+-- every set of tags matching the given selector.+htmls :: (Ord str, TagSoup.StringLike str, Selectable s)+      => s -> Scraper str [str] htmls s = MkScraper $ withAll tagsToHTML . select_ s +-- | The 'innerHTML' function takes a selector and returns the inner html string+-- from the set of tags described by the given selector. Inner html here meaning+-- the html within but not including the selected tags.+--+-- This function will match only the first set of tags matching the selector, to+-- match every set of tags, use 'innerHTMLs'.+innerHTML :: (Ord str, TagSoup.StringLike str, Selectable s)+          => s -> Scraper str str+innerHTML s = MkScraper $ withHead tagsToInnerHTML . select_ s++-- | The 'innerHTMLs' function takes a selector and returns the inner html+-- string from every set of tags matching the given selector.+innerHTMLs :: (Ord str, TagSoup.StringLike str, Selectable s)+           => s -> Scraper str [str]+innerHTMLs s = MkScraper $ withAll tagsToInnerHTML . select_ s+ -- | The 'attr' function takes an attribute name and a selector and returns the -- value of the attribute of the given name for the first opening tag that -- matches the given selector.@@ -141,6 +161,9 @@  tagsToHTML :: TagSoup.StringLike str => [TagSoup.Tag str] -> str tagsToHTML = TagSoup.renderTags++tagsToInnerHTML :: TagSoup.StringLike str => [TagSoup.Tag str] -> str+tagsToInnerHTML = tagsToHTML . reverse . drop 1 . reverse . drop 1  tagsToAttr :: (Show str, TagSoup.StringLike str)            => str -> [TagSoup.Tag str] -> Maybe str

@@ -1,4 +1,3 @@-{-# LANGUAGE ImpredicativeTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TupleSections #-} {-# OPTIONS_HADDOCK hide #-}

@@ -8,6 +8,7 @@ ,   (@=) ,   (@=~) ,   hasClass+,   match ) where  import Text.HTML.Scalpel.Internal.Select.Types@@ -65,3 +66,11 @@             where textClass   = TagSoup.castString clazz                   textClasses = TagSoup.castString classes                   classList   = T.split (== ' ') textClasses++-- | The 'match' function allows for the creation of arbitrary+-- 'AttributePredicate's. The argument is a function that takes the attribute+-- key followed by the attribute value and returns a boolean indicating if the+-- attribute satisfies the predicate.+match :: (String -> String -> Bool) -> AttributePredicate+match f = MkAttributePredicate $ \(attrKey, attrValue) ->+              f (TagSoup.toString attrKey) (TagSoup.toString attrValue)

@@ -11,11 +11,7 @@ import qualified Text.Regex.TDFA  -main = exit . failures =<< runTestTT (TestList [-        scrapeTests-    ,   scrapeHtmlsTests-    ,   scrapeHtmlTests-    ])+main = exit . failures =<< runTestTT (TestList [scrapeTests])  exit :: Int -> IO () exit 0 = exitSuccess@@ -106,6 +102,11 @@             (htmls (Any @: [Any @= "value"]))      ,   scrapeTest+            "<a foo=\"bar\">1</a><a foo=\"foo\">2</a><a bar=\"bar\">3</a>"+            (Just ["<a foo=\"foo\">2</a>", "<a bar=\"bar\">3</a>"])+            (htmls (Any @: [match (==)]))++    ,   scrapeTest             "<a>foo</a>"             (Just "foo")             (text "a")@@ -178,40 +179,55 @@             "<a B=C>foo</a>"             Nothing             (texts $ "A" @: ["b" @= "c"])-    ] -scrapeTest :: (Eq a, Show a) => String -> Maybe a -> Scraper String a -> Test-scrapeTest html expected scraper = label ~: expected @=? actual-    where-        label  = "scrape (" ++ show html ++ ")"-        actual = scrape scraper (TagSoup.parseTags html)--scrapeHtmlTests = "scrapeTests" ~: TestList [-        scrapeTest+    ,   scrapeTest             "<a>foo</a>"             (Just "<a>foo</a>")             (html "a")+     ,   scrapeTest             "<body><div><ul><li>1</li><li>2</li></ul></div></body>"             (Just "<li>1</li>")             (html "li")+     ,   scrapeTest             "<body><div></div></body>"             (Just "<div></div>")             (html "div")-    ] -scrapeHtmlsTests = "scrapeTests" ~: TestList [-        scrapeTest+    ,   scrapeTest             "<a>foo</a><a>bar</a>"             (Just ["<a>foo</a>","<a>bar</a>"])             (htmls "a")+     ,   scrapeTest             "<body><div><ul><li>1</li><li>2</li></ul></div></body>"             (Just ["<li>1</li>", "<li>2</li>"])             (htmls "li")+     ,   scrapeTest             "<body><div></div></body>"             (Just ["<div></div>"])             (htmls "div")++    ,   scrapeTest+            "<a>1<b>2</b>3</a>"+            (Just "1<b>2</b>3")+            (innerHTML Any)++    ,   scrapeTest+            "<a>"+            (Just "")+            (innerHTML Any)++    ,   scrapeTest+            "<a>foo</a><a>bar</a>"+            (Just ["foo","bar"])+            (innerHTMLs "a")     ]++scrapeTest :: (Eq a, Show a) => String -> Maybe a -> Scraper String a -> Test+scrapeTest html expected scraper = label ~: expected @=? actual+    where+        label  = "scrape (" ++ show html ++ ")"+        actual = scrape scraper (TagSoup.parseTags html)