diff --git a/scalpel.cabal b/scalpel.cabal --- a/scalpel.cabal +++ b/scalpel.cabal @@ -1,5 +1,5 @@ name: scalpel -version: 0.1.2 +version: 0.1.3 synopsis: A high level web scraping library for Haskell. description: Scalpel is a web scraping library inspired by libraries like Parsec and @@ -21,7 +21,7 @@ source-repository this type: git location: https://github.com/fimad/scalpel.git - tag: v0.1.1 + tag: v0.1.3 library other-extensions: diff --git a/src/Text/HTML/Scalpel.hs b/src/Text/HTML/Scalpel.hs --- a/src/Text/HTML/Scalpel.hs +++ b/src/Text/HTML/Scalpel.hs @@ -116,6 +116,8 @@ -- ** Primitives , attr , attrs +, html +, htmls , text , texts , chroot diff --git a/src/Text/HTML/Scalpel/Internal/Scrape.hs b/src/Text/HTML/Scalpel/Internal/Scrape.hs --- a/src/Text/HTML/Scalpel/Internal/Scrape.hs +++ b/src/Text/HTML/Scalpel/Internal/Scrape.hs @@ -3,6 +3,8 @@ Scraper (..) , attr , attrs +, html +, htmls , text , texts , chroot @@ -84,6 +86,19 @@ texts :: (TagSoup.StringLike str, Selectable str s) => s -> Scraper str [str] texts s = MkScraper $ withAll tagsToText . select s +-- | The 'html' function takes a selector and returns the html string from the +-- set of tags described by the given selector. +-- +-- This function will match only the first set of tags matching the selector, to +-- match every set of tags, use 'htmls'. +html :: (TagSoup.StringLike str, Selectable str s) => s -> Scraper str str +html s = MkScraper $ withHead tagsToHTML . select s + +-- | The 'htmls' function takes a selector and returns the html string from every +-- set of tags matching the given selector. +htmls :: (TagSoup.StringLike str, Selectable str s) => s -> Scraper str [str] +htmls s = MkScraper $ withAll tagsToHTML . select s + -- | The 'attr' function takes an attribute name and a selector and returns the -- value of the attribute of the given name for the first opening tag that -- matches the given selector. @@ -111,6 +126,9 @@ tagsToText :: TagSoup.StringLike str => [TagSoup.Tag str] -> str tagsToText = TagSoup.innerText + +tagsToHTML :: TagSoup.StringLike str => [TagSoup.Tag str] -> str +tagsToHTML = TagSoup.renderTags tagsToAttr :: (Show str, TagSoup.StringLike str) => str -> [TagSoup.Tag str] -> Maybe str diff --git a/tests/TestMain.hs b/tests/TestMain.hs --- a/tests/TestMain.hs +++ b/tests/TestMain.hs @@ -14,6 +14,8 @@ main = exit . failures =<< runTestTT (TestList [ scrapeTests , selectTests + , scrapeHtmlsTests + , scrapeHtmlTests ]) exit :: Int -> IO () @@ -173,3 +175,33 @@ where label = "scrape (" ++ show html ++ ")" actual = scrape scraper (TagSoup.parseTags html) + +scrapeHtmlTests = "scrapeTests" ~: TestList [ + scrapeTest + "foo" + (Just "foo") + (html "a") + , scrapeTest + "
" + (Just "
  • 1
    • ") + (html "li") + , scrapeTest + "
    " + (Just "
    ") + (html "div") + ] + +scrapeHtmlsTests = "scrapeTests" ~: TestList [ + scrapeTest + "foobar" + (Just ["foo","bar"]) + (htmls "a") + , scrapeTest + "
    " + (Just ["
  • 1
    • ", "
  • 2
    • "]) + (htmls "li") + , scrapeTest + "
    " + (Just ["
    "]) + (htmls "div") + ]