scalpel 0.1.1 → 0.1.2

raw patch · 7 files changed

+45/−17 lines, 7 filesdep +curlPVP ok

version bump matches the API change (PVP)

Dependencies added: curl

API changes (from Hackage documentation)

Files

• scalpel.cabal +5/−3
• src/Text/HTML/Scalpel.hs +2/−1
• src/Text/HTML/Scalpel/Internal/Scrape.hs +1/−2
• src/Text/HTML/Scalpel/Internal/Scrape/URL.hs +18/−3
• src/Text/HTML/Scalpel/Internal/Select.hs +13/−6
• src/Text/HTML/Scalpel/Internal/Select/Combinators.hs +1/−1
• tests/TestMain.hs +5/−1

@@ -1,5 +1,5 @@ name:                scalpel-version:             0.1.1+version:             0.1.2 synopsis:            A high level web scraping library for Haskell. description:     Scalpel is a web scraping library inspired by libraries like Parsec and@@ -41,6 +41,7 @@   build-depends:           base          >= 4.6 && < 5       ,   bytestring+      ,   curl       ,   download-curl >= 0.1.4       ,   regex-base       ,   regex-tdfa@@ -57,10 +58,11 @@   main-is:          TestMain.hs   hs-source-dirs:   src/, tests/   build-depends:-          base          >= 4.6 && < 5+          HUnit+      ,   base          >= 4.6 && < 5       ,   bytestring+      ,   curl       ,   download-curl >= 0.1.4-      ,   HUnit       ,   regex-base       ,   regex-tdfa       ,   tagsoup       >= 0.8

@@ -72,7 +72,7 @@ -- >     = TextComment Author String -- >     | ImageComment Author URL -- >--- > allComments :: IO (Maybe [Comments])+-- > allComments :: IO (Maybe [Comment]) -- > allComments = scrapeURL "http://example.com/article.html" comments -- >    where -- >        comments :: Scraper String [Comment]@@ -125,6 +125,7 @@ ,   scrapeStringLike ,   URL ,   scrapeURL+,   scrapeURLWithOpts ) where  import Text.HTML.Scalpel.Internal.Scrape

@@ -69,8 +69,7 @@ chroots :: (TagSoup.StringLike str, Selectable str s)         => s -> Scraper str a -> Scraper str [a] chroots selector (MkScraper inner) = MkScraper-                                   $ return . catMaybes-                                   . map inner . select selector+                                   $ return . mapMaybe inner . select selector  -- | The 'text' function takes a selector and returns the inner text from the -- set of tags described by the given selector.

@@ -2,13 +2,16 @@ module Text.HTML.Scalpel.Internal.Scrape.URL (     URL ,   scrapeURL+,   scrapeURLWithOpts ) where  import Text.HTML.Scalpel.Internal.Scrape  import Control.Applicative +import qualified Network.Curl as Curl import qualified Network.Curl.Download as Curl+import qualified Text.HTML.TagSoup as TagSoup import qualified Text.StringLike as TagSoup  @@ -17,6 +20,18 @@ -- | The 'scrapeURL' function downloads the contents of the given URL and -- executes a 'Scraper' on it. scrapeURL :: TagSoup.StringLike str => URL -> Scraper str a -> IO (Maybe a)-scrapeURL url scraper = (eitherToMaybeA . toStr) <$> Curl.openAsTags url-    where eitherToMaybeA = either (const Nothing) (scrape scraper)-          toStr = (((TagSoup.castString <$>) <$>) <$>)+scrapeURL = scrapeURLWithOpts [Curl.CurlFollowLocation True]++-- | The 'scrapeURLWithOpts' function take a list of curl options and downloads+-- the contents of the given URL and executes a 'Scraper' on it.+scrapeURLWithOpts :: TagSoup.StringLike str+                  => [Curl.CurlOption] -> URL -> Scraper str a -> IO (Maybe a)+scrapeURLWithOpts options url scraper = do+    maybeTags <- downloadAsTags url+    return (maybeTags >>= scrape scraper)+    where+        downloadAsTags url = do+            maybeBytes <- maybeRight <$> Curl.openURIWithOpts options url+            return $ (TagSoup.parseTags . TagSoup.castString) <$> maybeBytes+        maybeRight (Right a) = Just a+        maybeRight _         = Nothing

@@ -21,7 +21,7 @@  selectNodes :: TagSoup.StringLike str             => [SelectNode str] -> [TagSoup.Tag str] -> [[TagSoup.Tag str]]-selectNodes nodes tags = head' $ reverse $ results+selectNodes nodes tags = head' $ reverse results     where results = [concatMap (selectNode s) ts | s  <- nodes                                                  | ts <- [tags] : results]           head' []    = []@@ -38,25 +38,32 @@ -- returns true if a given tag matches the supplied name and predicates. checkTag :: TagSoup.StringLike str           => str -> [AttributePredicate str] -> [TagSoup.Tag str] -> Bool-checkTag name preds tags@((TagSoup.TagOpen str _):_)+checkTag name preds tags@(TagSoup.TagOpen str _:_)     = name == str && checkPreds preds tags checkTag _ _ _ = False  checkPreds :: TagSoup.StringLike str             => [AttributePredicate str] -> [TagSoup.Tag str] -> Bool-checkPreds preds ((TagSoup.TagOpen _ attrs):_)+checkPreds preds (TagSoup.TagOpen _ attrs:_)     = and [or [p attr | attr <- attrs] | p <- preds] checkPreds _ _ = False --- Given a list of tags, return the prefix that of the tags up to the closing--- tag that corresponds to the initial tag.+-- Given a list of tags, return the prefix of the tags up to the closing tag+-- that corresponds to the initial tag. extractTagBlock :: TagSoup.StringLike str                 => [TagSoup.Tag str] -> [[TagSoup.Tag str]] extractTagBlock [] = [] extractTagBlock (openTag : tags)     | not $ TagSoup.isTagOpen openTag = []-    | otherwise                       = map (openTag :)+    | otherwise                       = fakeClosingTag openTag+                                      $ map (openTag :)                                       $ splitBlock (getTagName openTag) 0 tags+    where+        -- To handle tags that do not have a closing tag, fake an empty block by+        -- adding a closing tag.+        fakeClosingTag openTag@(TagSoup.TagOpen name _) []+            = [[openTag, TagSoup.TagClose name]]+        fakeClosingTag _ blocks = blocks  splitBlock _    _       []                          = [] splitBlock name depth  (tag : tags)

@@ -56,7 +56,7 @@ -- if the given class appears anywhere in the space separated list of classes. hasClass :: TagSoup.StringLike str => str -> AttributePredicate str hasClass clazz (attrName, classes)-    | "class" == TagSoup.toString attrName = any (== textClass) classList+    | "class" == TagSoup.toString attrName = textClass `elem` classList     | otherwise                            = False     where textClass   = TagSoup.castString clazz           textClasses = TagSoup.castString classes

@@ -49,7 +49,7 @@     ,   selectTest             ("a" @: [])             "<a>foo"-            []+            ["<a></a>"]      ,   selectTest             ("a" @: ["key" @= "value"])@@ -162,6 +162,10 @@             "<a><b>foo</b></a><a><c>bar</c></a>"             (Just "bar")             ((text $ "a" // "d") <|> (text $ "a" // "c"))++    ,   scrapeTest "<img src='foobar'>" (Just "foobar") (attr "src" $ "img")++    ,   scrapeTest "<img src='foobar' />" (Just "foobar") (attr "src" $ "img")     ]  scrapeTest :: (Eq a, Show a) => String -> Maybe a -> Scraper String a -> Test