feed-crawl 0.1.1.2 → 0.1.2.0

raw patch · 3 files changed

+14/−4 lines, 3 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• Text/Feed/Crawl.hs +9/−2
• Text/Feed/Crawl/Common.hs +4/−1
• feed-crawl.cabal +1/−1

@@ -21,13 +21,20 @@ import Data.Monoid import Control.Monad (join) +-- | Spoof a Safari Browser because some sites don't even serve feeds to +--   an http-conduit client+userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A"+ -- |The main function crawlURL :: String            -> IO CrawlResult crawlURL url = do     request <- parseUrl url+    let request' = request {+          requestHeaders = ("User-Agent", userAgent):(requestHeaders request)+        }     let settings = mkManagerSettings (TLSSettingsSimple True False False) Nothing-    (mkCrawlResult url =<< withRedirectTracking settings request)+    (mkCrawlResult url =<< withRedirectTracking settings request')       `E.catch` (\e -> return . Left . CrawlHttpError $ e)      @@ -46,7 +53,7 @@       else do           links <- findFeedLinks (BL.unpack . responseBody $ resp)           -- TODO this ignore any the <base> tag in the html page-          return . Left . CrawlFoundFeedLinks +          return . Left . CrawlFoundFeedLinks (responseBody resp)                  . map (ensureLinkIsAbsolute lastUrl) $ links   where ensureLinkIsAbsolute :: B.ByteString -> Link -> Link         ensureLinkIsAbsolute linkBaseUrl x@Link{..} = 

@@ -8,7 +8,10 @@ type CrawlResult = Either CrawlFail CrawlSuccess  data CrawlFail = -    CrawlFoundFeedLinks [Link] +    CrawlFoundFeedLinks {+      crawlNotFeedResponse :: BL.ByteString+    , crawlNotFeedWithLinks :: [Link] +    }   | CrawlHttpError HttpException   deriving Show 

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                feed-crawl-version:             0.1.1.2+version:             0.1.2.0 synopsis:            Utility for fetching feeds with redirect info and HTML link detection description:         Utility for fetching feeds with redirect info and HTML link detection  homepage:            https://github.com/danchoi/feed-crawl