feed-crawl 0.1.0.0 → 0.1.1.0

raw patch · 3 files changed

+34/−5 lines, 3 filesdep +network-uridep ~basePVP ok

version bump matches the API change (PVP)

Dependencies added: network-uri

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• Text/Feed/Crawl.hs +28/−1
• Text/Feed/Crawl/DetectLink.hs +1/−0
• feed-crawl.cabal +5/−4

@@ -16,6 +16,10 @@ import Control.Monad.IO.Class import Data.Maybe (listToMaybe, catMaybes) import qualified Control.Exception as E+import Network.URI+import Control.Applicative +import Data.Monoid+import Control.Monad (join)  -- |The main function crawlURL :: String @@ -63,9 +67,22 @@    let location = lookup hLocation . responseHeaders $ res    case (req2, location) of        (Just req2', Just location') -> do+          -- location might be relative path+          let sLocation = B.unpack `fmap` location+          let location' =+                case sLocation of+                  Nothing -> Nothing+                  Just sLocation' -> +                    case parseURI sLocation' of+                      Just _ -> location -- is a valid full URI+                      Nothing ->  +                        (\x -> B.pack $ uriToString id x "") <$> +                          (nonStrictRelativeTo +                                <$> (parseRelativeReference sLocation')+                                <*> (baseURL req))           let st = Status {               sStatusCode = statusCode (responseStatus res)-            , sLocation = lookup hLocation . responseHeaders $ res+            , sLocation = location'              , sContentType = lookup hContentType . responseHeaders $ res             }           modify (st:)@@ -75,4 +92,14 @@ isFeed :: Status -> Bool isFeed Status{..} = isFeedContentType sContentType  +baseURL :: Request -> Maybe URI+baseURL req = +    let protocol = if secure req then "https://" else "http://"+    in parseURI $ mconcat [+                    protocol+                  , B.unpack . host $ req+                  , case port req of +                      80 -> "" +                      n -> ":" ++ show n+                  ] 

@@ -13,6 +13,7 @@       >>>       scrapePage )  +-- TODO fix relative links scrapePage =        deep (isElem >>> hasName "head") >>>       deep (isElem 

@@ -2,9 +2,9 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                feed-crawl-version:             0.1.0.0-synopsis:            Feed crawling utilities--- description:         +version:             0.1.1.0+synopsis:            Utility for fetching feeds with redirect info and HTML link detection+description:         Utility for fetching feeds with redirect info and HTML link detection  homepage:            https://github.com/danchoi/feed-crawl license:             MIT license-file:        LICENSE@@ -20,7 +20,7 @@   exposed-modules:     Text.Feed.Crawl                      , Text.Feed.Crawl.DetectLink                      , Text.Feed.Crawl.Common-  build-depends:       base >=4.7 && <4.8+  build-depends:       base >=4.6 && <4.8                      , http-conduit >= 2.1.5                      , conduit                      , connection@@ -29,6 +29,7 @@                      , transformers                      , http-types                      , hxt+                     , network-uri    -- hs-source-dirs:         default-language:    Haskell2010