shpider 0.0.3 → 0.0.4

raw patch · 4 files changed

+52/−10 lines, 4 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• Network/Shpider.hs +20/−7
• Network/Shpider/Options.hs +30/−0
• Network/Shpider/State.hs +1/−2
• shpider.cabal +1/−1

@@ -19,6 +19,7 @@    , sendForm    , getLinksByText    , getLinksByTextRegex+   , getLinksByAddressRegex    , getFormsByAction    , currentLinks    , currentForms@@ -92,17 +93,22 @@  curlDownload url = do    shpider <- get-   ( curlCode , html ) <- liftIO $ do-                             curlGetString url $ curlOpts shpider-   p <- parsePage url html+   res <- liftIO $ curlGetString url $ curlOpts shpider+   mkRes url res++mkRes url ( curlCode , html ) = do+   p <- if curlCode == CurlOK+           then+              parsePage url html+           else+              return emptyPage    return ( ccToSh curlCode , p ) + curlDownloadPost url fields = do    shpider <- get-   ( curlCode , html ) <- liftIO $ do-                             curlGetString url $ CurlPostFields ( map toPostField fields ) : curlOpts shpider-   p <- parsePage url html-   return ( ccToSh curlCode , p )+   res <- liftIO $ curlGetString url $ CurlPostFields ( map toPostField fields ) : curlOpts shpider+   mkRes url res   curlDownloadHead urlStr = do@@ -254,3 +260,10 @@                               fs          )          murl++-- | Get all links whose address matches this regex.+getLinksByAddressRegex :: String -> Shpider [ Link ]+getLinksByAddressRegex r = do+   cls <- currentLinks+   return $ filter ( flip (=~) r . linkAddress )+                   cls

@@ -3,6 +3,9 @@  import Data.Maybe +import Network.Shpider.Curl.Opts+import Network.Shpider.Curl.Types+ import Network.Shpider.State import Network.Shpider.URL import Network.Shpider.TextUtils@@ -14,6 +17,33 @@    put $ shpider { dontLeaveDomain =                       b                  }++-- | Set the CurlTimeout option.  Requests will TimeOut after this number of seconds.+setTimeOut :: Long -> Shpider ( )+setTimeOut s = do+   shpider <- get+   let isTimeout c =+          case c of+             ( CurlTimeout _ ) ->+                True+             _ ->+                False+       timeoutPresent =+          not $ null $ filter isTimeout $ curlOpts shpider+   put $ shpider { curlOpts =+                      if not timeoutPresent+                         then+                            CurlTimeout s : curlOpts shpider+                         else+                            map ( \ c ->+                                     if isTimeout c+                                        then+                                           CurlTimeout s+                                        else+                                           c+                                )+                                ( curlOpts shpider )+                 }    -- | Set the start page of your shpidering antics. -- The start page must be an absolute URL, if not, this will raise an error.

@@ -54,8 +54,7 @@    SS { startPage = ""       , htmlOnlyDownloads = False       , dontLeaveDomain = False-      , curlOpts = [ CurlTimeout 3-                   , CurlCookieFile "cookies"+      , curlOpts = [ CurlCookieFile "cookies"                    , CurlCookieJar "cookies"                    ]       , currentPage = emptyPage 

@@ -1,5 +1,5 @@ Name:               shpider -Version:            0.0.3+Version:            0.0.4 Synopsis:           Web automation library in Haskell. Description:     Shpider is a web automation library for Haskell.   It allows you to quickly write crawlers, and for simple cases ( like following links ) even without reading the page source.