shpider 0.0.5 → 0.0.6
raw patch · 6 files changed
+79/−20 lines, 6 filesdep ~basedep ~urlPVP ok
version bump matches the API change (PVP)
Dependency ranges changed: base, url
API changes (from Hackage documentation)
Files
- Network/Shpider.hs +2/−1
- Network/Shpider/Curl/Curl.hs +46/−10
- Network/Shpider/Curl/Debug.hs +1/−1
- Network/Shpider/Curl/Easy.hs +8/−0
- Network/Shpider/Curl/Types.hs +19/−5
- shpider.cabal +3/−3
Network/Shpider.hs view
@@ -94,7 +94,8 @@ curlDownload url = do shpider <- get res <- liftIO $ curlGetString url $ curlOpts shpider- mkRes url res+ r <- mkRes url res+ return r mkRes url ( curlCode , html ) = do p <- if curlCode == CurlOK
Network/Shpider/Curl/Curl.hs view
@@ -175,7 +175,7 @@ -- | 'curlGet' perform a basic GET, dumping the output on stdout. -- The list of options are set prior performing the GET request. curlGet :: URLString -> [CurlOption] -> IO ()-curlGet url opts = initialize >>= \ h -> do+curlGet url opts = initialize_no_cleanup >>= \ h -> do setopt h (CurlFailOnError True) setopt h (CurlURL url) -- Note: later options may (and should, probably) override these defaults.@@ -201,7 +201,7 @@ curlGetString :: URLString -> [CurlOption] -> IO (CurlCode, String)-curlGetString url opts = initialize >>= \ h -> do+curlGetString url opts = initialize_no_cleanup >>= \ h -> do ref <- newIORef [] -- Note: later options may (and should, probably) override these defaults. setopt h (CurlFailOnError True)@@ -218,7 +218,7 @@ => URLString -> [CurlOption] -> IO (CurlCode, ty)-curlGetString_ url opts = initialize >>= \ h -> do+curlGetString_ url opts = initialize_no_cleanup >>= \ h -> do (finalBody, gatherBody) <- newIncoming setopt h (CurlFailOnError True) setDefaultSSLOpts h url@@ -254,7 +254,7 @@ -> [CurlOption] -> IO (CurlResponse_ hdr ty) curlGetResponse_ url opts = do- h <- initialize+ h <- initialize_no_cleanup -- Note: later options may (and should, probably) override these defaults. setopt h (CurlFailOnError True) setDefaultSSLOpts h url@@ -262,7 +262,7 @@ mapM_ (setopt h) opts -- note that users cannot over-write the body and header handler -- which makes sense because otherwise we will return a bogus reposnse.- perform_with_response_ h + manual_perform_with_response_ h {-# DEPRECATED curlGetResponse "Switch to using curlGetResponse_" #-} curlGetResponse :: URLString@@ -287,6 +287,42 @@ -- 'CurlWriteFunction' and 'CurlHeaderFunction' options. -- The returned payload is overloaded over the representation of -- both headers and body via the 'CurlResponse_' type.+-- Deterministic cleanup+manual_perform_with_response_ :: (CurlHeader headerTy, CurlBuffer bodyTy)+ => Curl+ -> IO (CurlResponse_ headerTy bodyTy)+manual_perform_with_response_ h = do+ (finalHeader, gatherHeader) <- newIncomingHeader+ (finalBody, gatherBody) <- newIncoming++ -- Instead of allocating a separate handler for each+ -- request we could just set this options one and forall+ -- and just clear the IORefs.++ setopt h (CurlWriteFunction (gatherOutput_ gatherBody))+ setopt h (CurlHeaderFunction (gatherOutput_ gatherHeader))+ rc <- perform h+ manual_cleanup h+ rspCode <- getResponseCode h+ (st,hs) <- finalHeader+ bs <- finalBody+ return CurlResponse+ { respCurlCode = rc+ , respStatus = rspCode+ , respStatusLine = st+ , respHeaders = hs+ , respBody = bs + -- note: we're holding onto the handle here..+ -- note: with this interface this is not neccessary.+ , respGetInfo = getInfo h+ }++-- | Perform the actions already specified on the handle.+-- Collects useful information about the returned message.+-- Note that this function sets the+-- 'CurlWriteFunction' and 'CurlHeaderFunction' options.+-- The returned payload is overloaded over the representation of+-- both headers and body via the 'CurlResponse_' type. perform_with_response_ :: (CurlHeader headerTy, CurlBuffer bodyTy) => Curl -> IO (CurlResponse_ headerTy bodyTy)@@ -301,7 +337,6 @@ setopt h (CurlWriteFunction (gatherOutput_ gatherBody)) setopt h (CurlHeaderFunction (gatherOutput_ gatherHeader)) rc <- perform h- manual_cleanup h rspCode <- getResponseCode h (st,hs) <- finalHeader bs <- finalBody@@ -316,6 +351,7 @@ , respGetInfo = getInfo h } + -- | Performs a curl request using an exisitng curl handle. -- The provided URL will overwride any 'CurlURL' options that -- are provided in the list of options. See also: 'perform_with_response'.@@ -339,7 +375,7 @@ -- | Get the headers associated with a particular URL. -- Returns the status line and the key-value pairs for the headers. curlHead :: URLString -> [CurlOption] -> IO (String,[(String,String)])-curlHead url opts = initialize >>= \ h -> +curlHead url opts = initialize_no_cleanup >>= \ h -> do ref <- newIORef [] -- setopt h (CurlVerbose True) setopt h (CurlURL url)@@ -357,7 +393,7 @@ => URLString -> [CurlOption] -> IO (String, headers)-curlHead_ url opts = initialize >>= \ h -> do+curlHead_ url opts = initialize_no_cleanup >>= \ h -> do (finalHeader, gatherHeader) <- newIncomingHeader -- setopt h (CurlVerbose True) setopt h (CurlURL url)@@ -396,7 +432,7 @@ -- | 'curlMultiPost' perform a multi-part POST submission. curlMultiPost :: URLString -> [CurlOption] -> [HttpPost] -> IO ()-curlMultiPost s os ps = initialize >>= \ h -> do+curlMultiPost s os ps = initialize_no_cleanup >>= \ h -> do setopt h (CurlVerbose True) setopt h (CurlURL s) setopt h (CurlHttpPost ps)@@ -409,7 +445,7 @@ -- | 'curlPost' performs. a common POST operation, namely that -- of submitting a sequence of name=value pairs. curlPost :: URLString -> [String] -> IO ()-curlPost s ps = initialize >>= \ h -> do+curlPost s ps = initialize_no_cleanup >>= \ h -> do setopt h (CurlVerbose True) setopt h (CurlPostFields ps) setopt h (CurlCookieJar "cookies")
Network/Shpider/Curl/Debug.hs view
@@ -15,7 +15,7 @@ import System.IO debugging :: Bool-debugging = False+debugging = False debug :: String -> IO () debug msg
Network/Shpider/Curl/Easy.hs view
@@ -18,6 +18,7 @@ -------------------------------------------------------------------- module Network.Shpider.Curl.Easy ( initialize -- :: IO Curl+ , initialize_no_cleanup -- :: IO Curl , perform -- :: Curl -> IO CurlCode , setopt -- :: Curl -> CurlOption -> IO CurlCode , duphandle -- :: Curl -> IO Curl@@ -50,6 +51,13 @@ initialize = do h <- easy_initialize mkCurl h ++-- | Initialize a curl instance+-- | No automatic garbage colelction+initialize_no_cleanup :: IO Curl+initialize_no_cleanup = do+ h <- easy_initialize+ mkCurl_no_cleanup h -- XXX: Is running cleanup here OK? reset :: Curl -> IO ()
Network/Shpider/Curl/Types.hs view
@@ -19,7 +19,7 @@ ( CurlH, URLString, Port, Long, LLong, Slist_ , Curl, curlPrim, mkCurl, mkCurlWithCleanup , OptionMap, shareCleanup, runCleanup, updateCleanup- , manual_cleanup+ , manual_cleanup , mkCurl_no_cleanup ) where import Network.Shpider.Curl.Debug@@ -59,8 +59,20 @@ -- | Allocates a Haskell handle from a C handle.+-- | No garbage collection so cookies are written deterministically+mkCurl_no_cleanup :: CurlH -> IO Curl+mkCurl_no_cleanup h = do+ debug "ALLOC: CURL"+ fh <- newForeignPtr_ h + v1 <- newMVar fh+ v2 <- newIORef om_empty + let new_h = Curl { curlH = v1, curlCleanup = v2 }+ return new_h++-- | Allocates a Haskell handle from a C handle. mkCurl :: CurlH -> IO Curl-mkCurl h = mkCurlWithCleanup h om_empty+mkCurl h = + mkCurlWithCleanup h om_empty -- | Allocates a Haskell handle from a C handle. mkCurlWithCleanup :: CurlH -> OptionMap -> IO Curl@@ -70,9 +82,11 @@ v1 <- newMVar fh v2 <- newIORef clean let new_h = Curl { curlH = v1, curlCleanup = v2 }- + fin <- mkIOfin $ do debug "FREE: CURL" runCleanup v2+ withForeignPtr fh easy_cleanup+ addForeignPtrFinalizer fin fh return new_h@@ -80,9 +94,9 @@ -- Manually cleanup a curl instance, writing cookies etc. manual_cleanup curl = do fh <- readMVar $ curlH curl+ debug "FREE: CURL"+ runCleanup $ curlCleanup curl withForeignPtr fh easy_cleanup- finalizeForeignPtr fh- -- Admin code for cleaning up marshalled data. -- Note that these functions assume that they are running atomically,
shpider.cabal view
@@ -1,5 +1,5 @@ Name: shpider -Version: 0.0.5+Version: 0.0.6 Synopsis: Web automation library in Haskell. Description: Shpider is a web automation library for Haskell. It allows you to quickly write crawlers, and for simple cases ( like following links ) even without reading the page source.@@ -17,7 +17,7 @@ > "occupation" =: "unemployed Haskell programmer" > "location" =: "mother's house" .- Shpider contains a patched version of the curl package, to fix cookie handling. The curl licence is therefore distributed with this package.+ Shpider contains a patched version of the curl package ( the original package's garbage-collection caused non-deterministic behaviour ). The curl licence is therefore distributed with this package. Category: Web License: BSD3 License-file: LICENSE@@ -29,7 +29,7 @@ Extra-source-files: LICENSE.CURL, configure, configure.ac, curl.buildinfo.in Library- Build-depends: base , regex-posix , tagsoup-parsec , url==2 , containers , tagsoup , mtl , bytestring+ Build-depends: base < 5, regex-posix , tagsoup-parsec , url>=2 , containers , tagsoup , mtl , bytestring Exposed-modules: Network.Shpider.URL Network.Shpider.Code Network.Shpider.State