crawlchain 0.3.0.0 → 0.3.0.1

raw patch · 4 files changed

+10/−6 lines, 4 files

Files

crawlchain.cabal +3/−1
src/Network/CrawlChain/Crawling.hs +4/−4
src/Network/URI/Util.hs +1/−1
test/src/HtmlFilteringTests.hs +2/−0

crawlchain.cabal view

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                crawlchain-version:             0.3.0.0+version:             0.3.0.1 synopsis:            Simulation user crawl paths description:         Library for simulating user crawl paths (trees) with selectors - takes an initial action and a chain of processing actions to crawl a tree (lazy, depth first) searching for a matching branch. stability:           experimental@@ -52,9 +52,11 @@   main-is: HtmlFilteringTests.hs   other-modules:       Text.HTML.CrawlChain.HtmlFiltering                      , Network.CrawlChain.CrawlAction+                     , Network.URI.Util   hs-source-dirs:      src, test/src   build-depends:       base                      , crawlchain+                     , network-uri                      , split                      , tagsoup   default-language:    Haskell2010

src/Network/CrawlChain/Crawling.hs view

@@ -11,7 +11,7 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Network.Http.Client as C-import Network.URI (URI (..), parseURI)+import Network.URI (URI (..), parseURI, escapeURIString, isUnescapedInURI)  import Network.CrawlChain.CrawlAction import Network.CrawlChain.CrawlResult@@ -81,11 +81,11 @@     where       u = parseURL url where         parseURL :: C.URL -> URI-        parseURL r' =+        parseURL r' =  -- TODO use Network.URI.Util to have only one piece of code doing this           case parseURI r of           Just u'  -> u'-          Nothing -> error ("Can't parse URI " ++ r)-          where r = T.unpack $ T.decodeUtf8 r'+          Nothing -> error ("Can't parse URI - FIXME Crawling?: " ++ r)+          where r = escapeURIString isUnescapedInURI $ T.unpack $ T.decodeUtf8 r'       q = C.buildRequest1 $ do         C.http C.POST (path u)         C.setAccept $ BC.pack "*/*"

src/Network/URI/Util.hs view

@@ -4,4 +4,4 @@ import Network.URI  toURI :: String -> URI-toURI url = fromMaybe (error $ "invalid URI '" ++ url ++ "'") (parseURI url)+toURI url = fromMaybe (error $ "invalid URI - FIXME Network.URI.Util? '" ++ url ++ "'") (parseURI $ escapeURIString isUnescapedInURI url)

test/src/HtmlFilteringTests.hs view

@@ -3,6 +3,7 @@ import System.Exit (exitFailure)  import Network.CrawlChain.CrawlAction+import Network.URI.Util import Text.HTML.CrawlChain.HtmlFiltering  main :: IO ()@@ -21,6 +22,7 @@   match extractTagsContent "<div>sdf</div>" [("div", "sdf", [])]   match extractTagsContent "<div attr=\"1\">sdf</div>" [("div", "sdf", [("attr", "1")])]   match extractTagsContent "<script attr=\"some_url\">content</script><div>bla</div>" [("script", "content", [("attr","some_url")]),("div", "bla", [])]+  match toURI "http://sdf.sdf/α" (toURI "http://sdf.sdf/α")     where       match method input expected =         if actual /= expected