crawlchain 0.1.1.5 → 0.1.1.7

raw patch · 5 files changed

+27/−16 lines, 5 files

Files

• crawlchain.cabal +2/−1
• src/Network/CrawlChain/BasicTemplates.hs +3/−4
• src/Network/CrawlChain/CrawlChain.hs +17/−11
• src/Network/CrawlChain/CrawlChains.hs +4/−0
• src/Network/CrawlChain/CrawlDirective.hs +1/−0

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                crawlchain-version:             0.1.1.5+version:             0.1.1.7 synopsis:            Simulation user crawl paths description:         Library for simulating user crawl paths (trees) with selectors - takes an initial action and a chain of processing actions to crawl a tree (lazy, depth first) searching for a matching branch. stability:           experimental@@ -44,3 +44,4 @@                      , tagsoup                      , time   default-language:    Haskell2010+  ghc-options:         -Wall

@@ -3,14 +3,11 @@   searchWebTemplateAndProcessHits   ) where -import Data.List (nub, isInfixOf, isSuffixOf)-import Data.List.Split (splitOn)-import Data.Maybe (maybeToList)+import Data.List (nub, isInfixOf)  import Network.CrawlChain.CrawlAction import Network.CrawlChain.CrawlResult import Network.CrawlChain.CrawlDirective-import Network.CrawlChain.CrawlingParameters import Text.HTML.CrawlChain.HtmlFiltering  -- generic search template@@ -49,8 +46,10 @@ filterToUrlsContainingText textFilter marker =   retainActionsContaining marker . extractLinksFilteringAll noUrlFilter noAttrFilter textFilter +{- unused - temporary? filterToUrlsContaining :: String -> String -> [CrawlAction] filterToUrlsContaining = filterToUrlsContainingText noTextFilter+-}  retainActionsContaining :: String -> [CrawlAction] -> [CrawlAction] retainActionsContaining marker = filter ((marker `isInfixOf`) . crawlUrl)

@@ -1,8 +1,8 @@-module Network.CrawlChain.CrawlChain-       (executeActions, crawlChain, crawlForUrl,-        executeCrawlChain -- visible for tests-       )-       where+module Network.CrawlChain.CrawlChain (+  crawlChain, crawlChains, -- primary interface+  executeActions, crawlForUrl, -- legacy, deprecated+  executeCrawlChain -- visible for tests+  ) where  import Data.List (intersperse) import Data.List.Split (splitOn)@@ -33,13 +33,19 @@        (Just _) -> putStrLn "POST result processing not implemented" >> return Nothing        Nothing -> return Nothing +{-|+ Returns only the first result of a completely matching branch of the crawling directive.+-} crawlChain :: CrawlingParameters -> IO (Maybe CrawlAction) crawlChain args = do-  results <- crawlingChain+  results <- crawlChains args   logAndReturnFirstOk results-  where-    crawlingChain :: IO [DirectiveChainResult]-    crawlingChain =++{-|+ Returns all possible results of the craling directive - meant to be used with lazyness in mind as needed.+-}+crawlChains :: CrawlingParameters -> IO [DirectiveChainResult]+crawlChains args =       executeCrawlChain context (paramInitialAction args) (paramCrawlDirective args)         where           context = if paramDoStore args then storingContext else defaultContext@@ -56,7 +62,7 @@ putDetailsOnFailure :: Maybe CrawlAction -> [DirectiveChainResult] -> IO () putDetailsOnFailure firstSuccess results =   case firstSuccess of-   Just _ -> return ()+   Just a -> putStr "  Using result: " >> print a    Nothing -> do-     putStrLn $ "no results found - details: " ++ showAllFailures where+     putStrLn $ "  No results found - details: " ++ showAllFailures where        showAllFailures = concat $ intersperse "\n\n" $ map showResultPath results

@@ -66,6 +66,10 @@         else return a1Results     followDirective' (RestartChainDirective restart) =       uncurry (followDirective collectedResults reportPath context) restart+    followDirective' (GuardDirective guard) =+      if guard crawlAction+      then putStrLn ("  guard: accepting "++(crawlUrl crawlAction)) >> return [DirectiveChainResult reportPath [crawlAction]]+      else putStrLn ("  guard: rejecting "++(crawlUrl crawlAction)) >> return []     -- actual crawling happens here (and only here):     crawlAndSearch :: (CrawlResult -> [CrawlAction]) -> IO [DirectiveChainResult]     crawlAndSearch searchLogic = crawler context crawlAction >>= processCrawlingResult

@@ -11,4 +11,5 @@   | RetryDirective Int CrawlDirective                  -- | if given directive yields no results use add. retries   | AlternativeDirective CrawlDirective CrawlDirective -- | fallback to second argument if first yields no results   | RestartChainDirective (CrawlAction, CrawlDirective)-- | the possibility to start a new chain (when using alternative)+  | GuardDirective (CrawlAction -> Bool)               -- | not crawling anything, just a blacklisting option   | DirectiveSequence [CrawlDirective]                 -- | chaining of directives