packages feed

Webrexp 1.1 → 1.1.1

raw patch · 18 files changed

+663/−308 lines, 18 filesdep +aesondep +textdep +unordered-containersdep −AttoJsondep ~HTTPdep ~HaXmldep ~array

Dependencies added: aeson, text, unordered-containers, vector

Dependencies removed: AttoJson

Dependency ranges changed: HTTP, HaXml, array, base, bytestring, containers, directory, filepath, hxt, mtl, network, process, regex-pcre-builtin, transformers

Files

Text/Webrexp.hs view
@@ -1,11 +1,19 @@ {-# LANGUAGE ScopedTypeVariables #-}
 -- | Generic module for using Webrexp as a user.
+-- the main functions for the user are queryDocument to perform an in-memory
+-- evaluation, and evalWebRexpDepthFirst
 module Text.Webrexp ( 
+               -- * In memory evaluation
+                 ParseableType( .. )
+               , queryDocument
+               , queryDocumentM
+
                -- * Default evaluation
-                 evalWebRexp
-               , evalWebRexpDepthFirst 
+               , evalWebRexp
+               , evalWebRexpDepthFirst
                , parseWebRexp
                , evalParsedWebRexp
+               , executeParsedWebRexp 
 
                -- * Crawling configuration
                , Conf (..)
@@ -15,10 +23,14 @@ 
 import Control.Monad
 import Control.Monad.IO.Class
+import Control.Monad.ST
 import Text.Parsec
 import System.IO
 import System.Exit
 
+import Data.Array.ST
+import Data.Array.IO
+
 import Text.Webrexp.Exprtypes
 import Text.Webrexp.Parser( webRexpParser )
 
@@ -28,10 +40,12 @@ import Text.Webrexp.UnionNode
 import Text.Webrexp.DirectoryNode
 
+import Text.Webrexp.GraphWalker
 import Text.Webrexp.ResourcePath
 import Text.Webrexp.WebContext
-
 import Text.Webrexp.WebRexpAutomata
+import Text.Webrexp.Remote.MimeTypes
+import qualified Text.Webrexp.ProjectByteString as B
 
 data Conf = Conf
     { hammeringDelay :: Int
@@ -62,14 +76,50 @@     UnionNode (UnionNode  HxtNode  HaXmLNode)
               (UnionNode JsonNode DirectoryNode)
 
-type Crawled a =
-            WebCrawler CrawledNode ResourcePath a
+type Crawled a = WebCrawler IOArray CrawledNode ResourcePath a
+type MemoryCrawl s a = WebContextT (STArray s) CrawledNode ResourcePath (ST s) a
 
 initialState :: IO (EvalState CrawledNode ResourcePath)
 initialState = do
     node <- currentDirectoryNode 
     return . Node $ repurposeNode (UnionRight . UnionRight) node
 
+-- | Query a document in memory and retrieve the results, you can use it in combination
+-- to the quasiquoting facility to embed the webrexp in haskell :
+--
+-- > {-# LANGUAGE QuasiQuotes #-}
+-- > import Text.Webrexp
+-- > import Text.Webrexp.Quote
+-- > import qualified Data.ByteString.Char8 as B
+-- >
+-- > main :: IO ()
+-- > main = print $ queryDocument ParseableJson document [webrexpParse| some things [.] |]
+-- >     where document = B.pack "{ \"some\": { \"things\": \"a phrase\" } }"
+-- 
+-- The returned values contain possible errors as 'Left' and real value as 'Right.
+--
+queryDocument :: ParseableType -> B.ByteString -> WebRexp -> [Either String String]
+queryDocument docType str query = runST $ queryDocumentM docType str query
+
+-- | Exactly same thing as 'queryDocument', but in ST
+queryDocumentM :: forall s . ParseableType -> B.ByteString -> WebRexp 
+               -> ST s [Either String String]
+queryDocumentM docType str query = executeWithEmptyContext todo
+    where ignoreLog _ = return ()
+          loggers = (ignoreLog, ignoreLog, ignoreLog)
+          todo :: MemoryCrawl s Bool
+          todo = do
+            initialNode <- parseUnion loggers (Just docType) (Local "") str
+            case initialNode of
+                AccessError -> return False
+                DataBlob _ _ -> return False
+                Result rezPath a -> 
+                    let initNode = NodeContext { rootRef = rezPath
+                                               , this = a
+                                               , parents = ImmutableHistory [] }
+                    in do setLogLevel Quiet
+                          evalDepthFirst (Node initNode) query
+
 -- | Prepare a webrexp.
 -- This function is useful if the expression has
 -- to be applied many times.
@@ -81,16 +131,24 @@ 
 -- | Evaluation for pre-parsed webrexp.
 -- Best method if a webrexp has to be evaluated
--- many times.
+-- many times. Evaluated using breadth first method.
 evalParsedWebRexp :: WebRexp -> IO Bool
 evalParsedWebRexp wexpr = evalWithEmptyContext crawled
  where crawled :: Crawled Bool = evalBreadthFirst (Text "") wexpr
 
+-- | Evaluate a webrexp and return all the dumped text as 'Right'
+-- and all errors as 'Left'. Evaluated using depth first method.
+executeParsedWebRexp :: WebRexp -> IO [Either String String]
+executeParsedWebRexp wexpr = executeWithEmptyContext crawled
+ where crawled :: Crawled Bool = evalDepthFirst (Text "") wexpr
+
 -- | Simple evaluation function, evaluation is
 -- the breadth first type.
 evalWebRexp :: String -> IO Bool
 evalWebRexp = evalWebRexpWithEvaluator $ evalBreadthFirst (Text "")
 
+-- | Evaluate a webrexp in depth first fashion, returning a success
+-- status telling if the evaluation got up to the end.
 evalWebRexpDepthFirst :: String -> IO Bool
 evalWebRexpDepthFirst = evalWebRexpWithEvaluator $ evalDepthFirst (Text "")
 
@@ -108,6 +166,7 @@         let crawled :: Crawled Bool = evaluator wexpr
         in evalWithEmptyContext crawled
 
+-- | Function used in the command line program.
 evalWebRexpWithConf :: Conf -> IO Bool
 evalWebRexpWithConf conf =
   case runParser webRexpParser () "expr" (expr conf) of
Text/Webrexp/DirectoryNode.hs view
@@ -6,10 +6,11 @@                             ) where
 
 import Control.Exception
-import Control.Monad.IO.Class
+import Data.Maybe( fromMaybe )
 import System.Directory
 import System.FilePath
 
+import Text.Webrexp.IOMock
 import Text.Webrexp.GraphWalker
 import Text.Webrexp.ResourcePath
 import Text.Webrexp.UnionNode
@@ -97,12 +98,13 @@     indirectLinks (Directory _ _) = []
 
     accessGraph _ _ = return AccessError
+    rawAccess _ _ = return AccessError
 
     isHistoryMutable _ = True
 
     childrenOf (File _ _) = return []
     childrenOf (Directory (FullPath path) _) =
-        liftIO $ listDirectory path
+         performIO (listDirectory path) >>= return . fromMaybe []
Text/Webrexp/Eval.hs view
@@ -1,4 +1,5 @@ {-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE FlexibleContexts #-}
 module Text.Webrexp.Eval
     (
     -- * Functions
@@ -10,20 +11,27 @@ import Control.Monad
 import Data.List
 import Data.Maybe
+import Data.Array.MArray
+import qualified Data.Set as Set
 
+import Text.Webrexp.IOMock
 import Text.Webrexp.GraphWalker
 import Text.Webrexp.Exprtypes
 import Text.Webrexp.WebContext
 import Text.Webrexp.Eval.Action
-import Text.Webrexp.Log
 
+import qualified Text.Webrexp.ProjectByteString as B
+
 -- | Given a node search for valid children, check for their
 -- validity against the requirement.
-searchRefIn :: (GraphWalker node rezPath)
+searchRefIn :: ( GraphWalker node rezPath
+               , IOMockable (WebContextT array node rezPath m)
+               , Functor m
+               , Monad m )
             => Bool                         -- ^ Do we recurse?
             -> WebRef                       -- ^ Ref to find
             -> NodeContext node rezPath     -- ^ The root nood for the search
-            -> WebCrawler node rezPath
+            -> WebContextT array node rezPath m
                         [NodeContext node rezPath]   -- ^ The found nodes.
 searchRefIn False Wildcard n = do
     children <- childrenOf $ this n
@@ -66,9 +74,12 @@ -- | Evaluate the leaf nodes of a webrexp, this way the code
 -- can be shared between the Breadth first evaluator and the
 -- Depth first one.
-evalWebRexpFor :: (GraphWalker node rezPath)
+evalWebRexpFor :: ( GraphWalker node rezPath
+                  , IOMockable (WebContextT array node rezPath m)
+                  , Functor m
+                  , MArray array (Set.Set String) m)
                => WebRexp -> EvalState node rezPath
-               -> WebCrawler node rezPath (Bool, [EvalState node rezPath])
+               -> WebContextT array node rezPath m (Bool, [EvalState node rezPath])
 evalWebRexpFor (Str str) _ = do
     debugLog "> '\"...\"'"
     return (True, [Text str])
@@ -130,6 +141,11 @@     e' <- diggLinks e
     return (not $ null e', e')
 
+evalWebRexpFor DumpLink e = do
+    debugLog "> ->"
+    dumpLinks e
+    return (True, [])
+
 evalWebRexpFor NextSibling e = do
   debugLog "> '+'"
   subs <- siblingAccessor 1 e 
@@ -173,9 +189,12 @@ evalWebRexpFor (Range _ _) _ =
      error "evalWebRexpFor - non terminal in terminal function."
 
-downLinks :: (GraphWalker node rezPath)
+downLinks :: ( GraphWalker node rezPath
+             , IOMockable (WebContextT array node rezPath m)
+             , Functor m, Monad m
+             )
           => rezPath
-          -> WebCrawler node rezPath [EvalState node rezPath]
+          -> WebContextT array node rezPath m [EvalState node rezPath]
 downLinks path = do
     loggers <- prepareLogger
     down <- accessGraph loggers path
@@ -193,22 +212,75 @@ --------------------------------------------------
 ----            Helper functions
 --------------------------------------------------
-diggLinks :: (GraphWalker node rezPath)
+diggLinks :: (GraphWalker node rezPath
+             ,IOMockable (WebContextT array node rezPath m)
+             ,Functor m, Monad m)
           => EvalState node rezPath
-          -> WebCrawler node rezPath [EvalState node rezPath]
+          -> WebContextT array node rezPath m [EvalState node rezPath]
 diggLinks (Node n) =
     concat <$> sequence
             [ downLinks $ rootRef n <//> indir
                                 | indir <- indirectLinks $ this n ]
+
 diggLinks (Text str) = case importPath str of
         Nothing -> return []
         Just p -> downLinks p
 diggLinks _ = return []
 
+
+dumpLinks :: forall array node rezPath m.
+             ( GraphWalker node rezPath
+             , IOMockable (WebContextT array node rezPath m)
+             , Functor m, Monad m )
+          => EvalState node rezPath -> WebContextT array node rezPath m ()
+dumpLinks (Node n) = mapM_ downLink links
+    where indirs = indirectLinks $ this n
+          links = [ rootRef n <//> indir | indir <- indirs ]
+          downLink lnk = do
+              loggers@(normalLog, _, _) <- prepareLogger
+              down <- (rawAccess :: Loggers (WebContextT array node rezPath m)
+                                 -> rezPath -> WebContextT array node rezPath m 
+                                        (AccessResult node rezPath)) loggers lnk
+              case down of
+                DataBlob _ b -> do
+                    let localized = localizePath lnk
+                    dumpPath <- if not $ null localized
+                        then return localized
+                        else getUniqueName
+
+                    normalLog $ "\nDumping at path \"" ++ dumpPath  ++ "\""
+                    _ <- performIO $ B.writeFile dumpPath b
+                    return ()
+
+                _ -> return ()
+
+dumpLinks (Text str) = case importPath str of
+    Nothing -> return ()
+    Just lnk -> do
+        loggers@(normalLog, _, _) <- prepareLogger
+        down <- (rawAccess :: Loggers (WebContextT array node rezPath m)
+                            -> rezPath -> WebContextT array node rezPath m 
+                                (AccessResult node rezPath)) loggers lnk
+        case down of
+            DataBlob _ b -> do
+                let localized = localizePath lnk
+                dumpPath <- if not $ null localized
+                    then return localized
+                    else getUniqueName
+                normalLog $ "Dumping at path \"" ++ dumpPath  ++ "\""
+                _ <- performIO $ B.writeFile dumpPath b
+                return ()
+
+            _ -> return ()
+
+dumpLinks _ = return ()
+
 -- | Let access sibling nodes with a predefined index.
-siblingAccessor :: (GraphWalker node rezPath)
+siblingAccessor :: ( GraphWalker node rezPath
+                   , IOMockable (WebContextT array node rezPath m) 
+                   , Monad m )
                 => Int -> EvalState node rezPath
-                -> WebCrawler node rezPath
+                -> WebContextT array node rezPath m
                              (Maybe (EvalState node rezPath))
 siblingAccessor 0   node@(Node _) = return $ Just node
 siblingAccessor idx (Node node)=
Text/Webrexp/Eval/Action.hs view
@@ -1,10 +1,10 @@+{-# LANGUAGE FlexibleContexts #-}
 module Text.Webrexp.Eval.Action( evalAction 
                           , dumpActionVal
                           , isActionResultValid ) where
 
 import Control.Applicative
 import Control.Monad
-import Control.Monad.IO.Class
 import Data.List
 import Text.Regex.PCRE
 
@@ -12,16 +12,20 @@ import Text.Webrexp.Exprtypes
 import Text.Webrexp.WebContext
 import Text.Webrexp.Eval.ActionFunc
+import Text.Webrexp.IOMock
 
-import Text.Webrexp.Log
 import qualified Text.Webrexp.ProjectByteString as B
 
-binArith :: (GraphWalker node rezPath)
+binArith :: ( GraphWalker node rezPath
+            , IOMockable (WebContextT array node rezPath m)
+            , Functor m
+            , Monad m)
          => (ActionValue -> ActionValue -> ActionValue) -- ^ Function to cal result
          -> Maybe (EvalState node rezPath) -- Actually evaluated element
          -> ActionExpr       -- Left subaction (tree-like)
          -> ActionExpr      -- Right subaction (tree-like)
-         -> WebCrawler node rezPath (ActionValue, Maybe (EvalState node rezPath))
+         -> WebContextT array node rezPath m 
+                        (ActionValue, Maybe (EvalState node rezPath))
 binArith _ Nothing _ _ = return (ATypeError, Nothing)
 binArith f e sub1 sub2 = do
     (v1,e') <- evalAction sub1 e
@@ -67,15 +71,19 @@ isActionResultValid ATypeError = False
 isActionResultValid _ = True
 
-dumpActionVal :: ActionValue -> WebCrawler node rezPath ()
+dumpActionVal :: (IOMockable (WebContextT array node rezPath m), Monad m)
+              => ActionValue -> WebContextT array node rezPath m ()
 dumpActionVal (AString s) = textOutput s
 dumpActionVal (AInt i) = textOutput $ show i
 dumpActionVal _ = return ()
 
-dumpContent :: (GraphWalker node rezPath)
+dumpContent :: ( GraphWalker node rezPath
+               , IOMockable (WebContextT array node rezPath m)
+               , Functor m
+               , Monad m)
             => Bool     -- ^ If we convert recursively data.
             -> Maybe (EvalState node rezPath)   -- ^ Node to be dumped
-            -> WebCrawler node rezPath (ActionValue, Maybe (EvalState node rezPath))
+            -> WebContextT array node rezPath m (ActionValue, Maybe (EvalState node rezPath))
 dumpContent _ Nothing = return (ABool False, Nothing)
 dumpContent recursive e@(Just (Node ns)) =
   case (indirectLinks (this ns), recursive) of
@@ -90,15 +98,18 @@ dumpContent _ e@(Just (Blob b)) = do
     (norm, _, _) <- prepareLogger
     let filename = localizePath $ sourcePath b
-    liftIO . norm $ "Dumping blob in " ++ filename
-    liftIO $ B.writeFile filename (blobData b)
+    norm $ "Dumping blob in " ++ filename
+    _ <- performIO $ B.writeFile filename (blobData b)
     return (ABool True, e)
 
 -- | Evaluate embedded action in WebRexp
-evalAction :: (GraphWalker node rezPath)
+evalAction :: ( GraphWalker node rezPath
+              , IOMockable (WebContextT array node rezPath m)
+              , Functor m
+              , Monad m )
            => ActionExpr
            -> Maybe (EvalState node rezPath)
-           -> WebCrawler node rezPath
+           -> WebContextT array node rezPath m
                         (ActionValue, Maybe (EvalState node rezPath))
 evalAction (ActionExprs actions) e = do
     rez <- foldM eval (ABool True, e) actions
@@ -124,6 +135,12 @@          AString s -> return (ABool True, Just $ Text s)
          ATypeError -> return (val, el)
          
+evalAction NodeNameOutputAction el@(Just (Node e)) =
+    return (maybe (AString "") AString . nameOf $ this e, el)
+
+evalAction NodeNameOutputAction (Just _) = return (ATypeError, Nothing)
+evalAction NodeNameOutputAction Nothing = return (ATypeError, Nothing)
+
 evalAction (CstI i) n = return (AInt i, n)
 evalAction (CstS s) n = return (AString s, n)
 evalAction OutputAction e = dumpContent False e
@@ -181,10 +198,12 @@ evalAction (Call BuiltinSubsitute subs) e = actionFunEval substituteFunc subs e
 evalAction (Call BuiltinSystem subs) e = actionFunEvalM funcSysCall subs e
 
-actionFunEval :: (GraphWalker node rezPath)
+actionFunEval :: ( GraphWalker node rezPath, IOMockable (WebContextT array node rezPath m)
+                 , Functor m
+                 , Monad m )
               => ActionFunc node rezPath
               -> [ActionExpr] -> Maybe (EvalState node rezPath)
-              -> WebCrawler node rezPath
+              -> WebContextT array node rezPath m
                           (ActionValue, Maybe (EvalState node rezPath))
 actionFunEval f actions st =  do
     vals <- mapM (`evalAction` st) actions
@@ -194,10 +213,13 @@        else return (ATypeError, Nothing)
 
 
-actionFunEvalM :: (GraphWalker node rezPath)
-               => ActionFuncM node rezPath
+actionFunEvalM :: ( GraphWalker node rezPath
+                  , IOMockable (WebContextT array node rezPath m)
+                  , Functor m
+                  , Monad m )
+               => ActionFuncM array node rezPath m
                -> [ActionExpr] -> Maybe (EvalState node rezPath)
-               -> WebCrawler node rezPath
+               -> WebContextT array node rezPath m
                           (ActionValue, Maybe (EvalState node rezPath))
 actionFunEvalM f actions st = do
     vals <- mapM (`evalAction` st) actions
Text/Webrexp/Eval/ActionFunc.hs view
@@ -1,4 +1,5 @@ {-# LANGUAGE PatternGuards #-}
+{-# LANGUAGE FlexibleContexts #-}
 module Text.Webrexp.Eval.ActionFunc( ActionValue(..)
                               , ActionFunc
                               , ActionFuncM
@@ -13,13 +14,13 @@                               ) where
 
 import Control.Applicative
-import Control.Monad.IO.Class
 import Data.Char
 import System.Process
 import System.Exit
 
 import Debug.Trace
 
+import Text.Webrexp.IOMock
 import Text.Webrexp.WebContext
 
 -- | Data used for the evaluation of actions. Represent the
@@ -38,10 +39,10 @@      -> Maybe (EvalState node rezPath) -- ^ Pipeline argument
      -> (ActionValue, Maybe (EvalState node rezPath)) -- ^ Result
 
-type ActionFuncM node rezPath
+type ActionFuncM array node rezPath m
      = [ActionValue]                 -- ^ Argument list
      -> Maybe (EvalState node rezPath) -- ^ Pipeline argument
-     -> WebCrawler node rezPath
+     -> WebContextT array node rezPath m
                     (ActionValue, Maybe (EvalState node rezPath)) -- ^ Result
 
 -- | Typecast operation, from :
@@ -159,11 +160,12 @@     (AString $ substitute s what by, e)
 substituteFunc _ e = (ATypeError, e)
 
-funcSysCall :: ActionFuncM node rezPath
+funcSysCall :: (IOMockable (WebContextT array node rezPath m), Monad m)
+            => ActionFuncM array node rezPath m
 funcSysCall [AString s] e = do
-    code <- liftIO $ system s
+    code <- performIO $ system s
     case code of
-         ExitSuccess -> return (ABool True, e)
+         Just ExitSuccess -> return (ABool True, e)
          _ -> return (ABool False, e)
 funcSysCall _ e = return (ATypeError, e)
Text/Webrexp/Exprtypes.hs view
@@ -165,6 +165,9 @@     -- | Translate a node and all it's children into text.
     | DeepOutputAction
 
+    -- | Retrieve a node name
+    | NodeNameOutputAction
+
     -- | funcName(..., ...)
     | Call BuiltinFunc [ActionExpr]
     deriving (Eq, Show)
@@ -219,6 +222,11 @@     -- to follow hyper link
     | DiggLink
 
+    -- | \'->\' operator in the language, used to
+    -- follow hyper link and dump the resulting
+    -- content on hard drive (if permited).
+    | DumpLink
+
     -- | \'+\' operator in the language, used
     -- to select the next sibling node.
     | NextSibling
@@ -278,6 +286,7 @@ foldWebRexp f acc e@(Range _ _) = f acc e
 foldWebRexp f acc e@(Ref _) = f acc e
 foldWebRexp f acc e@DiggLink = f acc e
+foldWebRexp f acc e@DumpLink = f acc e
 foldWebRexp f acc e@NextSibling = f acc e
 foldWebRexp f acc e@PreviousSibling = f acc e
 foldWebRexp f acc e@Parent = f acc e
@@ -378,6 +387,7 @@ instance Lift ActionExpr where
     lift OutputAction = [| OutputAction |]
     lift DeepOutputAction = [| DeepOutputAction |]
+    lift NodeNameOutputAction = [| NodeNameOutputAction |]
     lift (ActionExprs lst) = [| ActionExprs lst |]
     lift (ARef str) = [| ARef str |]
     lift (CstI i) = [| CstI i |]
Text/Webrexp/GraphWalker.hs view
@@ -25,8 +25,8 @@     , findFirstNamedPure
     ) where
 
+import Text.Webrexp.IOMock
 import Control.Applicative
-import Control.Monad.IO.Class
 import qualified Text.Webrexp.ProjectByteString as B
 
 -- | Represent the path used to find the node
@@ -35,10 +35,10 @@ 
 -- | Type used to propagate different logging
 -- level across the software.
-type Logger = String -> IO ()
+type Logger m = String -> m ()
 
 -- | Normal/Err/verbose loggers.
-type Loggers = (Logger, Logger, Logger)
+type Loggers m = (Logger m, Logger m, Logger m)
 
 -- | Represent indirect links or links which
 -- necessitate the use of the IO monad to walk
@@ -55,8 +55,8 @@ 
     -- | Move semantic, try to dump the pointed
     -- resource to the current folder.
-    dumpDataAtPath :: (Monad m, MonadIO m)
-                   => Loggers -> a
+    dumpDataAtPath :: (Monad m, IOMockable m)
+                   => Loggers m -> a
                    -> m ()
 
     -- | Given a graphpath, transform it to
@@ -94,7 +94,7 @@ 
     -- | Get all the children of the current
     -- node.
-    childrenOf :: (MonadIO m) => a -> m [a]
+    childrenOf :: (IOMockable m, Monad m) => a -> m [a]
 
     -- | Retrieve the value of the tag (textual)
     valueOf :: a -> String
@@ -109,9 +109,14 @@     -- resource, so an updated name can be given.
     -- The given function is there to log information,
     -- the second is to log errors
-    accessGraph :: (MonadIO m, Functor m)
-                => Loggers -> rezPath -> m (AccessResult a rezPath)
+    accessGraph :: (IOMockable m, Functor m, Monad m)
+                => Loggers m -> rezPath -> m (AccessResult a rezPath)
 
+    -- | Same as accessGraph, but don't try to return
+    -- a structured result, only blobs.
+    rawAccess :: (IOMockable m, Functor m, Monad m)
+              => Loggers m -> rezPath -> m (AccessResult a rezPath)
+
     -- | Tell if the history associated is fixed or not.
     -- If the history is not fixed and can change (if you
     -- are querying the filesystem for example, it should
@@ -120,14 +125,14 @@ 
     -- | Like value of, but force the node to collect the
     -- value of all it's children in the process.
-    deepValueOf :: (MonadIO m, Functor m) => a -> m String
+    deepValueOf :: (IOMockable m, Functor m, Monad m) => a -> m String
     deepValueOf node = (valueOf node ++) <$> childrenText
         where childrenText = concat <$> (childrenOf node >>= mapM deepValueOf)
 
 -- | Return a list of all the "children"/linked node of a given node.
 -- The given node is not included in the list.
 -- A list of node with the taken path is returned.
-descendants :: (MonadIO m, GraphWalker a r) => a -> m [(a, [(a, Int)])]
+descendants :: (IOMockable m, Monad m, GraphWalker a r) => a -> m [(a, [(a, Int)])]
 descendants node = findDescendants (node, [])
    where findDescendants (a, hist) = do
              children <- childrenOf a
@@ -144,7 +149,7 @@ -- the returned list must contain : the node itself if
 -- it match the name, and all the children containing the
 -- good name.
-findNamed :: (Functor m, MonadIO m, GraphWalker a r)
+findNamed :: (Functor m, Monad m, IOMockable m, GraphWalker a r)
           => String -> a -> m [(a, [(a, Int)])]
 findNamed name node = if nameOf node == Just name
                          then ((node, []) :) <$> validChildren
@@ -153,7 +158,7 @@                        <$> descendants node
 
 -- | Return the first found node if any.
-findFirstNamed :: (Functor m, MonadIO m, GraphWalker a r)
+findFirstNamed :: (Functor m, Monad m, IOMockable m, GraphWalker a r)
                => String -> [a] -> m (Maybe (a, [(a,Int)]))
 findFirstNamed name lst = do
     nameList <- mapM (findNamed name) lst
Text/Webrexp/HaXmlNode.hs view
@@ -1,10 +1,10 @@ {-# LANGUAGE TypeSynonymInstances #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE FlexibleInstances #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 module Text.Webrexp.HaXmlNode( HaXmLNode ) where
 
 import Control.Applicative
-import Control.Monad.IO.Class
 import Data.Maybe( catMaybes )
 import Data.List( find )
 import Network.HTTP
@@ -17,6 +17,7 @@ 
 import qualified Text.Webrexp.ProjectByteString as B
 
+import Text.Webrexp.IOMock
 import Text.Webrexp.GraphWalker
 import Text.Webrexp.ResourcePath
 import Text.Webrexp.Remote.MimeTypes
@@ -63,6 +64,8 @@        (CString _ txt _:_) -> txt
        _ -> ""
 
+    rawAccess = accessResourcePath
+
 pureChildren :: HaXmLNode -> [HaXmLNode]
 pureChildren (CElem (Elem _ _ children) _) = children
 pureChildren _ = []
@@ -81,20 +84,21 @@ parserOfKind (Just ParseableJson) datapath = DataBlob datapath
 
 -- | Given a resource path, do the required loading
-loadHtml :: (MonadIO m)
-         => Loggers -> ResourcePath
+loadHtml :: (Monad m, IOMockable m)
+         => Loggers m -> ResourcePath
          -> m (AccessResult HaXmLNode ResourcePath)
 loadHtml (logger, _errLog, _verbose) datapath@(Local s) = do
-    liftIO . logger $ "Opening file : '" ++ s ++ "'"
-    realFile <- liftIO $ doesFileExist s
-    if not realFile
-       then return AccessError
-       else do file <- liftIO $ B.readFile s
-       	       let kind = getParseKind s
-       	       return $ parserOfKind kind datapath file
+    logger $ "Opening file : '" ++ s ++ "'"
+    realFile <- performIO $ doesFileExist s
+    case realFile of
+        Just True -> performIO (B.readFile s) >>=
+            maybe (return AccessError)
+                  (let kind = getParseKind s 
+                   in return . parserOfKind kind datapath)
+        _         -> return AccessError
 
 loadHtml loggers@(logger, _, verbose) datapath@(Remote uri) = do
-  liftIO . logger $ "Downloading URL : '" ++ show uri ++ "'"
+  logger $ "Downloading URL : '" ++ show uri ++ "'"
   (u, rsp) <- downloadBinary loggers uri
   let contentType = retrieveHeaders HdrContentType rsp
   case contentType of
@@ -104,6 +108,6 @@                         ++ hdrValue hdr ++ "] "
            
        	   kind = getParseKind (uriPath uri)
-       in do liftIO $ verbose logString
+       in do verbose logString
              return . parserOfKind kind datapath $ rspBody rsp
Text/Webrexp/HxtNode.hs view
@@ -1,11 +1,11 @@ {-# LANGUAGE TypeSynonymInstances #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE FlexibleInstances #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 -- | This module implement the GraphWalker type class
 -- for node of HxT (Haskell Xml parser).
 module Text.Webrexp.HxtNode ( HxtNode ) where
 
-import Control.Monad.IO.Class
 import Data.Maybe
 import Data.Tree.NTree.TypeDefs
 import Network.HTTP
@@ -16,6 +16,7 @@ 
 import qualified Text.Webrexp.ProjectByteString as B
 
+import Text.Webrexp.IOMock
 import Text.Webrexp.GraphWalker
 import Text.Webrexp.ResourcePath
 import Text.Webrexp.Remote.MimeTypes
@@ -43,6 +44,7 @@     valueOf = valueOfNode
     nameOf = getName
     indirectLinks = hyperNode
+    rawAccess = accessResourcePath
 
 deepValue :: HxtNode -> String
 deepValue (NTree (XText txt) _) = txt
@@ -93,20 +95,20 @@ parserOfKind _ datapath = DataBlob datapath
 
 -- | Given a resource path, do the required loading
-loadHtml :: (MonadIO m)
-         => Loggers -> ResourcePath
+loadHtml :: (Monad m, IOMockable m)
+         => Loggers m -> ResourcePath
          -> m (AccessResult HxtNode ResourcePath)
 loadHtml (logger, _errLog, _verbose) (Local s) = do
-    liftIO . logger $ "Opening file : '" ++ s ++ "'"
-    realFile <- liftIO $ doesFileExist s
-    if not realFile
-       then return AccessError
-       else do file <- liftIO $ readFile s
-       	       return . Result (Local s)
-                      $ parseToHTMLNode file
+    logger $ "Opening file : '" ++ s ++ "'"
+    realFile <- performIO $ doesFileExist s
+    case realFile of
+        Just True -> performIO (readFile s) >>=
+            maybe (return AccessError)
+                  (return . Result (Local s) . parseToHTMLNode)
+        _         -> return AccessError
 
 loadHtml loggers@(logger, _,  verbose) datapath@(Remote uri) = do
-  liftIO . logger $ "Downloading URL : '" ++ show uri ++ "'"
+  logger $ "Downloading URL : '" ++ show uri ++ "'"
   (u, rsp) <- downloadBinary loggers uri
   let contentType = retrieveHeaders HdrContentType rsp
   case contentType of
@@ -116,6 +118,6 @@                         ++ hdrValue hdr ++ "] "
            
        	   kind = getParseKind (uriPath uri)
-       in do liftIO $ verbose logString
+       in do verbose logString
              return . parserOfKind kind datapath $ rspBody rsp
Text/Webrexp/JsonNode.hs view
@@ -1,45 +1,58 @@ {-# LANGUAGE TypeSynonymInstances #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE FlexibleInstances #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 module Text.Webrexp.JsonNode( JsonNode ) where
 
 import Control.Arrow
 import Control.Applicative
-import Control.Monad.IO.Class
 import Data.Maybe( catMaybes )
-import qualified Data.Map as Map
+import qualified Data.HashMap.Strict as Map
 import Network.HTTP
 import System.Directory
-import Text.JSON.AttoJSON
+import Data.Aeson( decode
+                 , Value( Object
+                        , Array
+                        , String
+                        , Number
+                        , Bool
+                        , Null)
+                 )
 
+import qualified Data.Text as T
+import qualified Data.Vector as V
 import qualified Text.Webrexp.ProjectByteString as B
+import qualified Data.ByteString.Lazy.Char8 as L
 
+import Text.Webrexp.IOMock
 import Text.Webrexp.GraphWalker
 import Text.Webrexp.ResourcePath
 import Text.Webrexp.UnionNode
 import Text.Webrexp.Remote.MimeTypes
 
-type JsonNode = (Maybe String, JSValue)
+type JsonNode = (Maybe String, Value)
 
 instance PartialGraph JsonNode ResourcePath where
     isResourceParseable _ _ ParseableJson = True
     isResourceParseable _ _ _ = False
 
-    parseResource _ _ ParseableJson binData = return $ (,) Nothing <$> readJSON binData
+    parseResource _ _ ParseableJson binData =
+        return $ (,) Nothing <$> decode (L.fromChunks [binData])
     parseResource _ _ _ _ = error "Wrong kind of parser used"
 
 instance GraphWalker JsonNode ResourcePath where
     accessGraph = loadJson
+    rawAccess = accessResourcePath
 
-    attribOf attrName (_, JSObject obj) =
-        valueOf . none <$> Map.lookup (B.pack attrName) obj
+    attribOf attrName (_, Object obj) =
+        valueOf . none <$> Map.lookup (T.pack attrName) obj
             where none a = (Nothing :: Maybe String, a)
     attribOf _ _ = Nothing
 
-    childrenOf (_, JSArray children) =
-        return $ (,) Nothing <$> children
-    childrenOf (_, JSObject obj) =
-        return $ first (Just . B.unpack) <$> Map.assocs obj
+    childrenOf (_, Array children) =
+        return $ (,) Nothing <$> V.toList children
+    childrenOf (_, Object obj) =
+        return $ first (Just . T.unpack) <$> Map.toList obj
     childrenOf _ = return []
 
     nameOf (Just s, _) = Just s
@@ -51,42 +64,43 @@ 
     isHistoryMutable _ = False
 
-    valueOf (_, JSString s) = B.unpack s
-    valueOf (_, JSNumber i) = show i
-    valueOf (_, JSBool b) = show b
-    valueOf (_, JSArray _) = ""
-    valueOf (_, JSObject _) = ""
-    valueOf (_, JSNull) = ""
+    valueOf (_, String s) = T.unpack s
+    valueOf (_, Number i) = show i
+    valueOf (_, Bool b) = show b
+    valueOf (_, Array _) = ""
+    valueOf (_, Object _) = ""
+    valueOf (_, Null) = ""
 
-parseJson :: (MonadIO m)
-          => Loggers -> ResourcePath -> B.ByteString
+parseJson :: (Monad m) => Loggers m -> ResourcePath -> B.ByteString
           -> m (AccessResult JsonNode ResourcePath)
 parseJson (_, errLog, _) datapath file =
-    case parseJSON file of
-      Left err -> do liftIO . errLog $ "> JSON Parsing error " ++ err
-       	             return AccessError
-      Right valid -> return $ Result datapath (Nothing, valid)
+    case decode $ L.fromChunks [file] of
+      Nothing    -> do errLog "> JSON Parsing error"
+                       return AccessError
+      Just valid -> return $ Result datapath (Nothing, valid)
 
 -- | Given a resource path, do the required loading
-loadJson :: (MonadIO m)
-         => Loggers -> ResourcePath
+loadJson :: (IOMockable m, Monad m)
+         => Loggers m -> ResourcePath
          -> m (AccessResult JsonNode ResourcePath)
 loadJson loggers@(logger, _errLog, _verbose) datapath@(Local s) = do
-    liftIO . logger $ "Opening file : '" ++ s ++ "'"
-    realFile <- liftIO $ doesFileExist s
-    if not realFile
-       then return AccessError
-       else do file <- liftIO $ B.readFile s
-       	       parseJson loggers datapath file
+    logger $ "Opening file : '" ++ s ++ "'"
+    realFile <- performIO $ doesFileExist s
+    case realFile of
+        Just True -> performIO (B.readFile s) >>=
+                maybe (return AccessError)
+                      (\file -> parseJson loggers datapath file)
 
+        _         -> return AccessError
+
 loadJson loggers@(logger, _, verbose) datapath@(Remote uri) = do
-  liftIO . logger $ "Downloading URL : '" ++ show uri ++ "'"
+  logger $ "Downloading URL : '" ++ show uri ++ "'"
   (u, rsp) <- downloadBinary loggers uri
   let contentType = retrieveHeaders HdrContentType rsp
   case contentType of
     [] -> return AccessError
     (hdr:_) ->
-       do liftIO . verbose $ "Downloaded (" ++ show u ++ ") ["
+       do verbose $ "Downloaded (" ++ show u ++ ") ["
                                 ++ hdrValue hdr ++ "] "
           parseJson loggers datapath $ rspBody rsp
− Text/Webrexp/Log.hs 
@@ -1,35 +0,0 @@-
--- | Implementation of the (rather basic) logging facility. 
---
--- Avoid using 'putStrLn' or 'putStr' in the project, favor
--- using this module.
-module Text.Webrexp.Log ( 
-      debugLog
-    , textOutput 
-    ) where
-
-import System.IO
-import Control.Exception( IOException )
-import qualified Control.Exception as Ex
-import Control.Monad
-import Control.Monad.IO.Class
-import Text.Webrexp.WebContext
-
--- | Debugging function, only displayed in verbose
--- logging mode.
-debugLog :: String -> WebCrawler node rezPath ()
-debugLog str = do
-    verb <- isVerbose
-    when verb (liftIO $ putStrLn str)
-
-
--- | If a webrexp output some text, it must go through
--- this function. It ensure the writting in the correct
--- file.
-textOutput :: String -> WebCrawler node rezPath ()
-textOutput str = do
-    handle <- getOutput
-    liftIO $ Ex.catch (hPutStr handle str)
-             (\e -> hPutStrLn stderr $ "Writing error : " ++ 
-                                       show (e :: IOException))
-
Text/Webrexp/Parser.hs view
@@ -49,7 +49,7 @@          (haskellStyle { P.reservedOpNames = [ "&", "|", "<", ">"
                                              , "*", "/", "+", "-"
                                              , "^", "=", "!", ":"
-                                             , "_", "$", "~"
+                                             , "_", "$", "~", "?"
                                              ]
                        , P.identStart = letter
                        } )
@@ -121,6 +121,7 @@ 
 webrexpOp :: Parsed st WebRexp
 webrexpOp = (DiggLink <$ string ">>")
+         <|> (DumpLink <$ string "->")
          <|> (PreviousSibling <$ char '~')
          <|> (NextSibling <$ char '+')
          <|> (Parent <$ char '<')
@@ -186,6 +187,7 @@           <|> parens actionExpr
           <|> (CstS <$> stringLiteral)
           <|> (OutputAction <$ char '.')
+          <|> (NodeNameOutputAction <$ char '?')
           <|> (DeepOutputAction <$ char '#')
           <|> actionCall
           <?> "actionTerm"
Text/Webrexp/Quote.hs view
@@ -9,9 +9,15 @@ import Text.Webrexp
 import Text.Webrexp.WebRexpAutomata 
 
--- | QuasiQuotation to transform a wabrexp to
--- it's AST representation, resulting type is
--- :: WebRexp 
+-- | QuasiQuotation to transform a webrexp to
+-- it's AST representation, resulting type is :: Webrexp.
+-- You can use it the following way :
+--
+-- > {-# LANGUAGE QuasiQuotes #-}
+-- > import Text.Webrexp.Quote
+-- >
+-- > [webrexpParse| some webrexp [.] |]
+--
 webrexpParse :: QuasiQuoter
 webrexpParse = QuasiQuoter 
         { quoteExp = parser
@@ -25,6 +31,14 @@     Nothing -> fail "Invalid webrexp syntax"
     Just w -> lift w
 
+-- | Quasi quote to transform a webrexp into it's compiled representation.
+-- You can use it the following way :
+--
+-- > {-# LANGUAGE QuasiQuotes #-}
+-- > import Text.Webrexp.Quote
+-- >
+-- > [webrexpCompile| some webrexp [.] |]
+--
 webrexpCompile :: QuasiQuoter
 webrexpCompile = QuasiQuoter 
         { quoteExp = compiler
Text/Webrexp/ResourcePath.hs view
@@ -5,15 +5,21 @@     ( ResourcePath (..)
     , rezPathToString
     , downloadBinary
+    , accessResourcePath
+    , rawLoadData
     ) where
 
 import Text.Webrexp.GraphWalker
+import Text.Webrexp.IOMock
 import Control.Applicative
-import Control.Monad.IO.Class
-import Data.Maybe
 import Network.HTTP
 import Network.Browser
-import Network.URI
+import Network.URI( URI
+                  , relativeTo
+                  , isRelativeReference
+                  , parseRelativeReference
+                  , uriPath
+                  , parseURI )
 import System.Directory
 import System.FilePath
 import qualified Text.Webrexp.ProjectByteString as B
@@ -54,29 +60,31 @@ -- but also handle URI.
 combinePath :: ResourcePath -> ResourcePath -> ResourcePath
 combinePath (Local a) (Local b) = Local . normalise $ dropFileName a </> b
-combinePath (Remote a) (Remote b) =
-    case b `relativeTo` a of
-         -- TODO : find another way for this
-         Nothing -> error "Can't merge resourcepath" 
-                   -- Remote a
-         Just c -> Remote c
-
-combinePath (Remote a) (Local b)
-    | isRelativeReference b = case parseRelativeReference b of
-        Just r -> Remote . fromJust $ r `relativeTo` a
+combinePath (Remote a) (Remote b) = Remote $ b `relativeTo` a
+combinePath (Remote a) (Local b) =
+    case parseRelativeReference $ substSlash b of
+        Just r -> Remote $ r `relativeTo` a
         Nothing -> error "Not possible, checked before"
 combinePath (Local _) b@(Remote _) = b
-combinePath _ _ = error "Mixing local/remote path"
 
+substSlash :: String -> String
+substSlash [] = []
+substSlash ('\\':xs) = '/' : substSlash xs
+substSlash (x:xs) = x : substSlash xs
+
 extractFileName :: ResourcePath -> String
 extractFileName (Remote a) = snd . splitFileName $ uriPath a
 extractFileName (Local c) = snd $ splitFileName c
 
-dumpResourcePath :: (Monad m, MonadIO m)
-                 => Loggers -> ResourcePath -> m ()
+dumpResourcePath :: (Monad m, IOMockable m)
+                 => Loggers m -> ResourcePath -> m ()
 dumpResourcePath _ src@(Local source) = do
-    cwd <- liftIO getCurrentDirectory
-    liftIO . copyFile source $ cwd </> extractFileName src
+    maybeDirectory <- performIO getCurrentDirectory
+    case maybeDirectory of
+        Nothing -> return ()
+        Just cwd -> do
+            _ <- performIO . copyFile source $ cwd </> extractFileName src
+            return ()
 
 dumpResourcePath loggers@(logger,_,_) p@(Remote a) = do
   (_, rsp) <- downloadBinary loggers a
@@ -87,18 +95,52 @@                             (hdrValue hdr) rawFilename
                  
 
-  liftIO . logger $ "Downloading '" ++ show a ++ "' in '" ++ filename
-  liftIO . B.writeFile filename $ rspBody rsp
+  logger $ "Downloading '" ++ show a ++ "' in '" ++ filename
+  _ <- performIO . B.writeFile filename $ rspBody rsp
+  return ()
 
+accessResourcePath :: (Monad m, IOMockable m, Functor m)
+                   => Loggers m -> ResourcePath -> m (AccessResult a ResourcePath)
+accessResourcePath loggers rpath = maybe AccessError (DataBlob rpath)
+                                <$> rawLoadData loggers rpath
+
+-- | Extract a blob of data from a resourcepath and return
+-- the result.
+rawLoadData :: (Monad m, IOMockable m)
+            => Loggers m -> ResourcePath -> m (Maybe B.ByteString)
+rawLoadData (logger, _errLog, _verbose) (Local s) = do
+    logger $ "Opening file : '" ++ s ++ "'"
+    realFile <- performIO $ doesFileExist s
+    case realFile of
+      Just True -> performIO (B.readFile s)
+      _         -> return Nothing
+
+rawLoadData loggers@(logger, _, _verbose) (Remote uri) = do
+  logger $ "Downloading URL : '" ++ show uri ++ "'"
+  (_u, rsp) <- downloadBinary loggers uri
+  if rspBody rsp == B.empty 
+  	then return Nothing
+  	else return . Just $ rspBody rsp
+
+
 -- | Helper function to grab a resource on internet and returning
 -- it's binary representation, and it's real place if any.
-downloadBinary :: (Monad m, MonadIO m)
-               => Loggers -> URI -> m (URI, Response B.ByteString)
-downloadBinary (_, errLog, verbose) url =
-    liftIO . browse $ do
+downloadBinary :: (Monad m, IOMockable m)
+               => Loggers m -> URI -> m (URI, Response B.ByteString)
+downloadBinary (_, _errLog, _verbose) url = do
+    rez <- performIO . browse $ do
         setAllowRedirects True
-        setErrHandler errLog
-        setOutHandler verbose
+        -- setErrHandler errLog
+        -- TODO find a way to use that
+        -- setOutHandler verbose
+        setOutHandler . const $ return ()
+        setErrHandler . const $ return ()
         request $ defaultGETRequest_ url
+    case rez of
+        Nothing -> return (url, Response { rspCode = (4, 0, 4)
+                                         , rspReason = "Not allowed" 
+                                         , rspHeaders = []
+                                         , rspBody = B.empty })
+        Just r -> return r
Text/Webrexp/UnionNode.hs view
@@ -8,14 +8,14 @@ -- | This module has for aim to create new node type by combining
 -- different GraphWalkers. The idea is to be able to walk from an
 -- XML file to a Json file and so forth.
-module Text.Webrexp.UnionNode( PartialGraph( .. ), UnionNode ( .. ) ) where
+module Text.Webrexp.UnionNode( PartialGraph( .. ), UnionNode ( .. ), parseUnion ) where
 
 import Control.Applicative
-import Control.Monad.IO.Class
 import Network.HTTP
 import System.Directory
 
 import Text.Webrexp.GraphWalker
+import Text.Webrexp.IOMock
 import Text.Webrexp.Remote.MimeTypes
 import Text.Webrexp.ResourcePath
 import qualified Text.Webrexp.ProjectByteString as B
@@ -31,8 +31,9 @@     -- | The real parsing function.
     -- The IO monad is only here to provide a way to log information
     -- TODO : find a better way.
-    parseResource :: (MonadIO m)
-                  => Loggers -> rezPath -> ParseableType -> B.ByteString -> m (Maybe a)
+    parseResource :: (IOMockable m, Monad m)
+                  => Loggers m -> rezPath -> ParseableType -> B.ByteString 
+                  -> m (Maybe a)
 
 -- | Data type which is an instance of graphwalker.
 -- Use it to combine two other node types.
@@ -64,6 +65,8 @@     nameOf (UnionLeft a) = nameOf a
     nameOf (UnionRight a) = nameOf a
 
+    rawAccess = accessResourcePath
+
     childrenOf (UnionLeft a) =
         childrenOf a >>= \c -> return $ UnionLeft <$> c
     childrenOf (UnionRight a) =
@@ -83,11 +86,12 @@     deepValueOf (UnionLeft a) = deepValueOf a
     deepValueOf (UnionRight a) = deepValueOf a
 
+-- | Function which can be used to bootstrap an in-memory parsing.
 parseUnion :: forall a b m.
-              ( MonadIO m, Functor m
+              ( IOMockable m, Functor m, Monad m
               , PartialGraph a ResourcePath
               , PartialGraph b ResourcePath )
-           => Loggers -> Maybe ParseableType -> ResourcePath -> B.ByteString
+           => Loggers m -> Maybe ParseableType -> ResourcePath -> B.ByteString
            -> m (AccessResult (UnionNode a b) ResourcePath)
 parseUnion _ Nothing datapath binaryData =
     return $ DataBlob datapath binaryData
@@ -107,33 +111,37 @@ 
 
 
-loadData :: ( MonadIO m, Functor m
+loadData :: ( IOMockable m, Functor m, Monad m
             , PartialGraph a ResourcePath
             , PartialGraph b ResourcePath )
-         => Loggers -> ResourcePath
+         => Loggers m -> ResourcePath
          -> m (AccessResult (UnionNode a b) ResourcePath)
 loadData loggers@(logger, _errLog, verbose) datapath@(Local s) = do
-    liftIO . logger $ "Opening file : '" ++ s ++ "'"
-    realFile <- liftIO $ doesFileExist s
-    if not realFile
-       then do
-           liftIO . verbose $ "Unable to open file : " ++ s
+    logger $ "Opening file : '" ++ s ++ "'"
+    realFile <- performIO $ doesFileExist s
+    case realFile of
+        Just True -> performIO (B.readFile s) >>=
+            
+            maybe (return AccessError)
+                  (\file -> do
+                        let kind = getParseKind s
+                        verbose $ "Found kind " ++ show kind ++ " for (" ++ s ++ ")"
+                        parseUnion loggers kind datapath file)
+
+        _    -> do
+           verbose $ "Unable to open file : " ++ s
            return AccessError
-       else do file <- liftIO $ B.readFile s
-       	       let kind = getParseKind s
-       	       liftIO . verbose $ "Found kind " ++ show kind ++ " for (" ++ s ++ ")"
-       	       parseUnion loggers kind datapath file
 
 loadData loggers@(logger, _, verbose) (Remote uri) = do
-  liftIO . logger $ "Downloading URL : '" ++ show uri ++ "'"
+  logger $ "Downloading URL : '" ++ show uri ++ "'"
   (u, rsp) <- downloadBinary loggers uri
   let contentType = retrieveHeaders HdrContentType rsp
       binaryData = rspBody rsp
   case contentType of
     [] -> return $ DataBlob (Remote u) binaryData
     (hdr:_) -> do
-        liftIO . verbose $ "Downloaded (" ++ show u ++ ") ["
-                                      ++ hdrValue hdr ++ "] "
+        verbose $ "Downloaded (" ++ show u ++ ") ["
+                                 ++ hdrValue hdr ++ "] "
         parseUnion loggers
                    (getParserForMimeType $ hdrValue hdr)
                    (Remote u) binaryData
Text/Webrexp/WebContext.hs view
@@ -1,4 +1,6 @@ {-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE FlexibleInstances #-}
 -- | This module define the state carryied during the webrexp
 -- evaluation. This state is implemented as a monad transformer
 -- on top of 'IO'.
@@ -28,6 +30,7 @@     -- * Crawling configuration
     , LogLevel (..)
     , setLogLevel 
+    , getUniqueName
     , getUserAgent 
     , setUserAgent 
     , setOutput 
@@ -38,6 +41,7 @@ 
     -- * User function
     , evalWithEmptyContext
+    , executeWithEmptyContext 
     , repurposeNode 
     
     -- * Implementation info
@@ -66,6 +70,10 @@     , incrementGetRangeCounter 
     , hasResourceBeenVisited
     , setResourceVisited
+
+    -- * Log system
+    , debugLog
+    , textOutput
     )
     where
 
@@ -73,21 +81,25 @@ import Control.Applicative
 import Control.Arrow( first )
 import Control.Monad
+import Control.Monad.ST
 import Control.Monad.IO.Class
 import Control.Monad.Trans.Class
 import Data.Functor.Identity
-import Data.Array.IO
+import Data.Array.MArray
 import qualified Data.Set as Set
+import Control.Exception( IOException )
+import qualified Control.Exception as Ex
 
 import qualified Text.Webrexp.ProjectByteString as B
 import Text.Webrexp.GraphWalker
+import Text.Webrexp.IOMock
 
 -- | Typical use of the WebContextT monad transformer
 -- allowing to download information
-type WebCrawler node rezPath a = WebContextT node rezPath IO a
+type WebCrawler array node rezPath a = WebContextT array node rezPath IO a
 
 -- | WebContext is 'WebContextT' as a simple Monad
-type WebContext node rezPath a = WebContextT node rezPath Identity a
+type WebContext array node rezPath a = WebContextT array node rezPath Identity a
 
 -- | Record a graph path in a document, from the last indirect
 -- node to this one.
@@ -185,7 +197,7 @@ type StateNumber = Int
 
 -- | Internal data context.
-data Context node rezPath = Context
+data Context array node rezPath = Context
     { -- | Context stack used in breadth-first evaluation
       contextStack :: [([EvalState node rezPath], Counter)]
 
@@ -199,35 +211,43 @@ 
       -- | Buckets used for uniqueness pruning, all
       -- evaluation kind.
-    , uniqueBucket :: IOArray Int (Set.Set String) 
+    , uniqueBucket :: array Int (Set.Set String)
 
       -- | Counters used for range evaluation in DFS
-    , countBucket :: IOUArray Int Counter
+    , countBucket :: array Int Counter
 
+      -- | Tell if we must keep the found information in memory
+      -- instead of directly dumping it on screen.
+    , mustGatherData :: Bool
+
+      -- | If you want to run a webrexp in library mode
+    , gatheredData :: [Either String String]
+
       -- | Current log level
     , logLevel :: LogLevel
     , httpDelay :: Int
     , httpUserAgent :: String
     , defaultOutput :: Handle
+    , uniqueNameCount :: Int
     }
 
 --------------------------------------------------
 ----            Monad definitions
 --------------------------------------------------
-newtype (Monad m) => WebContextT node rezPath m a =
-    WebContextT { runWebContextT :: Context node rezPath
-                                 -> m (a, Context node rezPath ) }
+newtype (Monad m) => WebContextT array node rezPath m a =
+    WebContextT { runWebContextT :: Context array node rezPath
+                                 -> m (a, Context array node rezPath) }
 
-instance (Functor m, Monad m) => Functor (WebContextT node rezPath m) where
+instance (Functor m, Monad m) => Functor (WebContextT array node rezPath m) where
     {-# INLINE fmap #-}
     fmap f a = WebContextT $ \c ->
         fmap (first f) $ runWebContextT a c
 
-instance (Functor m, Monad m) => Applicative (WebContextT node rezPath m) where
+instance (Functor m, Monad m) => Applicative (WebContextT array node rezPath m) where
     pure = return
     (<*>) = ap
 
-instance (Monad m) => Monad (WebContextT node rezPath m) where
+instance (Monad m) => Monad (WebContextT array node rezPath m) where
     {-# INLINE return #-}
     return a =
         WebContextT $ \c -> return (a, c)
@@ -237,19 +257,25 @@         (val', c') <- val c
         runWebContextT (f val') c'
 
-instance MonadTrans (WebContextT node rezPath) where
+instance MonadTrans (WebContextT array node rezPath) where
     lift m = WebContextT $ \c -> do
         a <- m
         return (a, c)
 
-instance (MonadIO m) => MonadIO (WebContextT node rezPath m) where
+instance (MonadIO m) => MonadIO (WebContextT array node rezPath m) where
     liftIO = lift . liftIO
 
+instance IOMockable (WebContextT array node rezPath IO) where
+    performIO act = Just <$> liftIO act
+
+instance IOMockable (WebContextT array node rezPath (ST s)) where
+    performIO _ = return Nothing
+
 --------------------------------------------------
 ----            Context manipulation
 --------------------------------------------------
 
-emptyContext :: Context node rezPath
+emptyContext :: Context array node rezPath
 emptyContext = Context
     { contextStack = []
     , waitingStates = []
@@ -257,67 +283,81 @@     , logLevel = Normal
     , httpDelay = 1500
     , httpUserAgent = ""
+    , mustGatherData = False
+    , gatheredData = []
     , defaultOutput = stdout
     , uniqueBucket = undefined
     , countBucket = undefined
+    , uniqueNameCount = 1
     }
 
 --------------------------------------------------
 ----            Getter/Setter
 --------------------------------------------------
 
+-- | Return an "pseudo" unique, a filename not used during the
+-- run of the current expression.
+getUniqueName :: (Monad m) => WebContextT array node rezPath m FilePath
+getUniqueName = WebContextT $ \c ->
+    let i = uniqueNameCount c
+    in return ("file_" ++ show i, c { uniqueNameCount = i + 1 })
+
 -- | Setter for the wait time between two indirect
 -- operations.
 --
 -- The value is stored but not used yet.
-setHttpDelay :: (Monad m) => Int -> WebContextT node rezPath m ()
+setHttpDelay :: (Monad m) => Int -> WebContextT array node rezPath m ()
 setHttpDelay delay = WebContextT $ \c ->
         return ((), c{ httpDelay = delay })
 
 -- | return the value set by 'setHttpDelay'
-getHttpDelay :: (Monad m) => WebContextT node rezPath m Int
+getHttpDelay :: (Monad m) => WebContextT array node rezPath m Int
 getHttpDelay = WebContextT $ \c -> return (httpDelay c, c)
 
 -- | Define the text output for written text.
-setOutput :: (Monad m) => Handle -> WebContextT node rezPath m ()
+setOutput :: (Monad m) => Handle -> WebContextT array node rezPath m ()
 setOutput handle = WebContextT $ \c ->
         return ((), c{ defaultOutput = handle })
 
 -- | Retrieve the default file output used for text.
-getOutput :: (Monad m) => WebContextT node rezPath m Handle
+getOutput :: (Monad m) => WebContextT array node rezPath m Handle
 getOutput = WebContextT $ \c -> return (defaultOutput c, c)
 
+isDataOutputedDirectly :: (Monad m) => WebContextT array node rezPath m Bool
+isDataOutputedDirectly = WebContextT $ \c ->
+        return (not $ mustGatherData c, c)
+
 -- | Set the user agent which must be used for indirect operations
 --
 -- The value is stored but not used yet.
-setUserAgent :: (Monad m) => String -> WebContextT node rezPath m ()
+setUserAgent :: (Monad m) => String -> WebContextT array node rezPath m ()
 setUserAgent usr = WebContextT $ \c ->
     return ((), c{ httpUserAgent = usr })
 
 -- | return the value set by 'setUserAgent'
-getUserAgent :: (Monad m) => WebContextT node rezPath m String
+getUserAgent :: (Monad m) => WebContextT array node rezPath m String
 getUserAgent = WebContextT $ \c -> return (httpUserAgent c, c)
 
 -- | Set the value of the logging level.
-setLogLevel :: (Monad m) => LogLevel -> WebContextT node rezPath m ()
+setLogLevel :: (Monad m) => LogLevel -> WebContextT array node rezPath m ()
 setLogLevel lvl = WebContextT $ \c ->
     return ((), c{logLevel = lvl})
 
 -- | Tell if the current 'LoggingLevel' is set to 'Verbose'
-isVerbose :: (Monad m) => WebContextT node rezPath m Bool
+isVerbose :: (Monad m) => WebContextT array node rezPath m Bool
 isVerbose = WebContextT $ \c -> 
     return (logLevel c == Verbose, c)
 
 -- | TODO : write documentation
 accumulateCurrentState :: (Monad m)
                        => ([EvalState node rezPath], StateNumber)
-                       -> WebContextT node rezPath m ()
+                       -> WebContextT array node rezPath m ()
 accumulateCurrentState lst = WebContextT $ \c ->
         return ((), c{ contextStack = lst : contextStack c })
 
 -- | TODO : write documentation
 popAccumulation :: (Monad m)
-                => WebContextT node rezPath m ([EvalState node rezPath], StateNumber)
+                => WebContextT array node rezPath m ([EvalState node rezPath], StateNumber)
 popAccumulation = WebContextT $ \c ->
     case contextStack c of
          []     -> error "Empty context stack, implementation bug"
@@ -329,14 +369,14 @@ -- retrieval.
 pushCurrentState :: (Monad m)
                  => [EvalState node rezPath]
-                 -> WebContextT node rezPath m ()
+                 -> WebContextT array node rezPath m ()
 pushCurrentState lst = WebContextT $ \c ->
         return ((), c{ contextStack = (lst, 0) : contextStack c })
 
 -- | Inverse operation of 'pushCurrentState', retrieve
 -- stored nodes.
 popCurrentState :: (Monad m)
-                => WebContextT node rezPath m [EvalState node rezPath]
+                => WebContextT array node rezPath m [EvalState node rezPath]
 popCurrentState = WebContextT $ \c ->
     case contextStack c of
          []     -> error "Empty context stack, implementation bug"
@@ -346,36 +386,79 @@ -- | Helper function used to start the evaluation of a webrexp
 -- with a default context, with sane defaults.
 evalWithEmptyContext :: (Monad m)
-                     => WebContextT node rezPath m a -> m a
+                     => WebContextT array node rezPath m a -> m a
 evalWithEmptyContext val = do
     (finalVal, _context) <- runWebContextT val emptyContext
     return finalVal
 
+-- | Helper function used to evaluate a webrexp and get back
+-- data with a default context with sane defaults.
+executeWithEmptyContext :: (Monad m)
+                        => WebContextT array node rezPath m a -> m [Either String String]
+executeWithEmptyContext val = do
+    (_, context) <- runWebContextT val (emptyContext { mustGatherData = True })
+    return $ gatheredData context
+
 -- | Return normal, error, verbose logger
-prepareLogger :: (Monad m)
-              => WebContextT node rezPath m (Logger, Logger, Logger)
+prepareLogger :: (Monad m, IOMockable (WebContextT array node rezPath m))
+              => WebContextT array node rezPath m 
+                    (Logger (WebContextT array node rezPath m)
+                    ,Logger (WebContextT array node rezPath m)
+                    ,Logger (WebContextT array node rezPath m))
 prepareLogger = WebContextT $ \c ->
     let silenceLog _ = return ()
-        errLog = hPutStrLn stderr
-        normalLog = putStrLn
-    in case logLevel c of
-      Quiet -> return ((silenceLog, errLog, silenceLog), c)
-      Normal -> return ((normalLog, errLog, silenceLog), c)
-      Verbose -> return ((normalLog, errLog, normalLog), c)
+        errLog msg = performIO (hPutStrLn stderr msg) >> return ()
+        normalLog = textOutput
+    in case (mustGatherData c, logLevel c) of
+      (True, _) -> return ((silenceLog, gatheringLog . Left, silenceLog), c)
+      (_, Quiet) -> return ((silenceLog, errLog, silenceLog), c)
+      (_, Normal) -> return ((normalLog, errLog, silenceLog), c)
+      (_, Verbose) -> return ((normalLog, errLog, normalLog), c)
 
+-- | Debugging function, only displayed in verbose
+-- logging mode.
+debugLog :: (IOMockable (WebContextT array node rezPath m), Monad m)
+         => String -> WebContextT array node rezPath m ()
+debugLog str = do
+    verb <- isVerbose
+    when verb (performIO (putStrLn str) >> return ())
+
+
+-- | If a webrexp output some text, it must go through
+-- this function. It ensure the writting in the correct
+-- file.
+textOutput :: (Monad m, IOMockable (WebContextT array node rezPath m)) 
+           => String -> WebContextT array node rezPath m ()
+textOutput str = do
+    direct <- isDataOutputedDirectly 
+    if not direct
+       then gatheringLog $ Right str
+       else do handle <- getOutput
+               _ <- performIO $ Ex.catch (hPutStr handle str)
+                       (\e -> hPutStrLn stderr $ "Writing error : " ++ 
+                                                show (e :: IOException))
+               return ()
+
+-- | Keep track of an error or a normal log in the application monad
+-- transformer.
+gatheringLog :: (Monad m) => Either String String -> WebContextT array node rezPath m ()
+gatheringLog d = WebContextT $ \c ->
+    return ((), c { gatheredData = gatheredData c ++ [d] })
+
+
 --------------------------------------------------
 ----            Depth First evaluation
 --------------------------------------------------
 
 -- | Record a node in the context for the DFS evaluation.
 recordNode :: (Monad m) 
-           => (EvalState node rezPath, StateNumber) -> WebContextT node rezPath m ()
+           => (EvalState node rezPath, StateNumber) -> WebContextT array node rezPath m ()
 recordNode n = WebContextT $ \c ->
     return ((), c{ waitingStates = n : waitingStates c })
 
 -- | Get the last record from the top of the stack
 popLastRecord :: (Monad m)
-              => WebContextT node rezPath m (EvalState node rezPath, StateNumber)
+              => WebContextT array node rezPath m (EvalState node rezPath, StateNumber)
 popLastRecord = WebContextT $ \c ->
     case waitingStates c of
       [] -> error "popLAst Record - Empty stack!!!"
@@ -395,14 +478,16 @@ -- for other frame manipulation functions.
 pushToBranchContext :: (Monad m)
                     => (EvalState node rezPath, SeenCounter, ValidSeenCounter)
-                    -> WebContextT node rezPath m ()
+                    -> WebContextT array node rezPath m ()
 pushToBranchContext cont = WebContextT $ \c ->
     return ((), c{ branchContext = cont : branchContext c })
 
 -- | Retrieve the frame on the top of the stack.
 -- for more information regarding frames see 'pushToBranchContext'
 popBranchContext :: (Monad m)
-                 => WebContextT node rezPath m (EvalState node rezPath, SeenCounter, ValidSeenCounter)
+                 => WebContextT array node rezPath m ( EvalState node rezPath
+                                                     , SeenCounter
+                                                     , ValidSeenCounter )
 popBranchContext = WebContextT $ \c ->
     case branchContext c of
       [] -> error "popBranchContext - empty branch context"
@@ -413,7 +498,8 @@ --
 -- for more information regarding frames see 'pushToBranchContext'
 addToBranchContext :: (Monad m)
-                   => SeenCounter -> ValidSeenCounter -> WebContextT node rezPath m ()
+                   => SeenCounter -> ValidSeenCounter 
+                   -> WebContextT array node rezPath m ()
 addToBranchContext count validCount = WebContextT $ \c ->
     case branchContext c of
       [] -> error "addToBranchContext - empty context stack"
@@ -430,23 +516,26 @@ -- Inform the monad of the number of 'Unique' bucket in the
 -- expression, permitting the allocation of the required number
 -- of Set to hold them.
-setBucketCount :: (Monad m, MonadIO m)
+setBucketCount :: ( Monad m
+                  , MArray array Counter m
+                  , MArray array (Set.Set String) m
+                  )
                => Int -- ^ Unique bucket count
                -> Int -- ^ Range counter count
-               -> WebContextT node rezPath m ()
+               -> WebContextT array node rezPath m ()
 setBucketCount uniquecount rangeCount = WebContextT $ \c -> do
-    arr <- liftIO $ newArray (0, uniquecount - 1) Set.empty
-    counter <- liftIO $ newArray (0, rangeCount - 1) 0
-    return ((), c{ uniqueBucket = arr
-                 , countBucket = counter })
+    setArray <- newArray (0, uniquecount - 1) Set.empty
+    counterArray <- newArray (0, rangeCount - 1) 0
+    return ((), c{ uniqueBucket = setArray
+                 , countBucket = counterArray } )
 
 -- | Used for node range, return the current value of the
 -- counter and increment it.
-incrementGetRangeCounter :: (Monad m, MonadIO m)
-                         => Int -> WebContextT node rezPath m Int
+incrementGetRangeCounter :: (Monad m, MArray array Counter m)
+                         => Int -> WebContextT array node rezPath m Int
 incrementGetRangeCounter bucket = WebContextT $ \c -> do
-    num <- liftIO $ countBucket c `readArray`bucket
-    liftIO . (countBucket c `writeArray` bucket) $ num + 1
+    num <- countBucket c `readArray` bucket
+    writeArray (countBucket c) bucket $ num + 1
     return (num, c)
 
 -- | Tell if a string has already been recorded for a bucket ID.
@@ -454,19 +543,19 @@ --
 -- Return False, unless 'setResourceVisited' has been called with the same
 -- string before.
-hasResourceBeenVisited :: (Monad m, MonadIO m)
-                       => Int -> String -> WebContextT node rezPath m Bool
+hasResourceBeenVisited :: (Monad m, MArray array (Set.Set String) m)
+                       => Int -> String -> WebContextT array node rezPath m Bool
 hasResourceBeenVisited bucketId str = WebContextT $ \c -> do
-    set <- liftIO $ uniqueBucket c `readArray`bucketId
+    set <- uniqueBucket c `readArray` bucketId
     return (str `Set.member` set, c)
 
 -- | Record the visit of a string. 'hasResourceBeenVisited' will return True
 -- for the same string after this call.
-setResourceVisited :: (Monad m, MonadIO m)
-                   => Int -> String -> WebContextT node rezPath m ()
+setResourceVisited :: (Monad m, MArray array (Set.Set String) m)
+                   => Int -> String -> WebContextT array node rezPath m ()
 setResourceVisited bucketId str = WebContextT $ \c -> do
-    set <- liftIO $ uniqueBucket c `readArray` bucketId
+    set <- uniqueBucket c `readArray` bucketId
     let newSet = str `Set.insert` set
-    liftIO $ (uniqueBucket c `writeArray`bucketId) newSet
+    writeArray (uniqueBucket c) bucketId newSet
     return ((), c)
Text/Webrexp/WebRexpAutomata.hs view
@@ -1,5 +1,6 @@ {-# LANGUAGE QuasiQuotes #-}
 {-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE FlexibleContexts #-}
 module Text.Webrexp.WebRexpAutomata ( -- * Types
                                  Automata
                                , StateIndex
@@ -18,14 +19,16 @@ 
 import Control.Monad
 import Data.Array
+import Data.Array.MArray
 import qualified Data.Array.Unboxed as U
+import qualified Data.Set as Set
 import System.IO
 
-import Text.Webrexp.Log
 import Text.Webrexp.Eval
 import Text.Webrexp.GraphWalker
 import Text.Webrexp.WebContext
 import Text.Webrexp.Exprtypes
+import Text.Webrexp.IOMock
 
 import Language.Haskell.TH.Syntax
 
@@ -298,6 +301,8 @@     (free + 1, free, ((free, AutoState (AutoSimple rest) onTrue onFalse):))
 toAutomata rest@DiggLink  free (onTrue, onFalse) =
     (free + 1, free, ((free, AutoState (AutoSimple rest) onTrue onFalse):))
+toAutomata rest@DumpLink free (onTrue, onFalse) =
+    (free + 1, free, ((free, AutoState (AutoSimple rest) onTrue onFalse):))
 toAutomata rest@NextSibling  free (onTrue, onFalse) =
     (free + 1, free, ((free, AutoState (AutoSimple rest) onTrue onFalse):))
 toAutomata rest@PreviousSibling  free (onTrue, onFalse) =
@@ -310,9 +315,13 @@ --------------------------------------------------
 
 -- | Simple function performing a depth first evaluation
-evalDepthFirst :: (GraphWalker node rezPath)
+evalDepthFirst :: ( GraphWalker node rezPath
+                  , IOMockable (WebContextT array node rezPath m)
+                  , MArray array (Set.Set String) m
+                  , MArray array Counter m 
+                  , Functor m )
                => EvalState node rezPath -> WebRexp
-               -> WebCrawler node rezPath Bool
+               -> WebContextT array node rezPath m Bool
 evalDepthFirst initialState expr = do
     debugLog $ "[Depth first, starting at " ++ show begin ++ "]"
     setBucketCount count rangeCount
@@ -322,12 +331,17 @@               (count, rangeCount, neorexp) = assignWebrexpIndices expr
 
 -- | Main Evaluation function
-evalAutomataDFS :: (GraphWalker node rezPath)
-             => Automata                 -- ^ Automata to evaluate
-             -> StateIndex               -- ^ State to evaluate
-             -> Bool                     -- ^ Are we coming from a true link.
-             -> EvalState node rezPath   -- ^ Current evaluated element
-             -> WebCrawler node rezPath Bool
+evalAutomataDFS :: ( GraphWalker node rezPath
+                   , IOMockable (WebContextT array node rezPath m)
+                   , MArray array (Set.Set String) m
+                   , MArray array Counter m
+                   , Functor m
+                   , Monad m )
+                => Automata                 -- ^ Automata to evaluate
+                -> StateIndex               -- ^ State to evaluate
+                -> Bool                     -- ^ Are we coming from a true link.
+                -> EvalState node rezPath   -- ^ Current evaluated element
+                -> WebContextT array node rezPath m Bool
 evalAutomataDFS auto i fromTrue e
     | i < 0 = return fromTrue
     | otherwise = do
@@ -336,20 +350,29 @@                     (autoStates auto ! i) fromTrue e
 
 -- | Pop a record and start evaluation for him.
-scheduleNextElement :: (GraphWalker node rezPath)
-                    => Automata -> WebCrawler node rezPath Bool
+scheduleNextElement :: ( GraphWalker node rezPath
+                       , IOMockable (WebContextT array node rezPath m)
+                       , MArray array (Set.Set String) m
+                       , MArray array Counter m
+                       , Monad m, Functor m )
+                    => Automata -> WebContextT array node rezPath m Bool
 scheduleNextElement a = do
     (e, idx) <- popLastRecord
     evalAutomataDFS a idx True e
 
 
 -- | Evaluation function for an element.
-evalStateDFS :: (GraphWalker node rezPath)
-                  => Automata       -- ^ Evaluation automata
-                  -> AutomataState  -- ^ Current state in the automata
-                  -> Bool           -- ^ If we are coming from a True link or a False one
-                  -> EvalState node rezPath -- ^ Currently evaluated element
-                  -> WebCrawler node rezPath Bool
+evalStateDFS :: ( GraphWalker node rezPath
+                , IOMockable (WebContextT array node rezPath m)
+                , MArray array Counter m
+                , MArray array (Set.Set String) m
+                , Functor m
+                , Monad m)
+             => Automata       -- ^ Evaluation automata
+             -> AutomataState  -- ^ Current state in the automata
+             -> Bool           -- ^ If we are coming from a True link or a False one
+             -> EvalState node rezPath -- ^ Currently evaluated element
+             -> WebContextT array node rezPath m Bool
 evalStateDFS a (AutoState (Gather _) onTrue onFalse) valid e = do
     debugLog "> Gather"
     evalAutomataDFS a (if valid then onTrue else onFalse) valid e
@@ -431,9 +454,14 @@ 
 -- | Main function to evaluate the expression in breadth
 -- first order.
-evalBreadthFirst :: (GraphWalker node rezPath)
+evalBreadthFirst :: ( GraphWalker node rezPath
+                    , IOMockable (WebContextT array node rezPath m)
+                    , MArray array (Set.Set String) m
+                    , MArray array Counter m
+                    , Functor m
+                    )
                  => EvalState node rezPath -> WebRexp
-                 -> WebCrawler node rezPath Bool
+                 -> WebContextT array node rezPath m Bool
 evalBreadthFirst initialState expr = do
     debugLog $ "[Breadth first, starting at " ++ show begin ++ "]"
     setBucketCount count 0
@@ -442,12 +470,16 @@               begin = beginState auto
               (count, _, neorexp) = assignWebrexpIndices expr
 
-evalAutomataBFS :: (GraphWalker node rezPath)
+evalAutomataBFS :: ( GraphWalker node rezPath
+                   , IOMockable (WebContextT array node rezPath m)
+                   , MArray array (Set.Set String) m
+                   , Functor m, Monad m
+                   )
                 => Automata                 -- ^ Automata to evaluate
                 -> StateIndex               -- ^ State to evaluate
                 -> Bool                     -- ^ Are we coming from a true link.
                 -> [EvalState node rezPath] -- ^ Current evaluated element
-                -> WebCrawler node rezPath Bool
+                -> WebContextT array node rezPath m Bool
 evalAutomataBFS auto i fromTrue e
     | i < 0 = return fromTrue
     | otherwise = do
@@ -457,12 +489,16 @@ 
 
 -- | Main evaluation function for BFS evaluation.
-evalStateBFS :: (GraphWalker node rezPath)
+evalStateBFS :: ( GraphWalker node rezPath
+                , IOMockable (WebContextT array node rezPath m)
+                , MArray array (Set.Set String) m
+                , Functor m
+                )
              => Automata       -- ^ Evaluation automata
              -> AutomataState  -- ^ Current state in the automata
              -> Bool           -- ^ If we are coming from a True link or a False one
              -> [EvalState node rezPath]   -- ^ Currently evaluated elements
-             -> WebCrawler node rezPath Bool
+             -> WebContextT array node rezPath m Bool
 
 evalStateBFS a (AutoState (Gather idxs) onTrue onFalse) valid e = do
     debugLog "> Gather"
Webrexp.cabal view
@@ -1,5 +1,5 @@ Name:       Webrexp
-Version:    1.1
+Version:    1.1.1
 Synopsis:   Regexp-like engine to scrap web data
 Build-Type: Simple
 License:    BSD3
@@ -20,21 +20,25 @@   if flag(optimize)
       Ghc-options:-O3
 
-  Build-Depends: base >= 2.0 && < 5.0, mtl,
-                 HTTP >= 4000.1.1 && < 4000.2,
-                 parsec >= 3.1 && < 3.2,
-                 transformers >= 0.2.2 && < 0.3,
-                 network >= 2.3 && < 2.4,
-                 directory >= 1.0,
-                 bytestring >= 0.9.1.7 && < 1.0,
-                 containers >= 0.4 && < 0.4.1,
-                 array >= 0.3.0.2 && < 0.3.0.3,
-                 regex-pcre-builtin >= 0.94,
-                 HaXml >= 1.22.5 && < 1.23,
-                 hxt >= 9.1.2 && < 9.2,
-                 AttoJson >= 0.5.1 && < 1.0,
-                 process >= 1.0.1.3 && < 1.1,
-                 filepath >= 1.2.0 && < 1.3,
+  Build-Depends: base               >= 4.0      && < 5.0,
+                 mtl                >= 2.1      && < 2.2,
+                 HTTP               >= 4000.1.1 && < 4000.3,
+                 aeson              >= 0.6      && < 0.7,
+                 parsec             >= 3.1      && < 3.2,
+                 transformers       >= 0.3      && < 0.4,
+                 network            >= 2.3      && < 2.5,
+                 directory          >= 1.1      && < 1.3,
+                 bytestring         >= 0.9.1.7  && < 0.11,
+                 containers         >= 0.4      && < 0.6,
+                 array              >= 0.4      && < 0.5,
+                 regex-pcre-builtin >= 0.94     && < 0.95,
+                 HaXml              >= 1.23     && < 1.24,
+                 hxt                >= 9.2      && < 9.4,
+                 process            >= 1.1      && < 1.2,
+                 filepath           >= 1.3      && < 1.4,
+                 unordered-containers >= 0.2    && < 0.3,
+                 text               >= 0.11     && < 0.12,
+                 vector             >= 0.9      && < 0.10,
                  template-haskell
                  -- Webrexp
 
@@ -56,27 +60,30 @@             Text.Webrexp.HaXmlNode
             Text.Webrexp.HxtNode
             Text.Webrexp.JsonNode
-            Text.Webrexp.Log
             Text.Webrexp.ProjectByteString
             Text.Webrexp.Remote.MimeTypes
             Text.Webrexp.UnionNode
             Text.Webrexp.WebContext
             Text.Webrexp.WebRexpAutomata
 
-  Build-Depends: base >= 2.0 && < 5.0, mtl,
-                 HTTP >= 4000.1.1 && < 4000.2,
-                 HaXml >= 1.22.5 && < 1.23,
-                 hxt >= 9.1.2 && < 9.2,
-                 parsec >= 3.1 && < 3.2,
-                 transformers >= 0.2.2 && < 0.3,
-                 network >= 2.3 && < 2.4,
-                 directory >= 1.0,
-                 bytestring >= 0.9.1.7 && < 1.0,
-                 containers >= 0.4 && < 0.4.1,
-                 array >= 0.3.0.2 && < 0.3.0.3,
-                 regex-pcre-builtin >= 0.94,
-                 AttoJson >= 0.5.1 && < 1.0,
-                 process >= 1.0.1.3 && < 1.1,
-                 filepath >= 1.2.0 && < 1.3,
+  Build-Depends: base               >= 4.0      && < 5.0,
+                 mtl                >= 2.1      && < 2.2,
+                 HTTP               >= 4000.1.1 && < 4000.3,
+                 aeson              >= 0.6      && < 0.7,
+                 parsec             >= 3.1      && < 3.2,
+                 transformers       >= 0.3      && < 0.4,
+                 network            >= 2.3      && < 2.5,
+                 directory          >= 1.1      && < 1.3,
+                 bytestring         >= 0.9.1.7  && < 0.11,
+                 containers         >= 0.4      && < 0.6,
+                 array              >= 0.4      && < 0.5,
+                 regex-pcre-builtin >= 0.94     && < 0.95,
+                 HaXml              >= 1.23     && < 1.24,
+                 hxt                >= 9.2      && < 9.4,
+                 process            >= 1.1      && < 1.2,
+                 filepath           >= 1.3      && < 1.4,
+                 unordered-containers >= 0.2    && < 0.3,
+                 text               >= 0.11     && < 0.12,
+                 vector             >= 0.9      && < 0.10,
                  template-haskell