robots-txt 0.2.0.0 → 0.3.0.0

raw patch · 2 files changed

+15/−4 lines, 2 files

Files

• robots-txt.cabal +1/−1
• src/Network/HTTP/Robots.hs +14/−3

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                robots-txt-version:             0.2.0.0+version:             0.3.0.0 synopsis:            Parser for robots.txt description:         This is an attoparsec parser for robots.txt files homepage:            http://github.com/meanpath/robots

@@ -7,9 +7,12 @@ import Control.Applicative import Data.List(find) import Data.Maybe(catMaybes)+import Data.Either(partitionEithers) -type Robot = [([UserAgent], [Directive])]+type Robot = ([([UserAgent], [Directive])], [Unparsable]) +type Unparsable = ByteString+ data UserAgent = Wildcard | Literal ByteString   deriving (Show,Eq) type Path = ByteString@@ -27,8 +30,16 @@               . BS.lines  robotP :: Parser Robot-robotP = many ((,) <$> many1 agentP <*> many1 directiveP) <?> "robot"+robotP = do+  (dirs, unparsable) <- partitionEithers <$> many  (eitherP agentDirectiveP unparsableP) <?> "robot"+  return (dirs, filter (/= "") unparsable) +unparsableP = takeTill (=='\n') <* char '\n'++agentDirectiveP = (,) <$> many1 agentP <*> many1 directiveP <?> "agentDirective"+++ skipSpace :: Parser () skipSpace = skipWhile (\x -> x==' ' || x == '\t') @@ -66,7 +77,7 @@ -- I lack the art to make this prettier. canAccess :: ByteString -> Robot -> Path -> Bool canAccess _ _ "/robots.txt" = True -- special-cased-canAccess agent robot path = case stanzas of+canAccess agent (robot,_) path = case stanzas of   [] -> True   ((_,directives):_) -> matchingDirective directives   where stanzas = catMaybes [find ((Literal agent `elem`) . fst) robot,