robots-txt (empty) → 0.1.0.0

raw patch · 5 files changed

+123/−0 lines, 5 filesdep +QuickCheckdep +attoparsecdep +basesetup-changed

Dependencies added: QuickCheck, attoparsec, base, bytestring, directory, hspec, robots-txt, transformers

Files

• LICENSE +30/−0
• Setup.hs +2/−0
• robots-txt.cabal +43/−0
• src/Network/HTTP/Robots.hs +46/−0
• test/Spec.hs +2/−0

@@ -0,0 +1,30 @@+Copyright (c) 2013, Mark Wotton++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Mark Wotton nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

@@ -0,0 +1,43 @@+-- Initial robots-txt.cabal generated by cabal init.  For further+-- documentation, see http://haskell.org/cabal/users-guide/++name:                robots-txt+version:             0.1.0.0+synopsis:            Parser for robots.txt+description:         This is an attoparsec parser for robots.txt files+homepage:            http://github.com/meanpath/robots-txt+license:             BSD3+license-file:        LICENSE+author:              Mark Wotton+maintainer:          mwotton@gmail.com+bug-reports:         https://github.com/meanpath/robots-txt+-- copyright:+category:            Web+build-type:          Simple+cabal-version:       >=1.8++source-repository head+  type: git+  location: git://github.com/meanpath/robots-txt.git++library+  exposed-modules:     Network.HTTP.Robots+  -- other-modules:+  hs-source-dirs:         src+  build-depends:        base ==4.6.*+                      , bytestring+                      , attoparsec++test-suite tests+  type:            exitcode-stdio-1.0+  ghc-options:     -Wall+  hs-source-dirs:  test/+  main-is:         Spec.hs+  build-depends:   base >= 4 && <= 5+                   , hspec+                   , QuickCheck+                   , bytestring+                   , robots-txt+                   , directory+                   , transformers+                   , attoparsec

@@ -0,0 +1,46 @@+{-# LANGUAGE OverloadedStrings #-}+module Network.HTTP.Robots where++import qualified Data.ByteString.Char8 as BS+import           Data.ByteString.Char8(ByteString)+import Data.Attoparsec.Char8+import Control.Applicative++type Robot = [(UserAgent, [Directive])]++data UserAgent = Wildcard | Literal ByteString+  deriving (Show,Eq)+type Path = ByteString++data Directive = Allow Path+               | Disallow Path+               | CrawlDelay Int+  deriving (Show,Eq)+-- this is an ugly hack, but i don't want to handle comments+-- everywhere.+parseRobots :: ByteString -> Either String Robot+parseRobots = parseOnly robotP+              . BS.unlines+              . filter ( (\x -> BS.null x || BS.head x /= '#' ) . BS.dropWhile (==' '))+              . BS.lines++data Hole = Hole+robotP :: Parser Robot+robotP = many ((,) <$> agentP <*> many directiveP) <?> "robot"+++directiveP :: Parser Directive+directiveP = choice [ Allow <$>      (string "Allow:"       >> skipSpace >> tokenP)+                    , Disallow <$>   (string "Disallow:"    >> skipSpace >> tokenP)+                    , CrawlDelay <$> (string "Crawl-delay:" >>  skipSpace >>decimal)+                    ] <* skipSpace <?> "directive"++agentP :: Parser UserAgent+agentP = do+  string "User-agent:"+  skipSpace+  ((string "*" >> return Wildcard) <|>+   (Literal  <$> tokenP)) <* skipSpace <?> "agent"++tokenP :: Parser ByteString+tokenP = skipSpace >> takeTill isSpace <* skipSpace

@@ -0,0 +1,2 @@+-- file test/Spec.hs+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}