html-tokenizer 0.2.1.2 → 0.3.0.0

raw patch · 2 files changed

+31/−45 lines, 2 filesdep −html-entitiesdep ~conversion-text

Dependencies removed: html-entities

Dependency ranges changed: conversion-text

Files

• html-tokenizer.cabal +3/−4
• library/HTMLTokenizer/Parser.hs +28/−41

@@ -1,7 +1,7 @@ name:   html-tokenizer version:-  0.2.1.2+  0.3.0.0 synopsis:   An "attoparsec"-based HTML tokenizer description:@@ -9,7 +9,7 @@   or for streaming.   E.g., by composing it with    <http://hackage.haskell.org/package/list-t-attoparsec the "list-t-attoparsec" library>-  you can produce a token stream with HTML-entities decoded,+  you can produce a token stream,   thus becoming able to implement a highly efficient stream-parser,   which works in a single pass, constant memory and is capable of early termination.   <http://hackage.haskell.org/package/list-t-html-parser "list-t-html-parser"> is such a parser.@@ -49,9 +49,8 @@   exposed-modules:     HTMLTokenizer.Parser   build-depends:-    html-entities == 1.0.*,     conversion >= 1.0.1 && < 2,-    conversion-text == 1.0.*,+    conversion-text >= 1.0.1 && < 2,     conversion-case-insensitive == 1.*,     case-insensitive == 1.2.*,     text >= 1 && < 1.3,

@@ -3,21 +3,20 @@   -- * Model   Token(..),   OpeningTag,-  ClosingTag,+  Identifier,   Attribute,   -- * Parsers   token, ) where -import BasePrelude+import BasePrelude hiding (takeWhile) import Conversion import Conversion.Text import Conversion.CaseInsensitive import Data.Text (Text) import Data.Text.Lazy.Builder (Builder) import Data.CaseInsensitive (CI)-import HTMLEntities.Parser import Data.Attoparsec.Text import qualified Data.Text @@ -30,7 +29,7 @@   Token_OpeningTag OpeningTag |   -- |   -- A closing tag.-  Token_ClosingTag ClosingTag |+  Token_ClosingTag Identifier |   -- |   -- A text between tags with HTML-entities decoded.   Token_Text Text |@@ -42,75 +41,63 @@ -- | -- An opening tag name, attributes and whether it is closed. type OpeningTag =-  (CI Text, [Attribute], Bool)+  (Identifier, [Attribute], Bool)  -- |--- A closing tag name.-type ClosingTag =+-- A case-insensitive identifier.+type Identifier =   CI Text  -- |--- A tag attribute identifier and a value with HTML-entities decoded.+-- A tag attribute identifier and a value. type Attribute =-  (CI Text, Maybe Text)+  (Identifier, Maybe Text)  -- | -- A token parser. token :: Parser Token token =-  skipSpace *> (-    Token_Comment <$> comment <|>-    Token_ClosingTag <$> closingTag <|>-    Token_OpeningTag <$> openingTag <|>-    Token_Text <$> text-  )+  Token_Comment <$> comment <|>+  Token_ClosingTag <$> closingTag <|>+  Token_OpeningTag <$> openingTag <|>+  Token_Text <$> text  openingTag :: Parser OpeningTag openingTag =   do     char '<'     skipSpace-    name <- identifier+    theIdentifier <- identifier     attributes <- many $ space *> skipSpace *> attribute     skipSpace     closed <- convert <$> optional (char '/')     char '>'-    return (convert name, attributes, closed)+    return (theIdentifier, attributes, closed)  attribute :: Parser Attribute attribute =   do-    identifierValue <- identifier+    theIdentifier <- identifier     value <-       optional $ do         skipSpace         char '='         skipSpace-        msum (map quotedValue quotChars) <|> entityQuotedValue <|> unquotedValue-    return (convert identifierValue, value)+        msum (map quotedValue ['"', '\'', '`']) <|> entityQuotedValue <|> unquotedValue+    return (theIdentifier, value)   where     quotedValue q =-      do-        char q-        value <- -          fmap ((convert :: Builder -> Text) . mconcat) $ many $ -          fmap convert htmlEntity <|> fmap convert (satisfy (/= q))-        char q-        return value+      char q *> takeWhile (/= q) <* char q     unquotedValue =-      takeWhile1 isAlphaNum+      takeWhile1 $ flip all [not . isSpace, not . flip elem ['=', '<', '>', '/']] . (&)     entityQuotedValue =-      do-        q <- htmlEntity >>= \c -> bool mzero (return c) (elem c (map convert quotChars))-        fmap ((convert :: Builder -> Text) . mconcat) $ -          manyTill' (fmap convert anyChar) (htmlEntity >>= guard . (==) q)-    quotChars =-      ['"', '\'', '`']-+      fmap convert $ q *> manyTill' anyChar q+      where+        q = asciiCI "&quot;" -identifier :: Parser Text+identifier :: Parser Identifier identifier = -  takeWhile1 (flip any [isAlphaNum, flip elem ['_', '-', '!', '?']] . flip ($))+  fmap convert $ takeWhile1 (flip any [isAlphaNum, flip elem ['_', '-', '!', '?']] . flip ($))  comment :: Parser Text comment =@@ -125,19 +112,19 @@             (fmap convert (char '-'))             (content)))) -closingTag :: Parser ClosingTag+closingTag :: Parser Identifier closingTag =-  string "</" *> skipSpace *> fmap convert identifier <* skipSpace <* char '>'+  string "</" *> skipSpace *> identifier <* skipSpace <* char '>'  text :: Parser Text text =   fmap ((convert :: Builder -> Text) . mconcat) $ many1 $-  convert <$> htmlEntity <|> convert <$> nonTagChar+  convert <$> nonTagChar   where     nonTagChar =       shouldFail comment *> shouldFail closingTag *> shouldFail openingTag *> anyChar  shouldFail :: Parser a -> Parser () shouldFail p =-  ((p $> False) <|> pure True) >>= bool empty (pure ())+  join $ (p $> empty) <|> pure (pure ())