diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (c)2011, Chris Smith + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of Chris Smith nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/src/Text/Blaze/Renderer/XmlHtml.hs b/src/Text/Blaze/Renderer/XmlHtml.hs new file mode 100644 --- /dev/null +++ b/src/Text/Blaze/Renderer/XmlHtml.hs @@ -0,0 +1,74 @@ +-- | Renderer that supports rendering to xmlhtml forests. This is a port of +-- the Hexpat renderer. +-- +-- Warning: because this renderer doesn't directly create the output, but rather +-- an XML tree representation, it is impossible to render pre-escaped text. This +-- means that @preEscapedString@ will produce the same output as @string@. This +-- also applies to the functions @preEscapedText@, @preEscapedTextValue@... +-- +module Text.Blaze.Renderer.XmlHtml (renderHtml) where + +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import Text.Blaze.Internal +import Text.XmlHtml + + +-- | Render a 'ChoiceString' to Text. This is only meant to be used for +-- shorter strings, since it is inefficient for large strings. +-- +fromChoiceStringText :: ChoiceString -> Text +fromChoiceStringText (Static s) = getText s +fromChoiceStringText (String s) = T.pack s +fromChoiceStringText (Text s) = s +fromChoiceStringText (ByteString s) = T.decodeUtf8 s +fromChoiceStringText (PreEscaped s) = fromChoiceStringText s +fromChoiceStringText (External s) = fromChoiceStringText s +fromChoiceStringText (AppendChoiceString x y) = + fromChoiceStringText x `T.append` fromChoiceStringText y +fromChoiceStringText EmptyChoiceString = T.empty +{-# INLINE fromChoiceStringText #-} + + +-- | Render a 'ChoiceString' to an appending list of nodes +-- +fromChoiceString :: ChoiceString -> [Node] -> [Node] +fromChoiceString s@(Static _) = (TextNode (fromChoiceStringText s) :) +fromChoiceString s@(String _) = (TextNode (fromChoiceStringText s) :) +fromChoiceString s@(Text _) = (TextNode (fromChoiceStringText s) :) +fromChoiceString s@(ByteString _) = (TextNode (fromChoiceStringText s) :) +fromChoiceString (PreEscaped s) = fromChoiceString s +fromChoiceString (External s) = fromChoiceString s +fromChoiceString (AppendChoiceString x y) = + fromChoiceString x . fromChoiceString y +fromChoiceString EmptyChoiceString = id +{-# INLINE fromChoiceString #-} + + +-- | Render some 'Html' to an appending list of nodes +-- +renderNodes :: Html -> [Node] -> [Node] +renderNodes = go [] + where + go :: [(Text, Text)] -> HtmlM b -> [Node] -> [Node] + go attrs (Parent tag _ _ content) = + (Element (getText tag) attrs (go [] content []) :) + go attrs (Leaf tag _ _) = + (Element (getText tag) attrs [] :) + go attrs (AddAttribute key _ value content) = + go ((getText key, fromChoiceStringText value) : attrs) content + go attrs (AddCustomAttribute key _ value content) = + go ((fromChoiceStringText key, fromChoiceStringText value) : attrs) content + go _ (Content content) = fromChoiceString content + go attrs (Append h1 h2) = go attrs h1 . go attrs h2 + go _ Empty = id + {-# NOINLINE go #-} +{-# INLINE renderNodes #-} + +-- | Render HTML to an xmlhtml Document +-- +renderHtml :: Html -> Document +renderHtml html = HtmlDocument UTF8 Nothing (renderNodes html []) +{-# INLINE renderHtml #-} + diff --git a/src/Text/XmlHtml.hs b/src/Text/XmlHtml.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml.hs @@ -0,0 +1,93 @@ +------------------------------------------------------------------------------ +-- | Parsers and renderers for XML and HTML 5. Although the formats are +-- treated differently, the data types used by each are the same, which +-- makes it easy to write code that works with the element structure of +-- either XML or HTML 5 documents. +-- +-- Limitations: +-- +-- * The XML parser does not parse internal DOCTYPE subsets. They are just +-- stored as blocks of text, with minimal scanning done to match quotes +-- and brackets to determine the end. +-- +-- * Since DTDs are not parsed, the XML parser fails on entity references, +-- except for those defined internally. You cannot use this library for +-- parsing XML documents with entity references outside the predefined set. +-- +-- * The HTML 5 parser is not a compliant HTML parser. Instead, it is a +-- parser for valid HTML 5 content. It should only be used on content +-- that you have reason to believe is probably correct, since the +-- compatibility features of HTML 5 are missing. This is the wrong +-- library on which to build a web spider. +-- +-- * Both parsers accept fragments of documents, by which is meant that +-- they do not enforce the top-level structure of the document. Files +-- may contain more than one root element, for example. +module Text.XmlHtml ( + -- * Types + Document(..), + Node(..), + DocType(..), + ExternalID(..), + InternalSubset(..), + Encoding(..), + + -- * Manipulating documents + isTextNode, + isComment, + isElement, + tagName, + getAttribute, + hasAttribute, + setAttribute, + nodeText, + childNodes, + childElements, + childElementsTag, + childElementTag, + descendantNodes, + descendantElements, + descendantElementsTag, + descendantElementTag, + + -- * Parsing + parseXML, + parseHTML, + + -- * Rendering + render + ) where + +------------------------------------------------------------------------------ +import Blaze.ByteString.Builder (Builder) +import Data.ByteString (ByteString) + +import Text.XmlHtml.Common +import Text.XmlHtml.TextParser + +import qualified Text.XmlHtml.XML.Parse as XML +import qualified Text.XmlHtml.XML.Render as XML + +import qualified Text.XmlHtml.HTML.Parse as HTML +import qualified Text.XmlHtml.HTML.Render as HTML + + +------------------------------------------------------------------------------ +-- | Parses the given XML fragment. +parseXML :: String -> ByteString -> Either String Document +parseXML = parse XML.docFragment + + +------------------------------------------------------------------------------ +-- | Parses the given HTML fragment. This enables HTML quirks mode, which +-- changes the parsing algorithm to parse valid HTML 5 documents correctly. +parseHTML :: String -> ByteString -> Either String Document +parseHTML = parse HTML.docFragment + + +------------------------------------------------------------------------------ +-- | Renders a 'Document'. +render :: Document -> Builder +render (XmlDocument e dt ns) = XML.render e dt ns +render (HtmlDocument e dt ns) = HTML.render e dt ns + diff --git a/src/Text/XmlHtml/Common.hs b/src/Text/XmlHtml/Common.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/Common.hs @@ -0,0 +1,225 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} + +module Text.XmlHtml.Common where + +import Blaze.ByteString.Builder +import Data.Maybe + +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.Text.Encoding.Error as TE + +import Data.ByteString (ByteString) + + +------------------------------------------------------------------------------ +-- | Represents a document fragment, including the format, encoding, and +-- document type declaration as well as its content. +data Document = XmlDocument { + docEncoding :: !Encoding, + docType :: !(Maybe DocType), + docContent :: ![Node] + } + | HtmlDocument { + docEncoding :: !Encoding, + docType :: !(Maybe DocType), + docContent :: ![Node] + } + deriving (Eq, Show) + + +------------------------------------------------------------------------------ +-- | A node of a document structure. A node can be text, a comment, or an +-- element. XML processing instructions are intentionally omitted as a +-- simplification, and CDATA and plain text are both text nodes, since they +-- ought to be semantically interchangeable. +data Node = TextNode !Text + | Comment !Text + | Element { + elementTag :: !Text, + elementAttrs :: ![(Text, Text)], + elementChildren :: ![Node] + } + deriving (Eq, Show) + + +------------------------------------------------------------------------------ +-- | Determines whether the node is text or not. +isTextNode :: Node -> Bool +isTextNode (TextNode _) = True +isTextNode _ = False + + +------------------------------------------------------------------------------ +-- | Determines whether the node is a comment or not. +isComment :: Node -> Bool +isComment (Comment _) = True +isComment _ = False + + +------------------------------------------------------------------------------ +-- | Determines whether the node is an element or not. +isElement :: Node -> Bool +isElement (Element _ _ _) = True +isElement _ = False + + +------------------------------------------------------------------------------ +-- | Gives the tag name of an element, or 'Nothing' if the node isn't an +-- element. +tagName :: Node -> Maybe Text +tagName (Element t _ _) = Just t +tagName _ = Nothing + + +------------------------------------------------------------------------------ +-- | Retrieves the attribute with the given name. If the 'Node' is not an +-- element, the result is always 'Nothing' +getAttribute :: Text -> Node -> Maybe Text +getAttribute name (Element _ attrs _) = lookup name attrs +getAttribute _ _ = Nothing + + +------------------------------------------------------------------------------ +-- | Checks if a given attribute exists in a 'Node'. +hasAttribute :: Text -> Node -> Bool +hasAttribute name = isJust . getAttribute name + + +------------------------------------------------------------------------------ +-- | Sets the attribute name to the given value. If the 'Node' is not an +-- element, this is the identity. +setAttribute :: Text -> Text -> Node -> Node +setAttribute name val (Element t a c) = Element t newAttrs c + where newAttrs = (name, val) : filter ((/= name) . fst) a +setAttribute _ _ n = n + + +------------------------------------------------------------------------------ +-- | Gives the entire text content of a node, ignoring markup. +nodeText :: Node -> Text +nodeText (TextNode t) = t +nodeText (Comment _) = "" +nodeText (Element _ _ c) = T.concat (map nodeText c) + + +------------------------------------------------------------------------------ +-- | Gives the child nodes of the given node. Only elements have child nodes. +childNodes :: Node -> [Node] +childNodes (Element _ _ c) = c +childNodes _ = [] + + +------------------------------------------------------------------------------ +-- | Gives the child elements of the given node. +childElements :: Node -> [Node] +childElements = filter isElement . childNodes + + +------------------------------------------------------------------------------ +-- | Gives all of the child elements of the node with the given tag +-- name. +childElementsTag :: Text -> Node -> [Node] +childElementsTag tag = filter ((== Just tag) . tagName) . childNodes + + +------------------------------------------------------------------------------ +-- | Gives the first child element of the node with the given tag name, +-- or 'Nothing' if there is no such child element. +childElementTag :: Text -> Node -> Maybe Node +childElementTag tag = listToMaybe . childElementsTag tag + + +------------------------------------------------------------------------------ +-- | Gives the descendants of the given node in the order that they begin in +-- the document. +descendantNodes :: Node -> [Node] +descendantNodes = concatMap (\n -> n : descendantNodes n) . childNodes + +------------------------------------------------------------------------------ +-- | Gives the descendant elements of the given node, in the order that their +-- start tags appear in the document. +descendantElements :: Node -> [Node] +descendantElements = filter isElement . descendantNodes + + +------------------------------------------------------------------------------ +-- | Gives the descendant elements with a given tag name. +descendantElementsTag :: Text -> Node -> [Node] +descendantElementsTag tag = filter ((== Just tag) . tagName) . descendantNodes + + +------------------------------------------------------------------------------ +-- | Gives the first descendant element of the node with the given tag name, +-- or 'Nothing' if there is no such element. +descendantElementTag :: Text -> Node -> Maybe Node +descendantElementTag tag = listToMaybe . descendantElementsTag tag + + +------------------------------------------------------------------------------ +-- | A document type declaration. Note that DTD internal subsets are +-- currently unimplemented. +data DocType = DocType !Text !ExternalID !InternalSubset + deriving (Eq, Show) + + +------------------------------------------------------------------------------ +-- | An external ID, as in a document type declaration. This can be a +-- SYSTEM identifier, or a PUBLIC identifier, or can be omitted. +data ExternalID = Public !Text !Text + | System !Text + | NoExternalID + deriving (Eq, Show) + + +------------------------------------------------------------------------------ +-- | The internal subset is unparsed, but preserved in case it's actually +-- wanted. +data InternalSubset = InternalText !Text + | NoInternalSubset + deriving (Eq, Show) + + +------------------------------------------------------------------------------ +-- | The character encoding of a document. Currently only the required +-- character encodings are implemented. +data Encoding = UTF8 | UTF16BE | UTF16LE deriving (Eq, Show) + + +------------------------------------------------------------------------------ +-- | Retrieves the preferred name of a character encoding for embedding in +-- a document. +encodingName :: Encoding -> Text +encodingName UTF8 = "UTF-8" +encodingName UTF16BE = "UTF-16" +encodingName UTF16LE = "UTF-16" + + +------------------------------------------------------------------------------ +-- | Gets the encoding function from 'Text' to 'ByteString' for an encoding. +encoder :: Encoding -> Text -> ByteString +encoder UTF8 = T.encodeUtf8 +encoder UTF16BE = T.encodeUtf16BE +encoder UTF16LE = T.encodeUtf16LE + + +------------------------------------------------------------------------------ +-- | Gets the decoding function from 'ByteString' to 'Text' for an encoding. +decoder :: Encoding -> ByteString -> Text +decoder UTF8 = T.decodeUtf8With (TE.replace '\xFFFF') +decoder UTF16BE = T.decodeUtf16BEWith (TE.replace '\xFFFF') +decoder UTF16LE = T.decodeUtf16LEWith (TE.replace '\xFFFF') + + +------------------------------------------------------------------------------ +isUTF16 :: Encoding -> Bool +isUTF16 e = e == UTF16BE || e == UTF16LE + + +------------------------------------------------------------------------------ +fromText :: Encoding -> Text -> Builder +fromText e t = fromByteString (encoder e t) + diff --git a/src/Text/XmlHtml/Cursor.hs b/src/Text/XmlHtml/Cursor.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/Cursor.hs @@ -0,0 +1,409 @@ +------------------------------------------------------------------------------ +-- | A zipper for navigating and modifying XML trees. This is nearly the +-- same exposed interface as the @xml@ package in @Text.XML.Light.Cursor@, +-- with modifications as needed to adapt to different types. +module Text.XmlHtml.Cursor ( + -- * Cursor type + Cursor, + + -- * Conversion to and from cursors + fromNode, + fromNodes, + topNode, + topNodes, + current, + siblings, + + -- * Cursor navigation + parent, + root, + getChild, + firstChild, + lastChild, + left, + right, + nextDF, + + -- * Search + findChild, + findLeft, + findRight, + findRec, + + -- * Node classification + isRoot, + isFirst, + isLast, + isLeaf, + isChild, + hasChildren, + getNodeIndex, + + -- * Updates + setNode, + modifyNode, + modifyNodeM, + + -- * Insertions + insertLeft, + insertRight, + insertManyLeft, + insertManyRight, + insertFirstChild, + insertLastChild, + insertManyFirstChild, + insertManyLastChild, + insertGoLeft, + insertGoRight, + + -- * Deletions + removeLeft, + removeRight, + removeGoLeft, + removeGoRight, + removeGoUp + ) where + +import Control.Monad +import Data.Maybe +import Data.Text (Text) +import Text.XmlHtml + +------------------------------------------------------------------------------ +-- | Just the tag of an element +type Tag = (Text, [(Text, Text)]) + + +------------------------------------------------------------------------------ +-- | Reconstructs an element from a tag and a list of its children. +fromTag :: Tag -> [Node] -> Node +fromTag (t,a) c = Element t a c + + +------------------------------------------------------------------------------ +-- | A zipper for XML document forests. +data Cursor = Cursor { + current :: !Node, -- ^ Retrieves the current node of a 'Cursor' + lefts :: ![Node], -- right to left + rights :: ![Node], -- left to right + parents :: ![([Node], Tag, [Node])] -- parent's tag and siblings + } + deriving (Eq) + + +------------------------------------------------------------------------------ +-- | Builds a 'Cursor' for navigating a tree. That is, a forest with a single +-- root 'Node'. +fromNode :: Node -> Cursor +fromNode n = Cursor n [] [] [] + + +------------------------------------------------------------------------------ +-- | Builds a 'Cursor' for navigating a forest with the given list of roots. +-- The cursor is initially positioned at the left-most node. Gives 'Nothing' +-- if the list is empty. +fromNodes :: [Node] -> Maybe Cursor +fromNodes (n:ns) = Just (Cursor n [] ns []) +fromNodes [] = Nothing + + +------------------------------------------------------------------------------ +-- | Retrieves the root node containing the current cursor position. +topNode :: Cursor -> Node +topNode cur = current (root cur) + + +------------------------------------------------------------------------------ +-- | Retrieves the entire forest of 'Node's corresponding to a 'Cursor'. +topNodes :: Cursor -> [Node] +topNodes cur = siblings (root cur) + + +------------------------------------------------------------------------------ +-- | Retrieves a list of the 'Node's at the same level as the current position +-- of a cursor, including the current node. +siblings :: Cursor -> [Node] +siblings (Cursor cur ls rs _) = foldl (flip (:)) (cur:rs) ls + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' to its parent in the document. +parent :: Cursor -> Maybe Cursor +parent c@(Cursor _ _ _ ((ls,t,rs):ps)) + = Just (Cursor (fromTag t (siblings c)) ls rs ps) +parent _ = Nothing + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' up through parents to reach the root level. +root :: Cursor -> Cursor +root = until isRoot (fromJust . parent) + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' down to the indicated child index. +getChild :: Int -> Cursor -> Maybe Cursor +getChild i (Cursor n ls rs ps) = + case n of + Element t a cs -> let (lls, rest) = splitAt i cs in + if i >= length cs + then Nothing + else Just $ Cursor (head rest) + (reverse lls) + (tail rest) + ((ls, (t,a), rs):ps) + _ -> Nothing + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' down to its first child. +firstChild :: Cursor -> Maybe Cursor +firstChild = getChild 0 + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' down to its last child. +lastChild :: Cursor -> Maybe Cursor +lastChild (Cursor (Element t a c) ls rs ps) | not (null c) + = let rc = reverse c + in Just $ Cursor (head rc) (tail rc) [] ((ls, (t,a), rs):ps) +lastChild _ + = Nothing + + +------------------------------------------------------------------------------ +-- | Moves a 'Cursor' to its left sibling. +left :: Cursor -> Maybe Cursor +left (Cursor c (l:ls) rs ps) = Just (Cursor l ls (c:rs) ps) +left _ = Nothing + + +------------------------------------------------------------------------------ +-- | Moves a 'Cursor' to its right sibling. +right :: Cursor -> Maybe Cursor +right (Cursor c ls (r:rs) ps) = Just (Cursor r (c:ls) rs ps) +right _ = Nothing + + +------------------------------------------------------------------------------ +-- | Moves a 'Cursor' to the next node encountered in a depth-first search. +-- If it has children, this is equivalent to 'firstChild'. Otherwise, if it +-- has a right sibling, then this is equivalent to 'right'. Otherwise, the +-- cursor moves to the first right sibling of one of its parents. +nextDF :: Cursor -> Maybe Cursor +nextDF c = firstChild c `mplus` up c + where up x = right x `mplus` (up =<< parent x) + + +------------------------------------------------------------------------------ +-- | Repeats the given move until a 'Cursor' is obtained that matches the +-- predicate. +search :: (Cursor -> Bool) -- ^ predicate + -> (Cursor -> Maybe Cursor) -- ^ move + -> Cursor -- ^ starting point + -> Maybe Cursor +search p move c | p c = return c + | otherwise = search p move =<< move c + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' to the first child that matches the predicate. +findChild :: (Cursor -> Bool) -> Cursor -> Maybe Cursor +findChild p cur = search p right =<< firstChild cur + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' to the nearest left sibling that matches a +-- predicate. +findLeft :: (Cursor -> Bool) -> Cursor -> Maybe Cursor +findLeft p cur = search p left =<< left cur + + +------------------------------------------------------------------------------ +-- | Navigates a 'Cursor' to the nearest right sibling that matches a predicate. +findRight :: (Cursor -> Bool) -> Cursor -> Maybe Cursor +findRight p cur = search p right =<< right cur + + +------------------------------------------------------------------------------ +-- | Does a depth-first search for a descendant matching the predicate. This +-- can match the current cursor position. +findRec :: (Cursor -> Bool) -> Cursor -> Maybe Cursor +findRec p = search p nextDF + + +------------------------------------------------------------------------------ +-- | Determines if the 'Cursor' is at a root node. +isRoot :: Cursor -> Bool +isRoot cur = null (parents cur) + + +------------------------------------------------------------------------------ +-- | Determines if the 'Cursor' is at a first child. +isFirst :: Cursor -> Bool +isFirst cur = null (lefts cur) + + +------------------------------------------------------------------------------ +-- | Determines if the 'Cursor' is at a last child. +isLast :: Cursor -> Bool +isLast cur = null (rights cur) + + +------------------------------------------------------------------------------ +-- | Determines if the 'Cursor' is at a leaf node. +isLeaf :: Cursor -> Bool +isLeaf (Cursor (Element _ _ c) _ _ _) = null c +isLeaf _ = True + + +------------------------------------------------------------------------------ +-- | Determines if the 'Cursor' is at a child node (i.e., if it has a parent). +isChild :: Cursor -> Bool +isChild = not . isRoot + + +------------------------------------------------------------------------------ +-- | Determines if the 'Cursor' is at a non-leaf node (i.e., if it has +-- children). +hasChildren :: Cursor -> Bool +hasChildren = not . isLeaf + + +------------------------------------------------------------------------------ +-- | Gets the index of the 'Cursor' among its siblings. +getNodeIndex :: Cursor -> Int +getNodeIndex cur = length (lefts cur) + + +------------------------------------------------------------------------------ +-- | Replaces the current node. +setNode :: Node -> Cursor -> Cursor +setNode n cur = cur { current = n } + + +------------------------------------------------------------------------------ +-- | Modifies the current node by applying a function. +modifyNode :: (Node -> Node) -> Cursor -> Cursor +modifyNode f cur = setNode (f (current cur)) cur + + +------------------------------------------------------------------------------ +-- | Modifies the current node by applying an action in some functor. +modifyNodeM :: Functor m => (Node -> m Node) -> Cursor -> m Cursor +modifyNodeM f cur = flip setNode cur `fmap` f (current cur) + + +------------------------------------------------------------------------------ +-- | Inserts a new 'Node' to the left of the current position. +insertLeft :: Node -> Cursor -> Cursor +insertLeft n (Cursor nn ls rs ps) = Cursor nn (n:ls) rs ps + + +------------------------------------------------------------------------------ +-- | Inserts a new 'Node' to the right of the current position. +insertRight :: Node -> Cursor -> Cursor +insertRight n (Cursor nn ls rs ps) = Cursor nn ls (n:rs) ps + + +------------------------------------------------------------------------------ +-- | Inserts a list of new 'Node's to the left of the current position. +insertManyLeft :: [Node] -> Cursor -> Cursor +insertManyLeft ns (Cursor nn ls rs ps) = Cursor nn (reverse ns ++ ls) rs ps + + +------------------------------------------------------------------------------ +-- | Inserts a list of new 'Node's to the right of the current position. +insertManyRight :: [Node] -> Cursor -> Cursor +insertManyRight ns (Cursor nn ls rs ps) = Cursor nn ls (ns ++ rs) ps + + +------------------------------------------------------------------------------ +-- | Inserts a 'Node' as the first child of the current element. +insertFirstChild :: Node -> Cursor -> Maybe Cursor +insertFirstChild n (Cursor (Element t a c) ls rs ps) + = Just (Cursor (Element t a (n:c)) ls rs ps) +insertFirstChild _ _ + = Nothing + + +------------------------------------------------------------------------------ +-- | Inserts a 'Node' as the last child of the current element. +insertLastChild :: Node -> Cursor -> Maybe Cursor +insertLastChild n (Cursor (Element t a c) ls rs ps) + = Just (Cursor (Element t a (c ++ [n])) ls rs ps) +insertLastChild _ _ + = Nothing + + +------------------------------------------------------------------------------ +-- | Inserts a list of 'Node's as the first children of the current element. +insertManyFirstChild :: [Node] -> Cursor -> Maybe Cursor +insertManyFirstChild ns (Cursor (Element t a c) ls rs ps) + = Just (Cursor (Element t a (ns ++ c)) ls rs ps) +insertManyFirstChild _ _ + = Nothing + + +------------------------------------------------------------------------------ +-- | Inserts a list of 'Node's as the last children of the current element. +insertManyLastChild :: [Node] -> Cursor -> Maybe Cursor +insertManyLastChild ns (Cursor (Element t a c) ls rs ps) + = Just (Cursor (Element t a (c ++ ns)) ls rs ps) +insertManyLastChild _ _ + = Nothing + + +------------------------------------------------------------------------------ +-- | Inserts a new 'Node' to the left of the current position, and moves +-- left to the new node. +insertGoLeft :: Node -> Cursor -> Cursor +insertGoLeft n (Cursor nn ls rs ps) = Cursor n ls (nn:rs) ps + + +------------------------------------------------------------------------------ +-- | Inserts a new 'Node' to the right of the current position, and moves +-- right to the new node. +insertGoRight :: Node -> Cursor -> Cursor +insertGoRight n (Cursor nn ls rs ps) = Cursor n (nn:ls) rs ps + + +------------------------------------------------------------------------------ +-- | Removes the 'Node' to the left of the current position, if any. +removeLeft :: Cursor -> Maybe (Node, Cursor) +removeLeft (Cursor n (l:ls) rs ps) = Just (l, Cursor n ls rs ps) +removeLeft _ = Nothing + + +------------------------------------------------------------------------------ +-- | Removes the 'Node' to the right of the current position, if any. +removeRight :: Cursor -> Maybe (Node, Cursor) +removeRight (Cursor n ls (r:rs) ps) = Just (r, Cursor n ls rs ps) +removeRight _ = Nothing + + +------------------------------------------------------------------------------ +-- | Removes the current 'Node', and moves the Cursor to its left sibling, +-- if any. +removeGoLeft :: Cursor -> Maybe Cursor +removeGoLeft (Cursor _ (l:ls) rs ps) = Just (Cursor l ls rs ps) +removeGoLeft _ = Nothing + + +------------------------------------------------------------------------------ +-- | Removes the current 'Node', and moves the Cursor to its right sibling, +-- if any. +removeGoRight :: Cursor -> Maybe Cursor +removeGoRight (Cursor _ ls (r:rs) ps) = Just (Cursor r ls rs ps) +removeGoRight _ = Nothing + + +------------------------------------------------------------------------------ +-- | Removes the current 'Node', and moves the Cursor to its parent, if any. +removeGoUp :: Cursor -> Maybe Cursor +removeGoUp (Cursor _ ls rs ((lls, (t,a), rrs):ps)) + = Just (Cursor (Element t a children) lls rrs ps) + where + children = foldl (flip (:)) (rs) ls +removeGoUp _ = Nothing + diff --git a/src/Text/XmlHtml/HTML/Meta.hs b/src/Text/XmlHtml/HTML/Meta.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/HTML/Meta.hs @@ -0,0 +1,2431 @@ +{-# OPTIONS_GHC -O0 -fno-case-merge -fno-strictness -fno-cse #-} +{-# LANGUAGE OverloadedStrings #-} + +module Text.XmlHtml.HTML.Meta where + +import Data.Monoid +import Data.Map (Map) +import qualified Data.Map as M +import Data.Set (Set) +import qualified Data.Set as S +import Data.Text (Text) + +------------------------------------------------------------------------------ +-- Metadata used for HTML5 quirks mode. -- +------------------------------------------------------------------------------ + +voidTags :: Set Text +voidTags = S.fromAscList [ + "area", "base", "br", "col", "command", "embed", "hr", "img", "input", + "keygen", "link", "meta", "param", "source", "track", "wbr" + ] + +rawTextTags :: Set Text +rawTextTags = S.fromAscList [ "script", "style" ] + +rcdataTags :: Set Text +rcdataTags = S.fromAscList [ "textarea", "title" ] + +------------------------------------------------------------------------------ +-- | Tags which can be implicitly ended in case they are the last element in +-- their parent. This list actually includes all of the elements that have +-- any kind of omittable end tag, since in general when an element with an +-- omittable end tag isn't specified to be omittable in this way, it's just +-- because in a complete document it isn't expected to ever be the last thing +-- in its parent. We aren't interested in enforcing element structure rules, +-- so we'll allow it anyway. +endOmittableLast :: Set Text +endOmittableLast = S.fromAscList [ + "body", "colgroup", "dd", "dt", "head", "html", "li", "optgroup", + "option", "p", "rp", "rt", "tbody", "td", "tfoot", "th", "thead", "tr" + ] + +------------------------------------------------------------------------------ +-- | Tags which should be considered automatically ended in case one of a +-- certain set of tags pops up. +endOmittableNext :: Map Text (Set Text) +endOmittableNext = M.fromAscList [ + ("colgroup", S.fromAscList ["caption", "colgroup", "tbody", + "thead", "tfoot", "tr"]), + ("dd", S.fromAscList ["dd", "dt"]), + ("dt", S.fromAscList ["dd", "dt"]), + ("head", S.fromAscList ["body"]), + ("li", S.fromAscList ["li"]), + ("optgroup", S.fromAscList ["optgroup"]), + ("option", S.fromAscList ["optgroup", "option"]), + ("p", S.fromAscList ["address", "article", "aside", "blockquote", + "dir", "div", "dl", "fieldset", "footer", + "form", "h1", "h2", "h3", "h4", "h5", "h6", + "header", "hgroup", "hr", "menu", "nav", "ol", + "p", "pre", "section", "table", "ul"]), + ("rp", S.fromAscList ["rp", "rt"]), + ("rt", S.fromAscList ["rp", "rt"]), + ("tbody", S.fromAscList ["tbody", "tfoot", "thead"]), + ("td", S.fromAscList ["td", "th"]), + ("tfoot", S.fromAscList ["tbody", "tfoot", "thead"]), + ("th", S.fromAscList ["td", "th"]), + ("thead", S.fromAscList ["tbody", "tfoot", "thead"]), + ("tr", S.fromAscList ["tr"]) + ] + +predefinedRefs :: Map Text Text +predefinedRefs = mconcat $ map M.fromAscList [ + reftab1 + , reftab2 + , reftab3 + , reftab4 + , reftab5 + , reftab6 + , reftab7 + , reftab8 + , reftab9 + , reftab10 + , reftab11 + , reftab12 + , reftab13 + , reftab14 + , reftab15 + , reftab16 + , reftab17 + , reftab18 + , reftab19 + , reftab20 + , reftab21 + , reftab22 + , reftab23 + , reftab24 + , reftab25 + , reftab26 + , reftab27 + , reftab28 + , reftab29 + , reftab30 + , reftab31 + , reftab32 + , reftab33 + , reftab34 + , reftab35 + , reftab36 + , reftab37 + , reftab38 + , reftab39 + , reftab40 + , reftab41 + , reftab42 + , reftab43 + , reftab44 + , reftab45 + , reftab46 + , reftab47 + , reftab48 + , reftab49 + , reftab50 + , reftab51 + , reftab52 + , reftab53 + , reftab54 + , reftab55 + , reftab56 + , reftab57 + , reftab58 ] + + +reftab1 :: [(Text,Text)] +reftab1 = + [ ("AElig", "\x000C6"), + ("AMP", "\x00026"), + ("Aacute", "\x000C1"), + ("Abreve", "\x00102"), + ("Acirc", "\x000C2"), + ("Acy", "\x00410"), + ("Afr", "\x1D504"), + ("Agrave", "\x000C0"), + ("Alpha", "\x00391"), + ("Amacr", "\x00100"), + ("And", "\x02A53"), + ("Aogon", "\x00104"), + ("Aopf", "\x1D538"), + ("ApplyFunction", "\x02061"), + ("Aring", "\x000C5"), + ("Ascr", "\x1D49C"), + ("Assign", "\x02254"), + ("Atilde", "\x000C3"), + ("Auml", "\x000C4"), + ("Backslash", "\x02216"), + ("Barv", "\x02AE7"), + ("Barwed", "\x02306"), + ("Bcy", "\x00411"), + ("Because", "\x02235"), + ("Bernoullis", "\x0212C"), + ("Beta", "\x00392"), + ("Bfr", "\x1D505"), + ("Bopf", "\x1D539"), + ("Breve", "\x002D8"), + ("Bscr", "\x0212C"), + ("Bumpeq", "\x0224E"), + ("CHcy", "\x00427"), + ("COPY", "\x000A9"), + ("Cacute", "\x00106"), + ("Cap", "\x022D2"), + ("CapitalDifferentialD", "\x02145"), + ("Cayleys", "\x0212D") ] + +reftab2 :: [(Text,Text)] +reftab2 = + [ ("Ccaron", "\x0010C"), + ("Ccedil", "\x000C7"), + ("Ccirc", "\x00108"), + ("Cconint", "\x02230"), + ("Cdot", "\x0010A"), + ("Cedilla", "\x000B8"), + ("CenterDot", "\x000B7"), + ("Cfr", "\x0212D"), + ("Chi", "\x003A7"), + ("CircleDot", "\x02299"), + ("CircleMinus", "\x02296"), + ("CirclePlus", "\x02295"), + ("CircleTimes", "\x02297"), + ("ClockwiseContourIntegral", "\x02232"), + ("CloseCurlyDoubleQuote", "\x0201D"), + ("CloseCurlyQuote", "\x02019"), + ("Colon", "\x02237"), + ("Colone", "\x02A74"), + ("Congruent", "\x02261"), + ("Conint", "\x0222F"), + ("ContourIntegral", "\x0222E"), + ("Copf", "\x02102"), + ("Coproduct", "\x02210"), + ("CounterClockwiseContourIntegral", "\x02233"), + ("Cross", "\x02A2F"), + ("Cscr", "\x1D49E"), + ("Cup", "\x022D3"), + ("CupCap", "\x0224D"), + ("DD", "\x02145"), + ("DDotrahd", "\x02911"), + ("DJcy", "\x00402"), + ("DScy", "\x00405"), + ("DZcy", "\x0040F"), + ("Dagger", "\x02021"), + ("Darr", "\x021A1"), + ("Dashv", "\x02AE4"), + ("Dcaron", "\x0010E") ] + +reftab3 :: [(Text,Text)] +reftab3 = + [ ("Dcy", "\x00414"), + ("Del", "\x02207"), + ("Delta", "\x00394"), + ("Dfr", "\x1D507"), + ("DiacriticalAcute", "\x000B4"), + ("DiacriticalDot", "\x002D9"), + ("DiacriticalDoubleAcute", "\x002DD"), + ("DiacriticalGrave", "\x00060"), + ("DiacriticalTilde", "\x002DC"), + ("Diamond", "\x022C4"), + ("DifferentialD", "\x02146"), + ("Dopf", "\x1D53B"), + ("Dot", "\x000A8"), + ("DotDot", "\x020DC"), + ("DotEqual", "\x02250"), + ("DoubleContourIntegral", "\x0222F"), + ("DoubleDot", "\x000A8"), + ("DoubleDownArrow", "\x021D3"), + ("DoubleLeftArrow", "\x021D0"), + ("DoubleLeftRightArrow", "\x021D4"), + ("DoubleLeftTee", "\x02AE4"), + ("DoubleLongLeftArrow", "\x027F8"), + ("DoubleLongLeftRightArrow", "\x027FA"), + ("DoubleLongRightArrow", "\x027F9"), + ("DoubleRightArrow", "\x021D2"), + ("DoubleRightTee", "\x022A8"), + ("DoubleUpArrow", "\x021D1"), + ("DoubleUpDownArrow", "\x021D5"), + ("DoubleVerticalBar", "\x02225"), + ("DownArrow", "\x02193"), + ("DownArrowBar", "\x02913"), + ("DownArrowUpArrow", "\x021F5"), + ("DownBreve", "\x00311"), + ("DownLeftRightVector", "\x02950"), + ("DownLeftTeeVector", "\x0295E"), + ("DownLeftVector", "\x021BD"), + ("DownLeftVectorBar", "\x02956") ] + +reftab4 :: [(Text,Text)] +reftab4 = + [ ("DownRightTeeVector", "\x0295F"), + ("DownRightVector", "\x021C1"), + ("DownRightVectorBar", "\x02957"), + ("DownTee", "\x022A4"), + ("DownTeeArrow", "\x021A7"), + ("Downarrow", "\x021D3"), + ("Dscr", "\x1D49F"), + ("Dstrok", "\x00110"), + ("ENG", "\x0014A"), + ("ETH", "\x000D0"), + ("Eacute", "\x000C9"), + ("Ecaron", "\x0011A"), + ("Ecirc", "\x000CA"), + ("Ecy", "\x0042D"), + ("Edot", "\x00116"), + ("Efr", "\x1D508"), + ("Egrave", "\x000C8"), + ("Element", "\x02208"), + ("Emacr", "\x00112"), + ("EmptySmallSquare", "\x025FB"), + ("EmptyVerySmallSquare", "\x025AB"), + ("Eogon", "\x00118"), + ("Eopf", "\x1D53C"), + ("Epsilon", "\x00395"), + ("Equal", "\x02A75"), + ("EqualTilde", "\x02242"), + ("Equilibrium", "\x021CC"), + ("Escr", "\x02130"), + ("Esim", "\x02A73"), + ("Eta", "\x00397"), + ("Euml", "\x000CB"), + ("Exists", "\x02203"), + ("ExponentialE", "\x02147"), + ("Fcy", "\x00424"), + ("Ffr", "\x1D509"), + ("FilledSmallSquare", "\x025FC"), + ("FilledVerySmallSquare", "\x025AA") ] + +reftab5 :: [(Text,Text)] +reftab5 = + [ ("Fopf", "\x1D53D"), + ("ForAll", "\x02200"), + ("Fouriertrf", "\x02131"), + ("Fscr", "\x02131"), + ("GJcy", "\x00403"), + ("GT", "\x0003E"), + ("Gamma", "\x00393"), + ("Gammad", "\x003DC"), + ("Gbreve", "\x0011E"), + ("Gcedil", "\x00122"), + ("Gcirc", "\x0011C"), + ("Gcy", "\x00413"), + ("Gdot", "\x00120"), + ("Gfr", "\x1D50A"), + ("Gg", "\x022D9"), + ("Gopf", "\x1D53E"), + ("GreaterEqual", "\x02265"), + ("GreaterEqualLess", "\x022DB"), + ("GreaterFullEqual", "\x02267"), + ("GreaterGreater", "\x02AA2"), + ("GreaterLess", "\x02277"), + ("GreaterSlantEqual", "\x02A7E"), + ("GreaterTilde", "\x02273"), + ("Gscr", "\x1D4A2"), + ("Gt", "\x0226B"), + ("HARDcy", "\x0042A"), + ("Hacek", "\x002C7"), + ("Hat", "\x0005E"), + ("Hcirc", "\x00124"), + ("Hfr", "\x0210C"), + ("HilbertSpace", "\x0210B"), + ("Hopf", "\x0210D"), + ("HorizontalLine", "\x02500"), + ("Hscr", "\x0210B"), + ("Hstrok", "\x00126"), + ("HumpDownHump", "\x0224E"), + ("HumpEqual", "\x0224F") ] + +reftab6 :: [(Text,Text)] +reftab6 = + [ ("IEcy", "\x00415"), + ("IJlig", "\x00132"), + ("IOcy", "\x00401"), + ("Iacute", "\x000CD"), + ("Icirc", "\x000CE"), + ("Icy", "\x00418"), + ("Idot", "\x00130"), + ("Ifr", "\x02111"), + ("Igrave", "\x000CC"), + ("Im", "\x02111"), + ("Imacr", "\x0012A"), + ("ImaginaryI", "\x02148"), + ("Implies", "\x021D2"), + ("Int", "\x0222C"), + ("Integral", "\x0222B"), + ("Intersection", "\x022C2"), + ("InvisibleComma", "\x02063"), + ("InvisibleTimes", "\x02062"), + ("Iogon", "\x0012E"), + ("Iopf", "\x1D540"), + ("Iota", "\x00399"), + ("Iscr", "\x02110"), + ("Itilde", "\x00128"), + ("Iukcy", "\x00406"), + ("Iuml", "\x000CF"), + ("Jcirc", "\x00134"), + ("Jcy", "\x00419"), + ("Jfr", "\x1D50D"), + ("Jopf", "\x1D541"), + ("Jscr", "\x1D4A5"), + ("Jsercy", "\x00408"), + ("Jukcy", "\x00404"), + ("KHcy", "\x00425"), + ("KJcy", "\x0040C"), + ("Kappa", "\x0039A"), + ("Kcedil", "\x00136"), + ("Kcy", "\x0041A") ] + +reftab7 :: [(Text,Text)] +reftab7 = + [ ("Kfr", "\x1D50E"), + ("Kopf", "\x1D542"), + ("Kscr", "\x1D4A6"), + ("LJcy", "\x00409"), + ("LT", "\x0003C"), + ("Lacute", "\x00139"), + ("Lambda", "\x0039B"), + ("Lang", "\x027EA"), + ("Laplacetrf", "\x02112"), + ("Larr", "\x0219E"), + ("Lcaron", "\x0013D"), + ("Lcedil", "\x0013B"), + ("Lcy", "\x0041B"), + ("LeftAngleBracket", "\x027E8"), + ("LeftArrow", "\x02190"), + ("LeftArrowBar", "\x021E4"), + ("LeftArrowRightArrow", "\x021C6"), + ("LeftCeiling", "\x02308"), + ("LeftDoubleBracket", "\x027E6"), + ("LeftDownTeeVector", "\x02961"), + ("LeftDownVector", "\x021C3"), + ("LeftDownVectorBar", "\x02959"), + ("LeftFloor", "\x0230A"), + ("LeftRightArrow", "\x02194"), + ("LeftRightVector", "\x0294E"), + ("LeftTee", "\x022A3"), + ("LeftTeeArrow", "\x021A4"), + ("LeftTeeVector", "\x0295A"), + ("LeftTriangle", "\x022B2"), + ("LeftTriangleBar", "\x029CF"), + ("LeftTriangleEqual", "\x022B4"), + ("LeftUpDownVector", "\x02951"), + ("LeftUpTeeVector", "\x02960"), + ("LeftUpVector", "\x021BF"), + ("LeftUpVectorBar", "\x02958"), + ("LeftVector", "\x021BC"), + ("LeftVectorBar", "\x02952") ] + +reftab8 :: [(Text,Text)] +reftab8 = + [ ("Leftarrow", "\x021D0"), + ("Leftrightarrow", "\x021D4"), + ("LessEqualGreater", "\x022DA"), + ("LessFullEqual", "\x02266"), + ("LessGreater", "\x02276"), + ("LessLess", "\x02AA1"), + ("LessSlantEqual", "\x02A7D"), + ("LessTilde", "\x02272"), + ("Lfr", "\x1D50F"), + ("Ll", "\x022D8"), + ("Lleftarrow", "\x021DA"), + ("Lmidot", "\x0013F"), + ("LongLeftArrow", "\x027F5"), + ("LongLeftRightArrow", "\x027F7"), + ("LongRightArrow", "\x027F6"), + ("Longleftarrow", "\x027F8"), + ("Longleftrightarrow", "\x027FA"), + ("Longrightarrow", "\x027F9"), + ("Lopf", "\x1D543"), + ("LowerLeftArrow", "\x02199"), + ("LowerRightArrow", "\x02198"), + ("Lscr", "\x02112"), + ("Lsh", "\x021B0"), + ("Lstrok", "\x00141"), + ("Lt", "\x0226A"), + ("Map", "\x02905"), + ("Mcy", "\x0041C"), + ("MediumSpace", "\x0205F"), + ("Mellintrf", "\x02133"), + ("Mfr", "\x1D510"), + ("MinusPlus", "\x02213"), + ("Mopf", "\x1D544"), + ("Mscr", "\x02133"), + ("Mu", "\x0039C"), + ("NJcy", "\x0040A"), + ("Nacute", "\x00143"), + ("Ncaron", "\x00147") ] + +reftab9 :: [(Text,Text)] +reftab9 = + [ ("Ncedil", "\x00145"), + ("Ncy", "\x0041D"), + ("NegativeMediumSpace", "\x0200B"), + ("NegativeThickSpace", "\x0200B"), + ("NegativeThinSpace", "\x0200B"), + ("NegativeVeryThinSpace", "\x0200B"), + ("NestedGreaterGreater", "\x0226B"), + ("NestedLessLess", "\x0226A"), + ("NewLine", "\x0000A"), + ("Nfr", "\x1D511"), + ("NoBreak", "\x02060"), + ("NonBreakingSpace", "\x000A0"), + ("Nopf", "\x02115"), + ("Not", "\x02AEC"), + ("NotCongruent", "\x02262"), + ("NotCupCap", "\x0226D"), + ("NotDoubleVerticalBar", "\x02226"), + ("NotElement", "\x02209"), + ("NotEqual", "\x02260"), + ("NotEqualTilde", "\x02242\x00338"), + ("NotExists", "\x02204"), + ("NotGreater", "\x0226F"), + ("NotGreaterEqual", "\x02271"), + ("NotGreaterFullEqual", "\x02267\x00338"), + ("NotGreaterGreater", "\x0226B\x00338"), + ("NotGreaterLess", "\x02279"), + ("NotGreaterSlantEqual", "\x02A7E\x00338"), + ("NotGreaterTilde", "\x02275"), + ("NotHumpDownHump", "\x0224E\x00338"), + ("NotHumpEqual", "\x0224F\x00338"), + ("NotLeftTriangle", "\x022EA"), + ("NotLeftTriangleBar", "\x029CF\x00338"), + ("NotLeftTriangleEqual", "\x022EC"), + ("NotLess", "\x0226E"), + ("NotLessEqual", "\x02270"), + ("NotLessGreater", "\x02278"), + ("NotLessLess", "\x0226A\x00338") ] + +reftab10 :: [(Text,Text)] +reftab10 = + [ ("NotLessSlantEqual", "\x02A7D\x00338"), + ("NotLessTilde", "\x02274"), + ("NotNestedGreaterGreater", "\x02AA2\x00338"), + ("NotNestedLessLess", "\x02AA1\x00338"), + ("NotPrecedes", "\x02280"), + ("NotPrecedesEqual", "\x02AAF\x00338"), + ("NotPrecedesSlantEqual", "\x022E0"), + ("NotReverseElement", "\x0220C"), + ("NotRightTriangle", "\x022EB"), + ("NotRightTriangleBar", "\x029D0\x00338"), + ("NotRightTriangleEqual", "\x022ED"), + ("NotSquareSubset", "\x0228F\x00338"), + ("NotSquareSubsetEqual", "\x022E2"), + ("NotSquareSuperset", "\x02290\x00338"), + ("NotSquareSupersetEqual", "\x022E3"), + ("NotSubset", "\x02282\x020D2"), + ("NotSubsetEqual", "\x02288"), + ("NotSucceeds", "\x02281"), + ("NotSucceedsEqual", "\x02AB0\x00338"), + ("NotSucceedsSlantEqual", "\x022E1"), + ("NotSucceedsTilde", "\x0227F\x00338"), + ("NotSuperset", "\x02283\x020D2"), + ("NotSupersetEqual", "\x02289"), + ("NotTilde", "\x02241"), + ("NotTildeEqual", "\x02244"), + ("NotTildeFullEqual", "\x02247"), + ("NotTildeTilde", "\x02249"), + ("NotVerticalBar", "\x02224"), + ("Nscr", "\x1D4A9"), + ("Ntilde", "\x000D1"), + ("Nu", "\x0039D"), + ("OElig", "\x00152"), + ("Oacute", "\x000D3"), + ("Ocirc", "\x000D4"), + ("Ocy", "\x0041E"), + ("Odblac", "\x00150"), + ("Ofr", "\x1D512") ] + +reftab11 :: [(Text,Text)] +reftab11 = + [ ("Ograve", "\x000D2"), + ("Omacr", "\x0014C"), + ("Omega", "\x003A9"), + ("Omicron", "\x0039F"), + ("Oopf", "\x1D546"), + ("OpenCurlyDoubleQuote", "\x0201C"), + ("OpenCurlyQuote", "\x02018"), + ("Or", "\x02A54"), + ("Oscr", "\x1D4AA"), + ("Oslash", "\x000D8"), + ("Otilde", "\x000D5"), + ("Otimes", "\x02A37"), + ("Ouml", "\x000D6"), + ("OverBar", "\x0203E"), + ("OverBrace", "\x023DE"), + ("OverBracket", "\x023B4"), + ("OverParenthesis", "\x023DC"), + ("PartialD", "\x02202"), + ("Pcy", "\x0041F"), + ("Pfr", "\x1D513"), + ("Phi", "\x003A6"), + ("Pi", "\x003A0"), + ("PlusMinus", "\x000B1"), + ("Poincareplane", "\x0210C"), + ("Popf", "\x02119"), + ("Pr", "\x02ABB"), + ("Precedes", "\x0227A"), + ("PrecedesEqual", "\x02AAF"), + ("PrecedesSlantEqual", "\x0227C"), + ("PrecedesTilde", "\x0227E"), + ("Prime", "\x02033"), + ("Product", "\x0220F"), + ("Proportion", "\x02237"), + ("Proportional", "\x0221D"), + ("Pscr", "\x1D4AB"), + ("Psi", "\x003A8"), + ("QUOT", "\x00022") ] + +reftab12 :: [(Text,Text)] +reftab12 = + [ ("Qfr", "\x1D514"), + ("Qopf", "\x0211A"), + ("Qscr", "\x1D4AC"), + ("RBarr", "\x02910"), + ("REG", "\x000AE"), + ("Racute", "\x00154"), + ("Rang", "\x027EB"), + ("Rarr", "\x021A0"), + ("Rarrtl", "\x02916"), + ("Rcaron", "\x00158"), + ("Rcedil", "\x00156"), + ("Rcy", "\x00420"), + ("Re", "\x0211C"), + ("ReverseElement", "\x0220B"), + ("ReverseEquilibrium", "\x021CB"), + ("ReverseUpEquilibrium", "\x0296F"), + ("Rfr", "\x0211C"), + ("Rho", "\x003A1"), + ("RightAngleBracket", "\x027E9"), + ("RightArrow", "\x02192"), + ("RightArrowBar", "\x021E5"), + ("RightArrowLeftArrow", "\x021C4"), + ("RightCeiling", "\x02309"), + ("RightDoubleBracket", "\x027E7"), + ("RightDownTeeVector", "\x0295D"), + ("RightDownVector", "\x021C2"), + ("RightDownVectorBar", "\x02955"), + ("RightFloor", "\x0230B"), + ("RightTee", "\x022A2"), + ("RightTeeArrow", "\x021A6"), + ("RightTeeVector", "\x0295B"), + ("RightTriangle", "\x022B3"), + ("RightTriangleBar", "\x029D0"), + ("RightTriangleEqual", "\x022B5"), + ("RightUpDownVector", "\x0294F"), + ("RightUpTeeVector", "\x0295C"), + ("RightUpVector", "\x021BE") ] + +reftab13 :: [(Text,Text)] +reftab13 = + [ ("RightUpVectorBar", "\x02954"), + ("RightVector", "\x021C0"), + ("RightVectorBar", "\x02953"), + ("Rightarrow", "\x021D2"), + ("Ropf", "\x0211D"), + ("RoundImplies", "\x02970"), + ("Rrightarrow", "\x021DB"), + ("Rscr", "\x0211B"), + ("Rsh", "\x021B1"), + ("RuleDelayed", "\x029F4"), + ("SHCHcy", "\x00429"), + ("SHcy", "\x00428"), + ("SOFTcy", "\x0042C"), + ("Sacute", "\x0015A"), + ("Sc", "\x02ABC"), + ("Scaron", "\x00160"), + ("Scedil", "\x0015E"), + ("Scirc", "\x0015C"), + ("Scy", "\x00421"), + ("Sfr", "\x1D516"), + ("ShortDownArrow", "\x02193"), + ("ShortLeftArrow", "\x02190"), + ("ShortRightArrow", "\x02192"), + ("ShortUpArrow", "\x02191"), + ("Sigma", "\x003A3"), + ("SmallCircle", "\x02218"), + ("Sopf", "\x1D54A"), + ("Sqrt", "\x0221A"), + ("Square", "\x025A1"), + ("SquareIntersection", "\x02293"), + ("SquareSubset", "\x0228F"), + ("SquareSubsetEqual", "\x02291"), + ("SquareSuperset", "\x02290"), + ("SquareSupersetEqual", "\x02292"), + ("SquareUnion", "\x02294"), + ("Sscr", "\x1D4AE"), + ("Star", "\x022C6") ] + +reftab14 :: [(Text,Text)] +reftab14 = + [ ("Sub", "\x022D0"), + ("Subset", "\x022D0"), + ("SubsetEqual", "\x02286"), + ("Succeeds", "\x0227B"), + ("SucceedsEqual", "\x02AB0"), + ("SucceedsSlantEqual", "\x0227D"), + ("SucceedsTilde", "\x0227F"), + ("SuchThat", "\x0220B"), + ("Sum", "\x02211"), + ("Sup", "\x022D1"), + ("Superset", "\x02283"), + ("SupersetEqual", "\x02287"), + ("Supset", "\x022D1"), + ("THORN", "\x000DE"), + ("TRADE", "\x02122"), + ("TSHcy", "\x0040B"), + ("TScy", "\x00426"), + ("Tab", "\x00009"), + ("Tau", "\x003A4"), + ("Tcaron", "\x00164"), + ("Tcedil", "\x00162"), + ("Tcy", "\x00422"), + ("Tfr", "\x1D517"), + ("Therefore", "\x02234"), + ("Theta", "\x00398"), + ("ThickSpace", "\x0205F\x0200A"), + ("ThinSpace", "\x02009"), + ("Tilde", "\x0223C"), + ("TildeEqual", "\x02243"), + ("TildeFullEqual", "\x02245"), + ("TildeTilde", "\x02248"), + ("Topf", "\x1D54B"), + ("TripleDot", "\x020DB"), + ("Tscr", "\x1D4AF"), + ("Tstrok", "\x00166"), + ("Uacute", "\x000DA"), + ("Uarr", "\x0219F") ] + +reftab15 :: [(Text,Text)] +reftab15 = + [ ("Uarrocir", "\x02949"), + ("Ubrcy", "\x0040E"), + ("Ubreve", "\x0016C"), + ("Ucirc", "\x000DB"), + ("Ucy", "\x00423"), + ("Udblac", "\x00170"), + ("Ufr", "\x1D518"), + ("Ugrave", "\x000D9"), + ("Umacr", "\x0016A"), + ("UnderBar", "\x0005F"), + ("UnderBrace", "\x023DF"), + ("UnderBracket", "\x023B5"), + ("UnderParenthesis", "\x023DD"), + ("Union", "\x022C3"), + ("UnionPlus", "\x0228E"), + ("Uogon", "\x00172"), + ("Uopf", "\x1D54C"), + ("UpArrow", "\x02191"), + ("UpArrowBar", "\x02912"), + ("UpArrowDownArrow", "\x021C5"), + ("UpDownArrow", "\x02195"), + ("UpEquilibrium", "\x0296E"), + ("UpTee", "\x022A5"), + ("UpTeeArrow", "\x021A5"), + ("Uparrow", "\x021D1"), + ("Updownarrow", "\x021D5"), + ("UpperLeftArrow", "\x02196"), + ("UpperRightArrow", "\x02197"), + ("Upsi", "\x003D2"), + ("Upsilon", "\x003A5"), + ("Uring", "\x0016E"), + ("Uscr", "\x1D4B0"), + ("Utilde", "\x00168"), + ("Uuml", "\x000DC"), + ("VDash", "\x022AB"), + ("Vbar", "\x02AEB"), + ("Vcy", "\x00412") ] + +reftab16 :: [(Text,Text)] +reftab16 = + [ ("Vdash", "\x022A9"), + ("Vdashl", "\x02AE6"), + ("Vee", "\x022C1"), + ("Verbar", "\x02016"), + ("Vert", "\x02016"), + ("VerticalBar", "\x02223"), + ("VerticalLine", "\x0007C"), + ("VerticalSeparator", "\x02758"), + ("VerticalTilde", "\x02240"), + ("VeryThinSpace", "\x0200A"), + ("Vfr", "\x1D519"), + ("Vopf", "\x1D54D"), + ("Vscr", "\x1D4B1"), + ("Vvdash", "\x022AA"), + ("Wcirc", "\x00174"), + ("Wedge", "\x022C0"), + ("Wfr", "\x1D51A"), + ("Wopf", "\x1D54E"), + ("Wscr", "\x1D4B2"), + ("Xfr", "\x1D51B"), + ("Xi", "\x0039E"), + ("Xopf", "\x1D54F"), + ("Xscr", "\x1D4B3"), + ("YAcy", "\x0042F"), + ("YIcy", "\x00407"), + ("YUcy", "\x0042E"), + ("Yacute", "\x000DD"), + ("Ycirc", "\x00176"), + ("Ycy", "\x0042B"), + ("Yfr", "\x1D51C"), + ("Yopf", "\x1D550"), + ("Yscr", "\x1D4B4"), + ("Yuml", "\x00178"), + ("ZHcy", "\x00416"), + ("Zacute", "\x00179"), + ("Zcaron", "\x0017D"), + ("Zcy", "\x00417") ] + +reftab17 :: [(Text,Text)] +reftab17 = + [ ("Zdot", "\x0017B"), + ("ZeroWidthSpace", "\x0200B"), + ("Zeta", "\x00396"), + ("Zfr", "\x02128"), + ("Zopf", "\x02124"), + ("Zscr", "\x1D4B5"), + ("aacute", "\x000E1"), + ("abreve", "\x00103"), + ("ac", "\x0223E"), + ("acE", "\x0223E\x00333"), + ("acd", "\x0223F"), + ("acirc", "\x000E2"), + ("acute", "\x000B4"), + ("acy", "\x00430"), + ("aelig", "\x000E6"), + ("af", "\x02061"), + ("afr", "\x1D51E"), + ("agrave", "\x000E0"), + ("alefsym", "\x02135"), + ("aleph", "\x02135"), + ("alpha", "\x003B1"), + ("amacr", "\x00101"), + ("amalg", "\x02A3F"), + ("amp", "\x00026"), + ("and", "\x02227"), + ("andand", "\x02A55"), + ("andd", "\x02A5C"), + ("andslope", "\x02A58"), + ("andv", "\x02A5A"), + ("ang", "\x02220"), + ("ange", "\x029A4"), + ("angle", "\x02220"), + ("angmsd", "\x02221"), + ("angmsdaa", "\x029A8"), + ("angmsdab", "\x029A9"), + ("angmsdac", "\x029AA"), + ("angmsdad", "\x029AB") ] + +reftab18 :: [(Text,Text)] +reftab18 = + [ ("angmsdae", "\x029AC"), + ("angmsdaf", "\x029AD"), + ("angmsdag", "\x029AE"), + ("angmsdah", "\x029AF"), + ("angrt", "\x0221F"), + ("angrtvb", "\x022BE"), + ("angrtvbd", "\x0299D"), + ("angsph", "\x02222"), + ("angst", "\x000C5"), + ("angzarr", "\x0237C"), + ("aogon", "\x00105"), + ("aopf", "\x1D552"), + ("ap", "\x02248"), + ("apE", "\x02A70"), + ("apacir", "\x02A6F"), + ("ape", "\x0224A"), + ("apid", "\x0224B"), + ("apos", "\x00027"), + ("approx", "\x02248"), + ("approxeq", "\x0224A"), + ("aring", "\x000E5"), + ("ascr", "\x1D4B6"), + ("ast", "\x0002A"), + ("asymp", "\x02248"), + ("asympeq", "\x0224D"), + ("atilde", "\x000E3"), + ("auml", "\x000E4"), + ("awconint", "\x02233"), + ("awint", "\x02A11"), + ("bNot", "\x02AED"), + ("backcong", "\x0224C"), + ("backepsilon", "\x003F6"), + ("backprime", "\x02035"), + ("backsim", "\x0223D"), + ("backsimeq", "\x022CD"), + ("barvee", "\x022BD"), + ("barwed", "\x02305") ] + +reftab19 :: [(Text,Text)] +reftab19 = + [ ("barwedge", "\x02305"), + ("bbrk", "\x023B5"), + ("bbrktbrk", "\x023B6"), + ("bcong", "\x0224C"), + ("bcy", "\x00431"), + ("bdquo", "\x0201E"), + ("becaus", "\x02235"), + ("because", "\x02235"), + ("bemptyv", "\x029B0"), + ("bepsi", "\x003F6"), + ("bernou", "\x0212C"), + ("beta", "\x003B2"), + ("beth", "\x02136"), + ("between", "\x0226C"), + ("bfr", "\x1D51F"), + ("bigcap", "\x022C2"), + ("bigcirc", "\x025EF"), + ("bigcup", "\x022C3"), + ("bigodot", "\x02A00"), + ("bigoplus", "\x02A01"), + ("bigotimes", "\x02A02"), + ("bigsqcup", "\x02A06"), + ("bigstar", "\x02605"), + ("bigtriangledown", "\x025BD"), + ("bigtriangleup", "\x025B3"), + ("biguplus", "\x02A04"), + ("bigvee", "\x022C1"), + ("bigwedge", "\x022C0"), + ("bkarow", "\x0290D"), + ("blacklozenge", "\x029EB"), + ("blacksquare", "\x025AA"), + ("blacktriangle", "\x025B4"), + ("blacktriangledown", "\x025BE"), + ("blacktriangleleft", "\x025C2"), + ("blacktriangleright", "\x025B8"), + ("blank", "\x02423"), + ("blk12", "\x02592") ] + +reftab20 :: [(Text,Text)] +reftab20 = + [ ("blk14", "\x02591"), + ("blk34", "\x02593"), + ("block", "\x02588"), + ("bne", "\x0003D\x020E5"), + ("bnequiv", "\x02261\x020E5"), + ("bnot", "\x02310"), + ("bopf", "\x1D553"), + ("bot", "\x022A5"), + ("bottom", "\x022A5"), + ("bowtie", "\x022C8"), + ("boxDL", "\x02557"), + ("boxDR", "\x02554"), + ("boxDl", "\x02556"), + ("boxDr", "\x02553"), + ("boxH", "\x02550"), + ("boxHD", "\x02566"), + ("boxHU", "\x02569"), + ("boxHd", "\x02564"), + ("boxHu", "\x02567"), + ("boxUL", "\x0255D"), + ("boxUR", "\x0255A"), + ("boxUl", "\x0255C"), + ("boxUr", "\x02559"), + ("boxV", "\x02551"), + ("boxVH", "\x0256C"), + ("boxVL", "\x02563"), + ("boxVR", "\x02560"), + ("boxVh", "\x0256B"), + ("boxVl", "\x02562"), + ("boxVr", "\x0255F"), + ("boxbox", "\x029C9"), + ("boxdL", "\x02555"), + ("boxdR", "\x02552"), + ("boxdl", "\x02510"), + ("boxdr", "\x0250C"), + ("boxh", "\x02500"), + ("boxhD", "\x02565") ] + +reftab21 :: [(Text,Text)] +reftab21 = + [ ("boxhU", "\x02568"), + ("boxhd", "\x0252C"), + ("boxhu", "\x02534"), + ("boxminus", "\x0229F"), + ("boxplus", "\x0229E"), + ("boxtimes", "\x022A0"), + ("boxuL", "\x0255B"), + ("boxuR", "\x02558"), + ("boxul", "\x02518"), + ("boxur", "\x02514"), + ("boxv", "\x02502"), + ("boxvH", "\x0256A"), + ("boxvL", "\x02561"), + ("boxvR", "\x0255E"), + ("boxvh", "\x0253C"), + ("boxvl", "\x02524"), + ("boxvr", "\x0251C"), + ("bprime", "\x02035"), + ("breve", "\x002D8"), + ("brvbar", "\x000A6"), + ("bscr", "\x1D4B7"), + ("bsemi", "\x0204F"), + ("bsim", "\x0223D"), + ("bsime", "\x022CD"), + ("bsol", "\x0005C"), + ("bsolb", "\x029C5"), + ("bsolhsub", "\x027C8"), + ("bull", "\x02022"), + ("bullet", "\x02022"), + ("bump", "\x0224E"), + ("bumpE", "\x02AAE"), + ("bumpe", "\x0224F"), + ("bumpeq", "\x0224F"), + ("cacute", "\x00107"), + ("cap", "\x02229"), + ("capand", "\x02A44"), + ("capbrcup", "\x02A49") ] + +reftab22 :: [(Text,Text)] +reftab22 = + [ ("capcap", "\x02A4B"), + ("capcup", "\x02A47"), + ("capdot", "\x02A40"), + ("caps", "\x02229\x0FE00"), + ("caret", "\x02041"), + ("caron", "\x002C7"), + ("ccaps", "\x02A4D"), + ("ccaron", "\x0010D"), + ("ccedil", "\x000E7"), + ("ccirc", "\x00109"), + ("ccups", "\x02A4C"), + ("ccupssm", "\x02A50"), + ("cdot", "\x0010B"), + ("cedil", "\x000B8"), + ("cemptyv", "\x029B2"), + ("cent", "\x000A2"), + ("centerdot", "\x000B7"), + ("cfr", "\x1D520"), + ("chcy", "\x00447"), + ("check", "\x02713"), + ("checkmark", "\x02713"), + ("chi", "\x003C7"), + ("cir", "\x025CB"), + ("cirE", "\x029C3"), + ("circ", "\x002C6"), + ("circeq", "\x02257"), + ("circlearrowleft", "\x021BA"), + ("circlearrowright", "\x021BB"), + ("circledR", "\x000AE"), + ("circledS", "\x024C8"), + ("circledast", "\x0229B"), + ("circledcirc", "\x0229A"), + ("circleddash", "\x0229D"), + ("cire", "\x02257"), + ("cirfnint", "\x02A10"), + ("cirmid", "\x02AEF"), + ("cirscir", "\x029C2") ] + +reftab23 :: [(Text,Text)] +reftab23 = + [ ("clubs", "\x02663"), + ("clubsuit", "\x02663"), + ("colon", "\x0003A"), + ("colone", "\x02254"), + ("coloneq", "\x02254"), + ("comma", "\x0002C"), + ("commat", "\x00040"), + ("comp", "\x02201"), + ("compfn", "\x02218"), + ("complement", "\x02201"), + ("complexes", "\x02102"), + ("cong", "\x02245"), + ("congdot", "\x02A6D"), + ("conint", "\x0222E"), + ("copf", "\x1D554"), + ("coprod", "\x02210"), + ("copy", "\x000A9"), + ("copysr", "\x02117"), + ("crarr", "\x021B5"), + ("cross", "\x02717"), + ("cscr", "\x1D4B8"), + ("csub", "\x02ACF"), + ("csube", "\x02AD1"), + ("csup", "\x02AD0"), + ("csupe", "\x02AD2"), + ("ctdot", "\x022EF"), + ("cudarrl", "\x02938"), + ("cudarrr", "\x02935"), + ("cuepr", "\x022DE"), + ("cuesc", "\x022DF"), + ("cularr", "\x021B6"), + ("cularrp", "\x0293D"), + ("cup", "\x0222A"), + ("cupbrcap", "\x02A48"), + ("cupcap", "\x02A46"), + ("cupcup", "\x02A4A"), + ("cupdot", "\x0228D") ] + +reftab24 :: [(Text,Text)] +reftab24 = + [ ("cupor", "\x02A45"), + ("cups", "\x0222A\x0FE00"), + ("curarr", "\x021B7"), + ("curarrm", "\x0293C"), + ("curlyeqprec", "\x022DE"), + ("curlyeqsucc", "\x022DF"), + ("curlyvee", "\x022CE"), + ("curlywedge", "\x022CF"), + ("curren", "\x000A4"), + ("curvearrowleft", "\x021B6"), + ("curvearrowright", "\x021B7"), + ("cuvee", "\x022CE"), + ("cuwed", "\x022CF"), + ("cwconint", "\x02232"), + ("cwint", "\x02231"), + ("cylcty", "\x0232D"), + ("dArr", "\x021D3"), + ("dHar", "\x02965"), + ("dagger", "\x02020"), + ("daleth", "\x02138"), + ("darr", "\x02193"), + ("dash", "\x02010"), + ("dashv", "\x022A3"), + ("dbkarow", "\x0290F"), + ("dblac", "\x002DD"), + ("dcaron", "\x0010F"), + ("dcy", "\x00434"), + ("dd", "\x02146"), + ("ddagger", "\x02021"), + ("ddarr", "\x021CA"), + ("ddotseq", "\x02A77"), + ("deg", "\x000B0"), + ("delta", "\x003B4"), + ("demptyv", "\x029B1"), + ("dfisht", "\x0297F"), + ("dfr", "\x1D521"), + ("dharl", "\x021C3") ] + +reftab25 :: [(Text,Text)] +reftab25 = + [ ("dharr", "\x021C2"), + ("diam", "\x022C4"), + ("diamond", "\x022C4"), + ("diamondsuit", "\x02666"), + ("diams", "\x02666"), + ("die", "\x000A8"), + ("digamma", "\x003DD"), + ("disin", "\x022F2"), + ("div", "\x000F7"), + ("divide", "\x000F7"), + ("divideontimes", "\x022C7"), + ("divonx", "\x022C7"), + ("djcy", "\x00452"), + ("dlcorn", "\x0231E"), + ("dlcrop", "\x0230D"), + ("dollar", "\x00024"), + ("dopf", "\x1D555"), + ("dot", "\x002D9"), + ("doteq", "\x02250"), + ("doteqdot", "\x02251"), + ("dotminus", "\x02238"), + ("dotplus", "\x02214"), + ("dotsquare", "\x022A1"), + ("doublebarwedge", "\x02306"), + ("downarrow", "\x02193"), + ("downdownarrows", "\x021CA"), + ("downharpoonleft", "\x021C3"), + ("downharpoonright", "\x021C2"), + ("drbkarow", "\x02910"), + ("drcorn", "\x0231F"), + ("drcrop", "\x0230C"), + ("dscr", "\x1D4B9"), + ("dscy", "\x00455"), + ("dsol", "\x029F6"), + ("dstrok", "\x00111"), + ("dtdot", "\x022F1"), + ("dtri", "\x025BF") ] + +reftab26 :: [(Text,Text)] +reftab26 = + [ ("dtrif", "\x025BE"), + ("duarr", "\x021F5"), + ("duhar", "\x0296F"), + ("dwangle", "\x029A6"), + ("dzcy", "\x0045F"), + ("dzigrarr", "\x027FF"), + ("eDDot", "\x02A77"), + ("eDot", "\x02251"), + ("eacute", "\x000E9"), + ("easter", "\x02A6E"), + ("ecaron", "\x0011B"), + ("ecir", "\x02256"), + ("ecirc", "\x000EA"), + ("ecolon", "\x02255"), + ("ecy", "\x0044D"), + ("edot", "\x00117"), + ("ee", "\x02147"), + ("efDot", "\x02252"), + ("efr", "\x1D522"), + ("eg", "\x02A9A"), + ("egrave", "\x000E8"), + ("egs", "\x02A96"), + ("egsdot", "\x02A98"), + ("el", "\x02A99"), + ("elinters", "\x023E7"), + ("ell", "\x02113"), + ("els", "\x02A95"), + ("elsdot", "\x02A97"), + ("emacr", "\x00113"), + ("empty", "\x02205"), + ("emptyset", "\x02205"), + ("emptyv", "\x02205"), + ("emsp", "\x02003"), + ("emsp13", "\x02004"), + ("emsp14", "\x02005"), + ("eng", "\x0014B"), + ("ensp", "\x02002") ] + +reftab27 :: [(Text,Text)] +reftab27 = + [ ("eogon", "\x00119"), + ("eopf", "\x1D556"), + ("epar", "\x022D5"), + ("eparsl", "\x029E3"), + ("eplus", "\x02A71"), + ("epsi", "\x003B5"), + ("epsilon", "\x003B5"), + ("epsiv", "\x003F5"), + ("eqcirc", "\x02256"), + ("eqcolon", "\x02255"), + ("eqsim", "\x02242"), + ("eqslantgtr", "\x02A96"), + ("eqslantless", "\x02A95"), + ("equals", "\x0003D"), + ("equest", "\x0225F"), + ("equiv", "\x02261"), + ("equivDD", "\x02A78"), + ("eqvparsl", "\x029E5"), + ("erDot", "\x02253"), + ("erarr", "\x02971"), + ("escr", "\x0212F"), + ("esdot", "\x02250"), + ("esim", "\x02242"), + ("eta", "\x003B7"), + ("eth", "\x000F0"), + ("euml", "\x000EB"), + ("euro", "\x020AC"), + ("excl", "\x00021"), + ("exist", "\x02203"), + ("expectation", "\x02130"), + ("exponentiale", "\x02147"), + ("fallingdotseq", "\x02252"), + ("fcy", "\x00444"), + ("female", "\x02640"), + ("ffilig", "\x0FB03"), + ("fflig", "\x0FB00"), + ("ffllig", "\x0FB04") ] + +reftab28 :: [(Text,Text)] +reftab28 = + [ ("ffr", "\x1D523"), + ("filig", "\x0FB01"), + ("fjlig", "\x00066\x0006A"), + ("flat", "\x0266D"), + ("fllig", "\x0FB02"), + ("fltns", "\x025B1"), + ("fnof", "\x00192"), + ("fopf", "\x1D557"), + ("forall", "\x02200"), + ("fork", "\x022D4"), + ("forkv", "\x02AD9"), + ("fpartint", "\x02A0D"), + ("frac12", "\x000BD"), + ("frac13", "\x02153"), + ("frac14", "\x000BC"), + ("frac15", "\x02155"), + ("frac16", "\x02159"), + ("frac18", "\x0215B"), + ("frac23", "\x02154"), + ("frac25", "\x02156"), + ("frac34", "\x000BE"), + ("frac35", "\x02157"), + ("frac38", "\x0215C"), + ("frac45", "\x02158"), + ("frac56", "\x0215A"), + ("frac58", "\x0215D"), + ("frac78", "\x0215E"), + ("frasl", "\x02044"), + ("frown", "\x02322"), + ("fscr", "\x1D4BB"), + ("gE", "\x02267"), + ("gEl", "\x02A8C"), + ("gacute", "\x001F5"), + ("gamma", "\x003B3"), + ("gammad", "\x003DD"), + ("gap", "\x02A86"), + ("gbreve", "\x0011F") ] + +reftab29 :: [(Text,Text)] +reftab29 = + [ ("gcirc", "\x0011D"), + ("gcy", "\x00433"), + ("gdot", "\x00121"), + ("ge", "\x02265"), + ("gel", "\x022DB"), + ("geq", "\x02265"), + ("geqq", "\x02267"), + ("geqslant", "\x02A7E"), + ("ges", "\x02A7E"), + ("gescc", "\x02AA9"), + ("gesdot", "\x02A80"), + ("gesdoto", "\x02A82"), + ("gesdotol", "\x02A84"), + ("gesl", "\x022DB\x0FE00"), + ("gesles", "\x02A94"), + ("gfr", "\x1D524"), + ("gg", "\x0226B"), + ("ggg", "\x022D9"), + ("gimel", "\x02137"), + ("gjcy", "\x00453"), + ("gl", "\x02277"), + ("glE", "\x02A92"), + ("gla", "\x02AA5"), + ("glj", "\x02AA4"), + ("gnE", "\x02269"), + ("gnap", "\x02A8A"), + ("gnapprox", "\x02A8A"), + ("gne", "\x02A88"), + ("gneq", "\x02A88"), + ("gneqq", "\x02269"), + ("gnsim", "\x022E7"), + ("gopf", "\x1D558"), + ("grave", "\x00060"), + ("gscr", "\x0210A"), + ("gsim", "\x02273"), + ("gsime", "\x02A8E"), + ("gsiml", "\x02A90") ] + +reftab30 :: [(Text,Text)] +reftab30 = + [ ("gt", "\x0003E"), + ("gtcc", "\x02AA7"), + ("gtcir", "\x02A7A"), + ("gtdot", "\x022D7"), + ("gtlPar", "\x02995"), + ("gtquest", "\x02A7C"), + ("gtrapprox", "\x02A86"), + ("gtrarr", "\x02978"), + ("gtrdot", "\x022D7"), + ("gtreqless", "\x022DB"), + ("gtreqqless", "\x02A8C"), + ("gtrless", "\x02277"), + ("gtrsim", "\x02273"), + ("gvertneqq", "\x02269\x0FE00"), + ("gvnE", "\x02269\x0FE00"), + ("hArr", "\x021D4"), + ("hairsp", "\x0200A"), + ("half", "\x000BD"), + ("hamilt", "\x0210B"), + ("hardcy", "\x0044A"), + ("harr", "\x02194"), + ("harrcir", "\x02948"), + ("harrw", "\x021AD"), + ("hbar", "\x0210F"), + ("hcirc", "\x00125"), + ("hearts", "\x02665"), + ("heartsuit", "\x02665"), + ("hellip", "\x02026"), + ("hercon", "\x022B9"), + ("hfr", "\x1D525"), + ("hksearow", "\x02925"), + ("hkswarow", "\x02926"), + ("hoarr", "\x021FF"), + ("homtht", "\x0223B"), + ("hookleftarrow", "\x021A9"), + ("hookrightarrow", "\x021AA"), + ("hopf", "\x1D559") ] + +reftab31 :: [(Text,Text)] +reftab31 = + [ ("horbar", "\x02015"), + ("hscr", "\x1D4BD"), + ("hslash", "\x0210F"), + ("hstrok", "\x00127"), + ("hybull", "\x02043"), + ("hyphen", "\x02010"), + ("iacute", "\x000ED"), + ("ic", "\x02063"), + ("icirc", "\x000EE"), + ("icy", "\x00438"), + ("iecy", "\x00435"), + ("iexcl", "\x000A1"), + ("iff", "\x021D4"), + ("ifr", "\x1D526"), + ("igrave", "\x000EC"), + ("ii", "\x02148"), + ("iiiint", "\x02A0C"), + ("iiint", "\x0222D"), + ("iinfin", "\x029DC"), + ("iiota", "\x02129"), + ("ijlig", "\x00133"), + ("imacr", "\x0012B"), + ("image", "\x02111"), + ("imagline", "\x02110"), + ("imagpart", "\x02111"), + ("imath", "\x00131"), + ("imof", "\x022B7"), + ("imped", "\x001B5"), + ("in", "\x02208"), + ("incare", "\x02105"), + ("infin", "\x0221E"), + ("infintie", "\x029DD"), + ("inodot", "\x00131"), + ("int", "\x0222B"), + ("intcal", "\x022BA"), + ("integers", "\x02124"), + ("intercal", "\x022BA") ] + +reftab32 :: [(Text,Text)] +reftab32 = + [ ("intlarhk", "\x02A17"), + ("intprod", "\x02A3C"), + ("iocy", "\x00451"), + ("iogon", "\x0012F"), + ("iopf", "\x1D55A"), + ("iota", "\x003B9"), + ("iprod", "\x02A3C"), + ("iquest", "\x000BF"), + ("iscr", "\x1D4BE"), + ("isin", "\x02208"), + ("isinE", "\x022F9"), + ("isindot", "\x022F5"), + ("isins", "\x022F4"), + ("isinsv", "\x022F3"), + ("isinv", "\x02208"), + ("it", "\x02062"), + ("itilde", "\x00129"), + ("iukcy", "\x00456"), + ("iuml", "\x000EF"), + ("jcirc", "\x00135"), + ("jcy", "\x00439"), + ("jfr", "\x1D527"), + ("jmath", "\x00237"), + ("jopf", "\x1D55B"), + ("jscr", "\x1D4BF"), + ("jsercy", "\x00458"), + ("jukcy", "\x00454"), + ("kappa", "\x003BA"), + ("kappav", "\x003F0"), + ("kcedil", "\x00137"), + ("kcy", "\x0043A"), + ("kfr", "\x1D528"), + ("kgreen", "\x00138"), + ("khcy", "\x00445"), + ("kjcy", "\x0045C"), + ("kopf", "\x1D55C"), + ("kscr", "\x1D4C0") ] + +reftab33 :: [(Text,Text)] +reftab33 = + [ ("lAarr", "\x021DA"), + ("lArr", "\x021D0"), + ("lAtail", "\x0291B"), + ("lBarr", "\x0290E"), + ("lE", "\x02266"), + ("lEg", "\x02A8B"), + ("lHar", "\x02962"), + ("lacute", "\x0013A"), + ("laemptyv", "\x029B4"), + ("lagran", "\x02112"), + ("lambda", "\x003BB"), + ("lang", "\x027E8"), + ("langd", "\x02991"), + ("langle", "\x027E8"), + ("lap", "\x02A85"), + ("laquo", "\x000AB"), + ("larr", "\x02190"), + ("larrb", "\x021E4"), + ("larrbfs", "\x0291F"), + ("larrfs", "\x0291D"), + ("larrhk", "\x021A9"), + ("larrlp", "\x021AB"), + ("larrpl", "\x02939"), + ("larrsim", "\x02973"), + ("larrtl", "\x021A2"), + ("lat", "\x02AAB"), + ("latail", "\x02919"), + ("late", "\x02AAD"), + ("lates", "\x02AAD\x0FE00"), + ("lbarr", "\x0290C"), + ("lbbrk", "\x02772"), + ("lbrace", "\x0007B"), + ("lbrack", "\x0005B"), + ("lbrke", "\x0298B"), + ("lbrksld", "\x0298F"), + ("lbrkslu", "\x0298D"), + ("lcaron", "\x0013E") ] + +reftab34 :: [(Text,Text)] +reftab34 = + [ ("lcedil", "\x0013C"), + ("lceil", "\x02308"), + ("lcub", "\x0007B"), + ("lcy", "\x0043B"), + ("ldca", "\x02936"), + ("ldquo", "\x0201C"), + ("ldquor", "\x0201E"), + ("ldrdhar", "\x02967"), + ("ldrushar", "\x0294B"), + ("ldsh", "\x021B2"), + ("le", "\x02264"), + ("leftarrow", "\x02190"), + ("leftarrowtail", "\x021A2"), + ("leftharpoondown", "\x021BD"), + ("leftharpoonup", "\x021BC"), + ("leftleftarrows", "\x021C7"), + ("leftrightarrow", "\x02194"), + ("leftrightarrows", "\x021C6"), + ("leftrightharpoons", "\x021CB"), + ("leftrightsquigarrow", "\x021AD"), + ("leftthreetimes", "\x022CB"), + ("leg", "\x022DA"), + ("leq", "\x02264"), + ("leqq", "\x02266"), + ("leqslant", "\x02A7D"), + ("les", "\x02A7D"), + ("lescc", "\x02AA8"), + ("lesdot", "\x02A7F"), + ("lesdoto", "\x02A81"), + ("lesdotor", "\x02A83"), + ("lesg", "\x022DA\x0FE00"), + ("lesges", "\x02A93"), + ("lessapprox", "\x02A85"), + ("lessdot", "\x022D6"), + ("lesseqgtr", "\x022DA"), + ("lesseqqgtr", "\x02A8B"), + ("lessgtr", "\x02276") ] + +reftab35 :: [(Text,Text)] +reftab35 = + [ ("lesssim", "\x02272"), + ("lfisht", "\x0297C"), + ("lfloor", "\x0230A"), + ("lfr", "\x1D529"), + ("lg", "\x02276"), + ("lgE", "\x02A91"), + ("lhard", "\x021BD"), + ("lharu", "\x021BC"), + ("lharul", "\x0296A"), + ("lhblk", "\x02584"), + ("ljcy", "\x00459"), + ("ll", "\x0226A"), + ("llarr", "\x021C7"), + ("llcorner", "\x0231E"), + ("llhard", "\x0296B"), + ("lltri", "\x025FA"), + ("lmidot", "\x00140"), + ("lmoust", "\x023B0"), + ("lmoustache", "\x023B0"), + ("lnE", "\x02268"), + ("lnap", "\x02A89"), + ("lnapprox", "\x02A89"), + ("lne", "\x02A87"), + ("lneq", "\x02A87"), + ("lneqq", "\x02268"), + ("lnsim", "\x022E6"), + ("loang", "\x027EC"), + ("loarr", "\x021FD"), + ("lobrk", "\x027E6"), + ("longleftarrow", "\x027F5"), + ("longleftrightarrow", "\x027F7"), + ("longmapsto", "\x027FC"), + ("longrightarrow", "\x027F6"), + ("looparrowleft", "\x021AB"), + ("looparrowright", "\x021AC"), + ("lopar", "\x02985"), + ("lopf", "\x1D55D") ] + +reftab36 :: [(Text,Text)] +reftab36 = + [ ("loplus", "\x02A2D"), + ("lotimes", "\x02A34"), + ("lowast", "\x02217"), + ("lowbar", "\x0005F"), + ("loz", "\x025CA"), + ("lozenge", "\x025CA"), + ("lozf", "\x029EB"), + ("lpar", "\x00028"), + ("lparlt", "\x02993"), + ("lrarr", "\x021C6"), + ("lrcorner", "\x0231F"), + ("lrhar", "\x021CB"), + ("lrhard", "\x0296D"), + ("lrm", "\x0200E"), + ("lrtri", "\x022BF"), + ("lsaquo", "\x02039"), + ("lscr", "\x1D4C1"), + ("lsh", "\x021B0"), + ("lsim", "\x02272"), + ("lsime", "\x02A8D"), + ("lsimg", "\x02A8F"), + ("lsqb", "\x0005B"), + ("lsquo", "\x02018"), + ("lsquor", "\x0201A"), + ("lstrok", "\x00142"), + ("lt", "\x0003C"), + ("ltcc", "\x02AA6"), + ("ltcir", "\x02A79"), + ("ltdot", "\x022D6"), + ("lthree", "\x022CB"), + ("ltimes", "\x022C9"), + ("ltlarr", "\x02976"), + ("ltquest", "\x02A7B"), + ("ltrPar", "\x02996"), + ("ltri", "\x025C3"), + ("ltrie", "\x022B4"), + ("ltrif", "\x025C2") ] + +reftab37 :: [(Text,Text)] +reftab37 = + [ ("lurdshar", "\x0294A"), + ("luruhar", "\x02966"), + ("lvertneqq", "\x02268\x0FE00"), + ("lvnE", "\x02268\x0FE00"), + ("mDDot", "\x0223A"), + ("macr", "\x000AF"), + ("male", "\x02642"), + ("malt", "\x02720"), + ("maltese", "\x02720"), + ("map", "\x021A6"), + ("mapsto", "\x021A6"), + ("mapstodown", "\x021A7"), + ("mapstoleft", "\x021A4"), + ("mapstoup", "\x021A5"), + ("marker", "\x025AE"), + ("mcomma", "\x02A29"), + ("mcy", "\x0043C"), + ("mdash", "\x02014"), + ("measuredangle", "\x02221"), + ("mfr", "\x1D52A"), + ("mho", "\x02127"), + ("micro", "\x000B5"), + ("mid", "\x02223"), + ("midast", "\x0002A"), + ("midcir", "\x02AF0"), + ("middot", "\x000B7"), + ("minus", "\x02212"), + ("minusb", "\x0229F"), + ("minusd", "\x02238"), + ("minusdu", "\x02A2A"), + ("mlcp", "\x02ADB"), + ("mldr", "\x02026"), + ("mnplus", "\x02213"), + ("models", "\x022A7"), + ("mopf", "\x1D55E"), + ("mp", "\x02213"), + ("mscr", "\x1D4C2") ] + +reftab38 :: [(Text,Text)] +reftab38 = + [ ("mstpos", "\x0223E"), + ("mu", "\x003BC"), + ("multimap", "\x022B8"), + ("mumap", "\x022B8"), + ("nGg", "\x022D9\x00338"), + ("nGt", "\x0226B\x020D2"), + ("nGtv", "\x0226B\x00338"), + ("nLeftarrow", "\x021CD"), + ("nLeftrightarrow", "\x021CE"), + ("nLl", "\x022D8\x00338"), + ("nLt", "\x0226A\x020D2"), + ("nLtv", "\x0226A\x00338"), + ("nRightarrow", "\x021CF"), + ("nVDash", "\x022AF"), + ("nVdash", "\x022AE"), + ("nabla", "\x02207"), + ("nacute", "\x00144"), + ("nang", "\x02220\x020D2"), + ("nap", "\x02249"), + ("napE", "\x02A70\x00338"), + ("napid", "\x0224B\x00338"), + ("napos", "\x00149"), + ("napprox", "\x02249"), + ("natur", "\x0266E"), + ("natural", "\x0266E"), + ("naturals", "\x02115"), + ("nbsp", "\x000A0"), + ("nbump", "\x0224E\x00338"), + ("nbumpe", "\x0224F\x00338"), + ("ncap", "\x02A43"), + ("ncaron", "\x00148"), + ("ncedil", "\x00146"), + ("ncong", "\x02247"), + ("ncongdot", "\x02A6D\x00338"), + ("ncup", "\x02A42"), + ("ncy", "\x0043D"), + ("ndash", "\x02013") ] + +reftab39 :: [(Text,Text)] +reftab39 = + [ ("ne", "\x02260"), + ("neArr", "\x021D7"), + ("nearhk", "\x02924"), + ("nearr", "\x02197"), + ("nearrow", "\x02197"), + ("nedot", "\x02250\x00338"), + ("nequiv", "\x02262"), + ("nesear", "\x02928"), + ("nesim", "\x02242\x00338"), + ("nexist", "\x02204"), + ("nexists", "\x02204"), + ("nfr", "\x1D52B"), + ("ngE", "\x02267\x00338"), + ("nge", "\x02271"), + ("ngeq", "\x02271"), + ("ngeqq", "\x02267\x00338"), + ("ngeqslant", "\x02A7E\x00338"), + ("nges", "\x02A7E\x00338"), + ("ngsim", "\x02275"), + ("ngt", "\x0226F"), + ("ngtr", "\x0226F"), + ("nhArr", "\x021CE"), + ("nharr", "\x021AE"), + ("nhpar", "\x02AF2"), + ("ni", "\x0220B"), + ("nis", "\x022FC"), + ("nisd", "\x022FA"), + ("niv", "\x0220B"), + ("njcy", "\x0045A"), + ("nlArr", "\x021CD"), + ("nlE", "\x02266\x00338"), + ("nlarr", "\x0219A"), + ("nldr", "\x02025"), + ("nle", "\x02270"), + ("nleftarrow", "\x0219A"), + ("nleftrightarrow", "\x021AE"), + ("nleq", "\x02270") ] + +reftab40 :: [(Text,Text)] +reftab40 = + [ ("nleqq", "\x02266\x00338"), + ("nleqslant", "\x02A7D\x00338"), + ("nles", "\x02A7D\x00338"), + ("nless", "\x0226E"), + ("nlsim", "\x02274"), + ("nlt", "\x0226E"), + ("nltri", "\x022EA"), + ("nltrie", "\x022EC"), + ("nmid", "\x02224"), + ("nopf", "\x1D55F"), + ("not", "\x000AC"), + ("notin", "\x02209"), + ("notinE", "\x022F9\x00338"), + ("notindot", "\x022F5\x00338"), + ("notinva", "\x02209"), + ("notinvb", "\x022F7"), + ("notinvc", "\x022F6"), + ("notni", "\x0220C"), + ("notniva", "\x0220C"), + ("notnivb", "\x022FE"), + ("notnivc", "\x022FD"), + ("npar", "\x02226"), + ("nparallel", "\x02226"), + ("nparsl", "\x02AFD\x020E5"), + ("npart", "\x02202\x00338"), + ("npolint", "\x02A14"), + ("npr", "\x02280"), + ("nprcue", "\x022E0"), + ("npre", "\x02AAF\x00338"), + ("nprec", "\x02280"), + ("npreceq", "\x02AAF\x00338"), + ("nrArr", "\x021CF"), + ("nrarr", "\x0219B"), + ("nrarrc", "\x02933\x00338"), + ("nrarrw", "\x0219D\x00338"), + ("nrightarrow", "\x0219B"), + ("nrtri", "\x022EB") ] + +reftab41 :: [(Text,Text)] +reftab41 = + [ ("nrtrie", "\x022ED"), + ("nsc", "\x02281"), + ("nsccue", "\x022E1"), + ("nsce", "\x02AB0\x00338"), + ("nscr", "\x1D4C3"), + ("nshortmid", "\x02224"), + ("nshortparallel", "\x02226"), + ("nsim", "\x02241"), + ("nsime", "\x02244"), + ("nsimeq", "\x02244"), + ("nsmid", "\x02224"), + ("nspar", "\x02226"), + ("nsqsube", "\x022E2"), + ("nsqsupe", "\x022E3"), + ("nsub", "\x02284"), + ("nsubE", "\x02AC5\x00338"), + ("nsube", "\x02288"), + ("nsubset", "\x02282\x020D2"), + ("nsubseteq", "\x02288"), + ("nsubseteqq", "\x02AC5\x00338"), + ("nsucc", "\x02281"), + ("nsucceq", "\x02AB0\x00338"), + ("nsup", "\x02285"), + ("nsupE", "\x02AC6\x00338"), + ("nsupe", "\x02289"), + ("nsupset", "\x02283\x020D2"), + ("nsupseteq", "\x02289"), + ("nsupseteqq", "\x02AC6\x00338"), + ("ntgl", "\x02279"), + ("ntilde", "\x000F1"), + ("ntlg", "\x02278"), + ("ntriangleleft", "\x022EA"), + ("ntrianglelefteq", "\x022EC"), + ("ntriangleright", "\x022EB"), + ("ntrianglerighteq", "\x022ED"), + ("nu", "\x003BD"), + ("num", "\x00023") ] + +reftab42 :: [(Text,Text)] +reftab42 = + [ ("numero", "\x02116"), + ("numsp", "\x02007"), + ("nvDash", "\x022AD"), + ("nvHarr", "\x02904"), + ("nvap", "\x0224D\x020D2"), + ("nvdash", "\x022AC"), + ("nvge", "\x02265\x020D2"), + ("nvgt", "\x0003E\x020D2"), + ("nvinfin", "\x029DE"), + ("nvlArr", "\x02902"), + ("nvle", "\x02264\x020D2"), + ("nvlt", "\x0003C\x020D2"), + ("nvltrie", "\x022B4\x020D2"), + ("nvrArr", "\x02903"), + ("nvrtrie", "\x022B5\x020D2"), + ("nvsim", "\x0223C\x020D2"), + ("nwArr", "\x021D6"), + ("nwarhk", "\x02923"), + ("nwarr", "\x02196"), + ("nwarrow", "\x02196"), + ("nwnear", "\x02927"), + ("oS", "\x024C8"), + ("oacute", "\x000F3"), + ("oast", "\x0229B"), + ("ocir", "\x0229A"), + ("ocirc", "\x000F4"), + ("ocy", "\x0043E"), + ("odash", "\x0229D"), + ("odblac", "\x00151"), + ("odiv", "\x02A38"), + ("odot", "\x02299"), + ("odsold", "\x029BC"), + ("oelig", "\x00153"), + ("ofcir", "\x029BF"), + ("ofr", "\x1D52C"), + ("ogon", "\x002DB"), + ("ograve", "\x000F2") ] + +reftab43 :: [(Text,Text)] +reftab43 = + [ ("ogt", "\x029C1"), + ("ohbar", "\x029B5"), + ("ohm", "\x003A9"), + ("oint", "\x0222E"), + ("olarr", "\x021BA"), + ("olcir", "\x029BE"), + ("olcross", "\x029BB"), + ("oline", "\x0203E"), + ("olt", "\x029C0"), + ("omacr", "\x0014D"), + ("omega", "\x003C9"), + ("omicron", "\x003BF"), + ("omid", "\x029B6"), + ("ominus", "\x02296"), + ("oopf", "\x1D560"), + ("opar", "\x029B7"), + ("operp", "\x029B9"), + ("oplus", "\x02295"), + ("or", "\x02228"), + ("orarr", "\x021BB"), + ("ord", "\x02A5D"), + ("order", "\x02134"), + ("orderof", "\x02134"), + ("ordf", "\x000AA"), + ("ordm", "\x000BA"), + ("origof", "\x022B6"), + ("oror", "\x02A56"), + ("orslope", "\x02A57"), + ("orv", "\x02A5B"), + ("oscr", "\x02134"), + ("oslash", "\x000F8"), + ("osol", "\x02298"), + ("otilde", "\x000F5"), + ("otimes", "\x02297"), + ("otimesas", "\x02A36"), + ("ouml", "\x000F6"), + ("ovbar", "\x0233D") ] + +reftab44 :: [(Text,Text)] +reftab44 = + [ ("par", "\x02225"), + ("para", "\x000B6"), + ("parallel", "\x02225"), + ("parsim", "\x02AF3"), + ("parsl", "\x02AFD"), + ("part", "\x02202"), + ("pcy", "\x0043F"), + ("percnt", "\x00025"), + ("period", "\x0002E"), + ("permil", "\x02030"), + ("perp", "\x022A5"), + ("pertenk", "\x02031"), + ("pfr", "\x1D52D"), + ("phi", "\x003C6"), + ("phiv", "\x003D5"), + ("phmmat", "\x02133"), + ("phone", "\x0260E"), + ("pi", "\x003C0"), + ("pitchfork", "\x022D4"), + ("piv", "\x003D6"), + ("planck", "\x0210F"), + ("planckh", "\x0210E"), + ("plankv", "\x0210F"), + ("plus", "\x0002B"), + ("plusacir", "\x02A23"), + ("plusb", "\x0229E"), + ("pluscir", "\x02A22"), + ("plusdo", "\x02214"), + ("plusdu", "\x02A25"), + ("pluse", "\x02A72"), + ("plusmn", "\x000B1"), + ("plussim", "\x02A26"), + ("plustwo", "\x02A27"), + ("pm", "\x000B1"), + ("pointint", "\x02A15"), + ("popf", "\x1D561"), + ("pound", "\x000A3") ] + +reftab45 :: [(Text,Text)] +reftab45 = + [ ("pr", "\x0227A"), + ("prE", "\x02AB3"), + ("prap", "\x02AB7"), + ("prcue", "\x0227C"), + ("pre", "\x02AAF"), + ("prec", "\x0227A"), + ("precapprox", "\x02AB7"), + ("preccurlyeq", "\x0227C"), + ("preceq", "\x02AAF"), + ("precnapprox", "\x02AB9"), + ("precneqq", "\x02AB5"), + ("precnsim", "\x022E8"), + ("precsim", "\x0227E"), + ("prime", "\x02032"), + ("primes", "\x02119"), + ("prnE", "\x02AB5"), + ("prnap", "\x02AB9"), + ("prnsim", "\x022E8"), + ("prod", "\x0220F"), + ("profalar", "\x0232E"), + ("profline", "\x02312"), + ("profsurf", "\x02313"), + ("prop", "\x0221D"), + ("propto", "\x0221D"), + ("prsim", "\x0227E"), + ("prurel", "\x022B0"), + ("pscr", "\x1D4C5"), + ("psi", "\x003C8"), + ("puncsp", "\x02008"), + ("qfr", "\x1D52E"), + ("qint", "\x02A0C"), + ("qopf", "\x1D562"), + ("qprime", "\x02057"), + ("qscr", "\x1D4C6"), + ("quaternions", "\x0210D"), + ("quatint", "\x02A16"), + ("quest", "\x0003F") ] + +reftab46 :: [(Text,Text)] +reftab46 = + [ ("questeq", "\x0225F"), + ("quot", "\x00022"), + ("rAarr", "\x021DB"), + ("rArr", "\x021D2"), + ("rAtail", "\x0291C"), + ("rBarr", "\x0290F"), + ("rHar", "\x02964"), + ("race", "\x0223D\x00331"), + ("racute", "\x00155"), + ("radic", "\x0221A"), + ("raemptyv", "\x029B3"), + ("rang", "\x027E9"), + ("rangd", "\x02992"), + ("range", "\x029A5"), + ("rangle", "\x027E9"), + ("raquo", "\x000BB"), + ("rarr", "\x02192"), + ("rarrap", "\x02975"), + ("rarrb", "\x021E5"), + ("rarrbfs", "\x02920"), + ("rarrc", "\x02933"), + ("rarrfs", "\x0291E"), + ("rarrhk", "\x021AA"), + ("rarrlp", "\x021AC"), + ("rarrpl", "\x02945"), + ("rarrsim", "\x02974"), + ("rarrtl", "\x021A3"), + ("rarrw", "\x0219D"), + ("ratail", "\x0291A"), + ("ratio", "\x02236"), + ("rationals", "\x0211A"), + ("rbarr", "\x0290D"), + ("rbbrk", "\x02773"), + ("rbrace", "\x0007D"), + ("rbrack", "\x0005D"), + ("rbrke", "\x0298C"), + ("rbrksld", "\x0298E") ] + +reftab47 :: [(Text,Text)] +reftab47 = + [ ("rbrkslu", "\x02990"), + ("rcaron", "\x00159"), + ("rcedil", "\x00157"), + ("rceil", "\x02309"), + ("rcub", "\x0007D"), + ("rcy", "\x00440"), + ("rdca", "\x02937"), + ("rdldhar", "\x02969"), + ("rdquo", "\x0201D"), + ("rdquor", "\x0201D"), + ("rdsh", "\x021B3"), + ("real", "\x0211C"), + ("realine", "\x0211B"), + ("realpart", "\x0211C"), + ("reals", "\x0211D"), + ("rect", "\x025AD"), + ("reg", "\x000AE"), + ("rfisht", "\x0297D"), + ("rfloor", "\x0230B"), + ("rfr", "\x1D52F"), + ("rhard", "\x021C1"), + ("rharu", "\x021C0"), + ("rharul", "\x0296C"), + ("rho", "\x003C1"), + ("rhov", "\x003F1"), + ("rightarrow", "\x02192"), + ("rightarrowtail", "\x021A3"), + ("rightharpoondown", "\x021C1"), + ("rightharpoonup", "\x021C0"), + ("rightleftarrows", "\x021C4"), + ("rightleftharpoons", "\x021CC"), + ("rightrightarrows", "\x021C9"), + ("rightsquigarrow", "\x0219D"), + ("rightthreetimes", "\x022CC"), + ("ring", "\x002DA"), + ("risingdotseq", "\x02253"), + ("rlarr", "\x021C4") ] + +reftab48 :: [(Text,Text)] +reftab48 = + [ ("rlhar", "\x021CC"), + ("rlm", "\x0200F"), + ("rmoust", "\x023B1"), + ("rmoustache", "\x023B1"), + ("rnmid", "\x02AEE"), + ("roang", "\x027ED"), + ("roarr", "\x021FE"), + ("robrk", "\x027E7"), + ("ropar", "\x02986"), + ("ropf", "\x1D563"), + ("roplus", "\x02A2E"), + ("rotimes", "\x02A35"), + ("rpar", "\x00029"), + ("rpargt", "\x02994"), + ("rppolint", "\x02A12"), + ("rrarr", "\x021C9"), + ("rsaquo", "\x0203A"), + ("rscr", "\x1D4C7"), + ("rsh", "\x021B1"), + ("rsqb", "\x0005D"), + ("rsquo", "\x02019"), + ("rsquor", "\x02019"), + ("rthree", "\x022CC"), + ("rtimes", "\x022CA"), + ("rtri", "\x025B9"), + ("rtrie", "\x022B5"), + ("rtrif", "\x025B8"), + ("rtriltri", "\x029CE"), + ("ruluhar", "\x02968"), + ("rx", "\x0211E"), + ("sacute", "\x0015B"), + ("sbquo", "\x0201A"), + ("sc", "\x0227B"), + ("scE", "\x02AB4"), + ("scap", "\x02AB8"), + ("scaron", "\x00161"), + ("sccue", "\x0227D") ] + +reftab49 :: [(Text,Text)] +reftab49 = + [ ("sce", "\x02AB0"), + ("scedil", "\x0015F"), + ("scirc", "\x0015D"), + ("scnE", "\x02AB6"), + ("scnap", "\x02ABA"), + ("scnsim", "\x022E9"), + ("scpolint", "\x02A13"), + ("scsim", "\x0227F"), + ("scy", "\x00441"), + ("sdot", "\x022C5"), + ("sdotb", "\x022A1"), + ("sdote", "\x02A66"), + ("seArr", "\x021D8"), + ("searhk", "\x02925"), + ("searr", "\x02198"), + ("searrow", "\x02198"), + ("sect", "\x000A7"), + ("semi", "\x0003B"), + ("seswar", "\x02929"), + ("setminus", "\x02216"), + ("setmn", "\x02216"), + ("sext", "\x02736"), + ("sfr", "\x1D530"), + ("sfrown", "\x02322"), + ("sharp", "\x0266F"), + ("shchcy", "\x00449"), + ("shcy", "\x00448"), + ("shortmid", "\x02223"), + ("shortparallel", "\x02225"), + ("shy", "\x000AD"), + ("sigma", "\x003C3"), + ("sigmaf", "\x003C2"), + ("sigmav", "\x003C2"), + ("sim", "\x0223C"), + ("simdot", "\x02A6A"), + ("sime", "\x02243"), + ("simeq", "\x02243") ] + +reftab50 :: [(Text,Text)] +reftab50 = + [ ("simg", "\x02A9E"), + ("simgE", "\x02AA0"), + ("siml", "\x02A9D"), + ("simlE", "\x02A9F"), + ("simne", "\x02246"), + ("simplus", "\x02A24"), + ("simrarr", "\x02972"), + ("slarr", "\x02190"), + ("smallsetminus", "\x02216"), + ("smashp", "\x02A33"), + ("smeparsl", "\x029E4"), + ("smid", "\x02223"), + ("smile", "\x02323"), + ("smt", "\x02AAA"), + ("smte", "\x02AAC"), + ("smtes", "\x02AAC\x0FE00"), + ("softcy", "\x0044C"), + ("sol", "\x0002F"), + ("solb", "\x029C4"), + ("solbar", "\x0233F"), + ("sopf", "\x1D564"), + ("spades", "\x02660"), + ("spadesuit", "\x02660"), + ("spar", "\x02225"), + ("sqcap", "\x02293"), + ("sqcaps", "\x02293\x0FE00"), + ("sqcup", "\x02294"), + ("sqcups", "\x02294\x0FE00"), + ("sqsub", "\x0228F"), + ("sqsube", "\x02291"), + ("sqsubset", "\x0228F"), + ("sqsubseteq", "\x02291"), + ("sqsup", "\x02290"), + ("sqsupe", "\x02292"), + ("sqsupset", "\x02290"), + ("sqsupseteq", "\x02292"), + ("squ", "\x025A1") ] + +reftab51 :: [(Text,Text)] +reftab51 = + [ ("square", "\x025A1"), + ("squarf", "\x025AA"), + ("squf", "\x025AA"), + ("srarr", "\x02192"), + ("sscr", "\x1D4C8"), + ("ssetmn", "\x02216"), + ("ssmile", "\x02323"), + ("sstarf", "\x022C6"), + ("star", "\x02606"), + ("starf", "\x02605"), + ("straightepsilon", "\x003F5"), + ("straightphi", "\x003D5"), + ("strns", "\x000AF"), + ("sub", "\x02282"), + ("subE", "\x02AC5"), + ("subdot", "\x02ABD"), + ("sube", "\x02286"), + ("subedot", "\x02AC3"), + ("submult", "\x02AC1"), + ("subnE", "\x02ACB"), + ("subne", "\x0228A"), + ("subplus", "\x02ABF"), + ("subrarr", "\x02979"), + ("subset", "\x02282"), + ("subseteq", "\x02286"), + ("subseteqq", "\x02AC5"), + ("subsetneq", "\x0228A"), + ("subsetneqq", "\x02ACB"), + ("subsim", "\x02AC7"), + ("subsub", "\x02AD5"), + ("subsup", "\x02AD3"), + ("succ", "\x0227B"), + ("succapprox", "\x02AB8"), + ("succcurlyeq", "\x0227D"), + ("succeq", "\x02AB0"), + ("succnapprox", "\x02ABA"), + ("succneqq", "\x02AB6") ] + +reftab52 :: [(Text,Text)] +reftab52 = + [ ("succnsim", "\x022E9"), + ("succsim", "\x0227F"), + ("sum", "\x02211"), + ("sung", "\x0266A"), + ("sup", "\x02283"), + ("sup1", "\x000B9"), + ("sup2", "\x000B2"), + ("sup3", "\x000B3"), + ("supE", "\x02AC6"), + ("supdot", "\x02ABE"), + ("supdsub", "\x02AD8"), + ("supe", "\x02287"), + ("supedot", "\x02AC4"), + ("suphsol", "\x027C9"), + ("suphsub", "\x02AD7"), + ("suplarr", "\x0297B"), + ("supmult", "\x02AC2"), + ("supnE", "\x02ACC"), + ("supne", "\x0228B"), + ("supplus", "\x02AC0"), + ("supset", "\x02283"), + ("supseteq", "\x02287"), + ("supseteqq", "\x02AC6"), + ("supsetneq", "\x0228B"), + ("supsetneqq", "\x02ACC"), + ("supsim", "\x02AC8"), + ("supsub", "\x02AD4"), + ("supsup", "\x02AD6"), + ("swArr", "\x021D9"), + ("swarhk", "\x02926"), + ("swarr", "\x02199"), + ("swarrow", "\x02199"), + ("swnwar", "\x0292A"), + ("szlig", "\x000DF"), + ("target", "\x02316"), + ("tau", "\x003C4"), + ("tbrk", "\x023B4") ] + +reftab53 :: [(Text,Text)] +reftab53 = + [ ("tcaron", "\x00165"), + ("tcedil", "\x00163"), + ("tcy", "\x00442"), + ("tdot", "\x020DB"), + ("telrec", "\x02315"), + ("tfr", "\x1D531"), + ("there4", "\x02234"), + ("therefore", "\x02234"), + ("theta", "\x003B8"), + ("thetasym", "\x003D1"), + ("thetav", "\x003D1"), + ("thickapprox", "\x02248"), + ("thicksim", "\x0223C"), + ("thinsp", "\x02009"), + ("thkap", "\x02248"), + ("thksim", "\x0223C"), + ("thorn", "\x000FE"), + ("tilde", "\x002DC"), + ("times", "\x000D7"), + ("timesb", "\x022A0"), + ("timesbar", "\x02A31"), + ("timesd", "\x02A30"), + ("tint", "\x0222D"), + ("toea", "\x02928"), + ("top", "\x022A4"), + ("topbot", "\x02336"), + ("topcir", "\x02AF1"), + ("topf", "\x1D565"), + ("topfork", "\x02ADA"), + ("tosa", "\x02929"), + ("tprime", "\x02034"), + ("trade", "\x02122"), + ("triangle", "\x025B5"), + ("triangledown", "\x025BF"), + ("triangleleft", "\x025C3"), + ("trianglelefteq", "\x022B4"), + ("triangleq", "\x0225C") ] + +reftab54 :: [(Text,Text)] +reftab54 = + [ ("triangleright", "\x025B9"), + ("trianglerighteq", "\x022B5"), + ("tridot", "\x025EC"), + ("trie", "\x0225C"), + ("triminus", "\x02A3A"), + ("triplus", "\x02A39"), + ("trisb", "\x029CD"), + ("tritime", "\x02A3B"), + ("trpezium", "\x023E2"), + ("tscr", "\x1D4C9"), + ("tscy", "\x00446"), + ("tshcy", "\x0045B"), + ("tstrok", "\x00167"), + ("twixt", "\x0226C"), + ("twoheadleftarrow", "\x0219E"), + ("twoheadrightarrow", "\x021A0"), + ("uArr", "\x021D1"), + ("uHar", "\x02963"), + ("uacute", "\x000FA"), + ("uarr", "\x02191"), + ("ubrcy", "\x0045E"), + ("ubreve", "\x0016D"), + ("ucirc", "\x000FB"), + ("ucy", "\x00443"), + ("udarr", "\x021C5"), + ("udblac", "\x00171"), + ("udhar", "\x0296E"), + ("ufisht", "\x0297E"), + ("ufr", "\x1D532"), + ("ugrave", "\x000F9"), + ("uharl", "\x021BF"), + ("uharr", "\x021BE"), + ("uhblk", "\x02580"), + ("ulcorn", "\x0231C"), + ("ulcorner", "\x0231C"), + ("ulcrop", "\x0230F"), + ("ultri", "\x025F8") ] + +reftab55 :: [(Text,Text)] +reftab55 = + [ ("umacr", "\x0016B"), + ("uml", "\x000A8"), + ("uogon", "\x00173"), + ("uopf", "\x1D566"), + ("uparrow", "\x02191"), + ("updownarrow", "\x02195"), + ("upharpoonleft", "\x021BF"), + ("upharpoonright", "\x021BE"), + ("uplus", "\x0228E"), + ("upsi", "\x003C5"), + ("upsih", "\x003D2"), + ("upsilon", "\x003C5"), + ("upuparrows", "\x021C8"), + ("urcorn", "\x0231D"), + ("urcorner", "\x0231D"), + ("urcrop", "\x0230E"), + ("uring", "\x0016F"), + ("urtri", "\x025F9"), + ("uscr", "\x1D4CA"), + ("utdot", "\x022F0"), + ("utilde", "\x00169"), + ("utri", "\x025B5"), + ("utrif", "\x025B4"), + ("uuarr", "\x021C8"), + ("uuml", "\x000FC"), + ("uwangle", "\x029A7"), + ("vArr", "\x021D5"), + ("vBar", "\x02AE8"), + ("vBarv", "\x02AE9"), + ("vDash", "\x022A8"), + ("vangrt", "\x0299C"), + ("varepsilon", "\x003F5"), + ("varkappa", "\x003F0"), + ("varnothing", "\x02205"), + ("varphi", "\x003D5"), + ("varpi", "\x003D6"), + ("varpropto", "\x0221D") ] + +reftab56 :: [(Text,Text)] +reftab56 = + [ ("varr", "\x02195"), + ("varrho", "\x003F1"), + ("varsigma", "\x003C2"), + ("varsubsetneq", "\x0228A\x0FE00"), + ("varsubsetneqq", "\x02ACB\x0FE00"), + ("varsupsetneq", "\x0228B\x0FE00"), + ("varsupsetneqq", "\x02ACC\x0FE00"), + ("vartheta", "\x003D1"), + ("vartriangleleft", "\x022B2"), + ("vartriangleright", "\x022B3"), + ("vcy", "\x00432"), + ("vdash", "\x022A2"), + ("vee", "\x02228"), + ("veebar", "\x022BB"), + ("veeeq", "\x0225A"), + ("vellip", "\x022EE"), + ("verbar", "\x0007C"), + ("vert", "\x0007C"), + ("vfr", "\x1D533"), + ("vltri", "\x022B2"), + ("vnsub", "\x02282\x020D2"), + ("vnsup", "\x02283\x020D2"), + ("vopf", "\x1D567"), + ("vprop", "\x0221D"), + ("vrtri", "\x022B3"), + ("vscr", "\x1D4CB"), + ("vsubnE", "\x02ACB\x0FE00"), + ("vsubne", "\x0228A\x0FE00"), + ("vsupnE", "\x02ACC\x0FE00"), + ("vsupne", "\x0228B\x0FE00"), + ("vzigzag", "\x0299A"), + ("wcirc", "\x00175"), + ("wedbar", "\x02A5F"), + ("wedge", "\x02227"), + ("wedgeq", "\x02259"), + ("weierp", "\x02118"), + ("wfr", "\x1D534") ] + +reftab57 :: [(Text,Text)] +reftab57 = + [ ("wopf", "\x1D568"), + ("wp", "\x02118"), + ("wr", "\x02240"), + ("wreath", "\x02240"), + ("wscr", "\x1D4CC"), + ("xcap", "\x022C2"), + ("xcirc", "\x025EF"), + ("xcup", "\x022C3"), + ("xdtri", "\x025BD"), + ("xfr", "\x1D535"), + ("xhArr", "\x027FA"), + ("xharr", "\x027F7"), + ("xi", "\x003BE"), + ("xlArr", "\x027F8"), + ("xlarr", "\x027F5"), + ("xmap", "\x027FC"), + ("xnis", "\x022FB"), + ("xodot", "\x02A00"), + ("xopf", "\x1D569"), + ("xoplus", "\x02A01"), + ("xotime", "\x02A02"), + ("xrArr", "\x027F9"), + ("xrarr", "\x027F6"), + ("xscr", "\x1D4CD"), + ("xsqcup", "\x02A06"), + ("xuplus", "\x02A04"), + ("xutri", "\x025B3"), + ("xvee", "\x022C1"), + ("xwedge", "\x022C0"), + ("yacute", "\x000FD"), + ("yacy", "\x0044F"), + ("ycirc", "\x00177"), + ("ycy", "\x0044B"), + ("yen", "\x000A5"), + ("yfr", "\x1D536"), + ("yicy", "\x00457"), + ("yopf", "\x1D56A") ] + +reftab58 :: [(Text,Text)] +reftab58 = + [ ("yscr", "\x1D4CE"), + ("yucy", "\x0044E"), + ("yuml", "\x000FF"), + ("zacute", "\x0017A"), + ("zcaron", "\x0017E"), + ("zcy", "\x00437"), + ("zdot", "\x0017C"), + ("zeetrf", "\x02128"), + ("zeta", "\x003B6"), + ("zfr", "\x1D537"), + ("zhcy", "\x00436"), + ("zigrarr", "\x021DD"), + ("zopf", "\x1D56B"), + ("zscr", "\x1D4CF"), + ("zwj", "\x0200D"), + ("zwnj", "\x0200C") ] + diff --git a/src/Text/XmlHtml/HTML/Parse.hs b/src/Text/XmlHtml/HTML/Parse.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/HTML/Parse.hs @@ -0,0 +1,388 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TupleSections #-} + +module Text.XmlHtml.HTML.Parse where + +import Control.Applicative +import Control.Monad +import Data.Char +import Data.List +import Data.Maybe +import Text.XmlHtml.Common +import Text.XmlHtml.HTML.Meta +import Text.XmlHtml.TextParser +import qualified Text.XmlHtml.XML.Parse as XML + +import qualified Text.Parsec as P + +import qualified Data.Set as S +import qualified Data.Map as M + +import Data.Text (Text) +import qualified Data.Text as T + + +------------------------------------------------------------------------------ +-- | HTML version of document fragment parsing rule It differs only in that +-- it parses the HTML version of 'content' and returns an 'HtmlDocument'. +docFragment :: Encoding -> Parser Document +docFragment e = do + (dt, nodes1) <- prolog + (nodes2, Matched) <- content Nothing + return $ HtmlDocument e dt (nodes1 ++ nodes2) + + +------------------------------------------------------------------------------ +-- Parsing code -- +------------------------------------------------------------------------------ + +{- + The following are the differences between this code and the straight XML + parsing code. + + 1. HTML void tags (area, base, etc.) are always treated as empty tags, + regardless of whether they have the empty-tag slash. + + 2. HTML raw text tags (script and style) are parsed as straight text + with neither markup nor references, except that they end at the first + syntactically valid matching end tag. + + 3. HTML RCDATA tags (textarea and title) are parsed as text and references + but no other markup, except that they end at the first syntactically + valid matching end tag. + + 4. End tags need only match their corresponding start tags in a case + insensitive comparison. In case they are different, the start tag is + used for the element tag name. + + 5. Hexadecimal char references may use &#X...; (capital X) -- DONE + + 6. Attribute names are allowed to consist of any text except for control + characters, space, '\"', '\'', '>', '/', or '='. + + 7. Empty attribute syntax is allowed (an attribute not followed by an eq). + In this case, the attribute value is considered to be the empty string. + + 8. Quoted attribute syntax is relaxed to allow any character except for the + matching quote. References are allowed. + + 9. Attribute values may be unquoted. In this case, the attribute value may + not contain space, single or double quotes, '=', '<', '>', or '`', and + may not be the empty string. It can still contain references. + + 10. There are many more character references available. + + 11. Only "ambiguous" ampersands are prohibited in character data. This + means ampersands that parse like character or entity references. + + 12. Omittable end tags are inserted automatically. + + 13. DOCTYPE tags matched with case insensitive keywords. +-} + + +------------------------------------------------------------------------------ +prolog :: Parser (Maybe DocType, [Node]) +prolog = do + _ <- optional XML.xmlDecl + nodes1 <- many XML.misc + rest <- optional $ do + dt <- docTypeDecl + nodes2 <- many XML.misc + return (dt, nodes2) + case rest of + Nothing -> return (Nothing, catMaybes nodes1) + Just (dt, nodes2) -> return (Just dt, catMaybes (nodes1 ++ nodes2)) + + +------------------------------------------------------------------------------ +-- | Internal subset is parsed, but ignored since we don't have data types to +-- store it. +docTypeDecl :: Parser DocType +docTypeDecl = do + P.try $ do + _ <- text "' + return (DocType tag extid intsub) + + +------------------------------------------------------------------------------ +externalID :: Parser ExternalID +externalID = do + tok <- optional $ T.toLower <$> XML.name + case tok of + Just "system" -> systemID + Just "public" -> publicID + Just _ -> fail "Expected SYSTEM or PUBLIC" + Nothing -> return NoExternalID + where + systemID = do + XML.whiteSpace + System <$> XML.systemLiteral + publicID = do + XML.whiteSpace + pid <- XML.pubIdLiteral + XML.whiteSpace + sid <- XML.systemLiteral + return (Public pid sid) + + +------------------------------------------------------------------------------ +-- | Reads text and references, up until the passed-in parser succeeds. +-- (Generally, the passed in parser should be an end tag parser for the +-- RCDATA element.) +rcdata :: [Char] -> Parser a -> Parser Node +rcdata cs end = TextNode <$> T.concat <$> P.manyTill part end + where part = takeWhile1 (not . (`elem` cs)) + <|> reference + <|> T.singleton <$> P.anyChar + + +------------------------------------------------------------------------------ +-- | When parsing an element, three things can happen (besides failure): +-- +-- (1) The end tag matches the start tag. This is a Matched. +-- +-- (2) The end tag does not match, but the element has an end tag that can be +-- omitted when there is no more content in its parent. This is an +-- ImplicitLast. In this case, we need to remember the tag name of the +-- end tag that we did find, so as to match it later. +-- +-- (3) A start tag is found such that it implicitly ends the current element. +-- This is an ImplicitNext. In this case, we parse and remember the +-- entire element that comes next, so that it can be inserted after the +-- element being parsed. +data ElemResult = Matched + | ImplicitLast Text + | ImplicitNext Text [(Text, Text)] Bool + + +------------------------------------------------------------------------------ +finishElement :: Text -> [(Text, Text)] -> Bool -> Parser (Node, ElemResult) +finishElement t a b = do + if b then return (Element t a [], Matched) + else nonEmptyElem + where + nonEmptyElem + | T.map toLower t `S.member` rawTextTags = do + c <- XML.cdata "<" $ P.try (endTag t) + return (Element t a [c], Matched) + | T.map toLower t `S.member` rcdataTags = do + c <- rcdata "&<" $ P.try (endTag t) + return (Element t a [c], Matched) + | T.map toLower t `S.member` endOmittableLast = tagContents optional + | otherwise = tagContents (fmap Just) + tagContents modifier = do + (c,r1) <- content (Just t) + case r1 of + Matched -> do + r2 <- modifier (endTag t) + case r2 of + Nothing -> return (Element t a c, Matched) + Just rr -> return (Element t a c, rr) + ImplicitLast tag | T.map toLower tag == T.map toLower t -> do + return (Element t a c, Matched) + end -> do + return (Element t a c, end) + + +------------------------------------------------------------------------------ +emptyOrStartTag :: Parser (Text, [(Text, Text)], Bool) +emptyOrStartTag = do + t <- P.try $ P.char '<' *> XML.name + a <- many $ P.try $ do + XML.whiteSpace + attribute + when (hasDups a) $ fail "Duplicate attribute names in element" + _ <- optional XML.whiteSpace + e <- fmap isJust $ optional (P.char '/') + let e' = e || (T.map toLower t `S.member` voidTags) + _ <- P.char '>' + return (t, a, e') + where + hasDups a = length (nub (map fst a)) < length a + + +------------------------------------------------------------------------------ +attrName :: Parser Text +attrName = takeWhile1 isAttrName + where isAttrName c | c `elem` "\0 \"\'>/=" = False + | isControlChar c = False + | otherwise = True + + +------------------------------------------------------------------------------ +-- | From 8.2.2.3 of the HTML 5 spec, omitting the very high control +-- characters because they are unlikely to occur and I got tired of typing. +isControlChar :: Char -> Bool +isControlChar c | c >= '\x007F' && c <= '\x009F' = True + | c >= '\xFDD0' && c <= '\xFDEF' = True + | otherwise = False + + +------------------------------------------------------------------------------ +quotedAttrValue :: Parser Text +quotedAttrValue = singleQuoted <|> doubleQuoted + where + singleQuoted = P.char '\'' *> refTill "&\'" <* P.char '\'' + doubleQuoted = P.char '\"' *> refTill "&\"" <* P.char '\"' + refTill end = T.concat <$> many + (takeWhile1 (not . (`elem` end)) <|> reference) + + +------------------------------------------------------------------------------ +unquotedAttrValue :: Parser Text +unquotedAttrValue = refTill " \"\'=<>&`" + where + refTill end = T.concat <$> some + (takeWhile1 (not . (`elem` end)) <|> reference) + + +------------------------------------------------------------------------------ +attrValue :: Parser Text +attrValue = quotedAttrValue <|> unquotedAttrValue + + +------------------------------------------------------------------------------ +attribute :: Parser (Text, Text) +attribute = do + n <- attrName + _ <- optional XML.whiteSpace + v <- optional $ do + _ <- P.char '=' + _ <- optional XML.whiteSpace + attrValue + return $ maybe (n,"") (n,) v + + +------------------------------------------------------------------------------ +endTag :: Text -> Parser ElemResult +endTag s = do + _ <- text " found inside <" ++ T.unpack s ++ "> tag" + _ <- optional XML.whiteSpace + _ <- text ">" + return r + + +------------------------------------------------------------------------------ +content :: Maybe Text -> Parser ([Node], ElemResult) +content parent = do + (ns, end) <- readText + return (coalesceText (catMaybes ns), end) + where + readText = do + s <- optional XML.charData + t <- optional whileMatched + case t of + Nothing -> return ([s], Matched) + Just (tt, m) -> return (s:tt, m) + + whileMatched = do + (n,end) <- (,Matched) <$> (:[]) <$> Just <$> TextNode <$> reference + <|> (,Matched) <$> (:[]) <$> XML.cdSect + <|> (,Matched) <$> (:[]) <$> XML.processingInstruction + <|> (,Matched) <$> (:[]) <$> XML.comment + <|> doElement + case end of + Matched -> do + (ns, end') <- readText + return (n ++ ns, end') + _ -> do + return (n, end) + + doElement = do + (t,a,b) <- emptyOrStartTag + handle t a b + + handle t a b = do + if breaksTag t parent + then return ([Nothing], ImplicitNext t a b) + else do + (n,end) <- finishElement t a b + case end of + ImplicitNext t' a' b' -> do + (ns,end') <- handle t' a' b' + return (Just n : ns, end') + _ -> return ([Just n], end) + + breaksTag _ Nothing = False + breaksTag child (Just tag) = case M.lookup tag endOmittableNext of + Nothing -> False + Just s -> S.member child s + + coalesceText (TextNode s : TextNode t : ns) + = coalesceText (TextNode (T.append s t) : ns) + coalesceText (n:ns) + = n : coalesceText ns + coalesceText [] + = [] + + +------------------------------------------------------------------------------ +reference :: Parser Text +reference = do + _ <- P.char '&' + r <- (Left <$> P.try finishCharRef) + <|> (Right <$> P.try finishEntityRef) + <|> (Left <$> return '&') + case r of + Left c -> do + when (not (isValidChar c)) $ fail $ + "Reference is not a valid character" + return (T.singleton c) + Right nm -> case M.lookup nm predefinedRefs of + Nothing -> fail $ "Unknown entity reference: " ++ T.unpack nm + Just t -> return t + + +------------------------------------------------------------------------------ +finishCharRef :: Parser Char +finishCharRef = P.char '#' *> (hexCharRef <|> decCharRef) + where + decCharRef = do + ds <- some digit + _ <- P.char ';' + let c = chr $ foldl' (\a b -> 10 * a + b) 0 ds + return c + where + digit = do + d <- P.satisfy (\c -> c >= '0' && c <= '9') + return (ord d - ord '0') + hexCharRef = do + _ <- P.char 'x' <|> P.char 'X' + ds <- some digit + _ <- P.char ';' + let c = chr $ foldl' (\a b -> 16 * a + b) 0 ds + return c + where + digit = num <|> upper <|> lower + num = do + d <- P.satisfy (\c -> c >= '0' && c <= '9') + return (ord d - ord '0') + upper = do + d <- P.satisfy (\c -> c >= 'A' && c <= 'F') + return (10 + ord d - ord 'A') + lower = do + d <- P.satisfy (\c -> c >= 'a' && c <= 'f') + return (10 + ord d - ord 'a') + + +------------------------------------------------------------------------------ +finishEntityRef :: Parser Text +finishEntityRef = XML.name <* P.char ';' + diff --git a/src/Text/XmlHtml/HTML/Render.hs b/src/Text/XmlHtml/HTML/Render.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/HTML/Render.hs @@ -0,0 +1,148 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternGuards #-} + +module Text.XmlHtml.HTML.Render where + +import Blaze.ByteString.Builder +import Control.Applicative +import Data.Maybe +import Data.Monoid +import qualified Text.Parsec as P +import Text.XmlHtml.Common +import Text.XmlHtml.TextParser +import Text.XmlHtml.HTML.Meta +import qualified Text.XmlHtml.HTML.Parse as P +import Text.XmlHtml.XML.Render (docTypeDecl, entity) + +import Data.Text (Text) +import qualified Data.Text as T + +import qualified Data.Set as S + +------------------------------------------------------------------------------ +-- | And, the rendering code. +render :: Encoding -> Maybe DocType -> [Node] -> Builder +render e dt ns = byteOrder + `mappend` docTypeDecl e dt + `mappend` nodes + where byteOrder | isUTF16 e = fromText e "\xFEFF" -- byte order mark + | otherwise = mempty + nodes | null ns = mempty + | otherwise = firstNode e (head ns) + `mappend` (mconcat $ map (node e) (tail ns)) + + +------------------------------------------------------------------------------ +-- | HTML allows & so long as it is not "ambiguous" (i.e., looks like an +-- entity). So we have a special case for that. +escaped :: [Char] -> Encoding -> Text -> Builder +escaped _ _ "" = mempty +escaped bad e t = + let (p,s) = T.break (`elem` bad) t + r = T.uncons s + in fromText e p `mappend` case r of + Nothing + -> mempty + Just ('&',ss) | isLeft (parseText ambigAmp "" s) + -> fromText e "&" `mappend` escaped bad e ss + Just (c,ss) + -> entity e c `mappend` escaped bad e ss + where isLeft = either (const True) (const False) + ambigAmp = P.char '&' *> + (P.finishCharRef *> return () <|> P.finishEntityRef *> return ()) + + +------------------------------------------------------------------------------ +node :: Encoding -> Node -> Builder +node e (TextNode t) = escaped "<>&" e t +node e (Comment t) | "--" `T.isInfixOf` t = error "Invalid comment" + | "-" `T.isSuffixOf` t = error "Invalid comment" + | otherwise = fromText e "" +node e (Element t a c) = element e t a c + + +------------------------------------------------------------------------------ +-- | Process the first node differently to encode leading whitespace. This +-- lets us be sure that @parseHTML@ is a left inverse to @render@. +firstNode :: Encoding -> Node -> Builder +firstNode e (Comment t) = node e (Comment t) +firstNode e (Element t a c) = node e (Element t a c) +firstNode _ (TextNode "") = mempty +firstNode e (TextNode t) = let (c,t') = fromJust $ T.uncons t + in escaped "<>& \t\r\n" e (T.singleton c) + `mappend` node e (TextNode t') + + +------------------------------------------------------------------------------ +-- XXX: Should do something to avoid concatting large CDATA sections before +-- writing them to the output. +element :: Encoding -> Text -> [(Text, Text)] -> [Node] -> Builder +element e t a c + | t `S.member` voidTags && null c = + fromText e "<" + `mappend` fromText e t + `mappend` (mconcat $ map (attribute e) a) + `mappend` fromText e " />" + | t `S.member` voidTags = + error $ T.unpack t ++ " must be empty" + | t `S.member` rawTextTags, + all isTextNode c, + let s = T.concat (map nodeText c), + not ("" + `mappend` fromText e s + `mappend` fromText e "" + | t `S.member` rawTextTags, + [ TextNode _ ] <- c = + error $ T.unpack t ++ " cannot contain text looking like its end tag" + | t `S.member` rawTextTags = + error $ T.unpack t ++ " cannot contain child elements or comments" + | t `S.member` rcdataTags, + all isTextNode c = + fromText e "<" + `mappend` fromText e t + `mappend` (mconcat $ map (attribute e) a) + `mappend` fromText e ">" + `mappend` mconcat (map (escaped "<&" e . nodeText) c) + `mappend` fromText e "" + | t `S.member` rcdataTags = + error $ T.unpack t ++ " cannot contain child elements or comments" + | otherwise = + fromText e "<" + `mappend` fromText e t + `mappend` (mconcat $ map (attribute e) a) + `mappend` fromText e ">" + `mappend` (mconcat $ map (node e) c) + `mappend` fromText e "" + + +------------------------------------------------------------------------------ +attribute :: Encoding -> (Text, Text) -> Builder +attribute e (n,v) + | v == "" = + fromText e " " + `mappend` fromText e n + | not ("\'" `T.isInfixOf` v) = + fromText e " " + `mappend` fromText e n + `mappend` fromText e "=\'" + `mappend` escaped "&" e v + `mappend` fromText e "\'" + | otherwise = + fromText e " " + `mappend` fromText e n + `mappend` fromText e "=\"" + `mappend` escaped "&\"" e v + `mappend` fromText e "\"" + diff --git a/src/Text/XmlHtml/TextParser.hs b/src/Text/XmlHtml/TextParser.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/TextParser.hs @@ -0,0 +1,125 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} + +module Text.XmlHtml.TextParser where + +import Control.Applicative +import Data.Char +import Data.Maybe +import Text.XmlHtml.Common + +import Data.Text (Text) +import qualified Data.Text as T + +import Text.Parsec (Parsec) +import qualified Text.Parsec as P + +import Data.ByteString (ByteString) +import qualified Data.ByteString as B + + +------------------------------------------------------------------------------ +-- | Get an initial guess at document encoding from the byte order mark. If +-- the mark doesn't exist, guess UTF-8. Otherwise, guess according to the +-- mark. +guessEncoding :: ByteString -> (Encoding, ByteString) +guessEncoding b + | B.take 3 b == B.pack [ 0xEF, 0xBB, 0xBF ] = (UTF8, B.drop 3 b) + | B.take 2 b == B.pack [ 0xFE, 0xFF ] = (UTF16BE, B.drop 2 b) + | B.take 2 b == B.pack [ 0xFF, 0xFE ] = (UTF16LE, B.drop 2 b) + | otherwise = (UTF8, b) + + +------------------------------------------------------------------------------ +-- | Specialized type for the parsers we use here. +type Parser = Parsec Text () + + +------------------------------------------------------------------------------ +-- An (orphaned) instance for parsing Text with Parsec. +instance (Monad m) => P.Stream T.Text m Char where + uncons = return . T.uncons + + +------------------------------------------------------------------------------ +parse :: (Encoding -> Parser a) -> String -> ByteString -> Either String a +parse p src b = let (e, b') = guessEncoding b + t = decoder e b' + bad = T.find (not . isValidChar) t + in if isNothing bad + then parseText (p e <* P.eof) src t + else Left $ "Document contains invalid character:" + ++ " \\" ++ show (ord (fromJust bad)) + + +------------------------------------------------------------------------------ +-- | Checks if a document contains invalid characters. +-- +isValidChar :: Char -> Bool +isValidChar c | c < '\x9' = False + | c > '\xA' && c < '\xD' = False + | c > '\xD' && c < '\x20' = False + | c > '\xD7FF' && c < '\xE000' = False + | c > '\xFFFD' && c < '\x10000' = False + | otherwise = True + + +------------------------------------------------------------------------------ +-- | Parses a 'Text' value and gives back the result. The parser is expected +-- to match the entire string. +parseText :: Parser a -- ^ The parser to match + -> String -- ^ Name of the source file (can be @""@) + -> Text -- ^ Text to parse + -> Either String a -- Either an error message or the result +parseText p src t = inLeft show (P.parse p src t) + where inLeft :: (a -> b) -> Either a c -> Either b c + inLeft f (Left x) = Left (f x) + inLeft _ (Right x) = Right x + + +------------------------------------------------------------------------------ +-- | Consume input as long as the predicate returns 'True', and return the +-- consumed input. This parser does not fail. If it matches no input, it +-- will return an empty string. +takeWhile0 :: (Char -> Bool) -> Parser Text +takeWhile0 p = fmap T.pack $ P.many $ P.satisfy p + + +------------------------------------------------------------------------------ +-- | Consume input as long as the predicate returns 'True', and return the +-- consumed input. This parser requires the predicate to succeed on at least +-- one character of input. It will fail if the first character fails the +-- predicate. +takeWhile1 :: (Char -> Bool) -> Parser Text +takeWhile1 p = fmap T.pack $ P.many1 $ P.satisfy p + + +------------------------------------------------------------------------------ +-- | The equivalent of Parsec's string combinator, but for text. If there is +-- not a complete match, then no input is consumed. This matches the behavior +-- of @string@ from the attoparsec-text package. +text :: Text -> Parser Text +text t = P.try $ P.string (T.unpack t) *> return t + + +------------------------------------------------------------------------------ +-- | Represents the state of a text scanner, for use with the 'scanText' +-- parser combinator. +data ScanState = ScanNext (Char -> ScanState) + | ScanFinish + | ScanFail String + + +------------------------------------------------------------------------------ +-- | Scans text and progresses through a DFA, collecting the complete matching +-- text as it goes. +scanText :: (Char -> ScanState) -> Parser String +scanText f = do + P.try $ do + c <- P.anyChar + case f c of + ScanNext f' -> (c:) `fmap` scanText f' + ScanFinish -> return [c] + ScanFail err -> fail err + diff --git a/src/Text/XmlHtml/XML/Parse.hs b/src/Text/XmlHtml/XML/Parse.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/XML/Parse.hs @@ -0,0 +1,600 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Text.XmlHtml.XML.Parse where + +import Control.Applicative +import Control.Monad +import Data.Char +import Data.List +import Data.Maybe +import Text.XmlHtml.Common +import Text.XmlHtml.TextParser + +import qualified Text.Parsec as P + +import Data.Map (Map) +import qualified Data.Map as M + +import Data.Text (Text) +import qualified Data.Text as T + + +------------------------------------------------------------------------------ +-- | This is my best guess as to the best rule for handling document fragments +-- for processing. It is essentially modeled after document, but allowing +-- multiple nodes. +docFragment :: Encoding -> Parser Document +docFragment e = do + (dt, nodes1) <- prolog + nodes2 <- content + return $ XmlDocument e dt (nodes1 ++ nodes2) + + +------------------------------------------------------------------------------- +-- Everything from here forward is translated from the XML specification. -- +------------------------------------------------------------------------------- + +{- + Map from numbered productions in the XML specification to symbols here: + + PROD SPEC NAME PARSER NAME NOTES + -----|------------------|---------------------|------- + [1] document document + [2] Char {2} + [3] S whiteSpace + [4] NameStartChar isNameStartChar {1} + [4a] NameChar isNameChar {1} + [5] Name name + [6] Names names + [7] Nmtoken nmtoken + [8] Nmtokens nmtokens + [9] EntityValue {4} + [10] AttValue attrValue + [11] SystemLiteral systemLiteral + [12] PubidLiteral pubIdLiteral + [13] PubidChar isPubIdChar {1} + [14] CharData charData + [15] Comment comment + [16] PI processingInstruction + [17] PITarget piTarget + [18] CDSect cdSect + [19] CDStart cdSect {3} + [20] CData cdSect {3} + [21] CDEnd cdSect {3} + [22] prolog prolog + [23] XMLDecl xmlDecl + [24] VersionInfo versionInfo + [25] Eq eq + [26] VersionNum versionInfo {3} + [27] Misc misc + [28] doctypedecl docTypeDecl + [28a] DeclSep {4} + [28b] intSubset {4} + [29] markupdecl {4} + [30] extSubset {4} + [31] extSubsetDecl {4} + [32] SDDecl sdDecl + [39] element element + [40] STag emptyOrStartTag + [41] Attribute attribute + [42] ETag endTag + [43] content content + [44] EmptyElemTag emptyOrStartTag + [45] elementDecl {4} + [46] contentSpec {4} + [47] children {4} + [48] cp {4} + [49] choice {4} + [50] seq {4} + [51] Mixed {4} + [52] AttlistDecl {4} + [53] AttDef {4} + [54] AttType {4} + [55] StringType {4} + [56] TokenizedType {4} + [57] EnumeratedType {4} + [58] NotationType {4} + [59] Enumeration {4} + [60] DefaultDecl {4} + [61] conditionalSect {4} + [62] includeSect {4} + [63] ignoreSect {4} + [64] ignoreSectContents {4} + [65] Ignore {4} + [66] CharRef charRef + [67] Reference reference + [68] EntityRef entityRef + [69] PEReference {4} + [70] EntityDecl {4} + [71] GEDecl {4} + [72] PEDecl {4} + [73] EntityDef {4} + [74] PEDef {4} + [75] ExternalID externalID + [76] NDataDecl {4} + [77] TextDecl textDecl + [78] extParsedEnt extParsedEnt + [80] EncodingDecl encodingDecl + [81] EncName encodingDecl {3} + [82] NotationDecl {4} + [83] PublicID {4} + [84] Letter {5} + [85] BaseChar {5} + [86] Ideographic {5} + [87] CombiningChar {5} + [88] Digit {5} + [89] Extender {5} + + Notes: + {1} - These productions match single characters, and so are implemented + as predicates instead of parsers. + {3} - Denotes a production which is not exposed as a top-level symbol + because it is trivial and included in another definition. + {4} - This module does not contain a parser for the DTD subsets, so + grammar that occurs only in DTD subsets is not defined. + {5} - These are orphaned productions for character classes. +-} + + +------------------------------------------------------------------------------ +whiteSpace :: Parser () +whiteSpace = some (P.satisfy (`elem` " \t\r\n")) *> return () + + +------------------------------------------------------------------------------ +isNameStartChar :: Char -> Bool +isNameStartChar c | c == ':' = True + | c == '_' = True + | c >= 'a' && c <= 'z' = True + | c >= 'A' && c <= 'Z' = True + | c >= '\xc0' && c <= '\xd6' = True + | c >= '\xd8' && c <= '\xf6' = True + | c >= '\xf8' && c <= '\x2ff' = True + | c >= '\x370' && c <= '\x37d' = True + | c >= '\x37f' && c <= '\x1fff' = True + | c >= '\x200c' && c <= '\x200d' = True + | c >= '\x2070' && c <= '\x218f' = True + | c >= '\x2c00' && c <= '\x2fef' = True + | c >= '\x3001' && c <= '\xd7ff' = True + | c >= '\xf900' && c <= '\xfdcf' = True + | c >= '\xfdf0' && c <= '\xfffd' = True + | c >= '\x10000' && c <= '\xeffff' = True + | otherwise = False + + +------------------------------------------------------------------------------ +isNameChar :: Char -> Bool +isNameChar c | isNameStartChar c = True + | c == '-' = True + | c == '.' = True + | c == '\xb7' = True + | c >= '0' && c <= '9' = True + | c >= '\x300' && c <= '\x36f' = True + | c >= '\x203f' && c <= '\x2040' = True + | otherwise = False + + +------------------------------------------------------------------------------ +name :: Parser Text +name = do + c <- P.satisfy isNameStartChar + r <- takeWhile0 isNameChar + return $ T.cons c r + + +------------------------------------------------------------------------------ +attrValue :: Parser Text +attrValue = fmap T.concat (singleQuoted <|> doubleQuoted) + where + singleQuoted = P.char '\'' *> refTill "<&\'" <* P.char '\'' + doubleQuoted = P.char '\"' *> refTill "<&\"" <* P.char '\"' + refTill end = many (takeWhile1 (not . (`elem` end)) <|> reference) + + +------------------------------------------------------------------------------ +systemLiteral :: Parser Text +systemLiteral = singleQuoted <|> doubleQuoted + where + singleQuoted = do + _ <- P.char '\'' + x <- takeWhile0 (not . (== '\'')) + _ <- P.char '\'' + return x + doubleQuoted = do + _ <- P.char '\"' + x <- takeWhile0 (not . (== '\"')) + _ <- P.char '\"' + return x + + +------------------------------------------------------------------------------ +pubIdLiteral :: Parser Text +pubIdLiteral = singleQuoted <|> doubleQuoted + where + singleQuoted = do + _ <- P.char '\'' + x <- takeWhile0 (\c -> isPubIdChar c && c /= '\'') + _ <- P.char '\'' + return x + doubleQuoted = do + _ <- P.char '\"' + x <- takeWhile0 isPubIdChar + _ <- P.char '\"' + return x + + +------------------------------------------------------------------------------ +isPubIdChar :: Char -> Bool +isPubIdChar c | c >= 'a' && c <= 'z' = True + | c >= 'A' && c <= 'Z' = True + | c >= '0' && c <= '9' = True + | c `elem` " \r\n-\'()+,./:=?;!*#@$_%" = True + | otherwise = False + + +------------------------------------------------------------------------------ +-- | The requirement to not contain "]]>" is for SGML compatibility. We +-- deliberately choose to not enforce it. This makes the parser accept +-- strictly more documents than a standards-compliant parser. +charData :: Parser Node +charData = TextNode <$> takeWhile1 (not . (`elem` "<&")) + + +------------------------------------------------------------------------------ +comment :: Parser (Maybe Node) +comment = text "" + where + commentText = fmap T.concat $ many $ + nonDash <|> P.try (T.cons <$> P.char '-' <*> nonDash) + nonDash = takeWhile1 (not . (== '-')) + + +------------------------------------------------------------------------------ +-- | Always returns Nothing since there's no representation for a PI in the +-- document tree. +processingInstruction :: Parser (Maybe Node) +processingInstruction = do + _ <- text " contentEnd + return Nothing + where + emptyEnd = P.try (P.string "?>") + contentEnd = P.try $ do + _ <- whiteSpace + P.manyTill P.anyChar (P.try $ text "?>") + +------------------------------------------------------------------------------ +piTarget :: Parser () +piTarget = do + n <- name + when (T.map toLower n == "xml") $ fail "xml declaration can't occur here" + + +------------------------------------------------------------------------------ +cdata :: [Char] -> Parser a -> Parser Node +cdata cs end = TextNode <$> T.concat <$> P.manyTill part end + where part = takeWhile1 (not . (`elem` cs)) + <|> T.singleton <$> P.anyChar + + +------------------------------------------------------------------------------ +cdSect :: Parser (Maybe Node) +cdSect = Just <$> do + _ <- text "") + + +------------------------------------------------------------------------------ +prolog :: Parser (Maybe DocType, [Node]) +prolog = do + _ <- optional xmlDecl + nodes1 <- many misc + rest <- optional $ do + dt <- docTypeDecl + nodes2 <- many misc + return (dt, nodes2) + case rest of + Nothing -> return (Nothing, catMaybes nodes1) + Just (dt, nodes2) -> return (Just dt, catMaybes (nodes1 ++ nodes2)) + + +------------------------------------------------------------------------------ +-- | Return value is the encoding, if present. +xmlDecl :: Parser (Maybe Text) +xmlDecl = do + _ <- text "" + return e + + +------------------------------------------------------------------------------ +versionInfo :: Parser () +versionInfo = do + whiteSpace *> text "version" *> eq *> (singleQuoted <|> doubleQuoted) + where + singleQuoted = P.char '\'' *> versionNum <* P.char '\'' + doubleQuoted = P.char '\"' *> versionNum <* P.char '\"' + versionNum = do + _ <- text "1." + _ <- some (P.satisfy (\c -> c >= '0' && c <= '9')) + return () + + +------------------------------------------------------------------------------ +eq :: Parser () +eq = optional whiteSpace *> P.char '=' *> optional whiteSpace *> return () + + +------------------------------------------------------------------------------ +misc :: Parser (Maybe Node) +misc = comment <|> processingInstruction <|> (whiteSpace *> return Nothing) + + +------------------------------------------------------------------------------ +-- | Internal subset is parsed, but ignored since we don't have data types to +-- store it. +docTypeDecl :: Parser DocType +docTypeDecl = do + _ <- text "' + return (DocType tag extid intsub) + + +------------------------------------------------------------------------------ +-- | States for the DOCTYPE internal subset state machine. +data InternalDoctypeState = IDSStart + | IDSScanning Int + | IDSInQuote Int Char + | IDSCommentS1 Int + | IDSCommentS2 Int + | IDSCommentS3 Int + | IDSComment Int + | IDSCommentD1 Int + | IDSCommentE1 Int + + +------------------------------------------------------------------------------ +-- | Internal DOCTYPE subset. We don't actually parse this; just scan through +-- and look for the end, and store it in a block of text. +internalDoctype :: Parser InternalSubset +internalDoctype = InternalText <$> T.pack <$> scanText (dfa IDSStart) + <|> return NoInternalSubset + where dfa IDSStart '[' = ScanNext (dfa (IDSScanning 0)) + dfa IDSStart _ = ScanFail "Not a DOCTYPE internal subset" + dfa (IDSInQuote n c) d + | c == d = ScanNext (dfa (IDSScanning n)) + | otherwise = ScanNext (dfa (IDSInQuote n c)) + dfa (IDSScanning n) '[' = ScanNext (dfa (IDSScanning (n+1))) + dfa (IDSScanning 0) ']' = ScanFinish + dfa (IDSScanning n) ']' = ScanNext (dfa (IDSScanning (n-1))) + dfa (IDSScanning n) '\'' = ScanNext (dfa (IDSInQuote n '\'')) + dfa (IDSScanning n) '\"' = ScanNext (dfa (IDSInQuote n '\"')) + dfa (IDSScanning n) '<' = ScanNext (dfa (IDSCommentS1 n)) + dfa (IDSScanning n) _ = ScanNext (dfa (IDSScanning n)) + dfa (IDSCommentS1 n) '[' = ScanNext (dfa (IDSScanning (n+1))) + dfa (IDSCommentS1 0) ']' = ScanFinish + dfa (IDSCommentS1 n) ']' = ScanNext (dfa (IDSScanning (n-1))) + dfa (IDSCommentS1 n) '\'' = ScanNext (dfa (IDSInQuote n '\'')) + dfa (IDSCommentS1 n) '\"' = ScanNext (dfa (IDSInQuote n '\"')) + dfa (IDSCommentS1 n) '!' = ScanNext (dfa (IDSCommentS2 n)) + dfa (IDSCommentS1 n) _ = ScanNext (dfa (IDSScanning n)) + dfa (IDSCommentS2 n) '[' = ScanNext (dfa (IDSScanning (n+1))) + dfa (IDSCommentS2 0) ']' = ScanFinish + dfa (IDSCommentS2 n) ']' = ScanNext (dfa (IDSScanning (n-1))) + dfa (IDSCommentS2 n) '\'' = ScanNext (dfa (IDSInQuote n '\'')) + dfa (IDSCommentS2 n) '\"' = ScanNext (dfa (IDSInQuote n '\"')) + dfa (IDSCommentS2 n) '-' = ScanNext (dfa (IDSCommentS3 n)) + dfa (IDSCommentS2 n) _ = ScanNext (dfa (IDSScanning n)) + dfa (IDSCommentS3 n) '[' = ScanNext (dfa (IDSScanning (n+1))) + dfa (IDSCommentS3 0) ']' = ScanFinish + dfa (IDSCommentS3 n) ']' = ScanNext (dfa (IDSScanning (n-1))) + dfa (IDSCommentS3 n) '\'' = ScanNext (dfa (IDSInQuote n '\'')) + dfa (IDSCommentS3 n) '\"' = ScanNext (dfa (IDSInQuote n '\"')) + dfa (IDSCommentS3 n) '-' = ScanNext (dfa (IDSComment n)) + dfa (IDSCommentS3 n) _ = ScanNext (dfa (IDSScanning n)) + dfa (IDSComment n) '-' = ScanNext (dfa (IDSCommentD1 n)) + dfa (IDSComment n) _ = ScanNext (dfa (IDSComment n)) + dfa (IDSCommentD1 n) '-' = ScanNext (dfa (IDSCommentE1 n)) + dfa (IDSCommentD1 n) _ = ScanNext (dfa (IDSComment n)) + dfa (IDSCommentE1 n) '>' = ScanNext (dfa (IDSScanning n)) + dfa (IDSCommentE1 _) _ = ScanFail "Poorly formatted comment" + + +------------------------------------------------------------------------------ +sdDecl :: Parser () +sdDecl = do + _ <- P.try $ whiteSpace *> text "standalone" + eq + _ <- single <|> double + return () + where + single = P.char '\'' *> yesno <* P.char '\'' + double = P.char '\"' *> yesno <* P.char '\"' + yesno = text "yes" <|> text "no" + + +------------------------------------------------------------------------------ +element :: Parser Node +element = do + (t,a,b) <- emptyOrStartTag + if b then return (Element t a []) + else nonEmptyElem t a + where + nonEmptyElem t a = do + c <- content + endTag t + return (Element t a c) + + +------------------------------------------------------------------------------ +-- | Results are (tag name, attributes, isEmpty) +emptyOrStartTag :: Parser (Text, [(Text, Text)], Bool) +emptyOrStartTag = do + t <- P.try $ P.char '<' *> name + a <- many $ P.try $ do + whiteSpace + attribute + when (hasDups a) $ fail "Duplicate attribute names in element" + _ <- optional whiteSpace + e <- optional (P.char '/') + _ <- P.char '>' + return (t, a, isJust e) + where + hasDups a = length (nub (map fst a)) < length a + + +------------------------------------------------------------------------------ +attribute :: Parser (Text, Text) +attribute = do + n <- name + eq + v <- attrValue + return (n,v) + + +------------------------------------------------------------------------------ +endTag :: Text -> Parser () +endTag s = do + _ <- text " found inside <" ++ T.unpack s ++ "> tag" + _ <- optional whiteSpace + _ <- text ">" + return () + + +------------------------------------------------------------------------------ +content :: Parser [Node] +content = do + n <- optional charData + ns <- fmap concat $ many $ do + s <- ((Just <$> TextNode <$> reference) + <|> cdSect + <|> processingInstruction + <|> comment + <|> fmap Just element) + t <- optional charData + return [s,t] + return $ coalesceText $ catMaybes (n:ns) + where + coalesceText (TextNode s : TextNode t : ns) + = coalesceText (TextNode (T.append s t) : ns) + coalesceText (n:ns) + = n : coalesceText ns + coalesceText [] + = [] + + +------------------------------------------------------------------------------ +charRef :: Parser Text +charRef = hexCharRef <|> decCharRef + where + decCharRef = do + _ <- text "&#" + ds <- some digit + _ <- P.char ';' + let c = chr $ foldl' (\a b -> 10 * a + b) 0 ds + when (not (isValidChar c)) $ fail $ + "Reference is not a valid character" + return $ T.singleton c + where + digit = do + d <- P.satisfy (\c -> c >= '0' && c <= '9') + return (ord d - ord '0') + hexCharRef = do + _ <- text "&#x" + ds <- some digit + _ <- P.char ';' + let c = chr $ foldl' (\a b -> 16 * a + b) 0 ds + when (not (isValidChar c)) $ fail $ + "Reference is not a valid character" + return $ T.singleton c + where + digit = num <|> upper <|> lower + num = do + d <- P.satisfy (\c -> c >= '0' && c <= '9') + return (ord d - ord '0') + upper = do + d <- P.satisfy (\c -> c >= 'A' && c <= 'F') + return (10 + ord d - ord 'A') + lower = do + d <- P.satisfy (\c -> c >= 'a' && c <= 'f') + return (10 + ord d - ord 'a') + + +------------------------------------------------------------------------------ +reference :: Parser Text +reference = charRef <|> entityRef + + +------------------------------------------------------------------------------ +entityRef :: Parser Text +entityRef = do + _ <- P.char '&' + n <- name + _ <- P.char ';' + case M.lookup n entityRefLookup of + Nothing -> fail $ "Unknown entity reference: " ++ T.unpack n + Just t -> return t + where + entityRefLookup :: Map Text Text + entityRefLookup = M.fromList [ + ("amp", "&"), + ("lt", "<"), + ("gt", ">"), + ("apos", "\'"), + ("quot", "\"") + ] + + +------------------------------------------------------------------------------ +externalID :: Parser ExternalID +externalID = systemID <|> publicID <|> return NoExternalID + where + systemID = do + _ <- text "SYSTEM" + whiteSpace + fmap System systemLiteral + publicID = do + _ <- text "PUBLIC" + whiteSpace + pid <- pubIdLiteral + whiteSpace + sid <- systemLiteral + return (Public pid sid) + + +------------------------------------------------------------------------------ +encodingDecl :: Parser Text +encodingDecl = do + _ <- P.try $ whiteSpace *> text "encoding" + _ <- eq + singleQuoted <|> doubleQuoted + where + singleQuoted = P.char '\'' *> encName <* P.char '\'' + doubleQuoted = P.char '\"' *> encName <* P.char '\"' + encName = do + c <- P.satisfy isEncStart + cs <- takeWhile0 isEnc + return (T.cons c cs) + isEncStart c | c >= 'A' && c <= 'Z' = True + | c >= 'a' && c <= 'z' = True + | otherwise = False + isEnc c | c >= 'A' && c <= 'Z' = True + | c >= 'a' && c <= 'z' = True + | c >= '0' && c <= '9' = True + | c `elem` "._-" = True + | otherwise = False + diff --git a/src/Text/XmlHtml/XML/Render.hs b/src/Text/XmlHtml/XML/Render.hs new file mode 100644 --- /dev/null +++ b/src/Text/XmlHtml/XML/Render.hs @@ -0,0 +1,153 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Text.XmlHtml.XML.Render where + +import Blaze.ByteString.Builder +import Data.Char +import Data.Maybe +import Data.Monoid +import Text.XmlHtml.Common + +import Data.Text (Text) +import qualified Data.Text as T + + +------------------------------------------------------------------------------ +render :: Encoding -> Maybe DocType -> [Node] -> Builder +render e dt ns = byteOrder + `mappend` xmlDecl e + `mappend` docTypeDecl e dt + `mappend` nodes + where byteOrder | isUTF16 e = fromText e "\xFEFF" -- byte order mark + | otherwise = mempty + nodes | null ns = mempty + | otherwise = firstNode e (head ns) + `mappend` (mconcat $ map (node e) (tail ns)) + + +------------------------------------------------------------------------------ +xmlDecl :: Encoding -> Builder +xmlDecl e = fromText e "\n" + + +------------------------------------------------------------------------------ +docTypeDecl :: Encoding -> Maybe DocType -> Builder +docTypeDecl _ Nothing = mempty +docTypeDecl e (Just (DocType tag ext int)) = fromText e "" + + +------------------------------------------------------------------------------ +externalID :: Encoding -> ExternalID -> Builder +externalID _ NoExternalID = mempty +externalID e (System sid) = fromText e " SYSTEM " + `mappend` sysID e sid +externalID e (Public pid sid) = fromText e " PUBLIC " + `mappend` pubID e pid + `mappend` fromText e " " + `mappend` sysID e sid + + +------------------------------------------------------------------------------ +internalSubset :: Encoding -> InternalSubset -> Builder +internalSubset _ NoInternalSubset = mempty +internalSubset e (InternalText t) = fromText e " " `mappend` fromText e t + + +------------------------------------------------------------------------------ +sysID :: Encoding -> Text -> Builder +sysID e sid | not ("\'" `T.isInfixOf` sid) = fromText e "\'" + `mappend` fromText e sid + `mappend` fromText e "\'" + | not ("\"" `T.isInfixOf` sid) = fromText e "\"" + `mappend` fromText e sid + `mappend` fromText e "\"" + | otherwise = error "SYSTEM id is invalid" + + +------------------------------------------------------------------------------ +pubID :: Encoding -> Text -> Builder +pubID e sid | not ("\"" `T.isInfixOf` sid) = fromText e "\"" + `mappend` fromText e sid + `mappend` fromText e "\"" + | otherwise = error "PUBLIC id is invalid" + + +------------------------------------------------------------------------------ +node :: Encoding -> Node -> Builder +node e (TextNode t) = escaped "<>&" e t +node e (Comment t) | "--" `T.isInfixOf` t = error "Invalid comment" + | "-" `T.isSuffixOf` t = error "Invalid comment" + | otherwise = fromText e "" +node e (Element t a c) = element e t a c + + +------------------------------------------------------------------------------ +-- | Process the first node differently to encode leading whitespace. This +-- lets us be sure that @parseXML@ is a left inverse to @render@. +firstNode :: Encoding -> Node -> Builder +firstNode e (Comment t) = node e (Comment t) +firstNode e (Element t a c) = node e (Element t a c) +firstNode _ (TextNode "") = mempty +firstNode e (TextNode t) = let (c,t') = fromJust $ T.uncons t + in escaped "<>& \t\r\n" e (T.singleton c) + `mappend` node e (TextNode t') + + +------------------------------------------------------------------------------ +escaped :: [Char] -> Encoding -> Text -> Builder +escaped _ _ "" = mempty +escaped bad e t = let (p,s) = T.break (`elem` bad) t + r = T.uncons s + in fromText e p `mappend` case r of + Nothing -> mempty + Just (c,ss) -> entity e c `mappend` escaped bad e ss + + +------------------------------------------------------------------------------ +entity :: Encoding -> Char -> Builder +entity e '&' = fromText e "&" +entity e '<' = fromText e "<" +entity e '>' = fromText e ">" +entity e '\"' = fromText e """ +entity e c = fromText e "&#" + `mappend` fromText e (T.pack (show (ord c))) + `mappend` fromText e ";" + + +------------------------------------------------------------------------------ +element :: Encoding -> Text -> [(Text, Text)] -> [Node] -> Builder +element e t a [] = fromText e "<" + `mappend` fromText e t + `mappend` (mconcat $ map (attribute e) a) + `mappend` fromText e "/>" +element e t a c = fromText e "<" + `mappend` fromText e t + `mappend` (mconcat $ map (attribute e) a) + `mappend` fromText e ">" + `mappend` (mconcat $ map (node e) c) + `mappend` fromText e "" + + +------------------------------------------------------------------------------ +attribute :: Encoding -> (Text, Text) -> Builder +attribute e (n,v) | not ("\'" `T.isInfixOf` v) = fromText e " " + `mappend` fromText e n + `mappend` fromText e "=\'" + `mappend` escaped "<&" e v + `mappend` fromText e "\'" + | otherwise = fromText e " " + `mappend` fromText e n + `mappend` fromText e "=\"" + `mappend` escaped "<&\"" e v + `mappend` fromText e "\"" + diff --git a/xmlhtml.cabal b/xmlhtml.cabal new file mode 100644 --- /dev/null +++ b/xmlhtml.cabal @@ -0,0 +1,51 @@ +Name: xmlhtml +Version: 0.1 +Synopsis: XML parser and renderer with HTML 5 quirks mode +Description: Contains renderers and parsers for both XML and HTML 5 + document fragments, which share data structures wo that + it's easy to work with both. Document fragments are bits + of documents, which are not constrained by some of the + high-level structure rules (in particular, they may + contain more than one root element). + . + Note that this is not a compliant HTML 5 parser. Rather, + it is a parser for HTML 5 compliant documents. It does not + implement the HTML 5 parsing algorithm, and should + generally be expected to perform correctly only on + documents that you trust to conform to HTML 5. This is + not a suitable library for implementing web crawlers or + other software that will be exposed to documents from + outside sources. The result is also not the HTML 5 + node structure, but rather something closer to the physical + structure. For example, omitted start tags are not + inserted (and so, their corresponding end tags must also be + omitted). +License: BSD3 +License-file: LICENSE +Author: Chris Smith +Maintainer: Chris Smith +Category: Text +Build-type: Simple +Cabal-version: >=1.6 + +Library + Hs-source-dirs: src + Exposed-modules: Text.XmlHtml, + Text.XmlHtml.Cursor, + Text.Blaze.Renderer.XmlHtml + Other-modules: Text.XmlHtml.Common, + Text.XmlHtml.TextParser, + Text.XmlHtml.XML.Parse, + Text.XmlHtml.XML.Render, + Text.XmlHtml.HTML.Meta, + Text.XmlHtml.HTML.Parse, + Text.XmlHtml.HTML.Render + Build-depends: base == 4.*, + blaze-builder == 0.2.*, + blaze-html == 0.3.*, + bytestring == 0.9.*, + containers >= 0.3 && <0.5, + parsec >= 3.0 && < 3.2, + text >= 0.11 && < 0.12 + + Ghc-options: -Wall -fwarn-tabs