css-syntax (empty) → 0.0.1
raw patch · 5 files changed
+834/−0 lines, 5 filesdep +attoparsecdep +basedep +bytestringsetup-changed
Dependencies added: attoparsec, base, bytestring, directory, hspec, scientific, text
Files
- LICENSE +20/−0
- Setup.hs +2/−0
- css-syntax.cabal +59/−0
- src/Data/CSS/Syntax/Tokens.hs +493/−0
- test/Test.hs +260/−0
+ LICENSE view
@@ -0,0 +1,20 @@+Copyright (c) 2015 Tomas Carnecky++Permission is hereby granted, free of charge, to any person obtaining+a copy of this software and associated documentation files (the+"Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish,+distribute, sublicense, and/or sell copies of the Software, and to+permit persons to whom the Software is furnished to do so, subject to+the following conditions:++The above copyright notice and this permission notice shall be included+in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ css-syntax.cabal view
@@ -0,0 +1,59 @@+name: css-syntax+version: 0.0.1++synopsis: This package implments a parser for the CSS syntax+description:+ See https://drafts.csswg.org/css-syntax/+++license: MIT+license-file: LICENSE++author: Tomas Carnecky+maintainer: tomas.carnecky@gmail.com++category: Data++build-type: Simple+cabal-version: >=1.10+++source-repository head+ type: git+ location: git://github.com/wereHamster/haskell-css-syntax.git+++library+ hs-source-dirs: src+ default-language: Haskell2010++ exposed-modules:+ Data.CSS.Syntax.Tokens++ build-depends:+ base >=4 && <4.9+ , attoparsec >= 0.13+ , bytestring+ , scientific+ , text++ ghc-options: -Wall+++test-suite spec+ hs-source-dirs: test src+ default-language: Haskell2010+ ghc-options: -Wall++ type: exitcode-stdio-1.0+ main-is: Test.hs++ build-depends:+ base >=4 && <4.9+ , attoparsec >= 0.13+ , bytestring+ , scientific+ , text++ , hspec+ , directory
+ src/Data/CSS/Syntax/Tokens.hs view
@@ -0,0 +1,493 @@+{-# LANGUAGE OverloadedStrings #-}++module Data.CSS.Syntax.Tokens+ ( Token(..)+ , NumericValue(..)+ , HashFlag(..)+ , Unit++ , tokenize+ , serialize+ ) where+++import Control.Applicative+import Control.Monad++import Data.Text (Text)+import qualified Data.Text as T+import Data.Attoparsec.Text as AP+import Data.Monoid+import Data.Char+import Data.Scientific++import Prelude++++data Token+ = Whitespace++ | CDO -- CommentDelimiterOpen+ | CDC -- CommentDelimiterClose++ | Comma+ | Colon+ | Semicolon++ | LeftParen+ | RightParen+ | LeftSquareBracket+ | RightSquareBracket+ | LeftCurlyBracket+ | RightCurlyBracket++ | SuffixMatch+ | SubstringMatch+ | PrefixMatch+ | DashMatch+ | IncludeMatch++ | Column++ | String !Text+ | BadString !Text++ | Number !Text !NumericValue+ | Percentage !Text !NumericValue+ | Dimension !Text !NumericValue !Unit++ | Url !Text+ | BadUrl !Text++ | Ident !Text++ | AtKeyword !Text++ | Function !Text++ | Hash !HashFlag !Text++ | Delim !Char++ deriving (Show, Eq)+++data NumericValue+ = NVInteger !Scientific+ | NVNumber !Scientific+ deriving (Show, Eq)++data HashFlag = HId | HUnrestricted+ deriving (Show, Eq)++type Unit = Text++++-- Tokenization+-------------------------------------------------------------------------------+++-- | Parse a 'Text' into a list of 'Token's.+--+-- https://drafts.csswg.org/css-syntax/#tokenization++tokenize :: Text -> Either String [Token]+tokenize = parseOnly (many' parseToken) . preprocessInputStream++++-- | Before sending the input stream to the tokenizer, implementations must+-- make the following code point substitutions: (see spec)+--+-- https://drafts.csswg.org/css-syntax/#input-preprocessing++preprocessInputStream :: Text -> Text+preprocessInputStream = T.pack . f . T.unpack+ where+ f [] = []++ f ('\x000D':'\x000A':r) = '\x000A' : f r++ f ('\x000D':r) = '\x000A' : f r+ f ('\x000C':r) = '\x000A' : f r++ f ('\x0000':r) = '\xFFFD' : f r++ f (x:r) = x : f r++++-- Serialization+-------------------------------------------------------------------------------+++-- | Serialize a list of 'Token's back into 'Text'. Round-tripping is not+-- guaranteed to be identity. The tokenization step drops some information+-- from the source.+--+-- https://drafts.csswg.org/css-syntax/#serialization++serialize :: [Token] -> Text+serialize = mconcat . map renderToken+++renderToken :: Token -> Text+renderToken (Whitespace) = " "++renderToken (CDO) = "<!--"+renderToken (CDC) = "-->"++renderToken (Comma) = ","+renderToken (Colon) = ":"+renderToken (Semicolon) = ";"++renderToken (LeftParen) = "("+renderToken (RightParen) = ")"+renderToken (LeftSquareBracket) = "["+renderToken (RightSquareBracket) = "]"+renderToken (LeftCurlyBracket) = "{"+renderToken (RightCurlyBracket) = "}"++renderToken (SuffixMatch) = "$="+renderToken (SubstringMatch) = "*="+renderToken (PrefixMatch) = "^="+renderToken (DashMatch) = "|="+renderToken (IncludeMatch) = "~="++renderToken (Column) = "||"++renderToken (String x) = "'" <> x <> "'"+renderToken (BadString x) = "'" <> x <> "'"++renderToken (Number x _) = x+renderToken (Percentage x _) = x <> "%"+renderToken (Dimension x _ u) = x <> u++renderToken (Url x) = "url(" <> x <> ")"+renderToken (BadUrl x) = "url(" <> x <> ")"++renderToken (Ident x) = x++renderToken (AtKeyword x) = "@" <> x++renderToken (Function x) = x <> "("++renderToken (Hash _ x) = x++renderToken (Delim x) = T.singleton x++++parseComment :: Parser ()+parseComment = do+ void $ AP.string "/*"+ void $ AP.manyTill' AP.anyChar (void (AP.string "*/") <|> AP.endOfInput)++parseWhitespace :: Parser Token+parseWhitespace = do+ void $ AP.takeWhile1 isWhitespace+ return Whitespace++parseChar :: Token -> Char -> Parser Token+parseChar t c = do+ _ <- AP.char c+ return t++parseStr :: Token -> Text -> Parser Token+parseStr t str = AP.string str *> return t++escapedCodePoint :: Parser Char+escapedCodePoint = do+ mbChar <- AP.peekChar+ case mbChar of+ Nothing -> return $ '\xFFFD'+ Just ch -> do+ if isHexChar ch+ then do+ (t, _) <- AP.runScanner [] f+ case unhex (T.unpack t) of+ Nothing -> fail $ "escapedCodePoint: unable to parse hex " ++ (T.unpack t)+ Just cp -> do+ AP.peekChar >>= \c -> case c of+ Just nc -> if isWhitespace nc then void AP.anyChar else return ()+ _ -> return ()+ return $ if cp == 0 || cp > 0x10FFFF+ then chr 0xFFFD+ else chr cp+ else do+ if ch == '\n'+ then fail "A newline"+ else AP.anyChar >> return ch++ where+ f :: String -> Char -> Maybe String+ f acc c =+ if length acc < 6 && isHexChar c+ then Just $ c:acc+ else Nothing+++nextInputCodePoint :: Parser Char+nextInputCodePoint = escapedCodePoint' <|> AP.anyChar+++whenNext :: Char -> a -> Parser a+whenNext c a = do+ mbChar <- AP.peekChar+ if mbChar == Just c+ then return a+ else fail "whenNext"++-- 4.3.4. Consume a string token+parseString :: Char -> Parser Token+parseString endingCodePoint = do+ _ <- AP.char endingCodePoint+ go mempty++ where+ go acc = choice+ [ (AP.endOfInput <|> void (AP.char endingCodePoint)) *> return (String acc)+ , AP.string "\\\n" *> go acc+ , whenNext '\n' (BadString acc)+ , nextInputCodePoint >>= \ch -> go (acc <> T.singleton ch)+ ]+++parseHash :: Parser Token+parseHash = do+ _ <- AP.char '#'+ name <- parseName+ return $ Hash HId name++isNameStartCodePoint :: Char -> Bool+isNameStartCodePoint c = isLetter c || c >= '\x0080' || c == '_'++isNameCodePoint :: Char -> Bool+isNameCodePoint c = isNameStartCodePoint c || isDigit c || c == '-'++parseNumeric :: Parser Token+parseNumeric = do+ (repr, nv) <- parseNumericValue+ dimNum repr nv <|> pctNum repr nv <|> return (Number repr nv)+ where+ dimNum repr nv = do+ unit <- parseName+ return $ Dimension repr nv unit+ pctNum repr nv = do+ _ <- AP.char '%'+ return $ Percentage repr nv++nameCodePoint :: Parser Char+nameCodePoint = AP.satisfy isNameCodePoint++escapedCodePoint' :: Parser Char+escapedCodePoint' = do+ _ <- AP.char '\\'+ escapedCodePoint++parseName :: Parser Text+parseName = do+ chars <- AP.many1' $+ nameCodePoint <|> escapedCodePoint'++ case chars of+ '-':xs -> case xs of+ _:_ -> return $ T.pack chars+ _ -> fail "parseName: Not a valid name start"+ _ -> return $ T.pack chars+++parseSign :: Parser (Text, Int)+parseSign = do+ mbChar <- AP.peekChar+ case mbChar of+ Just '+' -> AP.anyChar >> return ("+", 1)+ Just '-' -> AP.anyChar >> return ("-", (-1))+ _ -> return ("", 1)++parseNumericValue :: Parser (Text, NumericValue)+parseNumericValue = do+ -- Sign+ (sS, s) <- parseSign++ -- Digits before the decimal dot. They are optional (".1em").+ (iS, i) <- do+ digits <- AP.takeWhile isDigit+ return $ if (T.null digits)+ then ("", 0)+ else (digits, read $ T.unpack digits)++ -- Decimal dot and digits after it. If the decimal dot is there then it+ -- MUST be followed by one or more digits. This is not allowed: "1.".+ (fS, f, fB) <- option ("", 0, False) $ do+ _ <- AP.char '.'+ digits <- AP.takeWhile1 isDigit+ return ("." <> digits, read $ T.unpack digits, True)++ -- Exponent (with optional sign).+ (tS, t, eS, e, eB) <- option ("", 1, "", 0, False) $ do+ e <- AP.char 'E' <|> AP.char 'e'+ (tS, t) <- parseSign+ eS <- AP.takeWhile1 isDigit++ return (T.singleton e <> tS, t, eS, read $ T.unpack eS, True)++ let repr = sS<>iS<>fS<>tS<>eS+ if T.null repr || repr == "-" || repr == "+" || T.head repr == 'e' || T.head repr == 'E'+ then fail "parseNumericValue: no parse"+ else do+ let v = fromIntegral s * (i + f*10^^(-(T.length fS - 1))) * 10^^(t*e)+ return $ if fB || eB+ then (repr, NVNumber v)+ else (repr, NVInteger v)+++parseUrl :: Parser Token+parseUrl = do+ _ <- AP.takeWhile isWhitespace+ go mempty++ where+ endOfUrl acc = (AP.endOfInput <|> void (AP.char ')')) *> return (Url acc)++ go acc = choice+ [ endOfUrl acc+ , (AP.char '"' <|> AP.char '\'' <|> AP.char '(') >>= \ch -> badUrl (acc <> T.singleton ch)+ , AP.string "\\\n" *> badUrl (acc <> "\\\n")+ , AP.takeWhile1 isWhitespace >>= \c -> (endOfUrl acc <|> badUrl (acc <> c))+ , nextInputCodePoint >>= \ch -> go (acc <> T.singleton ch)+ ]++ badUrl acc = choice+ [ (AP.endOfInput <|> void (AP.char ')')) *> return (BadUrl acc)+ , nextInputCodePoint >>= \ch -> badUrl (acc <> T.singleton ch)+ ]+++parseIdentLike :: Parser Token+parseIdentLike = do+ name <- parseName+ choice+ [ do+ -- Special handling of url() functions (they are not really+ -- functions, they have their own Token type).+ guard $ T.isPrefixOf "url" (T.map toLower name)++ void $ AP.char '('+ void $ AP.takeWhile isWhitespace++ whenNext '"' (Function name) <|> whenNext '\'' (Function name) <|> parseUrl++ , AP.char '(' *> return (Function name)+ , return (Ident name)+ ]+++parseEscapedIdentLike :: Parser Token+parseEscapedIdentLike = do+ mbChar <- AP.peekChar+ case mbChar of+ Just '\\' -> parseIdentLike <|> (AP.anyChar >> return (Delim '\\'))+ _ -> fail "parseEscapedIdentLike: Does not start with an escape code"++parseAtKeyword :: Parser Token+parseAtKeyword = do+ _ <- AP.char '@'+ name <- parseName+ return $ AtKeyword name+++parseToken :: Parser Token+parseToken = AP.many' parseComment *> choice+ [ parseWhitespace++ , AP.string "<!--" *> return CDO+ , AP.string "-->" *> return CDC++ , parseChar Comma ','+ , parseChar Colon ':'+ , parseChar Semicolon ';'+ , parseChar LeftParen '('+ , parseChar RightParen ')'+ , parseChar LeftSquareBracket '['+ , parseChar RightSquareBracket ']'+ , parseChar LeftCurlyBracket '{'+ , parseChar RightCurlyBracket '}'++ , parseStr SuffixMatch "$="+ , parseStr SubstringMatch "*="+ , parseStr PrefixMatch "^="+ , parseStr DashMatch "|="+ , parseStr IncludeMatch "~="++ , parseStr Column "||"++ , parseNumeric++ , parseEscapedIdentLike+ , parseIdentLike+ , parseHash++ , parseString '"'+ , parseString '\''++ , parseAtKeyword++ , AP.anyChar >>= return . Delim+ ] <?> "token"++++isWhitespace :: Char -> Bool+isWhitespace '\x0009' = True+isWhitespace '\x000A' = True+isWhitespace '\x0020' = True+isWhitespace _ = False+++isHexChar :: Char -> Bool+isHexChar ch+ | ch >= '0' && ch <= '9' = True+ | ch >= 'A' && ch <= 'F' = True+ | ch >= 'a' && ch <= 'f' = True+ | otherwise = False+++unhex :: Monad m => String -> m Int+unhex = fmap toInt . go []+ where++ go :: Monad m => [Int] -> String -> m [Int]+ go acc [] = return acc+ go acc (a:r) = do+ x <- c a+ go (x:acc) r++ toInt = sum . map (\(e, x) -> 16 ^ e * x) . zip [(0::Int)..]++ c :: Monad m => Char -> m Int+ c '0' = return 0+ c '1' = return 1+ c '2' = return 2+ c '3' = return 3+ c '4' = return 4+ c '5' = return 5+ c '6' = return 6+ c '7' = return 7+ c '8' = return 8+ c '9' = return 9+ c 'A' = return 10+ c 'B' = return 11+ c 'C' = return 12+ c 'D' = return 13+ c 'E' = return 14+ c 'F' = return 15+ c 'a' = return 10+ c 'b' = return 11+ c 'c' = return 12+ c 'd' = return 13+ c 'e' = return 14+ c 'f' = return 15+ c _ = fail "Invalid hex digit!"
+ test/Test.hs view
@@ -0,0 +1,260 @@+{-# LANGUAGE OverloadedStrings #-}++module Main where+++import Data.Monoid+import Data.CSS.Syntax.Tokens+import Test.Hspec+import Prelude++++main :: IO ()+main = hspec spec++spec :: Spec+spec = parallel $ do++ describe "Data.CSS.Syntax.Tokens" $ do+ it "Single Character" $ do+ tokenize "(" `shouldBe` Right [LeftParen]+ tokenize ")" `shouldBe` Right [RightParen]+ tokenize "[" `shouldBe` Right [LeftSquareBracket]+ tokenize "]" `shouldBe` Right [RightSquareBracket]+ tokenize ",," `shouldBe` Right [Comma, Comma]++ it "Multiple Character" $ do+ tokenize "~=" `shouldBe` Right [IncludeMatch]+ tokenize "||" `shouldBe` Right [Column]+ tokenize "|||" `shouldBe` Right [Column, Delim '|']+ tokenize "<!--" `shouldBe` Right [CDO]+ tokenize "<!---" `shouldBe` Right [CDO, Delim '-']+ tokenize "<!---->" `shouldBe` Right [CDO, CDC]+ tokenize "<!-- -->" `shouldBe` Right [CDO, Whitespace, CDC]++ it "Delimiter" $ do+ tokenize "^" `shouldBe` Right [Delim '^']+ tokenize "|" `shouldBe` Right [Delim '|']+ tokenize "\x7f" `shouldBe` Right [Delim '\x7f']+ tokenize "\1" `shouldBe` Right [Delim '\x1']+ tokenize "~-" `shouldBe` Right [Delim '~', Delim '-']+ tokenize "*^" `shouldBe` Right [Delim '*', Delim '^']++ it "Whitespace" $ do+ tokenize " " `shouldBe` Right [Whitespace]+ tokenize "\n\rS" `shouldBe` Right [Whitespace, Ident "S"]+ tokenize " *" `shouldBe` Right [Whitespace, Delim '*']+ tokenize "\n\r\f2" `shouldBe` Right [Whitespace, Number "2" (NVInteger 2)]++ it "Escapes" $ do+ tokenize "hel\\6Co" `shouldBe` Right [Ident "hello"]+ tokenize "\\26 B" `shouldBe` Right [Ident "&B"]+ tokenize "'hel\\6c o'" `shouldBe` Right [String "hello"]+ tokenize "'spac\\65\r\ns'" `shouldBe` Right [String "spaces"]+ tokenize "spac\\65\r\ns" `shouldBe` Right [Ident "spaces"]+ tokenize "spac\\65\n\rs" `shouldBe` Right [Ident "space", Whitespace, Ident "s"]+ tokenize "sp\\61\tc\\65\fs" `shouldBe` Right [Ident "spaces"]+ tokenize "hel\\6c o" `shouldBe` Right [Ident "hell", Whitespace, Ident "o"]+ tokenize "test\\\n" `shouldBe` Right [Ident "test", Delim '\\', Whitespace]+ tokenize "test\\D799" `shouldBe` Right [Ident "test\xD799"]+ tokenize "\\E000" `shouldBe` Right [Ident "\xe000"]+ tokenize "te\\s\\t" `shouldBe` Right [Ident "test"]+ tokenize "spaces\\ in\\\tident" `shouldBe` Right [Ident "spaces in\tident"]+ tokenize "\\.\\,\\:\\!" `shouldBe` Right [Ident ".,:!"]+ tokenize "\\\r" `shouldBe` Right [Delim '\\', Whitespace]+ tokenize "\\\f" `shouldBe` Right [Delim '\\', Whitespace]+ tokenize "\\\r\n" `shouldBe` Right [Delim '\\', Whitespace]+ -- let replacement = "\xFFFD"+ tokenize "null\\\0" `shouldBe` Right [Ident "null\xfffd"]+ tokenize "null\\\0\0" `shouldBe` Right [Ident $ "null" <> "\xfffd" <> "\xfffd"]+ tokenize "null\\0" `shouldBe` Right [Ident $ "null" <> "\xfffd"]+ tokenize "null\\0000" `shouldBe` Right [Ident $ "null" <> "\xfffd"]+ tokenize "large\\110000" `shouldBe` Right [Ident $ "large" <> "\xfffd"]+ tokenize "large\\23456a" `shouldBe` Right [Ident $ "large" <> "\xfffd"]+ tokenize "surrogate\\D800" `shouldBe` Right [Ident $ "surrogate" <> "\xfffd"]+ tokenize "surrogate\\0DABC" `shouldBe` Right [Ident $ "surrogate" <> "\xfffd"]+ tokenize "\\00DFFFsurrogate" `shouldBe` Right [Ident $ "\xfffd" <> "surrogate"]+ tokenize "\\10fFfF" `shouldBe` Right [Ident "\x10ffff"]+ tokenize "\\10fFfF0" `shouldBe` Right [Ident $ "\x10ffff" <> "0"]+ tokenize "\\10000000" `shouldBe` Right [Ident $ "\x100000" <> "00"]+ tokenize "eof\\" `shouldBe` Right [Ident "eof\xfffd"]++ it "Ident" $ do+ tokenize "simple-ident" `shouldBe` Right [Ident "simple-ident"]+ tokenize "testing123" `shouldBe` Right [Ident "testing123"]+ tokenize "hello!" `shouldBe` Right [Ident "hello", Delim '!']+ tokenize "world\5" `shouldBe` Right [Ident "world", Delim '\5']+ tokenize "_under score" `shouldBe` Right [Ident "_under", Whitespace, Ident "score"]+ tokenize "-_underscore" `shouldBe` Right [Ident "-_underscore"]+ tokenize "-text" `shouldBe` Right [Ident "-text"]+ tokenize "-\\6d" `shouldBe` Right [Ident "-m"]+ tokenize "--abc" `shouldBe` Right [Ident "--abc"]+ tokenize "--" `shouldBe` Right [Ident "--"]+ tokenize "--11" `shouldBe` Right [Ident "--11"]+ tokenize "---" `shouldBe` Right [Ident "---"]+ tokenize "\x2003" `shouldBe` Right [Ident "\x2003"] -- em-space+ tokenize "\xA0" `shouldBe` Right [Ident "\xA0"] -- non-breaking space+ tokenize "\x1234" `shouldBe` Right [Ident "\x1234"]+ tokenize "\x12345" `shouldBe` Right [Ident "\x12345"]+ tokenize "\0" `shouldBe` Right [Ident "\xfffd"]+ tokenize "ab\0c" `shouldBe` Right [Ident $ "ab\xfffd" <> "c"]++ it "Function" $ do+ tokenize "scale(2)" `shouldBe` Right [Function "scale", Number "2" (NVInteger 2), RightParen]+ tokenize "foo-bar\\ baz(" `shouldBe` Right [Function "foo-bar baz"]+ tokenize "fun\\(ction(" `shouldBe` Right [Function "fun(ction"]+ tokenize "-foo(" `shouldBe` Right [Function "-foo"]+ tokenize "url(\"foo.gif\"" `shouldBe` Right [Function "url", String "foo.gif"]+ tokenize "foo( \'bar.gif\'" `shouldBe` Right [Function "foo", Whitespace, String "bar.gif"]+ -- // To simplify implementation we drop the whitespace in function(url),whitespace,string()+ tokenize "url( \'bar.gif\'" `shouldBe` Right [Function "url", String "bar.gif"]++ it "AtKeyword" $ do+ tokenize "@at-keyword" `shouldBe` Right [AtKeyword "at-keyword"]+ tokenize "@hello!" `shouldBe` Right [AtKeyword "hello", Delim '!']+ tokenize "@-text" `shouldBe` Right [AtKeyword "-text"]+ tokenize "@--abc" `shouldBe` Right [AtKeyword "--abc"]+ tokenize "@--" `shouldBe` Right [AtKeyword "--"]+ tokenize "@--11" `shouldBe` Right [AtKeyword "--11"]+ tokenize "@---" `shouldBe` Right [AtKeyword "---"]+ tokenize "@\\ " `shouldBe` Right [AtKeyword " "]+ tokenize "@-\\ " `shouldBe` Right [AtKeyword "- "]+ tokenize "@@" `shouldBe` Right [ Delim '@', Delim '@']+ -- tokenize "@2" `shouldBe` Right [ Delim '@', Number "2" (NVInteger 2)]+ -- tokenize "@-1" `shouldBe` Right [ Delim '@', Number "-1" (NVInteger (-1))]+++ it "Url" $ do+ tokenize "url(foo.gif)" `shouldBe` Right [Url "foo.gif"]+ tokenize "urL(https://example.com/cats.png)" `shouldBe` Right [Url "https://example.com/cats.png"]+ tokenize "uRl(what-a.crazy^URL~this\\ is!)" `shouldBe` Right [Url "what-a.crazy^URL~this is!"]+ tokenize "uRL(123#test)" `shouldBe` Right [Url "123#test"]+ tokenize "Url(escapes\\ \\\"\\'\\)\\()" `shouldBe` Right [Url "escapes \"')("]+ tokenize "UrL( whitespace )" `shouldBe` Right [Url "whitespace"]+ tokenize "URl( whitespace-eof " `shouldBe` Right [Url "whitespace-eof"]+ tokenize "URL(eof" `shouldBe` Right [Url "eof"]+ tokenize "url(not/*a*/comment)" `shouldBe` Right [Url "not/*a*/comment"]+ tokenize "urL()" `shouldBe` Right [Url ""]+ tokenize "uRl(white space)," `shouldBe` Right [BadUrl "white space", Comma]+ tokenize "Url(b(ad)," `shouldBe` Right [BadUrl "b(ad", Comma]+ tokenize "uRl(ba'd):" `shouldBe` Right [BadUrl "ba'd", Colon]+ tokenize "urL(b\"ad):" `shouldBe` Right [BadUrl "b\"ad", Colon]+ tokenize "uRl(b\"ad):" `shouldBe` Right [BadUrl "b\"ad", Colon]+ tokenize "Url(b\\\rad):" `shouldBe` Right [BadUrl "b\\\nad", Colon]+ tokenize "url(b\\\nad):" `shouldBe` Right [BadUrl "b\\\nad", Colon]+ tokenize "url(/*'bad')*/" `shouldBe` Right [BadUrl "/*'bad'", Delim '*', Delim '/']+ tokenize "url(ba'd\\\\))" `shouldBe` Right [BadUrl "ba'd\\", RightParen]++ it "String" $ do+ tokenize "'text'" `shouldBe` Right [String "text"]+ tokenize "\"text\"" `shouldBe` Right [String "text"]+ tokenize "'testing, 123!'" `shouldBe` Right [String "testing, 123!"]+ tokenize "'es\\'ca\\\"pe'" `shouldBe` Right [String "es'ca\"pe"]+ tokenize "'\"quotes\"'" `shouldBe` Right [String "\"quotes\""]+ tokenize "\"'quotes'\"" `shouldBe` Right [String "'quotes'"]+ tokenize "\"mismatch'" `shouldBe` Right [String "mismatch'"]+ tokenize "'text\5\t\xb'" `shouldBe` Right [String "text\5\t\xb"]+ tokenize "\"end on eof" `shouldBe` Right [String "end on eof"]+ tokenize "'esca\\\nped'" `shouldBe` Right [String "escaped"]+ tokenize "\"esc\\\faped\"" `shouldBe` Right [String "escaped"]+ tokenize "'new\\\rline'" `shouldBe` Right [String "newline"]+ tokenize "\"new\\\r\nline\"" `shouldBe` Right [String "newline"]+ tokenize "'bad\nstring" `shouldBe` Right [BadString "bad", Whitespace, Ident "string"]+ tokenize "'bad\rstring" `shouldBe` Right [BadString "bad", Whitespace, Ident "string"]+ tokenize "'bad\r\nstring" `shouldBe` Right [BadString "bad", Whitespace, Ident "string"]+ tokenize "'bad\fstring" `shouldBe` Right [BadString "bad", Whitespace, Ident "string"]+ tokenize "'\0'" `shouldBe` Right [String "\xFFFD"]+ tokenize "'hel\0lo'" `shouldBe` Right [String "hel\xfffdlo"]+ tokenize "'h\\65l\0lo'" `shouldBe` Right [String "hel\xfffdlo"]++ it "Hash" $ do+ tokenize "#id-selector" `shouldBe` Right [Hash HId "id-selector"]+ tokenize "#FF7700" `shouldBe` Right [Hash HId "FF7700"]+ -- tokenize "#3377FF" `shouldBe` Right [Hash HUnrestricted "3377FF"]+ tokenize "#\\ " `shouldBe` Right [Hash HId " "]+ tokenize "# " `shouldBe` Right [Delim '#', Whitespace]+ tokenize "#\\\n" `shouldBe` Right [Delim '#', Delim '\\', Whitespace]+ tokenize "#\\\r\n" `shouldBe` Right [Delim '#', Delim '\\', Whitespace]+ tokenize "#!" `shouldBe` Right [Delim '#', Delim '!']++ it "Number" $ do+ tokenize "10" `shouldBe` Right [Number "10" (NVInteger 10)]+ tokenize "12.0" `shouldBe` Right [Number "12.0" (NVNumber 12)]+ tokenize "+45.6" `shouldBe` Right [Number "+45.6" (NVNumber 45.6)]+ tokenize "-7" `shouldBe` Right [Number "-7" (NVInteger (-7))]+ tokenize "010" `shouldBe` Right [Number "010" (NVInteger 10)]+ tokenize "10e0" `shouldBe` Right [Number "10e0" (NVNumber 10)]+ tokenize "12e3" `shouldBe` Right [Number "12e3" (NVNumber 12000)]+ tokenize "3e+1" `shouldBe` Right [Number "3e+1" (NVNumber 30)]+ tokenize "12E-1" `shouldBe` Right [Number "12E-1" (NVNumber 1.2)]+ tokenize ".7" `shouldBe` Right [Number ".7" (NVNumber 0.7)]+ tokenize "-.3" `shouldBe` Right [Number "-.3" (NVNumber (-0.3))]+ tokenize "+637.54e-2" `shouldBe` Right [Number "+637.54e-2" (NVNumber 6.3754)]+ tokenize "-12.34E+2" `shouldBe` Right [Number "-12.34E+2" (NVNumber (-1234))]+ tokenize "+ 5" `shouldBe` Right [Delim '+', Whitespace, Number "5" (NVInteger 5)]+ tokenize "-+12" `shouldBe` Right [Delim '-', Number "+12" (NVInteger 12)]+ tokenize "+-21" `shouldBe` Right [Delim '+', Number "-21" (NVInteger (-21))]+ tokenize "++22" `shouldBe` Right [Delim '+', Number "+22" (NVInteger 22)]+ tokenize "13." `shouldBe` Right [Number "13" (NVInteger 13), Delim ('.')]+ tokenize "1.e2" `shouldBe` Right [Number "1" (NVInteger 1), Delim '.', Ident "e2"]+ tokenize "2e3.5" `shouldBe` Right [Number "2e3" (NVNumber 2e3), Number ".5" (NVNumber 0.5)]+ tokenize "2e3." `shouldBe` Right [Number "2e3" (NVNumber 2e3), Delim '.']+ tokenize "1000000000000000000000000" `shouldBe` Right [Number "1000000000000000000000000" (NVInteger 1e24)]++ it "Dimension" $ do+ tokenize "10px" `shouldBe` Right [Dimension "10" (NVInteger 10) "px"]+ tokenize "12.0em" `shouldBe` Right [Dimension "12.0" (NVNumber 12) "em"]+ tokenize "-12.0em" `shouldBe` Right [Dimension "-12.0" (NVNumber (-12)) "em"]+ tokenize "+45.6__qem" `shouldBe` Right [Dimension "+45.6" (NVNumber 45.6) "__qem"]+ tokenize "5e" `shouldBe` Right [Dimension "5" (NVInteger 5) "e"]+ tokenize "5px-2px" `shouldBe` Right [Dimension "5" (NVInteger 5) "px-2px"]+ tokenize "5e-" `shouldBe` Right [Dimension "5" (NVInteger 5) "e-"]+ tokenize "5\\ " `shouldBe` Right [Dimension "5" (NVInteger 5) " "]+ tokenize "40\\70\\78" `shouldBe` Right [Dimension "40" (NVInteger 40) "px"]+ tokenize "4e3e2" `shouldBe` Right [Dimension "4e3" (NVNumber 4e3) "e2"]+ tokenize "0x10px" `shouldBe` Right [Dimension "0" (NVInteger 0) "x10px"]+ tokenize "4unit " `shouldBe` Right [Dimension "4" (NVInteger 4) "unit", Whitespace]+ tokenize "5e+" `shouldBe` Right [Dimension "5" (NVInteger 5) "e", Delim '+']+ tokenize "2e.5" `shouldBe` Right [Dimension "2" (NVInteger 2) "e", Number ".5" (NVNumber 0.5)]+ tokenize "2e+.5" `shouldBe` Right [Dimension "2" (NVInteger 2) "e", Number "+.5" (NVNumber 0.5)]++ it "Percentage" $ do+ tokenize "10%" `shouldBe` Right [Percentage "10" $ NVInteger 10]+ tokenize "+12.0%" `shouldBe` Right [Percentage "+12.0" $ NVNumber 12.0]+ tokenize "-48.99%" `shouldBe` Right [Percentage "-48.99" $ NVNumber (-48.99)]+ tokenize "6e-1%" `shouldBe` Right [Percentage "6e-1" $ NVNumber 6e-1]+ tokenize "5%%" `shouldBe` Right [Percentage "5" (NVInteger 5), Delim '%']+++-- 402 TEST(CSSTokenizerTest, UnicodeRangeToken)+-- 403 {+-- 404 TEST_TOKENS("u+012345-123456", unicodeRange(0x012345, 0x123456));+-- 405 TEST_TOKENS("U+1234-2345", unicodeRange(0x1234, 0x2345));+-- 406 TEST_TOKENS("u+222-111", unicodeRange(0x222, 0x111));+-- 407 TEST_TOKENS("U+CafE-d00D", unicodeRange(0xcafe, 0xd00d));+-- 408 TEST_TOKENS("U+2??", unicodeRange(0x200, 0x2ff));+-- 409 TEST_TOKENS("U+ab12??", unicodeRange(0xab1200, 0xab12ff));+-- 410 TEST_TOKENS("u+??????", unicodeRange(0x000000, 0xffffff));+-- 411 TEST_TOKENS("u+??", unicodeRange(0x00, 0xff));+-- 412+-- 413 TEST_TOKENS("u+222+111", unicodeRange(0x222, 0x222), number(IntegerValueType, 111, PlusSign));+-- 414 TEST_TOKENS("u+12345678", unicodeRange(0x123456, 0x123456), number(IntegerValueType, 78, NoSign));+-- 415 TEST_TOKENS("u+123-12345678", unicodeRange(0x123, 0x123456), number(IntegerValueType, 78, NoSign));+-- 416 TEST_TOKENS("u+cake", unicodeRange(0xca, 0xca), ident("ke"));+-- 417 TEST_TOKENS("u+1234-gggg", unicodeRange(0x1234, 0x1234), ident("-gggg"));+-- 418 TEST_TOKENS("U+ab12???", unicodeRange(0xab1200, 0xab12ff), delim('?'));+-- 419 TEST_TOKENS("u+a1?-123", unicodeRange(0xa10, 0xa1f), number(IntegerValueType, -123, MinusSign));+-- 420 TEST_TOKENS("u+1??4", unicodeRange(0x100, 0x1ff), number(IntegerValueType, 4, NoSign));+-- 421 TEST_TOKENS("u+z", ident("u"), delim('+'), ident("z"));+-- 422 TEST_TOKENS("u+", ident("u"), delim('+'));+-- 423 TEST_TOKENS("u+-543", ident("u"), delim('+'), number(IntegerValueType, -543, MinusSign));++ it "Comment" $ do+ tokenize "/*comment*/a" `shouldBe` Right [Ident "a"]+ tokenize "/**\\2f**//" `shouldBe` Right [Delim '/']+ tokenize "/**y*a*y**/ " `shouldBe` Right [Whitespace]+ tokenize ",/* \n :) \n */)" `shouldBe` Right [Comma, RightParen]+ tokenize ":/*/*/" `shouldBe` Right [Colon]+ tokenize "/**/*" `shouldBe` Right [Delim '*']+ tokenize ";/******" `shouldBe` Right [Semicolon]