diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,47 @@ # Revision history for skylighting and skylighting-core +## 0.9 + + * Use a pure Haskell regex implementation (in unexported module + Text.Regex.KDE) instead of pcre. The implementation is not + as efficient as pcre, but it seems good enough for this + application, and it is desirable to avoid depending on a C + library. (Available Haskell libraries weren't up to the + task, because they don't do back-references, captures, + lookahead/behind.) Some benchmarks (old/new): + haskell (4.6/7.9) java (13.4/23.3) c (2.8/3.7) rhtml + (4.7/6.1) lua (10.6/13.2) javascript (4.2/6.6). + Though this is a significant slowdown, the tradeoff seems + worth it to have a pure Haskell implementation. + + * Removed old `system-pcre` flag. + + * More efficient treatment of dynamic regexes. + We put something in the Regex itself to represent the `%1`, + and modify it later. This allows us to cache dynamic + regexes in a way we couldn't before. + + * Add support for TOML (#105, Shiming Wang), + GraphQL, and Nim syntax (#102, Daniel Pozo Escalona). + + * Update xml definitions for actionscript, bash, boo, c, + cmake, elm, erlang, glsl, isocpp, java, lua, m4, mediawiki, + perl, powershell, scala, tcsh, xul, zsh. + + * Fix fallthrough behavior (don't always consume a token). + + * Fix word boundary detection. + + * Remove RegexException. (API change) + + * Skylighting.Regex now exports `isWordChar` and `testRegex`, + as well as the constructors underlying the new `Regex` type. + + * Remove some obsolete xml definition patches. + + * Fix escaped % in dynamic regex. + + ## 0.8.5 * Respect dynamic flag on StringDetect elements (#99, Albert diff --git a/skylighting-core.cabal b/skylighting-core.cabal --- a/skylighting-core.cabal +++ b/skylighting-core.cabal @@ -1,5 +1,5 @@ name: skylighting-core -version: 0.8.5 +version: 0.9 synopsis: syntax highlighting library description: Skylighting is a syntax highlighting library. It derives its tokenizers from XML syntax @@ -109,6 +109,10 @@ Skylighting.Format.ANSI Skylighting.Format.HTML Skylighting.Format.LaTeX + other-modules: Regex.KDE + Regex.KDE.Regex + Regex.KDE.Compile + Regex.KDE.Match other-extensions: CPP, Arrows build-depends: base >= 4.8 && < 5.0, mtl, @@ -126,26 +130,21 @@ safe, base64-bytestring, blaze-html >= 0.5, - containers, + containers >= 0.5.8.2, + utf8-string, ansi-terminal >= 0.7, colour >= 2.0 - if flag(system-pcre) - build-depends: regex-pcre - else - build-depends: regex-pcre-builtin >= 0.95 hs-source-dirs: src ghc-prof-options: -fprof-auto-exported default-language: Haskell2010 ghc-options: -Wall + if impl(ghc >= 8.4) + ghc-options: -fhide-source-paths Flag executable Description: Build skylighting-extract tool Default: False -Flag system-pcre - Description: Use regex-pcre instead of regex-pcre-builtin - Default: False - test-suite test-skylighting type: exitcode-stdio-1.0 main-is: test-skylighting.hs @@ -161,6 +160,7 @@ random, Diff, text, + utf8-string, pretty-show, aeson >= 1.0, bytestring, @@ -182,6 +182,8 @@ directory, criterion >= 1.0 && < 1.6 Ghc-Options: -rtsopts -Wall -fno-warn-unused-do-bind + if impl(ghc >= 8.4) + ghc-options: -fhide-source-paths Default-Language: Haskell2010 executable skylighting-extract @@ -201,10 +203,6 @@ directory, ansi-terminal >= 0.7, colour >= 2.0 - if flag(system-pcre) - build-depends: regex-pcre - else - build-depends: regex-pcre-builtin if flag(executable) buildable: True else @@ -214,3 +212,5 @@ default-language: Haskell2010 other-extensions: CPP ghc-options: -Wall + if impl(ghc >= 8.4) + ghc-options: -fhide-source-paths diff --git a/src/Regex/KDE.hs b/src/Regex/KDE.hs new file mode 100644 --- /dev/null +++ b/src/Regex/KDE.hs @@ -0,0 +1,24 @@ +{-# LANGUAGE OverloadedStrings #-} +module Regex.KDE + (Regex(..), compileRegex, matchRegex, testRegex, isWordChar) + where + +import Regex.KDE.Regex +import Regex.KDE.Compile +import Regex.KDE.Match +import qualified Data.ByteString.UTF8 as U +import qualified Data.IntMap.Strict as M +import qualified Data.ByteString as B +import Data.List (sortOn) + +testRegex :: Bool -> String -> String -> Maybe (String, [(Int, String)]) +testRegex caseSensitive re s = + let bs = U.fromString s + toSlice (off,len) = U.toString $ B.take len $ B.drop off bs + in case compileRegex caseSensitive (U.fromString re) of + Right r -> + case matchRegex r bs of + Nothing -> Nothing + Just (m,cs) -> Just (U.toString m, sortOn fst + (M.toList (M.map toSlice cs))) + Left e -> error e diff --git a/src/Regex/KDE/Compile.hs b/src/Regex/KDE/Compile.hs new file mode 100644 --- /dev/null +++ b/src/Regex/KDE/Compile.hs @@ -0,0 +1,250 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} +module Regex.KDE.Compile + (compileRegex) + where + +import Data.Word (Word8) +import qualified Data.ByteString as B +import Data.ByteString (ByteString) +import qualified Data.ByteString.UTF8 as U +import Safe +import Data.Attoparsec.ByteString as A hiding (match) +import Data.Char +import Control.Applicative +import Regex.KDE.Regex +import Control.Monad.State.Strict +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>)) +#endif + +-- | Compile a UTF-8 encoded ByteString as a Regex. If the first +-- parameter is True, then the Regex will be case sensitive. +compileRegex :: Bool -> ByteString -> Either String Regex +compileRegex caseSensitive bs = + let !res = parseOnly (evalStateT parser 0) bs + in res + where + parser = do + !re <- pRegex caseSensitive + (re <$ lift A.endOfInput) <|> + do rest <- lift A.takeByteString + fail $ "parse error at byte position " ++ + show (B.length bs - B.length rest) + +type RParser = StateT Int Parser + +pRegex :: Bool -> RParser Regex +pRegex caseSensitive = + foldr MatchAlt + <$> (pAltPart caseSensitive) + <*> (many $ lift (char '|') *> (pAltPart caseSensitive <|> pure mempty)) + +pAltPart :: Bool -> RParser Regex +pAltPart caseSensitive = mconcat <$> many1 (pRegexPart caseSensitive) + +char :: Char -> Parser Char +char c = + c <$ satisfy (== fromIntegral (ord c)) + +pRegexPart :: Bool -> RParser Regex +pRegexPart caseSensitive = + (lift (pRegexChar caseSensitive) <|> pParenthesized caseSensitive) >>= + lift . pSuffix + +pParenthesized :: Bool -> RParser Regex +pParenthesized caseSensitive = do + _ <- lift (satisfy (== 40)) + modifier <- lift (satisfy (== 63) *> pGroupModifiers) + <|> (MatchCapture <$> (modify (+ 1) *> get)) + contents <- pRegex caseSensitive + _ <- lift (satisfy (== 41)) + return $ modifier contents + +pGroupModifiers :: Parser (Regex -> Regex) +pGroupModifiers = + (id <$ char ':') + <|> + do dir <- option Forward $ Backward <$ char '<' + (AssertPositive dir <$ char '=') <|> (AssertNegative dir <$ char '!') + +pSuffix :: Regex -> Parser Regex +pSuffix re = option re $ do + w <- satisfy (\x -> x == 42 || x == 43 || x == 63 || x == 123) + (case w of + 42 -> return $ MatchAlt (MatchSome re) MatchNull + 43 -> return $ MatchSome re + 63 -> return $ MatchAlt re MatchNull + 123 -> do + let isDig x = x >= 48 && x < 58 + minn <- option Nothing $ readMay . U.toString <$> A.takeWhile isDig + maxn <- option Nothing $ char ',' *> + (readMay . U.toString <$> A.takeWhile isDig) + _ <- char '}' + return $! + case (minn, maxn) of + (Nothing, Nothing) -> atleast 0 re + (Just n, Nothing) -> atleast n re + (Nothing, Just n) -> atmost n re + (Just m, Just n) -> between m n re + _ -> fail "pSuffix encountered impossible byte") >>= pSuffix + where + atmost 0 _ = MatchNull + atmost n r = MatchAlt (mconcat (replicate n r)) (atmost (n-1) r) + + between 0 n r = atmost n r + between m n r = mconcat (replicate m r) <> atmost (n - m) r + + atleast n r = mconcat (replicate n r) <> MatchAlt (MatchSome r) MatchNull + +pRegexChar :: Bool -> Parser Regex +pRegexChar caseSensitive = do + w <- satisfy $ const True + case w of + 46 -> return MatchAnyChar + 37 -> (do -- dynamic %1 %2 + ds <- A.takeWhile1 (\x -> x >= 48 && x <= 57) + case readMay (U.toString ds) of + Just !n -> return $ MatchDynamic n + Nothing -> fail "not a number") + <|> return (MatchChar (== '%')) + 92 -> pRegexEscapedChar + 36 -> return AssertEnd + 94 -> return AssertBeginning + 91 -> pRegexCharClass + _ | w < 128 + , not (isSpecial w) + -> do let c = chr $ fromIntegral w + return $! MatchChar $ + if caseSensitive + then (== c) + else (\d -> toLower d == toLower c) + | w >= 0xc0 -> do + rest <- case w of + _ | w >= 0xf0 -> A.take 3 + | w >= 0xe0 -> A.take 2 + | otherwise -> A.take 1 + case U.uncons (B.cons w rest) of + Just (d, _) -> return $! MatchChar $ + if caseSensitive + then (== d) + else (\e -> toLower e == toLower d) + Nothing -> fail "could not decode as UTF8" + | otherwise -> mzero + +pRegexEscapedChar :: Parser Regex +pRegexEscapedChar = do + c <- anyChar + (case c of + 'b' -> return AssertWordBoundary + '{' -> do -- captured pattern: \1 \2 \{12} + ds <- A.takeWhile1 (\x -> x >= 48 && x <= 57) + _ <- char '}' + case readMay (U.toString ds) of + Just !n -> return $ MatchCaptured $ n + Nothing -> fail "not a number" + 'd' -> return $ MatchChar isDigit + 'D' -> return $ MatchChar (not . isDigit) + 's' -> return $ MatchChar isSpace + 'S' -> return $ MatchChar (not . isSpace) + 'w' -> return $ MatchChar isWordChar + 'W' -> return $ MatchChar (not . isWordChar) + _ | c >= '0' && c <= '9' -> + return $! MatchCaptured (ord c - ord '0') + | otherwise -> mzero) <|> (MatchChar . (==) <$> pEscaped c) + +pEscaped :: Char -> Parser Char +pEscaped c = + case c of + '\\' -> return c + 'a' -> return '\a' + 'f' -> return '\f' + 'n' -> return '\n' + 'r' -> return '\r' + 't' -> return '\t' + 'v' -> return '\v' + '0' -> do -- \0ooo matches octal ooo + ds <- A.take 3 + case readMay ("'\\o" ++ U.toString ds ++ "'") of + Just x -> return x + Nothing -> fail "invalid octal character escape" + 'z' -> do -- \zhhhh matches unicode hex char hhhh + ds <- A.take 4 + case readMay ("'\\x" ++ U.toString ds ++ "'") of + Just x -> return x + Nothing -> fail "invalid hex character escape" + _ | c >= '1' && c <= '7' -> do -- \ooo octal undocument form but works + ds <- A.take 2 + case readMay ("'\\o" ++ c : U.toString ds ++ "'") of + Just x -> return x + Nothing -> fail "invalid octal character escape" + | otherwise -> return c + +pRegexCharClass :: Parser Regex +pRegexCharClass = do + negated <- option False $ True <$ satisfy (== 94) -- '^' + let getEscapedClass = do + _ <- satisfy (== 92) -- backslash + (isDigit <$ char 'd') + <|> (not . isDigit <$ char 'D') + <|> (isSpace <$ char 's') + <|> (not . isSpace <$ char 'S') + <|> (isWordChar <$ char 'w') + <|> (not . isWordChar <$ char 'W') + let getPosixClass = do + _ <- string "[:" + localNegated <- option False $ True <$ satisfy (== 94) -- '^' + res <- (isAlphaNum <$ string "alnum") + <|> (isAlpha <$ string "alpha") + <|> (isAscii <$ string "ascii") + <|> ((\c -> isSpace c && c `notElem` ['\n','\r','\f','\v']) <$ + string "blank") + <|> (isControl <$ string "cntrl") + <|> ((\c -> isPrint c || isSpace c) <$ string "graph:") + <|> (isLower <$ string "lower") + <|> (isUpper <$ string "upper") + <|> (isPrint <$ string "print") + <|> (isPunctuation <$ string "punct") + <|> (isSpace <$ string "space") + <|> ((\c -> isAlphaNum c || + generalCategory c == ConnectorPunctuation) + <$ string "word:") + <|> (isHexDigit <$ string "xdigit") + _ <- string ":]" + return $! if localNegated then not . res else res + let getC = (satisfy (== 92) *> anyChar >>= pEscaped) <|> + (chr . fromIntegral <$> satisfy (\x -> x /= 92 && x /= 93)) -- \ ] + let getCRange = do + c <- getC + (\d -> (\x -> x >= c && x <= d)) <$> (char '-' *> getC) <|> + return (== c) + brack <- option [] $ [(==']')] <$ char ']' + fs <- many (getEscapedClass <|> getPosixClass <|> getCRange) + _ <- satisfy (== 93) -- ] + let f c = any ($ c) $ brack ++ fs + return $! MatchChar (if negated then (not . f) else f) + +anyChar :: Parser Char +anyChar = do + w <- satisfy (const True) + return $! chr $ fromIntegral w + +isSpecial :: Word8 -> Bool +isSpecial 92 = True -- '\\' +isSpecial 63 = True -- '?' +isSpecial 42 = True -- '*' +isSpecial 43 = True -- '+' +isSpecial 123 = True -- '{' +isSpecial 125 = True -- '}' +isSpecial 91 = True -- '[' +isSpecial 93 = True -- ']' +isSpecial 37 = True -- '%' +isSpecial 40 = True -- '(' +isSpecial 41 = True -- ')' +isSpecial 124 = True -- '|' +isSpecial 46 = True -- '.' +isSpecial 36 = True -- '$' +isSpecial 94 = True -- '^' +isSpecial _ = False + diff --git a/src/Regex/KDE/Match.hs b/src/Regex/KDE/Match.hs new file mode 100644 --- /dev/null +++ b/src/Regex/KDE/Match.hs @@ -0,0 +1,160 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE BinaryLiterals #-} +module Regex.KDE.Match + ( matchRegex + ) where + +import qualified Data.ByteString as B +import Data.ByteString (ByteString) +import qualified Data.ByteString.UTF8 as U +import qualified Data.Set as Set +import Data.Set (Set) +import Regex.KDE.Regex +import qualified Data.IntMap.Strict as M +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>)) +#endif + +-- Note that all matches are from the beginning of the string. +-- The ^ anchor is implicit at the beginning of the regex. + +data Match = + Match { matchBytes :: !ByteString + , matchOffset :: !Int + , matchCaptures :: !(M.IntMap (Int, Int)) + -- starting offset, length in bytes + } deriving (Show, Eq) + +-- longer matches are <= +instance Ord Match where + m1 <= m2 = matchOffset m1 >= matchOffset m2 && + matchCaptures m1 >= matchCaptures m2 + +mapMatching :: (Match -> Match) -> Set Match -> Set Match +mapMatching f = Set.filter ((>= 0) . matchOffset) . Set.map f + +-- we take the n longest matches to avoid pathological slowdown +sizeLimit :: Int +sizeLimit = 2000 + +-- prune matches if it gets out of hand +prune :: Set Match -> Set Match +prune ms = if Set.size ms > sizeLimit + then Set.take sizeLimit ms + else ms + +exec :: Direction -> Regex -> Set Match -> Set Match +exec _ MatchNull = id +exec dir (MatchDynamic n) = -- if this hasn't been replaced, match literal + exec dir (MatchChar (== '%') <> + mconcat (map (\c -> MatchChar (== c)) (show n))) +exec _ AssertEnd = Set.filter (\m -> matchOffset m == B.length (matchBytes m)) +exec _ AssertBeginning = Set.filter (\m -> matchOffset m == 0) +exec _ (AssertPositive dir regex) = + Set.filter (\m -> not (null (exec dir regex (Set.singleton m)))) +exec _ (AssertNegative dir regex) = + Set.filter (\m -> null (exec dir regex (Set.singleton m))) +exec _ AssertWordBoundary = Set.filter atWordBoundary +exec Forward MatchAnyChar = mapMatching $ \m -> + case U.decode (B.drop (matchOffset m) (matchBytes m)) of + Nothing -> m{ matchOffset = - 1} + Just (_,n) -> m{ matchOffset = matchOffset m + n } +exec Backward MatchAnyChar = mapMatching $ \m -> + case lastCharOffset (matchBytes m) (matchOffset m) of + Nothing -> m{ matchOffset = -1 } + Just off -> m{ matchOffset = off } +exec Forward (MatchChar f) = mapMatching $ \m -> + case U.decode (B.drop (matchOffset m) (matchBytes m)) of + Just (c,n) | f c -> m{ matchOffset = matchOffset m + n } + _ -> m{ matchOffset = -1 } +exec Backward (MatchChar f) = mapMatching $ \m -> + case lastCharOffset (matchBytes m) (matchOffset m) of + Nothing -> m{ matchOffset = -1 } + Just off -> + case U.decode (B.drop off (matchBytes m)) of + Just (c,_) | f c -> m{ matchOffset = off } + _ -> m{ matchOffset = -1 } +exec dir (MatchConcat (MatchConcat r1 r2) r3) = + exec dir (MatchConcat r1 (MatchConcat r2 r3)) +exec Forward (MatchConcat r1 r2) = -- TODO longest match first + exec Forward r2 . prune . exec Forward r1 +exec Backward (MatchConcat r1 r2) = exec Backward r1 . exec Backward r2 +exec dir (MatchAlt r1 r2) = \ms -> exec dir r1 ms <> exec dir r2 ms +exec dir (MatchSome re) = go + where + go ms = case exec dir re ms of + ms' | Set.null ms' -> Set.empty + | ms' == ms -> ms + | otherwise -> let ms'' = prune ms' + in ms'' <> go ms'' +exec dir (MatchCapture i re) = + Set.foldr Set.union Set.empty . + Set.map (\m -> + Set.map (captureDifference m) (exec dir re (Set.singleton m))) + where + captureDifference m m' = + let len = matchOffset m' - matchOffset m + in m'{ matchCaptures = M.insert i (matchOffset m, len) + (matchCaptures m') } +exec dir (MatchCaptured n) = mapMatching matchCaptured + where + matchCaptured m = + case M.lookup n (matchCaptures m) of + Just (offset, len) -> + let capture = B.take len $ B.drop offset $ matchBytes m + in case dir of + Forward | B.isPrefixOf capture + (B.drop (matchOffset m) (matchBytes m)) + -> m{ matchOffset = matchOffset m + B.length capture } + Backward | B.isSuffixOf capture + (B.take (matchOffset m) (matchBytes m)) + -> m{ matchOffset = matchOffset m - B.length capture } + _ -> m{ matchOffset = -1 } + Nothing -> m{ matchOffset = -1 } + + +atWordBoundary :: Match -> Bool +atWordBoundary m = + case matchOffset m of + 0 -> True + n | n == B.length (matchBytes m) -> True + | otherwise -> + case lastCharOffset (matchBytes m) (matchOffset m) of + Nothing -> True + Just off -> + case U.toString (B.drop (off - 1) (matchBytes m)) of + (prev:cur:next:_) -> + (isWordChar cur /= isWordChar next) || + (isWordChar cur /= isWordChar prev) + _ -> True + +lastCharOffset :: ByteString -> Int -> Maybe Int +lastCharOffset _ 0 = Nothing +lastCharOffset _ 1 = Nothing +lastCharOffset bs n = + case B.index bs (n - 2) of + w | w < 0b10000000 -> Just (n - 1) + | w >= 0b11000000 -> Just (n - 1) + | otherwise -> lastCharOffset bs (n - 1) + +-- | Match a Regex against a (presumed UTF-8 encoded) ByteString, +-- returning the matched text and a map of (offset, size) +-- pairs for captures. Note that all matches are from the +-- beginning of the string (a @^@ anchor is implicit). Note +-- also that to avoid pathological performance in certain cases, +-- the matcher is limited to considering 2000 possible matches +-- at a time; when that threshold is reached, it discards +-- smaller matches. Hence certain regexes may incorrectly fail to +-- match: e.g. @a*a{3000}$@ on a string of 3000 @a@s. +matchRegex :: Regex + -> ByteString + -> Maybe (ByteString, M.IntMap (Int, Int)) +matchRegex re bs = + toResult <$> Set.lookupMin + (exec Forward re (Set.singleton (Match bs 0 M.empty))) + where + toResult m = (B.take (matchOffset m) (matchBytes m), (matchCaptures m)) + diff --git a/src/Regex/KDE/Regex.hs b/src/Regex/KDE/Regex.hs new file mode 100644 --- /dev/null +++ b/src/Regex/KDE/Regex.hs @@ -0,0 +1,62 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} +module Regex.KDE.Regex + ( Direction(..) + , Regex(..) + , isWordChar + ) where + +import Data.Char +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>), Semigroup) +#endif + +data Direction = Forward | Backward + deriving (Show, Eq) + +data Regex = + MatchAnyChar | + MatchDynamic !Int | + MatchChar (Char -> Bool) | + MatchSome !Regex | + MatchAlt !Regex !Regex | + MatchConcat !Regex !Regex | + MatchCapture !Int !Regex | + MatchCaptured !Int | + AssertWordBoundary | + AssertBeginning | + AssertEnd | + AssertPositive !Direction !Regex | + AssertNegative !Direction !Regex | + MatchNull + +instance Show Regex where + show MatchAnyChar = "MatchAnyChar" + show (MatchDynamic i) = "MatchDynamic " <> show i + show (MatchChar _) = "(MatchChar )" + show (MatchSome re) = "(MatchSome " <> show re <> ")" + show (MatchAlt r1 r2) = "(MatchAlt " <> show r1 <> " " <> show r2 <> ")" + show (MatchConcat r1 r2) = "(MatchConcat " <> show r1 <> " " <> show r2 <> + ")" + show (MatchCapture i re) = "(MatchCapture " <> show i <> " " <> + show re <> ")" + show (MatchCaptured n) = "(MatchCaptured " <> show n <> ")" + show AssertWordBoundary = "AssertWordBoundary" + show AssertBeginning = "AssertBeginning" + show AssertEnd = "AssertEnd" + show (AssertPositive dir re) = "(AssertPositive " <> show dir <> " " <> + show re <> ")" + show (AssertNegative dir re) = "(AssertNegativeLookahead " <> + show dir <> " " <> show re <> ")" + show MatchNull = "MatchNull" + +instance Semigroup Regex where + (<>) = MatchConcat + +instance Monoid Regex where + mempty = MatchNull + mappend = (<>) + +isWordChar :: Char -> Bool +isWordChar c = isAlphaNum c || generalCategory c == ConnectorPunctuation + diff --git a/src/Skylighting/Parser.hs b/src/Skylighting/Parser.hs --- a/src/Skylighting/Parser.hs +++ b/src/Skylighting/Parser.hs @@ -284,7 +284,7 @@ let column = if tildeRegex then Just (0 :: Int) else readMay column' - let re = RegExpr RE{ reString = fromString $ convertOctalEscapes str + let re = RegExpr RE{ reString = fromString str , reCaseSensitive = not insensitive } let (incsyntax, inccontext) = case break (=='#') context of @@ -293,7 +293,7 @@ let mbmatcher = case name of "DetectChar" -> Just $ DetectChar char0 "Detect2Chars" -> Just $ Detect2Chars char0 char1 - "AnyChar" -> Just $ AnyChar str + "AnyChar" -> Just $ AnyChar $ Set.fromList str "RangeDetect" -> Just $ RangeDetect char0 char1 "StringDetect" -> Just $ StringDetect $ Text.pack str "WordDetect" -> Just $ WordDetect $ Text.pack str diff --git a/src/Skylighting/Regex.hs b/src/Skylighting/Regex.hs --- a/src/Skylighting/Regex.hs +++ b/src/Skylighting/Regex.hs @@ -5,36 +5,26 @@ {-# LANGUAGE ScopedTypeVariables #-} module Skylighting.Regex ( - Regex - , RegexException + Regex(..) , RE(..) , compileRegex , matchRegex - , convertOctalEscapes + , testRegex + , isWordChar ) where -import qualified Control.Exception as E import Data.Aeson import Data.Binary (Binary) import qualified Data.ByteString.Base64 as Base64 import qualified Data.ByteString.Char8 as BS -import Data.ByteString.UTF8 (toString) import Data.Data import qualified Data.Text as Text import qualified Data.Text.Encoding as TE import GHC.Generics (Generic) -import System.IO.Unsafe (unsafePerformIO) -import Text.Printf -import Text.Regex.PCRE.ByteString #if !MIN_VERSION_base(4,13,0) import Control.Monad.Fail (MonadFail) #endif - --- | An exception in compiling or executing a regex. -newtype RegexException = RegexException String - deriving (Show, Typeable, Generic) - -instance E.Exception RegexException +import Regex.KDE -- | A representation of a regular expression. data RE = RE{ @@ -51,56 +41,6 @@ parseJSON = withObject "RE" $ \v -> RE <$> ((v .: "reString") >>= decodeFromText) <*> v .: "reCaseSensitive" - --- | Compile a PCRE regex. If the first parameter is True, the regex is --- case-sensitive, otherwise caseless. The regex is compiled from --- a bytestring interpreted as UTF-8. If the regex cannot be compiled, --- a 'RegexException' is thrown. -compileRegex :: Bool -> BS.ByteString -> Regex -compileRegex caseSensitive regexpStr = - let opts = compAnchored + compUTF8 + - if caseSensitive then 0 else compCaseless - in case unsafePerformIO $ compile opts (execNotEmpty) regexpStr of - Left (off,msg) -> E.throw $ RegexException $ - "Error compiling regex /" ++ toString regexpStr ++ - "/ at offset " ++ show off ++ "\n" ++ msg - Right r -> r - --- | Convert octal escapes to the form pcre wants. Note: --- need at least pcre 8.34 for the form \o{dddd}. --- So we prefer \ddd or \x{...}. -convertOctalEscapes :: String -> String -convertOctalEscapes [] = "" -convertOctalEscapes ('\\':'0':x:y:z:rest) - | all isOctalDigit [x,y,z] = '\\':x:y:z: convertOctalEscapes rest -convertOctalEscapes ('\\':x:y:z:rest) - | all isOctalDigit [x,y,z] ='\\':x:y:z: convertOctalEscapes rest -convertOctalEscapes ('\\':'o':'{':zs) = - case break (=='}') zs of - (ds, '}':rest) | all isOctalDigit ds && not (null ds) -> - case reads ('0':'o':ds) of - ((n :: Int,[]):_) -> - printf "\\x{%x}" n ++ convertOctalEscapes rest - _ -> E.throw $ RegexException $ - "Unable to read octal number: " ++ ds - _ -> '\\':'o':'{': convertOctalEscapes zs -convertOctalEscapes (x:xs) = x : convertOctalEscapes xs - -isOctalDigit :: Char -> Bool -isOctalDigit c = c >= '0' && c <= '7' - --- | Match a 'Regex' against a bytestring. Returns 'Nothing' if --- no match, otherwise 'Just' a nonempty list of bytestrings. The first --- bytestring in the list is the match, the others the captures, if any. --- If there are errors in executing the regex, a 'RegexException' is --- thrown. -matchRegex :: Regex -> BS.ByteString -> Maybe [BS.ByteString] -matchRegex r s = case unsafePerformIO (regexec r s) of - Right (Just (_, mat, _ , capts)) -> - Just (mat : capts) - Right Nothing -> Nothing - -- treat match error as no match, like Kate: #81 - Left (_rc, _msg) -> Nothing -- functions to marshall bytestrings to text diff --git a/src/Skylighting/Tokenizer.hs b/src/Skylighting/Tokenizer.hs --- a/src/Skylighting/Tokenizer.hs +++ b/src/Skylighting/Tokenizer.hs @@ -20,8 +20,9 @@ import qualified Data.ByteString.Char8 as BS import qualified Data.ByteString.UTF8 as UTF8 import Data.CaseInsensitive (mk) -import Data.Char (isAlphaNum, isAscii, isDigit, isLetter, isPrint, isSpace, ord) +import Data.Char (isAlphaNum, isAscii, isDigit, isLetter, isSpace, ord) import qualified Data.Map as Map +import qualified Data.IntMap as IntMap import Data.Maybe (catMaybes) import qualified Data.Set as Set import Data.Text (Text) @@ -30,7 +31,6 @@ import Debug.Trace import Skylighting.Regex import Skylighting.Types -import Text.Printf (printf) #if !MIN_VERSION_base(4,11,0) import Data.Semigroup #endif @@ -43,7 +43,7 @@ , endline :: Bool , prevChar :: Char , contextStack :: ContextStack - , captures :: [ByteString] + , captures :: IntMap.IntMap ByteString , column :: Int , lineContinuation :: Bool , firstNonspaceColumn :: Maybe Int @@ -211,7 +211,7 @@ , endline = True , prevChar = '\n' , contextStack = ContextStack [] - , captures = [] + , captures = mempty , column = 0 , lineContinuation = False , firstNonspaceColumn = Nothing @@ -250,18 +250,14 @@ gets endline >>= guard . not context <- currentContext msum (map (\r -> tryRule r inp) (cRules context)) <|> - if cFallthrough context - then do - let fallthroughContext = case cFallthroughContext context of - [] -> [Pop] - cs -> cs - doContextSwitches fallthroughContext - getToken - else do - t <- normalChunk - let mbtok = Just (cAttribute context, t) - info $ "FALLTHROUGH " ++ show mbtok - return mbtok + case cFallthroughContext context of + [] | cFallthrough context -> Nothing <$ doContextSwitches [Pop] + | otherwise -> do + t <- normalChunk + let mbtok = Just (cAttribute context, t) + info $ "FALLTHROUGH " ++ show mbtok + return mbtok + cs -> Nothing <$ doContextSwitches cs takeChars :: Int -> TokenizerM Text takeChars 0 = mzero @@ -512,39 +508,43 @@ takeChars 1 else mzero -anyChar :: [Char] -> ByteString -> TokenizerM Text +anyChar :: Set.Set Char -> ByteString -> TokenizerM Text anyChar cs inp = do case UTF8.uncons inp of - Just (x, _) | x `elem` cs -> takeChars 1 + Just (x, _) | x `Set.member` cs -> takeChars 1 _ -> mzero regExpr :: Bool -> RE -> ByteString -> TokenizerM Text regExpr dynamic re inp = do - reStr <- if dynamic - then do - reStr' <- subDynamic (reString re) - info $ "Dynamic regex: " ++ show reStr' - return reStr' - else return (reString re) + -- return $! traceShowId $! (reStr, inp) + let reStr = reString re when (BS.take 2 reStr == "\\b") $ wordBoundary inp - regex <- if dynamic - then return $ compileRegex (reCaseSensitive re) reStr - else do - compiledREs <- gets compiledRegexes - case Map.lookup re compiledREs of - Nothing -> do - let cre = compileRegex (reCaseSensitive re) reStr - modify $ \st -> st{ compiledRegexes = - Map.insert re cre (compiledRegexes st) } - return cre - Just cre -> return cre - case matchRegex regex inp of - Just (match:capts) -> do - unless (null capts) $ - modify $ \st -> st{ captures = capts } - takeChars (UTF8.length match) - _ -> mzero + compiledREs <- gets compiledRegexes + regex <- case Map.lookup re compiledREs of + Nothing -> do + cre <- case compileRegex (reCaseSensitive re) reStr of + Right r -> return r + Left e -> throwError $ + "Error compiling regex " ++ + UTF8.toString reStr ++ ": " ++ e + modify $ \st -> st{ compiledRegexes = + Map.insert re cre (compiledRegexes st) } + return cre + Just cre -> return cre + regex' <- if dynamic + then subDynamic regex + else return regex + case matchRegex regex' inp of + Just (matchedBytes, capts) -> do + unless (null capts) $ + modify $ \st -> st{ captures = + IntMap.map (toSlice matchedBytes) capts } + takeChars (UTF8.length matchedBytes) + _ -> mzero +toSlice :: ByteString -> (Int, Int) -> ByteString +toSlice bs (off, len) = BS.take len $ BS.drop off bs + wordBoundary :: ByteString -> TokenizerM () wordBoundary inp = do case UTF8.uncons inp of @@ -553,14 +553,8 @@ c <- gets prevChar guard $ isWordBoundary c d --- TODO is this right? isWordBoundary :: Char -> Char -> Bool -isWordBoundary c d = - (isAlphaNum c && not (isAlphaNum d)) - || (isAlphaNum d && not (isAlphaNum c)) - || (isSpace d && not (isSpace c)) - || (isSpace c && not (isSpace d)) - +isWordBoundary c d = isWordChar c /= isWordChar d decodeBS :: ByteString -> TokenizerM Text decodeBS bs = case decodeUtf8' bs of @@ -570,47 +564,30 @@ -- Substitute out %1, %2, etc. in regex string, escaping -- appropriately.. -subDynamic :: ByteString -> TokenizerM ByteString -subDynamic bs = - case BS.break (=='%') bs of - (y,z) - | BS.null z -> return y - | otherwise -> (y <>) <$> - case BS.unpack (BS.take 2 z) of - ['%',x] | x >= '0' && x <= '9' -> do - let capNum = ord x - ord '0' - replacement <- getCapture capNum - (escapeRegex (encodeUtf8 replacement) <>) <$> - subDynamic (BS.drop 2 z) - _ -> BS.cons '%' <$> (subDynamic (BS.drop 1 z)) - -escapeRegex :: BS.ByteString -> BS.ByteString -escapeRegex = BS.concatMap escapeRegexChar - -escapeRegexChar :: Char -> BS.ByteString -escapeRegexChar '^' = "\\^" -escapeRegexChar '$' = "\\$" -escapeRegexChar '\\' = "\\\\" -escapeRegexChar '[' = "\\[" -escapeRegexChar ']' = "\\]" -escapeRegexChar '(' = "\\(" -escapeRegexChar ')' = "\\)" -escapeRegexChar '{' = "\\{" -escapeRegexChar '}' = "\\}" -escapeRegexChar '*' = "\\*" -escapeRegexChar '+' = "\\+" -escapeRegexChar '.' = "\\." -escapeRegexChar '?' = "\\?" -escapeRegexChar c - | isAscii c && isPrint c = BS.singleton c - | otherwise = BS.pack $ printf "\\x{%x}" (ord c) +subDynamic :: Regex -> TokenizerM Regex +subDynamic (MatchDynamic capNum) = do + replacement <- getCapture capNum + return $ mconcat $ map (MatchChar . (==)) $ Text.unpack replacement +subDynamic (MatchAlt r1 r2) = + MatchAlt <$> subDynamic r1 <*> subDynamic r2 +subDynamic (MatchConcat r1 r2) = + MatchConcat <$> subDynamic r1 <*> subDynamic r2 +subDynamic (MatchSome r) = + MatchSome <$> subDynamic r +subDynamic (MatchCapture i r) = + MatchCapture i <$> subDynamic r +subDynamic (AssertPositive dir r) = + AssertPositive dir <$> subDynamic r +subDynamic (AssertNegative dir r) = + AssertNegative dir <$> subDynamic r +subDynamic x = return x getCapture :: Int -> TokenizerM Text getCapture capnum = do capts <- gets captures - if length capts < capnum - then mzero - else decodeBS $ capts !! (capnum - 1) + case IntMap.lookup capnum capts of + Nothing -> mzero + Just x -> decodeBS x keyword :: KeywordAttr -> WordSet Text -> ByteString -> TokenizerM Text keyword kwattr kws inp = do diff --git a/src/Skylighting/Types.hs b/src/Skylighting/Types.hs --- a/src/Skylighting/Types.hs +++ b/src/Skylighting/Types.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE BangPatterns #-} {-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE FlexibleInstances #-} @@ -70,16 +71,16 @@ -- | Attributes controlling how keywords are interpreted. data KeywordAttr = - KeywordAttr { keywordCaseSensitive :: Bool - , keywordDelims :: Set.Set Char + KeywordAttr { keywordCaseSensitive :: !Bool + , keywordDelims :: !(Set.Set Char) } deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary KeywordAttr -- | A set of "words," possibly case insensitive. -data WordSet a = CaseSensitiveWords (Set.Set a) - | CaseInsensitiveWords (Set.Set a) +data WordSet a = CaseSensitiveWords !(Set.Set a) + | CaseInsensitiveWords !(Set.Set a) deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary a => Binary (WordSet a) @@ -96,14 +97,14 @@ -- | Matchers correspond to the element types in a context. data Matcher = - DetectChar Char - | Detect2Chars Char Char - | AnyChar [Char] - | RangeDetect Char Char - | StringDetect Text - | WordDetect Text - | RegExpr RE - | Keyword KeywordAttr (WordSet Text) + DetectChar !Char + | Detect2Chars !Char !Char + | AnyChar !(Set.Set Char) + | RangeDetect !Char !Char + | StringDetect !Text + | WordDetect !Text + | RegExpr !RE + | Keyword KeywordAttr !(WordSet Text) | Int | Float | HlCOct @@ -111,7 +112,7 @@ | HlCStringChar | HlCChar | LineContinue - | IncludeRules ContextName + | IncludeRules !ContextName | DetectSpaces | DetectIdentifier deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) @@ -120,7 +121,7 @@ -- | A context switch, either pops or pushes a context. data ContextSwitch = - Pop | Push ContextName + Pop | Push !ContextName deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary ContextSwitch @@ -128,16 +129,16 @@ -- | A rule corresponds to one of the elements of a Kate syntax -- highlighting "context." data Rule = Rule{ - rMatcher :: Matcher - , rAttribute :: TokenType - , rIncludeAttribute :: Bool - , rDynamic :: Bool - , rCaseSensitive :: Bool - , rChildren :: [Rule] - , rLookahead :: Bool - , rFirstNonspace :: Bool - , rColumn :: Maybe Int - , rContextSwitch :: [ContextSwitch] + rMatcher :: !Matcher + , rAttribute :: !TokenType + , rIncludeAttribute :: !Bool + , rDynamic :: !Bool + , rCaseSensitive :: !Bool + , rChildren :: ![Rule] + , rLookahead :: !Bool + , rFirstNonspace :: !Bool + , rColumn :: !(Maybe Int) + , rContextSwitch :: ![ContextSwitch] } deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary Rule @@ -145,15 +146,15 @@ -- | A syntax corresponds to a complete Kate syntax description. -- The 'sShortname' field is derived from the filename. data Syntax = Syntax{ - sName :: Text - , sFilename :: String - , sShortname :: Text - , sContexts :: Map.Map Text Context - , sAuthor :: Text - , sVersion :: Text - , sLicense :: Text - , sExtensions :: [String] - , sStartingContext :: Text + sName :: !Text + , sFilename :: !String + , sShortname :: !Text + , sContexts :: !(Map.Map Text Context) + , sAuthor :: !Text + , sVersion :: !Text + , sLicense :: !Text + , sExtensions :: ![String] + , sStartingContext :: !Text } deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary Syntax @@ -164,16 +165,16 @@ -- | A Context corresponds to a context element in a Kate -- syntax description. data Context = Context{ - cName :: Text - , cSyntax :: Text - , cRules :: [Rule] - , cAttribute :: TokenType - , cLineEmptyContext :: [ContextSwitch] - , cLineEndContext :: [ContextSwitch] - , cLineBeginContext :: [ContextSwitch] - , cFallthrough :: Bool - , cFallthroughContext :: [ContextSwitch] - , cDynamic :: Bool + cName :: !Text + , cSyntax :: !Text + , cRules :: ![Rule] + , cAttribute :: !TokenType + , cLineEmptyContext :: ![ContextSwitch] + , cLineEndContext :: ![ContextSwitch] + , cLineBeginContext :: ![ContextSwitch] + , cFallthrough :: !Bool + , cFallthroughContext :: ![ContextSwitch] + , cDynamic :: !Bool } deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary Context @@ -247,11 +248,11 @@ -- | A 'TokenStyle' determines how a token is to be rendered. data TokenStyle = TokenStyle { - tokenColor :: Maybe Color - , tokenBackground :: Maybe Color - , tokenBold :: Bool - , tokenItalic :: Bool - , tokenUnderline :: Bool + tokenColor :: !(Maybe Color) + , tokenBackground :: !(Maybe Color) + , tokenBold :: !Bool + , tokenItalic :: !Bool + , tokenUnderline :: !Bool } deriving (Show, Read, Ord, Eq, Data, Typeable, Generic) instance Binary TokenStyle @@ -665,11 +666,11 @@ -- color for normal tokens. Line numbers can have a different -- color and background color. data Style = Style { - tokenStyles :: Map.Map TokenType TokenStyle - , defaultColor :: Maybe Color - , backgroundColor :: Maybe Color - , lineNumberColor :: Maybe Color - , lineNumberBackgroundColor :: Maybe Color + tokenStyles :: !(Map.Map TokenType TokenStyle) + , defaultColor :: !(Maybe Color) + , backgroundColor :: !(Maybe Color) + , lineNumberColor :: !(Maybe Color) + , lineNumberBackgroundColor :: !(Maybe Color) } deriving (Read, Show, Eq, Ord, Data, Typeable, Generic) instance Binary Style @@ -720,14 +721,14 @@ -- | Options for formatting source code. data FormatOptions = FormatOptions{ - numberLines :: Bool -- ^ Number lines - , startNumber :: Int -- ^ Number of first line - , lineAnchors :: Bool -- ^ Anchors on each line number - , titleAttributes :: Bool -- ^ Html titles with token types - , codeClasses :: [Text] -- ^ Additional classes for Html code tag - , containerClasses :: [Text] -- ^ Additional classes for Html container tag - , lineIdPrefix :: Text -- ^ Prefix for id attributes on lines - , ansiColorLevel :: ANSIColorLevel -- ^ Level of ANSI color support to use + numberLines :: !Bool -- ^ Number lines + , startNumber :: !Int -- ^ Number of first line + , lineAnchors :: !Bool -- ^ Anchors on each line number + , titleAttributes :: !Bool -- ^ Html titles with token types + , codeClasses :: ![Text] -- ^ Additional classes for Html code tag + , containerClasses :: ![Text] -- ^ Additional classes for Html container tag + , lineIdPrefix :: !Text -- ^ Prefix for id attributes on lines + , ansiColorLevel :: !ANSIColorLevel -- ^ Level of ANSI color support to use } deriving (Show, Read, Eq, Ord, Data, Typeable, Generic) instance Binary FormatOptions diff --git a/test/expected/abc.javascript.native b/test/expected/abc.javascript.native --- a/test/expected/abc.javascript.native +++ b/test/expected/abc.javascript.native @@ -99,7 +99,8 @@ , ( NormalTok , " (index " ) , ( OperatorTok , "!==" ) , ( NormalTok , " " ) - , ( DecValTok , "-1" ) + , ( OperatorTok , "-" ) + , ( DecValTok , "1" ) , ( NormalTok , ") {" ) ] , [ ( NormalTok , " length" ) , ( OperatorTok , "--;" ) ] diff --git a/test/expected/life.lua.native b/test/expected/life.lua.native --- a/test/expected/life.lua.native +++ b/test/expected/life.lua.native @@ -10,8 +10,7 @@ , [ ( CommentTok , "-- modified to use for instead of while" ) ] , [] , [ ( KeywordTok , "local" ) - , ( NormalTok , " " ) - , ( FunctionTok , "write" ) + , ( NormalTok , " write" ) , ( OperatorTok , "=" ) , ( FunctionTok , "io.write" ) ] diff --git a/test/test-skylighting.hs b/test/test-skylighting.hs --- a/test/test-skylighting.hs +++ b/test/test-skylighting.hs @@ -3,7 +3,6 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} module Main where -import qualified Control.Exception as E import Data.Aeson (decode, encode) import Data.Algorithm.Diff import qualified Data.ByteString.Lazy as BL @@ -23,7 +22,7 @@ import Test.Tasty.HUnit import Test.Tasty.QuickCheck (testProperty) import Text.Show.Pretty - +import GHC.IO.Encoding (setLocaleEncoding) import Skylighting.Core readTextFile :: FilePath -> IO Text @@ -40,6 +39,7 @@ main :: IO () main = do + setLocaleEncoding utf8 sMap <- do result <- loadSyntaxesFromDir xmlPath case result of @@ -78,11 +78,6 @@ , testCase "round trip style -> theme -> style" $ Just kate @=? decode (encode kate) ] - , testGroup "Skylighting.Regex" $ - [ testCase "convertOctalEscapes" $ - "a\\700b\\700c\\x{800}" @=? - convertOctalEscapes "a\\700b\\0700c\\o{4000}" - ] , testGroup "Skylighting" $ [ testCase "syntaxesByFilename" $ ["Perl"] @=? @@ -93,6 +88,7 @@ , testGroup "Doesn't hang or drop text on fuzz" $ map (\syn -> testProperty (Text.unpack (sName syn)) (p_no_drop defConfig syn)) syntaxes + , testGroup "Regex module" $ map regexTest regexTests , testGroup "Regression tests" $ let perl = maybe (error "could not find Perl syntax") id (lookupSyntax "Perl" sMap) @@ -193,24 +189,21 @@ noDropTest :: TokenizerConfig -> [Text] -> Syntax -> TestTree noDropTest cfg inps syntax = - localOption (mkTimeout 15000000) + localOption (mkTimeout 25000000) $ testCase (Text.unpack (sName syntax)) $ mapM_ go inps where go inp = - E.catch - (case tokenize cfg syntax inp of + case tokenize cfg syntax inp of Right ts -> assertBool ("Text has been dropped:\n" ++ diffs) (inplines == toklines) where inplines = Text.lines inp toklines = map (mconcat . map tokToText) ts diffs = makeDiff "expected" inplines toklines Left e -> - assertFailure ("Unexpected error: " ++ e ++ "\ninput = " ++ show inp)) - (\(e :: RegexException) -> - assertFailure (show e ++ "\ninput = " ++ show inp)) + assertFailure ("Unexpected error: " ++ e ++ "\ninput = " ++ show inp) tokenizerTest :: TokenizerConfig -> SyntaxMap -> Bool -> FilePath -> TestTree -tokenizerTest cfg sMap regen inpFile = localOption (mkTimeout 15000000) $ +tokenizerTest cfg sMap regen inpFile = localOption (mkTimeout 25000000) $ goldenTest testname getExpected getActual (compareValues referenceFile) updateGolden where testname = lang ++ " tokenizing of " ++ inpFile @@ -231,6 +224,55 @@ casesdir = "test" "cases" referenceFile = expecteddir inpFile <.> "native" lang = drop 1 $ takeExtension inpFile + +regexTest :: (String, String, Maybe (String, [(Int,String)])) -> TestTree +regexTest (re, inp, expected) = + testCase ("/" ++ re ++ "/ " ++ inp) $ + expected @=? testRegex True re inp + +regexTests :: [(String, String, Maybe (String, [(Int,String)]))] +regexTests = + [ (".", "aab", Just ("a", [])) + , ("ab", "aab", Nothing) + , ("ab", "abb", Just ("ab", [])) + , ("a(b)", "abb", Just ("ab", [(1,"b")])) + , ("a(b.)*", "abbbcb", Just ("abbbc", [(1,"bc")])) + , ("a(?:b.)*", "abbbcb", Just ("abbbc", [])) + , ("a(?=b)", "abb", Just ("a", [])) + , ("a(?=b)", "acb", Nothing) + , ("a(?!b)", "abb", Nothing) + , ("a(?!b)", "acb", Just ("a", [])) + , ("a?b+", "bbb", Just ("bbb", [])) + , ("a?b+", "abbb", Just ("abbb", [])) + , ("a?b+", "ac", Nothing) + , ("a*", "bbb", Just ("", [])) + , ("abc|ab$", "ab", Just ("ab", [])) + , ("abc|ab$", "abcd", Just ("abc", [])) + , ("abc|ab$", "abd", Nothing) + , ("(?:ab)*|a.*", "abababa", Just ("abababa", [])) + , ("a[b-e]*", "abcdefg", Just ("abcde", [])) + , ("a[b-e\\n-]*", "abcde\nb-bcfg", Just ("abcde\nb-bc", [])) + , ("^\\s+\\S+\\s+$", " abc ", Just (" abc ", [])) + , ("\\$", "$$", Just ("$", [])) + , ("[\\z12bb]", "\x12bb", Just ("\x12bb", [])) + , ("\\bhello\\b|hell", "hello there", Just ("hello", [])) + , ("\\bhello\\b|hell", "hellothere", Just ("hell", [])) + , ("[[:space:]]{2,4}.", " abc", Just (" a", [])) + , ("[[:space:]]{2,4}.", " abc", Nothing) + , ("[[:space:]]{2,4}.", " abc", Just (" ", [])) + , ("((..)\\+\\2)", "aa+aabb+bbbc+cb", + Just ("aa+aa", [(1,"aa+aa"), (2,"aa")])) + , ("(\\d+)/(\\d+) == \\{1}", "22/2 == 22", + Just ("22/2 == 22", [(1,"22"), (2,"2")])) + , ("([a-z]+){2}", "htabc", Just ("htabc", [(1,"c")])) + , ("((.+)(.+)(.+))*", (replicate 400 'a'), + Just (replicate 400 'a', + [(1, replicate 400 'a') + ,(2,replicate 398 'a') + ,(3,"a") + ,(4,"a")])) + ] + vividize :: Diff Text -> Text vividize (Both s _) = " " <> s diff --git a/xml/actionscript.xml b/xml/actionscript.xml --- a/xml/actionscript.xml +++ b/xml/actionscript.xml @@ -1,6 +1,6 @@ - + @@ -269,8 +269,8 @@ - - + + diff --git a/xml/bash.xml b/xml/bash.xml --- a/xml/bash.xml +++ b/xml/bash.xml @@ -9,7 +9,7 @@ ]> - + - + + @@ -606,7 +607,8 @@ - + + diff --git a/xml/boo.xml b/xml/boo.xml --- a/xml/boo.xml +++ b/xml/boo.xml @@ -2,7 +2,7 @@ - + import @@ -163,11 +163,13 @@ - - + + - - + + + + @@ -185,7 +187,7 @@ - + diff --git a/xml/c.xml b/xml/c.xml --- a/xml/c.xml +++ b/xml/c.xml @@ -1,8 +1,8 @@ - + + @@ -10,7 +10,7 @@ ]> - + - - - - + + + + diff --git a/xml/cmake.xml b/xml/cmake.xml --- a/xml/cmake.xml +++ b/xml/cmake.xml @@ -31,7 +31,7 @@ break cmake_host_system_information + cmake_language cmake_minimum_required cmake_parse_arguments cmake_policy @@ -185,6 +186,11 @@ TOTAL_PHYSICAL_MEMORY TOTAL_VIRTUAL_MEMORY + + CALL + CODE + EVAL + FATAL_ERROR VERSION @@ -218,6 +224,7 @@ AND + COMMAND DEFINED EQUAL EXISTS @@ -250,6 +257,8 @@ COMMAND COMMAND_ECHO + ECHO_ERROR_VARIABLE + ECHO_OUTPUT_VARIABLE ENCODING ERROR_FILE ERROR_QUIET @@ -275,8 +284,12 @@ UTF8 + @ONLY APPEND + ARCHIVE_CREATE + ARCHIVE_EXTRACT CONDITION + CONFIGURE CONFIGURE_DEPENDS CONTENT COPY @@ -285,13 +298,16 @@ DIRECTORY_PERMISSIONS DOWNLOAD ENCODING + ESCAPE_QUOTES EXCLUDE EXPECTED_HASH EXPECTED_MD5 + FILES FILES_MATCHING FILE_PERMISSIONS FOLLOW_SYMLINKS FOLLOW_SYMLINK_CHAIN + FORMAT GENERATE GET_RUNTIME_DEPENDENCIES GLOB @@ -309,13 +325,16 @@ LIMIT_INPUT LIMIT_OUTPUT LIST_DIRECTORIES + LIST_ONLY LOCK LOG MAKE_DIRECTORY MD5 + MTIME NETRC NETRC_FILE NEWLINE_CONSUME + NEWLINE_STYLE NO_HEX_CONVERSION NO_SOURCE_PERMISSIONS OFFSET @@ -353,19 +372,28 @@ TOUCH_NOCREATE TO_CMAKE_PATH TO_NATIVE_PATH + TYPE UPLOAD USERPWD USE_SOURCE_PERMISSIONS UTC + VERBOSE WRITE + 7zip + BZip2 + CRLF + DOS FILE FUNCTION GROUP_EXECUTE GROUP_READ GROUP_WRITE + GZip IGNORED + LF + None OPTIONAL OWNER_EXECUTE OWNER_READ @@ -374,14 +402,23 @@ REQUIRED SETGID SETUID + UNIX UTF-16BE UTF-16LE UTF-32B UTF-32LE UTF-8 + WIN32 WORLD_EXECUTE WORLD_READ WORLD_WRITE + XZ + Zstd + gnutar + pax + paxr + raw + zip CMAKE_FIND_ROOT_PATH_BOTH @@ -398,6 +435,7 @@ ONLY_CMAKE_FIND_ROOT_PATH PATHS PATH_SUFFIXES + REQUIRED CMAKE_FIND_ROOT_PATH_BOTH @@ -415,6 +453,7 @@ ONLY_CMAKE_FIND_ROOT_PATH PATHS PATH_SUFFIXES + REQUIRED CMAKE_FIND_ROOT_PATH_BOTH @@ -458,6 +497,7 @@ ONLY_CMAKE_FIND_ROOT_PATH PATHS PATH_SUFFIXES + REQUIRED CMAKE_FIND_ROOT_PATH_BOTH @@ -475,6 +515,7 @@ ONLY_CMAKE_FIND_ROOT_PATH PATHS PATH_SUFFIXES + REQUIRED IN @@ -521,11 +562,13 @@ SET SOURCE TARGET + TARGET_DIRECTORY TEST VARIABLE AND + COMMAND DEFINED EQUAL EXISTS @@ -601,6 +644,7 @@ DESCENDING FILE_BASENAME INSENSITIVE + NATURAL SENSITIVE STRING @@ -649,6 +693,7 @@ PROPERTY SOURCE TARGET + TARGET_DIRECTORY TEST VARIABLE @@ -679,6 +724,7 @@ GENEX_STRIP GREATER GREATER_EQUAL + HEX JOIN LENGTH LESS @@ -722,6 +768,7 @@ AND + COMMAND DEFINED EQUAL EXISTS @@ -866,6 +913,10 @@ NAMESPACE TARGETS + + DIRECTORY + TARGET_DIRECTORY + AFTER BEFORE @@ -968,7 +1019,9 @@ Swift + DIRECTORY PROPERTIES + TARGET_DIRECTORY PROPERTIES @@ -1221,6 +1274,7 @@ RETURN_VALUE SCHEDULE_RANDOM START + STOP_ON_FAILURE STOP_TIME STRIDE TEST_LOAD @@ -1244,6 +1298,9 @@ ANDROID APPLE + ARGC + ARGN + ARGV BORLAND BUILD_SHARED_LIBS BUILD_TESTING @@ -1322,6 +1379,7 @@ CMAKE_CROSS_CONFIGS CMAKE_CTEST_ARGUMENTS CMAKE_CTEST_COMMAND + CMAKE_CUDA_ARCHITECTURES CMAKE_CUDA_COMPILE_FEATURES CMAKE_CUDA_EXTENSIONS CMAKE_CUDA_HOST_COMPILER @@ -1413,6 +1471,7 @@ CMAKE_Fortran_MODDIR_FLAG CMAKE_Fortran_MODOUT_FLAG CMAKE_Fortran_MODULE_DIRECTORY + CMAKE_Fortran_PREPROCESS CMAKE_GENERATOR CMAKE_GENERATOR_INSTANCE CMAKE_GENERATOR_NO_COMPILER_ENV @@ -1424,7 +1483,6 @@ CMAKE_GLOBAL_AUTORCC_TARGET_NAME CMAKE_GNUtoMS CMAKE_HAS_ANSI_STRING_STREAM - CMAKE_HOME_DIRECTORY CMAKE_HOST_APPLE CMAKE_HOST_SOLARIS CMAKE_HOST_SYSTEM @@ -1493,7 +1551,6 @@ CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP CMAKE_INSTALL_UCRT_LIBRARIES - CMAKE_INTERNAL_PLATFORM_ABI CMAKE_INTERPROCEDURAL_OPTIMIZATION CMAKE_IOS_INSTALL_COMBINED CMAKE_JOB_POOLS @@ -1535,7 +1592,6 @@ CMAKE_NETRC CMAKE_NETRC_FILE CMAKE_NINJA_OUTPUT_PATH_PREFIX - CMAKE_NOT_USING_CONFIG_FLAGS CMAKE_NO_ANSI_FOR_SCOPE CMAKE_NO_ANSI_STREAM_HEADERS CMAKE_NO_ANSI_STRING_STREAM @@ -1548,6 +1604,7 @@ CMAKE_OSX_SYSROOT CMAKE_PARENT_LIST_FILE CMAKE_PATCH_VERSION + CMAKE_PCH_WARN_INVALID CMAKE_PDB_OUTPUT_DIRECTORY CMAKE_POSITION_INDEPENDENT_CODE CMAKE_PREFIX_PATH @@ -1592,8 +1649,6 @@ CMAKE_STATIC_LINKER_FLAGS_INIT CMAKE_SUBLIME_TEXT_2_ENV_SETTINGS CMAKE_SUBLIME_TEXT_2_EXCLUDE_BUILD_TREE - CMAKE_SUPPRESS_DEVELOPER_ERRORS - CMAKE_SUPPRESS_DEVELOPER_WARNINGS CMAKE_SUPPRESS_REGENERATION CMAKE_SWIG_FLAGS CMAKE_SWIG_OUTDIR @@ -1634,7 +1689,6 @@ CMAKE_VS_GLOBALS CMAKE_VS_INCLUDE_INSTALL_TO_DEFAULT_BUILD CMAKE_VS_INCLUDE_PACKAGE_TO_DEFAULT_BUILD - CMAKE_VS_INTEL_Fortran_PROJECT_VERSION CMAKE_VS_JUST_MY_CODE_DEBUGGING CMAKE_VS_MSBUILD_COMMAND CMAKE_VS_NsightTegra_VERSION @@ -1678,6 +1732,7 @@ CPACK_ABSOLUTE_DESTINATION_FILES CPACK_ARCHIVE_COMPONENT_INSTALL CPACK_ARCHIVE_FILE_NAME + CPACK_ARCHIVE_THREADS CPACK_BUILD_SOURCE_DIRS CPACK_BUNDLE_APPLE_CERT_APP CPACK_BUNDLE_APPLE_CODESIGN_FILES @@ -1742,9 +1797,9 @@ CPACK_DMG_SLA_LANGUAGES CPACK_DMG_VOLUME_NAME CPACK_ERROR_ON_ABSOLUTE_INSTALL_DESTINATION - CPACK_EXT_ENABLE_STAGING - CPACK_EXT_PACKAGE_SCRIPT - CPACK_EXT_REQUESTED_VERSIONS + CPACK_EXTERNAL_ENABLE_STAGING + CPACK_EXTERNAL_PACKAGE_SCRIPT + CPACK_EXTERNAL_REQUESTED_VERSIONS CPACK_GENERATOR CPACK_IFW_ADMIN_TARGET_DIRECTORY CPACK_IFW_BINARYCREATOR_EXECUTABLE @@ -1788,7 +1843,6 @@ CPACK_INSTALL_CMAKE_PROJECTS CPACK_INSTALL_COMMANDS CPACK_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS - CPACK_INSTALL_SCRIPT CPACK_INSTALL_SCRIPTS CPACK_MONOLITHIC_INSTALL CPACK_NSIS_COMPRESSOR @@ -1807,6 +1861,7 @@ CPACK_NSIS_INSTALLED_ICON_NAME CPACK_NSIS_INSTALLER_MUI_ICON_CODE CPACK_NSIS_INSTALL_ROOT + CPACK_NSIS_MANIFEST_DPI_AWARE CPACK_NSIS_MENU_LINKS CPACK_NSIS_MODIFY_PATH CPACK_NSIS_MUI_FINISHPAGE_RUN @@ -1838,7 +1893,6 @@ CPACK_NUGET_PACKAGE_VERSION CPACK_OUTPUT_CONFIG_FILE CPACK_PACKAGE_CHECKSUM - CPACK_PACKAGE_CONTACT CPACK_PACKAGE_DESCRIPTION CPACK_PACKAGE_DESCRIPTION_FILE CPACK_PACKAGE_DESCRIPTION_SUMMARY @@ -1850,7 +1904,6 @@ CPACK_PACKAGE_INSTALL_DIRECTORY CPACK_PACKAGE_INSTALL_REGISTRY_KEY CPACK_PACKAGE_NAME - CPACK_PACKAGE_RELOCATABLE CPACK_PACKAGE_VENDOR CPACK_PACKAGE_VERSION CPACK_PACKAGE_VERSION_MAJOR @@ -1926,8 +1979,10 @@ CPACK_RPM_PACKAGE_VENDOR CPACK_RPM_PACKAGE_VERSION CPACK_RPM_POST_INSTALL_SCRIPT_FILE + CPACK_RPM_POST_TRANS_SCRIPT_FILE CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE CPACK_RPM_PRE_INSTALL_SCRIPT_FILE + CPACK_RPM_PRE_TRANS_SCRIPT_FILE CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE CPACK_RPM_RELOCATION_PATHS CPACK_RPM_SOURCE_PKG_BUILD_PARAMS @@ -2023,6 +2078,8 @@ CTEST_P4_COMMAND CTEST_P4_OPTIONS CTEST_P4_UPDATE_OPTIONS + CTEST_RESOURCE_SPEC_FILE + CTEST_RUN_CURRENT_SCRIPT CTEST_SCP_COMMAND CTEST_SITE CTEST_SOURCE_DIRECTORY @@ -2120,6 +2177,21 @@ XCODE_VERSION + + CMAKE_HOME_DIRECTORY + CMAKE_INTERNAL_PLATFORM_ABI + CMAKE_NOT_USING_CONFIG_FLAGS + CMAKE_SUPPRESS_DEVELOPER_ERRORS + CMAKE_SUPPRESS_DEVELOPER_WARNINGS + CMAKE_VS_INTEL_Fortran_PROJECT_VERSION + CPACK_INSTALL_PREFIX + CPACK_INSTALL_SCRIPT + CPACK_PACKAGE_CONTACT + CPACK_PACKAGE_RELOCATABLE + CPACK_TEMPORARY_DIRECTORY + CPACK_TOPLEVEL_DIRECTORY + + CC CFLAGS @@ -2293,6 +2365,7 @@ COMPILE_PDB_NAME COMPILE_PDB_OUTPUT_DIRECTORY CROSSCOMPILING_EMULATOR + CUDA_ARCHITECTURES CUDA_EXTENSIONS CUDA_PTX_COMPILATION CUDA_RESOLVE_DEVICE_SYMBOLS @@ -2324,6 +2397,7 @@ FRAMEWORK_VERSION Fortran_FORMAT Fortran_MODULE_DIRECTORY + Fortran_PREPROCESS GENERATOR_FILE_NAME GNUtoMS HAS_CXX @@ -2382,6 +2456,8 @@ LINK_SEARCH_START_STATIC LINK_WHAT_YOU_USE LOCATION + MACHO_COMPATIBILITY_VERSION + MACHO_CURRENT_VERSION MACOSX_BUNDLE MACOSX_BUNDLE_INFO_PLIST MACOSX_FRAMEWORK_INFO_PLIST @@ -2395,6 +2471,7 @@ OSX_COMPATIBILITY_VERSION OSX_CURRENT_VERSION OUTPUT_NAME + PCH_WARN_INVALID PDB_NAME PDB_OUTPUT_DIRECTORY POSITION_INDEPENDENT_CODE @@ -2425,6 +2502,7 @@ UNITY_BUILD_BATCH_SIZE UNITY_BUILD_CODE_AFTER_INCLUDE UNITY_BUILD_CODE_BEFORE_INCLUDE + UNITY_BUILD_MODE VERSION VISIBILITY_INLINES_HIDDEN VS_CONFIGURATION_TYPE @@ -2448,12 +2526,14 @@ VS_MOBILE_EXTENSIONS_VERSION VS_NO_SOLUTION_DEPLOY VS_PACKAGE_REFERENCES + VS_PLATFORM_TOOLSET VS_PROJECT_IMPORT VS_SCC_AUXPATH VS_SCC_LOCALPATH VS_SCC_PROJECTNAME VS_SCC_PROVIDER VS_SDK_REFERENCES + VS_SOLUTION_DEPLOY VS_USER_PROPS VS_WINDOWS_TARGET_PLATFORM_MIN_VERSION VS_WINRT_COMPONENT @@ -2496,6 +2576,7 @@ COMPILE_OPTIONS EXTERNAL_OBJECT Fortran_FORMAT + Fortran_PREPROCESS GENERATED HEADER_FILE_ONLY INCLUDE_DIRECTORIES @@ -2520,6 +2601,7 @@ VS_DEPLOYMENT_LOCATION VS_INCLUDE_IN_VSIX VS_RESOURCE_GENERATOR + VS_SETTINGS VS_SHADER_DISABLE_OPTIMIZATIONS VS_SHADER_ENABLE_DEBUG VS_SHADER_ENTRYPOINT @@ -2609,6 +2691,10 @@ COMPILE_FEATURES COMPILE_LANG_AND_ID COMPILE_LANGUAGE + LINK_LANG_AND_ID + LINK_LANGUAGE + DEVICE_LINK + HOST_LINK ANGLE-R COMMA SEMICOLON @@ -2660,6 +2746,7 @@ + @@ -2667,11 +2754,11 @@ - - - - - + + + + + @@ -2679,17 +2766,17 @@ - - + + - + - + @@ -2703,7 +2790,7 @@ - + @@ -2789,6 +2876,14 @@ + + + + + + + + @@ -3370,6 +3465,7 @@ + @@ -3723,6 +3819,7 @@ + @@ -3757,6 +3854,7 @@ + @@ -3784,15 +3882,16 @@ + - - + + + + @@ -3821,7 +3920,10 @@ + + + @@ -3831,7 +3933,6 @@ - @@ -3873,7 +3974,6 @@ - @@ -3887,6 +3987,7 @@ + @@ -3995,18 +4096,12 @@ - + - - - - - - @@ -4014,7 +4109,8 @@ - + + @@ -4025,6 +4121,7 @@ + @@ -4071,6 +4168,7 @@ + @@ -4115,6 +4213,7 @@ + diff --git a/xml/elm.xml b/xml/elm.xml --- a/xml/elm.xml +++ b/xml/elm.xml @@ -1,6 +1,6 @@ - + type @@ -56,7 +56,7 @@ - + diff --git a/xml/erlang.xml b/xml/erlang.xml --- a/xml/erlang.xml +++ b/xml/erlang.xml @@ -213,7 +213,7 @@ - + diff --git a/xml/erlang.xml.patch b/xml/erlang.xml.patch deleted file mode 100644 --- a/xml/erlang.xml.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/skylighting-core/xml/erlang.xml b/skylighting-core/xml/erlang.xml -index 7bbcbc9..b631afd 100644 ---- a/skylighting-core/xml/erlang.xml -+++ b/skylighting-core/xml/erlang.xml -@@ -213,7 +213,7 @@ - - - -- -+ - - - diff --git a/xml/glsl.xml b/xml/glsl.xml --- a/xml/glsl.xml +++ b/xml/glsl.xml @@ -1,6 +1,6 @@ - + break @@ -1173,8 +1173,8 @@ - - + + diff --git a/xml/graphql.xml b/xml/graphql.xml new file mode 100644 --- /dev/null +++ b/xml/graphql.xml @@ -0,0 +1,83 @@ + + + + + + + enum + extend + fragment + implements + input + interface + mutation + on + query + scalar + schema + subscription + type + union + + + null + true + false + + + Boolean + Float + ID + Int + String + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xml/isocpp.xml b/xml/isocpp.xml --- a/xml/isocpp.xml +++ b/xml/isocpp.xml @@ -1,8 +1,8 @@ - + + @@ -24,7 +24,7 @@ - + @@ -392,16 +392,16 @@ - + - - - - + + + + @@ -702,7 +702,7 @@ - + diff --git a/xml/java.xml b/xml/java.xml --- a/xml/java.xml +++ b/xml/java.xml @@ -4,7 +4,7 @@ ]> - + ACTIVE @@ -3786,9 +3786,9 @@ - + - + diff --git a/xml/lua.xml b/xml/lua.xml --- a/xml/lua.xml +++ b/xml/lua.xml @@ -48,7 +48,7 @@ - NOTE, FIXME, TODO alerts added on comments - improved highlighting --> - + and @@ -149,7 +149,7 @@ coroutine.create coroutine.isyieldable coroutine.resume - coroutine.kill + coroutine.close coroutine.running coroutine.status coroutine.wrap @@ -423,6 +423,8 @@ __call __tostring + __name + __close __pairs __metatable @@ -477,6 +479,7 @@ + @@ -484,11 +487,11 @@ - + + + - - - + @@ -504,6 +507,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -556,6 +583,7 @@ + diff --git a/xml/lua.xml.patch b/xml/lua.xml.patch --- a/xml/lua.xml.patch +++ b/xml/lua.xml.patch @@ -1,11 +1,11 @@ ---- xml/lua.xml.orig 2018-06-08 09:28:41.000000000 -0700 -+++ xml/lua.xml 2018-06-08 09:28:33.000000000 -0700 -@@ -518,7 +518,7 @@ +--- lua.xml.orig 2020-07-29 11:31:42.000000000 -0700 ++++ lua.xml 2020-07-29 11:31:57.000000000 -0700 +@@ -582,7 +582,7 @@ - + + - diff --git a/xml/m4.xml b/xml/m4.xml --- a/xml/m4.xml +++ b/xml/m4.xml @@ -38,7 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> - + __gnu__ @@ -165,7 +165,7 @@ - + diff --git a/xml/mediawiki.xml b/xml/mediawiki.xml --- a/xml/mediawiki.xml +++ b/xml/mediawiki.xml @@ -6,7 +6,7 @@ ]> - + @@ -15,7 +15,8 @@ - + + @@ -29,7 +30,7 @@ - + @@ -58,13 +59,15 @@ - + + - + + @@ -83,7 +86,8 @@ - + + @@ -101,7 +105,8 @@ - + + @@ -119,7 +124,8 @@ - + + @@ -284,7 +290,8 @@ - + + @@ -306,7 +313,8 @@ - + + diff --git a/xml/nim.xml b/xml/nim.xml new file mode 100644 --- /dev/null +++ b/xml/nim.xml @@ -0,0 +1,317 @@ + + + + + + + + + + const + export + import + include + lambda + let + var + + + + assert + asm + atomic + block + break + case + cast + compiles + continue + declared + declaredinscope + defined + discard + do + echo + elif + else + end + except + finally + for + from + if + mixin + bind + new + raise + return + sizeof + try + when + while + quit + using + yield + + + + addr + and + as + div + in + is + isnot + mod + not + notin + of + or + shl + shr + xor + + + + array + bool + byte + cchar + cdouble + char + cfloat + cint + clong + cshort + cstring + cuint + distinct + expr + float + float32 + float64 + generic + int + int8 + int16 + int32 + int64 + interface + openarray + pointer + set + seq + stmt + string + tuple + typedesc + uint + uint8 + uint16 + uint32 + uint64 + varargs + void + untyped + typed + + + + out + ptr + ref + shared + static + + + + false + inf + nil + true + on + off + + + + result + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xml/perl.xml b/xml/perl.xml --- a/xml/perl.xml +++ b/xml/perl.xml @@ -39,7 +39,7 @@ Enhance tr/// and y/// support. --> - + if @@ -465,7 +465,7 @@ - + @@ -508,7 +508,7 @@ - + @@ -625,7 +625,7 @@ - + @@ -674,7 +674,7 @@ - + @@ -694,8 +694,8 @@ + - diff --git a/xml/perl.xml.patch b/xml/perl.xml.patch deleted file mode 100644 --- a/xml/perl.xml.patch +++ /dev/null @@ -1,29 +0,0 @@ ---- ../syntax-highlighting/data/syntax/perl.xml 2016-12-14 12:44:11.000000000 +0100 -+++ perl.xml 2017-02-16 16:14:04.000000000 +0100 -@@ -452,7 +452,7 @@ - - - -- -+ - - - -@@ -495,7 +495,7 @@ - - - -- -+ - - - -@@ -600,7 +600,7 @@ - - - -- -+ - - - diff --git a/xml/powershell.xml b/xml/powershell.xml --- a/xml/powershell.xml +++ b/xml/powershell.xml @@ -1,7 +1,7 @@ - - + + diff --git a/xml/scala.xml b/xml/scala.xml --- a/xml/scala.xml +++ b/xml/scala.xml @@ -1,6 +1,6 @@ - - - + + diff --git a/xml/tcsh.xml b/xml/tcsh.xml --- a/xml/tcsh.xml +++ b/xml/tcsh.xml @@ -655,7 +655,7 @@ - + diff --git a/xml/tcsh.xml.patch b/xml/tcsh.xml.patch deleted file mode 100644 --- a/xml/tcsh.xml.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- tcsh.xml.orig 2018-06-08 09:32:02.000000000 -0700 -+++ tcsh.xml 2018-06-08 09:32:06.000000000 -0700 -@@ -674,7 +674,7 @@ - - - -- -+ - - - diff --git a/xml/toml.xml b/xml/toml.xml new file mode 100644 --- /dev/null +++ b/xml/toml.xml @@ -0,0 +1,135 @@ + + + + + + + + + +]> + + + + + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xml/xul.xml b/xml/xul.xml --- a/xml/xul.xml +++ b/xml/xul.xml @@ -4,7 +4,7 @@ ]> - + @@ -520,7 +520,7 @@ - + diff --git a/xml/zsh.xml b/xml/zsh.xml --- a/xml/zsh.xml +++ b/xml/zsh.xml @@ -8,7 +8,7 @@ ]> - + @@ -600,7 +600,8 @@ - + +