skylighting-core 0.7.0.1 → 0.7.0.2
raw patch · 6 files changed
+61/−36 lines, 6 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- changelog.md +10/−0
- skylighting-core.cabal +3/−1
- src/Skylighting/Tokenizer.hs +40/−34
- test/cases/issue41.djangotemplate +3/−0
- test/expected/issue41.djangotemplate.native +4/−0
- xml/haskell.xml +1/−1
changelog.md view
@@ -1,5 +1,15 @@ # Revision history for skylighting-core +## 0.7.0.2 -- 2018-03-06++ * Ensure that regex captures are not overwritten by regexes+ without captures.+ * Fixed bug in subDynamic (#41).+ * Added tracing information about rules tried and dynamic+ regexes, for debugging.+ * Fix highlighting for Haskell chars which breaks `-XDataKinds` (#40,+ Artyom Kazak).+ ## 0.7.0.1 -- 2018-03-03 * Updated changelog.md.
skylighting-core.cabal view
@@ -1,5 +1,5 @@ name: skylighting-core-version: 0.7.0.1+version: 0.7.0.2 synopsis: syntax highlighting library description: Skylighting is a syntax highlighting library. It derives its tokenizers from XML syntax@@ -59,6 +59,7 @@ test/cases/life.lua test/cases/hk91.html test/cases/if.cmake+ test/cases/issue41.djangotemplate test/expected/abc.ada.native test/expected/abc.agda.native test/expected/abc.c.native@@ -89,6 +90,7 @@ test/expected/life.lua.native test/expected/hk91.html.native test/expected/if.cmake.native+ test/expected/issue41.djangotemplate.native cabal-version: >=1.10
src/Skylighting/Tokenizer.hs view
@@ -272,6 +272,7 @@ tryRule :: Rule -> ByteString -> TokenizerM (Maybe Token) tryRule _ "" = mzero tryRule rule inp = do+ info $ "Trying rule " ++ show rule case rColumn rule of Nothing -> return () Just n -> gets column >>= guard . (== n)@@ -496,7 +497,10 @@ regExpr :: Bool -> RE -> ByteString -> TokenizerM Text regExpr dynamic re inp = do reStr <- if dynamic- then subDynamic (reString re)+ then do+ reStr' <- subDynamic (reString re)+ info $ "Dynamic regex: " ++ show reStr'+ return reStr' else return (reString re) when (BS.take 2 reStr == "\\b") $ wordBoundary inp regex <- if dynamic@@ -512,7 +516,8 @@ Just cre -> return cre case matchRegex regex inp of Just (match:capts) -> do- modify $ \st -> st{ captures = capts }+ unless (null capts) $+ modify $ \st -> st{ captures = capts } takeChars (UTF8.length match) _ -> mzero @@ -542,38 +547,39 @@ -- Substitute out %1, %2, etc. in regex string, escaping -- appropriately.. subDynamic :: ByteString -> TokenizerM ByteString-subDynamic bs- | BS.null bs = return BS.empty- | otherwise =- case BS.unpack (BS.take 2 bs) of- ['%',x] | x >= '0' && x <= '9' -> do- let capNum = ord x - ord '0'- let escapeRegexChar :: Char -> BS.ByteString- escapeRegexChar '^' = "\\^"- escapeRegexChar '$' = "\\$"- escapeRegexChar '\\' = "\\\\"- escapeRegexChar '[' = "\\["- escapeRegexChar ']' = "\\]"- escapeRegexChar '(' = "\\("- escapeRegexChar ')' = "\\)"- escapeRegexChar '{' = "\\{"- escapeRegexChar '}' = "\\}"- escapeRegexChar '*' = "\\*"- escapeRegexChar '+' = "\\+"- escapeRegexChar '.' = "\\."- escapeRegexChar '?' = "\\?"- escapeRegexChar c- | isAscii c && isPrint c = BS.singleton c- | otherwise = BS.pack $ printf "\\x{%x}" (ord c)- let escapeRegex = BS.concatMap escapeRegexChar- replacement <- getCapture capNum- (escapeRegex (encodeUtf8 replacement) <>) <$>- subDynamic (BS.drop 2 bs)- _ -> case BS.break (=='%') bs of- (y,z)- | BS.null y -> BS.cons '%' <$> subDynamic z- | BS.null z -> return y- | otherwise -> (y <>) <$> subDynamic z+subDynamic bs =+ case BS.break (=='%') bs of+ (y,z)+ | BS.null z -> return y+ | otherwise -> (y <>) <$>+ case BS.unpack (BS.take 2 z) of+ ['%',x] | x >= '0' && x <= '9' -> do+ let capNum = ord x - ord '0'+ replacement <- getCapture capNum+ (escapeRegex (encodeUtf8 replacement) <>) <$>+ subDynamic (BS.drop 2 z)+ _ -> BS.cons '%' <$> (subDynamic (BS.drop 1 z))++escapeRegex :: BS.ByteString -> BS.ByteString+escapeRegex = BS.concatMap escapeRegexChar++escapeRegexChar :: Char -> BS.ByteString+escapeRegexChar '^' = "\\^"+escapeRegexChar '$' = "\\$"+escapeRegexChar '\\' = "\\\\"+escapeRegexChar '[' = "\\["+escapeRegexChar ']' = "\\]"+escapeRegexChar '(' = "\\("+escapeRegexChar ')' = "\\)"+escapeRegexChar '{' = "\\{"+escapeRegexChar '}' = "\\}"+escapeRegexChar '*' = "\\*"+escapeRegexChar '+' = "\\+"+escapeRegexChar '.' = "\\."+escapeRegexChar '?' = "\\?"+escapeRegexChar c+ | isAscii c && isPrint c = BS.singleton c+ | otherwise = BS.pack $ printf "\\x{%x}" (ord c) getCapture :: Int -> TokenizerM Text getCapture capnum = do
+ test/cases/issue41.djangotemplate view
@@ -0,0 +1,3 @@+{% if var %}+string+{% endif %}
+ test/expected/issue41.djangotemplate.native view
@@ -0,0 +1,4 @@+[ [ ( FunctionTok , "{% if var %}" ) ]+, [ ( NormalTok , "string" ) ]+, [ ( FunctionTok , "{% endif %}" ) ]+]
xml/haskell.xml view
@@ -482,7 +482,7 @@ <RegExpr attribute="Octal" context="#stay" String="0[Oo][0-7]+"/> <RegExpr attribute="Hex" context="#stay" String="0[Xx][0-9A-Fa-f]+"/> <Int attribute="Decimal" context="#stay" />- <DetectChar attribute="Char" context="char" char="'" />+ <RegExpr attribute="Char" context="#stay" String="'(\\'|\\[^']+|[^\\\n])'" /> <DetectChar attribute="String" context="string" char=""" /> <DetectChar attribute="Function Infix" context="infix" char="`"/>