diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,37 +1,47 @@ # Revision history for skylighting -## 0.1.0.0 -- 2016-12-23 +## 0.1.1.4 -- 2017-01-31 -* Initial release + * Properly escape characters in subDynamic. + This fixes a problem that caused failures with -## 0.1.0.1 -- 2016-12-23 + echo -e "s\0b\0c" | skylighting -s perl -* Fixed bug in LaTeX renderer (backslash in wrong position). + * More efficient rewrite of char escaping for regexes. -## 0.1.1 -- 2017-01-19 +## 0.1.1.3 -- 2017-01-29 -* Added breezeDark style, from breeze-dark.theme. -* Fixed performance bug in regex application (#1). This gives a - significant speedup, especially for inputs with long lines. + * Increase test timeout to 6s. + * Avoid double {{}} in latex macros. + * Fixed problem compiling Format.LaTeX on older ghc versions (7.8.3) + that can't take a Text as PrintfArg. +## 0.1.1.2 -- 2017-01-28 + + * Added much more extensive testing to ensure that tokenizers + don't hang or drop input. + * Fixed some issues with line-end and fallthrough context + handling. + * Fixed a bug in WordDetect handling which caused it to drop + input (#2). + ## 0.1.1.1 -- 2017-01-21 -* Optimized. Speed is now comparable to highlighting-kate - and often better. -* Added benchmarks. + * Optimized. Speed is now comparable to highlighting-kate + and often better. + * Added benchmarks. -## 0.1.1.2 -- 2017-01-28 +## 0.1.1 -- 2017-01-19 -* Added much more extensive testing to ensure that tokenizers - don't hang or drop input. -* Fixed some issues with line-end and fallthrough context - handling. -* Fixed a bug in WordDetect handling which caused it to drop - input (#2). + * Added breezeDark style, from breeze-dark.theme. + * Fixed performance bug in regex application (#1). This gives a + significant speedup, especially for inputs with long lines. -## 0.1.1.3 -- 2017-01-29 +## 0.1.0.1 -- 2016-12-23 -* Increase test timeout to 6s. -* Avoid double {{}} in latex macros. -* Fixed problem compiling Format.LaTeX on older ghc versions (7.8.3) - that can't take a Text as PrintfArg. + * Fixed bug in LaTeX renderer (backslash in wrong position). + +## 0.1.0.0 -- 2016-12-23 + + * Initial release + diff --git a/skylighting.cabal b/skylighting.cabal --- a/skylighting.cabal +++ b/skylighting.cabal @@ -1,5 +1,5 @@ name: skylighting -version: 0.1.1.3 +version: 0.1.1.4 synopsis: syntax highlighting library description: Skylighting is a syntax highlighting library with support for over one hundred languages. It derives diff --git a/src/Skylighting/Tokenizer.hs b/src/Skylighting/Tokenizer.hs --- a/src/Skylighting/Tokenizer.hs +++ b/src/Skylighting/Tokenizer.hs @@ -11,7 +11,7 @@ import qualified Data.ByteString.Char8 as BS import Data.ByteString.Char8 (ByteString) import Data.CaseInsensitive (mk) -import Data.Char (isAlphaNum, isAscii, isLetter, isSpace, ord) +import Data.Char (isAlphaNum, isAscii, isLetter, isSpace, ord, isPrint) import qualified Data.Map as Map import Data.Maybe (catMaybes, fromMaybe) import Data.Monoid @@ -23,6 +23,7 @@ import Debug.Trace import Skylighting.Regex import Skylighting.Types +import Text.Printf (printf) info :: String -> TokenizerM () info s = do @@ -480,13 +481,27 @@ case BS.unpack (BS.take 2 bs) of ['%',x] | x >= '0' && x <= '9' -> do let capNum = ord x - ord '0' - let escapeRegexChar c - | c `elem` ['^','$','\\','[',']','(',')','{','}','*','+','.','?'] - = BS.pack ['\\',c] - | otherwise = BS.singleton c + let escapeRegexChar :: Char -> BS.ByteString + escapeRegexChar '^' = "\\^" + escapeRegexChar '$' = "\\$" + escapeRegexChar '\\' = "\\\\" + escapeRegexChar '[' = "\\[" + escapeRegexChar ']' = "\\]" + escapeRegexChar '(' = "\\(" + escapeRegexChar ')' = "\\)" + escapeRegexChar '{' = "\\{" + escapeRegexChar '}' = "\\}" + escapeRegexChar '*' = "\\*" + escapeRegexChar '+' = "\\+" + escapeRegexChar '.' = "\\." + escapeRegexChar '?' = "\\?" + escapeRegexChar c + | isAscii c && isPrint c = BS.singleton c + | otherwise = BS.pack $ printf "\\x{%x}" (ord c) let escapeRegex = BS.concatMap escapeRegexChar replacement <- getCapture capNum - (escapeRegex (encodeUtf8 replacement) <>) <$> subDynamic (BS.drop 2 bs) + (escapeRegex (encodeUtf8 replacement) <>) <$> + subDynamic (BS.drop 2 bs) _ -> case BS.break (=='%') bs of (y,z) | BS.null y -> BS.cons '%' <$> subDynamic z diff --git a/test/test-skylighting.hs b/test/test-skylighting.hs --- a/test/test-skylighting.hs +++ b/test/test-skylighting.hs @@ -73,7 +73,16 @@ , testGroup "Doesn't hang or drop text on fuzz" $ map (noDropTest randomText) syntaxes , testGroup "Regression tests" $ - [ testCase "perl quoting case" $ Right + let perl = maybe (error "could not find Perl syntax") id + (lookupSyntax "Perl" defaultSyntaxMap) in + [ testCase "perl NUL case" $ Right + [[(KeywordTok,"s\NUL") + ,(OtherTok,"b") + ,(KeywordTok,"\NUL") + ,(StringTok,"c") + ,(KeywordTok,"\NUL")]] + @=? tokenize defConfig perl "s\0b\0c\0" + , testCase "perl quoting case" $ Right [ [ ( KeywordTok , "my" ) , ( NormalTok , " " ) , ( DataTypeTok , "$foo" ) @@ -92,10 +101,7 @@ , ( KeywordTok , "'" ) , ( NormalTok , ";" ) ] - ] @=? tokenize TokenizerConfig{ syntaxMap = defaultSyntaxMap - , traceOutput = False } - (maybe (error "could not find Perl syntax") id - (lookupSyntax "Perl" defaultSyntaxMap)) + ] @=? tokenize defConfig perl "my $foo = q/bar/;\nmy $baz = 'quux';\n" ] ] @@ -138,9 +144,7 @@ Just s -> return s Nothing -> fail $ "Could not find syntax definition for " ++ lang - case tokenize TokenizerConfig{ - traceOutput = False - , syntaxMap = defaultSyntaxMap } syntax $! code of + case tokenize defConfig syntax $! code of Left e -> fail e Right ls -> return $ Text.pack $ ppShow ls ++ "\n" updateGolden = if regen