skylighting-core 0.10 → 0.10.0.1
raw patch · 4 files changed
+15/−3 lines, 4 files
Files
- changelog.md +4/−0
- skylighting-core.cabal +1/−1
- src/Skylighting/Tokenizer.hs +3/−2
- test/test-skylighting.hs +7/−0
changelog.md view
@@ -1,5 +1,9 @@ # Revision history for skylighting and skylighting-core +## 0.10.0.1++ * Fix identifier detection in non-ASCII context (#110).+ ## 0.10 * Add instructions for submitting patches upstream to KDE (#106).
skylighting-core.cabal view
@@ -1,5 +1,5 @@ name: skylighting-core-version: 0.10+version: 0.10.0.1 synopsis: syntax highlighting library description: Skylighting is a syntax highlighting library. It derives its tokenizers from XML syntax
src/Skylighting/Tokenizer.hs view
@@ -495,9 +495,10 @@ detectIdentifier :: ByteString -> TokenizerM Text detectIdentifier inp = do case BS.uncons inp of- Just (c, t) | isLetter c || c == '_' ->+ Just (c, t) | (isAscii c && isLetter c) || c == '_' -> takeChars $ 1 + maybe (BS.length t) id- (BS.findIndex (\d -> not (isAlphaNum d || d == '_')) t)+ (BS.findIndex (\d -> isAscii d &&+ not (isAlphaNum d || d == '_')) t) _ -> mzero lineContinue :: ByteString -> TokenizerM Text
test/test-skylighting.hs view
@@ -110,6 +110,8 @@ , testGroup "Regression tests" $ let perl = maybe (error "could not find Perl syntax") id (lookupSyntax "Perl" sMap)+ html = maybe (error "could not find HTML syntax") id+ (lookupSyntax "html" sMap) cpp = maybe (error "could not find CPP syntax") id (lookupSyntax "cpp" sMap) c = maybe (error "could not find C syntax") id@@ -170,6 +172,11 @@ , testCase "c very long integer (#81)" $ Right [ [ (DecValTok, "1111111111111111111111") ] ] @=? tokenize defConfig c "1111111111111111111111"++ , testCase "Chinese characters in HTML (#110)" $ Right+ [ [ ( NormalTok , "\35797\65306" ) , ( KeywordTok , "<a>" ) ]+ ] @=? tokenize defConfig html "试：<a>"+ ] ]