diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,10 +3,13 @@ ## Unreleased changes ## Unreleased changes -0.2.3.0 Jan 12 2020 +0.3.0.0 Jan 15 2021 + - Support default syntaxes from skylighting. + +0.2.3.0 Jan 12 2021 - fixed minor issue that disabled inline rendering. -0.2.2.0 Jan 12 2020 +0.2.2.0 Jan 12 2021 - Fixed problem with declaring too few columns in LaTeX - Inline code formatting - Removed indent marks from parenthesised operators: "`(+)" diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -14,6 +14,7 @@ import qualified Data.Text as T import qualified Token.Haskell (tokenizer) +import qualified Token.Skylighting import Filter (renderBlock, renderInline) import FindColumns (findColumns) import Alignment (Processed) @@ -77,7 +78,9 @@ -- | Pick tokenizer depending on options. getTokenizer attrs | isHaskell attrs = Token.Haskell.tokenizer - | otherwise = \_ -> Nothing +getTokenizer (_,classes,_) = case Token.Skylighting.lookupTokenizer classes of + Just syntax -> Token.Skylighting.tokenizer syntax + Nothing -> \_ -> Nothing -- | Check if the code block is tagged as Haskell. isHaskell :: (Foldable t, Eq a1, IsString a1) => diff --git a/pandoc-filter-indent.cabal b/pandoc-filter-indent.cabal --- a/pandoc-filter-indent.cabal +++ b/pandoc-filter-indent.cabal @@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack -- --- hash: ad9eff137e5b4615f1392709efb99056fe4ba12434353892298d8e7d8d4099b0 +-- hash: 54950c09cdef3ae347be5565c51f866814024d8f3304665a2873a0acdf7c41a2 name: pandoc-filter-indent -version: 0.2.3.0 +version: 0.3.0.0 synopsis: Pandoc filter formatting Haskell code fragments using GHC lexer. description: Formats marked code fragments, and allows `pandoc` to safely process rest of your literate program: . @@ -58,6 +58,7 @@ Render.Latex Token Token.Haskell + Token.Skylighting Tuples Util other-modules: @@ -73,6 +74,7 @@ , optics-core , optics-th , pandoc-types + , skylighting , text ==1.2.4.0 default-language: Haskell2010 @@ -95,6 +97,7 @@ , optparse-applicative , pandoc-filter-indent , pandoc-types + , skylighting , text ==1.2.4.0 default-language: Haskell2010 @@ -118,5 +121,6 @@ , pandoc-filter-indent , pandoc-types , quickcheck-text + , skylighting , text ==1.2.4.0 default-language: Haskell2010 diff --git a/src/Token/Skylighting.hs b/src/Token/Skylighting.hs new file mode 100644 --- /dev/null +++ b/src/Token/Skylighting.hs @@ -0,0 +1,159 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE ViewPatterns #-} +{-# LANGUAGE FlexibleContexts #-} +-- | Skylighting code tokenizer +module Token.Skylighting(lookupTokenizer, tokenizer) where + +import Control.Arrow(first) +import Text.Pandoc.JSON () +import Text.Pandoc.Definition () +import Data.Maybe(listToMaybe, catMaybes) +import Data.String (IsString) +import Data.Text (Text) +import qualified Data.Text as T +import Prelude hiding(getLine) +import Optics.Core + +import qualified Skylighting.Types as Sky +import Skylighting.Types (TokenType(..), Syntax, SourceLine, Token) +import qualified Skylighting.Syntax as Sky(defaultSyntaxMap) +import qualified Skylighting.Tokenizer as Sky(tokenize, TokenizerConfig(..)) +import qualified Skylighting.Core as Sky(lookupSyntax, syntaxByShortName) + +import Token ( MyLoc(MyLoc), MyTok(..), unTikzMark, mark ) + +rightToMaybe (Left err ) = Nothing +rightToMaybe (Right result) = Just result + +lookupTokenizer :: [Text] -> Maybe Syntax +lookupTokenizer = listToMaybe + . catMaybes + . fmap (Sky.syntaxByShortName Sky.defaultSyntaxMap) + +-- * Haskell tokenizer frontend +-- | Attempt to tokenize input, +-- returns `Nothing` if unsuccessful, +-- so the processor can just pass input +-- further when tokenizer fails. +tokenizer :: Syntax -- Skylighting syntax description + -> Text -- ^ Input text of code block + -> Maybe [(MyTok, MyLoc, Text)] +tokenizer syntax = + fmap ( joinEscapedOperators + . splitTokens + . restoreLocations + . recognizeTokens ) + . rightToMaybe + . Sky.tokenize tokenizerOpts syntax + where + tokenizerOpts = Sky.TokenizerConfig Sky.defaultSyntaxMap False + +-- | Recognize tokens from all source lines. +recognizeTokens :: [SourceLine] -> [[(MyTok, Text)]] +recognizeTokens = map $ map $ first skyTok + +-- | Convert token type of `ghc-lib` into tokens recognized by the filter. +skyTok :: TokenType -> MyTok +skyTok FloatTok = TNum +skyTok FloatTok = TNum +skyTok DecValTok = TNum +skyTok BaseNTok = TNum +skyTok StringTok = TOther +skyTok VerbatimStringTok = TOther +skyTok SpecialStringTok = TOther +skyTok ConstantTok = TOther +skyTok KeywordTok = TKeyword +skyTok BuiltInTok = TKeyword +skyTok CharTok = TOther +skyTok SpecialCharTok = TOther +skyTok CommentTok = TBlank +skyTok DocumentationTok = TBlank +skyTok CommentTok = TOther +skyTok OperatorTok = TOperator +skyTok SpecialCharTok = TOperator +skyTok RegionMarkerTok = TOperator +skyTok AnnotationTok = TOther +skyTok ControlFlowTok = TKeyword +skyTok VariableTok = TVar +skyTok DataTypeTok = TCons +skyTok other = TOther + +-- FIXME: generalize for GHC tokenizer and Skylighting +-- | Restore locations +-- TESTME: test +-- 1. Without newlines should return a list of indices up to length +-- 2. Of the same length as number of tokens +-- 3. With newlines should return line indices up to number of lines. +-- 4. Same for a list of lists of words without newlines joined as lines +restoreLocations :: [[(MyTok, Text)]] -> [(MyTok, MyLoc, Text)] +restoreLocations srcLines = concat + $ zipWith (`go` 1) [1..] srcLines + where + go line col [] = [] + go line col ((tok, txt):ls) = + (tok, MyLoc line col (isMark tok), txt):go newLine newCol ls + where + isMark TBlank = False + isMark _ = True + newLine = line + lineIncr + lineIncr = T.length $ T.filter (=='\n') txt + newCol | lineIncr == 0 = col + T.length txt + | otherwise = (+1) + $ T.length + $ fst + $ T.break (=='\n') + $ T.reverse txt + +-- * Likely common with other tokenizers +-- | Split tokens into one blank per line. +-- TESTME: assures that no token has '\n' before the end of text. +splitTokens :: [(MyTok, MyLoc, Text)] -> [(MyTok, MyLoc, Text)] +splitTokens = mconcat + . fmap splitter + where + splitter :: (MyTok, MyLoc, Text) -> [(MyTok, MyLoc, Text)] + splitter (TBlank, loc@(MyLoc line _ _), txt) | T.filter (=='\n') txt /= "" = + withLocs withNewLines + where + split, withNewLines :: [Text] + split = T.lines txt + withNewLines = fmap (<>"\n") (init split) + <> [last split] + withLocs :: [Text] -> [(MyTok, MyLoc, Text)] + withLocs (l:ls) = (TBlank, set mark True $ loc, l) + : zipWith mkEntry [line+1..] ls + mkEntry :: Int -> Text -> (MyTok, MyLoc, Text) + mkEntry i t = (TBlank, MyLoc i 1 True, t) + splitter other@(_, loc@(MyLoc line 1 x), txt) = [set (_2 % mark) True other] + splitter other = [other] + + +unmark :: Field2 a a MyLoc MyLoc => a -> a +unmark = set (_2 % mark) False + +-- FIXME: use no-indent-mark instead. +joinEscapedOperators :: (Eq c, IsString c, Semigroup c) => [(MyTok, MyLoc, c)] -> [(MyTok, MyLoc, c)] +joinEscapedOperators (a@(_, _, "("):b@(_, _, _):c@(_, _, ")"):rest) = + a:unmark b:unmark c:joinEscapedOperators rest +joinEscapedOperators (a@(_, loc, "`"):b@(_, _, _):c@(_, _, "`"):rest) = + a:unmark b:unmark c:joinEscapedOperators rest +joinEscapedOperators (a@(_, _, "("):b@(TOperator, _, _):rest) = + a:unmark b:joinEscapedOperators rest +joinEscapedOperators (a@(_, _, _):b@(_, _, ")"):rest) = + a:unmark b:joinEscapedOperators rest +joinEscapedOperators (tok:rest) = tok:joinEscapedOperators rest +joinEscapedOperators [] = [] + +{- + +-- | Recognize token using both token type from `ghc-lib`, +-- and text content. +-- Only TikZ marks are recognized by looking up text content. +recognizeToken :: [Token] -> (MyTok, Text) +recognizeToken (CommentTok, tokText@(unTikzMark -> Just mark)) = + (TTikz mark, tokText) +recognizeToken (tokType, tokText) = + (skyTok tokType, tokText) + -} +