pandoc-filter-indent 0.2.0.0 → 0.2.1.0

raw patch · 15 files changed

+120/−40 lines, 15 files

Files

• ChangeLog.md +6/−2
• app/Main.hs +21/−12
• pandoc-filter-indent.cabal +2/−2
• src/Alignment.hs +3/−0
• src/Filter.hs +10/−13
• src/FindColumns.hs +10/−0
• src/Render/ColSpan.hs +6/−0
• src/Render/Common.hs +2/−1
• src/Render/Debug.hs +5/−0
• src/Render/HTML.hs +8/−3
• src/Render/Latex.hs +12/−2
• src/Token.hs +13/−3
• src/Token/Haskell.hs +12/−2
• src/Tuples.hs +3/−0
• src/Util.hs +7/−0

@@ -1,9 +1,13 @@ # Changelog for pandoc-filter-indent  ## Unreleased changes-0.0.1 Dec 1 2020+0.1.0.0 Dec 1 2020   - Initial release supporting LaTeX and HTML output -0.0.2 Dec 2 2020+0.2.0.0 Dec 2 2020   - Preliminary support for Tikzmarks   - Updated README++0.2.1.0 Dec 2 2020+  - Extended code documentation+  - Cleanups

@@ -1,14 +1,16 @@ {-# LANGUAGE OverloadedStrings #-}-+{-# LANGUAGE ViewPatterns      #-} module Main where  import           Text.Pandoc.JSON import           Text.Pandoc.Definition () import           Data.String (fromString, IsString)-import           Data.Text (Text)+import           Data.Maybe  (fromMaybe)+import           Data.Text   (Text) import qualified Data.Text as T---import Debug.Trace(trace) +import Debug.Trace(trace)+ import Token.Haskell import GHC.SyntaxHighlighter import Filter@@ -19,19 +21,26 @@  -- | Select the desired format output then process it. blockFormatter :: Maybe Format -> Block -> Block-blockFormatter  Nothing               (CodeBlock attrs content) = -- debugging mode-    codeFormatter "text" attrs content-blockFormatter (Just (Format format)) (CodeBlock attrs content)-    | isHaskell attrs = codeFormatter format attrs content+blockFormatter format (CodeBlock attrs content) =+    codeFormatter (fromMaybe (Format "text") format) attrs content -- Do not touch other blocks than 'CodeBlock'-blockFormatter _ x = x+blockFormatter _       x                        = x --- | Select formatter-codeFormatter :: Text -> Attr -> Text -> Block+-- | Run tokenizer, analysis, and formatter.+--   Fallback to original input, if failed to tokenize.+codeFormatter :: Format -- ^ Output format, defaults to "text" if not found+              -> Attr -- ^ Code block attributes+              -> Text -- ^ Code block content+              -> Block codeFormatter format attrs content =-        render format attrs (filterCodeBlock content)+  case fmap findColumns $ aTokenizer content of+    Just processed -> render format attrs processed+    Nothing        -> CodeBlock attrs content -- fallback+  where+    aTokenizer | isHaskell attrs = Token.Haskell.tokenizer+               | otherwise       = \_ -> Nothing ---  (Text, [Text], [(Text, Text)])+-- | Check if the code block is tagged as Haskell. isHaskell :: (Foldable t, Eq a1, IsString a1) =>                    (a2, t a1, c) -> Bool isHaskell (_, classes, _) = "haskell" `elem` classes

@@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack ----- hash: 3a5612ddfa28673f5b83117e2fde59d631d99c9de58e1b04aaddaf53c7559d58+-- hash: 556363e65a61ebed2e66c3756278177ca371c24b119a21fbad59d9b26059d56e  name:           pandoc-filter-indent-version:        0.2.0.0+version:        0.2.1.0 synopsis:       Pandoc filter formatting Haskell code fragments using GHC lexer. description:    Formats marked code fragments, and allows `pandoc` to safely process rest of your literate program:                 .

@@ -1,4 +1,7 @@ {-# LANGUAGE FlexibleContexts #-}+-- | Alignment options,+--   and extracting common columns+--   from tuples used in analysis. module Alignment where  import Data.Text (Text)

@@ -4,6 +4,7 @@ {-# LANGUAGE ViewPatterns          #-} {-# LANGUAGE FlexibleInstances     #-} {-# LANGUAGE FlexibleContexts      #-}+-- | Filtering a single code block. module Filter where  import Text.Pandoc.JSON@@ -21,28 +22,24 @@  import Debug.Trace(trace) -filterCodeBlock = withTokens findColumns ("haskell", tokenizer)---- | Apply function if tokenization succeeded, otherwise return same output-withTokens :: Show a => (t -> p) -> (String, a -> Maybe t) -> a -> p-withTokens f (tokenizerName, tokenizer) src@(tokenizer -> Nothing) = error $ mconcat ["Tokenizer ", tokenizerName, " failed for ", show src]-withTokens f (tokenizerName, tokenizer)     (tokenizer -> Just tokens) = f tokens--render ::  Text       -- ^ Format string+-- | Render a list of `Processed` token records into the target output format.+render ::  Format     -- ^ Format string        ->  Attr       -- ^ Attributes        -> [Processed] -- ^ Data about alignment        ->  Block --render "text" attrs aligned = RawBlock (Format "latex") $ processLatex aligned -- debug-render "text"  attrs = CodeBlock attrs           . Render.Debug.render-render "latex" attrs = RawBlock (Format "latex") . processLatex-render "html"  attrs = RawBlock (Format "html" ) . processHTML+render (Format "text" ) attrs = CodeBlock attrs           . Render.Debug.render+render (Format "latex") attrs = RawBlock (Format "latex") . processLatex+render (Format "html" ) attrs = RawBlock (Format "html" ) . processHTML -- Debugging option render other   attrs = CodeBlock attrs . T.pack . show +-- | Convert a list of input token records to raw LaTeX. processLatex :: [Processed] -> T.Text processLatex processed = Render.Latex.latexFromColSpans (length $ tableColumns processed)                        $ colspans processed +-- | Convert a list of input token records to raw HTML. processHTML :: [Processed] -> T.Text-processHTML processed = Render.HTML.htmlFromColSpans (length $ tableColumns processed)-                      $ colspans processed+processHTML  = Render.HTML.htmlFromColSpans+             . colspans

@@ -4,6 +4,7 @@ {-# LANGUAGE TypeSynonymInstances  #-} {-# LANGUAGE FlexibleInstances     #-} {-# LANGUAGE FlexibleContexts      #-}+-- | Find alignment columns. module FindColumns(findColumns, getLine, getCol, getLineCol, alignPos, tableColumns) where  import Text.Pandoc.JSON@@ -42,6 +43,7 @@ align :: Field5 a a (Maybe b) (Maybe b) => Lens' a (Maybe b) align  = _5 +-- | From a record with alignment option, select this option. alignPos :: Field5 a a (Maybe (Align, Int)) (Maybe (Align, Int)) => Lens' a (Maybe (Align, Int)) alignPos  = _5 @@ -58,8 +60,10 @@     . map addLineIndent     . grouping getLine +-- | Extract both line and column where a token starts. getLineCol x = (getLine x, getCol x) +-- | Mark alignment boundaries. markBoundaries :: [Unanalyzed] -> [Aligned] markBoundaries = map markIndent                . concat@@ -79,9 +83,12 @@            (myTok,  myLoc              , txt, Just indent, Just ALeft) markIndent other                                                               = other +-- | Append alignment option to unanalyzed token record. withAlign :: Maybe Align -> Unanalyzed -> Aligned withAlign  = flip annex +-- | Given a list of unanalyzed token records in a single column,+--   check the alignment option that applies to this column. alignBlock :: [Unanalyzed] -> [Aligned] alignBlock [a]                            = withAlign  Nothing       <$> [a] alignBlock opList | all isOperator opList = withAlign (Just ACenter) <$> opList@@ -120,6 +127,7 @@     hasAlignment (_, Nothing) = False     hasAlignment (_, Just _)  = True +-- | Extract information about depth of indent in a line where token is. getIndent = view _4  -- | Detect uninterrupted stretches that cover consecutive columns.@@ -150,6 +158,8 @@     extractColumn  []                           = Nothing     extractColumn  ((_,   MyLoc line col, _):_) = Just col +-- | Given a list of aligned records, and a list of marker columns,+--   add table column indices to each record. columnIndices :: ([Aligned], [(Int, b)]) -> [Processed] columnIndices (allAligned, map fst -> markerColumns) = map addIndex allAligned   where

@@ -2,6 +2,9 @@ {-# LANGUAGE PartialTypeSignatures #-} {-# LANGUAGE FlexibleInstances     #-} {-# LANGUAGE FlexibleContexts      #-}+-- | Helper functions for taking a list of+--   processed records, and returning+--   the same list splitted into separate colspans. module Render.ColSpan where  import Data.Function(on)@@ -16,6 +19,8 @@ import Token(MyTok) import Util ( maybeLens ) +-- | A list of tokens to be put into a single colspan,+--   Number of table columns in this colspan, and alignment option. type TokensWithColSpan = ([(MyTok, Text)], Int, Align)  -- | Find colspan parameters@@ -44,6 +49,7 @@     extractToken tok = (view tokenType tok, view textContent tok) -- FIXME: split lines before colspans! +-- | Given a `Processed` record, extract number of table columns. getAlignCol :: Processed -> Int getAlignCol = view (alignPos % maybeLens (ALeft, 0) % _2) 

@@ -1,8 +1,9 @@+-- | Common definitions shared by Render modules. module Render.Common where  import Data.Text(Text) import Alignment import Token -type TextWithColSpan = (Text, Int, Align)+-- | A single colspan with its tokens. type TokensWithColSpan = ([(MyTok,Text)], Int, Align)

@@ -16,16 +16,21 @@ import Alignment ( textContent, Align(..), Processed ) import Util ( safeTail ) +-- | Insert element at a given index of the list. insertAt       :: Show a => Int -> a -> [a] -> [a] insertAt i e ls = case maybeInsertAt i e ls of                     Nothing -> error $ "Failed in insertAt " <> show i <> " " <> show e <> " " <> show ls                     Just r  -> r +-- | Insert element at a given index of the list, or return error. maybeInsertAt :: Int -> a -> [a] -> Maybe [a] maybeInsertAt 0 e    ls  = pure $ e:ls maybeInsertAt i e (l:ls) = (l:) <$> maybeInsertAt (i-1) e ls maybeInsertAt i e    []  = Nothing +-- | Render text of code blocks+--   with marker columns inserted+--   to indicate alignment boundaries. render :: [Processed] -> Text render ps = T.concat $ go ps   where

@@ -1,5 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ViewPatterns      #-}+-- | Render analyzed tokens into HTML table. module Render.HTML(htmlFromColSpans) where  import Prelude hiding(span, id)@@ -15,19 +16,22 @@ import Token        (MyTok(..)) import Util ( preformatTokens, unbrace ) -htmlFromColSpans ::   p-                 -> [[TokensWithColSpan]]+-- | Given a list of lists of colspans in each table row, return an HTML text.+htmlFromColSpans :: [[TokensWithColSpan]]                  ->   Text-htmlFromColSpans cols =+htmlFromColSpans =     LT.toStrict   . renderHtml   . table   . tbody   . mapM_ renderTr +-- | Given a list of colspans within a single table row, render it to HTML tree+--   of `<tr/>` element renderTr :: [TokensWithColSpan] -> Html renderTr colspans = tr (mapM_ renderColSpan colspans) +-- | Render single colspan as a single `<td/>` cell. renderColSpan :: TokensWithColSpan -> Html renderColSpan ([(TBlank, txt)], colSpan, AIndent) = -- indentation     td (toHtml txt)@@ -45,6 +49,7 @@     alignMark ALeft   = "left"  -- TODO: braced operators+-- | Given a list of tokens in a colspan, render HTML fragment. formatTokens :: [(MyTok, Text)] -> Html formatTokens  = mapM_ formatToken               . preformatTokens

@@ -1,5 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ViewPatterns      #-}+-- | Render analyzed input into LaTeX table. module Render.Latex(latexFromColSpans) where  import Data.Text(Text)@@ -11,8 +12,9 @@ import Token(MyTok(..)) import Util(unbrace) ---import Debug.Trace(trace)-+-- | Given a number of table columns,+--   and a list of lists of colspans for each table row,+--   return raw LaTeX code. latexFromColSpans :: Int -> [[TokensWithColSpan]] -> Text latexFromColSpans cols =     wrapTable cols@@ -21,6 +23,7 @@          . T.intercalate " & "          . fmap renderColSpan ) +-- | Render a single colspan as LaTeX \multicolumn. renderColSpan :: TokensWithColSpan -> Text renderColSpan ([(TBlank, txt)], colSpan, AIndent) = -- indentation     T.concat [ "\\multicolumn{",    T.pack $ show colSpan@@ -36,6 +39,7 @@     alignMark ACenter = "c"     alignMark ALeft   = "l" +-- | Wrap a LaTeX table content into \begin{tabular} environment. wrapTable :: Int -> Text -> Text wrapTable cols txt =   mconcat [-- "\\newlength{\\tabcolsepBACKUP}\n"@@ -50,10 +54,14 @@  -- Decrease column spacing: \\setlength{\\tabcolsep}{1ex} -- TODO: braced operators+-- | Preprocesses functions converted to operator syntax and joins them into a single token.+-- FIXME: deduplicate preformatTokens []                                                     = [] preformatTokens ((TOperator,"`"):(TVar, "elem"):(TOperator, "`"):rest) = (TOperator, "elem"):preformatTokens rest preformatTokens (a                                              :rest) =  a                 :preformatTokens rest +-- | Format a list of tokens within a colspan.+--   Preprocesses then and calls `formatToken` for each. formatTokens :: [(MyTok, Text)] -> Text formatTokens  = T.concat               . fmap formatToken@@ -61,6 +69,8 @@               -- . (\t -> trace ("Tokens: " <> show t) t) -- debug  -- Workaround with joinEscapedOperators til w consider spaces only.+-- | Render a simple token.+formatToken :: (MyTok, Text) -> Text formatToken (TOperator,unbrace -> Just op) = "(" <> formatToken (TOperator, op) <> ")" formatToken (TKeyword, "forall") = mathop "forall" formatToken (TVar,     "mempty") = mathop "emptyset"

@@ -13,8 +13,11 @@ import           Optics.TH  -- * Common tokens and locations--- Location is just line and column-data MyLoc = MyLoc { _line, _col :: Int }+--   We keep them here, so we can translate output from tokenizers to common format.+-- | Location is just line and column (not a slice.)+data MyLoc = MyLoc { _line :: Int -- ^ Line number starting from 1+                   , _col  :: Int -- ^ Column number starting from 1+                   }   deriving (Eq, Ord, Show)  makeLenses ''MyLoc@@ -34,13 +37,20 @@ -- | Records tokenized and converted to common token format. type Tokenized = (MyTok, MyLoc, Text) +-- | Unpack a Haskell comment with a TikZ mark indicator. unTikzMark    :: Text -> Maybe Text unTikzMark txt =   unwrap "{->" "-}" txt >>= \case     ""   -> Nothing     mark -> Just mark -unwrap :: Text -> Text -> Text -> Maybe Text+-- | Given opening text, and closing text,+--   check that input is "braced" by these, and strip them.+--   Return `Nothing` if input text does not match.+unwrap :: Text -- ^ Opening text+       -> Text -- ^ Closing text+       -> Text -- ^ Input to match+       -> Maybe Text unwrap starter trailer   txt  |   starter `T.isPrefixOf` txt &&   trailer `T.isPrefixOf` txt  =

@@ -21,19 +21,27 @@ import Token  -- * Haskell tokenizer frontend-tokenizer :: Text -> Maybe [(MyTok, MyLoc, Text)]+-- | Attempt to tokenize input,+--   returns `Nothing` if unsuccessful,+--   so the processor can just pass input+--   further when tokenizer fails.+tokenizer :: Text -- ^ Input text of code block+          -> Maybe [(MyTok, MyLoc, Text)] tokenizer  = fmap ( joinEscapedOperators                   . splitTokens                   . restoreLocations                   . fmap recognizeToken)            . tokenizeHaskell -+-- | Recognize token using both token type from `ghc-lib`,+--   and text content.+--   Only TikZ marks are recognized by looking up text content. recognizeToken (CommentTok, tokText@(unTikzMark -> Just mark)) =   (TTikz mark,           tokText) recognizeToken (tokType, tokText) =   (haskellTok   tokType, tokText) +-- | Convert token type of `ghc-lib` into tokens recognized by the filter. haskellTok SpaceTok       = TBlank haskellTok CommentTok     = TBlank haskellTok PragmaTok      = TBlank@@ -45,7 +53,9 @@ haskellTok IntegerTok     = TNum haskellTok t              = TOther +-- | Extract line number from `ghc-lib` slice location. locLine (Loc startLineNo startColNo _ _) = startLineNo+-- | Extract column number from `ghc-lib` slice location. locCol  (Loc startLineNo startColNo _ _) = startColNo  -- | Split tokens into one blank per line.

@@ -1,7 +1,10 @@ {-# LANGUAGE FunctionalDependencies #-} {-# LANGUAGE FlexibleInstances      #-}+-- | Tuple utilities. module Tuples where +-- | Class of tuples that can be expanded+--   by a single element. class Annex c b a | c -> a, c -> b where   annex :: b -> a -> c 

@@ -6,6 +6,7 @@ {-# LANGUAGE TypeSynonymInstances  #-} {-# LANGUAGE FlexibleInstances     #-} {-# LANGUAGE FlexibleContexts      #-}+-- | Shared utilities that may be moved to upstream libraries. module Util where  import           Data.Function(on)@@ -46,9 +47,15 @@ safeTail []     = [] safeTail (_:ls) = ls +-- | Take text in braces, and return its inner part.+--   Fail if the given text does not start with opening brace,+--   and end in closing brace. unbrace txt | T.head txt == '(' && T.last txt == ')' && T.length txt > 2 = Just $ T.tail $ T.init txt unbrace _ = Nothing +-- | Preprocess tokens before formatting+--   in order to detect tokens like functions converted to operator syntax.+--   These are merged into a single token. preformatTokens []                                                     = [] preformatTokens ((TOperator,"`"):(TVar, "elem"):(TOperator, "`"):rest) = (TOperator, "elem"):preformatTokens rest preformatTokens (a                                              :rest) =  a                 :preformatTokens rest