diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## GHC syntax highlighter 0.0.8.0 + +* Uses `ghc-lib-parser-9.2.1.x`. + ## GHC syntax highlighter 0.0.7.0 * Uses `ghc-lib-parser-9.0.1.x`. diff --git a/GHC/SyntaxHighlighter.hs b/GHC/SyntaxHighlighter.hs --- a/GHC/SyntaxHighlighter.hs +++ b/GHC/SyntaxHighlighter.hs @@ -16,8 +16,8 @@ -- The module allows you to decompose a 'Text' stream containing Haskell -- source code into a stream of 'Text' chunks tagged with 'Token'. -- --- This library uses GHC's lexer, so the result is guaranteed to be 100% --- correct, as if it was parsed by GHC itself. +-- This library uses the GHC's lexer, so the result is guaranteed to be 100% +-- correct, as if it were parsed by GHC itself. module GHC.SyntaxHighlighter ( Token (..), Loc (..), @@ -27,20 +27,16 @@ where import Control.Monad -import Data.List (foldl', unfoldr) +import Data.List (unfoldr) import Data.Maybe (isJust) import Data.Text (Text) import qualified Data.Text as T import qualified GHC.Data.EnumSet as ES import GHC.Data.FastString (mkFastString) import GHC.Data.StringBuffer -import GHC.Driver.Session as DynFlags import GHC.LanguageExtensions import qualified GHC.Parser.Lexer as L -import GHC.Settings import GHC.Types.SrcLoc -import GHC.Utils.Fingerprint (fingerprint0) -import GHC.Version (cProjectVersion) ---------------------------------------------------------------------------- -- Data types @@ -75,7 +71,7 @@ OtherTok deriving (Eq, Ord, Enum, Bounded, Show) --- | Start and end positions of a span. The arguments of the data +-- | The start and end positions of a span. The arguments of the data -- constructor contain in order: -- -- * Line number of start position of a span @@ -92,7 +88,7 @@ -- | Tokenize Haskell source code. If the code cannot be parsed, return -- 'Nothing'. Otherwise return the original input tagged by 'Token's. --- 'Nothing' is rarely returned if ever, because it looks like the lexer is +-- 'Nothing' is rarely returned, if ever, because it looks like the lexer is -- capable of interpreting almost any text as a stream of GHC tokens. -- -- The parser does not require the input source code to form a valid Haskell @@ -134,35 +130,17 @@ where location = mkRealSrcLoc (mkFastString "") 1 1 buffer = stringToStringBuffer (T.unpack input) - parseState = L.mkPStatePure parserFlags buffer location - parserFlags = L.mkParserFlags (foldl' xopt_set initialDynFlags enabledExts) - initialDynFlags = - DynFlags - { warningFlags = ES.empty, - generalFlags = - ES.fromList - [ Opt_Haddock, - Opt_KeepRawTokenStream - ], - extensions = [], - extensionFlags = ES.empty, - safeHaskell = Sf_Safe, - language = Just Haskell2010, - ghcNameVersion = - GhcNameVersion - { ghcNameVersion_programName = "ghc", - ghcNameVersion_projectVersion = cProjectVersion - }, - fileSettings = FileSettings {}, - toolSettings = - ToolSettings - { toolSettings_opt_P_fingerprint = fingerprint0, - toolSettings_pgm_F = "" - }, - platformMisc = PlatformMisc {} - } + parseState = L.initParserState parserOpts buffer location + parserOpts = + L.mkParserOpts + ES.empty + (ES.fromList enabledExts) + True -- safe imports + True -- keep Haddock tokens + True -- keep comment tokens + False -- lex LINE and COLUMN pragmas --- | Haskell lexer. +-- | The Haskell lexer. pLexer :: L.P [(Token, Loc)] pLexer = go where @@ -175,12 +153,12 @@ Nothing -> go Just x -> (x :) <$> go --- | Convert @'Located' 'L.Token'@ representation to a more convenient for +-- | Convert @'Located' 'L.Token'@ representation into a more convenient for -- us form. fixupToken :: Located L.Token -> Maybe (Token, Loc) fixupToken (L srcSpan tok) = (classifyToken tok,) <$> srcSpanToLoc srcSpan --- | Convert 'SrcSpan' to 'Loc'. +-- | Convert 'SrcSpan' into 'Loc'. srcSpanToLoc :: SrcSpan -> Maybe Loc srcSpanToLoc (RealSrcSpan s _) = let srcSpanSLine = srcSpanStartLine s @@ -268,7 +246,6 @@ L.ITline_prag _ -> PragmaTok L.ITcolumn_prag _ -> PragmaTok L.ITscc_prag _ -> PragmaTok - L.ITgenerated_prag _ -> PragmaTok L.ITunpack_prag _ -> PragmaTok L.ITnounpack_prag _ -> PragmaTok L.ITann_prag _ -> PragmaTok @@ -319,6 +296,7 @@ L.ITbackquote -> SymbolTok L.ITsimpleQuote -> SymbolTok L.ITpercent -> SymbolTok + L.ITproj _ -> SymbolTok -- NOTE GHC thinks these are reserved symbols, but I classify them as -- operators. L.ITminus -> OperatorTok @@ -374,13 +352,13 @@ L.ITunknown _ -> OtherTok L.ITeof -> OtherTok -- normally is not included in results -- Documentation annotations - L.ITdocCommentNext _ -> CommentTok - L.ITdocCommentPrev _ -> CommentTok - L.ITdocCommentNamed _ -> CommentTok - L.ITdocSection _ _ -> CommentTok - L.ITdocOptions _ -> CommentTok - L.ITlineComment _ -> CommentTok - L.ITblockComment _ -> CommentTok + L.ITdocCommentNext {} -> CommentTok + L.ITdocCommentPrev {} -> CommentTok + L.ITdocCommentNamed {} -> CommentTok + L.ITdocSection {} -> CommentTok + L.ITdocOptions {} -> CommentTok + L.ITlineComment {} -> CommentTok + L.ITblockComment {} -> CommentTok ---------------------------------------------------------------------------- -- Text traversing diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Stackage LTS](http://stackage.org/package/ghc-syntax-highlighter/badge/lts)](http://stackage.org/lts/package/ghc-syntax-highlighter) ![CI](https://github.com/mrkkrp/ghc-syntax-highlighter/workflows/CI/badge.svg?branch=master) -This is a syntax highlighter library for Haskell using lexer of GHC itself. +This is a syntax highlighter library for Haskell using the lexer of GHC. Here is a blog post announcing the package, the readme is mostly derived from it: @@ -16,9 +16,10 @@ ## Motivation Parsing Haskell is hard, because Haskell is a complex language with -countless features. The only way to get it right 100% is to use parser of -GHC itself. Fortunately, now there is the [`ghc`][ghc] package, which as of -version 8.4.1 exports enough of GHC's source code to allow us use its lexer. +countless features. The only way to get it right 100% is to use the parser +of GHC itself. Fortunately, now there is the [`ghc`][ghc] package, which as +of version 8.4.1 exports enough of GHC's source code to allow us use its +lexer. Alternative approaches, even decent ones like [`highlight.js`][hljs] either don't support cutting-edge features or do their work without sufficient diff --git a/ghc-syntax-highlighter.cabal b/ghc-syntax-highlighter.cabal --- a/ghc-syntax-highlighter.cabal +++ b/ghc-syntax-highlighter.cabal @@ -1,15 +1,15 @@ cabal-version: 1.18 name: ghc-syntax-highlighter -version: 0.0.7.0 +version: 0.0.8.0 license: BSD3 license-file: LICENSE.md maintainer: Mark Karpov author: Mark Karpov -tested-with: ghc ==8.8.4 ghc ==8.10.4 ghc ==9.0.1 +tested-with: ghc ==8.10.7 ghc ==9.0.1 ghc ==9.2.1 homepage: https://github.com/mrkkrp/ghc-syntax-highlighter bug-reports: https://github.com/mrkkrp/ghc-syntax-highlighter/issues -synopsis: Syntax highlighter for Haskell using lexer of GHC itself -description: Syntax highlighter for Haskell using lexer of GHC itself. +synopsis: Syntax highlighter for Haskell using the lexer of GHC +description: Syntax highlighter for Haskell using the lexer of GHC. category: Text build-type: Simple data-files: data/*.hs @@ -31,7 +31,7 @@ default-language: Haskell2010 build-depends: base >=4.13 && <5.0, - ghc-lib-parser >=9.0 && <9.1, + ghc-lib-parser >=9.2 && <9.3, text >=0.2 && <1.3 if flag(dev)