diff --git a/INSTALL.md b/INSTALL.md --- a/INSTALL.md +++ b/INSTALL.md @@ -215,66 +215,46 @@ `pandoc` executable into `~/.local/bin`, which you should add to your `PATH`. This process will take a while, and will consume a considerable amount of disk space. + If you also want the `pandoc-server` executable, add + `--flag pandoc:server` to the above command. + ### Quick cabal method -1. Install the [Haskell platform]. This will give you [GHC] and - the [cabal-install] build tool. Note that pandoc requires - GHC >= 7.10 and cabal >= 2.0. +1. Install [ghcup](https://www.haskell.org/ghcup/install/). + This will give you `ghc` and `cabal`. 2. Update your package database: cabal update -3. Check your cabal version with - - cabal --version - - If you have a version less than 2.0, install the latest with: - - cabal install cabal-install - -4. Use `cabal` to install pandoc and its dependencies: +3. Use `cabal` to install pandoc and its dependencies: cabal install pandoc This procedure will install the released version of pandoc, which will be downloaded automatically from HackageDB. + The `pandoc` executable will be placed in `$HOME/.cabal/bin` + on linux/unix/macOS and in `%APPDATA%\cabal\bin` on Windows. + Make sure this directory is in your path. + If you also want the `pandoc-server` executable, add + `-fserver` to the above command. + If you want to install a modified or development version of pandoc instead, switch to the source directory and do as above, but without the 'pandoc': cabal install -5. Make sure the `$CABALDIR/bin` directory is in your path. You should - now be able to run `pandoc`: +4. You should now be able to run `pandoc`: pandoc --help - [Not sure where `$CABALDIR` is?](https://wiki.haskell.org/Cabal-Install#The_cabal-install_configuration_file) - -5. By default `pandoc` uses the "i;unicode-casemap" method - to sort bibliography entries (RFC 5051). If you would like to - use the locale-sensitive unicode collation algorithm instead, - specify the `icu` flag (which affects the dependency `citeproc`): - - cabal install pandoc -ficu - - Note that this requires the `text-icu` library, which in turn - depends on the C library `icu4c`. Installation directions - vary by platform. Here is how it might work on macOS with Homebrew: - - brew install icu4c - stack install pandoc \ - --flag "citeproc:icu" \ - --extra-lib-dirs=/usr/local/opt/icu4c/lib \ - --extra-include-dirs=/usr/local/opt/icu4c/include - -6. The `pandoc.1` man page will be installed automatically. cabal shows - you where it is installed: you may need to set your `MANPATH` - accordingly. If `MANUAL.txt` has been modified, the man page can be - rebuilt: `make man/pandoc.1`. +5. Cabal does not install the `pandoc.1` man page, but you can + copy it from the `man/` directory of the source code to + `/usr/local/share/man/man1/` or wherever man pages go on + your system. ### Custom cabal method @@ -312,6 +292,8 @@ - `lua53`: embed support for Lua 5.3 instead of 5.4. + - `server`: build the `pandoc-server` executable. + 3. Build: cabal build @@ -321,19 +303,6 @@ cabal haddock --html-location=URL --hyperlink-source -5. Copy the files: - - cabal copy --destdir=PATH - - The default destdir is `/`. - -6. Register pandoc as a GHC package: - - cabal register - - Package managers may want to use the `--gen-script` option to - generate a script that can be run to register the package at - install time. ### Creating a relocatable binary diff --git a/MANUAL.txt b/MANUAL.txt --- a/MANUAL.txt +++ b/MANUAL.txt @@ -1,7 +1,7 @@ --- title: Pandoc User's Guide author: John MacFarlane -date: August 3, 2022 +date: August 18, 2022 --- # Synopsis @@ -7050,6 +7050,25 @@ Some document formats also include a unique identifier. For EPUB, this can be set explicitly by setting the `identifier` metadata field (see [EPUB Metadata], above). + +# Running pandoc as a web server + +If you rename (or symlink) the pandoc executable to +`pandoc-server`, it will start up a web server with a JSON +API. This server exposes most of the conversion functionality +of pandoc. For full documentation, see the [pandoc-server] +man page. + +If you rename (or symlink) the pandoc executable to +`pandoc-server.cgi`, it will function as a CGI program +exposing the same API as `pandoc-server`. + +`pandoc-server` is designed to be maximally secure; it uses +Haskell's type system to provide strong guarantees that no I/O +will be performed on the server during pandoc conversions. + +[pandoc-server]: https://github.com/jgm/pandoc/blob/master/doc/pandoc-server.md + # A note on security diff --git a/app/pandoc.hs b/app/pandoc.hs --- a/app/pandoc.hs +++ b/app/pandoc.hs @@ -14,7 +14,20 @@ import qualified Control.Exception as E import Text.Pandoc.App (convertWithOpts, defaultOpts, options, parseOptions) import Text.Pandoc.Error (handleError) +import Text.Pandoc.Server (ServerOpts(..), parseServerOpts, app) +import Safe (readDef) +import System.Environment (getProgName, lookupEnv) +import qualified Network.Wai.Handler.CGI as CGI +import qualified Network.Wai.Handler.Warp as Warp +import Network.Wai.Middleware.Timeout (timeout) main :: IO () -main = E.catch (parseOptions options defaultOpts >>= convertWithOpts) - (handleError . Left) +main = E.handle (handleError . Left) $ do + prg <- getProgName + cgiTimeout <- maybe 2 (readDef 2) <$> lookupEnv "PANDOC_SERVER_TIMEOUT" + case prg of + "pandoc-server.cgi" -> CGI.run (timeout cgiTimeout app) + "pandoc-server" -> do + sopts <- parseServerOpts + Warp.run (serverPort sopts) (timeout (serverTimeout sopts) app) + _ -> parseOptions options defaultOpts >>= convertWithOpts diff --git a/cabal.project b/cabal.project --- a/cabal.project +++ b/cabal.project @@ -2,4 +2,3 @@ tests: True flags: +embed_data_files constraints: aeson >= 2.0.1.0 - diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,86 @@ # Revision history for pandoc +## pandoc 2.19.1 (2022-08-18) + + * Add server capabilities. + + + New exported module Text.Pandoc.Server [API change]. + + The pandoc executable now starts up a web server when renamed or + symlinked as `pandoc-server`, and functions as a CGI program when + renamed or symlinked as `pandoc-server.cgi`. See the man page for + `pandoc-server` for full documentation. + + * Text.Pandoc.App.Opts: Redo `FromJSON` for `Opt` so that optional + values can be omitted (in which case the values from + `defaultOptions` are used). + + * Org reader: treat "abstract" block as metadata (Albert Krewinkel, #8204). + A block of type "abstract" is assumed to define the document's abstract. + It is transferred from the main text to the metadata. + + * Org template: add abstract from metadata as block of type "abstract" + (#8204). + + * HTML writer: use `flex` property for column widths + (Albert Krewinkel, #8232). + + * LaTeX writer: + + + Add label to tables that have an identifier (Albert Krewinkel, #8219). + Tables with an identifier are marked with a `\label`. A caption is + always included in this case, even if the caption is empty. + + Use `\textquotesingle` for straight quotes in text. + + Fix widths of multicolumn cells (#8218). + + * LaTeX template: fix behavior of `colorlinks` variable (Albert + Krewinkel, #8226). Fixes a regression in 2.19 that required the + `boxlinks` variable to be set in addition to the usual link coloring + variables. Otherwise links were never colored in LaTeX PDF output. + + * Text.Pandoc.Highlighting: Export `lookupHighlightingStyle` + [API change]. Previously this lived in an unexported module + Text.Pandoc.App.CommandLineOptions, under the name + `lookupHighlightStyle`. + + * Text.Pandoc.App: + + + Remove unneeded MonadIO constraints in readSources. + + Factor out `convertWithOpts'` from `convertWithOpts`. + This runs in any PandocMonad, MonadIO, MonadMask instance. + So far it is not exported, but it might find a use later. + + * Support `--strip-comments` in commonmark/gfm (#8222). + This change makes the commonmark reader sensitive to + `readerStripComments`. + + * Lua: add function `pandoc.utils.citeproc` (Albert Krewinkel). + The function runs the *citeproc* processor on a Pandoc document. + Exposing this functionality to Lua allows to make citation processing + part of a filter or writer, simplifies the creation of multiple + bibliographies, and enables the use of varying citation styles in + different parts of a document. + + * Refactor `linux/make_artifacts.sh`. + + * Update INSTALL.md installation from source instructions. + + * Use base64 package instead of base64-bytestring. It is supposed to be + faster and more standards-compliant. + + * trypandoc improvements: + + + Add dropdown with canned examples. + + Add citeproc support. + + Support csv, bibliographic and binary formats. + + Add load from file. + + Add permalink. Don't always reload page. + + Use vanilla JS and CSS + the new `pandoc-server.cgi`. + + * Allow haddock-library-1.11.0. + + * Convert `tool/extract-changes.hs` to a Lua filter. + + ## pandoc 2.19 (2022-08-03) * Add `--embed-resources` flag (Elliot Bobrow, #7331). This can be diff --git a/data/templates/default.latex b/data/templates/default.latex --- a/data/templates/default.latex +++ b/data/templates/default.latex @@ -403,10 +403,11 @@ filecolor={$if(filecolor)$$filecolor$$else$Maroon$endif$}, citecolor={$if(citecolor)$$citecolor$$else$Blue$endif$}, urlcolor={$if(urlcolor)$$urlcolor$$else$Blue$endif$}, -$endif$ +$else$ $if(boxlinks)$ $else$ hidelinks, +$endif$ $endif$ pdfcreator={LaTeX via pandoc}} diff --git a/data/templates/default.org b/data/templates/default.org --- a/data/templates/default.org +++ b/data/templates/default.org @@ -13,6 +13,11 @@ $header-includes$ $endfor$ +$if(abstract)$ +#+begin_abstract +$abstract$ +#+end_abstract +$endif$ $for(include-before)$ $include-before$ diff --git a/man/pandoc.1 b/man/pandoc.1 --- a/man/pandoc.1 +++ b/man/pandoc.1 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pandoc 2.18 +.\" Automatically generated by Pandoc 2.19 .\" .\" Define V font for inline verbatim, using C font in formats .\" that render this, and otherwise B font. @@ -14,7 +14,7 @@ . ftr VB CB . ftr VBI CBI .\} -.TH "Pandoc User\[cq]s Guide" "" "August 3, 2022" "pandoc 2.19" "" +.TH "Pandoc User\[cq]s Guide" "" "August 18, 2022" "pandoc 2.19.1" "" .hy .SH NAME pandoc - general markup converter @@ -7960,6 +7960,20 @@ Some document formats also include a unique identifier. For EPUB, this can be set explicitly by setting the \f[V]identifier\f[R] metadata field (see EPUB Metadata, above). +.SH RUNNING PANDOC AS A WEB SERVER +.PP +If you rename (or symlink) the pandoc executable to +\f[V]pandoc-server\f[R], it will start up a web server with a JSON API. +This server exposes most of the conversion functionality of pandoc. +For full documentation, see the pandoc-server man page. +.PP +If you rename (or symlink) the pandoc executable to +\f[V]pandoc-server.cgi\f[R], it will function as a CGI program exposing +the same API as \f[V]pandoc-server\f[R]. +.PP +\f[V]pandoc-server\f[R] is designed to be maximally secure; it uses +Haskell\[cq]s type system to provide strong guarantees that no I/O will +be performed on the server during pandoc conversions. .SH A NOTE ON SECURITY .IP "1." 3 Although pandoc itself will not create or modify any files other than diff --git a/pandoc.cabal b/pandoc.cabal --- a/pandoc.cabal +++ b/pandoc.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: pandoc -version: 2.19 +version: 2.19.1 build-type: Simple license: GPL-2.0-or-later license-file: COPYING.md @@ -28,7 +28,8 @@ - TeX formats (LaTeX, ConTeXt) - XML formats (DocBook 4 and 5, JATS, TEI Simple, OpenDocument) - Outline formats (OPML) - - Bibliography formats (BibTeX, BibLaTeX, CSL JSON, CSL YAML) + - Bibliography formats (BibTeX, BibLaTeX, CSL JSON, CSL YAML, + RIS) - Word processor formats (Docx, RTF, ODT) - Interactive notebook formats (Jupyter notebook ipynb) - Page layout formats (InDesign ICML) @@ -36,7 +37,7 @@ Vimwiki, XWiki, ZimWiki, Jira wiki, Creole) - Slide show formats (LaTeX Beamer, PowerPoint, Slidy, reveal.js, Slideous, S5, DZSlides) - - Data formats (CSV tables) + - Data formats (CSV and TSV tables) - PDF (via external programs such as pdflatex or wkhtmltopdf) . Pandoc can convert mathematical content in documents @@ -205,8 +206,8 @@ man/pandoc.1.before man/pandoc.1.after -- trypandoc - trypandoc/Makefile trypandoc/index.html + trypandoc/trypandoc.js -- tests test/bodybg.gif test/*.native @@ -428,10 +429,6 @@ Description: Embed Lua 5.3 instead of 5.4. Default: False -flag trypandoc - Description: Build trypandoc cgi executable. - Default: False - flag nightly Description: Add '-nightly-COMPILEDATE' to the output of '--version'. Default: False @@ -474,7 +471,6 @@ aeson-pretty >= 0.8.9 && < 0.9, array >= 0.5 && < 0.6, attoparsec >= 0.12 && < 0.15, - base64-bytestring >= 0.1 && < 1.3, binary >= 0.7 && < 0.11, blaze-html >= 0.9 && < 0.10, blaze-markup >= 0.8 && < 0.9, @@ -491,12 +487,13 @@ directory >= 1.2.3 && < 1.4, doclayout >= 0.4 && < 0.5, doctemplates >= 0.10 && < 0.11, + base64 >= 0.4 && < 0.5, emojis >= 0.1 && < 0.2, exceptions >= 0.8 && < 0.11, file-embed >= 0.0 && < 0.1, filepath >= 1.1 && < 1.5, - gridtables >= 0.0.2 && < 0.1, - haddock-library >= 1.10 && < 1.11, + gridtables >= 0.0.3 && < 0.1, + haddock-library >= 1.10 && < 1.12, hslua-module-doclayout>= 1.0.4 && < 1.1, hslua-module-path >= 1.0 && < 1.1, hslua-module-system >= 1.0 && < 1.1, @@ -537,9 +534,12 @@ xml-types >= 0.3 && < 0.4, yaml >= 0.11 && < 0.12, zip-archive >= 0.2.3.4 && < 0.5, - zlib >= 0.5 && < 0.7 + zlib >= 0.5 && < 0.7, + servant-server, + wai >= 0.3 + if !os(windows) - build-depends: unix >= 2.4 && < 2.8 + build-depends: unix >= 2.4 && < 2.9 if flag(nightly) cpp-options: -DNIGHTLY build-depends: template-haskell @@ -563,6 +563,7 @@ Text.Pandoc.MediaBag, Text.Pandoc.Error, Text.Pandoc.Filter, + Text.Pandoc.Server, Text.Pandoc.Readers, Text.Pandoc.Readers.HTML, Text.Pandoc.Readers.LaTeX, @@ -788,20 +789,9 @@ main-is: pandoc.hs buildable: True other-modules: Paths_pandoc - -executable trypandoc - import: common-executable - main-is: trypandoc.hs - hs-source-dirs: trypandoc - if flag(trypandoc) - build-depends: aeson, - http-types, - text, - wai >= 0.3, - wai-extra >= 3.0.24 - buildable: True - else - buildable: False + build-depends: wai-extra >= 3.0.24, + warp, + safe test-suite test-pandoc import: common-executable diff --git a/src/Text/Pandoc/App.hs b/src/Text/Pandoc/App.hs --- a/src/Text/Pandoc/App.hs +++ b/src/Text/Pandoc/App.hs @@ -28,6 +28,7 @@ import qualified Control.Exception as E import Control.Monad ( (>=>), when, forM_ ) import Control.Monad.Trans ( MonadIO(..) ) +import Control.Monad.Catch ( MonadMask ) import Control.Monad.Except (throwError, catchError) import qualified Data.ByteString as BS import qualified Data.ByteString.Char8 as B8 @@ -77,6 +78,7 @@ convertWithOpts :: Opt -> IO () convertWithOpts opts = do + let outputFile = fromMaybe "-" (optOutputFile opts) datadir <- case optDataDir opts of Nothing -> do d <- defaultUserDataDir @@ -86,292 +88,297 @@ else Nothing Just _ -> return $ optDataDir opts - let outputFile = fromMaybe "-" (optOutputFile opts) - let filters = optFilters opts - let verbosity = optVerbosity opts - when (optDumpArgs opts) $ do UTF8.hPutStrLn stdout (T.pack outputFile) mapM_ (UTF8.hPutStrLn stdout . T.pack) (fromMaybe ["-"] $ optInputFiles opts) exitSuccess - let sources = case optInputFiles opts of - Just xs | not (optIgnoreArgs opts) -> xs - _ -> ["-"] #ifdef _WINDOWS let istty = True #else istty <- liftIO $ queryTerminal stdOutput #endif - res <- runIO $ do + res <- runIO $ convertWithOpts' istty datadir opts + case res of + Left e -> E.throwIO e + Right (output, reports) -> do + case optLogFile opts of + Nothing -> return () + Just logfile -> BL.writeFile logfile (encodeLogMessages reports) + let isWarning msg = messageVerbosity msg == WARNING + when (optFailIfWarnings opts && any isWarning reports) $ + E.throwIO PandocFailOnWarningError + let eol = case optEol opts of + CRLF -> IO.CRLF + LF -> IO.LF + Native -> nativeNewline + case output of + TextOutput t -> writerFn eol outputFile t + BinaryOutput bs -> writeFnBinary outputFile bs - setTrace (optTrace opts) - setVerbosity verbosity - setUserDataDir datadir - setResourcePath (optResourcePath opts) +convertWithOpts' :: (PandocMonad m, MonadIO m, MonadMask m) + => Bool + -> Maybe FilePath + -> Opt + -> m (PandocOutput, [LogMessage]) +convertWithOpts' istty datadir opts = do + let outputFile = fromMaybe "-" (optOutputFile opts) + let filters = optFilters opts + let verbosity = optVerbosity opts - setInputFiles (fromMaybe ["-"] (optInputFiles opts)) - setOutputFile (optOutputFile opts) + let sources = case optInputFiles opts of + Just xs | not (optIgnoreArgs opts) -> xs + _ -> ["-"] + setTrace (optTrace opts) + setVerbosity verbosity + setUserDataDir datadir + setResourcePath (optResourcePath opts) - -- assign reader and writer based on options and filenames - readerName <- case optFrom opts of - Just f -> return f - Nothing -> case formatFromFilePaths sources of - Just f' -> return f' - Nothing | sources == ["-"] -> return "markdown" - | any (isURI . T.pack) sources -> return "html" - | otherwise -> do - report $ CouldNotDeduceFormat - (map (T.pack . takeExtension) sources) "markdown" - return "markdown" + setInputFiles (fromMaybe ["-"] (optInputFiles opts)) + setOutputFile (optOutputFile opts) - let readerNameBase = T.takeWhile (\c -> c /= '+' && c /= '-') readerName + -- assign reader and writer based on options and filenames + readerName <- case optFrom opts of + Just f -> return f + Nothing -> case formatFromFilePaths sources of + Just f' -> return f' + Nothing | sources == ["-"] -> return "markdown" + | any (isURI . T.pack) sources -> return "html" + | otherwise -> do + report $ CouldNotDeduceFormat + (map (T.pack . takeExtension) sources) "markdown" + return "markdown" - let makeSandboxed pureReader = - let files = maybe id (:) (optReferenceDoc opts) . - maybe id (:) (optEpubMetadata opts) . - maybe id (:) (optEpubCoverImage opts) . - maybe id (:) (optCSL opts) . - maybe id (:) (optCitationAbbreviations opts) $ - optEpubFonts opts ++ - optBibliography opts - in case pureReader of - TextReader r -> TextReader $ \o t -> sandbox files (r o t) - ByteStringReader r - -> ByteStringReader $ \o t -> sandbox files (r o t) + let readerNameBase = T.takeWhile (\c -> c /= '+' && c /= '-') readerName - (reader, readerExts) <- - if ".lua" `T.isSuffixOf` readerName - then return (TextReader (readCustom (T.unpack readerName)), mempty) - else if optSandbox opts - then case runPure (getReader readerName) of - Left e -> throwError e - Right (r, rexts) -> return (makeSandboxed r, rexts) - else getReader readerName + let makeSandboxed pureReader = + let files = maybe id (:) (optReferenceDoc opts) . + maybe id (:) (optEpubMetadata opts) . + maybe id (:) (optEpubCoverImage opts) . + maybe id (:) (optCSL opts) . + maybe id (:) (optCitationAbbreviations opts) $ + optEpubFonts opts ++ + optBibliography opts + in case pureReader of + TextReader r -> TextReader $ \o t -> sandbox files (r o t) + ByteStringReader r + -> ByteStringReader $ \o t -> sandbox files (r o t) - outputSettings <- optToOutputSettings opts - let format = outputFormat outputSettings - let writer = outputWriter outputSettings - let writerName = outputWriterName outputSettings - let writerNameBase = T.takeWhile (\c -> c /= '+' && c /= '-') writerName - let writerOptions = outputWriterOptions outputSettings + (reader, readerExts) <- + if ".lua" `T.isSuffixOf` readerName + then return (TextReader (readCustom (T.unpack readerName)), mempty) + else if optSandbox opts + then case runPure (getReader readerName) of + Left e -> throwError e + Right (r, rexts) -> return (makeSandboxed r, rexts) + else getReader readerName - let pdfOutput = isJust $ outputPdfProgram outputSettings + outputSettings <- optToOutputSettings opts + let format = outputFormat outputSettings + let writer = outputWriter outputSettings + let writerName = outputWriterName outputSettings + let writerNameBase = T.takeWhile (\c -> c /= '+' && c /= '-') writerName + let writerOptions = outputWriterOptions outputSettings - let bibOutput = writerNameBase == "bibtex" || - writerNameBase == "biblatex" || - writerNameBase == "csljson" + let pdfOutput = isJust $ outputPdfProgram outputSettings - let standalone = optStandalone opts || - not (isTextFormat format) || - pdfOutput || - bibOutput + let bibOutput = writerNameBase == "bibtex" || + writerNameBase == "biblatex" || + writerNameBase == "csljson" - when (pdfOutput && readerNameBase == "latex") $ - case optInputFiles opts of - Just (inputFile:_) -> report $ UnusualConversion $ T.pack $ - "to convert a .tex file to PDF, you get better results by using pdflatex " - <> "(or lualatex or xelatex) directly, try `pdflatex " <> inputFile - <> "` instead of `pandoc " <> inputFile <> " -o " <> outputFile <> "`." - _ -> return () + let standalone = optStandalone opts || + not (isTextFormat format) || + pdfOutput || + bibOutput - -- We don't want to send output to the terminal if the user - -- does 'pandoc -t docx input.txt'; though we allow them to - -- force this with '-o -'. On posix systems, we detect - -- when stdout is being piped and allow output to stdout - -- in that case, but on Windows we can't. - when ((pdfOutput || not (isTextFormat format)) && - istty && isNothing ( optOutputFile opts)) $ - throwError $ PandocAppError $ - "Cannot write " <> (if pdfOutput then "pdf" else format) <> - " output to terminal.\n" <> - "Specify an output file using the -o option, or " <> - "use '-o -' to force output to stdout." + when (pdfOutput && readerNameBase == "latex") $ + case optInputFiles opts of + Just (inputFile:_) -> report $ UnusualConversion $ T.pack $ + "to convert a .tex file to PDF, you get better results by using pdflatex " + <> "(or lualatex or xelatex) directly, try `pdflatex " <> inputFile + <> "` instead of `pandoc " <> inputFile <> " -o " <> outputFile <> "`." + _ -> return () + -- We don't want to send output to the terminal if the user + -- does 'pandoc -t docx input.txt'; though we allow them to + -- force this with '-o -'. On posix systems, we detect + -- when stdout is being piped and allow output to stdout + -- in that case, but on Windows we can't. + when ((pdfOutput || not (isTextFormat format)) && + istty && isNothing ( optOutputFile opts)) $ + throwError $ PandocAppError $ + "Cannot write " <> (if pdfOutput then "pdf" else format) <> + " output to terminal.\n" <> + "Specify an output file using the -o option, or " <> + "use '-o -' to force output to stdout." - abbrevs <- Set.fromList . filter (not . T.null) . T.lines . UTF8.toText <$> - case optAbbreviations opts of - Nothing -> readDataFile "abbreviations" - Just f -> readFileStrict f - case lookupMetaString "lang" (optMetadata opts) of - "" -> setTranslations $ Lang "en" Nothing (Just "US") [] [] [] - l -> case parseLang l of - Left _ -> report $ InvalidLang l - Right l' -> setTranslations l' + abbrevs <- Set.fromList . filter (not . T.null) . T.lines . UTF8.toText <$> + case optAbbreviations opts of + Nothing -> readDataFile "abbreviations" + Just f -> readFileStrict f - let readerOpts = def{ - readerStandalone = standalone - , readerColumns = optColumns opts - , readerTabStop = optTabStop opts - , readerIndentedCodeClasses = optIndentedCodeClasses opts - , readerDefaultImageExtension = - optDefaultImageExtension opts - , readerTrackChanges = optTrackChanges opts - , readerAbbreviations = abbrevs - , readerExtensions = readerExts - , readerStripComments = optStripComments opts - } + case lookupMetaString "lang" (optMetadata opts) of + "" -> setTranslations $ Lang "en" Nothing (Just "US") [] [] [] + l -> case parseLang l of + Left _ -> report $ InvalidLang l + Right l' -> setTranslations l' - metadataFromFile <- - case optMetadataFiles opts of - [] -> return mempty - paths -> do - -- If format is markdown or commonmark, use the enabled extensions, - -- otherwise treat metadata as pandoc markdown (see #7926, #6832) - let readerOptsMeta = - if readerNameBase == "markdown" || readerNameBase == "commonmark" - then readerOpts - else readerOpts{ readerExtensions = pandocExtensions } - mconcat <$> mapM - (\path -> do raw <- readMetadataFile path - yamlToMeta readerOptsMeta (Just path) raw) paths + let readerOpts = def{ + readerStandalone = standalone + , readerColumns = optColumns opts + , readerTabStop = optTabStop opts + , readerIndentedCodeClasses = optIndentedCodeClasses opts + , readerDefaultImageExtension = + optDefaultImageExtension opts + , readerTrackChanges = optTrackChanges opts + , readerAbbreviations = abbrevs + , readerExtensions = readerExts + , readerStripComments = optStripComments opts + } - let transforms = (case optShiftHeadingLevelBy opts of - 0 -> id - x -> (headerShift x :)) . - (if optStripEmptyParagraphs opts - then (stripEmptyParagraphs :) - else id) . - (if extensionEnabled Ext_east_asian_line_breaks - readerExts && - not (extensionEnabled Ext_east_asian_line_breaks - (writerExtensions writerOptions) && - writerWrapText writerOptions == WrapPreserve) - then (eastAsianLineBreakFilter :) - else id) . - (case optIpynbOutput opts of - _ | readerNameBase /= "ipynb" -> id - IpynbOutputAll -> id - IpynbOutputNone -> (filterIpynbOutput Nothing :) - IpynbOutputBest -> (filterIpynbOutput (Just $ - if htmlFormat format - then Format "html" - else - case format of - "latex" -> Format "latex" - "beamer" -> Format "latex" - _ -> Format format) :)) - $ [] + metadataFromFile <- + case optMetadataFiles opts of + [] -> return mempty + paths -> do + -- If format is markdown or commonmark, use the enabled extensions, + -- otherwise treat metadata as pandoc markdown (see #7926, #6832) + let readerOptsMeta = + if readerNameBase == "markdown" || readerNameBase == "commonmark" + then readerOpts + else readerOpts{ readerExtensions = pandocExtensions } + mconcat <$> mapM + (\path -> do raw <- readMetadataFile path + yamlToMeta readerOptsMeta (Just path) raw) paths - let convertTabs = tabFilter (if optPreserveTabs opts || - readerNameBase == "t2t" || - readerNameBase == "man" || - readerNameBase == "tsv" - then 0 - else optTabStop opts) + let transforms = (case optShiftHeadingLevelBy opts of + 0 -> id + x -> (headerShift x :)) . + (if optStripEmptyParagraphs opts + then (stripEmptyParagraphs :) + else id) . + (if extensionEnabled Ext_east_asian_line_breaks + readerExts && + not (extensionEnabled Ext_east_asian_line_breaks + (writerExtensions writerOptions) && + writerWrapText writerOptions == WrapPreserve) + then (eastAsianLineBreakFilter :) + else id) . + (case optIpynbOutput opts of + _ | readerNameBase /= "ipynb" -> id + IpynbOutputAll -> id + IpynbOutputNone -> (filterIpynbOutput Nothing :) + IpynbOutputBest -> (filterIpynbOutput (Just $ + if htmlFormat format + then Format "html" + else + case format of + "latex" -> Format "latex" + "beamer" -> Format "latex" + _ -> Format format) :)) + $ [] + let convertTabs = tabFilter (if optPreserveTabs opts || + readerNameBase == "t2t" || + readerNameBase == "man" || + readerNameBase == "tsv" + then 0 + else optTabStop opts) - when (readerNameBase == "markdown_github" || - writerNameBase == "markdown_github") $ - report $ Deprecated "markdown_github" "Use gfm instead." - mapM_ (uncurry setRequestHeader) (optRequestHeaders opts) + when (readerNameBase == "markdown_github" || + writerNameBase == "markdown_github") $ + report $ Deprecated "markdown_github" "Use gfm instead." - setNoCheckCertificate (optNoCheckCertificate opts) + mapM_ (uncurry setRequestHeader) (optRequestHeaders opts) - let isPandocCiteproc (JSONFilter f) = takeBaseName f == "pandoc-citeproc" - isPandocCiteproc _ = False + setNoCheckCertificate (optNoCheckCertificate opts) - when (any isPandocCiteproc filters) $ - report $ Deprecated "pandoc-citeproc filter" - "Use --citeproc instead." + let isPandocCiteproc (JSONFilter f) = takeBaseName f == "pandoc-citeproc" + isPandocCiteproc _ = False - let cslMetadata = - maybe id (setMeta "csl") (optCSL opts) . - (case optBibliography opts of - [] -> id - xs -> setMeta "bibliography" xs) . - maybe id (setMeta "citation-abbreviations") - (optCitationAbbreviations opts) $ mempty + when (any isPandocCiteproc filters) $ + report $ Deprecated "pandoc-citeproc filter" + "Use --citeproc instead." - let filterEnv = Environment readerOpts writerOptions + let cslMetadata = + maybe id (setMeta "csl") (optCSL opts) . + (case optBibliography opts of + [] -> id + xs -> setMeta "bibliography" xs) . + maybe id (setMeta "citation-abbreviations") + (optCitationAbbreviations opts) $ mempty - inputs <- readSources sources + let filterEnv = Environment readerOpts writerOptions - doc <- (case reader of - TextReader r - | readerNameBase == "json" -> - mconcat <$> - mapM (inputToText convertTabs - >=> r readerOpts . (:[])) inputs - | optFileScope opts -> - mconcat <$> mapM - (inputToText convertTabs - >=> r readerOpts . (:[])) - inputs - | otherwise -> mapM (inputToText convertTabs) inputs - >>= r readerOpts - ByteStringReader r -> - mconcat <$> mapM (r readerOpts . inputToLazyByteString) inputs) - >>= ( return . adjustMetadata (metadataFromFile <>) - >=> return . adjustMetadata (<> optMetadata opts) - >=> return . adjustMetadata (<> cslMetadata) - >=> applyTransforms transforms - >=> applyFilters filterEnv filters [T.unpack format] - >=> (if not (optSandbox opts) && - (isJust (optExtractMedia opts) - || writerNameBase == "docx") -- for fallback pngs - then fillMediaBag - else return) - >=> maybe return extractMedia (optExtractMedia opts) - ) + inputs <- readSources sources - when (writerNameBase == "docx" && not (optSandbox opts)) $ do - -- create fallback pngs for svgs - items <- mediaItems <$> getMediaBag - forM_ items $ \(fp, mt, bs) -> - case T.takeWhile (/=';') mt of - "image/svg+xml" -> do - res <- svgToPng (writerDpi writerOptions) bs - case res of - Right bs' -> do - let fp' = fp <> ".png" - insertMedia fp' (Just "image/png") bs' - Left e -> report $ CouldNotConvertImage (T.pack fp) (tshow e) - _ -> return () + doc <- (case reader of + TextReader r + | readerNameBase == "json" -> + mconcat <$> + mapM (inputToText convertTabs + >=> r readerOpts . (:[])) inputs + | optFileScope opts -> + mconcat <$> mapM + (inputToText convertTabs + >=> r readerOpts . (:[])) + inputs + | otherwise -> mapM (inputToText convertTabs) inputs + >>= r readerOpts + ByteStringReader r -> + mconcat <$> mapM (r readerOpts . inputToLazyByteString) inputs) + >>= ( return . adjustMetadata (metadataFromFile <>) + >=> return . adjustMetadata (<> optMetadata opts) + >=> return . adjustMetadata (<> cslMetadata) + >=> applyTransforms transforms + >=> applyFilters filterEnv filters [T.unpack format] + >=> (if not (optSandbox opts) && + (isJust (optExtractMedia opts) + || writerNameBase == "docx") -- for fallback pngs + then fillMediaBag + else return) + >=> maybe return extractMedia (optExtractMedia opts) + ) - output <- case writer of - ByteStringWriter f -> BinaryOutput <$> f writerOptions doc - TextWriter f -> case outputPdfProgram outputSettings of - Just pdfProg -> do - res <- makePDF pdfProg (optPdfEngineOpts opts) f - writerOptions doc - case res of - Right pdf -> return $ BinaryOutput pdf - Left err' -> throwError $ PandocPDFError $ - TL.toStrict (TE.decodeUtf8With TE.lenientDecode err') + when (writerNameBase == "docx" && not (optSandbox opts)) $ do + -- create fallback pngs for svgs + items <- mediaItems <$> getMediaBag + forM_ items $ \(fp, mt, bs) -> + case T.takeWhile (/=';') mt of + "image/svg+xml" -> do + res <- svgToPng (writerDpi writerOptions) bs + case res of + Right bs' -> do + let fp' = fp <> ".png" + insertMedia fp' (Just "image/png") bs' + Left e -> report $ CouldNotConvertImage (T.pack fp) (tshow e) + _ -> return () - Nothing -> do - let ensureNl t - | standalone = t - | T.null t || T.last t /= '\n' = t <> T.singleton '\n' - | otherwise = t - textOutput <- ensureNl <$> f writerOptions doc - if (optSelfContained opts || optEmbedResources opts) && htmlFormat format - then TextOutput <$> makeSelfContained textOutput - else return $ TextOutput textOutput - reports <- getLog - return (output, reports) + output <- case writer of + ByteStringWriter f -> BinaryOutput <$> f writerOptions doc + TextWriter f -> case outputPdfProgram outputSettings of + Just pdfProg -> do + res <- makePDF pdfProg (optPdfEngineOpts opts) f + writerOptions doc + case res of + Right pdf -> return $ BinaryOutput pdf + Left err' -> throwError $ PandocPDFError $ + TL.toStrict (TE.decodeUtf8With TE.lenientDecode err') - case res of - Left e -> E.throwIO e - Right (output, reports) -> do - case optLogFile opts of - Nothing -> return () - Just logfile -> BL.writeFile logfile (encodeLogMessages reports) - let isWarning msg = messageVerbosity msg == WARNING - when (optFailIfWarnings opts && any isWarning reports) $ - E.throwIO PandocFailOnWarningError - let eol = case optEol opts of - CRLF -> IO.CRLF - LF -> IO.LF - Native -> nativeNewline - case output of - TextOutput t -> writerFn eol outputFile t - BinaryOutput bs -> writeFnBinary outputFile bs + Nothing -> do + let ensureNl t + | standalone = t + | T.null t || T.last t /= '\n' = t <> T.singleton '\n' + | otherwise = t + textOutput <- ensureNl <$> f writerOptions doc + if (optSelfContained opts || optEmbedResources opts) && htmlFormat format + then TextOutput <$> makeSelfContained textOutput + else return $ TextOutput textOutput + reports <- getLog + return (output, reports) data PandocOutput = TextOutput Text | BinaryOutput BL.ByteString deriving (Show) @@ -393,13 +400,13 @@ applyTransforms :: Monad m => [Transform] -> Pandoc -> m Pandoc applyTransforms transforms d = return $ foldr ($) d transforms -readSources :: (PandocMonad m, MonadIO m) +readSources :: PandocMonad m => [FilePath] -> m [(FilePath, (BS.ByteString, Maybe MimeType))] readSources srcs = mapM (\fp -> do t <- readSource fp return (if fp == "-" then "" else fp, t)) srcs -readSource :: (PandocMonad m, MonadIO m) +readSource :: PandocMonad m => FilePath -> m (BS.ByteString, Maybe MimeType) readSource "-" = (,Nothing) <$> readStdinStrict readSource src = diff --git a/src/Text/Pandoc/App/CommandLineOptions.hs b/src/Text/Pandoc/App/CommandLineOptions.hs --- a/src/Text/Pandoc/App/CommandLineOptions.hs +++ b/src/Text/Pandoc/App/CommandLineOptions.hs @@ -22,12 +22,10 @@ , parseOptionsFromArgs , options , engines - , lookupHighlightStyle , setVariable ) where import Control.Monad import Control.Monad.Trans -import Control.Monad.Except (throwError) import Control.Monad.State.Strict import Data.Aeson.Encode.Pretty (encodePretty', Config(..), keyOrder, defConfig, Indent(..), NumberFormat(..)) @@ -41,7 +39,7 @@ import Data.Text (Text) import HsLua (Exception, getglobal, openlibs, peek, run, top) import Safe (tailDef) -import Skylighting (Style, Syntax (..), defaultSyntaxMap, parseTheme) +import Skylighting (Syntax (..), defaultSyntaxMap) import System.Console.GetOpt import System.Environment (getArgs, getProgName) import System.Exit (exitSuccess) @@ -54,7 +52,7 @@ DefaultsState (..), applyDefaults, fullDefaultsPath) import Text.Pandoc.Filter (Filter (..)) -import Text.Pandoc.Highlighting (highlightingStyles) +import Text.Pandoc.Highlighting (highlightingStyles, lookupHighlightingStyle) import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDir) import Text.Printf @@ -946,7 +944,7 @@ (ReqArg (\arg opt -> do let write = maybe B.putStr B.writeFile $ optOutputFile opt - sty <- runIOorExplode $ lookupHighlightStyle arg + sty <- runIOorExplode $ lookupHighlightingStyle arg write $ encodePretty' defConfig{confIndent = Spaces 4 ,confCompare = keyOrder @@ -1058,20 +1056,6 @@ splitField :: String -> (String, String) splitField = second (tailDef "true") . break (`elemText` ":=") - -lookupHighlightStyle :: PandocMonad m => String -> m Style -lookupHighlightStyle s - | takeExtension s == ".theme" = -- attempt to load KDE theme - do contents <- readFileLazy s - case parseTheme contents of - Left _ -> throwError $ PandocOptionError $ T.pack $ - "Could not read highlighting theme " ++ s - Right sty -> return sty - | otherwise = - case lookup (T.toLower $ T.pack s) highlightingStyles of - Just sty -> return sty - Nothing -> throwError $ PandocOptionError $ T.pack $ - "Unknown highlight-style " ++ s deprecatedOption :: String -> String -> IO () deprecatedOption o msg = diff --git a/src/Text/Pandoc/App/Opt.hs b/src/Text/Pandoc/App/Opt.hs --- a/src/Text/Pandoc/App/Opt.hs +++ b/src/Text/Pandoc/App/Opt.hs @@ -54,7 +54,8 @@ import qualified Data.Map as M import qualified Data.ByteString.Char8 as B8 import Text.Pandoc.Definition (Meta(..), MetaValue(..)) -import Data.Aeson (defaultOptions, Options(..), Result(..), fromJSON, camelTo2) +import Data.Aeson (defaultOptions, Options(..), Result(..), camelTo2, + genericToJSON, fromJSON) import Data.Aeson.TH (deriveJSON) import Control.Applicative ((<|>)) import Data.Yaml @@ -156,8 +157,90 @@ , optSandbox :: Bool } deriving (Generic, Show) -$(deriveJSON - defaultOptions{ fieldLabelModifier = camelTo2 '-' . drop 3 } ''Opt) +instance FromJSON Opt where + parseJSON = withObject "Opt" $ \o -> + Opt + <$> o .:? "tab-stop" .!= optTabStop defaultOpts + <*> o .:? "preserve-tabs" .!= optPreserveTabs defaultOpts + <*> o .:? "standalone" .!= optStandalone defaultOpts + <*> o .:? "from" + <*> o .:? "to" + <*> o .:? "table-of-contents" .!= optTableOfContents defaultOpts + <*> o .:? "shift-heading-level-by" .!= optShiftHeadingLevelBy defaultOpts + <*> o .:? "template" + <*> o .:? "variables" .!= optVariables defaultOpts + <*> o .:? "metadata" .!= optMetadata defaultOpts + <*> o .:? "metadata-files" .!= optMetadataFiles defaultOpts + <*> o .:? "output-file" + <*> o .:? "input-files" + <*> o .:? "number-sections" .!= optNumberSections defaultOpts + <*> o .:? "number-offset" .!= optNumberOffset defaultOpts + <*> o .:? "section-divs" .!= optSectionDivs defaultOpts + <*> o .:? "incremental" .!= optIncremental defaultOpts + <*> o .:? "self-contained" .!= optSelfContained defaultOpts + <*> o .:? "embed-resources" .!= optEmbedResources defaultOpts + <*> o .:? "html-q-tags" .!= optHtmlQTags defaultOpts + <*> o .:? "highlight-style" + <*> o .:? "syntax-definitions" .!= optSyntaxDefinitions defaultOpts + <*> o .:? "top-level-division" .!= optTopLevelDivision defaultOpts + <*> o .:? "html-math-method" .!= optHTMLMathMethod defaultOpts + <*> o .:? "abbreviations" + <*> o .:? "reference-doc" + <*> o .:? "epub-subdirectory" .!= optEpubSubdirectory defaultOpts + <*> o .:? "epub-metadata" + <*> o .:? "epub-fonts" .!= optEpubFonts defaultOpts + <*> o .:? "epub-chapter-level" .!= optEpubChapterLevel defaultOpts + <*> o .:? "epub-cover-image" + <*> o .:? "toc-depth" .!= optTOCDepth defaultOpts + <*> o .:? "dump-args" .!= optDumpArgs defaultOpts + <*> o .:? "ignore-args" .!= optIgnoreArgs defaultOpts + <*> o .:? "verbosity" .!= optVerbosity defaultOpts + <*> o .:? "trace" .!= optTrace defaultOpts + <*> o .:? "log-file" + <*> o .:? "fail-if-warnings" .!= optFailIfWarnings defaultOpts + <*> o .:? "reference-links" .!= optReferenceLinks defaultOpts + <*> o .:? "reference-location" .!= optReferenceLocation defaultOpts + <*> o .:? "dpi" .!= optDpi defaultOpts + <*> o .:? "wrap" .!= optWrap defaultOpts + <*> o .:? "columns" .!= optColumns defaultOpts + <*> o .:? "filters" .!= optFilters defaultOpts + <*> o .:? "email-obfuscation" .!= optEmailObfuscation defaultOpts + <*> o .:? "identifier-prefix" .!= optIdentifierPrefix defaultOpts + <*> o .:? "strip-empty-paragraphs" .!= optStripEmptyParagraphs defaultOpts + <*> o .:? "indented-code-classes" .!= optIndentedCodeClasses defaultOpts + <*> o .:? "data-dir" + <*> o .:? "cite-method" .!= optCiteMethod defaultOpts + <*> o .:? "listings" .!= optListings defaultOpts + <*> o .:? "pdf-engine" + <*> o .:? "pdf-engine-opts" .!= optPdfEngineOpts defaultOpts + <*> o .:? "slide-level" + <*> o .:? "setext-headers" .!= optSetextHeaders defaultOpts + <*> o .:? "ascii" .!= optAscii defaultOpts + <*> o .:? "default-image-extension" .!= optDefaultImageExtension defaultOpts + <*> o .:? "extract-media" + <*> o .:? "track-changes" .!= optTrackChanges defaultOpts + <*> o .:? "file-scope" .!= optFileScope defaultOpts + <*> o .:? "title-prefix" .!= optTitlePrefix defaultOpts + <*> o .:? "css" .!= optCss defaultOpts + <*> o .:? "ipynb-output" .!= optIpynbOutput defaultOpts + <*> o .:? "include-before-body" .!= optIncludeBeforeBody defaultOpts + <*> o .:? "include-after-body" .!= optIncludeAfterBody defaultOpts + <*> o .:? "include-in-header" .!= optIncludeInHeader defaultOpts + <*> o .:? "resource-path" .!= optResourcePath defaultOpts + <*> o .:? "request-headers" .!= optRequestHeaders defaultOpts + <*> o .:? "no-check-certificate" .!= optNoCheckCertificate defaultOpts + <*> o .:? "eol" .!= optEol defaultOpts + <*> o .:? "strip-comments" .!= optStripComments defaultOpts + <*> o .:? "csl" + <*> o .:? "bibliography" .!= optBibliography defaultOpts + <*> o .:? "citation-abbreviations" + <*> o .:? "sandbox" .!= optSandbox defaultOpts + +instance ToJSON Opt where + toJSON = genericToJSON defaultOptions{ + fieldLabelModifier = camelTo2 '-' . drop 3, + omitNothingFields = True } + instance FromJSON (Opt -> Opt) where parseJSON (Object m) = diff --git a/src/Text/Pandoc/App/OutputSettings.hs b/src/Text/Pandoc/App/OutputSettings.hs --- a/src/Text/Pandoc/App/OutputSettings.hs +++ b/src/Text/Pandoc/App/OutputSettings.hs @@ -37,8 +37,8 @@ import Text.Pandoc import Text.Pandoc.App.FormatHeuristics (formatFromFilePaths) import Text.Pandoc.App.Opt (Opt (..)) -import Text.Pandoc.App.CommandLineOptions (engines, lookupHighlightStyle, - setVariable) +import Text.Pandoc.App.CommandLineOptions (engines, setVariable) +import Text.Pandoc.Highlighting (lookupHighlightingStyle) import Text.Pandoc.Writers.Custom (writeCustom) import qualified Text.Pandoc.UTF8 as UTF8 @@ -128,7 +128,8 @@ syntaxMap <- foldM addSyntaxMap defaultSyntaxMap (optSyntaxDefinitions opts) - hlStyle <- traverse (lookupHighlightStyle . T.unpack) $ optHighlightStyle opts + hlStyle <- traverse (lookupHighlightingStyle . T.unpack) $ + optHighlightStyle opts let setVariableM k v = return . setVariable k v diff --git a/src/Text/Pandoc/Class/IO.hs b/src/Text/Pandoc/Class/IO.hs --- a/src/Text/Pandoc/Class/IO.hs +++ b/src/Text/Pandoc/Class/IO.hs @@ -36,7 +36,7 @@ import Control.Monad.Except (throwError) import Control.Monad.IO.Class (MonadIO, liftIO) -import Data.ByteString.Base64 (decodeLenient) +import Data.ByteString.Base64.URL (decodeBase64Lenient) import Data.ByteString.Lazy (toChunks) import Data.Text (Text, pack, unpack) import Data.Time (TimeZone, UTCTime) @@ -125,7 +125,7 @@ let mime = T.takeWhile (/=',') u'' let contents = UTF8.fromString $ unEscapeString $ T.unpack $ T.drop 1 $ T.dropWhile (/=',') u'' - return (decodeLenient contents, Just mime) + return (decodeBase64Lenient contents, Just mime) | otherwise = do let toReqHeader (n, v) = (CI.mk (UTF8.fromText n), UTF8.fromText v) customHeaders <- map toReqHeader <$> getsCommonState stRequestHeaders diff --git a/src/Text/Pandoc/Highlighting.hs b/src/Text/Pandoc/Highlighting.hs --- a/src/Text/Pandoc/Highlighting.hs +++ b/src/Text/Pandoc/Highlighting.hs @@ -30,6 +30,7 @@ , breezeDark , haddock , Style + , lookupHighlightingStyle , fromListingsLanguage , toListingsLanguage ) where @@ -39,6 +40,10 @@ import qualified Data.Text as T import Skylighting import Text.Pandoc.Definition +import Text.Pandoc.Class (PandocMonad, readFileLazy) +import Text.Pandoc.Error (PandocError(..)) +import Control.Monad.Except (throwError) +import System.FilePath (takeExtension) import Text.Pandoc.Shared (safeRead) highlightingStyles :: [(T.Text, Style)] @@ -214,3 +219,21 @@ -- | Determine skylighting language name from listings language name. fromListingsLanguage :: T.Text -> Maybe T.Text fromListingsLanguage lang = M.lookup lang listingsToLangMap + +-- | Lookup style from a name. If the name is a standard style, +-- load it; if it ends in ".theme", attempt to load a KDE theme +-- from the file path specified. +lookupHighlightingStyle :: PandocMonad m => String -> m Style +lookupHighlightingStyle s + | takeExtension s == ".theme" = -- attempt to load KDE theme + do contents <- readFileLazy s + case parseTheme contents of + Left _ -> throwError $ PandocOptionError $ T.pack $ + "Could not read highlighting theme " ++ s + Right sty -> return sty + | otherwise = + case lookup (T.toLower $ T.pack s) highlightingStyles of + Just sty -> return sty + Nothing -> throwError $ PandocOptionError $ T.pack $ + "Unknown highlight-style " ++ s + diff --git a/src/Text/Pandoc/Lua/Module/Utils.hs b/src/Text/Pandoc/Lua/Module/Utils.hs --- a/src/Text/Pandoc/Lua/Module/Utils.hs +++ b/src/Text/Pandoc/Lua/Module/Utils.hs @@ -25,7 +25,7 @@ import Data.Version (Version) import HsLua as Lua import HsLua.Module.Version (peekVersionFuzzy, pushVersion) -import Text.Pandoc.Citeproc (getReferences) +import Text.Pandoc.Citeproc (getReferences, processCitations) import Text.Pandoc.Definition import Text.Pandoc.Error (PandocError) import Text.Pandoc.Lua.Marshal.AST @@ -58,6 +58,16 @@ "blocks" "" <#> opt (parameter (peekList peekInline) "list of inlines" "inline" "") =#> functionResult pushInlines "list of inlines" "" + + , defun "citeproc" + ### unPandocLua . processCitations + <#> parameter peekPandoc "Pandoc" "doc" "document" + =#> functionResult pushPandoc "Pandoc" "processed document" + #? T.unwords + [ "Process the citations in the file, replacing them with " + , "rendered citations and adding a bibliography. " + , "See the manual section on citation rendering for details." + ] , defun "equals" ### equal diff --git a/src/Text/Pandoc/PDF.hs b/src/Text/Pandoc/PDF.hs --- a/src/Text/Pandoc/PDF.hs +++ b/src/Text/Pandoc/PDF.hs @@ -58,6 +58,7 @@ #endif import Data.List (isPrefixOf, find) import Text.Pandoc.Class (fillMediaBag, getVerbosity, + readFileLazy, readFileStrict, fileExists, report, extractMedia, PandocMonad) import Text.Pandoc.Logging @@ -335,21 +336,21 @@ => Maybe String -> String -> m (Maybe ByteString, Maybe ByteString) getResultingPDF logFile pdfFile = do - pdfExists <- liftIO $ doesFileExist pdfFile + pdfExists <- fileExists pdfFile pdf <- if pdfExists -- We read PDF as a strict bytestring to make sure that the -- temp directory is removed on Windows. -- See https://github.com/jgm/pandoc/issues/1192. then (Just . BL.fromChunks . (:[])) `fmap` - liftIO (BS.readFile pdfFile) + (readFileStrict pdfFile) else return Nothing -- Note that some things like Missing character warnings -- appear in the log but not on stderr, so we prefer the log: log' <- case logFile of Just logFile' -> do - logExists <- liftIO $ doesFileExist logFile' + logExists <- fileExists logFile' if logExists - then liftIO $ Just <$> BL.readFile logFile' + then Just <$> readFileLazy logFile' else return Nothing Nothing -> return Nothing return (log', pdf) diff --git a/src/Text/Pandoc/Readers/CommonMark.hs b/src/Text/Pandoc/Readers/CommonMark.hs --- a/src/Text/Pandoc/Readers/CommonMark.hs +++ b/src/Text/Pandoc/Readers/CommonMark.hs @@ -34,7 +34,10 @@ runF, defaultParserState, option, many1, anyChar, Sources(..), ToSources(..), ParserT, Future, sourceName, sourceLine, incSourceLine) +import Text.Pandoc.Walk (walk) import qualified Data.Text as T +import qualified Data.Attoparsec.Text as A +import Control.Applicative ((<|>)) -- | Parse a CommonMark formatted string into a 'Pandoc' structure. readCommonMark :: (PandocMonad m, ToSources a) @@ -86,15 +89,41 @@ Right (Cm bls :: Cm () Blocks) -> return $ return $ B.toMetaValue bls readCommonMarkBody :: PandocMonad m => ReaderOptions -> Sources -> [Tok] -> m Pandoc -readCommonMarkBody opts s toks - | isEnabled Ext_sourcepos opts = - case runIdentity (parseCommonmarkWith (specFor opts) toks) of - Left err -> throwError $ PandocParsecError s err - Right (Cm bls :: Cm SourceRange Blocks) -> return $ B.doc bls - | otherwise = - case runIdentity (parseCommonmarkWith (specFor opts) toks) of - Left err -> throwError $ PandocParsecError s err - Right (Cm bls :: Cm () Blocks) -> return $ B.doc bls +readCommonMarkBody opts s toks = + (if readerStripComments opts + then walk stripBlockComments . walk stripInlineComments + else id) <$> + if isEnabled Ext_sourcepos opts + then case runIdentity (parseCommonmarkWith (specFor opts) toks) of + Left err -> throwError $ PandocParsecError s err + Right (Cm bls :: Cm SourceRange Blocks) -> return $ B.doc bls + else case runIdentity (parseCommonmarkWith (specFor opts) toks) of + Left err -> throwError $ PandocParsecError s err + Right (Cm bls :: Cm () Blocks) -> return $ B.doc bls + +stripBlockComments :: Block -> Block +stripBlockComments (RawBlock (B.Format "html") s) = + RawBlock (B.Format "html") (removeComments s) +stripBlockComments x = x + +stripInlineComments :: Inline -> Inline +stripInlineComments (RawInline (B.Format "html") s) = + RawInline (B.Format "html") (removeComments s) +stripInlineComments x = x + +removeComments :: Text -> Text +removeComments s = + either (const s) id $ A.parseOnly pRemoveComments s + where + pRemoveComments = mconcat <$> A.many' + ("" <$ (A.string " - - - - -
+
-
-

Try pandoc!

+
+

Try pandoc!

-
+ 

     

   

   

-    

-      
+    

+      
        
       
       
-      
-      
+      

+      
+      
       

-      
+      
+      

+      
+      
+
     

-    

+    

       
       
+      
+      
+      
+      
       

+      

       

     

   

 

 
+
+
+
 
 
diff --git a/trypandoc/trypandoc.hs b/trypandoc/trypandoc.hs
deleted file mode 100644
--- a/trypandoc/trypandoc.hs
+++ /dev/null
@@ -1,76 +0,0 @@
-{-# LANGUAGE OverloadedStrings #-}
-{- |
-   Module      : Main
-   Copyright   : © 2014-2022 John MacFarlane 
-   License     : GNU GPL, version 2 or above
-
-   Maintainer  : John MacFarlane 
-   Stability   : alpha
-   Portability : portable
-
-Provides a webservice which allows to try pandoc in the browser.
--}
-module Main where
-import Network.Wai.Handler.CGI
-import Network.Wai.Middleware.Timeout (timeout)
-import Network.Wai
-import Data.Maybe (fromMaybe)
-import Network.HTTP.Types.Status (status200)
-import Network.HTTP.Types.Header (hContentType)
-import Network.HTTP.Types.URI (queryToQueryText)
-import Text.Pandoc
-import Text.Pandoc.Highlighting (pygments)
-import Text.Pandoc.Shared (tabFilter)
-import Data.Aeson
-import qualified Data.Text as T
-import Data.Text (Text)
-
-main :: IO ()
-main = run $ timeout 2 app
-
-app :: Application
-app req respond = do
-  let query = queryToQueryText $ queryString req
-  let getParam x = maybe (error $ T.unpack x ++ " parameter not set")
-                       return $ lookup x query
-  text <- getParam "text" >>= checkLength . fromMaybe T.empty
-  fromFormat <- fromMaybe "" <$> getParam "from"
-  toFormat <- fromMaybe "" <$> getParam "to"
-  standalone <- (==) "1" . fromMaybe "" <$> getParam "standalone"
-  compiledTemplate <- runIO . compileDefaultTemplate $ toFormat
-  let template = if standalone then either (const Nothing) Just compiledTemplate else Nothing
-  let reader = case runPure $ getReader fromFormat of
-                    Right (TextReader r, es) -> r readerOpts{
-                       readerExtensions = es }
-                    _ -> error $ "could not find reader for "
-                                  ++ T.unpack fromFormat
-  let writer = case runPure $ getWriter toFormat of
-                    Right (TextWriter w, es) -> w writerOpts{
-                       writerExtensions = es, writerTemplate = template }
-                    _ -> error $ "could not find writer for " ++
-                           T.unpack toFormat
-  let result = case runPure $ reader (tabFilter 4 text) >>= writer of
-                    Right s   -> s
-                    Left  err -> error (show err)
-  let output = encode $ object [ "html" .= result
-                               , "name" .=
-                                  if fromFormat == "markdown_strict"
-                                     then T.pack "pandoc (strict)"
-                                     else T.pack "pandoc"
-                               , "version" .= pandocVersion]
-  respond $ responseLBS status200 [(hContentType,"text/json; charset=UTF-8")] output
-
-checkLength :: Text -> IO Text
-checkLength t =
-  if T.length t > 10000
-     then error "exceeds length limit of 10,000 characters"
-     else return t
-
-writerOpts :: WriterOptions
-writerOpts = def { writerReferenceLinks = True,
-                   writerEmailObfuscation = NoObfuscation,
-                   writerHTMLMathMethod = MathJax defaultMathJaxURL,
-                   writerHighlightStyle = Just pygments }
-
-readerOpts :: ReaderOptions
-readerOpts = def
diff --git a/trypandoc/trypandoc.js b/trypandoc/trypandoc.js
new file mode 100644
--- /dev/null
+++ b/trypandoc/trypandoc.js
@@ -0,0 +1,161 @@
+"use strict";
+
+var params = {
+  text: '"hello *world*"',
+  to: 'html5',
+  from: 'markdown',
+  standalone: false,
+  citeproc: false };
+
+function permalink() {
+  let input = document.getElementById("text").value;
+  let from = document.getElementById("from").value;
+  let to = document.getElementById("to").value;
+  let standalone = document.getElementById("standalone").checked ? true : false;
+  let citeproc = document.getElementById("citeproc").checked ? true : false;
+  let href = window.location.href;
+  const URLparams = new URLSearchParams(Object.entries({text: input, from: from, to: to, standalone: standalone, citeproc: citeproc}));
+  return href.replace(/([?].*)?$/,"?" + URLparams);
+}
+
+const binaryFormats = {
+   docx: { extension: "docx",
+           mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document" },
+    odt: { extension: "odt",
+           mime: "application/vnd.oasis.opendocument.text" },
+    pptx: { extension: "pptx",
+            mime: "application/vnd.openxmlformats-officedocument.presentationml.presentation" },
+    epub:  { extension: "epub",
+             mime: "application/epub+zip" },
+    epub2: { extension: "epub",
+             mime: "application/epub+zip" },
+    epub3: { extension: "epub",
+             mime: "application/epub+zip" }
+};
+
+const binaryMimeTypes = {
+  ["application/epub+zip"]: true,
+  ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]: true,
+  ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]: true,
+  ["application/vnd.oasis.opendocument.text"]: true
+};
+
+function paramsFromURL() {
+  if (window.location.search.length > 0) {
+    const uparams = new URLSearchParams(window.location.search);
+    params.text = uparams.get("text") || "";
+    params.from = uparams.get("from") || "markdown";
+    params.to = uparams.get("to") || "html5";
+    params.standalone = uparams.get("standalone") === "true";
+    params.citeproc = uparams.get("citeproc") === "true";
+  }
+}
+
+function handleErrors(response) {
+    if (response.status == 503 || response.status == 500) {
+        throw Error("Conversion timed out.")
+    } else if (!response.ok) {
+        throw Error(response.statusText);
+    }
+    return response;
+}
+
+function convert() {
+    let text = document.getElementById("text").value;
+    let from = document.getElementById("from").value;
+    let to = document.getElementById("to").value;
+    let standalone = document.getElementById("standalone").checked;
+    let citeproc = document.getElementById("citeproc").checked;
+    params = { text: text, from: from, to: to, standalone: standalone,
+               citeproc: citeproc };
+
+    if (text && text != "") {
+       let commandString = "pandoc"
+         + " --from " + from + " --to " + to
+         + (standalone ? " --standalone" : "")
+         + (citeproc ? " --citeproc" : "") ;
+       document.getElementById("command").textContent = commandString;
+       fetch("/cgi-bin/pandoc-server.cgi", {
+         method: "POST",
+         headers: {"Content-Type": "application/json"},
+         body: JSON.stringify(params)
+        })
+       .then(handleErrors)
+       .then(response => response.text())
+       .then(restext => {
+            let binary = binaryFormats[to];
+            if (binary) {
+            document.getElementById("results").innerHTML =
+                'click to download trypandoc.' + binary.extension + '';
+          } else {
+            document.getElementById("results").textContent = restext;
+          }
+          document.getElementById("permalink").href = permalink();
+       })
+       .catch(error => {
+         document.getElementById("results").textContent = error;
+       }
+       );
+    };
+}
+
+(function() {
+    paramsFromURL();
+    document.getElementById("text").value = params.text;
+    document.getElementById("from").value = params.from;
+    document.getElementById("to").value = params.to;
+    document.getElementById("standalone").checked = params.standalone;
+    document.getElementById("citeproc").checked = params.citeproc;
+
+    document.getElementById("convert").onclick = convert;
+    document.getElementById("from").onchange = convert;
+    document.getElementById("to").onchange = convert;
+    document.getElementById("standalone").onchange = convert;
+    document.getElementById("citeproc").onchange = convert;
+
+    document.getElementById("examples").onchange =
+      (e => window.location.href = e.target.value );
+
+    const fileInput = document.getElementById('loadfile');
+
+    // Listen for the change event so we can capture the file
+    fileInput.addEventListener('change', (e) => {
+      // Get a reference to the file
+      const file = e.target.files[0];
+      const mimetype = file.type;
+      let binary = binaryMimeTypes[mimetype];
+
+      // Encode the file using the FileReader API
+      const reader = new FileReader();
+      let inputtext = document.getElementById("text");
+      reader.onloadend = () => {
+        // Use a regex to remove data url part
+        if (binary) {
+          const base64String = reader.result
+           .replace('data:', '')
+           .replace(/^.+,/, '');
+          inputtext.value = base64String;
+	} else {
+          inputtext.value = reader.result;
+        }
+      };
+      if (binary) {
+        reader.readAsDataURL(file);
+      } else {
+        reader.readAsText(file);
+      }
+    });
+
+    fetch("/cgi-bin/pandoc-server.cgi/version")
+       .then(handleErrors)
+       .then(response => response.text())
+       .then(restext =>
+           document.getElementById("version").textContent = restext
+         );
+
+    convert();
+
+})();
+