diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ pandoc-pyplot uses [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +Release 2.1.2.0 +--------------- + +* Added the "flags" configuration option, which allows to pass command-line flags to the Python interpreter. For example, warnings can be suppressed using the `-Wignore` flag. +* Refactoring of the script check mechanism. It will be much easier to extend in the future. +* Updated the command-line help with an example combining pandoc-pyplot and pandoc-crossref +* Default Python interpreter is now "python" on Windows and __"python3" otherwise__. + Release 2.1.1.1 --------------- diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -135,13 +135,16 @@ include: mystyle.py format: jpeg dpi: 150 +flags: [-O, -Wignore] ``` These values override the default values, which are equivalent to: ```yaml # Defaults if no configuration is provided. +# Note that the default interpreter name on MacOS and Unix is python3 interpreter: python +flags: [] directory: generated/ format: png dpi: 80 @@ -187,7 +190,7 @@ ### Requirements -This filter only works with the Matplotlib plotting library. Therefore, you a Python interpreter and at least [Matplotlib](https://matplotlib.org/) installed. The name of the Python interpreter to use can be specified in a `.pandoc-pyplot.yml` file; by default, `pandoc-pyplot` will use the `"python"` name. +This filter only works with the Matplotlib plotting library. Therefore, you a Python interpreter and at least [Matplotlib](https://matplotlib.org/) installed. The name of the Python interpreter to use can be specified in a `.pandoc-pyplot.yml` file; by default, `pandoc-pyplot` will use the `"python"` name on Windows, and `"python3"` otherwise. You can use the filter with Pandoc as follows: diff --git a/executable/Main.hs b/executable/Main.hs --- a/executable/Main.hs +++ b/executable/Main.hs @@ -32,7 +32,6 @@ | Version | Formats | Manual - | InvalidFlag deriving (Eq) parseFlag :: [String] -> Maybe Flag @@ -49,14 +48,13 @@ | f == Version = showVersion | f == Formats = showFormats | f == Manual = showManual - | otherwise = showError -- Includes InvalidFlag + | otherwise = error "Unknown flag" where showHelp = putStrLn help showVersion = putStrLn (V.showVersion version) showFormats = putStrLn . mconcat . intersperse ", " . fmap show $ supportedSaveFormats showManual = writeSystemTempFile "pandoc-pyplot-manual.html" (T.unpack manualHtml) >>= \fp -> openBrowser ("file:///" <> fp) >> return () - showError = putStrLn "Invalid flag. Please read `pandoc-pyplot --help` for information on valid flags." help :: String help = @@ -73,13 +71,17 @@ \ -f, --formats Show supported output figure formats and exit.\n\ \ -m, --manual Open the manual page in the default web browser and exit.\n\ \\n\ - \ To use with pandoc: \n\ - \ pandoc -s --filter pandoc-pyplot input.md --output output.html\n\ + \ Example usage with pandoc: \n\ \\n\ + \ > pandoc --filter pandoc-pyplot input.md --output output.html\n\ + \\n\ \ If you use pandoc-pyplot in combination with other filters, you probably want\n\ - \ to run pandoc-pyplot first. See the manual (`pandoc-pyplot --manual`) for details.\n\ + \ to run pandoc-pyplot first. Here is an example with pandoc-crossref: \n\ \\n\ - \ More information can be found in the repository README, located at \n\ + \ > pandoc --filter pandoc-pyplot --filter pandoc-crossref -i input.md -o output.pdf\n\ + \\n\ + \ More information can be found via the manual (pandoc-pyplot --manual) or the\n\ + \ repository README, located at \n\ \ https://github.com/LaurentRDC/pandoc-pyplot\n" main :: IO () diff --git a/pandoc-pyplot.cabal b/pandoc-pyplot.cabal --- a/pandoc-pyplot.cabal +++ b/pandoc-pyplot.cabal @@ -1,5 +1,5 @@ name: pandoc-pyplot -version: 2.1.1.1 +version: 2.1.2.0 cabal-version: >= 1.12 synopsis: A Pandoc filter to include figures generated from Python code blocks description: A Pandoc filter to include figures generated from Python code blocks. Keep the document and Python code in the same location. Output from Matplotlib is captured and included as a figure. diff --git a/src/Text/Pandoc/Filter/Pyplot.hs b/src/Text/Pandoc/Filter/Pyplot.hs --- a/src/Text/Pandoc/Filter/Pyplot.hs +++ b/src/Text/Pandoc/Filter/Pyplot.hs @@ -126,31 +126,22 @@ import Paths_pandoc_pyplot (version) -import System.Directory (createDirectoryIfMissing, - doesFileExist) -import System.FilePath (makeValid, takeDirectory) +import System.FilePath (makeValid) import Text.Pandoc.Definition import Text.Pandoc.Walk (walkM) import Text.Pandoc.Filter.Pyplot.Internal - --- | Possible errors returned by the filter -data PandocPyplotError - = ScriptError Int -- ^ Running Python script has yielded an error - | BlockingCallError -- ^ Python script contains a block call to 'show()' - deriving (Eq) - -instance Show PandocPyplotError where - show (ScriptError exitcode) = "Script error: plot could not be generated. Exit code " <> (show exitcode) - show BlockingCallError = "Script contains a blocking call to show, like 'plt.show()'" +-- | Code block class that will trigger the filter +filterClass :: String +filterClass = "pyplot" -- | Determine inclusion specifications from Block attributes. -- Note that the @".pyplot"@ class is required, but all other parameters are optional parseFigureSpec :: Configuration -> Block -> IO (Maybe FigureSpec) parseFigureSpec config (CodeBlock (id', cls, attrs) content) - | "pyplot" `elem` cls = Just <$> figureSpec + | filterClass `elem` cls = Just <$> figureSpec | otherwise = return Nothing where attrs' = Map.fromList attrs @@ -166,24 +157,11 @@ format = fromMaybe (defaultSaveFormat config) $ join $ saveFormatFromString <$> Map.lookup saveFormatKey attrs' dir = makeValid $ Map.findWithDefault (defaultDirectory config) directoryKey attrs' dpi' = fromMaybe (defaultDPI config) $ read <$> Map.lookup dpiKey attrs' - blockAttrs' = (id', filter (/= "pyplot") cls, filteredAttrs) + blockAttrs' = (id', filter (/= filterClass) cls, filteredAttrs) return $ FigureSpec caption' fullScript format dir dpi' blockAttrs' parseFigureSpec _ _ = return Nothing --- | Run the Python script. In case the file already exists, we can safely assume --- there is no need to re-run it. -runScriptIfNecessary :: Configuration -> FigureSpec -> IO ScriptResult -runScriptIfNecessary config spec = do - createDirectoryIfMissing True . takeDirectory $ figurePath spec - fileAlreadyExists <- doesFileExist $ figurePath spec - result <- if fileAlreadyExists - then return ScriptSuccess - else runTempPythonScript (interpreter config) (addPlotCapture spec) - case result of - ScriptFailure code -> return $ ScriptFailure code - ScriptSuccess -> T.writeFile (sourceCodePath spec) (script spec) >> return ScriptSuccess - -- | Main routine to include Matplotlib plots. -- Code blocks containing the attributes @.pyplot@ are considered -- Python plotting scripts. All other possible blocks are ignored. @@ -192,13 +170,12 @@ parsed <- parseFigureSpec config block case parsed of Nothing -> return $ Right block - Just spec -> - if hasBlockingShowCall (script spec) - then return $ Left BlockingCallError - else handleResult spec <$> runScriptIfNecessary config spec + Just spec -> handleResult spec <$> runScriptIfNecessary config spec + where - handleResult _ (ScriptFailure code) = Left $ ScriptError code - handleResult spec ScriptSuccess = Right $ toImage spec + handleResult _ (ScriptChecksFailed msg) = Left $ ScriptChecksFailedError msg + handleResult _ (ScriptFailure code) = Left $ ScriptError code + handleResult spec ScriptSuccess = Right $ toImage spec -- | Highest-level function that can be walked over a Pandoc tree. -- All code blocks that have the '.pyplot' parameter will be considered diff --git a/src/Text/Pandoc/Filter/Pyplot/Configuration.hs b/src/Text/Pandoc/Filter/Pyplot/Configuration.hs --- a/src/Text/Pandoc/Filter/Pyplot/Configuration.hs +++ b/src/Text/Pandoc/Filter/Pyplot/Configuration.hs @@ -63,6 +63,7 @@ , defaultSaveFormat_ :: String , defaultDPI_ :: Int , interpreter_ :: String + , flags_ :: [String] } instance FromJSON ConfigPrecursor where @@ -72,6 +73,7 @@ <*> v .:? (T.pack saveFormatKey) .!= (extension $ defaultSaveFormat def) <*> v .:? (T.pack dpiKey) .!= (defaultDPI def) <*> v .:? "interpreter" .!= (interpreter def) + <*> v .:? "flags" .!= (flags def) parseJSON _ = fail "Could not parse the configuration" @@ -84,6 +86,7 @@ , defaultSaveFormat = saveFormat' , defaultDPI = defaultDPI_ prec , interpreter = interpreter_ prec + , flags = flags_ prec } diff --git a/src/Text/Pandoc/Filter/Pyplot/Scripting.hs b/src/Text/Pandoc/Filter/Pyplot/Scripting.hs --- a/src/Text/Pandoc/Filter/Pyplot/Scripting.hs +++ b/src/Text/Pandoc/Filter/Pyplot/Scripting.hs @@ -13,47 +13,83 @@ -} module Text.Pandoc.Filter.Pyplot.Scripting ( runTempPythonScript - , hasBlockingShowCall + , runScriptIfNecessary ) where import Data.Hashable (hash) -import Data.Monoid (Any(..)) +import Data.List (intersperse) +import Data.Monoid (Any(..), (<>)) import qualified Data.Text as T import qualified Data.Text.IO as T +import System.Directory (createDirectoryIfMissing, + doesFileExist) import System.Exit (ExitCode (..)) -import System.FilePath (()) +import System.FilePath ((), takeDirectory) import System.IO.Temp (getCanonicalTemporaryDirectory) import System.Process.Typed (runProcess, shell) import Text.Pandoc.Filter.Pyplot.Types +import Text.Pandoc.Filter.Pyplot.FigureSpec +-- | Detect the presence of a blocking show call, for example "plt.show()" +checkBlockingShowCall :: PythonScript -> CheckResult +checkBlockingShowCall script' = + if hasShowCall + then CheckFailed "The script has a blocking call to `matplotlib.pyplot.show`. " + else CheckPassed + where + scriptLines = T.lines script' + hasShowCall = getAny $ mconcat $ Any <$> + [ "plt.show()" `elem` scriptLines + , "pyplot.show()" `elem` scriptLines + , "matplotlib.pyplot.show()" `elem` scriptLines + ] + +-- | List of all script checks +-- This might be overkill right now but extension to other languages will be easier +scriptChecks :: [PythonScript -> CheckResult] +scriptChecks = [checkBlockingShowCall] + -- | Take a python script in string form, write it in a temporary directory, -- then execute it. runTempPythonScript :: String -- ^ Interpreter (e.g. "python" or "python35") + -> [String] -- ^ Command-line flags -> PythonScript -- ^ Content of the script - -> IO ScriptResult -- ^ Result with exit code. -runTempPythonScript interpreter' script' = do - -- We involve the script hash as a temporary filename - -- so that there is never any collision - scriptPath <- ( hashedPath) <$> getCanonicalTemporaryDirectory - T.writeFile scriptPath script' + -> IO ScriptResult -- ^ Result. +runTempPythonScript interpreter' flags' script' = case checkResult of + CheckFailed msg -> return $ ScriptChecksFailed msg + CheckPassed -> do + -- We involve the script hash as a temporary filename + -- so that there is never any collision + scriptPath <- ( hashedPath) <$> getCanonicalTemporaryDirectory + T.writeFile scriptPath script' - ec <- runProcess $ shell $ mconcat [interpreter', " ", show scriptPath] - case ec of - ExitSuccess -> return ScriptSuccess - ExitFailure code -> return $ ScriptFailure code + let command = mconcat . intersperse " " $ [interpreter'] <> flags' <> [show scriptPath] + + ec <- runProcess . shell $ command + case ec of + ExitSuccess -> return ScriptSuccess + ExitFailure code -> return $ ScriptFailure code where + checkResult = mconcat $ scriptChecks <*> [script'] hashedPath = show . hash $ script' --- | Detect the presence of a blocking show call, for example "plt.show()" -hasBlockingShowCall :: PythonScript -> Bool -hasBlockingShowCall script' = - anyOf - [ "plt.show()" `elem` scriptLines - , "pyplot.show()" `elem` scriptLines - , "matplotlib.pyplot.show()" `elem` scriptLines - ] - where - scriptLines = T.lines script' - anyOf xs = getAny $ mconcat $ Any <$> xs +-- | Run the Python script. In case the file already exists, we can safely assume +-- there is no need to re-run it. +runScriptIfNecessary :: Configuration -> FigureSpec -> IO ScriptResult +runScriptIfNecessary config spec = do + createDirectoryIfMissing True . takeDirectory $ figurePath spec + + fileAlreadyExists <- doesFileExist $ figurePath spec + result <- if fileAlreadyExists + then return ScriptSuccess + else runTempPythonScript (interpreter config) (flags config) scriptWithCapture + + case result of + ScriptSuccess -> T.writeFile (sourceCodePath spec) (script spec) >> return ScriptSuccess + ScriptFailure code -> return $ ScriptFailure code + ScriptChecksFailed msg -> return $ ScriptChecksFailed msg + + where + scriptWithCapture = addPlotCapture spec diff --git a/src/Text/Pandoc/Filter/Pyplot/Types.hs b/src/Text/Pandoc/Filter/Pyplot/Types.hs --- a/src/Text/Pandoc/Filter/Pyplot/Types.hs +++ b/src/Text/Pandoc/Filter/Pyplot/Types.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} {-| Module : Text.Pandoc.Filter.Pyplot.Types Copyright : (c) Laurent P René de Cotret, 2019 @@ -14,18 +15,53 @@ import Data.Char (toLower) import Data.Default.Class (Default, def) import Data.Hashable (Hashable, hashWithSalt) +import Data.Semigroup as Sem import Data.Text (Text) import Text.Pandoc.Definition (Attr) + -- | String representation of a Python script type PythonScript = Text + -- | Possible result of running a Python script data ScriptResult = ScriptSuccess + | ScriptChecksFailed String | ScriptFailure Int + +-- | Result of checking scripts for problems +data CheckResult + = CheckPassed + | CheckFailed String + deriving (Eq) + +instance Sem.Semigroup CheckResult where + (<>) CheckPassed a = a + (<>) a CheckPassed = a + (<>) (CheckFailed msg1) (CheckFailed msg2) = CheckFailed (msg1 <> msg2) + +instance Monoid CheckResult where + mempty = CheckPassed + +#if !(MIN_VERSION_base(4,11,0)) + mappend = (<>) +#endif + + +-- | Possible errors returned by the filter +data PandocPyplotError + = ScriptError Int -- ^ Running Python script has yielded an error + | ScriptChecksFailedError String -- ^ Python script did not pass all checks + deriving (Eq) + +instance Show PandocPyplotError where + show (ScriptError exitcode) = "Script error: plot could not be generated. Exit code " <> (show exitcode) + show (ScriptChecksFailedError msg) = "Script did not pass all checks: " <> msg + + -- | Generated figure file format supported by pandoc-pyplot. data SaveFormat = PNG @@ -62,6 +98,17 @@ extension :: SaveFormat -> String extension fmt = mconcat [".", fmap toLower . show $ fmt] +-- | Default interpreter should be Python 3, which has a different +-- name on Windows ("python") vs Unix ("python3") +-- +-- @since 2.1.2.0 +defaultPlatformInterpreter :: String +#if defined(mingw32_HOST_OS) +defaultPlatformInterpreter = "python" +#else +defaultPlatformInterpreter = "python3" +#endif + -- | Configuration of pandoc-pyplot, describing the default behavior -- of the filter. -- @@ -76,6 +123,7 @@ , defaultSaveFormat :: SaveFormat -- ^ The default save format of generated figures. , defaultDPI :: Int -- ^ The default dots-per-inch value for generated figures. , interpreter :: String -- ^ The name of the interpreter to use to render figures. + , flags :: [String] -- ^ Command-line flags to be passed to the Python interpreger, e.g. ["-O", "-Wignore"] } deriving (Eq, Show) @@ -85,9 +133,11 @@ , defaultIncludeScript = mempty , defaultSaveFormat = PNG , defaultDPI = 80 - , interpreter = "python" + , interpreter = defaultPlatformInterpreter + , flags = mempty } + -- | Datatype containing all parameters required to run pandoc-pyplot data FigureSpec = FigureSpec { caption :: String -- ^ Figure caption. @@ -105,4 +155,4 @@ , fromEnum . saveFormat $ spec , directory spec, dpi spec , blockAttrs spec - ) + ) diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -106,6 +106,7 @@ testCase "writes output files in appropriate directory" $ do tempDir <- ( "test-file-creation") <$> getCanonicalTemporaryDirectory ensureDirectoryExistsAndEmpty tempDir + let codeBlock = (addDirectory tempDir $ plotCodeBlock "import matplotlib.pyplot as plt\n") _ <- makePlot' def codeBlock filesCreated <- length <$> listDirectory tempDir @@ -118,6 +119,7 @@ testCase "includes plot inclusions" $ do tempDir <- ( "test-file-inclusion") <$> getCanonicalTemporaryDirectory ensureDirectoryExistsAndEmpty tempDir + let codeBlock = (addInclusion "test/fixtures/include.py" $ addDirectory tempDir $ plotCodeBlock "import matplotlib.pyplot as plt\n") @@ -134,6 +136,7 @@ testCase "saves in the appropriate format" $ do tempDir <- ( "test-safe-format") <$> getCanonicalTemporaryDirectory ensureDirectoryExistsAndEmpty tempDir + let codeBlock = (addSaveFormat JPG $ addDirectory tempDir $ @@ -151,15 +154,20 @@ testBlockingCallError :: TestTree testBlockingCallError = testCase "raises an exception for blocking calls" $ do - tempDir <- ( "test-blocking-call-error") <$>getCanonicalTemporaryDirectory - let codeBlock = plotCodeBlock "import matplotlib.pyplot as plt\nplt.show()" + tempDir <- ( "test-blocking-call-error") <$> getCanonicalTemporaryDirectory + ensureDirectoryExistsAndEmpty tempDir + + let codeBlock = addDirectory tempDir $ plotCodeBlock "import matplotlib.pyplot as plt\nplt.show()" result <- makePlot' def codeBlock case result of Right block -> assertFailure "did not catch the expected blocking call" Left error -> - if error == BlockingCallError + if isCheckError error then pure () - else assertFailure "did not catch the expected blocking call" + else assertFailure "An error was caught but not the expected blocking call" + where + isCheckError (ScriptChecksFailedError msg) = True + isCheckError _ = False ------------------------------------------------------------------------------- ------------------------------------------------------------------------------- @@ -169,6 +177,7 @@ testCase "appropriately parses Markdown captions" $ do tempDir <- ( "test-caption-parsing") <$> getCanonicalTemporaryDirectory ensureDirectoryExistsAndEmpty tempDir + -- Note that this test is fragile, in the sense that the expected result must be carefully -- constructed let expected = [B.Strong [B.Str "caption"]] @@ -191,6 +200,7 @@ testConfig = do tempDir <- ( "test-with-config") <$> getCanonicalTemporaryDirectory ensureDirectoryExistsAndEmpty tempDir + return $ def {defaultDirectory = tempDir, defaultSaveFormat = JPG} testOverridingConfiguration :: TestTree @@ -237,6 +247,7 @@ let config = def { defaultDirectory = "generated/other" , defaultSaveFormat = JPG , defaultDPI = 150 + , flags = ["-Wignore"] } parsedConfig <- configuration "test/fixtures/.pandoc-pyplot.yml" assertEqual "" config parsedConfig