pandoc-plot 0.4.0.1 → 0.5.0.0

raw patch · 14 files changed

+218/−68 lines, 14 filesdep +asyncdep +criteriondep −parallel-iodep ~basedep ~data-default-classdep ~pandoc-typesPVP ok

version bump matches the API change (PVP)

Dependencies added: async, criterion

Dependencies removed: parallel-io

Dependency ranges changed: base, data-default-class, pandoc-types, text

API changes (from Hackage documentation)

Files

• CHANGELOG.md +9/−0
• README.md +4/−4
• benchmark/MatplotlibGallery.hs +26/−0
• benchmark/bench.hs +31/−0
• executable/Main.hs +8/−7
• executable/ManPage.hs +1/−1
• pandoc-plot.cabal +18/−5
• src/Text/Pandoc/Filter/Plot.hs +59/−31
• src/Text/Pandoc/Filter/Plot/Configuration.hs +16/−8
• src/Text/Pandoc/Filter/Plot/Renderers.hs +4/−10
• src/Text/Pandoc/Filter/Plot/Renderers/Matplotlib.hs +19/−1
• src/Text/Pandoc/Filter/Plot/Types.hs +1/−1
• tests/Common.hs +21/−0
• tests/Main.hs +1/−0

@@ -2,6 +2,15 @@ 
 pandoc-plot uses [Semantic Versioning](http://semver.org/spec/v2.0.0.html)
 
+Release 0.5.0.0
+---------------
+
+* The `pandoc-plot` executable will now process documents in parallel. This should dramatically speed up processing of large documents with lots of figures. 
+This happens automatically through the function `plotTransform`.
+* Added a benchmarking suite.
+* Added `defaultConfiguration` so that people don't have to install the `data-default` package to get access to default configuration values. 
+* Added a check for the `matplotlib` toolkit, preventing users from using `matplotlib.pyplot.show` in figures. This would halt `pandoc-plot`.
+
 Release 0.4.0.1
 ---------------
 

@@ -178,11 +178,11 @@ 
 ### Performance
 
-`pandoc-plot` minimizes work, only generating figures if it absolutely must, i.e. if the content has changed. 
+`pandoc-plot` minimizes work, only generating figures if it absolutely must, i.e. if the content has changed. `pandoc-plot` will save the hash of the source code used to generate a figure in its filename. Before generating a figure, `pandoc-plot` will check it this figure already exists based on the hash of its source! This also means that there is no way to directly name figures.
 
-`pandoc-plot` will save the hash of the source code used to generate a figure in its filename. Before generating a figure, `pandoc-plot` will check it this figure already exists based on the hash of its source! This also means that there is no way to directly name figures.
+Moreover, starting with version 0.5.0.0, `pandoc-plot` takes advantage of multicore CPUs, rendering figures **in parallel**.
 
-Therefore, you can confidently run the filter on very large documents containing hundreds of figures --- like a book or a thesis --- and only the figures which have changed will be re-generated.
+Therefore, you can confidently run the filter on very large documents containing hundreds of figures, like a book or a thesis.
 
 ### Compatibility with pandoc-crossref
 
@@ -257,7 +257,7 @@ 
 A file like the above sets the **default** values; you can still override them in documents directly.
 
-Using `pandoc-plot --write-example-config` will write the default configuration to a file which you can then customize.
+Using `pandoc-plot write-example-config` will write the default configuration to a file which you can then customize.
 
 ### Executables
 

@@ -0,0 +1,26 @@+{-# LANGUAGE TemplateHaskell #-}
+
+module MatplotlibGallery where
+
+import           Data.String                (fromString)
+
+import qualified Data.Text                  as T
+import qualified Data.Text.IO               as T
+
+import           Language.Haskell.TH.Syntax
+
+
+galleryItem :: FilePath -> Q Exp
+galleryItem fp = do
+    qAddDependentFile fp
+    txt <- runIO $ T.readFile fp
+    strToExp $ T.unpack txt
+    where
+        strToExp :: String -> Q Exp
+        strToExp s = return $ VarE 'fromString `AppE` LitE (StringL s)
+
+galleryItem1 :: Q Exp
+galleryItem1 = galleryItem "benchmark/gallery_item_1.py"
+
+galleryItem2 :: Q Exp
+galleryItem2 = galleryItem "benchmark/gallery_item_2.py"

@@ -0,0 +1,31 @@+{-# LANGUAGE TemplateHaskell #-}
+
+import Criterion.Main
+
+import Text.Pandoc.Definition
+import Text.Pandoc.Filter.Plot
+import Text.Pandoc.Filter.Plot.Internal
+
+import MatplotlibGallery (galleryItem1, galleryItem2)
+
+main :: IO ()
+main = 
+    defaultMain [
+        envWithCleanup (return ()) (\_ -> cleanupEnv) $ \_ -> 
+            bgroup "main" [
+                  bench "filter" $ nfIO (plotTransform defaultConfiguration benchDoc)
+                ]
+    ]
+
+cleanupEnv :: IO ()
+cleanupEnv = cleanOutputDirs defaultConfiguration benchDoc >> return ()
+
+
+codeBlock :: Script -> Block
+codeBlock = CodeBlock (mempty, [cls Matplotlib], mempty)
+
+
+benchDoc :: Pandoc
+benchDoc = Pandoc mempty [ codeBlock $(galleryItem1)
+                         , codeBlock $(galleryItem2)
+                         ]

@@ -8,7 +8,6 @@ import           Control.Applicative              ((<|>))
 import           Control.Monad                    (join, forM_)
 
-import           Data.Default.Class               (def)
 import           Data.List                        (intersperse, (\\))
 import           Data.Monoid                      ((<>))
 import           Data.Text                        (unpack)
@@ -22,11 +21,13 @@ import           System.IO.Temp                   (writeSystemTempFile)
 
 import           Text.Pandoc.Filter.Plot          (availableToolkits,
-                                                   plotTransform)
-import           Text.Pandoc.Filter.Plot.Internal (cls, Configuration(..),
-                                                   supportedSaveFormats, 
-                                                   configuration, toolkits, 
-                                                   readDoc, cleanOutputDirs)
+                                                   plotTransform,
+                                                   defaultConfiguration, 
+                                                   configuration, 
+                                                   Configuration(..))
+import           Text.Pandoc.Filter.Plot.Internal (cls, supportedSaveFormats, 
+                                                   toolkits, readDoc, 
+                                                   cleanOutputDirs, )
 
 import           Text.Pandoc                      (pandocVersion)
 import           Text.Pandoc.Definition           (pandocTypesVersion)
@@ -143,7 +144,7 @@     configExists <- doesFileExist ".pandoc-plot.yml"
     if configExists
         then configuration ".pandoc-plot.yml" 
-        else return def
+        else return defaultConfiguration
 
 
 showFullVersion :: IO ()

@@ -1,4 +1,4 @@-{-# LANGUAGE TemplateHaskellQuotes #-}
+{-# LANGUAGE TemplateHaskell #-}
 {-|
 This module was inspired by pandoc-crossref
 |-}

@@ -1,5 +1,5 @@ name:           pandoc-plot
-version:        0.4.0.1
+version:        0.5.0.0
 cabal-version:  >= 1.12
 synopsis:       A Pandoc filter to include figures generated from code blocks using your plotting toolkit of choice.
 description:    A Pandoc filter to include figures generated from code blocks. Keep the document and code in the same location. Output is captured and included as a figure.
@@ -57,7 +57,7 @@         , hashable           >= 1     && < 2
         , pandoc             >= 2.8   && < 3
         , pandoc-types       >= 1.20  && < 2
-        , parallel-io        >= 0.3.3 && < 0.4
+        , async              >= 2     && < 3
         , shakespeare        >= 2.0   && < 3
         , temporary
         , text               >= 1     && < 2
@@ -75,11 +75,10 @@         Paths_pandoc_plot
     hs-source-dirs:
         executable
-    ghc-options: -Wall -Wcompat -rtsopts -threaded -with-rtsopts=-N
+    ghc-options: -Wall -Wcompat -rtsopts -O2 -threaded -with-rtsopts=-N
     build-depends:
           base                  >= 4.11 && <5
         , directory
-        , data-default-class    >= 0.1.2
         , deepseq
         , filepath
         , githash               >= 0.1.3.0 && < 1
@@ -114,4 +113,18 @@                    , text
                    , mtl                >= 2.2   && < 2.3
     default-language: Haskell2010
-  +  
+benchmark benchmark-pandoc-plot
+    type: exitcode-stdio-1.0
+    main-is: bench.hs
+    other-modules:
+        MatplotlibGallery
+    hs-source-dirs: benchmark
+    build-depends:   base
+                   , pandoc-plot
+                   , pandoc-types
+                   , criterion          >= 1.0 && < 1.6
+                   , template-haskell   > 2.7 && < 3
+                   , text
+    ghc-options:   -Wall -Wcompat -rtsopts -O2 -threaded -with-rtsopts=-N
+    default-language: Haskell2010

@@ -1,4 +1,6 @@ {-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE RecordWildCards   #-}
+{-# LANGUAGE FlexibleContexts  #-}
 {-|
 Module      : $header$
 Description : Pandoc filter to create figures from code blocks using your plotting toolkit of choice
@@ -69,42 +71,53 @@     , cleanOutputDirs
     -- * Runtime configuration
     , configuration
+    , defaultConfiguration
     , Configuration(..)
     , SaveFormat(..)
     , Script
     -- * For testing and internal purposes ONLY
     , make
+    , make'
+    , PandocPlotError(..)
     , readDoc
     , availableToolkits
     , unavailableToolkits
     ) where
 
-import           Control.Monad.IO.Class           (liftIO)
-import           Control.Monad.Reader             (runReaderT)
+import Control.Concurrent.Async          (mapConcurrently)
 
-import           System.IO                        (hPutStrLn, stderr)
+import Control.Monad.Reader              (runReaderT)
 
-import           Text.Pandoc.Definition
-import           Text.Pandoc.Walk                 (walkM)
+import System.IO                         (hPutStrLn, stderr)
 
-import           Text.Pandoc.Filter.Plot.Internal
+import Text.Pandoc.Definition            (Pandoc(..), Block)
+import Text.Pandoc.Walk                  (walkM, Walkable)
 
+import Text.Pandoc.Filter.Plot.Internal
 
+
 -- | Highest-level function that can be walked over a Pandoc tree.
 -- All code blocks that have the appropriate class names will be considered
 -- figures, e.g. @.matplotlib@.
 --
+-- This function can be made to operation on whole @Pandoc@ documents. However,
+-- you should prefer the @plotTransform@ function for whole documents, as it
+-- is optimized for parallel operations.
+--
 -- Failing to render a figure does not stop the filter, so that you may run the filter
 -- on documents without having all necessary toolkits installed. In this case, error
 -- messages are printed to stderr, and blocks are left unchanged.
-makePlot :: Configuration -- ^ Configuration for default values
-         -> Block 
-         -> IO Block
-makePlot conf block = maybe (return block) (\tk -> make tk conf block) (plotToolkit block)
+makePlot :: Walkable Block a
+         => Configuration -- ^ Configuration for default values
+         -> a             -- ^ Input block or document
+         -> IO a
+makePlot conf = walkM makePlot'
+    where
+        makePlot' block = maybe (return block) (\tk -> make tk conf block) (plotToolkit block)
 
 
--- | Walk over an entire Pandoc document, changing appropriate code blocks
--- into figures. 
+-- | Walk over an entire Pandoc document, transforming appropriate code blocks
+-- into figures. This function will operate on blocks in parallel if possible.
 --
 -- Failing to render a figure does not stop the filter, so that you may run the filter
 -- on documents without having all necessary toolkits installed. In this case, error
@@ -112,7 +125,8 @@ plotTransform :: Configuration -- ^ Configuration for default values
               -> Pandoc        -- ^ Input document
               -> IO Pandoc
-plotTransform conf = walkM $ makePlot conf
+plotTransform conf (Pandoc meta blocks) =
+    mapConcurrently (makePlot conf) blocks >>= return . Pandoc meta
 
 
 -- | Force to use a particular toolkit to render appropriate code blocks.
@@ -124,22 +138,36 @@      -> Configuration -- ^ Configuration for default values.
      -> Block 
      -> IO Block
-make tk conf block = runReaderT (makePlot' block) (PlotEnv tk conf)
+make tk conf blk = 
+    either (const (return blk) . showErr) return =<< make' tk conf blk
     where
-        makePlot' :: Block -> PlotM Block
-        makePlot' blk 
-            = parseFigureSpec blk 
-            >>= maybe 
-                    (return blk) 
-                    (\s -> runScriptIfNecessary s >>= handleResult s)
-            where                
-                handleResult spec ScriptSuccess         = return $ toImage (captionFormat conf) spec
-                handleResult _ (ScriptChecksFailed msg) = do
-                    liftIO $ hPutStrLn stderr $ " ERROR (pandoc-plot) The script check failed with message: " <> msg 
-                    return blk
-                handleResult _ (ScriptFailure _ code) = do
-                    liftIO $ hPutStrLn stderr $ "ERROR (pandoc-plot) The script failed with exit code " <> show code 
-                    return blk
-                handleResult _ (ToolkitNotInstalled tk') = do
-                    liftIO $ hPutStrLn stderr $ "ERROR (pandoc-plot) The " <> show tk' <> " toolkit is required but not installed."
-                    return blk+        showErr e = hPutStrLn stderr $ show e
+
+
+make' :: Toolkit 
+      -> Configuration 
+      -> Block 
+      -> IO (Either PandocPlotError Block)
+make' tk conf block = runReaderT (make'' block) (PlotEnv tk conf)
+    where
+        make'' :: Block -> PlotM (Either PandocPlotError Block)
+        make'' blk = parseFigureSpec blk
+                    >>= maybe
+                        (return $ Right blk)
+                        (\s -> runScriptIfNecessary s >>= handleResult s)
+            where
+                handleResult spec ScriptSuccess          = return $ Right $ toImage (captionFormat conf) spec
+                handleResult _ (ScriptFailure msg code)  = return $ Left (ScriptRuntimeError msg code) 
+                handleResult _ (ScriptChecksFailed msg)  = return $ Left (ScriptChecksFailedError msg)
+                handleResult _ (ToolkitNotInstalled tk') = return $ Left (ToolkitNotInstalledError tk') 
+
+
+data PandocPlotError
+    = ScriptRuntimeError String Int
+    | ScriptChecksFailedError String
+    | ToolkitNotInstalledError Toolkit
+
+instance Show PandocPlotError where
+    show (ScriptRuntimeError _ exitcode) = "ERROR (pandoc-plot) The script failed with exit code " <> show exitcode <> "."
+    show (ScriptChecksFailedError msg)   = "ERROR (pandoc-plot) A script check failed with message: " <> msg <> "."
+    show (ToolkitNotInstalledError tk)   = "ERROR (pandoc-plot) The " <> show tk <> " toolkit is required but not installed."

@@ -15,6 +15,7 @@ 
 module Text.Pandoc.Filter.Plot.Configuration (
       configuration
+    , defaultConfiguration
 ) where
 
 import           Data.Default.Class     (Default, def)
@@ -37,15 +38,22 @@ configuration fp = (loadYamlSettings [fp] [] ignoreEnv) >>= renderConfig
 
 
+-- | Default configuration values.
+--
+-- @since 0.5.0.0
+defaultConfiguration :: Configuration
+defaultConfiguration = def
+
+
 -- We define a precursor type because preambles are best specified as file paths,
 -- but we want to read those files before building a full
 -- @Configuration@ value.
 data ConfigPrecursor = ConfigPrecursor
-    { _defaultDirectory  :: !FilePath   -- ^ The default directory where figures will be saved.
-    , _defaultWithSource :: !Bool       -- ^ The default behavior of whether or not to include links to source code and high-res
-    , _defaultDPI        :: !Int        -- ^ The default dots-per-inch value for generated figures. Renderers might ignore this.
-    , _defaultSaveFormat :: !SaveFormat -- ^ The default save format of generated figures.
-    , _captionFormat     :: Format        -- ^ Caption format in Pandoc notation, e.g. "markdown+tex_math_dollars".
+    { _defaultDirectory  :: !FilePath    -- ^ The default directory where figures will be saved.
+    , _defaultWithSource :: !Bool        -- ^ The default behavior of whether or not to include links to source code and high-res
+    , _defaultDPI        :: !Int         -- ^ The default dots-per-inch value for generated figures. Renderers might ignore this.
+    , _defaultSaveFormat :: !SaveFormat  -- ^ The default save format of generated figures.
+    , _captionFormat     :: !Format      -- ^ Caption format in Pandoc notation, e.g. "markdown+tex_math_dollars".
     
     , _matplotlibPrec    :: !MatplotlibPrecursor
     , _matlabPrec        :: !MatlabPrecursor
@@ -137,9 +145,9 @@     parseJSON (Null) = return def -- In case of empty file
     parseJSON (Object v) = do
         
-        _defaultDirectory  <- v .:? (tshow DirectoryK)     .!= (defaultDirectory def)
-        _defaultWithSource <- v .:? (tshow WithSourceK)    .!= (defaultWithSource def)
-        _defaultDPI        <- v .:? (tshow DpiK)           .!= (defaultDPI def)
+        _defaultDirectory  <- v .:? (tshow DirectoryK)     .!= (_defaultDirectory def)
+        _defaultWithSource <- v .:? (tshow WithSourceK)    .!= (_defaultWithSource def)
+        _defaultDPI        <- v .:? (tshow DpiK)           .!= (_defaultDPI def)
         _defaultSaveFormat <- v .:? (tshow SaveFormatK)    .!= (_defaultSaveFormat def)
         _captionFormat     <- v .:? (tshow CaptionFormatK) .!= (_captionFormat def)
 

@@ -26,7 +26,7 @@     , unavailableToolkits
 ) where
 
-import           Control.Concurrent.ParallelIO.Local
+import           Control.Concurrent.Async                      (mapConcurrently)
 
 import           Data.List                                     ((\\))
 import           Data.Map.Strict                               (Map)
@@ -92,7 +92,8 @@ -- the figure is not rendered. This is to prevent, for example,
 -- blocking operations to occur.
 scriptChecks :: Toolkit -> [Script -> CheckResult]
-scriptChecks = const mempty
+scriptChecks Matplotlib = [matplotlibCheckIfShow]
+scriptChecks _ = mempty
 
 
 -- | Parse code block headers for extra attributes that are specific
@@ -138,14 +139,7 @@ -- | List of toolkits available on this machine.
 -- The executables to look for are taken from the configuration.
 availableToolkits :: Configuration -> IO [Toolkit]
-availableToolkits conf = do
-    -- Certain toolkits (e.g. Python-based toolkits)
-    -- may take a long time to startup.
-    -- Therefore, we check for available of toolkits in parallel.
-    -- TODO: benchmark. Is this overkill?
-    maybeToolkits <- withPool (length toolkits) $ 
-        \pool -> parallel pool (maybeToolkit <$> toolkits)
-    return $ catMaybes maybeToolkits
+availableToolkits conf = catMaybes <$> (mapConcurrently maybeToolkit toolkits)
     where
         maybeToolkit tk = do
             available <- toolkitAvailable tk conf

@@ -23,12 +23,14 @@     , matplotlibCapture
     , matplotlibExtraAttrs
     , matplotlibAvailable
+    , matplotlibCheckIfShow
 ) where
 
 import           Text.Pandoc.Filter.Plot.Renderers.Prelude
 
 import qualified Data.Map.Strict                           as M
-
+import           Data.Monoid                               (Any(..))
+import qualified Data.Text                                 as T
 
 matplotlibSupportedSaveFormats :: [SaveFormat]
 matplotlibSupportedSaveFormats = [PNG, PDF, SVG, JPG, EPS, GIF, TIF]
@@ -56,6 +58,22 @@ 
 matplotlibAvailable :: Configuration -> IO Bool
 matplotlibAvailable Configuration{..} = commandSuccess [st|#{matplotlibExe} -c "import matplotlib"|]
+
+
+-- | Check if `matplotlib.pyplot.show()` calls are present in the script,
+-- which would halt pandoc-plot
+matplotlibCheckIfShow :: Script -> CheckResult
+matplotlibCheckIfShow s = 
+    if getAny $ mconcat showPresent
+        then CheckFailed "encountered a call to `matplotlib.pyplot.show`."
+        else CheckPassed
+    where
+        showPresent = (\n -> Any (T.isInfixOf n s)) <$> [
+                  "matplotlib.pyplot.show()"
+                , "pyplot.show()"
+                , "plt.show()"
+            ]
+        
 
 
 -- | Flexible boolean parsing

@@ -96,7 +96,7 @@     , defaultWithSource     :: !Bool       -- ^ The default behavior of whether or not to include links to source code and high-res
     , defaultDPI            :: !Int        -- ^ The default dots-per-inch value for generated figures. Renderers might ignore this.
     , defaultSaveFormat     :: !SaveFormat -- ^ The default save format of generated figures.
-    , captionFormat         :: Format      -- ^ Caption format, in the same notation as Pandoc format, e.g. "markdown+tex_math_dollars"
+    , captionFormat         :: !Format     -- ^ Caption format, in the same notation as Pandoc format, e.g. "markdown+tex_math_dollars"
 
     , matplotlibPreamble    :: !Script     -- ^ The default preamble script for the matplotlib toolkit.
     , plotlyPythonPreamble  :: !Script     -- ^ The default preamble script for the Plotly/Python toolkit.

@@ -228,6 +228,27 @@         assertEqual "" outputDirExists False
 
 
+-------------------------------------------------------------------------------
+-- Test that toolkit checks failed when appropriate.
+testChecksFail :: Toolkit -> TestTree
+testChecksFail tk = 
+    testCase "script checks fail when appropriate" $ do
+        assertChecksFail tk
+    where
+        assertChecksFail Matplotlib = do
+            let postfix = unpack . cls $ tk
+            tempDir <- (</> "test-checks" <> postfix) <$> getCanonicalTemporaryDirectory
+            ensureDirectoryExistsAndEmpty tempDir
+
+            let cb = addDirectory tempDir $ codeBlock Matplotlib "plt.show()"
+            result <- (make' Matplotlib) def cb
+            let expectedCheck :: Either PandocPlotError a -> Bool
+                expectedCheck (Left (ScriptChecksFailedError _)) = True
+                expectedCheck _ = False
+            assertBool "" (expectedCheck result)
+
+        assertChecksFail _          = assertEqual "Test skipped" True True
+
 codeBlock :: Toolkit -> Script -> Block
 codeBlock tk script = CodeBlock (mempty, [cls tk], mempty) script
 

@@ -50,6 +50,7 @@         , testMarkdownFormattingCaption1
         , testMarkdownFormattingCaption2
         , testCleanOutputDirs
+        , testChecksFail
         ] <*> [tk]