diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,6 @@ +# Revision history for IPyNb + +0.1.0.0 -- 2017-11-21 + + * First version. Released on an unsuspecting world. + diff --git a/JuPyTer-notebook.cabal b/JuPyTer-notebook.cabal new file mode 100644 --- /dev/null +++ b/JuPyTer-notebook.cabal @@ -0,0 +1,45 @@ +-- Initial IPyNb.cabal generated by cabal init. For further documentation, +-- see http://haskell.org/cabal/users-guide/ + +name: JuPyTer-notebook +version: 0.1.0.0 +synopsis: JuPyTer notebook parser +description: JuPyTer also called IPython notebook. + . + It is cross-language interactive data science platform + that allows for interactive editing of code and visualizing its results. + . + This library allows to directly parse `.ipynb` files, and process them. +homepage: http://github.com/mgajda/ipynb +cabal-version: >=1.10 +license: BSD3 +license-file: LICENSE +author: Michal J. Gajda +maintainer: migamake@migamake.com +category: Data +build-type: Simple +extra-source-files: ChangeLog.md + README.md +source-repository head + type: git + location: https://github.com/mgajda/jupyter-notebook.git + +library + build-depends: base >=4.9 && <4.10, + json-autotype >=1.0 && <2.0 + hs-source-dirs: src + exposed-modules: Data.JuPyTer + default-language: Haskell2010 + +executable jupyter-extract + hs-source-dirs: src + main-is: Main.hs + --other-modules: Data. + --other-extensions: TemplateHaskell, + build-depends: base >=4.3 && <4.11, + text >=1.0 && <1.3, + json-autotype >=1.0 && <1.1, + aeson >=1.0 && <1.3, + bytestring >=0.10 && <1.0, + JuPyTer-notebook + default-language: Haskell2010 diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (c) 2017, Michal J. Gajda + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of Michal J. Gajda nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +JuPyTer notebook parser +======================= +[JuPyTer](https://jupyter.org/) was formerly called IPython notebook. + +It is cross-language interactive data science platform +that allows for interactive editing of code and visualizing its results. + +This library allows to directly parse `.ipynb` files, and process them. + +There is an example script `jupyter-extract` that just extracts +all the source code cells, and prints them. + +Initial version was generated with [`json-autotype`](https://hackage.haskell.org/package/json-autotype) diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/src/Data/JuPyTer.hs b/src/Data/JuPyTer.hs new file mode 100644 --- /dev/null +++ b/src/Data/JuPyTer.hs @@ -0,0 +1,173 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeOperators #-} +{-# LANGUAGE DeriveGeneric #-} + +module Data.JuPyTer where + +import System.Exit (exitFailure, exitSuccess) +import System.IO (stderr, hPutStrLn) +import qualified Data.ByteString.Lazy.Char8 as BSL +import System.Environment (getArgs) +import Control.Monad (forM_, mzero, join) +import Control.Applicative +import Data.Aeson.AutoType.Alternative +import Data.Aeson(decode, Value(..), FromJSON(..), ToJSON(..), + pairs, + (.:), (.:?), (.=), object) +import Data.Monoid +import Data.Text (Text) +import qualified Data.Text as Text +import qualified GHC.Generics + +-- | Workaround for https://github.com/bos/aeson/issues/287. +o .:?? val = fmap join (o .:? val) + + +data CodemirrorMode = CodemirrorMode { + codemirrorModeName :: Text, + codemirrorModeVersion :: Double + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON CodemirrorMode where + parseJSON (Object v) = CodemirrorMode <$> v .: "name" <*> v .: "version" + parseJSON _ = mzero + + +instance ToJSON CodemirrorMode where + toJSON (CodemirrorMode {..}) = object ["name" .= codemirrorModeName, "version" .= codemirrorModeVersion] + toEncoding (CodemirrorMode {..}) = pairs ("name" .= codemirrorModeName<>"version" .= codemirrorModeVersion) + + +data LanguageInfo = LanguageInfo { + languageInfoNbconvertExporter :: Text, + languageInfoFileExtension :: Text, + languageInfoPygmentsLexer :: Text, + languageInfoMimetype :: Text, + languageInfoName :: Text, + languageInfoVersion :: Text, + languageInfoCodemirrorMode :: CodemirrorMode + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON LanguageInfo where + parseJSON (Object v) = LanguageInfo <$> v .: "nbconvert_exporter" <*> v .: "file_extension" <*> v .: "pygments_lexer" <*> v .: "mimetype" <*> v .: "name" <*> v .: "version" <*> v .: "codemirror_mode" + parseJSON _ = mzero + + +instance ToJSON LanguageInfo where + toJSON (LanguageInfo {..}) = object ["nbconvert_exporter" .= languageInfoNbconvertExporter, "file_extension" .= languageInfoFileExtension, "pygments_lexer" .= languageInfoPygmentsLexer, "mimetype" .= languageInfoMimetype, "name" .= languageInfoName, "version" .= languageInfoVersion, "codemirror_mode" .= languageInfoCodemirrorMode] + toEncoding (LanguageInfo {..}) = pairs ("nbconvert_exporter" .= languageInfoNbconvertExporter<>"file_extension" .= languageInfoFileExtension<>"pygments_lexer" .= languageInfoPygmentsLexer<>"mimetype" .= languageInfoMimetype<>"name" .= languageInfoName<>"version" .= languageInfoVersion<>"codemirror_mode" .= languageInfoCodemirrorMode) + + +data Kernelspec = Kernelspec { + kernelspecDisplayName :: Text, + kernelspecName :: Text, + kernelspecLanguage :: Text + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON Kernelspec where + parseJSON (Object v) = Kernelspec <$> v .: "display_name" <*> v .: "name" <*> v .: "language" + parseJSON _ = mzero + + +instance ToJSON Kernelspec where + toJSON (Kernelspec {..}) = object ["display_name" .= kernelspecDisplayName, "name" .= kernelspecName, "language" .= kernelspecLanguage] + toEncoding (Kernelspec {..}) = pairs ("display_name" .= kernelspecDisplayName<>"name" .= kernelspecName<>"language" .= kernelspecLanguage) + + +data Slideshow = Slideshow { + slideshowSlideType :: Text + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON Slideshow where + parseJSON (Object v) = Slideshow <$> v .: "slide_type" + parseJSON _ = mzero + + +instance ToJSON Slideshow where + toJSON (Slideshow {..}) = object ["slide_type" .= slideshowSlideType] + toEncoding (Slideshow {..}) = pairs ("slide_type" .= slideshowSlideType) + + +data Metadata = Metadata { + metadataScrolled :: (Maybe (Bool:|:[(Maybe Value)])), + metadataCelltoolbar :: (Maybe (Text:|:[(Maybe Value)])), + metadataLanguageInfo :: (Maybe (LanguageInfo:|:[(Maybe Value)])), + metadataKernelspec :: (Maybe (Kernelspec:|:[(Maybe Value)])), + metadataEditable :: (Maybe (Bool:|:[(Maybe Value)])), + metadataDeletable :: (Maybe (Bool:|:[(Maybe Value)])), + metadataSlideshow :: (Maybe (Slideshow:|:[(Maybe Value)])), + metadataCollapsed :: (Maybe (Bool:|:[(Maybe Value)])) + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON Metadata where + parseJSON (Object v) = Metadata <$> v .:?? "scrolled" <*> v .:?? "celltoolbar" <*> v .:?? "language_info" <*> v .:?? "kernelspec" <*> v .:?? "editable" <*> v .:?? "deletable" <*> v .:?? "slideshow" <*> v .:?? "collapsed" + parseJSON _ = mzero + + +instance ToJSON Metadata where + toJSON (Metadata {..}) = object ["scrolled" .= metadataScrolled, "celltoolbar" .= metadataCelltoolbar, "language_info" .= metadataLanguageInfo, "kernelspec" .= metadataKernelspec, "editable" .= metadataEditable, "deletable" .= metadataDeletable, "slideshow" .= metadataSlideshow, "collapsed" .= metadataCollapsed] + toEncoding (Metadata {..}) = pairs ("scrolled" .= metadataScrolled<>"celltoolbar" .= metadataCelltoolbar<>"language_info" .= metadataLanguageInfo<>"kernelspec" .= metadataKernelspec<>"editable" .= metadataEditable<>"deletable" .= metadataDeletable<>"slideshow" .= metadataSlideshow<>"collapsed" .= metadataCollapsed) + +data CellsElt = CellsElt { + cellsEltExecutionCount :: (Maybe (Double:|:[(Maybe Value)])), + cellsEltOutputs :: (Maybe ([[(Maybe Value)]])), + cellsEltMetadata :: Metadata, + cellsEltSource :: [Text:|:[(Maybe Value)]], + cellsEltCellType :: Text + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON CellsElt where + parseJSON (Object v) = CellsElt <$> v .:?? "execution_count" <*> v .:?? "outputs" <*> v .: "metadata" <*> v .: "source" <*> v .: "cell_type" + parseJSON _ = mzero + + +instance ToJSON CellsElt where + toJSON (CellsElt {..}) = object ["execution_count" .= cellsEltExecutionCount, "outputs" .= cellsEltOutputs, "metadata" .= cellsEltMetadata, "source" .= cellsEltSource, "cell_type" .= cellsEltCellType] + toEncoding (CellsElt {..}) = pairs ("execution_count" .= cellsEltExecutionCount<>"outputs" .= cellsEltOutputs<>"metadata" .= cellsEltMetadata<>"source" .= cellsEltSource<>"cell_type" .= cellsEltCellType) + +data IPyNb = IPyNb { + iPyNbNbformatMinor :: Double, + iPyNbNbformat :: Double, + iPyNbCells :: [CellsElt], + iPyNbMetadata :: Metadata + } deriving (Show,Eq,GHC.Generics.Generic) + + +instance FromJSON IPyNb where + parseJSON (Object v) = IPyNb <$> v .: "nbformat_minor" <*> v .: "nbformat" <*> v .: "cells" <*> v .: "metadata" + parseJSON _ = mzero + + +instance ToJSON IPyNb where + toJSON (IPyNb {..}) = object ["nbformat_minor" .= iPyNbNbformatMinor, "nbformat" .= iPyNbNbformat, "cells" .= iPyNbCells, "metadata" .= iPyNbMetadata] + toEncoding (IPyNb {..}) = pairs ("nbformat_minor" .= iPyNbNbformatMinor<>"nbformat" .= iPyNbNbformat<>"cells" .= iPyNbCells<>"metadata" .= iPyNbMetadata) + + + + +parse :: FilePath -> IO IPyNb +parse filename = do input <- BSL.readFile filename + case decode input of + Nothing -> fatal $ case (decode input :: Maybe Value) of + Nothing -> "Invalid JSON file: " ++ filename + Just v -> "Mismatched JSON value from file: " ++ filename + Just r -> return (r :: IPyNb) + where + fatal :: String -> IO a + fatal msg = do hPutStrLn stderr msg + exitFailure + +main :: IO () +main = do + filenames <- getArgs + forM_ filenames (\f -> parse f >>= (\p -> p `seq` putStrLn $ "Successfully parsed " ++ f)) + exitSuccess diff --git a/src/Main.hs b/src/Main.hs new file mode 100644 --- /dev/null +++ b/src/Main.hs @@ -0,0 +1,39 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeOperators #-} +{-# LANGUAGE DeriveGeneric #-} + +module Main(main) where + +import System.Exit (exitFailure, exitSuccess) +import System.IO (stderr, hPutStrLn) +import qualified Data.ByteString.Lazy.Char8 as BSL +import System.Environment (getArgs) +import Control.Monad (forM_, mzero, join) +import Control.Applicative +import Data.Aeson.AutoType.Alternative +import Data.Aeson(decode, Value(..), FromJSON(..), ToJSON(..), + pairs, + (.:), (.:?), (.=), object) +import Data.Monoid +import Data.Text (Text) +import qualified Data.Text as Text +import qualified GHC.Generics + +import Data.JuPyTer(IPyNb(..), CellsElt(..), parse) + +main :: IO () +main = do + filenames <- getArgs + forM_ filenames (\f -> parse f >>= (\p -> p `seq` putStrLn $ "Successfully parsed " ++ f + ++ ":\n" ++ extract p)) + exitSuccess + +extract :: IPyNb -> String +extract = unlines + . map (concatMap (alt Text.unpack show) + . cellsEltSource) + . iPyNbCells +