diff --git a/Main.hs b/Main.hs --- a/Main.hs +++ b/Main.hs @@ -22,15 +22,32 @@ import qualified Data.Text.Lazy.Builder as B import qualified Data.Text.Lazy.Builder.Int as B import qualified Data.Text.Lazy.Builder.RealFloat as B +import qualified Options.Applicative as O +import qualified Text.CSV as CSV +data Options = Options { jsonExpr :: String, outputMode :: OutputMode } deriving Show + +data OutputMode = TSVOutput { delimiter :: String } | CSVOutput deriving (Show) + +parseOpts :: O.Parser Options +parseOpts = Options + <$> O.argument O.str (O.metavar "FIELDS") + <*> ((O.flag' CSVOutput (O.short 'c' <> O.long "csv" <> O.help "output CSV")) + <|> (TSVOutput <$> (O.strOption (O.metavar "DELIM" <> O.value "\t" <> O.short 'd' <> O.help "output field delimiter. Defaults to tab")))) + +opts = O.info (O.helper <*> parseOpts) + (O.fullDesc <> O.progDesc "Transform JSON objects to TSV" <> O.header "jsontsv") + main = do - x <- BL.getContents - let xs :: [Value] - xs = decodeStream x - (ks:_) <- getArgs - let ks' = parseKeyPath $ T.pack ks - -- Prelude.putStrLn $ "key Paths " ++ show ks' - mapM_ (TL.putStrLn . B.toLazyText . evalToLineBuilder ks') xs + Options expr mode <- O.execParser opts + x <- BL.getContents + let xs :: [Value] + xs = decodeStream x + let ks' = parseKeyPath $ T.pack expr + -- Prelude.putStrLn $ "key Paths " ++ show ks' + case mode of + TSVOutput delim -> mapM_ (TL.putStrLn . B.toLazyText . evalToLineBuilder delim ks') xs + CSVOutput -> Prelude.putStrLn . CSV.printCSV $ map (map T.unpack . evalToList ks') $ xs decodeStream :: (FromJSON a) => BL.ByteString -> [a] decodeStream bs = case decodeWith json bs of @@ -69,8 +86,8 @@ type KeyPath = [Key] data Key = Key Text | Index Int deriving (Eq, Show) -evalToLineBuilder :: [KeyPath] -> Value -> B.Builder -evalToLineBuilder ks v = mconcat $ intersperse (B.singleton '\t') $ map (flip evalToBuilder v) ks +evalToLineBuilder :: String -> [KeyPath] -> Value -> B.Builder +evalToLineBuilder delim ks v = mconcat $ intersperse (B.fromText . T.pack $ delim) $ map (flip evalToBuilder v) ks evalToList :: [KeyPath] -> Value -> [Text] evalToList ks v = map (flip evalToText v) ks diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # jsontsv -A simple tool to transform JSON into tab-separated line-oriented output -amenable to downstream Unix text processing. +Transforms JSON objects into tab-separated line-oriented output, which is more +convenient for downstream processing with Unix tools like `grep`, `awk`, +`diff`, etc., and for inspecting with spreadsheet programs. ## Synopsis @@ -32,29 +33,59 @@ } ``` +Note that this input is not actually JSON at the top-level. It is a stream of +JSON objects. It can be fed into `jsontsv`: + jsontsv 'title year stars ratings.imdb' < input.json -Outputs this tab-separated text: +outputs this tab-separated text: ```tsv Terminator 2: Judgement Day 1991 Arnold Schwarzenegger,Linda Hamilton 8.5 Interstellar 2014 Matthew McConaughey,Anne Hathaway 8.9 ``` -## Setup +You can pick off array elements using `[i]` syntax: -From the project directory, + jsontsv 'title year stars[0]' < input.json - cabal install +outputs +```tsv +Terminator 2: Judgement Day 1991 Arnold Schwarzenegger +Interstellar 2014 Matthew McConaughey +``` + +## Installation + +Assuming you have a recent version of the [Haskell +platform](https://www.haskell.org/platform/) on your system, + + cabal install jsontsv + Make sure the installed executable is on your PATH. ## Usage -Input should be a stream of JSON objects with mostly uniform keys, separated by -whitespace such as newlines. If the objects are wrapped in a JSON array at the -top level, use the `jq` tool by Stephan Dolan to unwrap the objects, e.g.: +``` +jsontsv +Usage: jsontsv FIELDS ([-c|--csv] | [-d DELIM]) + Transform JSON objects to TSV + +Available options: + -h,--help Show this help text + -c,--csv output CSV + -d DELIM output field delimiter. Defaults to tab +``` + +Input should be a stream of JSON objects of the same or mostly similar shape, +separated by whitespace such as newlines. If the objects are wrapped in a JSON +array at the top level or nested inside a top-level object, use the [jq][jq] +tool by Stephan Dolan to extract an object stream, e.g.: + +[jq]:http://stedolan.github.io/jq/ + curl -s "https://api.github.com/users/danchoi/repos?type=owner&sort=created&direction=desc" \ | jq '.[]' | jsontsv 'id name stargazers_count open_issues_count' @@ -73,20 +104,16 @@ JSON leaf values are printed as follows: -* Strings and numbers were copied to output. +* Strings and numbers are copied to output. * Boolean values are output as `t` or `f`. * null is printed as `null` -* Arrays of leaf values are concatenated into a single comma-separated string - -## Nested keys - - jsontsv 'title duplicates.Rental.HD duplicates.Rental.SD' < input.json - -## Using a file to designate columns: +* If the leaf value is an array, it is concatenated into a single comma-separated string - jsontsv -f keys < input.json +## Performing post-processing on field values -## Concatenating fields, truncating fields, etc. +Use awk! -This should be done downstream using a tool like AWK. +## Known alternatives +* [jsawk](https://github.com/micha/jsawk) +* [jq][jq] diff --git a/jsontsv.cabal b/jsontsv.cabal --- a/jsontsv.cabal +++ b/jsontsv.cabal @@ -2,7 +2,7 @@ -- see http://haskell.org/cabal/users-guide/ name: jsontsv -version: 0.1.0.0 +version: 0.1.1.0 synopsis: JSON to TSV transformer homepage: https://github.com/danchoi/jsontsv description: Transforms JSON into tab-separated line-oriented output, for easier processing in Unix-style pipelines. @@ -26,5 +26,6 @@ , text >= 1.2.0.0 , vector , scientific + , optparse-applicative + , csv >= 0.1.2 default-language: Haskell2010 - ghc-options: -O2