jsontsv 0.1.0.0 → 0.1.1.0
raw patch · 3 files changed
+76/−31 lines, 3 filesdep +csvdep +optparse-applicative
Dependencies added: csv, optparse-applicative
Files
- Main.hs +26/−9
- README.md +47/−20
- jsontsv.cabal +3/−2
Main.hs view
@@ -22,15 +22,32 @@ import qualified Data.Text.Lazy.Builder as B import qualified Data.Text.Lazy.Builder.Int as B import qualified Data.Text.Lazy.Builder.RealFloat as B+import qualified Options.Applicative as O+import qualified Text.CSV as CSV +data Options = Options { jsonExpr :: String, outputMode :: OutputMode } deriving Show++data OutputMode = TSVOutput { delimiter :: String } | CSVOutput deriving (Show)++parseOpts :: O.Parser Options+parseOpts = Options + <$> O.argument O.str (O.metavar "FIELDS")+ <*> ((O.flag' CSVOutput (O.short 'c' <> O.long "csv" <> O.help "output CSV"))+ <|> (TSVOutput <$> (O.strOption (O.metavar "DELIM" <> O.value "\t" <> O.short 'd' <> O.help "output field delimiter. Defaults to tab"))))++opts = O.info (O.helper <*> parseOpts)+ (O.fullDesc <> O.progDesc "Transform JSON objects to TSV" <> O.header "jsontsv")+ main = do- x <- BL.getContents - let xs :: [Value]- xs = decodeStream x- (ks:_) <- getArgs - let ks' = parseKeyPath $ T.pack ks- -- Prelude.putStrLn $ "key Paths " ++ show ks'- mapM_ (TL.putStrLn . B.toLazyText . evalToLineBuilder ks') xs+ Options expr mode <- O.execParser opts+ x <- BL.getContents + let xs :: [Value]+ xs = decodeStream x+ let ks' = parseKeyPath $ T.pack expr+ -- Prelude.putStrLn $ "key Paths " ++ show ks'+ case mode of + TSVOutput delim -> mapM_ (TL.putStrLn . B.toLazyText . evalToLineBuilder delim ks') xs+ CSVOutput -> Prelude.putStrLn . CSV.printCSV $ map (map T.unpack . evalToList ks') $ xs decodeStream :: (FromJSON a) => BL.ByteString -> [a] decodeStream bs = case decodeWith json bs of@@ -69,8 +86,8 @@ type KeyPath = [Key] data Key = Key Text | Index Int deriving (Eq, Show) -evalToLineBuilder :: [KeyPath] -> Value -> B.Builder -evalToLineBuilder ks v = mconcat $ intersperse (B.singleton '\t') $ map (flip evalToBuilder v) ks+evalToLineBuilder :: String -> [KeyPath] -> Value -> B.Builder +evalToLineBuilder delim ks v = mconcat $ intersperse (B.fromText . T.pack $ delim) $ map (flip evalToBuilder v) ks evalToList :: [KeyPath] -> Value -> [Text] evalToList ks v = map (flip evalToText v) ks
README.md view
@@ -1,7 +1,8 @@ # jsontsv -A simple tool to transform JSON into tab-separated line-oriented output-amenable to downstream Unix text processing. +Transforms JSON objects into tab-separated line-oriented output, which is more+convenient for downstream processing with Unix tools like `grep`, `awk`,+`diff`, etc., and for inspecting with spreadsheet programs. ## Synopsis @@ -32,29 +33,59 @@ } ``` +Note that this input is not actually JSON at the top-level. It is a stream of+JSON objects. It can be fed into `jsontsv`:+ jsontsv 'title year stars ratings.imdb' < input.json -Outputs this tab-separated text:+outputs this tab-separated text: ```tsv Terminator 2: Judgement Day 1991 Arnold Schwarzenegger,Linda Hamilton 8.5 Interstellar 2014 Matthew McConaughey,Anne Hathaway 8.9 ``` -## Setup+You can pick off array elements using `[i]` syntax: -From the project directory, + jsontsv 'title year stars[0]' < input.json - cabal install+outputs +```tsv+Terminator 2: Judgement Day 1991 Arnold Schwarzenegger+Interstellar 2014 Matthew McConaughey+```++## Installation++Assuming you have a recent version of the [Haskell+platform](https://www.haskell.org/platform/) on your system, ++ cabal install jsontsv+ Make sure the installed executable is on your PATH. ## Usage -Input should be a stream of JSON objects with mostly uniform keys, separated by-whitespace such as newlines. If the objects are wrapped in a JSON array at the-top level, use the `jq` tool by Stephan Dolan to unwrap the objects, e.g.: +```+jsontsv +Usage: jsontsv FIELDS ([-c|--csv] | [-d DELIM])+ Transform JSON objects to TSV++Available options:+ -h,--help Show this help text+ -c,--csv output CSV+ -d DELIM output field delimiter. Defaults to tab+```++Input should be a stream of JSON objects of the same or mostly similar shape,+separated by whitespace such as newlines. If the objects are wrapped in a JSON+array at the top level or nested inside a top-level object, use the [jq][jq]+tool by Stephan Dolan to extract an object stream, e.g.: ++[jq]:http://stedolan.github.io/jq/+ curl -s "https://api.github.com/users/danchoi/repos?type=owner&sort=created&direction=desc" \ | jq '.[]' | jsontsv 'id name stargazers_count open_issues_count' @@ -73,20 +104,16 @@ JSON leaf values are printed as follows: -* Strings and numbers were copied to output.+* Strings and numbers are copied to output. * Boolean values are output as `t` or `f`. * null is printed as `null`-* Arrays of leaf values are concatenated into a single comma-separated string--## Nested keys-- jsontsv 'title duplicates.Rental.HD duplicates.Rental.SD' < input.json--## Using a file to designate columns:+* If the leaf value is an array, it is concatenated into a single comma-separated string - jsontsv -f keys < input.json+## Performing post-processing on field values -## Concatenating fields, truncating fields, etc.+Use awk! -This should be done downstream using a tool like AWK.+## Known alternatives +* [jsawk](https://github.com/micha/jsawk) +* [jq][jq]
jsontsv.cabal view
@@ -2,7 +2,7 @@ -- see http://haskell.org/cabal/users-guide/ name: jsontsv-version: 0.1.0.0+version: 0.1.1.0 synopsis: JSON to TSV transformer homepage: https://github.com/danchoi/jsontsv description: Transforms JSON into tab-separated line-oriented output, for easier processing in Unix-style pipelines.@@ -26,5 +26,6 @@ , text >= 1.2.0.0 , vector , scientific+ , optparse-applicative+ , csv >= 0.1.2 default-language: Haskell2010- ghc-options: -O2