diff --git a/Main.hs b/Main.hs --- a/Main.hs +++ b/Main.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE OverloadedStrings, QuasiQuotes #-} module Main where import Data.Aeson import Data.Monoid @@ -26,6 +26,7 @@ import qualified Data.Text.Lazy.Builder.RealFloat as B import qualified Options.Applicative as O import qualified Text.CSV as CSV +import Data.String.QQ data Options = Options { jsonExpr :: String, arrayDelim :: String, outputMode :: OutputMode, showHeader :: Bool } deriving Show @@ -34,7 +35,7 @@ parseOpts :: O.Parser Options parseOpts = Options <$> O.argument O.str (O.metavar "FIELDS") - <*> O.strOption (O.metavar "STRING" <> O.value "," <> O.short 'a' <> O.help "Concatentated array elem delimiter. Defaults to comma.") + <*> O.strOption (O.metavar "DELIM" <> O.value "," <> O.short 'a' <> O.help "Concatentated array elem delimiter. Defaults to comma.") <*> (parseCSVMode <|> parseTSVMode) <*> O.flag False True (O.short 'H' <> O.help "Include headers") @@ -42,10 +43,14 @@ parseTSVMode = TSVOutput <$> (O.strOption - (O.metavar "STRING" <> O.value "\t" <> O.short 'd' <> O.help "Output field delimiter. Defaults to tab.")) + (O.metavar "DELIM" <> O.value "\t" <> O.short 'd' <> O.help "Output field delimiter. Defaults to tab.")) opts = O.info (O.helper <*> parseOpts) - (O.fullDesc <> O.progDesc "Transform JSON objects to TSV" <> O.header "jsontsv") + (O.fullDesc + <> O.progDesc [s| Transform JSON objects to TSV. + On STDIN provide an input stream of whitespace-separated JSON objects. |] + <> O.header "jsontsv v0.1.4.1" + <> O.footer "See https://github.com/danchoi/jsontsv for more information.") main = do Options expr arrayDelim mode showHeaders <- O.execParser opts diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ Transforms JSON objects into delimiter-separated line-oriented output, which is more convenient for downstream processing with Unix tools like `grep`, `awk`, -`diff`, etc. Also useful for converting JSON data into spreadsheet data. CSV-style -output is also supported. +`diff`, etc. Also useful for converting JSON data into spreadsheet data: CSV-style +output is supported. ## Synopsis @@ -65,6 +65,19 @@ Interstellar 2014 Matthew McConaughey ``` +Use the `-H` flag to output the headers and the Unix tool `column` to align the +columns for prettier output: + + cat input | jsontsv -H 'title year stars.name ratings.imdb' | column -s $'\t' -t + +outputs + +``` +title year stars.name ratings.imdb +Terminator 2: Judgment Day 1991 Arnold Schwarzenegger,Linda Hamilton 8.5 +Interstellar 2014 Matthew McConaughey,Anne Hathaway 8.9 +``` + ## Installation Assuming you have a recent version of the [Haskell @@ -73,21 +86,27 @@ cabal update cabal install jsontsv -Make sure the installed executable is on your PATH. +Alternatively, + git clone git@github.com:danchoi/jsontsv.git + cd jsontsv + cabal sandbox init + cabal install + # Now copy .cabal-sandbox/bin/jsontsv to your PATH + ## Usage ``` jsontsv -Usage: jsontsv FIELDS [-a STRING] ([-c|--csv] | [-d STRING]) [-H] +Usage: jsontsv FIELDS [-a DELIM] ([-c|--csv] | [-d DELIM]) [-H] Transform JSON objects to TSV Available options: -h,--help Show this help text - -a STRING Concatentated array elem delimiter. Defaults to comma. + -a DELIM Concatentated array elem delimiter. Defaults to comma. -c,--csv Output CSV - -d STRING Output field delimiter. Defaults to tab. + -d DELIM Output field delimiter. Defaults to tab. -H Include headers ``` @@ -98,21 +117,45 @@ [jq]:http://stedolan.github.io/jq/ - curl -s "https://api.github.com/users/danchoi/repos?sort=created&direction=desc" \ - | jq '.[]' | jsontsv 'id name stargazers_count open_issues_count' + curl -s "https://api.github.com/repos/rails/rails/issues" | + jq -M '.[]' | + jsontsv -H 'number title user.login state repository.name labels.name' outputs - 27397673 jsontsv 0 0 - 26033118 ngrender 24 1 - 25832026 rdoc 0 0 - 24756523 treehtml 0 0 - 24022588 heistexamples 0 0 - 24022042 hxtexamples 0 0 - 24005242 jdiff 0 0 - 23997156 https-types 0 0 - 22763562 podcasting 0 0 - 19294791 vimscript 3 0 +``` +number title user.login state repository.name labels.name +17894 Add default value for `create_table_definition` kamipo open null +17893 Vendor/assets/images not being precompiled runephilosof open null +17891 Removed use of mocha in the info_controller tests prathamesh-sonpatki open null +17887 Wrong instance object passed to lambda on has_many :through haruska open null +17885 Update postgresql_database_tasks.rb starbelly open null +17884 Routes with {trailing_slash: true} do not match if referenced as non-named routes dreyks open null +17880 Fix humanize for already upcased acronyms mintuhouse open null activesupport +17879 humanize doesn't respect Infector acronyms mintuhouse open null activesupport +17864 eager loading a has_many through association ignores order of through association jturkel open null +17859 Includes HABTM returns correct size now scambra open null +17858 test preloading a HABTM association with hash conditions scambra open null +17854 Bug when using find_in_batches and reverse_order robertjlooby open null activerecord +17853 Remove deprecated `reset_changes` and `reset_attribute!` methods. kaspth open null +17851 Support for any type primary key kamipo open null +17845 Don't leak Object constants in core_ext/module/qualified_const gsamokovarov open null +17825 Fix Sidekiq ActiveJob integration setup aripollak open null activejob +17824 AR::RecordNotSaved & RecordNotDestroyed from save!/destroy! should include an error message yuki24 open null +17822 Refactor `visit_ChangeColumnDefinition` kamipo open null +17820 Clear query cache on rollback fw42 open null +17819 handle_positional_args does not work properly in route with format: false option vevisystems open null actionpack +17817 Hide potentially sensitive ActiveJob params from logs aripollak open null activejob +17815 Remove custom errors page section from the guides yuki24 open null +17813 Changed welcome#index page overall look and feel wazery open null railties +17804 Null values will still be passed to custom serializers. xaviershay open null activerecord +17797 Don't remove mailer layouts files y-yagi open null +17795 ActiveRecord joins/includes bug dgobaud open null activerecord,regression +17793 Fix undesirable RangeError by Type::Integer. Add Type::UnsignedInteger. kamipo open null activerecord +17792 allow 'all' for :domain option in addition to :all rockrep open null +17788 Issue#17703 Test case for tempfile attribute sivagollapalli open null +17787 rails runner does not respect subdirectory / how to specify subdirectory? doits open null railties +``` JSON leaf values are printed as follows: @@ -125,6 +168,17 @@ ## Known alternatives * [jsawk](https://github.com/micha/jsawk) Jsawk is like awk, but for JSON. (nodejs) -* [jq][jq] +* [json2csv](https://github.com/jehiah/json2csv) made with Go +* [jq][jq] a lightweight and flexible command-line JSON processor * [jsoncsv](https://github.com/gradus/jsoncsv) a json to csv library in javascript/coffeescript -* [json](https://github.com/konklone/json) A free, in-browser JSON to CSV converter. +* [json](https://github.com/konklone/json) A free, in-browser JSON to CSV converter +* [tsvutils](https://github.com/brendano/tsvutils) utilities for processing tab-separated files + +## Related + +* [Seven command line tools for data + science](http://jeroenjanssens.com/2013/09/19/seven-command-line-tools-for-data-science.html) by Jeroen Janssens. + A blog post surveying tools like this. +* [Data Science at the Command Line: Facing the Future with Time-Tested Tools](http://shop.oreilly.com/product/0636920032823.do) + by Jeroen Janssens. A book published in September 2014 by O'Reilly. + diff --git a/jsontsv.cabal b/jsontsv.cabal --- a/jsontsv.cabal +++ b/jsontsv.cabal @@ -2,7 +2,7 @@ -- see http://haskell.org/cabal/users-guide/ name: jsontsv -version: 0.1.4.0 +version: 0.1.4.1 synopsis: JSON to TSV transformer homepage: https://github.com/danchoi/jsontsv description: Transforms JSON into tab-separated line-oriented output, for easier processing in Unix-style pipelines. @@ -27,5 +27,6 @@ , vector , scientific , optparse-applicative + , string-qq , csv >= 0.1.0 default-language: Haskell2010