diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -34,16 +34,6 @@ page](https://github.com/vmchale/wordchoice/releases) for 64-bit Linux, ARM Linux, and 64-bit Windows. -### Nix - -Install [nix](https://nixos.org/nix/), then type: - -```bash -nix-env -i wordchoice -``` - -to download the package for Mac or Linux. - ### Stack Install stack, following instructions @@ -54,7 +44,3 @@ ``` You might need to do a `stack setup` first. - -### Build - -This repo should build with cabal, stack, and nix-build. diff --git a/bench/Bench.hs b/bench/Bench.hs --- a/bench/Bench.hs +++ b/bench/Bench.hs @@ -4,9 +4,13 @@ import Data.Text.WordCount main = do - defaultMain [ env setupEnv $ \file -> + defaultMain [ env ulysses $ \file -> bgroup "ulysses all" + [ bench "226" $ whnf (fun 10000000) file ] + , env beowulf $ \file -> + bgroup "beowulf all" [ bench "226" $ whnf (fun 10000000) file ] ] where fun = topN - setupEnv = processFile "test/ulysses.txt" + ulysses = processFile "test/ulysses.txt" + beowulf = processFile "test/beowulf.txt" diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -1,14 +1,15 @@ { mkDerivation, base, bytestring, Chart, Chart-diagrams, containers -, lens, optparse-applicative, pandoc, stdenv, system-filepath, text +, Glob, lens, optparse-applicative, pandoc, stdenv, system-filepath +, text }: mkDerivation { pname = "wordchoice"; - version = "0.1.0.0"; + version = "0.1.0.5"; src = ./.; isLibrary = true; isExecutable = true; libraryHaskellDepends = [ - base bytestring Chart Chart-diagrams containers lens + base bytestring Chart Chart-diagrams containers Glob lens optparse-applicative pandoc system-filepath text ]; executableHaskellDepends = [ base ]; diff --git a/src/Data/Text/WordCount.hs b/src/Data/Text/WordCount.hs --- a/src/Data/Text/WordCount.hs +++ b/src/Data/Text/WordCount.hs @@ -3,6 +3,10 @@ module Data.Text.WordCount ( topN , displayWords + , filterTop + -- * Filters + , small + , common -- * For making graphs , makeFile , makeDistribution @@ -13,18 +17,18 @@ , buildFreq ) where -import qualified Data.Map.Lazy as M -import Data.Map.Lens -import Control.Lens hiding (argument) -import qualified Data.Text as TL -- .Lazy as TL -import Data.Tuple -import Data.Monoid -import Data.List -import Data.Ord -import Graphics.Rendering.Chart.Easy hiding (argument) -import Graphics.Rendering.Chart.Backend.Diagrams -import Data.Char -import Data.Text.WordCount.FileRead +import Control.Lens hiding (argument) +import Data.Char +import Data.List +import qualified Data.Map.Lazy as M +import Data.Map.Lens +import Data.Monoid +import Data.Ord +import qualified Data.Text as TL +import Data.Text.WordCount.FileRead +import Data.Tuple +import Graphics.Rendering.Chart.Backend.Diagrams +import Graphics.Rendering.Chart.Easy hiding (argument) -- | Return top n words and their frequencies -- @@ -35,6 +39,16 @@ topN :: Int -> TL.Text -> [(Int,TL.Text)] topN n = take n . order . buildFreq +-- | Return the top n words, with some filter applied. +filterTop :: Int -> (TL.Text -> Bool) -> TL.Text -> [(Int, TL.Text)] +filterTop n p = take n . filter (p . snd) . order . buildFreq + +small :: TL.Text -> Bool +small = (> 5) . TL.length . TL.filter (/= '\'') + +common :: TL.Text -> Bool +common = flip elem ["the","and","a","an","or","not","but","on","so","if","in","that","this","for"] + displayWords :: [(Int,TL.Text)] -> TL.Text displayWords [] = "" displayWords (pair:pairs) = display pair <> "\n" <> displayWords pairs @@ -50,10 +64,10 @@ count words = foldr ((.) . wordFunction) id words M.empty where wordFunction word map = case map ^. at word of Nothing -> at word ?~ 1 $ map - _ -> ix word %~ (+1) $ map + _ -> ix word %~ (+1) $ map -- | Make a bar graph from the word frequencies --- +-- -- @ -- makeFile :: IO () -- makeFile [(4,"it"),(3,"why")] "out.html" diff --git a/src/Data/Text/WordCount/Exec.hs b/src/Data/Text/WordCount/Exec.hs --- a/src/Data/Text/WordCount/Exec.hs +++ b/src/Data/Text/WordCount/Exec.hs @@ -1,18 +1,19 @@ module Data.Text.WordCount.Exec where - -import Data.Monoid -import qualified Data.Text.IO as TLIO -- .Lazy.IO as TLIO -import Paths_wordchoice -import Options.Applicative -import Data.Maybe -import Data.Version -import Data.Text.WordCount -import Data.Text.WordCount.FileRead +import Data.Maybe +import Data.Monoid +import qualified Data.Text.IO as TLIO +import Data.Text.WordCount +import Data.Text.WordCount.FileRead +import Data.Version +import Options.Applicative +import Paths_wordchoice + -- | Program datatype to be parsed -data Program = Program { file :: FilePath - , num :: Maybe Int - , output :: Maybe FilePath } -- TODO add option for separators +data Program = Program { file :: FilePath + , num :: Maybe Int + , output :: Maybe FilePath + , filterOutput :: Bool } -- TODO add option for separators -- | Command line argument parser program :: Parser Program @@ -31,11 +32,15 @@ <> long "output" <> metavar "OUTPUT" <> help "Filepath for output graph"))) + <*> switch + (short 'f' + <> long "filter" + <> help "Filter common English words from output.") -- | Parse for version info versionInfo :: Parser (a -> a) -versionInfo = infoOption - ("wordchoice version: " ++ showVersion version) +versionInfo = infoOption + ("wordchoice version: " ++ showVersion version) (short 'v' <> long "version" <> help "Show version") -- | Wraps parser with help parser @@ -53,7 +58,8 @@ pick :: Program -> IO () pick rec = let n = fromMaybe 25 (num rec) in do contents <- globFile (file rec) - TLIO.putStrLn . displayWords . topN n $ contents + let pick = if not (filterOutput rec) then topN n else filterTop n small + TLIO.putStrLn . displayWords . pick $ contents case output rec of (Just out) -> flip makeFile out . topN n $ contents - _ -> pure () + _ -> pure () diff --git a/stack.yaml b/stack.yaml --- a/stack.yaml +++ b/stack.yaml @@ -1,66 +1,8 @@ -# This file was automatically generated by 'stack init' -# -# Some commonly used options have been documented as comments in this file. -# For advanced use and comprehensive documentation of the format, please see: -# http://docs.haskellstack.org/en/stable/yaml_configuration/ - -# Resolver to choose a 'specific' stackage snapshot or a compiler version. -# A snapshot resolver dictates the compiler version and the set of packages -# to be used for project dependencies. For example: -# -# resolver: lts-3.5 -# resolver: nightly-2015-09-21 -# resolver: ghc-7.10.2 -# resolver: ghcjs-0.1.0_ghc-7.10.2 -# resolver: -# name: custom-snapshot -# location: "./custom-snapshot.yaml" -resolver: lts-8.18 - -# User packages to be built. -# Various formats can be used as shown in the example below. -# -# packages: -# - some-directory -# - https://example.com/foo/bar/baz-0.0.2.tar.gz -# - location: -# git: https://github.com/commercialhaskell/stack.git -# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a -# - location: https://github.com/commercialhaskell/stack/commit/e7b331f14bcffb8367cd58fbfc8b40ec7642100a -# extra-dep: true -# subdirs: -# - auto-update -# - wai -# -# A package marked 'extra-dep: true' will only be built if demanded by a -# non-dependency (i.e. a user package), and its test suites and benchmarks -# will not be run. This is useful for tweaking upstream packages. +resolver: lts-8.22 packages: - '.' -# Dependency packages to be pulled from upstream that are not in the resolver -# (e.g., acme-missiles-0.3) extra-deps: [] - -# Override default flag values for local packages and extra-deps -flags: {} - -# Extra package databases containing global packages +flags: + wordchoice: + llvm-fast: true extra-package-dbs: [] - -# Control whether we use the GHC we find on the path -# system-ghc: true -# -# Require a specific version of stack, using version ranges -# require-stack-version: -any # Default -# require-stack-version: ">=1.4" -# -# Override the architecture used by stack, especially useful on Windows -# arch: i386 -# arch: x86_64 -# -# Extra directories used by stack for building -# extra-include-dirs: [/path/to/dir] -# extra-lib-dirs: [/path/to/dir] -# -# Allow a newer minor version of GHC than the snapshot specifies -# compiler-check: newer-minor diff --git a/wordchoice.cabal b/wordchoice.cabal --- a/wordchoice.cabal +++ b/wordchoice.cabal @@ -1,81 +1,73 @@ -name: wordchoice -version: 0.1.0.5 -cabal-version: >=1.10 -build-type: Simple -license: BSD3 -license-file: LICENSE -copyright: 2017 Author name here -maintainer: example@example.com -homepage: https://github.com/githubuser/wordchoice#readme -synopsis: Get word counts and distributions -description: - A command line tool to compute the word distribution from various types of document, converting to text with pandoc. -category: Web -author: Author name here -extra-source-files: - README.md - stack.yaml - default.nix - release.nix - -source-repository head - type: git - location: https://github.com/githubuser/wordchoice +name: wordchoice +version: 0.1.1.0 +synopsis: Get word counts and distributions +description: A command line tool to compute the word distribution from various types of document, converting to text with pandoc. +homepage: https://github.com/githubuser/wordchoice#readme +license: BSD3 +license-file: LICENSE +author: Author name here +maintainer: example@example.com +copyright: 2017 Author name here +category: Web +build-type: Simple +extra-source-files: README.md + , stack.yaml + , default.nix + , release.nix +cabal-version: >=1.10 -flag llvm-fast - description: - Enable build with llvm backend - default: False +Flag llvm-fast { + Description: Enable build with llvm backend + Default: False +} library - exposed-modules: - Data.Text.WordCount - Data.Text.WordCount.Exec - build-depends: - base >=4.7 && <5, - pandoc >=1.19.2.1 && <1.20, - containers >=0.5.7.1 && <0.6, - Glob >=0.7.14 && <0.8, - text >=1.2.2.1 && <1.3, - optparse-applicative >=0.13.2.0 && <0.14, - Chart >=1.8.2 && <1.9, - bytestring >=0.10.8.1 && <0.11, - system-filepath >=0.4.13.4 && <0.5, - Chart-diagrams >=1.8.2 && <1.9, - lens >=4.15.1 && <4.16 - default-language: Haskell2010 - default-extensions: OverloadedStrings - hs-source-dirs: src - other-modules: - Paths_wordchoice - Data.Text.WordCount.FileRead + hs-source-dirs: src + exposed-modules: Data.Text.WordCount + , Data.Text.WordCount.Exec + other-modules: Paths_wordchoice + , Data.Text.WordCount.FileRead + build-depends: base >= 4.7 && < 5 + , pandoc + , containers + , Glob + , text + , optparse-applicative + , Chart + , bytestring + , system-filepath + , Chart-diagrams + , lens + default-language: Haskell2010 + default-extensions: OverloadedStrings executable wrd - main-is: Main.hs - build-depends: - base >=4.9.1.0 && <4.10, - wordchoice >=0.1.0.5 && <0.2 - default-language: Haskell2010 - hs-source-dirs: app - ghc-options: -threaded -rtsopts -with-rtsopts=-N + hs-source-dirs: app + main-is: Main.hs + ghc-options: -threaded -rtsopts -with-rtsopts=-N + build-depends: base + , wordchoice + default-language: Haskell2010 test-suite wordchoice-test - type: exitcode-stdio-1.0 - main-is: Spec.hs - build-depends: - base >=4.9.1.0 && <4.10, - wordchoice >=0.1.0.5 && <0.2 - default-language: Haskell2010 - hs-source-dirs: test - ghc-options: -threaded -rtsopts -with-rtsopts=-N + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: Spec.hs + build-depends: base + , wordchoice + ghc-options: -threaded -rtsopts -with-rtsopts=-N + default-language: Haskell2010 benchmark wordchoice-bench - type: exitcode-stdio-1.0 - main-is: Bench.hs - build-depends: - base >=4.9.1.0 && <4.10, - criterion >=1.1.4.0 && <1.2, - wordchoice >=0.1.0.5 && <0.2 - default-language: Haskell2010 - hs-source-dirs: bench - ghc-options: -threaded -rtsopts -with-rtsopts=-N -O3 + type: exitcode-stdio-1.0 + hs-source-dirs: bench + main-is: Bench.hs + build-depends: base + , criterion + , wordchoice + ghc-options: -threaded -rtsopts -with-rtsopts=-N -O3 + default-language: Haskell2010 + +source-repository head + type: git + location: https://github.com/githubuser/wordchoice