wordchoice 0.1.1.1 → 0.1.2.0
raw patch · 4 files changed
+80/−18 lines, 4 filesdep +binarydep +composition-preludedep +directory
Dependencies added: binary, composition-prelude, directory
Files
- cabal.project.local +3/−0
- src/Data/Text/WordCount.hs +33/−10
- src/Data/Text/WordCount/Exec.hs +38/−7
- wordchoice.cabal +6/−1
+ cabal.project.local view
@@ -0,0 +1,3 @@+constraints: wordchoice +fast-llvm+with-compiler: ghc-8.2.1+optimization: 2
src/Data/Text/WordCount.hs view
@@ -15,21 +15,33 @@ , globFile -- * Low level-ish , buildFreq+ -- * Keyed maps+ , indexed ) where -import Control.Lens hiding (argument)+import Control.Arrow ((&&&))+import Control.Composition (on)+import Control.Lens hiding (argument,+ indexed) import Data.Char+import qualified Data.IntMap as IM import Data.List import qualified Data.Map.Lazy as M import Data.Map.Lens import Data.Monoid import Data.Ord-import qualified Data.Text as TL+import qualified Data.Text as T+import qualified Data.Text.Lazy as TL import Data.Text.WordCount.FileRead import Data.Tuple import Graphics.Rendering.Chart.Backend.Diagrams-import Graphics.Rendering.Chart.Easy hiding (argument)+import Graphics.Rendering.Chart.Easy hiding (argument,+ indexed) +-- | Return an `IntMap` containing words indexed by their frequencies.+indexed :: TL.Text -> IM.IntMap [TL.Text]+indexed = orderM . buildFreq+ -- | Return top n words and their frequencies -- -- @@@ -49,22 +61,33 @@ common :: TL.Text -> Bool common = flip elem ["the","and","a","an","or","not","but","on","so","if","in","that","this","for"] -displayWords :: [(Int,TL.Text)] -> TL.Text+displayWords :: [(Int,TL.Text)] -> T.Text displayWords [] = "" displayWords (pair:pairs) = display pair <> "\n" <> displayWords pairs- where display (n,str) = (TL.pack . show) n <> ": " <> str+ where display (n,str) = (T.pack . show) n <> ": " <> TL.toStrict str buildFreq :: TL.Text -> M.Map TL.Text Int-buildFreq = count . TL.words . TL.map toLower+buildFreq = count . filter (/=" ") . TL.split (`elem` (" \n;:,./" :: String)) . TL.map toLower +orderM :: M.Map TL.Text Int -> IM.IntMap [TL.Text]+orderM = IM.fromList . fmap ((fst . head) &&& fmap snd) . groupBy go . sortBy (flip (comparing fst)) . fmap swap . M.toList+ where go :: (Int, TL.Text) -> (Int, TL.Text) -> Bool+ go = on (==) fst+ order :: M.Map TL.Text Int -> [(Int, TL.Text)] order = sortBy (flip (comparing fst)) . fmap swap . M.toList count :: [TL.Text] -> M.Map TL.Text Int-count words = foldr ((.) . wordFunction) id words M.empty- where wordFunction word map = case map ^. at word of- Nothing -> at word ?~ 1 $ map- _ -> ix word %~ (+1) $ map+count words = foldr ((.) . go) id words M.empty+ where go word m = case m^.at word of+ Nothing -> at word ?~ 1 $ m+ _ -> ix word %~ (+1) $ m++count' :: [TL.Text] -> M.Map TL.Text Int+count' = M.fromListWith (+) . (`zip` repeat 1)++count'' :: [TL.Text] -> [(TL.Text, Int)]+count'' = fmap (head &&& length) . group . sort -- | Make a bar graph from the word frequencies --
src/Data/Text/WordCount/Exec.hs view
@@ -1,19 +1,29 @@ module Data.Text.WordCount.Exec where +import Control.Arrow ((&&&))+import Control.Lens (over, _2)+import Data.Binary (decode, encode)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BSL+import qualified Data.IntMap as IM import Data.Maybe import Data.Monoid-import qualified Data.Text.IO as TLIO+import qualified Data.Text.IO as TIO+import qualified Data.Text.Lazy as TL import Data.Text.WordCount import Data.Text.WordCount.FileRead import Data.Version import Options.Applicative import Paths_wordchoice+import System.Directory (doesFileExist) -- | Program datatype to be parsed data Program = Program { file :: FilePath , num :: Maybe Int , output :: Maybe FilePath- , filterOutput :: Bool } -- TODO add option for separators+ , filterOutput :: Bool+ , cacheIndex :: Bool+ } -- TODO add option for separators -- | Command line argument parser program :: Parser Program@@ -36,11 +46,15 @@ (short 'f' <> long "filter" <> help "Filter common English words from output.")+ <*> switch+ (short 'd'+ <> long "dump"+ <> help "Cache word frequency indices") -- | Parse for version info versionInfo :: Parser (a -> a) versionInfo = infoOption- ("wordchoice version: " ++ showVersion version)+ ("wordchoice version: " <> showVersion version) (short 'v' <> long "version" <> help "Show version") -- | Wraps parser with help parser@@ -54,12 +68,29 @@ exec :: IO () exec = execParser wrapper >>= pick +-- TODO stream output nicely?+ -- | Run parsed record pick :: Program -> IO ()-pick rec = let n = fromMaybe 25 (num rec) in do- contents <- globFile (file rec)- let pick = if not (filterOutput rec) then topN n else filterTop n small- TLIO.putStrLn . displayWords . pick $ contents+pick rec = do+ let n = fromMaybe 25 (num rec)+ contents <- TL.fromStrict <$> globFile (file rec)+ pickContents <- case (filterOutput &&& num) rec of {+ (True, _) -> pure $ filterTop n small contents ;+ (False, Just x) -> pure $ topN x contents ;+ (False, Nothing) -> do {+ cacheExists <- doesFileExist "index.bin" ;+ let toDisplay = (>>= (\(i, ws) -> zip (repeat i) ws)) in+ if cacheExists+ then toDisplay . IM.toList . (decode :: BSL.ByteString -> IM.IntMap [TL.Text]) . BSL.fromStrict <$> BS.readFile "index.bin"+ else pure . toDisplay . IM.toList . indexed $ contents }+ }+ if cacheIndex rec+ then do {+ BS.writeFile "index.bin" . BSL.toStrict $ encode $ indexed contents ;+ putStrLn "...finished indexing" ;+ TIO.putStrLn . displayWords $ pickContents }+ else TIO.putStrLn . displayWords $ pickContents case output rec of (Just out) -> flip makeFile out . topN n $ contents _ -> pure ()
wordchoice.cabal view
@@ -1,5 +1,5 @@ name: wordchoice-version: 0.1.1.1+version: 0.1.2.0 synopsis: Get word counts and distributions description: A command line tool to compute the word distribution from various types of document, converting to text with pandoc. homepage: https://github.com/githubuser/wordchoice#readme@@ -12,6 +12,7 @@ build-type: Simple extra-source-files: README.md , stack.yaml+ , cabal.project.local , default.nix , release.nix cabal-version: >=1.10@@ -31,8 +32,12 @@ , pandoc , containers , Glob+ , bytestring+ , binary , text+ , directory , optparse-applicative+ , composition-prelude , Chart , bytestring , system-filepath