diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,5 @@ +Copyright © 2014 Blake Gardner github.com/gbgar + +This work is free. You can redistribute it and/or modify it under the terms of +the Do What The Fuck You Want To Public License, Version 2, as published by Sam +Hocevar. See for more details. diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/Wordlint.cabal b/Wordlint.cabal new file mode 100644 --- /dev/null +++ b/Wordlint.cabal @@ -0,0 +1,53 @@ +-- Initial Wordlint.cabal generated by cabal init. For further +-- documentation, see http://haskell.org/cabal/users-guide/ + +name: Wordlint +version: 0.1.0.0 +synopsis: Plaintext prose redundancy linter. + +description: Wordlint locates matching pairs of words repeated within a user-defined + distance. Text may be linted by distance between words (that is, by word + count), by line count, or by percentage of the total words in the file. For + details on running the program, see README or the included man page. + + Internally, Wordlint separates a text file (or stdin) into + a list of "Words", after processing user flags. An instance of the "Word" + datatype consisting of a String that contains a lemma; its line and column + coordinates; and its "position": an Int or Float depending on the type of check + invoked by the user. + + This list is first filtered by the user-defined minimum + length of the lemma. Next, items are matched by their lemma and exclusive + coordinates before being added to a list of "Wordpairs", a data structure + containing two Words and the difference between their respective "positions". + Finally, these Wordpairs are optionally filtered by the difference in their + positions (another user-specified option). All remaining Wordpairs are + processed for printing to stdout in machine-readable (default) or + human-readable format. A plugin also exists for integration with Vim + (https://github.com/gbgar/wordcheck.vim). + +homepage: https://github.com/gbgar/Wordlint +license: OtherLicense +license-file: LICENSE +author: GB Gardner +maintainer: gbgar@users.noreply.github.com +copyright: 2014 +category: Text +build-type: Simple +cabal-version: >=1.10 +extra-doc-files: doc/Wordlint.haddock + man/man1/wordlint.1 + +executable wordlint + main-is: Wordlint.hs + -- other-modules: + other-extensions: DeriveDataTypeable + build-depends: base >=4.7 && <4.8, cmdargs >=0.10 && <0.11, boxes >=0.1 && <0.2 + -- hs-source-dirs: + default-language: Haskell2010 + ghc-options: -O2 + +source-repository head + type: git + location: git://github.com/gbgar/Wordlint.git + diff --git a/Wordlint.hs b/Wordlint.hs new file mode 100644 --- /dev/null +++ b/Wordlint.hs @@ -0,0 +1,240 @@ +import Data.List +import System.Console.CmdArgs +import Text.PrettyPrint.Boxes +import Wordlint.Args +import Wordlint.Words +import Wordlint.Wordpairs + +main :: IO () +main = do + -- Execute command line arguments, retrieve flags + cargs <- cmdArgs cliargs + let sortflag = sort_ cargs + let wordlen = wordlength cargs + -- If human-readable flag is present, print header + checkIfHumanHeader cargs + -- Acquire String data from file or stdin + dat <- accessInputFileData . checkFileStdin $ file cargs + -- blacklist data for running filter function + blist' <- accessBlacklistFileData . checkFileStdin $ blacklist cargs + let blist = setBlacklistData blist' + -- + -- Choose checker and printer according to -t, -h flags + -- + case type_ cargs of + [] -> do + let checkedwords' = runWordCheck dat wordlen dist blist cargs + let checkedwords = checkSortFlag sortflag checkedwords' + if human cargs + then do putStrLn "No type chosen; running word-based check" + mapM_ putStrLn (processHumanWordData checkedwords) + putStrLn "" + summaryData (length checkedwords) (wordlength cargs) (length dat) + else + if sortflag == "vim" + then putStrLn $ processDataVim checkedwords + else processMachineWordData checkedwords + where dist = checkDistanceOrAll cargs + "word" -> do + let checkedwords' = runWordCheck dat wordlen dist blist cargs + let checkedwords = checkSortFlag sortflag checkedwords' + + if human cargs + then do mapM_ putStrLn (processHumanWordData checkedwords) + putStrLn "" + summaryData (length checkedwords) (wordlength cargs) (length dat) + else + if sortflag == "vim" + then putStrLn $ processDataVim checkedwords + else processMachineWordData checkedwords + where dist = checkDistanceOrAll cargs + "line" -> do + let checkedwords' = runLineCheck dat wordlen dist blist cargs + let checkedwords = checkSortFlag sortflag checkedwords' + if human cargs + then do mapM_ putStrLn (processHumanLineData checkedwords) + putStrLn "" + summaryData (length checkedwords) (wordlength cargs) (length dat) + else + if sortflag == "vim" + then putStrLn $ processDataVim checkedwords + else processMachineLineData checkedwords + where dist = checkDistanceOrAll cargs + "percentage" -> do + let checkedwords' = runPercentageCheck dat wordlen dist blist cargs + let checkedwords = checkSortFlag sortflag checkedwords' + if human cargs + then do mapM_ putStrLn (processHumanPercentageData checkedwords) + putStrLn "" + summaryData (length checkedwords) (wordlength cargs) (length dat) + else + if sortflag == "vim" + then putStrLn $ processDataVim checkedwords + else processMachinePercentageData checkedwords + where dist = checkDistanceOrAll cargs + + _ -> do + let checkedwords' = runWordCheck dat wordlen dist blist cargs + let checkedwords = checkSortFlag sortflag checkedwords' + if human cargs + then do putStrLn "No type chosen; running word-based check" + mapM_ putStrLn (processHumanWordData checkedwords) + putStrLn "" + summaryData (length checkedwords) (wordlength cargs) (length dat) + else + if sortflag == "vim" + then putStrLn $ processDataVim checkedwords + else processMachineWordData checkedwords + where dist = checkDistanceOrAll cargs +-- run*Check functions are run with a maybe to account for --all flag +-- process*Data functions return a human-readable format. +-- In the future, a "machine-readable" flag will be added to the latter +-- in order to output for use in text editor plugins. + +runWordCheck :: String -> Int -> Maybe Int -> Maybe [String] -> Arguments -> Wordpairs Int +runWordCheck inputdata wordlen distorall blist arg = case distorall of + Nothing -> instring + Just i -> filterWordpairsByDistance instring i + where instring = sortWordsByString . filterMatchingWords $ dwords + dwords = checkWordList fwords wordlen + fwords = runFilterFlags cwords arg blist + cwords = zipWords inputdata "word" + +processHumanWordData :: Wordpairs Int -> [String] +processHumanWordData [] = ["No (more) matches found"] +processHumanWordData (x:xs) = ("\'" ++ word ++ "\'" + ++ " at positions " + ++ position' + ++ " at coordinates " + ++ coordinates + ++ " with an intervening distance of " + ++ distance' + ++ " words") : processHumanWordData xs + where word = getWordPairString x + position' = show (getWordpairPositions x) + coordinates = show (getWordpairCoords x) + distance' = show (pdiff x) + +-- machine-based outputs drawn from +-- http://www.tedreed.info/programming/2012/06/02/how-to-use-textprettyprintboxes/ +processMachineWordData :: Wordpairs Int -> IO () +processMachineWordData x = printBox $ hsep 2 left (map (vcat left . map text) (transpose $ processMachineWordData' x)) + +processDataVim :: (NumOps a) => Wordpairs a -> String +processDataVim [] = "" +processDataVim (x:xs) = ("lnum=" ++ "\'" ++ linum1 ++ "\', " ++ "col=" ++ "\'" ++ colnum1 ++ "\', " ++ "text=" ++ "\'" ++ word ++ "\'\n") ++ + ("lnum=" ++ "\'" ++ linum2 ++ "\', " ++ "col=" ++ "\'" ++ colnum2 ++ "\', " ++ "text=" ++ "\'" ++ word ++ "\'\n") ++ + processDataVim xs + where word = getWordPairString x + coordinates' = getWordpairCoords x + coordinates1 ((r1,_),(_,_)) = show r1 + coordinates1' ((_,_),(r1,_)) = show r1 + linum1 = coordinates1 coordinates' + linum2 = coordinates1' coordinates' + coordinates2 ((_,s1),(_,_)) = show s1 + coordinates2' ((_,_),(_,s1)) = show s1 + colnum1 = coordinates2 coordinates' + colnum2 = coordinates2' coordinates' + + +processMachineWordData' :: Wordpairs Int -> [[String]] +processMachineWordData' [] = [] +processMachineWordData' (x:xs) = words (coordinates1 coordinates' + ++ " " + ++ coordinates2 coordinates' + ++ " " + ++ word + ++ " " + ++ distance') : processMachineWordData' xs + where word = getWordPairString x + coordinates' = getWordpairCoords x + coordinates1 ((r1,s1),(_,_)) = show r1 ++ "," ++ show s1 + coordinates2 ((_,_),(r2,s2)) = show r2 ++ "," ++ show s2 + distance' = show (pdiff x) + +runLineCheck :: String -> Int -> Maybe Int -> Maybe [String] -> Arguments-> Wordpairs Int +runLineCheck inputdata wordlen distorall blist arg = case distorall of + Nothing -> instring + Just i -> filterWordpairsByDistance instring i + where instring = sortWordsByString . filterMatchingWords $ dwords + dwords = checkWordList fwords wordlen + fwords = runFilterFlags cwords arg blist + cwords = zipWords inputdata "line" + +processHumanLineData :: Wordpairs Int -> [String] +processHumanLineData [] = ["No (more) matches found"] +processHumanLineData (x:xs) = ("\'" ++ word ++ "\'" + ++ " at coordinates " + ++ coordinates + ++ " with an intervening distance of " + ++ distance' + ++ " lines") : processHumanLineData xs + where word = getWordPairString x + coordinates = show (getWordpairCoords x) + distance' = show (pdiff x) + +processMachineLineData :: Wordpairs Int -> IO () +processMachineLineData x = printBox $ hsep 2 left (map (vcat left . map text) (transpose $ processMachineLineData' x)) + +processMachineLineData' :: Wordpairs Int -> [[String]] +processMachineLineData' [] = [] +processMachineLineData' (x:xs) = words (coordinates1 coordinates' + ++ " " + ++ coordinates2 coordinates' + ++ " " + ++ word + ++ " " + ++ distance') + : processMachineLineData' xs + where word = getWordPairString x + coordinates' = getWordpairCoords x + coordinates1 ((r1,s1),(_,_)) = show r1 ++ "," ++ show s1 + coordinates2 ((_,_),(r2,s2)) = show r2 ++ "," ++ show s2 + distance' = show (pdiff x) + +runPercentageCheck :: String -> Int -> Maybe Double -> Maybe [String] -> Arguments -> Wordpairs Double +runPercentageCheck inputdata wordlen distorall blist arg = case distorall of + Nothing -> instring + Just i -> filterWordpairsByDistance instring i + where instring = sortWordsByString . filterMatchingWords $ dwords + dwords = checkWordList fwords wordlen + fwords = runFilterFlags cwords arg blist + cwords = zipWords inputdata "percentage" + +processHumanPercentageData :: Wordpairs Double -> [String] +processHumanPercentageData [] = ["No (more) matches found"] +processHumanPercentageData (x:xs) = ("\'" ++ word ++ "\'" + ++ " at coordinates " + ++ coordinates + ++ " with an intervening distance of %" + ++ distance') : processHumanPercentageData xs + where word = getWordPairString x + coordinates = show (getWordpairCoords x) + distance' = take 7 (show (pdiff x)) + +processMachinePercentageData :: Wordpairs Double -> IO () +processMachinePercentageData x = printBox $ hsep 2 left (map (vcat left . map text) + (transpose $ processMachinePercentageData' x)) + +processMachinePercentageData' :: Wordpairs Double -> [[String]] +processMachinePercentageData' [] = [] +processMachinePercentageData' (x:xs) = words (coordinates1 coordinates' + ++ " " + ++ coordinates2 coordinates' + ++ " " + ++ word + ++ " " + ++ distance') : processMachinePercentageData' xs + where word = getWordPairString x + coordinates' = getWordpairCoords x + coordinates1 ((r1,s1),(_,_)) = show r1 ++ "," ++ show s1 + coordinates2 ((_,_),(r2,s2)) = show r2 ++ "," ++ show s2 + distance' = take 7 $ show (pdiff x) + +-- Function that provides summary totals when -h flag is passed +summaryData :: Int -> Int -> Int -> IO() +summaryData x y z = do + putStrLn "" + putStrLn $ "Found " ++ show x ++ " pairs of words " + ++ show y ++ " or more characters in length out of " + ++ show z ++ " words total." diff --git a/doc/Wordlint.haddock b/doc/Wordlint.haddock new file mode 100644 --- /dev/null +++ b/doc/Wordlint.haddock @@ -0,0 +1,120 @@ += Name +#name# + +wordlint - plaintext redundancy linter + += Synopsis +#synopsis# + +wordcheck [OPTIONS] [-f input-file] + += Description +#description# + +Wordlint locates matching pairs of words repeated within a user-defined +distance. Text may be linted by distance between words (that is, by word +count), by line count, or by percentage of the total words in the file. +The user may also choose a minimum word length for matches. + +Filters are available to remove punctuation, capitalization, and\/or a +user-defined list of words from the list of potential matches. + +Various modes exist for data output, which is machine-readable by +default. Results may be sorted by alphabetically by word, by position +(line number), or by intervening distance between matches; and may be +used with a human-readable mode. Additionally, a Vim mode provides +output for a plugin. + += Options +#options# + +[--help] + Display condensed help and exit. +[-f, --file /FILE/] + Specify an input file. If none is given, wordlint reads from stdin. + +== Linting Options +#linting-options# + +[-d, --distance /INT | FLOAT/] + Specify maximum intervening distance between returned word-pairs. + __If the type of lint is either \"word\" or \"line\", an integer + must be used__, while a \"percentage\" check will accept a float + value. Ignored if -a is used. Default is 250. +[-t, --type /word|line|percentage/] + Specify type of lint to perform, which affects which the calculation + of intervening distance between word pairs. Options are: + + > - word (default) + > - line + > - percentage + + A word-type check will define a word\'s \"position\" as it\'s word + count, while a line check uses the line number on which a word is + found. A percentage check sets this value according to a word\'s + count divided by the total count of words in the input. + +[-w ,--wordlength /NUMBER/] + Specify minimum length of words to be matched, i.e. to reduce hits + for \"there\". Default is 5. + +== Filters +#filters# + +[-b, --blacklist] + Specify a file containing a newline-separated list of words (no + spaces) to filter from matches. Pairs well with --nopunct, which is + applied before, but activated prior to application of --nocaps + filter. Thus, --nocaps will not interfere, for example, with proper + names given in the blacklist. +[--nocaps] + Ignore capitalization when determining matches. +[--nopunct] + Ignore punctuation when determining matches. + +== Output Options +#output-options# + +[-a, --all] + Return all matched pairs of words regardless of intervening + distance. Deactivates -d parameter. +[-h, --human] + Return human-readable output. Compatible with all sorting except for + @--show vim@, which will supersede @--human@. +[-s, --sort /word|position|distance|vim/] + Sort word pairs alphabetically, by line number, or by intervening + distance; or provides Vim plugin output respective to the following + options: + + > - word + > - position (default) + > - distance + > - vim + += Examples +#examples# + +> wordlint --type line --distance 20 --wordlength 7 --file file.txt + +Finds matching strings consisting of seven characters or more and which +have an intervening distance of twenty lines or less. Returns +machine-readable format. + +> cat file.txt | wordlint -t percentage -d 2.5 -a -s word -h + +Finds all matching, five-characters-or-longer strings within a 2.5% +distance of one-another within the file, and returns the output sorted +alphabetically and in human-readable form. + +> wordlint -f file.txt -b dir/blacklist.txt --nopunct --nocaps + +Finds matching strings consisting of 5 characters or more, and which +have had punctuation, a list of words, and all capitalization stripped +from the possible matches. Returns machine-readable table. + += See Also +#see-also# + +A Vim front-end to Wordlint, creatively named Wordlint.vim, is available +at https:\/\/github.com\/gbgar\/Wordlint.vim + diff --git a/man/man1/wordlint.1 b/man/man1/wordlint.1 new file mode 100644 --- /dev/null +++ b/man/man1/wordlint.1 @@ -0,0 +1,162 @@ +.TH "WORDLINT" "1" "2014\-11\-22" "0.1.0.0 +.SH Name +.PP +wordlint \- plaintext redundancy linter +.SH Synopsis +.PP +wordcheck [OPTIONS] [\-f input\-file] +.SH Description +.PP +Wordlint locates matching pairs of words repeated within a user\-defined +distance. +Text may be linted by distance between words (that is, by word count), +by line count, or by percentage of the total words in the file. +The user may also choose a minimum word length for matches. +.PP +Filters are available to remove punctuation, capitalization, and/or a +user\-defined list of words from the list of potential matches. +.PP +Various modes exist for data output, which is machine\-readable by +default. +Results may be sorted by alphabetically by word, by position (line +number), or by intervening distance between matches; and may be used +with a human\-readable mode. +Additionally, a Vim mode provides output for a plugin. +.SH Options +.TP +.B \-\-help +Display condensed help and exit. +.RS +.RE +.TP +.B \-f, \-\-file \f[I]FILE\f[] +Specify an input file. +If none is given, wordlint reads from stdin. +.RS +.RE +.SS Linting Options +.TP +.B \-d, \-\-distance \f[I]INT | FLOAT\f[] +Specify maximum intervening distance between returned word\-pairs. +\f[B]If the type of lint is either "word" or "line", an integer must be +used\f[], while a "percentage" check will accept a float value. +Ignored if \-a is used. +Default is 250. +.RS +.RE +.TP +.B \-t, \-\-type \f[I]word|line|percentage\f[] +Specify type of lint to perform, which affects which the calculation of +intervening distance between word pairs. +Options are: +.RS +.IP +.nf +\f[C] +\-\ word\ (default) +\-\ line +\-\ percentage +\f[] +.fi +.PP +A word\-type check will define a word\[aq]s "position" as it\[aq]s word +count, while a line check uses the line number on which a word is found. +A percentage check sets this value according to a word\[aq]s count +divided by the total count of words in the input. +.RE +.TP +.B \-w ,\-\-wordlength \f[I]NUMBER\f[] +Specify minimum length of words to be matched, i.e. +to reduce hits for "there". +Default is 5. +.RS +.RE +.SS Filters +.TP +.B \-b, \-\-blacklist +Specify a file containing a newline\-separated list of words (no spaces) +to filter from matches. +Pairs well with \-\-nopunct, which is applied before, but activated +prior to application of \-\-nocaps filter. +Thus, \-\-nocaps will not interfere, for example, with proper names +given in the blacklist. +.RS +.RE +.TP +.B \-\-nocaps +Ignore capitalization when determining matches. +.RS +.RE +.TP +.B \-\-nopunct +Ignore punctuation when determining matches. +.RS +.RE +.SS Output Options +.TP +.B \-a, \-\-all +Return all matched pairs of words regardless of intervening distance. +Deactivates \-d parameter. +.RS +.RE +.TP +.B \-h, \-\-human +Return human\-readable output. +Compatible with all sorting except for \f[C]\-\-show\ vim\f[], which +will supersede \f[C]\-\-human\f[]. +.RS +.RE +.TP +.B \-s, \-\-sort \f[I]word|position|distance|vim\f[] +Sort word pairs alphabetically, by line number, or by intervening +distance; or provides Vim plugin output respective to the following +options: +.RS +.IP +.nf +\f[C] +\-\ word +\-\ position\ (default) +\-\ distance +\-\ vim +\f[] +.fi +.RE +.SH Examples +.IP +.nf +\f[C] +wordlint\ \-\-type\ line\ \-\-distance\ 20\ \-\-wordlength\ 7\ \-\-file\ file.txt +\f[] +.fi +.PP +Finds matching strings consisting of seven characters or more and which +have an intervening distance of twenty lines or less. +Returns machine\-readable format. +.IP +.nf +\f[C] +cat\ file.txt\ |\ wordlint\ \-t\ percentage\ \-d\ 2.5\ \-a\ \-s\ word\ \-h\ +\f[] +.fi +.PP +Finds all matching, five\-characters\-or\-longer strings within a 2.5% +distance of one\-another within the file, and returns the output sorted +alphabetically and in human\-readable form. +.IP +.nf +\f[C] +wordlint\ \-f\ file.txt\ \-b\ dir/blacklist.txt\ \-\-nopunct\ \-\-nocaps +\f[] +.fi +.PP +Finds matching strings consisting of 5 characters or more, and which +have had punctuation, a list of words, and all capitalization stripped +from the possible matches. +Returns machine\-readable table. +.SH See Also +.PP +A Vim front\-end to Wordlint, creatively named Wordlint.vim, is +available at https://github.com/gbgar/Wordlint.vim +.SH AUTHOR +GB Gardner.