diff --git a/Wordlint.cabal b/Wordlint.cabal --- a/Wordlint.cabal +++ b/Wordlint.cabal @@ -2,13 +2,13 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: Wordlint -version: 0.1.0.2 +version: 0.2.0.0 synopsis: Plaintext prose redundancy linter. description: Wordlint locates matching pairs of words repeated within a user-defined distance. Text may be linted by distance between words (that is, by word - count), by line count, or by percentage of the total words in the file. For - details on running the program, see README or the included man page. + count), by line count, and/or by percentage of the total words in the file. + For details on running the program, see README or the included man page. Internally, Wordlint separates a text file (or stdin) into a list of "Words", after processing user flags. An instance of the "Word" @@ -20,18 +20,20 @@ length of the lemma. Next, items are matched by their lemma and exclusive coordinates before being added to a list of "Wordpairs", a data structure containing two Words and the difference between their respective "positions". + If multiple types are specified, all Int positions are converted to the Double + type and the resulting lists are intersected to produce the final results. Finally, these Wordpairs are optionally filtered by the difference in their positions (another user-specified option). All remaining Wordpairs are processed for printing to stdout in machine-readable (default) or human-readable format. A plugin also exists for integration with Vim - (https://github.com/gbgar/wordcheck.vim). + (https:\/\/github.com\/gbgar\/wordcheck.vim). homepage: https://github.com/gbgar/Wordlint license: OtherLicense license-file: LICENSE author: GB Gardner maintainer: gbgar@users.noreply.github.com -copyright: 2014 +copyright: 2014-2015 category: Text build-type: Simple cabal-version: >=1.10 diff --git a/Wordlint.hs b/Wordlint.hs --- a/Wordlint.hs +++ b/Wordlint.hs @@ -1,244 +1,13 @@ -import Data.List import System.Console.CmdArgs -import Text.PrettyPrint.Boxes import Wordlint.Args -import Wordlint.Words -import Wordlint.Wordpairs +import Wordlint.Linters +import Wordlint.Output main :: IO () main = do -- Execute command line arguments, retrieve flags cargs <- cmdArgs cliargs - let sortflag = sort_ cargs - let wordlen = wordlength cargs - let fname = file cargs - -- If human-readable flag is present, print header checkIfHumanHeader cargs - -- Acquire String data from file or stdin - dat <- accessInputFileData . checkFileStdin $ file cargs - -- blacklist data for running filter function - blist' <- accessBlacklistFileData . checkFileStdin $ blacklist cargs - let blist = setBlacklistData blist' - -- - -- Choose checker and printer according to -t, -h flags - -- - case type_ cargs of - [] -> do - let checkedwords' = runWordCheck dat wordlen dist blist cargs - let checkedwords = checkSortFlag sortflag checkedwords' - if human cargs - then do putStrLn "No type chosen; running word-based check" - mapM_ putStrLn (processHumanWordData checkedwords) - putStrLn "" - summaryData (length checkedwords) (wordlength cargs) (length dat) - else - if sortflag == "error" - then putStrLn $ processDataError checkedwords fname - else processMachineWordData checkedwords - where dist = checkDistanceOrAll cargs - "word" -> do - let checkedwords' = runWordCheck dat wordlen dist blist cargs - let checkedwords = checkSortFlag sortflag checkedwords' - - if human cargs - then do mapM_ putStrLn (processHumanWordData checkedwords) - putStrLn "" - summaryData (length checkedwords) (wordlength cargs) (length dat) - else - if sortflag == "error" - then putStrLn $ processDataError checkedwords fname - else processMachineWordData checkedwords - where dist = checkDistanceOrAll cargs - "line" -> do - let checkedwords' = runLineCheck dat wordlen dist blist cargs - let checkedwords = checkSortFlag sortflag checkedwords' - if human cargs - then do mapM_ putStrLn (processHumanLineData checkedwords) - putStrLn "" - summaryData (length checkedwords) (wordlength cargs) (length dat) - else - if sortflag == "error" - then putStrLn $ processDataError checkedwords fname - else processMachineLineData checkedwords - where dist = checkDistanceOrAll cargs - "percentage" -> do - let checkedwords' = runPercentageCheck dat wordlen dist blist cargs - let checkedwords = checkSortFlag sortflag checkedwords' - if human cargs - then do mapM_ putStrLn (processHumanPercentageData checkedwords) - putStrLn "" - summaryData (length checkedwords) (wordlength cargs) (length dat) - else - if sortflag == "error" - then putStrLn $ processDataError checkedwords fname - else processMachinePercentageData checkedwords - where dist = checkDistanceOrAll cargs - - _ -> do - let checkedwords' = runWordCheck dat wordlen dist blist cargs - let checkedwords = checkSortFlag sortflag checkedwords' - if human cargs - then do putStrLn "No type chosen; running word-based check" - mapM_ putStrLn (processHumanWordData checkedwords) - putStrLn "" - summaryData (length checkedwords) (wordlength cargs) (length dat) - else - if sortflag == "error" - then putStrLn $ processDataError checkedwords fname - else processMachineWordData checkedwords - where dist = checkDistanceOrAll cargs --- run*Check functions are run with a maybe to account for --all flag --- process*Data functions return a human-readable format. --- In the future, a "machine-readable" flag will be added to the latter --- in order to output for use in text editor plugins. - -runWordCheck :: String -> Int -> Maybe Int -> Maybe [String] -> Arguments -> Wordpairs Int -runWordCheck inputdata wordlen distorall blist arg = case distorall of - Nothing -> instring - Just i -> filterWordpairsByDistance instring i - where instring = sortWordsByString . filterMatchingWords $ dwords - dwords = checkWordList fwords wordlen - fwords = runFilterFlags cwords arg blist - cwords = zipWords inputdata "word" - -processHumanWordData :: Wordpairs Int -> [String] -processHumanWordData [] = ["No (more) matches found"] -processHumanWordData (x:xs) = ("\'" ++ word ++ "\'" - ++ " at positions " - ++ position' - ++ " at coordinates " - ++ coordinates - ++ " with an intervening distance of " - ++ distance' - ++ " words") : processHumanWordData xs - where word = getWordPairString x - position' = show (getWordpairPositions x) - coordinates = show (getWordpairCoords x) - distance' = show (pdiff x) - --- machine-based outputs drawn from --- http://www.tedreed.info/programming/2012/06/02/how-to-use-textprettyprintboxes/ -processMachineWordData :: Wordpairs Int -> IO () -processMachineWordData x = printBox $ hsep 2 left (map (vcat left . map text) - (transpose $ ["Coord1", "Coord2", "Word", "Distance"]:processMachineWordData' x)) - - - -processMachineWordData' :: Wordpairs Int -> [[String]] -processMachineWordData' [] = [] -processMachineWordData' (x:xs) = words (coordinates1 coordinates' - ++ " " - ++ coordinates2 coordinates' - ++ " " - ++ word - ++ " " - ++ distance') : processMachineWordData' xs - where word = getWordPairString x - coordinates' = getWordpairCoords x - coordinates1 ((r1,s1),(_,_)) = show r1 ++ "," ++ show s1 - coordinates2 ((_,_),(r2,s2)) = show r2 ++ "," ++ show s2 - distance' = show (pdiff x) - -runLineCheck :: String -> Int -> Maybe Int -> Maybe [String] -> Arguments-> Wordpairs Int -runLineCheck inputdata wordlen distorall blist arg = case distorall of - Nothing -> instring - Just i -> filterWordpairsByDistance instring i - where instring = sortWordsByString . filterMatchingWords $ dwords - dwords = checkWordList fwords wordlen - fwords = runFilterFlags cwords arg blist - cwords = zipWords inputdata "line" - -processHumanLineData :: Wordpairs Int -> [String] -processHumanLineData [] = ["No (more) matches found"] -processHumanLineData (x:xs) = ("\'" ++ word ++ "\'" - ++ " at coordinates " - ++ coordinates - ++ " with an intervening distance of " - ++ distance' - ++ " lines") : processHumanLineData xs - where word = getWordPairString x - coordinates = show (getWordpairCoords x) - distance' = show (pdiff x) - -processMachineLineData :: Wordpairs Int -> IO () -processMachineLineData x = printBox $ hsep 2 left (map (vcat left . map text) - (transpose $ ["Coord1", "Coord2", "Word", "Distance"]:processMachineLineData' x)) - -processMachineLineData' :: Wordpairs Int -> [[String]] -processMachineLineData' [] = [] -processMachineLineData' (x:xs) = words (coordinates1 coordinates' - ++ " " - ++ coordinates2 coordinates' - ++ " " - ++ word - ++ " " - ++ distance') - : processMachineLineData' xs - where word = getWordPairString x - coordinates' = getWordpairCoords x - coordinates1 ((r1,s1),(_,_)) = show r1 ++ "," ++ show s1 - coordinates2 ((_,_),(r2,s2)) = show r2 ++ "," ++ show s2 - distance' = show (pdiff x) - -runPercentageCheck :: String -> Int -> Maybe Double -> Maybe [String] -> Arguments -> Wordpairs Double -runPercentageCheck inputdata wordlen distorall blist arg = case distorall of - Nothing -> instring - Just i -> filterWordpairsByDistance instring i - where instring = sortWordsByString . filterMatchingWords $ dwords - dwords = checkWordList fwords wordlen - fwords = runFilterFlags cwords arg blist - cwords = zipWords inputdata "percentage" - -processHumanPercentageData :: Wordpairs Double -> [String] -processHumanPercentageData [] = ["No (more) matches found"] -processHumanPercentageData (x:xs) = ("\'" ++ word ++ "\'" - ++ " at coordinates " - ++ coordinates - ++ " with an intervening distance of %" - ++ distance') : processHumanPercentageData xs - where word = getWordPairString x - coordinates = show (getWordpairCoords x) - distance' = take 7 (show (pdiff x)) - -processMachinePercentageData :: Wordpairs Double -> IO () -processMachinePercentageData x = printBox $ hsep 2 left (map (vcat left . map text) - (transpose $ ["Coord1", "Coord2", "Word", "Distance"]:processMachinePercentageData' x)) - -processMachinePercentageData' :: Wordpairs Double -> [[String]] -processMachinePercentageData' [] = [] -processMachinePercentageData' (x:xs) = words (coordinates1 coordinates' - ++ " " - ++ coordinates2 coordinates' - ++ " " - ++ word - ++ " " - ++ distance') : processMachinePercentageData' xs - where word = getWordPairString x - coordinates' = getWordpairCoords x - coordinates1 ((r1,s1),(_,_)) = show r1 ++ "," ++ show s1 - coordinates2 ((_,_),(r2,s2)) = show r2 ++ "," ++ show s2 - distance' = take 7 $ show (pdiff x) - -processDataError :: (NumOps a) => Wordpairs a -> String -> String -processDataError [] _ = "" -processDataError (x:xs) fname = (fname ++ ":" ++ linum1 ++ ":" ++ colnum1 ++ ":" ++ word ++ "\n") ++ - (fname ++ ":" ++ linum2 ++ ":" ++ colnum2 ++ ":" ++ word ++ "\n") ++ - processDataError xs fname - where word = getWordPairString x - coordinates' = getWordpairCoords x - coordinates1 ((r1,_),(_,_)) = show r1 - coordinates1' ((_,_),(r1,_)) = show r1 - linum1 = coordinates1 coordinates' - linum2 = coordinates1' coordinates' - coordinates2 ((_,s1),(_,_)) = show s1 - coordinates2' ((_,_),(_,s1)) = show s1 - colnum1 = coordinates2 coordinates' - colnum2 = coordinates2' coordinates' - --- Function that provides summary totals when -h flag is passed -summaryData :: Int -> Int -> Int -> IO() -summaryData x y z = do - putStrLn "" - putStrLn $ "Found " ++ show x ++ " pairs of words " - ++ show y ++ " or more characters in length out of " - ++ show z ++ " words total." + linter <- getLinter cargs + let results = runAllLinters linter + produceOutput linter results diff --git a/changelog b/changelog --- a/changelog +++ b/changelog @@ -1,13 +1,45 @@ +2015-02-19 - version 0.2.0.0 +============================ + +This version brings breaking changes if you typically +use a non-default set of linting arguments. + +- Command line arguments -w/--wordlength, -d/--distance and -t/--type + have been removed. + +- The -m/--matchlength replaces -w/--wordlength + +- The-w/--words, -l/--lines, and -p/--percent take Int/Int/Double, respectively. + The parameter itself represents the type of lint you wish to perform, while the + argument provided by the user indicates the maximum distance between matches. + +- Intersecting lints have been added. The type/distance arguments provided (or not) + are used to create a set of lists. The intersection of these lists + is provided as the end result. + +- Some granularity in the human-readable data has been lost. Due to the addition of + intersecting lints, all final results are treated as Doubles, and the "intervening + distance" between word pairs are unlabeled. This is no issue if running one type of + lint, but with more, it is difficult to sort out which unit is being provided. + +- Wordlint.hs has been heavily manicured. + +- Wordlint.Linters and the `Linter` datatype and associated functions have been + taken the place of the `run*Linter` family of functions in Wordlint.hs + +- Wordlint.Output has been added to house the Human/Machine functions from Wordlint.hs. + These functions have been pruned and some higher-level functions added. + 2014-12-01 - version 0.1.0.2 ============================ -Changed "-s vim" to "-s error". +- Changed "-s vim" to "-s error". -Changed output of "-s error" to produce better, text editor-agnostic output. +- Changed output of "-s error" to produce better, text editor-agnostic output. 2014-11-25 - version 0.1.0.1 ============================= -Added labels to machine-readable output table. +- Added labels to machine-readable output table. -Minor documntation fixes + diff --git a/doc/Wordlint.haddock b/doc/Wordlint.haddock --- a/doc/Wordlint.haddock +++ b/doc/Wordlint.haddock @@ -13,18 +13,19 @@ Wordlint locates matching pairs of words repeated within a user-defined distance. Text may be linted by distance between words (that is, by word -count), by line count, or by percentage of the total words in the file. -The user may also choose a minimum word length for matches. +count), by line count, and\/or by percentage of the total words in the +file. Multiple lint types may be specified at one time. The user may +also choose a minimum word length for matches. Filters are available to remove punctuation, capitalization, and\/or a user-defined list of words from the list of potential matches. -Various modes exist for data output, which is machine-readable by -default with column-based formatting. Results may be sorted by -alphabetically by word, by position (line number), or by intervening -distance between matches; and may be used with a human-readable mode. -Additionally, an \"error\" mode may supersede these options to provide -output designed for easy integration with text editors. +Various modes exist for data output, which is column-based and +machine-readable by default. Results may be sorted by alphabetically by +word, by position (line number), or by intervening distance between +matches; and may be used with a human-readable mode. Additionally, an +\"error\" mode may supersede these options to provide output designed +for easy integration with text editors. = Options #options# @@ -37,32 +38,28 @@ == Linting Options #linting-options# -[-d, --distance /INT | FLOAT/] - Specify maximum intervening distance between returned word-pairs. - __If the type of lint is either \"word\" or \"line\", an integer - must be used__, while a \"percentage\" check will accept a float - value. Ignored if -a is used. Default is 250. -[-t, --type /word|line|percentage/] - Specify type of lint to perform, which affects which the calculation - of intervening distance between word pairs. Options are: - - > - word (default) - > - line - > - percentage - - A word-type check will define a word\'s \"position\" as it\'s word - count, while a line check uses the line number on which a word is - found. A percentage check sets this value according to a word\'s - count divided by the total count of words in the input. - -[-w ,--wordlength /NUMBER/] +[-w, --words /INT/] + Specify maximum intervening distance between returned word-pairs + measuring by word count. This may intersect with the --lines and + --percent options, but is ignored if -a is provided. Default is 250. +[-l, --distance /INT/] + Specify maximum intervening distance between returned word-pairs + measuring by line count. This may intersect with the --words and + --percent options, but is ignored if -a is provided. Default is 0 + (off). +[-p, --percent /DOUBLE/] + Specify maximum intervening distance between returned word-pairs + measuring by percentage of words. This may intersect with the + --words and --lines options, but is ignored if -a is provided. + Default is 0 (off). +[-m ,--matchlength /NUMBER/] Specify minimum length of words to be matched, i.e. to reduce hits - for \"there\". Default is 5. + for \"the\". Default is 5. == Filters #filters# -[-b, --blacklist] +[-b, --blacklist /FILE/] Specify a file containing a newline-separated list of words (no spaces) to filter from matches. Pairs well with --nopunct, which is applied before, but activated prior to application of --nocaps @@ -102,14 +99,20 @@ words. The results are in a machine-readable table format (i.e. for easy use with @awk@, @sed@, and the like). -> wordlint --type line --distance 20 --wordlength 7 --file file.txt +> wordlint --lines 20 --matchlength 7 --file file.txt Finds matching strings consisting of seven characters or more and which have an intervening distance of twenty lines or less. Returns machine-readable format. -> cat file.txt | wordlint -t percentage -d 2.5 -a -s word -h +> wordlint -w 100 -l 20 -m 7 -f file.txt +Finds matching strings consisting of seven characters or more and which +fall within an intervening distance of /both/ 100 words /and/ twenty +lines or less. Returns machine-readable format. + +> cat file.txt | wordlint --percent 2.5 -a -s word -h + Finds all matching, five-characters-or-longer strings within a 2.5% distance of one-another within the file, and returns the output sorted alphabetically and in \"human-readable form. @@ -125,5 +128,8 @@ #see-also# A Vim front-end to Wordlint, creatively named Wordlint.vim, is available -at https:\/\/github.com\/gbgar\/Wordlint.vim +at +A compiler and a flycheck interface are available for Emacs: + and +. diff --git a/man/man1/wordlint.1 b/man/man1/wordlint.1 --- a/man/man1/wordlint.1 +++ b/man/man1/wordlint.1 @@ -1,4 +1,4 @@ -.TH "WORDLINT" "1" "2014\-11\-22" "0.1.0.2 +.TH "WORDLINT" "1" "2015\-02\-24" "0.2.0.0 .SH Name .PP wordlint \- plaintext redundancy linter @@ -10,14 +10,15 @@ Wordlint locates matching pairs of words repeated within a user\-defined distance. Text may be linted by distance between words (that is, by word count), -by line count, or by percentage of the total words in the file. +by line count, and/or by percentage of the total words in the file. +Multiple lint types may be specified at one time. The user may also choose a minimum word length for matches. .PP Filters are available to remove punctuation, capitalization, and/or a user\-defined list of words from the list of potential matches. .PP -Various modes exist for data output, which is machine\-readable by -default with column\-based formatting. +Various modes exist for data output, which is column\-based and +machine\-readable by default. Results may be sorted by alphabetically by word, by position (line number), or by intervening distance between matches; and may be used with a human\-readable mode. @@ -37,44 +38,42 @@ .RE .SS Linting Options .TP -.B \-d, \-\-distance \f[I]INT | FLOAT\f[] -Specify maximum intervening distance between returned word\-pairs. -\f[B]If the type of lint is either "word" or "line", an integer must be -used\f[], while a "percentage" check will accept a float value. -Ignored if \-a is used. +.B \-w, \-\-words \f[I]INT\f[] +Specify maximum intervening distance between returned word\-pairs +measuring by word count. +This may intersect with the \-\-lines and \-\-percent options, but is +ignored if \-a is provided. Default is 250. .RS .RE .TP -.B \-t, \-\-type \f[I]word|line|percentage\f[] -Specify type of lint to perform, which affects which the calculation of -intervening distance between word pairs. -Options are: +.B \-l, \-\-distance \f[I]INT\f[] +Specify maximum intervening distance between returned word\-pairs +measuring by line count. +This may intersect with the \-\-words and \-\-percent options, but is +ignored if \-a is provided. +Default is 0 (off). .RS -.IP -.nf -\f[C] -\-\ word\ (default) -\-\ line -\-\ percentage -\f[] -.fi -.PP -A word\-type check will define a word\[aq]s "position" as it\[aq]s word -count, while a line check uses the line number on which a word is found. -A percentage check sets this value according to a word\[aq]s count -divided by the total count of words in the input. .RE .TP -.B \-w ,\-\-wordlength \f[I]NUMBER\f[] +.B \-p, \-\-percent \f[I]DOUBLE\f[] +Specify maximum intervening distance between returned word\-pairs +measuring by percentage of words. +This may intersect with the \-\-words and \-\-lines options, but is +ignored if \-a is provided. +Default is 0 (off). +.RS +.RE +.TP +.B \-m ,\-\-matchlength \f[I]NUMBER\f[] Specify minimum length of words to be matched, i.e. -to reduce hits for "there". +to reduce hits for "the". Default is 5. .RS .RE .SS Filters .TP -.B \-b, \-\-blacklist +.B \-b, \-\-blacklist \f[I]FILE\f[] Specify a file containing a newline\-separated list of words (no spaces) to filter from matches. Pairs well with \-\-nopunct, which is applied before, but activated @@ -139,7 +138,7 @@ .IP .nf \f[C] -wordlint\ \-\-type\ line\ \-\-distance\ 20\ \-\-wordlength\ 7\ \-\-file\ file.txt +wordlint\ \-\-lines\ 20\ \-\-matchlength\ 7\ \-\-file\ file.txt \f[] .fi .PP @@ -149,10 +148,21 @@ .IP .nf \f[C] -cat\ file.txt\ |\ wordlint\ \-t\ percentage\ \-d\ 2.5\ \-a\ \-s\ word\ \-h\ +wordlint\ \-w\ 100\ \-l\ 20\ \-m\ 7\ \-f\ file.txt \f[] .fi .PP +Finds matching strings consisting of seven characters or more and which +fall within an intervening distance of \f[I]both\f[] 100 words +\f[I]and\f[] twenty lines or less. +Returns machine\-readable format. +.IP +.nf +\f[C] +cat\ file.txt\ |\ wordlint\ \-\-percent\ 2.5\ \-a\ \-s\ word\ \-h\ +\f[] +.fi +.PP Finds all matching, five\-characters\-or\-longer strings within a 2.5% distance of one\-another within the file, and returns the output sorted alphabetically and in "human\-readable form. @@ -171,6 +181,10 @@ .SH See Also .PP A Vim front\-end to Wordlint, creatively named Wordlint.vim, is -available at https://github.com/gbgar/Wordlint.vim +available at +.PP +A compiler and a flycheck interface are available for Emacs: + and +. .SH AUTHOR GB Gardner.