wordsetdiff 0.0.2 → 0.0.3
raw patch · 2 files changed
+45/−12 lines, 2 files
Files
- Text/WordSetDiff/Main.hs +42/−10
- wordsetdiff.cabal +3/−2
Text/WordSetDiff/Main.hs view
@@ -98,6 +98,8 @@ -------------------------------------------------------------------------------- -- CONFIGURATION VARIABLES: +version = "0.0.3"+ -- How many consecutive words should we look at? default_word_sequence_size = 3 clump_distance = 10@@ -108,21 +110,27 @@ data CmdFlag = NoColor | NWords Int+ | UseLines | WithPunc+ | AlphaNumeric | AlphaOnly | CaseInsensitive options :: [OptDescr CmdFlag] options = [ Option ['n'] ["nwords"] (ReqArg (NWords . safeRead) "NUM") "use word tuples of length NUM"+ , Option ['l'] ["lines"] (NoArg UseLines) "compare using lines instead of a fixed size sliding window of words" , Option [] ["nocolor"] (NoArg NoColor) "disable ANSI color output" - , Option ['p'] ["withpunc"] (NoArg WithPunc) "use all non-whitespace chars as 'words'"- , Option [ ] ["alpha"] (NoArg AlphaOnly) "ignore all but A-Z characters when forming words [default]"+ , Option [ ] ["alpha"] (NoArg AlphaOnly) "ignore all but A-Z characters when forming words [default]"+ , Option ['a'] ["alphanumeric"] (NoArg AlphaNumeric) "ignore all but A-Z and 0-9"+ , Option ['p'] ["withpunc"] (NoArg WithPunc) "use all non-whitespace chars as 'words'"+ , Option ['i'] ["ignore-case"] (NoArg CaseInsensitive) "ignore case distinctions" ] -usage = "\nUsage: wordsetdiff [OPTION...] file1 file2 [file3 ...]\n\n"+++usage = "\nVersion "++version++"\n"+++ "Usage: wordsetdiff [OPTION...] file1 file2 [file3 ...]\n\n"++ "The wordsetdiff program subtracts the contents of file2, file3, ... from file1.\n\n"++ @@ -200,6 +208,25 @@ combine_entries ls = (pack_window (List.map fst ls), Set.singleton (combine_locs (List.map snd ls))) -- Perhaps not the most efficient... ++-- | Similar to wordmapN but use lines rather than a sliding window of words.+lineMap :: (Char -> Bool) -> BC.ByteString -> TupMap (Set.Set Loc)+lineMap isWordChar bs = + M.fromListWith Set.union $ + words_only+ where + -- Here's a little trick, we get labeled lines by forming them as "big words" separated by newlines:+ lns :: [(B.ByteString, Loc)]+ lns = words_wloc (not . (== '\n')) bs++ words_only :: [(Window, Set.Set Loc)]+ words_only = List.map (\ (bs,loc) -> (pack_window$ + List.filter (not . B.null) $+ B.splitWith (not . isWordChar . w2c) bs,+ Set.singleton loc)) + lns + + --sliding_win :: Int -> [ByteString] -> [Window] sliding_win :: Int -> [(B.ByteString,Loc)] -> [[(B.ByteString,Loc)]] --sliding_win :: Int -> [a] -> [[a]]@@ -278,14 +305,14 @@ Cfg { color :: Bool , word_sequence_size :: Int , case_insensitive :: Bool- , with_punctuation :: Bool+ , is_wordchar :: Char -> Bool } default_cfg = Cfg { color = True , word_sequence_size = default_word_sequence_size , case_insensitive = False- , with_punctuation = False+ , is_wordchar = isAlpha } main = @@ -312,8 +339,10 @@ case opt of NoColor -> cfg { color = False } NWords n -> cfg { word_sequence_size = n }- WithPunc -> cfg { with_punctuation = True }- AlphaOnly -> cfg { with_punctuation = False }+ UseLines -> cfg { word_sequence_size = 0 } -- A little convention here, 0 means lines.+ WithPunc -> cfg { is_wordchar = not . isSpace }+ AlphaOnly -> cfg { is_wordchar = isAlpha }+ AlphaNumeric -> cfg { is_wordchar = isAlphaNum } CaseInsensitive -> cfg { case_insensitive = True } _bs_left <- BC.readFile left @@ -321,15 +350,18 @@ let -- This is a sloppy way to implement case insesitivity. Do it at the outset:- isWordChar = if with_punctuation cfg then not . isSpace else isAlpha+ isWordChar = is_wordchar cfg lower = if case_insensitive cfg then BC.map toLower else id bs_left = lower _bs_left bs_rights = List.map lower _bs_rights- snips_left = wordmapN isWordChar (word_sequence_size cfg) bs_left + get_snips bs = if 0 == word_sequence_size cfg+ then lineMap isWordChar bs+ else wordmapN isWordChar (word_sequence_size cfg) bs+ snips_left = get_snips bs_left sub_one_file (remain,common) bs_right = let- snips_right = wordmapN isWordChar (word_sequence_size cfg) bs_right+ snips_right = get_snips bs_right remain' = difference remain snips_right common' = intersection common snips_right in (remain',common')
wordsetdiff.cabal view
@@ -2,9 +2,10 @@ -- CHANGELOG: -- version 0.0.1: initial release -- version 0.0.2: switched to HashMap+-- version 0.0.3: Added -l and -a options. Name: wordsetdiff-Version: 0.0.2+Version: 0.0.3 License: BSD3 License-file: LICENSE Stability: Beta@@ -14,7 +15,7 @@ homepage: http://people.csail.mit.edu/newton/wordsetdiff -- Copyright: Copyright (c) 2009-2010 Ryan Newton-Synopsis: Compare two files as sets of words or sets of N-tuples of words.+Synopsis: Compare two files as sets of N-tuples of words. Description: This utility is useful for finding out if some old, misplaced version of a file (say from your old laptop)