diff --git a/cgrep.cabal b/cgrep.cabal --- a/cgrep.cabal +++ b/cgrep.cabal @@ -1,6 +1,6 @@ Name: cgrep Description: Cgrep: a context-aware grep for source codes -Version: 6.6.15 +Version: 6.6.16 Synopsis: Command line tool Homepage: http://awgn.github.io/cgrep/ License: GPL-2 @@ -48,6 +48,8 @@ mtl >= 2.0, unix-compat >= 0.4, async >= 2.0, + utf8-string, + unicode-show, transformers, process Ghc-options: -O2 -Wall -threaded -rtsopts -with-rtsopts=-N diff --git a/src/CGrep/Output.hs b/src/CGrep/Output.hs --- a/src/CGrep/Output.hs +++ b/src/CGrep/Output.hs @@ -26,7 +26,11 @@ showFile, showBold) where +import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as C +import qualified Codec.Binary.UTF8.String as UC + +import Text.Show.Unicode import System.Console.ANSI #ifdef ENABLE_HINT @@ -179,7 +183,7 @@ ("#f", showFile conf opt out), ("#n", showLineCol opt out), ("#l", showLine conf opt out), - ("#t", show ts'), + ("#t", ushow ts'), ("##", unwords ts'), ("#,", intercalate "," ts'), ("#;", intercalate ";" ts'), @@ -216,7 +220,7 @@ "; file = " ++ show (showFile opt out) ++ "; row = " ++ show n ++ "; line = " ++ show (showLine conf opt ts l) ++ - "; tokens = " ++ show (map snd ts) ++ " in " ++ + "; tokens = " ++ ushow (map snd ts) ++ " in " ++ (fromJust $ hint opt) #endif @@ -244,14 +248,15 @@ showTokens :: Options -> Output -> String showTokens Options { show_match = st } out - | st = show (map snd (outTokens out)) + | st = ushow (map snd (outTokens out)) | otherwise = "" showLine :: Config -> Options -> Output -> String showLine conf Options { color = c, no_color = c' } out - | c && not c'= hilightLine conf (sortBy (flip compare `on` (length . snd )) (outTokens out)) (C.unpack (outLine out)) - | otherwise = C.unpack (outLine out) + | c && not c'= hilightLine conf (sortBy (flip compare `on` (length . snd )) (outTokens out)) line + | otherwise = line + where line = UC.decode $ B.unpack $ outLine out showFileName :: Config -> Options -> String -> String diff --git a/src/CGrep/Strategy/BoyerMoore.hs b/src/CGrep/Strategy/BoyerMoore.hs --- a/src/CGrep/Strategy/BoyerMoore.hs +++ b/src/CGrep/Strategy/BoyerMoore.hs @@ -20,7 +20,7 @@ module CGrep.Strategy.BoyerMoore (search) where -import qualified Data.ByteString.Char8 as C +import qualified Data.ByteString.Char8 as C import Control.Monad.Trans.Reader import Control.Monad.IO.Class @@ -55,22 +55,24 @@ , ignoreCase opt ] - putStrLevel1 $ "strategy : running Boyer-Moore search on " ++ filename ++ "..." + -- make shallow search let shallow = shallowSearch patterns text''' - runSearch opt filename (all notNull shallow) $ do + -- search for matching tokens - -- search for matching tokens + let tokens = concatMap (\(p, xs) -> let p' = C.unpack p in map (,p') xs ) $ zip patterns shallow - let tokens = concatMap (\(p, xs) -> let p' = C.unpack p in map (,p') xs ) $ zip patterns shallow + -- filter exact/partial matching tokens - -- filter exact/partial matching tokens + let tokens' = if word_match opt || prefix_match opt || suffix_match opt + then filter (checkToken opt text''') tokens + else tokens - tokens' = if word_match opt || prefix_match opt || suffix_match opt - then filter (checkToken opt text''') tokens - else tokens + putStrLevel1 $ "strategy : running Boyer-Moore search on " ++ filename ++ "..." + runSearch opt filename (all notNull shallow) $ do + -- print banners... putStrLevel2 $ "tokens : " ++ show tokens @@ -85,10 +87,9 @@ | word_match opt = (off - off', str) `elem` ts | prefix_match opt = any (\(o,s) -> str `isPrefixOf` s && o + off' == off) ts | suffix_match opt = any (\(o,s) -> str `isSuffixOf` s && o + off' + (length s - length str) == off) ts + | otherwise = undefined where (text',off') = getLineByOffset off text ts = T.tokenizer text' - -checkToken _ _ (_,_) = undefined splitLines :: Text8 -> [(Text8,Offset)] diff --git a/src/CGrep/Strategy/Cpp/Semantic.hs b/src/CGrep/Strategy/Cpp/Semantic.hs --- a/src/CGrep/Strategy/Cpp/Semantic.hs +++ b/src/CGrep/Strategy/Cpp/Semantic.hs @@ -19,6 +19,7 @@ module CGrep.Strategy.Cpp.Semantic (search) where import qualified Data.ByteString.Char8 as C + import qualified CGrep.Parser.Cpp.Token as Cpp import Control.Monad.Trans.Reader @@ -50,8 +51,13 @@ -- transform text let filt = (mkContextFilter opt) { getFilterComment = False } - text' = ignoreCase opt text + + let [text''', text'' , text', _] = scanr ($) text [ expandMultiline opt + , contextFilter (getFileLang opt filename) filt + , ignoreCase opt + ] + -- pre-process patterns patterns' = map (Cpp.tokenizer . contextFilter (Just Cpp) filt) patterns -- [ [t1,t2,..], [t1,t2...] ] @@ -73,26 +79,21 @@ putStrLevel2 $ "multicards: " ++ show patterns''' putStrLevel2 $ "identif : " ++ show identif - let text'' = contextFilter (getFileLang opt filename) filt text' - idpack = map C.pack identif + let idpack = map C.pack identif quick1 = all notNull $ shallowSearch idpack text' quick2 = all notNull $ shallowSearch idpack text'' runSearch opt filename (quick1 && quick2) $ do - let [text''', _ , _] = scanr ($) text' [ expandMultiline opt - , contextFilter (getFileLang opt filename) filt - ] - -- parse source code, get the Cpp.Token list... - let tokens = Cpp.tokenizer text''' + let tokens = Cpp.tokenizer text''' -- get matching tokens ... - tokens' = sortBy (compare `on` Cpp.toOffset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns''' + let tokens' = sortBy (compare `on` Cpp.toOffset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns''' - matches = map (\t -> let n = fromIntegral (Cpp.toOffset t) in (n, Cpp.toString t)) tokens' :: [(Int, String)] + let matches = map (\t -> let n = fromIntegral (Cpp.toOffset t) in (n, Cpp.toString t)) tokens' :: [(Int, String)] putStrLevel2 $ "tokens : " ++ show tokens' putStrLevel2 $ "matches : " ++ show matches diff --git a/src/CGrep/Strategy/Cpp/Tokenizer.hs b/src/CGrep/Strategy/Cpp/Tokenizer.hs --- a/src/CGrep/Strategy/Cpp/Tokenizer.hs +++ b/src/CGrep/Strategy/Cpp/Tokenizer.hs @@ -19,6 +19,7 @@ module CGrep.Strategy.Cpp.Tokenizer (search) where import qualified Data.ByteString.Char8 as C + import qualified CGrep.Parser.Cpp.Token as Cpp import Control.Monad.Trans.Reader @@ -50,7 +51,7 @@ let filt = (mkContextFilter opt) { getFilterComment = False } - [text''', _ , text', _] = scanr ($) text [ expandMultiline opt + let [text''', _ , text', _] = scanr ($) text [ expandMultiline opt , contextFilter (getFileLang opt filename) filt , ignoreCase opt ] diff --git a/src/CGrep/Strategy/Generic/Semantic.hs b/src/CGrep/Strategy/Generic/Semantic.hs --- a/src/CGrep/Strategy/Generic/Semantic.hs +++ b/src/CGrep/Strategy/Generic/Semantic.hs @@ -19,6 +19,7 @@ module CGrep.Strategy.Generic.Semantic (search) where import qualified Data.ByteString.Char8 as C + import qualified CGrep.Parser.Generic.Token as Generic import CGrep.Filter @@ -51,7 +52,11 @@ -- transform text - let text' = ignoreCase opt text + let [text''', text'', text', _ ] = scanr ($) text [ expandMultiline opt + , contextFilter (getFileLang opt filename) filt + , ignoreCase opt + ] + filt = (mkContextFilter opt) { getFilterComment = False } -- pre-process patterns @@ -74,26 +79,20 @@ putStrLevel2 $ "multicards: " ++ show patterns'' putStrLevel2 $ "identif : " ++ show identif - let text'' = contextFilter (getFileLang opt filename) filt text' - idpack = map C.pack identif + let idpack = map C.pack identif quick1 = all notNull $ shallowSearch idpack text' quick2 = all notNull $ shallowSearch idpack text'' runSearch opt filename (quick1 && quick2) $ do - -- expand multi-line - - let text''' = expandMultiline opt text'' - -- parse source code, get the Generic.Token list... - tokens = Generic.tokenizer text''' + let tokens = Generic.tokenizer text''' -- get matching tokens ... - tokens' = sortBy (compare `on` Generic.toOffset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns'' - - matches = map (\t -> let n = fromIntegral (Generic.toOffset t) in (n, Generic.toString t)) tokens' :: [(Int, String)] + let tokens' = sortBy (compare `on` Generic.toOffset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns'' + let matches = map (\t -> let n = fromIntegral (Generic.toOffset t) in (n, Generic.toString t)) tokens' :: [(Int, String)] putStrLevel2 $ "tokens : " ++ show tokens' putStrLevel2 $ "matches : " ++ show matches diff --git a/src/CGrep/Strategy/Levenshtein.hs b/src/CGrep/Strategy/Levenshtein.hs --- a/src/CGrep/Strategy/Levenshtein.hs +++ b/src/CGrep/Strategy/Levenshtein.hs @@ -56,7 +56,6 @@ -- filter tokens... patterns' = map C.unpack patterns - matches = filter (\t -> any (\p -> p ~== snd t) patterns') tokens' putStrLevel1 $ "strategy : running edit-distance (Levenshtein) search on " ++ filename ++ "..." diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -65,7 +65,9 @@ import Config import Reader +import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as C +import qualified Codec.Binary.UTF8.String as UC fileFilter :: Options -> [Lang] -> FilePath -> Bool @@ -184,7 +186,8 @@ case () of _ | json opts -> when m $ liftIO $ putStrLn "," | otherwise -> return () - prettyOutput out >>= mapM_ (liftIO . putStrLn) + let out' = map (\p -> p {outTokens = map (\(off, s) -> (length $ UC.decode $ B.unpack $ C.take off $ outLine p, UC.decodeString s)) $ outTokens p}) out + prettyOutput out' >>= mapM_ (liftIO . putStrLn) liftIO $ when (vim opts || editor opts) $ mapM_ (modifyIORef matchingFiles . Set.insert . outFilePath) out action n True ) 0 False @@ -242,7 +245,7 @@ -- load patterns: - patterns <- if null (file opts) then return $ (if isTermIn then (:[]) . head else id) $ map C.pack (others opts) + patterns <- if null (file opts) then return $ (if isTermIn then (:[]) . head else id) $ map (C.pack . UC.encodeString) (others opts) else readPatternsFromFile $ file opts let patterns' = map (if ignore_case opts then ic else id) patterns