diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Usage ----- -Cgrep 6.4.17 Usage: cgrep [OPTION] [PATTERN] files... +Cgrep 6.4.18 Usage: cgrep [OPTION] [PATTERN] files... cgrep [OPTIONS] [ITEM] diff --git a/cgrep.cabal b/cgrep.cabal --- a/cgrep.cabal +++ b/cgrep.cabal @@ -1,6 +1,6 @@ Name: cgrep Description: Cgrep: a context-aware grep for source codes -Version: 6.4.17 +Version: 6.4.18 Synopsis: Command line tool Homepage: http://awgn.github.io/cgrep/ License: GPL-2 diff --git a/src/CGrep/Semantic/Cpp/Token.hs b/src/CGrep/Semantic/Cpp/Token.hs --- a/src/CGrep/Semantic/Cpp/Token.hs +++ b/src/CGrep/Semantic/Cpp/Token.hs @@ -33,6 +33,7 @@ import qualified Data.HashMap.Strict as HM import qualified Data.ByteString.Char8 as C +import CGrep.Semantic.Token type TokenizerState = (Source, Offset, CppState) @@ -41,14 +42,14 @@ type Offset = Int -data Token = TokenIdentifier { toString :: String, offset :: Int } | - TokenDirective { toString :: String, offset :: Int } | - TokenKeyword { toString :: String, offset :: Int } | - TokenNumber { toString :: String, offset :: Int } | - TokenHeaderName { toString :: String, offset :: Int } | - TokenString { toString :: String, offset :: Int } | - TokenChar { toString :: String, offset :: Int } | - TokenOperOrPunct { toString :: String, offset :: Int } +data Token = TokenIdentifier { toString :: String, toOffset :: Int } | + TokenDirective { toString :: String, toOffset :: Int } | + TokenKeyword { toString :: String, toOffset :: Int } | + TokenNumber { toString :: String, toOffset :: Int } | + TokenHeaderName { toString :: String, toOffset :: Int } | + TokenString { toString :: String, toOffset :: Int } | + TokenChar { toString :: String, toOffset :: Int } | + TokenOperOrPunct { toString :: String, toOffset :: Int } deriving (Show, Eq, Ord) @@ -65,6 +66,19 @@ } deriving (Show,Read,Eq) + +instance SemanticToken Token where + tkIsIdentifier = isIdentifier + tkIsString = isString + tkIsChar = isChar + tkIsNumber = isLiteralNumber + tkIsKeyword = isKeyword + tkEquivalent = tokenCompare + tkToString = toString + tkToOffset = toOffset + tkToIdentif = TokenIdentifier + + -- Tokenize the source code in a list of Token -- Precondition: the C++ source code must be well-formed -- @@ -206,7 +220,7 @@ len = fromIntegral $ length tstring (xs', w) = dropWhite $ C.drop (fromIntegral len) xs in - (token { offset = off }, (xs', off + len + w, nextCppState tstring state)) + (token { toOffset = off }, (xs', off + len + w, nextCppState tstring state)) getTokenIdOrKeyword, getTokenNumber, diff --git a/src/CGrep/Semantic/Generic/Token.hs b/src/CGrep/Semantic/Generic/Token.hs --- a/src/CGrep/Semantic/Generic/Token.hs +++ b/src/CGrep/Semantic/Generic/Token.hs @@ -43,13 +43,26 @@ deriving (Eq, Enum, Show) -data Token = TokenAlpha { toString :: String, offset :: Offset } | - TokenDigit { toString :: String, offset :: Offset } | - TokenBracket { toString :: String, offset :: Offset } | - TokenLiteral { toString :: String, offset :: Offset } | - TokenOther { toString :: String, offset :: Offset } +data Token = TokenAlpha { toString :: String, toOffset :: Offset } | + TokenDigit { toString :: String, toOffset :: Offset } | + TokenBracket { toString :: String, toOffset :: Offset } | + TokenLiteral { toString :: String, toOffset :: Offset } | + TokenOther { toString :: String, toOffset :: Offset } deriving (Show, Eq, Ord) + +instance SemanticToken Token where + tkIsIdentifier = _isTokenAlpha + tkIsString = _isTokenLiteral + tkIsChar = _isTokenLiteral + tkIsNumber = _isTokenDigit + tkIsKeyword = const False + tkEquivalent = tokenCompare + tkToString = toString + tkToOffset = toOffset + tkToIdentif = TokenAlpha + + _isTokenAlpha, _isTokenDigit, _isTokenBracket, _isTokenOther, _isTokenLiteral :: Token -> Bool _isTokenAlpha (TokenAlpha _ _) = True @@ -195,14 +208,4 @@ | x == '\'' -> TokenAccum StateLit1 (off+1) 0 (DL.singleton x) (out `DL.snoc` mkToken TokenBracket off acc) | x == '"' -> TokenAccum StateLit2 (off+1) 0 (DL.singleton x) (out `DL.snoc` mkToken TokenBracket off acc) | otherwise -> TokenAccum StateOther (off+1) 0 (acc `DL.snoc` x) out - - -instance SemanticToken Token where - tkIsIdentifier = _isTokenAlpha - tkIsString = _isTokenLiteral - tkIsChar = _isTokenLiteral - tkIsNumber = _isTokenDigit - tkIsKeyword = const False - tkToString = toString - tkEquivalent = tokenCompare diff --git a/src/CGrep/Semantic/Token.hs b/src/CGrep/Semantic/Token.hs --- a/src/CGrep/Semantic/Token.hs +++ b/src/CGrep/Semantic/Token.hs @@ -25,7 +25,8 @@ tkIsChar :: t -> Bool tkIsNumber :: t -> Bool tkIsKeyword :: t -> Bool - tkToString :: t -> String tkEquivalent :: t -> t -> Bool - + tkToString :: t -> String + tkToOffset :: t -> Int + tkToIdentif :: String -> Int -> t diff --git a/src/CGrep/Semantic/WildCard.hs b/src/CGrep/Semantic/WildCard.hs --- a/src/CGrep/Semantic/WildCard.hs +++ b/src/CGrep/Semantic/WildCard.hs @@ -17,10 +17,12 @@ -- module CGrep.Semantic.WildCard (WildCard(..), MultiCard, + wildCardMap, + mkWildCardFromToken, + combineMultiCard, filterTokensWithMultiCards, wildCardMatch, isWildCardPattern, - rmWildCardEscape, multiCardMatch) where import qualified Data.Map as M @@ -46,9 +48,43 @@ IdentifCard String deriving (Show, Eq, Ord) + type MultiCard a = [WildCard a] +wildCardMap :: M.Map String (WildCard a) +wildCardMap = M.fromList + [ + ("ANY", AnyCard ), + ("KEY", KeyWordCard ), + ("OCT", OctCard ), + ("HEX", HexCard ), + ("NUM", NumberCard ), + ("CHR", CharCard ), + ("STR", StringCard ), + ("LIT", StringCard ) + ] + + +mkWildCardFromToken :: (SemanticToken a) => a -> WildCard a +mkWildCardFromToken t + | tkIsIdentifier t = case () of + _ | Just wc <- M.lookup str wildCardMap -> wc + | isWildCardPattern str -> IdentifCard str + | otherwise -> TokenCard $ tkToIdentif (rmWildCardEscape str) (tkToOffset t) + where str = tkToString t + | otherwise = TokenCard t + + +combineMultiCard :: (SemanticToken a) => [MultiCard a] -> [MultiCard a] +combineMultiCard (m1:r@(m2:m3:ms)) + | [TokenCard b] <- m2, tkToString b == "OR" = combineMultiCard $ (m1++m3):ms + | otherwise = m1 : combineMultiCard r +combineMultiCard [m1,m2] = [m1,m2] +combineMultiCard [m1] = [m1] +combineMultiCard [] = [] + + filterTokensWithMultiCards :: (SemanticToken a) => Options -> [MultiCard a] -> [a] -> [a] filterTokensWithMultiCards opt ws = filterTokensWithMultiCards' opt (spanOptionalCards ws) @@ -84,8 +120,8 @@ isWildCardPattern :: String -> Bool isWildCardPattern s = case () of - _ | (x:y:_) <- s -> wprefix x && not (wprefix y) - | [x] <- s -> wprefix x + _ | (x:y:_) <- s -> wprefix x && not (wprefix y) + | [x] <- s -> wprefix x | otherwise -> error "isWildCardPattern" where wprefix x = x == '$' || x == '_' diff --git a/src/CGrep/Strategy/Cpp/Semantic.hs b/src/CGrep/Strategy/Cpp/Semantic.hs --- a/src/CGrep/Strategy/Cpp/Semantic.hs +++ b/src/CGrep/Strategy/Cpp/Semantic.hs @@ -19,7 +19,6 @@ module CGrep.Strategy.Cpp.Semantic (search) where import qualified Data.ByteString.Char8 as C - import qualified CGrep.Semantic.Cpp.Token as Cpp import CGrep.Filter @@ -27,11 +26,8 @@ import CGrep.Common import CGrep.Output -import CGrep.Semantic.Token import CGrep.Semantic.WildCard -import qualified Data.Map as M - import Data.List import Data.Function import Data.Maybe @@ -39,6 +35,7 @@ import Options import Debug + search :: CgrepFunction search opt ps f = do @@ -55,7 +52,7 @@ -- pre-process patterns patterns = map (Cpp.tokenizer . contextFilter (Just Cpp) filt) ps -- [ [t1,t2,..], [t1,t2...] ] - patterns' = map (map mkWildCard) patterns -- [ [w1,w2,..], [w1,w2,..] ] + patterns' = map (map mkWildCardFromToken) patterns -- [ [w1,w2,..], [w1,w2,..] ] patterns'' = map (combineMultiCard . map (:[])) patterns' -- [ [m1,m2,..], [m1,m2,..] ] == [ [ [w1], [w2],..], [[w1],[w2],..]] -- quick Search... @@ -94,56 +91,13 @@ -- get matching tokens ... - tokens' = sortBy (compare `on` Cpp.offset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns'' + tokens' = sortBy (compare `on` Cpp.toOffset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns'' - matches = map (\t -> let n = fromIntegral (Cpp.offset t) in (n, Cpp.toString t)) tokens' :: [(Int, String)] + matches = map (\t -> let n = fromIntegral (Cpp.toOffset t) in (n, Cpp.toString t)) tokens' :: [(Int, String)] putStrLevel2 (debug opt) $ "tokens : " ++ show tokens' putStrLevel2 (debug opt) $ "matches : " ++ show matches putStrLevel3 (debug opt) $ "---\n" ++ C.unpack text''' ++ "\n---" return $ mkOutput opt filename text text''' matches - - -instance SemanticToken Cpp.Token where - tkIsIdentifier = Cpp.isIdentifier - tkIsString = Cpp.isString - tkIsChar = Cpp.isChar - tkIsNumber = Cpp.isLiteralNumber - tkIsKeyword = Cpp.isKeyword - tkToString = Cpp.toString - tkEquivalent = Cpp.tokenCompare - - -wildCardMap :: M.Map String (WildCard a) -wildCardMap = M.fromList - [ - ("ANY", AnyCard ), - ("KEY", KeyWordCard ), - ("OCT", OctCard ), - ("HEX", HexCard ), - ("NUM", NumberCard ), - ("CHR", CharCard ), - ("STR", StringCard ), - ("LIT", LiteralCard ) - ] - -mkWildCard :: Cpp.Token -> WildCard Cpp.Token -mkWildCard t@(Cpp.TokenIdentifier s off) = - case () of - _ | Just wc <- M.lookup str wildCardMap -> wc - | isWildCardPattern s -> IdentifCard str - | otherwise -> TokenCard $ Cpp.TokenIdentifier (rmWildCardEscape s) off - where str = tkToString t -mkWildCard t = TokenCard t - - -combineMultiCard :: [MultiCard Cpp.Token] -> [MultiCard Cpp.Token] -combineMultiCard (m1:r@(m2:m3:ms)) - | [TokenCard (Cpp.TokenIdentifier {Cpp.toString = "OR"})] <- m2 = combineMultiCard $ (m1++m3):ms - | otherwise = m1 : combineMultiCard r -combineMultiCard [m1,m2] = [m1,m2] -combineMultiCard [m1] = [m1] -combineMultiCard [] = [] - diff --git a/src/CGrep/Strategy/Cpp/Tokenizer.hs b/src/CGrep/Strategy/Cpp/Tokenizer.hs --- a/src/CGrep/Strategy/Cpp/Tokenizer.hs +++ b/src/CGrep/Strategy/Cpp/Tokenizer.hs @@ -84,7 +84,7 @@ -- convert Cpp.Tokens to CGrep.Tokens - matches = map (\t -> let off = fromIntegral (Cpp.offset t) in (off, Cpp.toString t)) tokens'' :: [(Int, String)] + matches = map (\t -> let off = fromIntegral (Cpp.toOffset t) in (off, Cpp.toString t)) tokens'' :: [(Int, String)] putStrLevel2 (debug opt) $ "tokens : " ++ show tokens putStrLevel2 (debug opt) $ "tokens' : " ++ show tokens' diff --git a/src/CGrep/Strategy/Generic/Semantic.hs b/src/CGrep/Strategy/Generic/Semantic.hs --- a/src/CGrep/Strategy/Generic/Semantic.hs +++ b/src/CGrep/Strategy/Generic/Semantic.hs @@ -19,6 +19,7 @@ module CGrep.Strategy.Generic.Semantic (search) where import qualified Data.ByteString.Char8 as C +import qualified CGrep.Semantic.Generic.Token as Generic import CGrep.Filter import CGrep.Lang @@ -28,8 +29,6 @@ import CGrep.Semantic.Token import CGrep.Semantic.WildCard -import qualified Data.Map as M - import Data.List import Data.Function import Data.Maybe @@ -37,7 +36,6 @@ import Options import Debug -import qualified CGrep.Semantic.Generic.Token as Generic search :: CgrepFunction search opt ps f = do @@ -55,7 +53,7 @@ -- pre-process patterns patterns = map (Generic.tokenizer . contextFilter (getLang opt filename) filt) ps -- [ [t1,t2,..], [t1,t2...] ] - patterns' = map (map mkWildCard) patterns -- [ [w1,w2,..], [w1,w2,..] ] + patterns' = map (map mkWildCardFromToken) patterns -- [ [w1,w2,..], [w1,w2,..] ] patterns'' = map (combineMultiCard . map (:[])) patterns' -- [ [m1,m2,..], [m1,m2,..] ] == [ [ [w1], [w2],..], [[w1],[w2],..]] -- quickSearch ... @@ -93,48 +91,13 @@ -- get matching tokens ... - tokens' = sortBy (compare `on` Generic.offset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns'' + tokens' = sortBy (compare `on` Generic.toOffset) $ nub $ concatMap (\ms -> filterTokensWithMultiCards opt ms tokens) patterns'' - matches = map (\t -> let n = fromIntegral (Generic.offset t) in (n, Generic.toString t)) tokens' :: [(Int, String)] + matches = map (\t -> let n = fromIntegral (Generic.toOffset t) in (n, Generic.toString t)) tokens' :: [(Int, String)] putStrLevel2 (debug opt) $ "tokens : " ++ show tokens' putStrLevel2 (debug opt) $ "matches : " ++ show matches - putStrLevel3 (debug opt) $ "---\n" ++ C.unpack text''' ++ "\n---" return $ mkOutput opt filename text text''' matches - - -wildCardMap :: M.Map String (WildCard a) -wildCardMap = M.fromList - [ - ("ANY", AnyCard ), - ("KEY", KeyWordCard ), - ("OCT", OctCard ), - ("HEX", HexCard ), - ("NUM", NumberCard ), - ("CHR", CharCard ), - ("STR", StringCard ), - ("LIT", StringCard ) - ] - - -mkWildCard :: Generic.Token -> WildCard Generic.Token -mkWildCard t@(Generic.TokenAlpha s off) = - case () of - _ | Just wc <- M.lookup str wildCardMap -> wc - | isWildCardPattern s -> IdentifCard str - | otherwise -> TokenCard $ Generic.TokenAlpha (rmWildCardEscape s) off - where str = tkToString t -mkWildCard t = TokenCard t - - -combineMultiCard :: [MultiCard Generic.Token] -> [MultiCard Generic.Token] -combineMultiCard (m1:r@(m2:m3:ms)) - | [TokenCard (Generic.TokenAlpha {Generic.toString = "OR"})] <- m2 = combineMultiCard $ (m1++m3):ms - | otherwise = m1 : combineMultiCard r -combineMultiCard [m1,m2] = [m1,m2] -combineMultiCard [m1] = [m1] -combineMultiCard [] = [] - diff --git a/src/Config.hs b/src/Config.hs --- a/src/Config.hs +++ b/src/Config.hs @@ -31,7 +31,7 @@ cgreprc = "cgreprc" version :: String -version = "6.4.17" +version = "6.4.18" data Config = Config