token-search (empty) → 0.1.0.0

raw patch · 13 files changed

+478/−0 lines, 13 filesdep +aesondep +basedep +bytestringsetup-changed

Dependencies added: aeson, base, bytestring, conduit, hashable, hspec, process, streaming-commons, text, token-search, unordered-containers

Files

• ChangeLog.md +3/−0
• LICENSE +7/−0
• README.md +63/−0
• Setup.hs +2/−0
• app/Main.hs +25/−0
• src/TokenSearch.hs +89/−0
• src/Trie.hs +66/−0
• src/Util.hs +11/−0
• src/WalkTrie.hs +53/−0
• test/Spec.hs +1/−0
• test/TokenSearch/TokenSearchSpec.hs +32/−0
• test/TokenSearch/TrieSpec.hs +34/−0
• token-search.cabal +92/−0

@@ -0,0 +1,3 @@+# Changelog for token-search++## Unreleased changes

@@ -0,0 +1,7 @@+Copyright (c) 2019 Josh Clayton++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@@ -0,0 +1,63 @@+# token-search++This is a library for efficient substring detection across a codebase.++## Motivation++[Unused] leverages [ctags]' token generation in conjunction with a tool to+search the file system (either [ripgrep] or [The Silver Searcher]).++During execution, Unused shells out from Haskell for each unique token and+searches the appropriate files for each token. This means each file searched is+searched thousands of times each run, which takes a significant amount of time.++[Unused]: https://unused.codes/+[ctags]: https://ctags.io/+[ripgrep]: https://github.com/BurntSushi/ripgrep+[The Silver Searcher]: https://geoff.greer.fm/ag/++## Approach++Instead of searching each file git tracks for each of potentially thousands of+tokens, `token-search` processes each file once.++With the tokens:++```+rem+or+lo+```++and the text:++```+lorem ipsum+dolor sit amet+```++`token-search` then:++1. Builds a trie with the three tokens+2. Iterates over each character in the text, while also:+   * creating a new copy of the trie+   * adding the created trie to a list of all non-terminated tries+   * walks each trie by the character+   * maintains a list of terminal nodes as tries are walked+   * increments a terminal node count for tokens as they're encountered++## Install++```sh+stack install+```++## Test++```sh+stack test+```++## License++Copyright 2019 Josh Clayton. See the [LICENSE](LICENSE).

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

@@ -0,0 +1,25 @@+module Main where++import Control.Monad.IO.Class (MonadIO, liftIO)+import qualified Data.Aeson as A+import qualified Data.ByteString.Lazy as BS+import qualified Data.List as L+import qualified Data.Text as T+import System.Process (readProcess)+import qualified TokenSearch++main :: MonadIO m => m ()+main = do+    tokens <- calculateTokens+    results <-+        TokenSearch.calculateResults tokens =<< TokenSearch.calculateFileNames+    liftIO $ BS.putStr $ A.encode results++calculateTokens :: MonadIO m => m [T.Text]+calculateTokens =+    tokensFromTags . T.pack <$> liftIO (readProcess "cat" [".git/tags"] [])++tokensFromTags :: T.Text -> [T.Text]+tokensFromTags = L.nub . tokenLocations+  where+    tokenLocations = map (head . T.splitOn "\t") . T.lines

@@ -0,0 +1,89 @@+module TokenSearch+    ( calculateResults+    , calculateFileNames+    ) where++import Conduit+import Control.Monad.IO.Class (MonadIO, liftIO)+import qualified Data.Bifunctor as BF+import qualified Data.ByteString as BS+import qualified Data.HashMap.Strict as Map+import Data.Hashable (Hashable)+import qualified Data.Streaming.FileRead as FR+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Text.Encoding.Error as T+import System.Process (readProcess)+import Trie+import WalkTrie++calculateFileNames :: MonadIO m => m [String]+calculateFileNames = lines <$> liftIO (readProcess "git" ["ls-files"] [])++calculateResults ::+       MonadIO m+    => [T.Text]+    -> [FilePath]+    -> m (Map.HashMap String (Map.HashMap FilePath Int))+calculateResults tokens filenames = do+    let newTrie = buildTrieWithTokens tokens+    transformMap . Map.unions <$> processAllFiles filenames newTrie++processAllFiles ::+       MonadIO m+    => [FilePath]+    -> Trie+    -> m [Map.HashMap FilePath (Map.HashMap String Int)]+processAllFiles filenames trie =+    liftIO $+    runConduitRes $+    pathAndContentsSource filenames .| processTextC trie .| sinkList++pathAndContentsSource ::+       MonadResource m => [FilePath] -> ConduitT () (FilePath, T.Text) m ()+pathAndContentsSource filenames =+    yieldMany filenames .| awaitForever sourceFileWithFilename .|+    mapC (BF.second lenientUtf8Decode)++lenientUtf8Decode :: BS.ByteString -> T.Text+lenientUtf8Decode = T.decodeUtf8With T.lenientDecode++sourceFileWithFilename ::+       MonadResource m => FilePath -> ConduitT i (FilePath, BS.ByteString) m ()+sourceFileWithFilename fp =+    bracketP (FR.openFile fp) FR.closeFile (loop BS.empty)+  where+    loop acc h = do+        bs <- liftIO $ FR.readChunk h+        if BS.null bs+            then yield (fp, acc)+            else loop (BS.append acc bs) h++processTextC ::+       Monad m+    => Trie+    -> ConduitT (FilePath, T.Text) (Map.HashMap String (Map.HashMap String Int)) m ()+processTextC trie = mapC (processTextWithFilename trie)++processTextWithFilename ::+       Trie+    -> (FilePath, T.Text)+    -> Map.HashMap FilePath (Map.HashMap String Int)+processTextWithFilename trie (filename, input) =+    Map.singleton filename $ processText trie input++transformMap ::+       (Hashable a, Hashable b, Eq a, Eq b)+    => Map.HashMap a (Map.HashMap b Int)+    -> Map.HashMap b (Map.HashMap a Int)+transformMap = Map.foldlWithKey' f Map.empty+  where+    f tokenToFilenamesAcc filename =+        Map.foldlWithKey'+            (\acc token count ->+                 Map.insertWith+                     Map.union+                     token+                     (Map.singleton filename count)+                     acc)+            tokenToFilenamesAcc

@@ -0,0 +1,66 @@+module Trie+    ( buildTrieWithTokens+    , isTerminal+    , findNodeFromTrie+    , findNodeFromChildren+    , Trie+    , Node+    ) where++import qualified Data.HashMap.Strict as Map+import qualified Data.Text as T++type NodeMap = Map.HashMap Char Node++newtype Trie =+    Root NodeMap+    deriving (Show, Eq)++data Node+    = Terminal !NodeMap+    | NonTerminal !NodeMap+    deriving (Show, Eq)++buildTrieWithTokens :: [T.Text] -> Trie+buildTrieWithTokens = foldl (flip add) buildTrie++findNodeFromTrie :: Trie -> Char -> Maybe Node+findNodeFromTrie (Root nodes) char = Map.lookup char nodes++findNodeFromChildren :: Node -> Char -> Maybe Node+findNodeFromChildren node char = Map.lookup char $ nodeChildren node++mergeSameNodes :: Node -> Node -> Node+mergeSameNodes (NonTerminal xs) (NonTerminal ys) =+    NonTerminal $ mergeNodeMaps xs ys+mergeSameNodes x y = Terminal $ mergeNodeMaps (nodeChildren x) (nodeChildren y)++mergeNodeMaps :: NodeMap -> NodeMap -> NodeMap+mergeNodeMaps = Map.unionWith mergeSameNodes++isTerminal :: Node -> Bool+isTerminal (Terminal _) = True+isTerminal (NonTerminal _) = False++nodeChildren :: Node -> NodeMap+nodeChildren (Terminal xs) = xs+nodeChildren (NonTerminal xs) = xs++addNode :: Trie -> Char -> Node -> Trie+addNode (Root nodes) char node =+    Root $ Map.insertWith mergeSameNodes char node nodes++buildTrie :: Trie+buildTrie = Root Map.empty++add :: T.Text -> Trie -> Trie+add input trie =+    case T.uncons input of+        Nothing -> trie+        Just (x, xs) -> addNode trie x $ createNode xs++createNode :: T.Text -> Node+createNode input =+    case T.uncons input of+        Nothing -> Terminal Map.empty+        Just (x, xs) -> NonTerminal $ Map.singleton x $ createNode xs

@@ -0,0 +1,11 @@+module Util+    ( groupBy+    ) where++import Control.Arrow ((&&&))+import Data.Function (on)+import qualified Data.List as L++groupBy :: Ord b => (a -> b) -> [a] -> [(b, [a])]+groupBy f =+    map (f . head &&& id) . L.groupBy ((==) `on` f) . L.sortBy (compare `on` f)

@@ -0,0 +1,53 @@+module WalkTrie+    ( processText+    , aggregateResults+    ) where++import Control.Arrow ((&&&))+import qualified Data.HashMap.Strict as Map+import qualified Data.Maybe as M+import qualified Data.Text as T+import Trie++data WalkedNode+    = Unwalked Trie+    | Walked String+             Node+    deriving (Show)++aggregateResults :: [Map.HashMap String Int] -> Map.HashMap String Int+aggregateResults = foldl1 (Map.unionWith (+))++processText :: Trie -> T.Text -> Map.HashMap String Int+processText trie = snd . T.foldl f ([], Map.empty)+  where+    newTrie char =+        case findNodeFromTrie trie char of+            Nothing -> []+            Just _ -> [Unwalked trie]+    f (state, map') char = advanceStates char map' $ newTrie char ++ state++advanceStates ::+       Char+    -> Map.HashMap String Int+    -> [WalkedNode]+    -> ([WalkedNode], Map.HashMap String Int)+advanceStates char map' =+    (id &&& foldl newMap map' . concatMap walkedTerminalResult) .+    M.mapMaybe (walk char)+  where+    newMap m word = Map.insertWith (+) word 1 m++walk :: Char -> WalkedNode -> Maybe WalkedNode+walk char (Unwalked trie) =+    case findNodeFromTrie trie char of+        Nothing -> Nothing+        Just node' -> Just $ Walked [char] node'+walk char (Walked string node) =+    case findNodeFromChildren node char of+        Nothing -> Nothing+        Just node' -> Just $ Walked (string ++ [char]) node'++walkedTerminalResult :: WalkedNode -> [String]+walkedTerminalResult (Walked base node) = [base | isTerminal node]+walkedTerminalResult _ = []

@@ -0,0 +1,1 @@+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}

@@ -0,0 +1,32 @@+module TokenSearch.TokenSearchSpec where++import qualified Data.HashMap.Strict as Map+import qualified Data.List as L+import Test.Hspec+import TokenSearch++main :: IO ()+main = hspec spec++spec :: Spec+spec =+    describe "TokenSearch.calculateResults" $+    it "calculates the correct counts" $ do+        let tokens = ["Person", "name", "age", "Place", "latitude", "longitude"]+        let paths =+                [ "test/data/long_file.rb"+                , "test/data/place.rb"+                , "test/data/person.rb"+                , "test/data/person_spec.rb"+                ]+        results <- calculateResults tokens paths+        L.sort . Map.toList <$>+            Map.lookup "Person" results `shouldBe`+            Just+                [ ("test/data/long_file.rb", 3)+                , ("test/data/person.rb", 1)+                , ("test/data/person_spec.rb", 3)+                ]+        Map.toList <$>+            Map.lookup "Place" results `shouldBe`+            Just [("test/data/place.rb", 1)]

@@ -0,0 +1,34 @@+module TokenSearch.TrieSpec where++import qualified Data.HashMap.Strict as Map+import Test.Hspec+import Trie+import WalkTrie++main :: IO ()+main = hspec spec++spec :: Spec+spec =+    describe "WalkTrie" $ do+        it "allows for processing text" $ do+            let trie = buildTrieWithTokens ["foo", "foobar", "Bar"]+            let outcome = processText trie "foobarfooBarbar"+            Map.lookup "foo" outcome `shouldBe` Just 2+            Map.lookup "foobar" outcome `shouldBe` Just 1+            Map.lookup "Bar" outcome `shouldBe` Just 1+            Map.lookup "non-token" outcome `shouldBe` Nothing+        it "supports insertion of shorter words later" $ do+            let trie = buildTrieWithTokens ["foobar", "foo"]+            let outcome = processText trie "foobar"+            Map.lookup "foo" outcome `shouldBe` Just 1+            Map.lookup "foobar" outcome `shouldBe` Just 1+        it "allows for aggregating results" $ do+            let trie = buildTrieWithTokens ["foo", "foobar", "Bar"]+            let outcome = processText trie "foobarfooBarbar"+            let otherOutcome = processText trie "BarBarBar foob"+            let combinedOutcome = aggregateResults [outcome, otherOutcome]+            Map.lookup "foo" combinedOutcome `shouldBe` Just 3+            Map.lookup "foobar" combinedOutcome `shouldBe` Just 1+            Map.lookup "Bar" combinedOutcome `shouldBe` Just 4+            Map.lookup "non-token" combinedOutcome `shouldBe` Nothing

@@ -0,0 +1,92 @@+cabal-version: 1.12++-- This file has been generated from package.yaml by hpack version 0.31.2.+--+-- see: https://github.com/sol/hpack+--+-- hash: c96df379ecbb0aa72fc51d10f9dd0bb8386a9f126987418d04a4850dde4d79b9++name:           token-search+version:        0.1.0.0+description:    Please see the README on GitHub at <https://github.com/joshuaclayton/token-search#readme>+homepage:       https://github.com/joshuaclayton/token-search#readme+bug-reports:    https://github.com/joshuaclayton/token-search/issues+author:         Josh Clayton+maintainer:     sayhi@joshuaclayton.me+copyright:      2019 Josh Clayton+license:        MIT+license-file:   LICENSE+build-type:     Simple+extra-source-files:+    README.md+    ChangeLog.md++source-repository head+  type: git+  location: https://github.com/joshuaclayton/token-search++library+  exposed-modules:+      TokenSearch+      Trie+      Util+      WalkTrie+  other-modules:+      Paths_token_search+  hs-source-dirs:+      src+  build-depends:+      base >=4.7 && <5+    , bytestring >0.10.8 && <1+    , conduit+    , hashable+    , process >=1.6 && <2+    , streaming-commons+    , text >=1.2.3+    , unordered-containers+  default-language: Haskell2010++executable token-search+  main-is: Main.hs+  other-modules:+      Paths_token_search+  hs-source-dirs:+      app+  default-extensions: OverloadedStrings+  ghc-options: -threaded -rtsopts -with-rtsopts=-N+  build-depends:+      aeson >=1.4.5 && <2+    , base >=4.7 && <5+    , bytestring >0.10.8 && <1+    , conduit+    , hashable+    , process+    , streaming-commons+    , text+    , token-search+    , unordered-containers+  default-language: Haskell2010++test-suite token-search-test+  type: exitcode-stdio-1.0+  main-is: Spec.hs+  other-modules:+      TokenSearch.TokenSearchSpec+      TokenSearch.TrieSpec+      Paths_token_search+  hs-source-dirs:+      test+  default-extensions: OverloadedStrings+  ghc-options: -threaded -rtsopts -with-rtsopts=-N+  build-depends:+      base >=4.7 && <5+    , bytestring >0.10.8 && <1+    , conduit+    , hashable+    , hspec+    , process >=1.6 && <2+    , streaming-commons+    , text >=1.2.3+    , token-search+    , unordered-containers+  default-language: Haskell2010