diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ After that, `takedouble ` so you could use `takedouble ~/` for example. +If there are common files you'd like to exclude (such as .git directories) you can pass a glob to exclude any matching patterns from the output. + +For example + +`takedouble "**/.git/**"` + # Is it Fast? On my ThinkPad with six Xeon cores, 128GB RAM, and a 1TB Samsung 970 Pro NVMe (via PCIe 3.0), I can check 34393 uncached files in 6.4 seconds. diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -1,23 +1,42 @@ module Main where import Data.List (sort) -import System.Environment (getArgs, getProgName) -import Takedouble (File (..), checkFullDuplicates, findPossibleDuplicates, getFileNames) -import Text.Printf (printf) +import System.Environment (getArgs) +import Takedouble + ( File (filepath), + checkFullDuplicates, + findPossibleDuplicates, + getFileNames, + ) +fetchResults :: [FilePath] -> Maybe String -> IO () +fetchResults filenames glob = do + putStrLn $ "comparing " <> show (length filenames) <> " files" + likelyDups <- findPossibleDuplicates filenames glob -- each element in the list is a list of files that are likely duplicates + let fnamesOnly = (filepath <$>) `fmap` likelyDups -- convert to a nested list of FilePath + dups <- mapM checkFullDuplicates fnamesOnly + mapM_ (\x -> putStr "\n" >> mapM_ putStrLn x) (sort $ concat dups) + main :: IO () main = do args <- getArgs case args of + ["help"] -> do + putStrLn "Usage: takedouble DIR [GLOB]" + putStrLn "Description:" + putStrLn "\tDIR - starting search directory" + putStrLn "\tGLOB - optional glob from filepattern library to exclude from results." + putStrLn "" + putStrLn "Examples:" + putStrLn "\ttakedouble '~/code/' '**/.git/**' # This will exclude .git paths from results." [dir] -> do putStrLn $ "reading " <> dir filenames <- getFileNames dir - putStrLn $ "comparing " <> show (length filenames) <> " files" - likelyDups <- findPossibleDuplicates filenames -- each element in the list is a list of files that are likely duplicates - let fnamesOnly = (filepath <$>) `fmap` likelyDups -- convert to a nested list of FilePath - dups <- mapM checkFullDuplicates fnamesOnly - mapM_ (\x -> putStr "\n" >> mapM_ putStrLn x) (sort $ concat dups) + fetchResults filenames Nothing + [dir, glob] -> do + putStrLn $ "reading " <> dir + filenames <- getFileNames dir + fetchResults filenames $ Just glob _ -> do - name <- getProgName - printf "Something went wrong - please use ./%s \n" name + putStrLn "Something went wrong - See 'takedouble help' for usage." diff --git a/src/Takedouble.hs b/src/Takedouble.hs --- a/src/Takedouble.hs +++ b/src/Takedouble.hs @@ -5,6 +5,7 @@ import Data.List (group, sort, sortOn) import Data.List.Extra (groupOn) import System.Directory (doesDirectoryExist, doesPathExist, listDirectory) +import System.FilePattern import System.FilePath.Posix (()) import System.IO ( Handle, @@ -40,10 +41,14 @@ show (File fp _ _ _) = show fp -- | (hopefully) lazy comparison of files by size, first, and last chunk. -findPossibleDuplicates :: [FilePath] -> IO [[File]] -findPossibleDuplicates filenames = do - files <- mapM loadFile filenames +findPossibleDuplicates :: [FilePath] -> Maybe String -> IO [[File]] +findPossibleDuplicates filenames glob = do + files <- mapM loadFile filteredFilenames pure $ filter (\x -> 1 < length x) $ group (sort files) + where filteredFilenames = + case glob of + Nothing -> filenames + Just g -> filter (\fn -> not (g ?== fn)) filenames checkFullDuplicates :: [FilePath] -> IO [[FilePath]] checkFullDuplicates fps = do diff --git a/takedouble.cabal b/takedouble.cabal --- a/takedouble.cabal +++ b/takedouble.cabal @@ -1,7 +1,7 @@ name: takedouble synopsis: duplicate file finder description: takedouble is a fast duplicate file finder that filters by file size, first and last 4k chunks before checking the full contents of files that pass the filter. -version: 0.0.1.1 +version: 0.0.2.0 homepage: https://github.com/shapr/takedouble license: BSD3 license-file: LICENSE @@ -22,19 +22,20 @@ hs-source-dirs: src default-language: Haskell2010 exposed-modules: Takedouble - ghc-options: -Wall -fno-warn-name-shadowing -O2 + ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-name-shadowing -O2 build-depends: base >= 4.11 && < 5 , bytestring , directory , extra , filepath , unix + , filepattern executable takedouble main-is: Main.hs hs-source-dirs: app default-language: Haskell2010 - ghc-options: -threaded -O2 -rtsopts "-with-rtsopts=-N -qg" + ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -threaded -O2 -rtsopts "-with-rtsopts=-N -qg" build-depends: base , takedouble @@ -43,7 +44,7 @@ hs-source-dirs: test main-is: Main.hs default-language: Haskell2010 - ghc-options: -Wall -threaded + ghc-options: -Wall -threaded -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints build-depends: base >=4.11 && < 5 , directory , extra diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -3,11 +3,9 @@ module Main where -import Control.Monad import Control.Monad.IO.Class import Hedgehog import Hedgehog.Main --- import System.Directory import System.FilePath.Posix (()) import System.IO.Temp import Takedouble @@ -19,23 +17,35 @@ prop_FileEq_no_check_filename = property $ do File "a" 1024 "abc" "def" === File "b" 1024 "abc" "def" --- prop_Find_Duplicates :: Property --- prop_Find_Duplicates = property $ --- do --- withTempDirectory "/tmp" "foo" $ do --- \dir -> do --- _ <- genTempFiles dir --- files <- findPossibleDuplicates [dir] --- if 1 < length files then pure () else error "no" +prop_Find_Duplicates :: Property +prop_Find_Duplicates = property $ + do + files <- liftIO $ + withTempDirectory "/tmp" "foo" $ + \dir -> do + _ <- genTempFiles dir + filenames <- getFileNames dir + findPossibleDuplicates filenames Nothing + all allTheSame files === True --- dir <- liftIO $ genTempFiles --- files <- liftIO $ findPossibleDuplicates [dir] --- True === (1 < length files) +prop_Glob :: Property +prop_Glob = property $ + do + files <- liftIO $ + withTempDirectory "/tmp" "foo" $ + \dir -> do + _ <- genTempFiles dir + filenames <- getFileNames dir + findPossibleDuplicates filenames $ Just "**" + length files === 0 genTempFiles :: FilePath -> IO FilePath genTempFiles dir = do - -- dir <- getTemporaryDirectory writeFile (dir "dup1") "duplicate file" writeFile (dir "dup2") "duplicate file" writeFile (dir "uniq1") "uniq file" pure dir + +allTheSame :: (Eq a) => [a] -> Bool +allTheSame [] = False -- For the duplicate finder we want this case to fail. Semantically this makes no sense. +allTheSame xs = all (== head xs) (tail xs)