takedouble 0.0.1.1 → 0.0.2.0

raw patch · 5 files changed

+72/−31 lines, 5 filesdep +filepatternPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: filepattern

API changes (from Hackage documentation)

Files

• README.md +6/−0
• app/Main.hs +29/−10
• src/Takedouble.hs +8/−3
• takedouble.cabal +5/−4
• test/Main.hs +24/−14

@@ -8,6 +8,12 @@  After that, `takedouble <dirname>` so you could use `takedouble ~/` for example. +If there are common files you'd like to exclude (such as .git directories) you can pass a glob to exclude any matching patterns from the output.++For example++`takedouble <dirname> "**/.git/**"`+ # Is it Fast?  On my ThinkPad with six Xeon cores, 128GB RAM, and a 1TB Samsung 970 Pro NVMe (via PCIe 3.0), I can check 34393 uncached files in 6.4 seconds.

@@ -1,23 +1,42 @@ module Main where  import Data.List (sort)-import System.Environment (getArgs, getProgName)-import Takedouble (File (..), checkFullDuplicates, findPossibleDuplicates, getFileNames)-import Text.Printf (printf)+import System.Environment (getArgs)+import Takedouble+  ( File (filepath),+    checkFullDuplicates,+    findPossibleDuplicates,+    getFileNames,+  ) +fetchResults :: [FilePath] -> Maybe String -> IO ()+fetchResults filenames glob = do+  putStrLn $ "comparing " <> show (length filenames) <> " files"+  likelyDups <- findPossibleDuplicates filenames glob -- each element in the list is a list of files that are likely duplicates+  let fnamesOnly = (filepath <$>) `fmap` likelyDups -- convert to a nested list of FilePath+  dups <- mapM checkFullDuplicates fnamesOnly+  mapM_ (\x -> putStr "\n" >> mapM_ putStrLn x) (sort $ concat dups)+ main :: IO () main = do   args <- getArgs   case args of+    ["help"] -> do+      putStrLn "Usage: takedouble DIR [GLOB]"+      putStrLn "Description:"+      putStrLn "\tDIR  - starting search directory"+      putStrLn "\tGLOB - optional glob from filepattern library to exclude from results."+      putStrLn ""+      putStrLn "Examples:"+      putStrLn "\ttakedouble '~/code/' '**/.git/**' # This will exclude .git paths from results."     [dir] ->       do         putStrLn $ "reading " <> dir         filenames <- getFileNames dir-        putStrLn $ "comparing " <> show (length filenames) <> " files"-        likelyDups <- findPossibleDuplicates filenames -- each element in the list is a list of files that are likely duplicates-        let fnamesOnly = (filepath <$>) `fmap` likelyDups -- convert to a nested list of FilePath-        dups <- mapM checkFullDuplicates fnamesOnly-        mapM_ (\x -> putStr "\n" >> mapM_ putStrLn x) (sort $ concat dups)+        fetchResults filenames Nothing+    [dir, glob] -> do+      putStrLn $ "reading " <> dir+      filenames <- getFileNames dir+      fetchResults filenames $ Just glob     _ -> do-      name <- getProgName-      printf "Something went wrong - please use ./%s <dir>\n" name+      putStrLn "Something went wrong - See 'takedouble help' for usage."

@@ -5,6 +5,7 @@ import Data.List (group, sort, sortOn) import Data.List.Extra (groupOn) import System.Directory (doesDirectoryExist, doesPathExist, listDirectory)+import System.FilePattern import System.FilePath.Posix ((</>)) import System.IO   ( Handle,@@ -40,10 +41,14 @@   show (File fp _ _ _) = show fp  -- | (hopefully) lazy comparison of files by size, first, and last chunk.-findPossibleDuplicates :: [FilePath] -> IO [[File]]-findPossibleDuplicates filenames = do-  files <- mapM loadFile filenames+findPossibleDuplicates :: [FilePath] -> Maybe String -> IO [[File]]+findPossibleDuplicates filenames glob = do+  files <- mapM loadFile filteredFilenames   pure $ filter (\x -> 1 < length x) $ group (sort files)+  where filteredFilenames =+          case glob of+            Nothing -> filenames+            Just g -> filter (\fn -> not (g ?== fn)) filenames  checkFullDuplicates :: [FilePath] -> IO [[FilePath]] checkFullDuplicates fps = do

@@ -1,7 +1,7 @@ name:                takedouble synopsis:            duplicate file finder description:         takedouble is a fast duplicate file finder that filters by file size, first and last 4k chunks before checking the full contents of files that pass the filter.-version:             0.0.1.1+version:             0.0.2.0 homepage:            https://github.com/shapr/takedouble license:             BSD3 license-file:        LICENSE@@ -22,19 +22,20 @@   hs-source-dirs:      src   default-language:    Haskell2010   exposed-modules:     Takedouble-  ghc-options:         -Wall -fno-warn-name-shadowing -O2+  ghc-options:         -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-name-shadowing -O2   build-depends:       base >= 4.11 && < 5                      , bytestring                      , directory                      , extra                      , filepath                      , unix+                     , filepattern  executable takedouble   main-is:            Main.hs   hs-source-dirs:     app   default-language:   Haskell2010-  ghc-options:        -threaded -O2 -rtsopts "-with-rtsopts=-N -qg"+  ghc-options:        -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -threaded -O2 -rtsopts "-with-rtsopts=-N -qg"   build-depends:      base                     , takedouble @@ -43,7 +44,7 @@   hs-source-dirs:   test   main-is:          Main.hs   default-language: Haskell2010-  ghc-options:      -Wall -threaded+  ghc-options:      -Wall -threaded -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints   build-depends:    base >=4.11 && < 5                   , directory                   , extra

@@ -3,11 +3,9 @@  module Main where -import Control.Monad import Control.Monad.IO.Class import Hedgehog import Hedgehog.Main--- import System.Directory import System.FilePath.Posix ((</>)) import System.IO.Temp import Takedouble@@ -19,23 +17,35 @@ prop_FileEq_no_check_filename = property $ do   File "a" 1024 "abc" "def" === File "b" 1024 "abc" "def" --- prop_Find_Duplicates :: Property--- prop_Find_Duplicates = property $---   do---     withTempDirectory "/tmp" "foo" $ do---       \dir -> do---         _ <- genTempFiles dir---         files <- findPossibleDuplicates [dir]---         if 1 < length files then pure () else error "no"+prop_Find_Duplicates :: Property+prop_Find_Duplicates = property $+  do+    files <- liftIO $+      withTempDirectory "/tmp" "foo" $+        \dir -> do+          _ <- genTempFiles dir+          filenames <- getFileNames dir+          findPossibleDuplicates filenames Nothing+    all allTheSame files === True --- dir <- liftIO $ genTempFiles--- files <- liftIO $ findPossibleDuplicates [dir]--- True === (1 < length files)+prop_Glob :: Property+prop_Glob = property $+  do+    files <- liftIO $+      withTempDirectory "/tmp" "foo" $+        \dir -> do+          _ <- genTempFiles dir+          filenames <- getFileNames dir+          findPossibleDuplicates filenames $ Just "**"+    length files === 0  genTempFiles :: FilePath -> IO FilePath genTempFiles dir = do-  -- dir <- getTemporaryDirectory   writeFile (dir </> "dup1") "duplicate file"   writeFile (dir </> "dup2") "duplicate file"   writeFile (dir </> "uniq1") "uniq file"   pure dir++allTheSame :: (Eq a) => [a] -> Bool+allTheSame [] = False -- For the duplicate finder we want this case to fail. Semantically this makes no sense.+allTheSame xs = all (== head xs) (tail xs)