diff --git a/CHANGELOG.md b/CHANGELOG.md
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
 Changelog for tmp-postgres
 
+1.23.0.3
+  #122 Try to minize failed deletes ... again.
+
 1.23.0.2
   #183 More Doc
 
diff --git a/src/Database/Postgres/Temp/Internal/Config.hs b/src/Database/Postgres/Temp/Internal/Config.hs
--- a/src/Database/Postgres/Temp/Internal/Config.hs
+++ b/src/Database/Postgres/Temp/Internal/Config.hs
@@ -379,12 +379,17 @@
   let ignoreDirIsMissing e
         | isDoesNotExistError e = return ()
         | otherwise = throwIO e
-  -- I'm trying to prevent new files getting added
-  -- to the dir as I am deleting the files.
-  let newName = mainDir <> "_removing"
-  handle ignoreDirIsMissing $ uninterruptibleMask_ $ do
-    renameDirectory mainDir newName
-    removeDirectoryRecursive newName
+
+  -- Files are continued to be written after the delete starts. This
+  -- seems to fix it. #122
+  -- TODO come up with a better way to deal with this. Probably
+  -- need to lock the directories recursively before deleting.
+  handle ignoreDirIsMissing $ do
+    try (removePathForcibly mainDir) >>= \case
+      Left (_ :: IOError) -> try (removePathForcibly mainDir) >>= \case
+        Left (_ :: IOError) -> removePathForcibly mainDir
+        Right _ -> pure ()
+      Right _ -> pure ()
 
 -- | Either remove a 'CTemporary' directory or do nothing to a 'CPermanent'
 -- one.
diff --git a/test/Main.hs b/test/Main.hs
--- a/test/Main.hs
+++ b/test/Main.hs
@@ -509,7 +509,7 @@
   it "throws if initdb is not on the path" $ do
     path <-  getEnv "PATH"
 
-    bracket (setEnv "PATH" "/foo") (const $ setEnv "PATH" path) $ \_ ->
+    bracket (setEnv "PATH" "/bin") (const $ setEnv "PATH" path) $ \_ ->
       withConfig defaultConfig (const $ pure ())
         `shouldThrow` isDoesNotExistError
 
@@ -517,10 +517,14 @@
     withTempDirectory "/tmp" "createdb-not-on-path-test" $ \dir -> do
       Just initDbPath   <- findExecutable "initdb"
       Just postgresPath <- findExecutable "postgres"
+      Just rmPath       <- findExecutable "rm"
+      Just chmod        <- findExecutable "chmod"
 
       -- create symlinks
       createSymbolicLink initDbPath $ dir <> "/initdb"
       createSymbolicLink postgresPath $ dir <> "/postgres"
+      createSymbolicLink rmPath $ dir <> "/rm"
+      createSymbolicLink chmod $ dir <> "/chmod"
 
       path <-  getEnv "PATH"
 
diff --git a/tmp-postgres.cabal b/tmp-postgres.cabal
--- a/tmp-postgres.cabal
+++ b/tmp-postgres.cabal
@@ -1,5 +1,5 @@
 name:                tmp-postgres
-version:             1.23.0.2
+version:             1.23.0.3
 synopsis: Start and stop a temporary postgres
 description: Start and stop a temporary postgres. See README.md
 homepage:            https://github.com/jfischoff/tmp-postgres#readme