diff --git a/CHANGELOG.md b/CHANGELOG.md
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Unreleased
 
+# v0.3.1.0
+
+* Fix bug where errors in Main module would show the path of the temporary file instead of the original file
+* Ignore binary files in test directory
+
 # v0.3.0.0
 
 * Fix bug when omitting signature after specifying a signature for a prior test
diff --git a/src/Test/Tasty/AutoCollect.hs b/src/Test/Tasty/AutoCollect.hs
--- a/src/Test/Tasty/AutoCollect.hs
+++ b/src/Test/Tasty/AutoCollect.hs
@@ -15,16 +15,14 @@
 processFile :: FilePath -> Text -> IO Text
 processFile path file =
   case parseModuleType file of
-    Just (ModuleMain cfg) -> generateMainModule cfg path
+    Just (ModuleMain cfg) -> addLinePragma <$> generateMainModule cfg path
     Just ModuleTest ->
-      pure . Text.unlines $
-        [ "{-# OPTIONS_GHC -fplugin=Test.Tasty.AutoCollect.ConvertTest #-}"
-        , file'
-        ]
-    Nothing -> pure file'
+      pure
+        . addLine "{-# OPTIONS_GHC -fplugin=Test.Tasty.AutoCollect.ConvertTest #-}"
+        . addLinePragma
+        $ file
+    Nothing -> pure $ addLinePragma file
   where
-    file' =
-      Text.unlines
-        [ "{-# LINE 1 " <> quoted (Text.pack path) <> " #-}"
-        , file
-        ]
+    addLine line f = line <> "\n" <> f
+    -- this is needed to tell GHC to use original path in error messages
+    addLinePragma = addLine $ "{-# LINE 1 " <> quoted (Text.pack path) <> " #-}"
diff --git a/src/Test/Tasty/AutoCollect/GenerateMain.hs b/src/Test/Tasty/AutoCollect/GenerateMain.hs
--- a/src/Test/Tasty/AutoCollect/GenerateMain.hs
+++ b/src/Test/Tasty/AutoCollect/GenerateMain.hs
@@ -6,12 +6,13 @@
   generateMainModule,
 ) where
 
+import qualified Data.ByteString as ByteString
 import Data.List (sortOn)
 import qualified Data.Map.Strict as Map
 import Data.Maybe (catMaybes, fromMaybe)
 import Data.Text (Text)
 import qualified Data.Text as Text
-import qualified Data.Text.IO as Text
+import qualified Data.Text.Encoding as Text
 import System.Directory (doesDirectoryExist, listDirectory)
 import System.FilePath (makeRelative, splitExtensions, takeDirectory, (</>))
 
@@ -76,10 +77,10 @@
     testDir = takeDirectory path
 
     toTestModule fp = do
-      fileContents <- Text.readFile fp
+      fileContentsBS <- ByteString.readFile fp
       return $
-        case (splitExtensions fp, parseModuleType fileContents) of
-          ((fpNoExt, ".hs"), Just ModuleTest) ->
+        case (splitExtensions fp, parseModuleType <$> Text.decodeUtf8' fileContentsBS) of
+          ((fpNoExt, ".hs"), Right (Just ModuleTest)) ->
             let moduleName = Text.replace "/" "." . Text.pack . makeRelative testDir $ fpNoExt
              in Just
                   TestModule
diff --git a/tasty-autocollect.cabal b/tasty-autocollect.cabal
--- a/tasty-autocollect.cabal
+++ b/tasty-autocollect.cabal
@@ -5,7 +5,7 @@
 -- see: https://github.com/sol/hpack
 
 name:           tasty-autocollect
-version:        0.3.0.0
+version:        0.3.1.0
 synopsis:       Autocollection of tasty tests.
 description:    Autocollection of tasty tests. See README.md for more details.
 category:       Testing
@@ -64,6 +64,7 @@
   ghc-options: -Wall
   build-depends:
       base >=4.14 && <5
+    , bytestring >=0.10 && <0.12
     , containers >=0.6.2.1 && <0.7
     , directory >=1.3.6.0 && <2
     , filepath >=1.4.2.1 && <2
diff --git a/test/Test/Tasty/AutoCollect/GenerateMainTest.hs b/test/Test/Tasty/AutoCollect/GenerateMainTest.hs
--- a/test/Test/Tasty/AutoCollect/GenerateMainTest.hs
+++ b/test/Test/Tasty/AutoCollect/GenerateMainTest.hs
@@ -5,8 +5,10 @@
   -- $AUTOCOLLECT.TEST.export$
 ) where
 
+import qualified Data.ByteString as ByteString
 import Data.Text (Text)
 import qualified Data.Text as Text
+import System.FilePath ((</>))
 import Test.Predicates
 import Test.Predicates.HUnit
 import Test.Tasty.HUnit
@@ -36,6 +38,20 @@
       , "test = testCase \"test\" $ return ()"
       ]
 
+test =
+  testCase "ignores binary files" $
+    assertSuccess_ $
+      runMainWith
+        ( \proj ->
+            proj
+              { preRunCallback = \tmpdir ->
+                  ByteString.writeFile
+                    (tmpdir </> "binary-file")
+                    (ByteString.pack [0 ..])
+              }
+        )
+        ["{- AUTOCOLLECT.MAIN -}"]
+
 test_batch =
   [ testGolden
     ("output for group_type = " <> groupType <> " is as expected")
@@ -235,6 +251,7 @@
           , testFile "FooTest"
           , testFile "BarTest"
           ]
+      , preRunCallback = \_ -> pure ()
       , entrypoint = "Main.hs"
       , runArgs = []
       }
diff --git a/test/TestUtils/Integration.hs b/test/TestUtils/Integration.hs
--- a/test/TestUtils/Integration.hs
+++ b/test/TestUtils/Integration.hs
@@ -76,6 +76,7 @@
   { dependencies :: [Text]
   , extraGhcArgs :: [Text]
   , files :: [(FilePath, FileContents)]
+  , preRunCallback :: FilePath -> IO ()
   , entrypoint :: FilePath
   , runArgs :: [Text]
   }
@@ -97,6 +98,8 @@
       createDirectoryIfMissing True (takeDirectory testFile)
       Text.writeFile testFile (Text.unlines contents)
 
+    preRunCallback tmpdir
+
     let ghcArgs =
           concat
             [ ["-hide-all-packages"]
@@ -136,6 +139,7 @@
           [ ("Test.hs", testFilePrefix ++ contents)
           , ("Main.hs", ["{- AUTOCOLLECT.MAIN -}"])
           ]
+      , preRunCallback = \_ -> pure ()
       , entrypoint = "Main.hs"
       , runArgs = []
       }