diff --git a/ChangeLog.md b/ChangeLog.md
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,3 +1,7 @@
+## 0.8.21.2
+
+* Fix wrong file not found exception in `Data.Yaml.Include` with pre-1.2.3.0 `directory` [#104](https://github.com/snoyberg/yaml/pull/104)
+
 ## 0.8.21.1
 
 * Add missing test files [#102](https://github.com/snoyberg/yaml/pull/102)
diff --git a/Data/Yaml/Include.hs b/Data/Yaml/Include.hs
--- a/Data/Yaml/Include.hs
+++ b/Data/Yaml/Include.hs
@@ -1,6 +1,13 @@
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE RankNTypes #-}
 module Data.Yaml.Include (decodeFile, decodeFileEither) where
 
+#if !MIN_VERSION_directory(1, 2, 3)
+import Control.Exception (handleJust)
+import Control.Monad (guard)
+import System.IO.Error (ioeGetFileName, ioeGetLocation, isDoesNotExistError)
+#endif
+
 import Control.Exception (throwIO)
 import Control.Monad (when)
 import Control.Monad.IO.Class (liftIO)
@@ -25,7 +32,7 @@
   where
     go :: MonadResource m => [FilePath] -> FilePath -> Producer m Event
     go seen fp = do
-        cfp <- liftIO $ canonicalizePath fp
+        cfp <- liftIO $ handleNotFound $ canonicalizePath fp
         when (cfp `elem` seen) $ do
             liftIO $ throwIO CyclicIncludes
         Y.decodeFile cfp $= do
@@ -36,6 +43,17 @@
                 _ -> yield event
 
     irrelevantEvents = [EventStreamStart, EventDocumentStart, EventDocumentEnd, EventStreamEnd]
+
+#if !MIN_VERSION_directory(1, 2, 3)
+    handleNotFound = handleJust
+        (\e -> do
+            guard (isDoesNotExistError e)
+            guard (ioeGetLocation e == "canonicalizePath")
+            ioeGetFileName e)
+        (throwIO . YamlException . ("Yaml file not found: " ++))
+#else
+    handleNotFound = id
+#endif
 
 -- | Like `Data.Yaml.decodeFile` but with support for relative and absolute
 -- includes.
diff --git a/test/Data/Yaml/IncludeSpec.hs b/test/Data/Yaml/IncludeSpec.hs
--- a/test/Data/Yaml/IncludeSpec.hs
+++ b/test/Data/Yaml/IncludeSpec.hs
@@ -2,12 +2,12 @@
 module Data.Yaml.IncludeSpec (main, spec) where
 
 import           Test.Hspec
-import           Data.Either.Compat
+import           Data.List (isPrefixOf)
 import           Data.Aeson
 import           Data.Aeson.QQ
-import           Data.Yaml (ParseException)
-
+import           Data.Yaml (ParseException(InvalidYaml))
 import           Data.Yaml.Include
+import           Text.Libyaml (YamlException(YamlException))
 
 main :: IO ()
 main = hspec spec
@@ -38,7 +38,17 @@
     it "aborts on cyclic includes" $ do
       (decodeFile "test/resources/loop/foo.yaml" :: IO (Maybe Value)) `shouldThrow` anyException
 
+    context "when file does not exist" $ do
+      it "throws Left (InvalidYaml (Just (YamlException \"Yaml file not found: ...\")))" $ do
+        (decodeFile "./does_not_exist.yaml" :: IO (Maybe Value)) `shouldThrow` isYamlFileNotFoundException
+
   describe "decodeFileEither" $ do
     context "when file does not exist" $ do
-      it "returns Left" $ do
-        (decodeFileEither "./does_not_exist.yaml" :: IO (Either ParseException Value)) >>= (`shouldSatisfy` isLeft)
+      it "returns Left (InvalidYaml (Just (YamlException \"Yaml file not found: ...\")))" $ do
+        (decodeFileEither "./does_not_exist.yaml" :: IO (Either ParseException Value)) >>=
+          (`shouldSatisfy` either isYamlFileNotFoundException (const False))
+
+isYamlFileNotFoundException :: ParseException -> Bool
+isYamlFileNotFoundException (InvalidYaml (Just (YamlException msg)))
+  | "Yaml file not found: " `isPrefixOf` msg = True
+isYamlFileNotFoundException _ = False
diff --git a/yaml.cabal b/yaml.cabal
--- a/yaml.cabal
+++ b/yaml.cabal
@@ -1,5 +1,5 @@
 name:            yaml
-version:         0.8.21.1
+version:         0.8.21.2
 license:         BSD3
 license-file:    LICENSE
 author:          Michael Snoyman <michael@snoyman.com>, Anton Ageev <antage@gmail.com>,Kirill Simonov