diff --git a/ChangeLog.md b/ChangeLog.md
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,5 +1,10 @@
 # ChangeLog for yaml
 
+## 0.11.1.0
+
+* Better error messages in the `Data.Yaml.Config` module [#168](https://github.com/snoyberg/yaml/issues/168)
+* Add `LoadSettingsException` exception and remove error printing from `loadYamlSettings` [#172](https://github.com/snoyberg/yaml/pull/172)
+
 ## 0.11.0.0
 
 * Split out the `libyaml` and `Text.Libyaml` code into its own package. [#145](https://github.com/snoyberg/yaml/issues/145)
diff --git a/src/Data/Yaml/Config.hs b/src/Data/Yaml/Config.hs
--- a/src/Data/Yaml/Config.hs
+++ b/src/Data/Yaml/Config.hs
@@ -180,9 +180,7 @@
     runValues <- forM runTimeFiles $ \fp -> do
         eres <- YI.decodeFileEither fp
         case eres of
-            Left e -> do
-                putStrLn $ "loadYamlSettings: Could not parse file as YAML: " ++ fp
-                throwIO e
+            Left e -> throwIO (Y.LoadSettingsException fp e)
             Right value -> return value
 
     value' <-
@@ -197,9 +195,9 @@
             UseCustomEnv env     -> return $ applyEnvValue   False env    value'
             RequireCustomEnv env -> return $ applyEnvValue   True  env    value'
 
-    case fromJSON value of
-        Error s -> error $ "Could not convert to expected type: " ++ s
-        Success settings -> return settings
+    case Y.parseEither parseJSON value of
+        Left s -> error $ "Could not convert to expected type: " ++ s
+        Right settings -> return settings
 
 -- | Same as @loadYamlSettings@, but get the list of runtime config files from
 -- the command line arguments.
diff --git a/src/Data/Yaml/Internal.hs b/src/Data/Yaml/Internal.hs
--- a/src/Data/Yaml/Internal.hs
+++ b/src/Data/Yaml/Internal.hs
@@ -62,6 +62,7 @@
                     | OtherParseException SomeException
                     | NonStringKeyAlias Y.AnchorName Value
                     | CyclicIncludes
+                    | LoadSettingsException FilePath ParseException
     deriving (Show, Typeable)
 
 instance Exception ParseException where
@@ -113,6 +114,7 @@
     , "  Value: " ++ show value
     ]
   CyclicIncludes -> "Cyclic includes"
+  LoadSettingsException fp exc -> "Could not parse file as YAML: " ++ fp ++ "\n" ++ prettyPrintParseException exc
 
 defineAnchor :: Value -> String -> ReaderT JSONPath (ConduitM e o Parse) ()
 defineAnchor value name = modify (modifyAnchors $ Map.insert name value)
diff --git a/test/Data/YamlSpec.hs b/test/Data/YamlSpec.hs
--- a/test/Data/YamlSpec.hs
+++ b/test/Data/YamlSpec.hs
@@ -8,6 +8,7 @@
 module Data.YamlSpec (main, spec) where
 
 import qualified Text.Libyaml as Y
+import qualified Data.ByteString as BS
 import qualified Data.ByteString.Char8 as B8
 import Data.Int (Int64)
 
@@ -55,6 +56,13 @@
   actual <- D.decodeThrow bs
   actual `shouldBe` expected
 
+testEncodeWith :: Y.FormatOptions -> [Y.Event] -> IO BS.ByteString
+testEncodeWith opts es = runConduitRes (CL.sourceList events .| Y.encodeWith opts)
+  where
+    events =
+      [Y.EventStreamStart, Y.EventDocumentStart] ++ es ++
+      [Y.EventDocumentEnd, Y.EventStreamEnd]
+
 main :: IO ()
 main = hspec spec
 
@@ -205,6 +213,103 @@
       go "12.3015e+02" (1230.15 :: Scientific)
       go "1230.15" (1230.15 :: Scientific)
 
+    describe "Text.Libyaml with default tag rendering" $ do
+      let enc = testEncodeWith Y.defaultFormatOptions
+      it "elides custom sequence tags" $
+        enc taggedSequence `shouldReturn` "[]\n"
+      it "elides custom mapping tags" $
+        enc taggedMapping `shouldReturn` "{}\n"
+      it "elides default sequence tags" $
+        enc defaultTaggedSequence `shouldReturn` "[]\n"
+      it "elides default mapping tags" $
+        enc defaultTaggedMapping `shouldReturn` "{}\n"
+      it "handles NoTag on sequences" $
+        enc untaggedSequence `shouldReturn` "[]\n"
+      it "handles NoTag on mappings" $
+        enc untaggedMapping `shouldReturn` "{}\n"
+      it "handles mixed tag usages but elides all mapping and sequence tags" $
+        enc mixedTagSampleA `shouldReturn` "- {}\n"
+      it "in combination of tags, anchors and styles, outputs only the scalar tags" $
+        enc mixedTagSampleB `shouldReturn` "&a\n&b !<bar> foo: &c [&d !!null '']\n"
+      it "outputs tags when double quoted" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.DoubleQuoted Nothing] `shouldReturn` "!!str \"foo\"\n"
+      it "outputs tags when single quoted" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.SingleQuoted Nothing] `shouldReturn` "!!str 'foo'\n"
+      it "outputs tags on literal text" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.Literal Nothing] `shouldReturn` "!!str |-\n  foo\n"
+      it "outputs tags on folded text" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.Folded Nothing] `shouldReturn` "!!str >-\n  foo\n"
+    describe "Text.Libyaml with all tags on" $ do
+      let enc = testEncodeWith $ Y.setTagRendering Y.renderAllTags Y.defaultFormatOptions
+      it "will output custom sequence tags" $
+        enc taggedSequence `shouldReturn` "!foo []\n"
+      it "will output custom mapping tags" $
+        enc taggedMapping `shouldReturn` "!foo {}\n"
+      it "will output default sequence tags" $
+        enc defaultTaggedSequence `shouldReturn` "!!seq []\n"
+      it "will output default mapping tags" $
+        enc defaultTaggedMapping `shouldReturn` "!!map {}\n"
+      it "handles NoTag on sequences" $
+        enc untaggedSequence `shouldReturn` "[]\n"
+      it "handles NoTag on mappings" $
+        enc untaggedMapping `shouldReturn` "{}\n"
+      it "handles mixed tag usages outputting all mapping and sequence tags" $
+        enc mixedTagSampleA `shouldReturn` "- !foo {}\n"
+      it "in combination of tags, anchors and styles, outputs all the tags" $
+        enc mixedTagSampleB `shouldReturn` "&a\n&b !<bar> foo: &c !baz [&d !!null '']\n"
+      it "outputs plain tags" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.Plain Nothing] `shouldReturn` "!!str foo\n"
+      it "respects PlainNoTag tags" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.PlainNoTag Nothing] `shouldReturn` "foo\n"
+    describe "Text.Libyaml with uri tags on" $ do
+      let enc = testEncodeWith $ Y.setTagRendering Y.renderUriTags Y.defaultFormatOptions
+      it "will output custom sequence tags" $
+        enc taggedSequence `shouldReturn` "!foo []\n"
+      it "will output custom mapping tags" $
+        enc taggedMapping `shouldReturn` "!foo {}\n"
+      it "will output default sequence tags" $
+        enc defaultTaggedSequence `shouldReturn` "[]\n"
+      it "will output default mapping tags" $
+        enc defaultTaggedMapping `shouldReturn` "{}\n"
+      it "handles NoTag on sequences" $
+        enc untaggedSequence `shouldReturn` "[]\n"
+      it "handles NoTag on mappings" $
+        enc untaggedMapping `shouldReturn` "{}\n"
+      it "handles mixed tag usages outputting all mapping and sequence tags" $
+        enc mixedTagSampleA `shouldReturn` "- !foo {}\n"
+      it "in combination of tags, anchors and styles, outputs all the tags" $
+        enc mixedTagSampleB `shouldReturn` "&a\n&b !<bar> foo: &c !baz [&d '']\n"
+    describe "Text.Libyaml with tags off" $ do
+      let enc = testEncodeWith $ Y.setTagRendering Y.renderNoTags Y.defaultFormatOptions
+      it "outputs plain tags" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.Plain Nothing] `shouldReturn` "foo\n"
+      it "respects PlainNoTag tags" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.PlainNoTag Nothing] `shouldReturn` "foo\n"
+      it "elides tags when double quoted" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.DoubleQuoted Nothing] `shouldReturn` "\"foo\"\n"
+      it "elides tags when single quoted" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.SingleQuoted Nothing] `shouldReturn` "'foo'\n"
+      it "elides tags on literal text" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.Literal Nothing] `shouldReturn` "|-\n  foo\n"
+      it "elides tags on folded text" $
+        enc [Y.EventScalar "foo" Y.StrTag Y.Folded Nothing] `shouldReturn` ">-\n  foo\n"
+    describe "Text.Libyaml with only UriTags set to render " $ do
+      let enc =
+            testEncodeWith $
+            Y.setTagRendering Y.renderUriTags $ Y.defaultFormatOptions
+      it "outputs only UriTags" $
+        enc
+          [ Y.EventSequenceStart Y.NoTag Y.FlowSequence Nothing
+          , Y.EventScalar "foo" Y.StrTag Y.DoubleQuoted Nothing
+          , Y.EventScalar "99" Y.IntTag Y.Plain Nothing
+          , Y.EventScalar "99.99" Y.FloatTag Y.Plain Nothing
+          , Y.EventScalar "bar" Y.NoTag Y.Plain Nothing
+          , Y.EventScalar "foo" (Y.UriTag "!foo") Y.DoubleQuoted Nothing
+          , Y.EventScalar "foo" (Y.UriTag "!foo") Y.Plain Nothing
+          , Y.EventSequenceEnd
+          ] `shouldReturn`
+        "[\"foo\", 99, 99.99, bar, !foo \"foo\", !foo foo]\n"
+
 specialStrings :: [T.Text]
 specialStrings =
     [ "fo\"o"
@@ -622,3 +727,50 @@
     D.encodeFile fp val
     res <- D.decodeFileEither fp
     either (Left . show) Right res `shouldBe` Right val
+
+
+taggedSequence :: [Y.Event]
+taggedSequence =
+  [ Y.EventSequenceStart (Y.UriTag "!foo") Y.FlowSequence Nothing
+  , Y.EventSequenceEnd
+  ]
+
+taggedMapping :: [Y.Event]
+taggedMapping =
+  [ Y.EventMappingStart (Y.UriTag "!foo") Y.FlowMapping Nothing
+  , Y.EventMappingEnd
+  ]
+
+defaultTaggedSequence :: [Y.Event]
+defaultTaggedSequence =
+  [Y.EventSequenceStart Y.SeqTag Y.FlowSequence Nothing, Y.EventSequenceEnd]
+
+defaultTaggedMapping :: [Y.Event]
+defaultTaggedMapping =
+  [Y.EventMappingStart Y.MapTag Y.FlowMapping Nothing, Y.EventMappingEnd]
+
+untaggedSequence :: [Y.Event]
+untaggedSequence =
+  [Y.EventSequenceStart Y.NoTag Y.FlowSequence Nothing, Y.EventSequenceEnd]
+
+untaggedMapping :: [Y.Event]
+untaggedMapping =
+  [Y.EventMappingStart Y.NoTag Y.FlowMapping Nothing, Y.EventMappingEnd]
+
+mixedTagSampleA :: [Y.Event]
+mixedTagSampleA =
+  [ Y.EventSequenceStart Y.NoTag Y.BlockSequence Nothing
+  , Y.EventMappingStart (Y.UriTag "!foo") Y.FlowMapping Nothing
+  , Y.EventMappingEnd
+  , Y.EventSequenceEnd
+  ]
+
+mixedTagSampleB :: [Y.Event]
+mixedTagSampleB =
+  [ Y.EventMappingStart Y.NoTag Y.BlockMapping (Just "a")
+  , Y.EventScalar "foo" (Y.UriTag "bar") Y.Plain (Just "b")
+  , Y.EventSequenceStart (Y.UriTag "!baz") Y.FlowSequence (Just "c")
+  , Y.EventScalar "" Y.NullTag Y.Plain (Just "d")
+  , Y.EventSequenceEnd
+  , Y.EventMappingEnd
+  ]
diff --git a/yaml.cabal b/yaml.cabal
--- a/yaml.cabal
+++ b/yaml.cabal
@@ -1,13 +1,13 @@
 cabal-version: 1.12
 
--- This file has been generated from package.yaml by hpack version 0.30.0.
+-- This file has been generated from package.yaml by hpack version 0.31.2.
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 44126ede97b8c2576f72b255726cbc3a06a12e3547a02cceca6e02c4933029cc
+-- hash: 3a4f2b1c1a7a978d5810ea6223b95b1c51f38f60610704c35e28f8bc667081ba
 
 name:           yaml
-version:        0.11.0.0
+version:        0.11.1.0
 synopsis:       Support for parsing and rendering YAML documents.
 description:    README and API documentation are available at <https://www.stackage.org/package/yaml>
 category:       Data
