diff --git a/ChangeLog.md b/ChangeLog.md
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,3 +1,7 @@
+## 0.8.18.1
+
+* Improve prettyPrintParseException when context is empty [#89](https://github.com/snoyberg/yaml/pull/89)
+
 ## 0.8.18
 
 * Switched yaml decode function for config file readers in `Data.Yaml.Config` to
diff --git a/Data/Yaml/Internal.hs b/Data/Yaml/Internal.hs
--- a/Data/Yaml/Internal.hs
+++ b/Data/Yaml/Internal.hs
@@ -88,11 +88,13 @@
     Nothing -> "Unspecified YAML error"
     Just yamlError -> case yamlError of
       YamlException s -> "YAML exception:\n" ++ s
-      YamlParseException problem context mark -> unlines
+      YamlParseException problem context mark -> concat
         [ "YAML parse exception at line " ++ show (yamlLine mark) ++
-          ", column " ++ show (yamlColumn mark) ++ ","
-        -- The context seems to include a leading "while" or similar.
-        , context ++ ":"
+          ", column " ++ show (yamlColumn mark)
+        , case context of
+            "" -> ":\n"
+            -- The context seems to include a leading "while" or similar.
+            _  -> ",\n" ++ context ++ ":\n"
         , problem
         ]
   AesonException s -> "Aeson exception:\n" ++ s
diff --git a/yaml.cabal b/yaml.cabal
--- a/yaml.cabal
+++ b/yaml.cabal
@@ -1,5 +1,5 @@
 name:            yaml
-version:         0.8.18
+version:         0.8.18.1
 license:         BSD3
 license-file:    LICENSE
 author:          Michael Snoyman <michael@snoyman.com>, Anton Ageev <antage@gmail.com>,Kirill Simonov