diff --git a/ChangeLog.md b/ChangeLog.md
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,3 +1,7 @@
+## 0.4.7.2
+
+* Fix question marks in string literals causing substitution errors (https://github.com/paul-rouse/mysql-simple/pull/54)
+
 ## 0.4.7.1
 
 * Use parseTimeM from Data.Time.Format instead of parseTime (https://github.com/paul-rouse/mysql-simple/pull/53)
diff --git a/Database/MySQL/Simple.hs b/Database/MySQL/Simple.hs
--- a/Database/MySQL/Simple.hs
+++ b/Database/MySQL/Simple.hs
@@ -80,6 +80,7 @@
     -- * Helper functions
     , formatMany
     , formatQuery
+    , splitQuery
     ) where
 
 import Blaze.ByteString.Builder (Builder, fromByteString, toByteString)
@@ -88,6 +89,7 @@
 import Control.Exception (Exception, bracket, onException, throw, throwIO)
 import Control.Monad.Fix (fix)
 import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as BS
 import Data.Int (Int64)
 import Data.List (intersperse)
 import Data.Monoid (mappend, mconcat)
@@ -169,18 +171,47 @@
         [caseless]
 
 buildQuery :: Connection -> Query -> ByteString -> [Action] -> IO Builder
-buildQuery conn q template xs = zipParams (split template) <$> mapM sub xs
+buildQuery conn q template xs = zipParams queryFragments <$> mapM sub xs
   where sub (Plain b)  = pure b
         sub (Escape s) = (inQuotes . fromByteString) <$> Base.escape conn s
         sub (Many ys)  = mconcat <$> mapM sub ys
-        split s = fromByteString h : if B.null t then [] else split (B.tail t)
-            where (h,t) = B.break (=='?') s
         zipParams (t:ts) (p:ps) = t `mappend` p `mappend` zipParams ts ps
         zipParams [t] []        = t
-        zipParams _ _ = fmtError (show (B.count '?' template) ++
+        zipParams _ _ = fmtError (show fragmentCount ++
                                   " '?' characters, but " ++
                                   show (length xs) ++ " parameters") q xs
+        fragmentCount = length queryFragments - 1
+        queryFragments = splitQuery template
 
+-- | Split a query into fragments separated by @?@ characters. Does not
+-- break a fragment if the question mark is in a string literal.
+splitQuery :: ByteString -> [Builder]
+splitQuery s =
+  reverse $ fmap (fromByteString . BS.pack . reverse) $
+    begin [] (BS.unpack s)
+  where
+  begin = normal []
+
+  normal ret acc [] =
+    acc : ret
+  normal ret acc (c : cs) =
+    case c of
+      '?' ->
+        normal (acc : ret) [] cs
+      '\'' ->
+        quotes ret (c : acc) cs
+      _ ->
+        normal ret (c : acc) cs
+
+  quotes ret acc [] =
+    acc : ret
+  quotes ret acc (c : cs) =
+    case c of
+      '\'' ->
+        normal ret (c : acc) cs
+      _ ->
+        quotes ret (c : acc) cs
+
 -- | Execute an @INSERT@, @UPDATE@, or other SQL query that is not
 -- expected to return results.
 --
@@ -373,7 +404,7 @@
 -- facility to address both ease of use and security.
 
 -- $querytype
--- 
+--
 -- A 'Query' is a @newtype@-wrapped 'ByteString'. It intentionally
 -- exposes a tiny API that is not compatible with the 'ByteString'
 -- API; this makes it difficult to construct queries from fragments of
diff --git a/Database/MySQL/Simple/Types.hs b/Database/MySQL/Simple/Types.hs
--- a/Database/MySQL/Simple/Types.hs
+++ b/Database/MySQL/Simple/Types.hs
@@ -31,26 +31,12 @@
 import qualified Data.ByteString as B
 
 -- | A placeholder for the SQL @NULL@ value.
---
--- Prior to @mysql-simple-0.5@, an 'Eq' instance was provided for 'Null'.
--- In an attempt to mirror SQL semantics, it unconditionally produced
--- 'False' for both '==' and '/='.  However, this is not correct SQL semantics,
--- and in any case it is unclear why those semantics in Haskell would be
--- caried over to a Haskell program manipulating a database.  A proposal has
--- been accepted to remove '/=' from the 'Eq' class, so this non-law-abiding
--- instance will become impossible.  Therefore, the 'Eq' instance will simply
--- be removed.
---
--- Currently a WARNING is produced for uses of the 'Null' type since it is
--- not possible to attach a DEPRECATED pragma to an instance.
---
 data Null = Null
           deriving (Read, Show, Typeable)
 
 instance Eq Null where
     _ == _ = False
     _ /= _ = False
-{-# WARNING Null "The Eq instance of Null is deprecated: see the documentation for Null in Database.MySQL.Simple.Types" #-}
 
 -- | A query string. This type is intended to make it difficult to
 -- construct a SQL query by concatenating string fragments, as that is
diff --git a/mysql-simple.cabal b/mysql-simple.cabal
--- a/mysql-simple.cabal
+++ b/mysql-simple.cabal
@@ -1,5 +1,5 @@
 name:           mysql-simple
-version:        0.4.7.1
+version:        0.4.7.2
 homepage:       https://github.com/paul-rouse/mysql-simple
 bug-reports:    https://github.com/paul-rouse/mysql-simple/issues
 synopsis:       A mid-level MySQL client library.
@@ -84,6 +84,7 @@
     ghc-options:     -Wall
     default-language: Haskell2010
     build-depends:   base                    >= 4 && < 5
+                   , bytestring
                    , blaze-builder
                    , hspec
                    , mysql-simple
diff --git a/stack.yaml b/stack.yaml
--- a/stack.yaml
+++ b/stack.yaml
@@ -4,4 +4,4 @@
 packages:
 - '.'
 extra-deps: []
-resolver: lts-18.6
+resolver: lts-18.28
diff --git a/test/main.hs b/test/main.hs
--- a/test/main.hs
+++ b/test/main.hs
@@ -1,31 +1,42 @@
 {-# LANGUAGE CPP, OverloadedStrings #-}
 
+
+{-# options_ghc -fno-warn-orphans #-}
+
+import Data.ByteString.Builder as BS
+import Control.Applicative   ((<|>))
 import Control.Exception (bracket)
 import Data.Text (Text)
 import Database.MySQL.Simple
 import Database.MySQL.Simple.Param
+import System.Environment    (getEnvironment)
 import Test.Hspec
 import Blaze.ByteString.Builder (toByteString)
-#if MIN_VERSION_base(4,8,2)
-#else
-import Control.Applicative
-import Data.Monoid
-#endif
 
+isCI :: IO Bool
+isCI = do
+    env <- getEnvironment
+    return $ case lookup "TRAVIS" env <|> lookup "CI" env of
+               Just "true" -> True
+               _ -> False
+
 -- This is how to connect to our test database
-testConn :: ConnectInfo
-testConn = defaultConnectInfo {
-               connectHost     = "127.0.0.1",
-               connectUser     = "test",
-               connectDatabase = "test"
-           }
+testConn :: Bool -> ConnectInfo
+testConn ci = defaultConnectInfo {
+                connectHost     = "127.0.0.1"
+              , connectUser     = "test"
+              , connectPassword = "test"
+              , connectDatabase = "test"
+              , connectPort     = if ci then 33306 else 3306
+              }
 
 main :: IO ()
-main =
-  bracket (connect testConn) close $ \conn ->
-    hspec $ do
-      unitSpec
-      integrationSpec conn
+main = do
+    ci <- isCI
+    bracket (connect $ testConn ci) close $ \conn ->
+      hspec $ do
+        describe "Database.MySQL.Simple.unitSpec" unitSpec
+        describe "Database.MySQL.Simple.integrationSpec" $ integrationSpec conn
 
 unitSpec :: Spec
 unitSpec = do
@@ -46,9 +57,51 @@
         Many [Plain _, Escape "foo", Plain _, Escape "bar", Plain _] -> pure ()
         _ -> expectationFailure "expected a Many with specific contents"
 
+  describe "splitQuery" $ do
+    it "works for a single question mark" $ do
+      splitQuery "select * from foo where name = ?"
+        `shouldBe`
+          ["select * from foo where name = ", ""]
+    it "works with a question mark in a string literal" $ do
+      splitQuery "select 'hello?'"
+        `shouldBe`
+          ["select 'hello?'"]
+    it "works with many question marks" $ do
+      splitQuery "select ? + ? + what from foo where bar = ?"
+        `shouldBe`
+          ["select ", " + ", " + what from foo where bar = ", ""]
+
+instance Show BS.Builder where
+  show = show . BS.toLazyByteString
+
+instance Eq BS.Builder where
+  a == b = BS.toLazyByteString a == BS.toLazyByteString b
+
 integrationSpec :: Connection -> Spec
 integrationSpec conn = do
-  describe "the library" $ do
+  describe "query_" $ do
     it "can connect to a database" $ do
       result <- query_ conn "select 1 + 1"
       result `shouldBe` [Only (2::Int)]
+    it "can have question marks in string literals" $ do
+      result <- query_ conn "select 'hello?'"
+      result `shouldBe` [Only ("hello?" :: Text)]
+  describe "query" $ do
+    it "can have question marks in string literals" $ do
+      result <- query conn "select 'hello?'" ()
+      result `shouldBe` [Only ("hello?" :: Text)]
+    describe "with too many query params" $ do
+      it "should have the right message" $ do
+        (query conn "select 'hello?'" (Only ['a']) :: IO [Only Text])
+          `shouldThrow`
+            (\e -> fmtMessage e == "0 '?' characters, but 1 parameters")
+    describe "with too few query params" $ do
+      it "should have the right message" $ do
+        (query conn "select 'hello?' = ?" () :: IO [Only Text])
+          `shouldThrow`
+            (\e -> fmtMessage e == "1 '?' characters, but 0 parameters")
+  describe "formatQuery" $ do
+    it "should not blow up on a question mark in string literal" $ do
+      formatQuery conn "select 'hello?'" ()
+        `shouldReturn`
+          "select 'hello?'"