diff --git a/CHANGELOG.md b/CHANGELOG.md
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,10 @@
 
 ## Unreleased
 
+## 0.3.0.0 - 2023-09-02
+
+- Throw errors instead of returning Maybe when creating metrics sharing a key.
+
 ## 0.2.0.0 - 2023-09-02
 
 - Fix gauges.
diff --git a/src/System/Metrics/StatsD.hs b/src/System/Metrics/StatsD.hs
--- a/src/System/Metrics/StatsD.hs
+++ b/src/System/Metrics/StatsD.hs
@@ -74,34 +74,26 @@
     }
 
 newStatCounter ::
-  (MonadIO m) => Stats -> Key -> Sampling -> m (Maybe StatCounter)
+  (MonadIO m) => Stats -> Key -> Sampling -> m StatCounter
 newStatCounter stats key sampling = do
-  success <- newMetric stats key (CounterData 0)
-  if success
-    then return $ Just $ StatCounter stats key sampling
-    else return Nothing
+  newMetric stats key (CounterData 0)
+  return $ StatCounter stats key sampling
 
 newStatGauge ::
-  (MonadIO m) => Stats -> Key -> Int -> m (Maybe StatGauge)
+  (MonadIO m) => Stats -> Key -> Int -> m StatGauge
 newStatGauge stats key ini = do
-  success <- newMetric stats key (GaugeData ini)
-  if success
-    then return $ Just $ StatGauge stats key
-    else return Nothing
+  newMetric stats key (GaugeData ini)
+  return $ StatGauge stats key
 
-newStatTiming :: (MonadIO m) => Stats -> Key -> Int -> m (Maybe StatTiming)
+newStatTiming :: (MonadIO m) => Stats -> Key -> Int -> m StatTiming
 newStatTiming stats key sampling = do
-  success <- newMetric stats key (TimingData [])
-  if success
-    then return $ Just $ StatTiming stats key sampling
-    else return Nothing
+  newMetric stats key (TimingData [])
+  return $ StatTiming stats key sampling
 
-newStatSet :: (MonadIO m) => Stats -> Key -> m (Maybe StatSet)
+newStatSet :: (MonadIO m) => Stats -> Key -> m StatSet
 newStatSet stats key = do
-  success <- newMetric stats key (SetData HashSet.empty)
-  if success
-    then return $ Just $ StatSet stats key
-    else return Nothing
+  newMetric stats key (SetData HashSet.empty)
+  return $ StatSet stats key
 
 incrementCounter :: (MonadIO m) => StatCounter -> Int -> m ()
 incrementCounter StatCounter {..} =
diff --git a/src/System/Metrics/StatsD/Internal.hs b/src/System/Metrics/StatsD/Internal.hs
--- a/src/System/Metrics/StatsD/Internal.hs
+++ b/src/System/Metrics/StatsD/Internal.hs
@@ -71,7 +71,7 @@
 import Network.Socket qualified as Net
 import Network.Socket.ByteString qualified as Net
 import Text.Printf (printf)
-import UnliftIO (MonadIO, liftIO)
+import UnliftIO (MonadIO, liftIO, throwIO)
 import UnliftIO.Concurrent (threadDelay)
 import UnliftIO.STM
   ( STM,
@@ -191,17 +191,22 @@
         then Just md
         else Nothing
 
-newMetric :: (MonadIO m) => Stats -> Key -> MetricData -> m Bool
+newMetric :: (MonadIO m) => Stats -> Key -> MetricData -> m ()
 newMetric stats key store
-  | validateKey key =
-      atomically $ do
+  | validateKey key = do
+      e <- atomically $ do
         exists <- HashMap.member key <$> readTVar stats.metrics
         if exists
-          then return False
+          then return True
           else do
             modifyTVar stats.metrics (addMetric stats.cfg key store)
-            return True
-  | otherwise = return False
+            return False
+      when e $
+        throwIO $
+          userError $
+            "A metric already exists with key: " <> key
+  | otherwise =
+      throwIO $ userError $ "Metric key is invalid: " <> key
 
 validateKey :: String -> Bool
 validateKey t = not (null t) && all valid t
diff --git a/statsd-rupp.cabal b/statsd-rupp.cabal
--- a/statsd-rupp.cabal
+++ b/statsd-rupp.cabal
@@ -5,7 +5,7 @@
 -- see: https://github.com/sol/hpack
 
 name:           statsd-rupp
-version:        0.2.0.0
+version:        0.3.0.0
 synopsis:       Simple StatsD Client
 description:    Please see the README on GitHub at <https://github.com/jprupp/statsd-rupp#readme>
 category:       System
diff --git a/test/Spec.hs b/test/Spec.hs
--- a/test/Spec.hs
+++ b/test/Spec.hs
@@ -23,12 +23,12 @@
 main = hspec $ around (withMockStats testConfig) $ do
   describe "counter" $ do
     it "creates two counters" $ \(stats, _) -> do
-      Just _ <- newStatCounter stats "planets" 1
-      Just _ <- newStatCounter stats "moons" 1
+      _ <- newStatCounter stats "planets" 1
+      _ <- newStatCounter stats "moons" 1
       return () :: Expectation
     it "reports on two counters" $ \(stats, report) -> do
-      Just planets <- newStatCounter stats "planets" 1
-      Just moons <- newStatCounter stats "moons" 1
+      planets <- newStatCounter stats "planets" 1
+      moons <- newStatCounter stats "moons" 1
       incrementCounter planets 10
       incrementCounter moons 20
       rs <- replicateM 2 report
@@ -38,7 +38,7 @@
             ]
       rs `shouldMatchList` rs'
     it "filters reported samples" $ \(stats, report) -> do
-      Just c <- newStatCounter stats "moons" 4
+      c <- newStatCounter stats "moons" 4
       let counts = [11 .. 30]
       forM_ counts (incrementCounter c)
       rs <- replicateM 5 report
@@ -51,7 +51,7 @@
       r <- report
       r.key `shouldNotBe` "moons"
     it "reports stats" $ \(stats, report) -> do
-      Just c <- newStatCounter stats "planets" 4
+      c <- newStatCounter stats "planets" 4
       replicateM_ 20 (incrementCounter c 10)
       replicateM_ 5 report
       do
@@ -71,13 +71,13 @@
   describe "timing" $ do
     it "reports events" $ \(stats, report) -> do
       let timings = [51 .. 55]
-      Just t <- newStatTiming stats "trips" 1
+      t <- newStatTiming stats "trips" 1
       forM_ timings (addTiming t)
       rs <- replicateM (length timings) report
       let rs' = map (\n -> Report "trips" (Timing n) 1.0) timings
       rs `shouldMatchList` rs'
     it "reports empty stats" $ \(stats, report) -> do
-      Just _ <- newStatTiming stats "holes" 1
+      _ <- newStatTiming stats "holes" 1
       rs <- replicateM 8 report
       let rs' =
             concatMap
@@ -90,7 +90,7 @@
       rs `shouldMatchList` rs'
     it "filters reported samples" $ \(stats, report) -> do
       let timings = [1001 .. 2000]
-      Just t <- newStatTiming stats "cats" 100
+      t <- newStatTiming stats "cats" 100
       forM_ timings (addTiming t)
       rs <- replicateM 10 report
       let rs' =
@@ -140,7 +140,7 @@
       median t95 `shouldBe` median' p95
     it "reports stats" $ \(stats, report) -> do
       let timings = [5 .. 1500]
-      Just t <- newStatTiming stats "kittens" 0
+      t <- newStatTiming stats "kittens" 0
       forM_ timings (addTiming t)
       let l = length timings
           r k v = Report ("stats.timers.kittens." <> k) v 1.0
@@ -186,14 +186,14 @@
   describe "gauge" $ do
     it "reports set events" $ \(stats, report) -> do
       let gauges = [51 .. 55]
-      Just g <- newStatGauge stats "speed" 50
+      g <- newStatGauge stats "speed" 50
       forM_ gauges (setGauge g)
       rs <- replicateM (length gauges) report
       let rs' = map (\n -> Report "speed" (Gauge n False) 1.0) gauges
       rs `shouldMatchList` rs'
     it "reports stats" $ \(stats, report) -> do
       let gauges = [51 .. 55]
-      Just g <- newStatGauge stats "radius" 50
+      g <- newStatGauge stats "radius" 50
       r <- report
       r `shouldBe` Report "stats.gauges.radius" (Gauge 50 False) 1.0
       forM_ gauges (setGauge g)
@@ -202,7 +202,7 @@
       let rs' = replicate 2 $ Report "stats.gauges.radius" (Gauge 55 False) 1.0
       rs `shouldMatchList` rs'
     it "increments and decrements value" $ \(stats, report) -> do
-      Just g <- newStatGauge stats "breadth" 50
+      g <- newStatGauge stats "breadth" 50
       incrementGauge g 5
       report `shouldReturn` Report "breadth" (Gauge 5 True) 1.0
       report `shouldReturn` Report "stats.gauges.breadth" (Gauge 55 False) 1.0
@@ -218,14 +218,14 @@
   describe "set" $ do
     it "reports events" $ \(stats, report) -> do
       let set = ["one", "two", "three", "three"]
-      Just s <- newStatSet stats "potatoes"
+      s <- newStatSet stats "potatoes"
       forM_ set (newSetElement s)
       rs <- replicateM (length set) report
       let rs' = map (\x -> Report "potatoes" (Set x) 1.0) set
       rs `shouldMatchList` rs'
     it "reports stats" $ \(stats, report) -> do
       let set = ["two", "three", "one", "two", "three", "three"]
-      Just s <- newStatSet stats "bananas"
+      s <- newStatSet stats "bananas"
       forM_ set (newSetElement s)
       let rs' =
             [ Report "stats.sets.bananas.count" (Counter 3) 1.0,