diff --git a/hs-opentelemetry-api.cabal b/hs-opentelemetry-api.cabal
--- a/hs-opentelemetry-api.cabal
+++ b/hs-opentelemetry-api.cabal
@@ -5,7 +5,7 @@
 -- see: https://github.com/sol/hpack
 
 name:           hs-opentelemetry-api
-version:        0.0.3.1
+version:        0.0.3.2
 synopsis:       OpenTelemetry API for use by libraries for direct instrumentation or wrapper packages.
 description:    Please see the README on GitHub at <https://github.com/iand675/hs-opentelemetry/tree/main/api#readme>
 category:       OpenTelemetry, Telemetry, Monitoring, Observability, Metrics
diff --git a/src/OpenTelemetry/Context.hs b/src/OpenTelemetry/Context.hs
--- a/src/OpenTelemetry/Context.hs
+++ b/src/OpenTelemetry/Context.hs
@@ -33,8 +33,10 @@
   , union
   , insertSpan
   , lookupSpan
+  , removeSpan
   , insertBaggage
   , lookupBaggage
+  , removeBaggage
   ) where
 import Control.Monad.IO.Class
 import Data.Maybe
@@ -88,6 +90,9 @@
 insertSpan :: Span -> Context -> Context
 insertSpan = insert spanKey
 
+removeSpan :: Context -> Context 
+removeSpan = delete spanKey
+
 baggageKey :: Key Baggage
 baggageKey = unsafePerformIO $ newKey "baggage"
 {-# NOINLINE baggageKey #-}
@@ -99,3 +104,6 @@
 insertBaggage b c = case lookup baggageKey c of
   Nothing -> insert baggageKey b c
   Just b' -> insert baggageKey (b <> b') c
+
+removeBaggage :: Context -> Context
+removeBaggage = delete baggageKey
diff --git a/src/OpenTelemetry/Trace/Core.hs b/src/OpenTelemetry/Trace/Core.hs
--- a/src/OpenTelemetry/Trace/Core.hs
+++ b/src/OpenTelemetry/Trace/Core.hs
@@ -351,7 +351,7 @@
         recordException s [] Nothing inner
       endSpan s Nothing
       adjustContext $ \ctx ->
-        maybe ctx (`insertSpan` ctx) parent
+        maybe (removeSpan ctx) (`insertSpan` ctx) parent
     )
     (\(_, s) -> f s)