diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,8 @@
+incremental
+====
+
+0.3
+----
+
+* Fixed the bug in the `Semigroup` instance for `Alter`
+* Renamed `Delete` to `Delete_` and `Delete` now contains the old value
diff --git a/incremental.cabal b/incremental.cabal
--- a/incremental.cabal
+++ b/incremental.cabal
@@ -1,5 +1,5 @@
 name:                incremental
-version:             0.2
+version:             0.3
 synopsis:            incremental update library
 description:         Generic interface for incremental updates
 homepage:            https://github.com/fumieval/incremental#readme
@@ -10,7 +10,7 @@
 copyright:           2020 Fumiaki Kinoshita
 category:            Data
 build-type:          Simple
-extra-source-files:  README.md
+extra-source-files:  README.md, CHANGELOG.md
 cabal-version:       >=1.10
 
 library
diff --git a/src/Data/Incremental.hs b/src/Data/Incremental.hs
--- a/src/Data/Incremental.hs
+++ b/src/Data/Incremental.hs
@@ -70,7 +70,11 @@
   patch (Const a) d = Const (patch a d)
   diff (Const a) (Const b) = diff a b
 
-data Alter a d = Insert a | Update d | Delete | Upsert a d
+data Alter a d = Insert a
+  | Update d
+  | Delete a -- ^ last value
+  | Delete_
+  | Upsert a d
   deriving (Generic, Functor)
 
 deriving instance (Show a, Show d) => Show (Alter a d)
@@ -81,14 +85,17 @@
 
 instance (Incremental a, d ~ Delta a, Semigroup d) => Semigroup (Alter a d) where
   _ <> Insert a = Insert a
-  _ <> Delete = Delete
+  _ <> Delete a = Delete a
+  _ <> Delete_ = Delete_
   Insert a <> Update d = Insert (patch a d)
   Insert a <> Upsert _ d = Insert (patch a d)
   Update c <> Update d = Update (c <> d)
   Update c <> Upsert a d = Upsert a (c <> d)
-  Delete <> Update _ = Delete
-  Delete <> Upsert a _ = Insert a
-  Upsert a d <> Update e = Upsert a (d <> e)
+  Delete a <> Update _ = Delete a
+  Delete _ <> Upsert a _ = Insert a
+  Delete_ <> Update _ = Delete_
+  Delete_ <> Upsert a _ = Insert a
+  Upsert a d <> Update e = Upsert (patch a e) (d <> e)
   Upsert a d <> Upsert _ e = Upsert a (d <> e)
 
 instance (Incremental a, Monoid d, d ~ Delta a) => Monoid (Alter a d) where
@@ -104,7 +111,7 @@
   patch _ _ = Nothing
   diff Nothing (Just a) = Just $ Insert a
   diff (Just a) (Just b) = Update <$> diff a b
-  diff (Just _) Nothing = Just Delete
+  diff (Just a) Nothing = Just (Delete a)
   diff _ _ = Nothing
 
 instance (Incremental a, Incremental b) => Incremental (a, b) where
@@ -124,7 +131,7 @@
       Insert a -> Just a
       Upsert a _ -> Just a
       _ -> Nothing
-  diff = (check.). IntMap.mergeWithKey (\_ a b -> Update <$> diff a b) (IntMap.map (const Delete)) (IntMap.map Insert)
+  diff = (check.). IntMap.mergeWithKey (\_ a b -> Update <$> diff a b) (IntMap.map Delete) (IntMap.map Insert)
     where
       check m = if IntMap.null m then Nothing else Just m
 
@@ -135,7 +142,7 @@
       Insert a -> Just a
       Upsert a _ -> Just a
       _ -> Nothing
-  diff = (check.). Map.mergeWithKey (\_ a b -> Update <$> diff a b) (Map.map (const Delete)) (Map.map Insert)
+  diff = (check.). Map.mergeWithKey (\_ a b -> Update <$> diff a b) (Map.map Delete) (Map.map Insert)
     where
       check m = if Map.null m then Nothing else Just m