diff --git a/delivery_auto_refresh/models/sale_order.py b/delivery_auto_refresh/models/sale_order.py
index ecb0a14a58..ac4387e5f1 100644
--- a/delivery_auto_refresh/models/sale_order.py
+++ b/delivery_auto_refresh/models/sale_order.py
@@ -59,7 +59,9 @@ def _get_param_auto_add_delivery_line(self):
     def _auto_refresh_delivery(self):
         self.ensure_one()
         # Make sure that if you have removed the carrier, the line is gone
-        if self.state in {"draft", "sent"}:
+        if self.state in {"draft", "sent"} and not self.env.context.get(
+            "deleting_delivery_line"
+        ):
             # Context added to avoid the recursive calls and save the new
             # value of carrier_id
             self.with_context(auto_refresh_delivery=True)._remove_delivery_line()
@@ -67,10 +69,12 @@ def _auto_refresh_delivery(self):
             if self.state in {"draft", "sent"}:
                 price_unit = self.carrier_id.rate_shipment(self)["price"]
                 if not self.is_all_service:
-                    self._create_delivery_line(self.carrier_id, price_unit)
-                self.with_context(auto_refresh_delivery=True).write(
-                    {"recompute_delivery_price": False}
-                )
+                    self.with_context(skip_validation_check=True)._create_delivery_line(
+                        self.carrier_id, price_unit
+                    )
+                self.with_context(
+                    auto_refresh_delivery=True, skip_validation_check=True
+                ).write({"recompute_delivery_price": False})
 
     @api.model
     def create(self, vals):
@@ -80,7 +84,15 @@ def create(self, vals):
         return order
 
     def write(self, vals):
-        """Create or refresh delivery line after saving."""
+        """Create or refresh the delivery line after saving."""
+        # Check if it's already deleting a delivery line to not
+        # delete it again inside `_auto_refresh_delivery()`
+        deleting_delivery_line = vals.get("order_line") and any(
+            i[0] == 2 and self.env["sale.order.line"].browse(i[1]).is_delivery
+            for i in vals["order_line"]
+        )
+        if deleting_delivery_line:
+            self = self.with_context(deleting_delivery_line=deleting_delivery_line)
         res = super().write(vals)
         if self._get_param_auto_add_delivery_line() and not self.env.context.get(
             "auto_refresh_delivery"
diff --git a/delivery_auto_refresh/tests/test_delivery_auto_refresh.py b/delivery_auto_refresh/tests/test_delivery_auto_refresh.py
index 9fcb528167..17102f92dc 100644
--- a/delivery_auto_refresh/tests/test_delivery_auto_refresh.py
+++ b/delivery_auto_refresh/tests/test_delivery_auto_refresh.py
@@ -303,8 +303,7 @@ def test_auto_refresh_so_and_unlink_line(self):
         self.assertFalse(delivery_line.exists())
 
     def test_auto_add_delivery_line_add_service(self):
-        """Delivery line should not be created because
-        there are only service products in SO"""
+        self.env["ir.config_parameter"].sudo().set_param(self.auto_add_delivery_line, 1)
         service = self.env["product.product"].create(
             {"name": "Service Test", "type": "service"}
         )
@@ -318,3 +317,11 @@ def test_auto_add_delivery_line_add_service(self):
         order = order_form.save()
         delivery_line = order.order_line.filtered("is_delivery")
         self.assertFalse(delivery_line.exists())
+
+    def test_auto_refresh_so_and_manually_unlink_delivery_line(self):
+        """Test that we are able to manually remove the delivery line"""
+        self._test_autorefresh_unlink_line()
+        sale_form = Form(self.order)
+        # Deleting the delivery line
+        sale_form.order_line.remove(1)
+        sale_form.save()