diff --git a/src/zeroconf/_handlers/record_manager.py b/src/zeroconf/_handlers/record_manager.py
index 566f0e8c..7d3c6a66 100644
--- a/src/zeroconf/_handlers/record_manager.py
+++ b/src/zeroconf/_handlers/record_manager.py
@@ -214,8 +214,6 @@ def async_remove_listener(self, listener: RecordUpdateListener) -> None:
 
         This function is not threadsafe and must be called in the eventloop.
         """
-        try:
-            self.listeners.remove(listener)
+        if listener in self.listeners:
+            self.listeners.discard(listener)
             self.zc.async_notify_all()
-        except ValueError as e:
-            log.exception("Failed to remove listener: %r", e)
diff --git a/tests/test_handlers.py b/tests/test_handlers.py
index 69f3c826..4cdc44dc 100644
--- a/tests/test_handlers.py
+++ b/tests/test_handlers.py
@@ -2107,3 +2107,17 @@ def async_update_records_complete(self) -> None:
     # This should not raise RuntimeError: set changed size during iteration
     zc.record_manager.async_updates_complete(False)
     await aiozc.async_close()
+
+
+@pytest.mark.asyncio
+async def test_async_remove_listener_missing_does_not_raise():
+    """Removing a listener that was never registered must not raise."""
+
+    aiozc = AsyncZeroconf(interfaces=["127.0.0.1"])
+    zc: Zeroconf = aiozc.zeroconf
+
+    listener = r.RecordUpdateListener()
+
+    zc.record_manager.async_remove_listener(listener)
+
+    await aiozc.async_close()