Require an explicit delete_missing spec key to delete anything

matthiask · matthiask · commit 189c1bed491d · 2022-02-20T10:49:28.000+01:00
diff --git a/README.rst b/README.rst
@@ -109,13 +109,15 @@ Model specs consist of the following fields:
   objects to remove after loading.
 - ``"force_insert"``: If present and truish, objects are inserted using new
   primary keys into the database instead of (potentially) overwriting
-  pre-existing objects. This flag also makes the loader **not** delete objects
-  it doesn't know about.
+  pre-existing objects.
+- ``"delete_missing"``: This flag makes the loader delete all objects matching
+  ``"filter"`` which do not exist in the dump.
 
 The dumps can be loaded back into the database by running::
 
     ./manage.py f3loaddata -v2 tmp/dump.json tmp/districts.json
 
 Each dump is processed in an individual transaction. The data is first loaded
 into the database; at the end, data *matching* the filters but whose primary
-key wasn't contained in the dump is deleted from the database.
+key wasn't contained in the dump is deleted from the database (if
+``"delete_missing": True``).
diff --git a/feincms3_data/data.py b/feincms3_data/data.py
@@ -30,7 +30,7 @@ class InvalidSpec(Exception):
     pass
 
 
-_valid_keys = {"model", "filter", "force_insert"}
+_valid_keys = {"model", "filter", "force_insert", "delete_missing"}
 
 
 def _validate_spec(spec):
@@ -114,7 +114,7 @@ def load_dump(data, *, progress=silence, ignorenonexistent=False):
             progress(f"Saved {len(objs)} {spec['model']} objects")
 
         for spec in data["specs"]:
-            if spec.get("force_insert"):
+            if not spec.get("delete_missing"):
                 continue
 
             queryset = _model_queryset(spec)
diff --git a/tests/testapp/models.py b/tests/testapp/models.py
@@ -4,12 +4,16 @@
 class Parent(models.Model):
     name = models.CharField(default="name", max_length=20)
 
+    class Meta:
+        ordering = ["id"]
+
 
 class Child(models.Model):
     name = models.CharField(default="name", max_length=20)
 
     class Meta:
         abstract = True
+        ordering = ["id"]
 
 
 class Child1(Child):
diff --git a/tests/testapp/test_data.py b/tests/testapp/test_data.py
@@ -19,7 +19,7 @@
 def parent_child1_set():
     return [
         (p.name, [c.name for c in p.child1_set.all()])
-        for p in Parent.objects.order_by("id").prefetch_related("child1_set")
+        for p in Parent.objects.prefetch_related("child1_set")
     ]
 
 
@@ -53,14 +53,14 @@ def test_specs_for_derived_models(self):
         self.assertNotIn({"model": "testapp.child"}, specs)
 
     def test_specs_for_app_models(self):
-        specs = list(specs_for_app_models("testapp"))
+        specs = list(specs_for_app_models("testapp", {"delete_missing": True}))
 
         self.assertCountEqual(
             specs,
             [
-                {"model": "testapp.parent"},
-                {"model": "testapp.child1"},
-                {"model": "testapp.child2"},
+                {"model": "testapp.parent", "delete_missing": True},
+                {"model": "testapp.child1", "delete_missing": True},
+                {"model": "testapp.child2", "delete_missing": True},
             ],
         )
 
@@ -126,7 +126,7 @@ def test_force_insert_simple(self):
 
     def test_force_insert_partial_graph(self):
         p1 = Parent.objects.create(name="blub-1")
-        p1.child1_set.create(name="blub-1-1")
+        c1 = p1.child1_set.create(name="blub-1-1")
 
         p2 = Parent.objects.create(name="blub-2")
         p2.child1_set.create(name="blub-2-1")
@@ -143,7 +143,11 @@ def test_force_insert_partial_graph(self):
             ),
             *specs_for_derived_models(
                 Child,
-                {"filter": {"parent__in": [p1.pk]}, "force_insert": True},
+                {
+                    "filter": {"parent__in": [p1.pk]},
+                    "force_insert": True,
+                    "delete_missing": True,
+                },
             ),
         ]
 
@@ -152,9 +156,12 @@ def test_force_insert_partial_graph(self):
 
         self.assertEqual(
             parent_child1_set(),
-            [("blub-1", ["blub-1-1", "blub-1-1"]), ("blub-2", ["blub-2-1"])],
+            [("blub-1", ["blub-1-1"]), ("blub-2", ["blub-2-1"])],
         )
 
+        c1_new = p1.child1_set.get()
+        self.assertNotEqual(c1.pk, c1_new.pk)
+
     def test_force_insert_full_graph(self):
         Parent.objects.create(name="other")
 
@@ -169,15 +176,24 @@ def test_force_insert_full_graph(self):
         specs = [
             *specs_for_models(
                 [Parent],
-                {"filter": {"pk__in": [p1.pk]}, "force_insert": True},
+                {
+                    "filter": {"pk__in": [p1.pk]},
+                    "force_insert": True,
+                    "delete_missing": False,  # Do not remove old items
+                },
             ),
             *specs_for_derived_models(
                 Child,
-                {"filter": {"parent__in": [p1.pk]}, "force_insert": True},
+                {
+                    "filter": {"parent__in": [p1.pk]},
+                    "force_insert": True,
+                    "delete_missing": False,  # Do not remove old items
+                },
             ),
         ]
 
         dump = json.loads(dump_specs(specs))
+        # from pprint import pprint; print(); pprint(dump)
         load_dump(dump)
 
         self.assertEqual(
