More IfcDiff optimisations (461s->247s), bump DeepDiff version, less false positives with more accurate number comparisons

Moult · Moult · commit a4d0378405ce · 2022-08-04T17:49:18.000+10:00
diff --git a/src/blenderbim/Makefile b/src/blenderbim/Makefile
@@ -210,9 +210,9 @@ endif
 
 	# Required by IFCDiff
 	mkdir dist/working
-	cd dist/working && wget https://github.com/Moult/deepdiff/archive/master.zip
-	cd dist/working && unzip master.zip
-	cp -r dist/working/deepdiff-master/deepdiff dist/blenderbim/libs/site/packages/
+	cd dist/working && wget https://files.pythonhosted.org/packages/0f/ca/caead2949fbb824c7142e3774fa841aa853bb4d4331b440da8c8514dfc6f/deepdiff-5.8.1.tar.gz
+	cd dist/working && tar -xzvf deepdiff*
+	cp -r dist/working/deepdiff-5.8.1/deepdiff dist/blenderbim/libs/site/packages/
 	rm -rf dist/working
 
 	# Required by deepdiff
diff --git a/src/ifcdiff/ifcdiff.py b/src/ifcdiff/ifcdiff.py
@@ -20,27 +20,32 @@
 
 # This can be packaged with `pyinstaller --onefile --clean --icon=icon.ico ifcdiff.py`
 
-import ifcopenshell
-from deepdiff import DeepDiff
 import time
 import json
+import logging
 import argparse
-import decimal
+import numpy as np
+import ifcopenshell
+import ifcopenshell.util.element
+import ifcopenshell.util.placement
+import ifcopenshell.util.classification
+from deepdiff import DeepDiff
 
 
 class IfcDiff:
-    def __init__(self, old_file, new_file, output_file, inverse_classes=None, is_shallow=False):
+    def __init__(self, old_file, new_file, output_file, relationships=None, is_shallow=True):
         self.old_file = old_file
         self.new_file = new_file
         self.output_file = output_file
         self.change_register = {}
-        self.representation_ids = set()
-        self.inverse_classes = inverse_classes
-        self.precision = 2
+        self.representation_ids = {}
+        self.relationships = relationships
+        self.precision = 1e-4
         self.is_shallow = is_shallow
 
     def diff(self):
         print("# IFC Diff")
+        logging.disable(logging.CRITICAL)
         self.load()
 
         self.precision = self.get_precision()
@@ -64,20 +69,19 @@ def diff(self):
             total_diffed += 1
             if total_diffed % 250 == 0:
                 print("{}/{} diffed ...".format(total_diffed, total_same_elements), end="\r", flush=True)
-            old_element = self.old.by_id(global_id)
-            new_element = self.new.by_id(global_id)
-            if self.diff_element(old_element, new_element) and self.is_shallow:
+            old = self.old.by_id(global_id)
+            new = self.new.by_id(global_id)
+            if self.diff_element(old, new) and self.is_shallow:
                 continue
-            if self.diff_element_inverse_relationships(old_element, new_element) and self.is_shallow:
+            if self.diff_element_relationships(old, new) and self.is_shallow:
                 continue
-            representation_id = self.get_representation_id(new_element)
-            if representation_id in self.representation_ids:
-                continue
-            self.representation_ids.add(representation_id)
-            self.diff_element_geometry(old_element, new_element)
+            diff = self.diff_element_geometry(old, new)
+            if diff:
+                self.change_register.setdefault(new.GlobalId, {}).update({"geometry_changed": True})
 
         print(" - {} item(s) were changed either geometrically or with data".format(len(self.change_register.keys())))
         print("# Diff finished in {:.2f} seconds".format(time.time() - start))
+        logging.disable(logging.NOTSET)
 
     def export(self):
         with open(self.output_file, "w", encoding="utf-8") as diff_file:
@@ -99,116 +103,123 @@ def load(self):
         self.new = ifcopenshell.open(self.new_file)
 
     def get_precision(self):
-        try:
-            precision = [c for c in self.new.by_type("IfcGeometricRepresentationContext") if c.ContextType == "Model"][
-                0
-            ].Precision
-            exponent = decimal.Decimal(str(precision)).as_tuple().exponent
-            if exponent < 0:
-                return abs(exponent)
-            return 0
-        except:
-            return 2
+        contexts = [c for c in self.new.by_type("IfcGeometricRepresentationContext") if c.ContextType == "Model"]
+        if contexts:
+            return contexts[0].Precision or 1e-4
+        return 1e-4
 
-    def diff_element(self, old_element, new_element):
+    def diff_element(self, old, new):
         diff = DeepDiff(
-            old_element,
-            new_element,
-            significant_digits=self.precision,
+            [a for a in old if not isinstance(a, (ifcopenshell.entity_instance, tuple))],
+            [a for a in new if not isinstance(a, (ifcopenshell.entity_instance, tuple))],
+            math_epsilon=self.precision,
             ignore_string_type_changes=True,
             ignore_numeric_type_changes=True,
-            exclude_regex_paths={
-                r"root.*id$",
-                r".*Representation.*",
-                r".*OwnerHistory.*",
-                r".*ObjectPlacement.*",
-            },
         )
-        if diff and new_element.GlobalId:
-            self.change_register.setdefault(new_element.GlobalId, {}).update(diff)
+        if diff and new.GlobalId:
+            self.change_register.setdefault(new.GlobalId, {}).update({"attributes_changed": True})
             return True
 
-    def diff_element_inverse_relationships(self, old_element, new_element):
-        if not self.inverse_classes:
+    def diff_element_relationships(self, old, new):
+        if not self.relationships:
             return
-        old_relationships_all = self.old.get_inverse(old_element)
-        new_relationships_all = self.new.get_inverse(new_element)
-        if self.inverse_classes[0] == "all":
-            old_relationships = old_relationships_all
-            new_relationships = new_relationships_all
-        else:
-            old_relationships = [x for x in old_relationships_all if x.is_a() in self.inverse_classes]
-            new_relationships = [x for x in new_relationships_all if x.is_a() in self.inverse_classes]
+        for relationship in self.relationships:
+            if relationship == "type":
+                if ifcopenshell.util.element.get_type(old) != ifcopenshell.util.element.get_type(new):
+                    self.change_register.setdefault(new.GlobalId, {}).update({"type_changed": True})
+                    return True
+            elif relationship == "property":
+                old_psets = ifcopenshell.util.element.get_psets(old)
+                new_psets = ifcopenshell.util.element.get_psets(new)
+                try:
+                    diff = DeepDiff(
+                        old_psets,
+                        new_psets,
+                        math_epsilon=self.precision,
+                        ignore_string_type_changes=True,
+                        ignore_numeric_type_changes=True,
+                        exclude_regex_paths=[r".*id$"],
+                    )
+                except:
+                    diff = True
+                if diff and new.GlobalId:
+                    self.change_register.setdefault(new.GlobalId, {}).update({"properties_changed": diff})
+                    return True
+            elif relationship == "container":
+                if ifcopenshell.util.element.get_container(old) != ifcopenshell.util.element.get_container(new):
+                    self.change_register.setdefault(new.GlobalId, {}).update({"container_changed": True})
+                    return True
+            elif relationship == "aggregate":
+                if ifcopenshell.util.element.get_aggregate(old) != ifcopenshell.util.element.get_aggregate(new):
+                    self.change_register.setdefault(new.GlobalId, {}).update({"aggregate_changed": True})
+                    return True
+            elif relationship == "classification":
+                old_id = "ItemReference" if self.old.schema == "IFC2X3" else "Identification"
+                new_id = "ItemReference" if self.new.schema == "IFC2X3" else "Identification"
+                old_refs = [getattr(r, old_id) for r in ifcopenshell.util.classification.get_references(old)]
+                new_refs = [getattr(r, new_id) for r in ifcopenshell.util.classification.get_references(new)]
+                if old_refs != new_refs:
+                    self.change_register.setdefault(new.GlobalId, {}).update({"classification_changed": True})
+                    return True
 
-        diff = DeepDiff(
-            old_relationships,
-            new_relationships,
-            significant_digits=self.precision,
-            ignore_string_type_changes=True,
-            ignore_numeric_type_changes=True,
-            exclude_regex_paths=[
-                r"root.*id$",
-                r".*GlobalId.*",
-                r".*OwnerHistory.*",
-                r".*RelatedObjects.*",
-                r".*RelatingObject.*",
-                r".*RelatingDefinitions.*",
-                r".*RelatedObjectsType.*",  # Deprecated in IFC4 anyway
-            ],
-        )
-        if diff and new_element.GlobalId:
-            self.change_register.setdefault(new_element.GlobalId, {}).update(diff)
+    def diff_element_geometry(self, old, new):
+        old_placement = ifcopenshell.util.placement.get_local_placement(old.ObjectPlacement)
+        new_placement = ifcopenshell.util.placement.get_local_placement(new.ObjectPlacement)
+        if not np.allclose(old_placement[:,3], new_placement[:,3], atol=self.precision):
+            return True
+        if not np.allclose(old_placement[0:3,0:3], new_placement[0:3,0:3], atol=1e-2):
+            return True
+        old_openings = [o.RelatedOpeningElement.GlobalId for o in getattr(old, "HasOpenings", []) or []]
+        new_openings = [o.RelatedOpeningElement.GlobalId for o in getattr(new, "HasOpenings", []) or []]
+        if old_openings != new_openings:
+            return True
+        old_projections = [o.RelatedFeatureElement.GlobalId for o in getattr(old, "HasProjections", []) or []]
+        new_projections = [o.RelatedFeatureElement.GlobalId for o in getattr(new, "HasProjections", []) or []]
+        if old_projections != new_projections:
             return True
+        old_rep_id = self.get_representation_id(old)
+        new_rep_id = self.get_representation_id(new)
+        rep_result = self.representation_ids.get(new_rep_id, None)
+        if rep_result is not None:
+            return rep_result
+        if type(old_rep_id) != type(new_rep_id):
+            self.representation_ids[new_rep_id] = True
+            return True
+        if new_rep_id is None:
+            return
+        result = self.diff_representation(old_rep_id, new_rep_id) or False
+        self.representation_ids[new_rep_id] = result
+        return result
+
+    def diff_representation(self, old_rep_id, new_rep_id):
+        old_rep = self.old.by_id(old_rep_id)
+        new_rep = self.new.by_id(new_rep_id)
+        if len(old_rep.Items) != len(new_rep.Items):
+            return True
+        for i, old_item in enumerate(old_rep.Items):
+            result = self.diff_representation_item(old_item, new_rep.Items[i])
+            if result is True:
+                return True
 
-    def diff_element_geometry(self, old_element, new_element):
+    def diff_representation_item(self, old_item, new_item):
+        if old_item.is_a() != new_item.is_a():
+            return True
         try:
-            DeepDiff(
-                old_element.ObjectPlacement,
-                new_element.ObjectPlacement,
-                terminate_on_first=True,
-                significant_digits=self.precision,
-                ignore_string_type_changes=True,
-                ignore_numeric_type_changes=True,
-                exclude_regex_paths=r"root.*id$",
-            )
-            DeepDiff(
-                old_element.HasOpenings,
-                new_element.HasOpenings,
-                terminate_on_first=True,
-                significant_digits=self.precision,
-                ignore_string_type_changes=True,
-                ignore_numeric_type_changes=True,
-                exclude_regex_paths=r"root.*id$",
-            )
-            DeepDiff(
-                old_element.HasProjections,
-                new_element.HasProjections,
-                terminate_on_first=True,
-                significant_digits=self.precision,
-                ignore_string_type_changes=True,
-                ignore_numeric_type_changes=True,
-                exclude_regex_paths=r"root.*id$",
-            )
-            DeepDiff(
-                old_element.Representation.get_info_2(recursive=True),
-                new_element.Representation.get_info_2(recursive=True),
-                terminate_on_first=True,
-                skip_after_n=500,  # Arbitrary value to "skim" check
-                significant_digits=self.precision,
-                ignore_string_type_changes=True,
-                ignore_numeric_type_changes=True,
-                exclude_regex_paths=[
-                    r"root.*id']$",
-                    r".*ContextOfItems.*",
-                ],
+            diff = DeepDiff(
+                old_item.get_info_2(recursive=True),
+                new_item.get_info_2(recursive=True),
+                custom_operators=[DiffTerminator()] if self.is_shallow else [],
+                math_epsilon=self.precision,
+                exclude_regex_paths=[r".*id']$"]
             )
         except:
-            if new_element.GlobalId:
-                return self.change_register.setdefault(new_element.GlobalId, {}).update({"has_geometry_change": True})
+            return True
+        if diff:
+            return True
 
     def get_representation_id(self, element):
         if not element.Representation:
-            return None
+            return
         for representation in element.Representation.Representations:
             if not representation.is_a("IfcShapeRepresentation"):
                 continue
@@ -221,6 +232,15 @@ def get_representation_id(self, element):
                 return representation.Items[0].MappingSource.MappedRepresentation.id()
 
 
+class DiffTerminator:
+    def match(self, level) -> bool:
+        return True
+
+    def give_up_diffing(self, level, diff_instance) -> bool:
+        if any(diff_instance.tree.values()):
+            raise Exception("Terminated")
+
+
 class DiffEncoder(json.JSONEncoder):
     def default(self, obj):
         try:
@@ -240,7 +260,7 @@ def default(self, obj):
         "-r",
         "--relationships",
         type=str,
-        help='A list of IFC classes to check in inverse relationships, like "IfcRelDefinesByProperties", or "all".',
+        help='A list of space-separated relationships, chosen from "type", "property", "container", "aggregate", "classification"',
         default="",
     )
     args = parser.parse_args()
