From 5a65a0cf99a14dae2042072e87e83b37266e32ef Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Mon, 28 Aug 2023 15:05:57 -0600 Subject: [PATCH 01/23] Post init tests & general setup --- default/tests/shared/test_annotation.py | 2057 ++++++++++--------- default/tests/shared/test_annotation_new.py | 103 + shared/annotation.py | 13 +- 3 files changed, 1150 insertions(+), 1023 deletions(-) create mode 100644 default/tests/shared/test_annotation_new.py diff --git a/default/tests/shared/test_annotation.py b/default/tests/shared/test_annotation.py index edc346eb4..58e5bbb06 100755 --- a/default/tests/shared/test_annotation.py +++ b/default/tests/shared/test_annotation.py @@ -125,69 +125,69 @@ def test_update_sequence_id_in_cache_list(self): self.assertEqual(ann_update.instance_list_kept_serialized[0]['sequence_id'], sequence.id) - def test_duplicate_instance_update_existing_false(self): - """ - We send duplicate instances with update_existing = False. - Expect: Just one of the instances should be saved. - :return: - """ + def test_detect_and_remove_collisions(self): label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) frame = data_mocking.create_file( {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, self.session) - inst1 = { - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } - inst2 = inst1.copy() - self.project.label_dict['label_file_id_list'] = [label_file.id] + instance1 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance2 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance3 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance1.hash_instance() + instance2.hash_instance() + instance3.hash_instance() video_data = { 'video_mode': True, 'video_file_id': file1.id, 'current_frame': frame.frame_number } + inst_list = [instance1, instance2, instance3] ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = [inst1, inst2], + instance_list_new = [], file = file1, - do_init_existing_instances = False + do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) + result = ann_update.detect_and_remove_collisions(inst_list) - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(ann_update.duplicate_hash_new_instance_list), 1) - self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].x_min, inst1['x_min']) - self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].x_max, inst1['x_max']) - self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].y_min, inst1['y_min']) - self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].y_max, inst1['y_max']) - self.assertEqual(len(deleted_instances), 0) + self.assertEqual(len(result), 1) + self.assertEqual(result[0], instance3) - def test_overlap_existing_instances(self): + def test_update_cache_single_instance_in_list_context(self): """ - 2 instances with ids are in different position and then one is placed on the exact position as the other one - Expect: new overlapped instance is soft deleted and original instance is preserved in cache. + Test that the instance gets serialized correctly and that if the instance has no ID + the function does not serializer anything. :return: """ - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) frame = data_mocking.create_file( {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, self.session) - inst1 = { + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + # 2 Exactly equal instances + instance1 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance2 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'soft_delete': True, + 'label_file_id': label_file.id}, + self.session + ) + self.project.label_dict['label_file_id_list'] = [label_file.id] + inst = { 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, @@ -197,102 +197,54 @@ def test_overlap_existing_instances(self): 'label_file_id': label_file.id, 'type': 'box' } - inst2 = { - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 5, - 'y_min': 5, - 'x_max': 55, - 'y_max': 55, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } - - self.project.label_dict['label_file_id_list'] = [label_file.id] + instance_data = [ + inst.copy(), + inst.copy() + ] video_data = { 'video_mode': True, 'video_file_id': file1.id, 'current_frame': frame.frame_number } - ann_update = Annotation_Update( - session = self.session, - project = self.project, - video_data = video_data, - instance_list_new = [inst1, inst2], - file = file1, - do_init_existing_instances = True - ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances - self.assertEqual(len(added_instances), 2) - self.assertEqual(len(new_instance_list), 2) - self.assertEqual(len(deleted_instances), 0) - - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertIsNotNone(new_instance_list[0]['id']) - - self.assertEqual(new_instance_list[1]['x_min'], inst2['x_min']) - self.assertEqual(new_instance_list[1]['y_min'], inst2['y_min']) - self.assertEqual(new_instance_list[1]['x_max'], inst2['x_max']) - self.assertEqual(new_instance_list[1]['y_max'], inst2['y_max']) - self.assertFalse(new_instance_list[1]['soft_delete']) - self.assertIsNotNone(new_instance_list[0]['id']) - # 2. Now place one instance on top of the other one - inst1['id'] = new_instance_list[0]['id'] - inst2['id'] = new_instance_list[1]['id'] - inst2['x_min'] = inst1['x_min'] - inst2['x_max'] = inst1['x_max'] - inst2['y_min'] = inst1['y_min'] - inst2['y_max'] = inst1['y_max'] ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = [inst1, inst2], + instance_list_new = instance_data, file = file1, do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances + with patch.object(instance1, 'serialize_with_label') as mock_1: + ann_update.instance = instance1 + ann_update.update_cache_single_instance_in_list_context() + mock_1.assert_called_once() - self.assertEqual(len(added_instances), 0) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(deleted_instances), 1) - self.assertEqual(deleted_instances[0], inst2['id']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertIsNotNone(new_instance_list[0]['id']) - self.assertEqual(new_instance_list[0]['id'], inst1['id']) - self.assertNotEqual(new_instance_list[0]['id'], inst2['id']) - updated_inst1 = Instance.get_by_id(self.session, instance_id = inst1['id']) - updated_inst2 = Instance.get_by_id(self.session, instance_id = inst2['id']) - self.assertFalse(updated_inst1.soft_delete) - self.assertTrue(updated_inst2.soft_delete) + with patch.object(instance2, 'serialize_with_label') as mock_1: + instance2.id = None + ann_update.instance = instance2 + ann_update.update_cache_single_instance_in_list_context() + self.assertEqual(mock_1.call_count, 0) - def test_update_move_and_undo_case(self): - """ - We create an instance, move it, save it, undo it, save it and redoit and save again - Expect:File cache should have the newest instance as this was the latests version of it - :return: - """ - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + def test_append_new_instance_list_hash(self): file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) frame = data_mocking.create_file( {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, self.session) - inst1 = { + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + # 2 Exactly equal instances + instance1 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance2 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'soft_delete': True, + 'label_file_id': label_file.id}, + self.session + ) + self.project.label_dict['label_file_id_list'] = [label_file.id] + inst = { 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, @@ -302,430 +254,326 @@ def test_update_move_and_undo_case(self): 'label_file_id': label_file.id, 'type': 'box' } - - self.project.label_dict['label_file_id_list'] = [label_file.id] + instance_data = [ + inst.copy(), + inst.copy() + ] video_data = { 'video_mode': True, 'video_file_id': file1.id, 'current_frame': frame.frame_number } + ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = [inst1], + instance_list_new = instance_data, file = file1, do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances - - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(deleted_instances), 0) + result = ann_update.append_new_instance_list_hash(instance = instance1) + result2 = ann_update.append_new_instance_list_hash(instance = instance2) - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertIsNotNone(new_instance_list[0]['id']) - old_id = int(new_instance_list[0]['id']) + self.assertTrue(result) + self.assertFalse(result2) - # 2. Move The Instance + def test_order_new_instances_by_date(self): + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) + frame = data_mocking.create_file( + {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, + self.session) + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) inst1 = { - 'id': old_id, 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'client_created_time': datetime.datetime(2020, 1, 1), 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } - ann_update = Annotation_Update( - session = self.session, - project = self.project, - video_data = video_data, - instance_list_new = [inst1], - file = file1, - do_init_existing_instances = True - ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances - - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(deleted_instances), 1) - self.assertEqual(deleted_instances[0], old_id) - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertNotEqual(new_instance_list[0]['id'], old_id) - moved_id = int(new_instance_list[0]['id']) - # 3. Undo the instance - inst_undone = { - 'id': moved_id, - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, - 'soft_delete': True, - 'label_file_id': label_file.id, - 'type': 'box' - } - inst1 = { - 'id': old_id, + inst2 = { 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, 'x_max': 18, 'y_max': 18, + 'client_created_time': datetime.datetime(2020, 1, 2), 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } - ann_update = Annotation_Update( - session = self.session, - project = self.project, - video_data = video_data, - instance_list_new = [inst_undone, inst1], - file = file1, - do_init_existing_instances = True - ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances - - self.assertEqual(len(added_instances), 2) - self.assertEqual(len(new_instance_list), 2) - self.assertEqual(len(deleted_instances), 1) - self.assertEqual(deleted_instances[0], inst_undone['id']) - self.assertEqual(new_instance_list[0]['x_min'], inst_undone['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst_undone['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst_undone['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst_undone['y_max']) - self.assertNotEqual(inst_undone['id'], new_instance_list[0]['id']) - self.assertNotEqual(new_instance_list[0]['id'], old_id) - self.assertTrue(new_instance_list[0]['soft_delete']) - - self.assertEqual(new_instance_list[1]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[1]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[1]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[1]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[1]['soft_delete']) - self.assertNotEqual(new_instance_list[1]['id'], old_id) - newest_id = int(new_instance_list[1]['id']) - deleted_id = int(new_instance_list[0]['id']) - # 4. Redo the instance - inst_redone = { - 'id': deleted_id, + inst3 = { 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'client_created_time': datetime.datetime(2020, 1, 3), 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } - inst1 = { - 'id': newest_id, + inst4 = { 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, 'x_max': 18, 'y_max': 18, - 'soft_delete': True, + 'client_created_time': None, + 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } + inst_list = [inst1, inst2, inst3, inst4] ann_update = Annotation_Update( session = self.session, project = self.project, - video_data = video_data, - instance_list_new = [inst_redone, inst1], + video_data = None, + instance_list_new = inst_list, file = file1, do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) + ann_update.instance_list_new = inst_list + ann_update.order_new_instances_by_date() - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances + self.assertEqual(ann_update.instance_list_new[0], inst3) + self.assertEqual(ann_update.instance_list_new[1], inst2) + self.assertEqual(ann_update.instance_list_new[2], inst1) + self.assertEqual(ann_update.instance_list_new[3], inst4) - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 2) - self.assertEqual(len(deleted_instances), 1) - self.assertEqual(deleted_instances[0], newest_id) - self.assertEqual(new_instance_list[0]['x_min'], inst_redone['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst_redone['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst_redone['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst_redone['y_max']) - self.assertNotEqual(inst_redone['id'], new_instance_list[0]['id']) - self.assertNotEqual(new_instance_list[0]['id'], deleted_id) - self.assertFalse(new_instance_list[0]['soft_delete']) + def test__check_all_instances_available_in_new_instance_list(self): + file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) + label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) + instance1 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance2 = data_mocking.create_instance( + {'x_min': 2, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) - def test_update_on_duplicate_instance_undo_case(self): - """ - We send 2 instances that are equal, one with an ID and the other one with no ID - expect: Instance with ID should be existing and new instance should be ignored - :return: - """ - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) - frame = data_mocking.create_file( - {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, - self.session) - inst1 = { - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } + instance3 = data_mocking.create_instance( + {'x_min': 3, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, + self.session + ) + instance1.hash_instance() + instance2.hash_instance() + instance3.hash_instance() + old_payload = [instance1, instance2, instance3] + new_list_payload = [x.serialize_with_label() for x in old_payload] + new_list_payload_wrong = [instance1.serialize_with_label()] - self.project.label_dict['label_file_id_list'] = [label_file.id] - video_data = { - 'video_mode': True, - 'video_file_id': file1.id, - 'current_frame': frame.frame_number - } + # Test Case where we don't want to run verification ann_update = Annotation_Update( session = self.session, project = self.project, - video_data = video_data, - instance_list_new = [inst1], + instance_list_new = new_list_payload, file = file1, - do_init_existing_instances = True + do_init_existing_instances = False ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) + result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances + self.assertTrue(result) - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(deleted_instances), 0) + # Now test case with validations + ann_update = Annotation_Update( + session = self.session, + project = self.project, + instance_list_new = new_list_payload, + file = file1, + do_init_existing_instances = True + ) + result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertIsNotNone(new_instance_list[0]['id']) - old_id = int(new_instance_list[0]['id']) + self.assertTrue(result) - # 2. Move The Instance - inst1 = { - 'id': old_id, - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } + # Now test case with validations and a wrong payload ann_update = Annotation_Update( session = self.session, project = self.project, - video_data = video_data, - instance_list_new = [inst1], + instance_list_new = new_list_payload_wrong, file = file1, do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) + result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances + self.assertTrue(result) + self.assertTrue(len(ann_update.log['warning'].keys()) > 0) + self.assertTrue('new_instance_list_missing_ids' in ann_update.log['warning']) + self.assertTrue('information' in ann_update.log['warning']) + self.assertTrue('missing_ids' in ann_update.log['warning']) + self.assertTrue(instance2.id in ann_update.log['warning']['missing_ids']) + self.assertTrue(instance3.id in ann_update.log['warning']['missing_ids']) - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(deleted_instances), 1) - self.assertEqual(deleted_instances[0], old_id) - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertNotEqual(new_instance_list[0]['id'], old_id) - # 3. Undo the instance - inst_undone = { - 'id': new_instance_list[0]['id'], - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, - 'soft_delete': True, - 'label_file_id': label_file.id, - 'type': 'box' - } - inst1 = { - 'id': old_id, - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } + # Now test case with validations and a wrong payload and some existing deleted instances + instance4 = data_mocking.create_instance( + {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'soft_delete': True, + 'label_file_id': label_file.id}, + self.session + ) ann_update = Annotation_Update( session = self.session, project = self.project, - video_data = video_data, - instance_list_new = [inst_undone, inst1], + instance_list_new = new_list_payload_wrong, file = file1, do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - - new_instance_list = updated_frame_file.cache_dict['instance_list'] - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances + result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() - self.assertEqual(len(added_instances), 2) - self.assertEqual(len(new_instance_list), 2) - self.assertEqual(len(deleted_instances), 1) - self.assertEqual(deleted_instances[0], inst_undone['id']) - self.assertEqual(new_instance_list[0]['x_min'], inst_undone['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst_undone['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst_undone['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst_undone['y_max']) - self.assertNotEqual(inst_undone['id'], new_instance_list[0]['id']) - self.assertNotEqual(new_instance_list[0]['id'], old_id) - self.assertTrue(new_instance_list[0]['soft_delete']) - - self.assertEqual(new_instance_list[1]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[1]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[1]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[1]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[1]['soft_delete']) + self.assertTrue(result) + self.assertTrue(len(ann_update.log['warning'].keys()) > 0) + self.assertTrue('new_instance_list_missing_ids' in ann_update.log['warning']) + self.assertTrue('information' in ann_update.log['warning']) + self.assertTrue('missing_ids' in ann_update.log['warning']) + self.assertTrue(instance2.id in ann_update.log['warning']['missing_ids']) + self.assertTrue(instance3.id in ann_update.log['warning']['missing_ids']) + self.assertTrue(instance4.id not in ann_update.log['warning']['missing_ids']) - def test_update_on_duplicate_instance_1_id_1_no_id(self): + def test_check_relations_id_existence(self): """ - We send 2 instances that are equal, one with an ID and the other one with no ID - expect: Instance with ID should be existing and new instance should be ignored + Check calls to check_relations_id_existence :return: """ - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'image'}, self.session) frame = data_mocking.create_file( {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, self.session) + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + # 2 Exactly equal instances + self.project.label_dict['label_file_id_list'] = [label_file.id] instance1 = data_mocking.create_instance( - {'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'x_max': 18, 'y_min': 1, 'y_max': 18, - 'file_id': file1.id, 'label_file_id': label_file.id}, + {'x_min': 1, + 'x_max': 10, + 'y_min': 1, + 'y_max': 10, + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'type': 'relation' + }, self.session ) + instance2 = data_mocking.create_instance( + {'x_min': 2, + 'x_max': 15, + 'y_min': 2, + 'y_max': 15, + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'type': 'box' + }, + self.session + ) + ann_update2 = Annotation_Update( + session = self.session, + project = self.project, + instance_list_new = [], + file = file1, + do_init_existing_instances = True + ) - inst1 = { - 'id': instance1.id, # This one has the ID - 'creation_ref_id': instance1.creation_ref_id, - 'x_min': instance1.x_min, - 'y_min': instance1.y_min, - 'x_max': instance1.x_max, - 'y_max': instance1.y_max, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } - - inst2 = { - 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, - 'soft_delete': False, - 'label_file_id': label_file.id, - 'type': 'box' - } + ann_update2.instance = instance1 + # Check Error State + ann_update2.check_relations_id_existence(None, None, None, None) + self.assertTrue('from_id' in ann_update2.log['error']) + # Check IDs available + ann_update2.log['error'] = {} + ann_update2.instance.from_instance_id = 1 + ann_update2.instance.to_instance_id = 1 + ann_update2.check_relations_id_existence(ann_update2.instance.from_instance_id, + ann_update2.instance.to_instance_id, + None, + None) + self.assertEqual(len(ann_update2.log['error'].keys()), 0) + self.assertEqual(len(ann_update2.new_instance_relations_list_no_ids), 0) + # Check IDs Not available + ann_update2.check_relations_id_existence(None, + None, + str(uuid.uuid4()), + str(uuid.uuid4())) + self.assertEqual(len(ann_update2.log['error'].keys()), 0) + self.assertEqual(len(ann_update2.new_instance_relations_list_no_ids), 1) + def test_add_missing_ids_to_new_relations(self): + """ + Tests calss to add_missing_ids_to_new_relations() + :return: + """ + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'image'}, self.session) + frame = data_mocking.create_file( + {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, + self.session) + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + # 2 Exactly equal instances self.project.label_dict['label_file_id_list'] = [label_file.id] - video_data = { - 'video_mode': True, - 'video_file_id': file1.id, - 'current_frame': frame.frame_number - } + instance1 = data_mocking.create_instance( + {'x_min': 1, + 'x_max': 10, + 'y_min': 1, + 'y_max': 10, + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'type': 'relation' + }, + self.session + ) + instance1.creation_ref_id = str(uuid.uuid4()) + instance2 = data_mocking.create_instance( + {'x_min': 2, + 'x_max': 15, + 'y_min': 2, + 'y_max': 15, + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'type': 'box' + }, + self.session + ) + relation = data_mocking.create_instance( + { + 'file_id': file1.id, + 'label_file_id': None, + 'type': 'relation' + }, + self.session + ) + relation.hash_instance() + old_hash = relation.hash + + instance2.creation_ref_id = str(uuid.uuid4()) + self.session.commit() ann_update = Annotation_Update( session = self.session, project = self.project, - video_data = video_data, - instance_list_new = [inst1, inst2], + instance_list_new = [], file = file1, do_init_existing_instances = True ) - updated_file = ann_update.annotation_update_main() - updated_frame_file = File.get_by_id(self.session, frame.id) - - new_instance_list = updated_frame_file.cache_dict['instance_list'] - - deleted_instances = ann_update.new_deleted_instances - added_instances = ann_update.new_added_instances - - self.assertEqual(len(added_instances), 1) - self.assertEqual(len(new_instance_list), 1) - self.assertEqual(len(deleted_instances), 0) - - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(new_instance_list[0]['soft_delete']) - self.assertNotEqual(new_instance_list[0]['id'], instance1.id) - - # Test with cache regenerated too - updated_frame_file.set_cache_key_dirty(cache_key = 'instance_list') - regenerated_instance_list = updated_frame_file.get_with_cache( - cache_key = 'instance_list', - cache_miss_function = updated_frame_file.serialize_instance_list_only, - session = self.session) - - self.assertEqual(len(regenerated_instance_list), 1) + ann_update.new_added_instances = [instance1, instance2] + ann_update.new_instance_relations_list_no_ids = [{'instance': relation, + 'from_ref': instance1.creation_ref_id, + 'to_ref': instance2.creation_ref_id}] + ann_update.add_missing_ids_to_new_relations() - self.assertEqual(regenerated_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(regenerated_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(regenerated_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(regenerated_instance_list[0]['y_max'], inst1['y_max']) - self.assertFalse(regenerated_instance_list[0]['soft_delete']) - self.assertNotEqual(regenerated_instance_list[0]['id'], instance1.id) + self.assertEqual(relation.from_instance_id, instance1.id) + self.assertEqual(relation.to_instance_id, instance2.id) + self.assertNotEqual(old_hash, relation.hash) - deleted_instance = Instance.get_by_id(session = self.session, instance_id = instance1.id) - self.assertFalse(deleted_instance.soft_delete) + """ + ====================================================================================================================== + Everything below here is for large tests validation overall functionality (mainly annotation_update_main). + These all need to be heavily refactored, mocked and broken down. + ====================================================================================================================== + """ - def test_update_on_duplicate_instance_no_ids(self): + def test_duplicate_instance_update_existing_false(self): """ - We send 2 instances that are equal and dont have ID - expect: File cache should just have one of them, other one should be ignored + We send duplicate instances with update_existing = False. + Expect: Just one of the instances should be saved. :return: """ label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) @@ -756,7 +604,7 @@ def test_update_on_duplicate_instance_no_ids(self): video_data = video_data, instance_list_new = [inst1, inst2], file = file1, - do_init_existing_instances = True + do_init_existing_instances = False ) updated_file = ann_update.annotation_update_main() updated_frame_file = File.get_by_id(self.session, frame.id) @@ -764,35 +612,19 @@ def test_update_on_duplicate_instance_no_ids(self): new_instance_list = updated_frame_file.cache_dict['instance_list'] deleted_instances = ann_update.new_deleted_instances added_instances = ann_update.new_added_instances - self.assertEqual(len(added_instances), 1) self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(ann_update.duplicate_hash_new_instance_list), 1) + self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].x_min, inst1['x_min']) + self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].x_max, inst1['x_max']) + self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].y_min, inst1['y_min']) + self.assertEqual(ann_update.duplicate_hash_new_instance_list[0].y_max, inst1['y_max']) self.assertEqual(len(deleted_instances), 0) - self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) - self.assertIsNotNone(new_instance_list[0]['id']) - - # Test with cache regenerated too - updated_frame_file.set_cache_key_dirty(cache_key = 'instance_list') - regenerated_instance_list = updated_frame_file.get_with_cache( - cache_key = 'instance_list', - cache_miss_function = updated_frame_file.serialize_instance_list_only, - session = self.session) - - self.assertEqual(len(regenerated_instance_list), 1) - - self.assertEqual(regenerated_instance_list[0]['x_min'], inst1['x_min']) - self.assertEqual(regenerated_instance_list[0]['y_min'], inst1['y_min']) - self.assertEqual(regenerated_instance_list[0]['x_max'], inst1['x_max']) - self.assertEqual(regenerated_instance_list[0]['y_max'], inst1['y_max']) - - def test_update_non_duplicate_instances_no_ids(self): + def test_overlap_existing_instances(self): """ - We send a list of non duplicate instances with no IDs - expect: File cache should have all those instances with new ID's + 2 instances with ids are in different position and then one is placed on the exact position as the other one + Expect: new overlapped instance is soft deleted and original instance is preserved in cache. :return: """ label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) @@ -812,14 +644,15 @@ def test_update_non_duplicate_instances_no_ids(self): } inst2 = { 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 50, - 'y_min': 50, - 'x_max': 120, - 'y_max': 120, + 'x_min': 5, + 'y_min': 5, + 'x_max': 55, + 'y_max': 55, 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } + self.project.label_dict['label_file_id_list'] = [label_file.id] video_data = { 'video_mode': True, @@ -840,7 +673,6 @@ def test_update_non_duplicate_instances_no_ids(self): new_instance_list = updated_frame_file.cache_dict['instance_list'] deleted_instances = ann_update.new_deleted_instances added_instances = ann_update.new_added_instances - self.assertEqual(len(added_instances), 2) self.assertEqual(len(new_instance_list), 2) self.assertEqual(len(deleted_instances), 0) @@ -849,77 +681,63 @@ def test_update_non_duplicate_instances_no_ids(self): self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[0]['soft_delete']) self.assertIsNotNone(new_instance_list[0]['id']) self.assertEqual(new_instance_list[1]['x_min'], inst2['x_min']) self.assertEqual(new_instance_list[1]['y_min'], inst2['y_min']) self.assertEqual(new_instance_list[1]['x_max'], inst2['x_max']) self.assertEqual(new_instance_list[1]['y_max'], inst2['y_max']) - self.assertIsNotNone(new_instance_list[1]['id']) - - def test_detect_and_remove_collisions(self): - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) - frame = data_mocking.create_file( - {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, - self.session) - instance1 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance2 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance3 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance1.hash_instance() - instance2.hash_instance() - instance3.hash_instance() - video_data = { - 'video_mode': True, - 'video_file_id': file1.id, - 'current_frame': frame.frame_number - } - inst_list = [instance1, instance2, instance3] + self.assertFalse(new_instance_list[1]['soft_delete']) + self.assertIsNotNone(new_instance_list[0]['id']) + + # 2. Now place one instance on top of the other one + inst1['id'] = new_instance_list[0]['id'] + inst2['id'] = new_instance_list[1]['id'] + inst2['x_min'] = inst1['x_min'] + inst2['x_max'] = inst1['x_max'] + inst2['y_min'] = inst1['y_min'] + inst2['y_max'] = inst1['y_max'] ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = [], + instance_list_new = [inst1, inst2], file = file1, do_init_existing_instances = True ) - result = ann_update.detect_and_remove_collisions(inst_list) + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - self.assertEqual(len(result), 1) - self.assertEqual(result[0], instance3) + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - def test_update_cache_single_instance_in_list_context(self): + self.assertEqual(len(added_instances), 0) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 1) + self.assertEqual(deleted_instances[0], inst2['id']) + self.assertFalse(new_instance_list[0]['soft_delete']) + self.assertIsNotNone(new_instance_list[0]['id']) + self.assertEqual(new_instance_list[0]['id'], inst1['id']) + self.assertNotEqual(new_instance_list[0]['id'], inst2['id']) + updated_inst1 = Instance.get_by_id(self.session, instance_id = inst1['id']) + updated_inst2 = Instance.get_by_id(self.session, instance_id = inst2['id']) + self.assertFalse(updated_inst1.soft_delete) + self.assertTrue(updated_inst2.soft_delete) + + def test_update_move_and_undo_case(self): """ - Test that the instance gets serialized correctly and that if the instance has no ID - the function does not serializer anything. + We create an instance, move it, save it, undo it, save it and redoit and save again + Expect:File cache should have the newest instance as this was the latests version of it :return: """ + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) frame = data_mocking.create_file( {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, self.session) - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - # 2 Exactly equal instances - instance1 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance2 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'soft_delete': True, - 'label_file_id': label_file.id}, - self.session - ) - self.project.label_dict['label_file_id_list'] = [label_file.id] - inst = { + inst1 = { 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, @@ -929,169 +747,198 @@ def test_update_cache_single_instance_in_list_context(self): 'label_file_id': label_file.id, 'type': 'box' } - instance_data = [ - inst.copy(), - inst.copy() - ] + + self.project.label_dict['label_file_id_list'] = [label_file.id] video_data = { 'video_mode': True, 'video_file_id': file1.id, 'current_frame': frame.frame_number } - ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = instance_data, + instance_list_new = [inst1], file = file1, do_init_existing_instances = True ) + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - with patch.object(instance1, 'serialize_with_label') as mock_1: - ann_update.instance = instance1 - ann_update.update_cache_single_instance_in_list_context() - mock_1.assert_called_once() + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - with patch.object(instance2, 'serialize_with_label') as mock_1: - instance2.id = None - ann_update.instance = instance2 - ann_update.update_cache_single_instance_in_list_context() - self.assertEqual(mock_1.call_count, 0) + self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 0) - def test_append_new_instance_list_hash(self): - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) - frame = data_mocking.create_file( - {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, - self.session) - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - # 2 Exactly equal instances - instance1 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance2 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'soft_delete': True, - 'label_file_id': label_file.id}, - self.session - ) - self.project.label_dict['label_file_id_list'] = [label_file.id] - inst = { + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[0]['soft_delete']) + self.assertIsNotNone(new_instance_list[0]['id']) + old_id = int(new_instance_list[0]['id']) + + # 2. Move The Instance + inst1 = { + 'id': old_id, 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } - instance_data = [ - inst.copy(), - inst.copy() - ] - video_data = { - 'video_mode': True, - 'video_file_id': file1.id, - 'current_frame': frame.frame_number - } - ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = instance_data, + instance_list_new = [inst1], file = file1, do_init_existing_instances = True ) - result = ann_update.append_new_instance_list_hash(instance = instance1) - result2 = ann_update.append_new_instance_list_hash(instance = instance2) + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - self.assertTrue(result) - self.assertFalse(result2) + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - def test_order_new_instance_list_by_date(self): - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) - frame = data_mocking.create_file( - {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, - self.session) - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - inst1 = { + self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 1) + self.assertEqual(deleted_instances[0], old_id) + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[0]['soft_delete']) + self.assertNotEqual(new_instance_list[0]['id'], old_id) + moved_id = int(new_instance_list[0]['id']) + # 3. Undo the instance + inst_undone = { + 'id': moved_id, 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, - 'client_created_time': datetime.datetime(2020, 1, 1), - 'soft_delete': False, + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, + 'soft_delete': True, 'label_file_id': label_file.id, 'type': 'box' } - inst2 = { + inst1 = { + 'id': old_id, 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, 'x_max': 18, 'y_max': 18, - 'client_created_time': datetime.datetime(2020, 1, 2), 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } - inst3 = { + ann_update = Annotation_Update( + session = self.session, + project = self.project, + video_data = video_data, + instance_list_new = [inst_undone, inst1], + file = file1, + do_init_existing_instances = True + ) + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) + + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances + + self.assertEqual(len(added_instances), 2) + self.assertEqual(len(new_instance_list), 2) + self.assertEqual(len(deleted_instances), 1) + self.assertEqual(deleted_instances[0], inst_undone['id']) + self.assertEqual(new_instance_list[0]['x_min'], inst_undone['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst_undone['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst_undone['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst_undone['y_max']) + self.assertNotEqual(inst_undone['id'], new_instance_list[0]['id']) + self.assertNotEqual(new_instance_list[0]['id'], old_id) + self.assertTrue(new_instance_list[0]['soft_delete']) + + self.assertEqual(new_instance_list[1]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[1]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[1]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[1]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[1]['soft_delete']) + self.assertNotEqual(new_instance_list[1]['id'], old_id) + newest_id = int(new_instance_list[1]['id']) + deleted_id = int(new_instance_list[0]['id']) + # 4. Redo the instance + inst_redone = { + 'id': deleted_id, 'creation_ref_id': str(uuid.uuid4()), - 'x_min': 1, - 'y_min': 1, - 'x_max': 18, - 'y_max': 18, - 'client_created_time': datetime.datetime(2020, 1, 3), + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, 'soft_delete': False, 'label_file_id': label_file.id, 'type': 'box' } - inst4 = { + inst1 = { + 'id': newest_id, 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, 'x_max': 18, 'y_max': 18, - 'client_created_time': None, - 'soft_delete': False, + 'soft_delete': True, 'label_file_id': label_file.id, 'type': 'box' } - inst_list = [inst1, inst2, inst3, inst4] ann_update = Annotation_Update( session = self.session, project = self.project, - video_data = None, - instance_list_new = inst_list, + video_data = video_data, + instance_list_new = [inst_redone, inst1], file = file1, do_init_existing_instances = True ) - ann_update.instance_list_new = inst_list - ann_update.order_new_instances_by_date() + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - self.assertEqual(ann_update.instance_list_new[0], inst3) - self.assertEqual(ann_update.instance_list_new[1], inst2) - self.assertEqual(ann_update.instance_list_new[2], inst1) - self.assertEqual(ann_update.instance_list_new[3], inst4) + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - def test_special__removing_duplicate_instances_in_new_instance_list(self): + self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 2) + self.assertEqual(len(deleted_instances), 1) + self.assertEqual(deleted_instances[0], newest_id) + self.assertEqual(new_instance_list[0]['x_min'], inst_redone['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst_redone['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst_redone['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst_redone['y_max']) + self.assertNotEqual(inst_redone['id'], new_instance_list[0]['id']) + self.assertNotEqual(new_instance_list[0]['id'], deleted_id) + self.assertFalse(new_instance_list[0]['soft_delete']) + + def test_update_on_duplicate_instance_undo_case(self): """ - This is an important test to test when the client is sending the same - instance data twice in the payload. Not handling this can lead to unexpected results. - Check: https://github.com/diffgram/diffgram/issues/226 + We send 2 instances that are equal, one with an ID and the other one with no ID + expect: Instance with ID should be existing and new instance should be ignored :return: """ + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) frame = data_mocking.create_file( {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, self.session) - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - # 2 Exactly equal instances - self.project.label_dict['label_file_id_list'] = [label_file.id] - inst = { + inst1 = { 'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'y_min': 1, @@ -1101,390 +948,408 @@ def test_special__removing_duplicate_instances_in_new_instance_list(self): 'label_file_id': label_file.id, 'type': 'box' } - instance_data = [ - inst.copy(), - inst.copy() - ] + + self.project.label_dict['label_file_id_list'] = [label_file.id] video_data = { 'video_mode': True, 'video_file_id': file1.id, 'current_frame': frame.frame_number } - ann_update = Annotation_Update( session = self.session, project = self.project, video_data = video_data, - instance_list_new = instance_data, + instance_list_new = [inst1], file = file1, do_init_existing_instances = True ) - ann_update.main() + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) + new_instance_list = updated_frame_file.cache_dict['instance_list'] deleted_instances = ann_update.new_deleted_instances added_instances = ann_update.new_added_instances self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 0) - def test__check_all_instances_available_in_new_instance_list(self): - file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) - label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) - instance1 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance2 = data_mocking.create_instance( - {'x_min': 2, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - - instance3 = data_mocking.create_instance( - {'x_min': 3, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'label_file_id': label_file.id}, - self.session - ) - instance1.hash_instance() - instance2.hash_instance() - instance3.hash_instance() - old_payload = [instance1, instance2, instance3] - new_list_payload = [x.serialize_with_label() for x in old_payload] - new_list_payload_wrong = [instance1.serialize_with_label()] + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[0]['soft_delete']) + self.assertIsNotNone(new_instance_list[0]['id']) + old_id = int(new_instance_list[0]['id']) - # Test Case where we don't want to run verification + # 2. Move The Instance + inst1 = { + 'id': old_id, + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } ann_update = Annotation_Update( session = self.session, project = self.project, - instance_list_new = new_list_payload, + video_data = video_data, + instance_list_new = [inst1], file = file1, - do_init_existing_instances = False + do_init_existing_instances = True ) - result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - self.assertTrue(result) + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - # Now test case with validations + self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 1) + self.assertEqual(deleted_instances[0], old_id) + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[0]['soft_delete']) + self.assertNotEqual(new_instance_list[0]['id'], old_id) + # 3. Undo the instance + inst_undone = { + 'id': new_instance_list[0]['id'], + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, + 'soft_delete': True, + 'label_file_id': label_file.id, + 'type': 'box' + } + inst1 = { + 'id': old_id, + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } ann_update = Annotation_Update( session = self.session, project = self.project, - instance_list_new = new_list_payload, + video_data = video_data, + instance_list_new = [inst_undone, inst1], file = file1, do_init_existing_instances = True ) - result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - self.assertTrue(result) + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - # Now test case with validations and a wrong payload - ann_update = Annotation_Update( - session = self.session, - project = self.project, - instance_list_new = new_list_payload_wrong, - file = file1, - do_init_existing_instances = True - ) - result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() + self.assertEqual(len(added_instances), 2) + self.assertEqual(len(new_instance_list), 2) + self.assertEqual(len(deleted_instances), 1) + self.assertEqual(deleted_instances[0], inst_undone['id']) + self.assertEqual(new_instance_list[0]['x_min'], inst_undone['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst_undone['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst_undone['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst_undone['y_max']) + self.assertNotEqual(inst_undone['id'], new_instance_list[0]['id']) + self.assertNotEqual(new_instance_list[0]['id'], old_id) + self.assertTrue(new_instance_list[0]['soft_delete']) - self.assertTrue(result) - self.assertTrue(len(ann_update.log['warning'].keys()) > 0) - self.assertTrue('new_instance_list_missing_ids' in ann_update.log['warning']) - self.assertTrue('information' in ann_update.log['warning']) - self.assertTrue('missing_ids' in ann_update.log['warning']) - self.assertTrue(instance2.id in ann_update.log['warning']['missing_ids']) - self.assertTrue(instance3.id in ann_update.log['warning']['missing_ids']) + self.assertEqual(new_instance_list[1]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[1]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[1]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[1]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[1]['soft_delete']) - # Now test case with validations and a wrong payload and some existing deleted instances - instance4 = data_mocking.create_instance( - {'x_min': 1, 'x_max': 10, 'y_min': 1, 'y_max': 10, 'file_id': file1.id, 'soft_delete': True, - 'label_file_id': label_file.id}, + def test_update_on_duplicate_instance_1_id_1_no_id(self): + """ + We send 2 instances that are equal, one with an ID and the other one with no ID + expect: Instance with ID should be existing and new instance should be ignored + :return: + """ + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) + frame = data_mocking.create_file( + {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, + self.session) + instance1 = data_mocking.create_instance( + {'creation_ref_id': str(uuid.uuid4()), 'x_min': 1, 'x_max': 18, 'y_min': 1, 'y_max': 18, + 'file_id': file1.id, 'label_file_id': label_file.id}, self.session ) + + inst1 = { + 'id': instance1.id, # This one has the ID + 'creation_ref_id': instance1.creation_ref_id, + 'x_min': instance1.x_min, + 'y_min': instance1.y_min, + 'x_max': instance1.x_max, + 'y_max': instance1.y_max, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } + + inst2 = { + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } + + self.project.label_dict['label_file_id_list'] = [label_file.id] + video_data = { + 'video_mode': True, + 'video_file_id': file1.id, + 'current_frame': frame.frame_number + } ann_update = Annotation_Update( session = self.session, project = self.project, - instance_list_new = new_list_payload_wrong, + video_data = video_data, + instance_list_new = [inst1, inst2], file = file1, do_init_existing_instances = True ) - result = ann_update._Annotation_Update__check_all_instances_available_in_new_instance_list() + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - self.assertTrue(result) - self.assertTrue(len(ann_update.log['warning'].keys()) > 0) - self.assertTrue('new_instance_list_missing_ids' in ann_update.log['warning']) - self.assertTrue('information' in ann_update.log['warning']) - self.assertTrue('missing_ids' in ann_update.log['warning']) - self.assertTrue(instance2.id in ann_update.log['warning']['missing_ids']) - self.assertTrue(instance3.id in ann_update.log['warning']['missing_ids']) - self.assertTrue(instance4.id not in ann_update.log['warning']['missing_ids']) + new_instance_list = updated_frame_file.cache_dict['instance_list'] - def test_hashing_algorithm_changes(self): - """ - This case handles when the instance.hash() function changes, ie we add or remove - an element for the hash creation. We check things like: + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - - Avoid tracking this hash change and marking it as a system event. - - Making sure this change is not attached to the user who saved the file - - Avoiding the "restored" state on the instance history. - - Checking overall instance history is valid. - :return: - """ - file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) - label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) - self.project.label_dict['label_file_id_list'] = [label_file.id] - # 1. Initial State 2 Saved Instances - instance1 = data_mocking.create_instance( - {'x_min': 1, - 'x_max': 10, - 'y_min': 1, - 'y_max': 10, - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'type': 'box' - }, - self.session - ) - instance2 = data_mocking.create_instance( - {'x_min': 2, - 'x_max': 15, - 'y_min': 2, - 'y_max': 15, - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'type': 'box' - }, - self.session - ) - instance1.hash_instance() - instance2.hash_instance() - self.session.commit() - instance_list = [x.serialize_with_label() for x in [instance1, instance2]] + self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 0) - # 2. We Edit One Instance, the other one stays the same - instance_list[1]['x_max'] = 25 - instance_list[1]['y_max'] = 25 + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(new_instance_list[0]['soft_delete']) + self.assertNotEqual(new_instance_list[0]['id'], instance1.id) - # 3. We change the hashing algorithm (with a mock) - def dummy_hashing_algorithm(instance): - hash_data = [ - instance.x_min, - instance.x_max, - instance.y_min, - instance.y_max, - ] + # Test with cache regenerated too + updated_frame_file.set_cache_key_dirty(cache_key = 'instance_list') + regenerated_instance_list = updated_frame_file.get_with_cache( + cache_key = 'instance_list', + cache_miss_function = updated_frame_file.serialize_instance_list_only, + session = self.session) - instance.hash = hashlib.sha256(json.dumps(hash_data, - sort_keys = True).encode('utf-8')).hexdigest() + self.assertEqual(len(regenerated_instance_list), 1) - with patch.object(Instance, 'hash_instance', dummy_hashing_algorithm): - # 4. Re-save the instance. - ann_update = Annotation_Update( - session = self.session, - project = self.project, - instance_list_new = instance_list, - file = file1, - do_init_existing_instances = True - ) - ann_update.main() - # 5. Expect just one changed instance (Even though hasing algo changed, it was not a user edit). - self.assertEqual(len(ann_update.new_added_instances), 1) + self.assertEqual(regenerated_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(regenerated_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(regenerated_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(regenerated_instance_list[0]['y_max'], inst1['y_max']) + self.assertFalse(regenerated_instance_list[0]['soft_delete']) + self.assertNotEqual(regenerated_instance_list[0]['id'], instance1.id) - def test_no_changes_in_hashes_by_default_values(self): + deleted_instance = Instance.get_by_id(session = self.session, instance_id = instance1.id) + self.assertFalse(deleted_instance.soft_delete) + + def test_update_on_duplicate_instance_no_ids(self): """ - Sometimes default values can make the hash change after - saving to DB. This test checks that is not happening. + We send 2 instances that are equal and dont have ID + expect: File cache should just have one of them, other one should be ignored :return: """ - file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) - label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) + frame = data_mocking.create_file( + {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, + self.session) + inst1 = { + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } + inst2 = inst1.copy() self.project.label_dict['label_file_id_list'] = [label_file.id] - # 1. Initial State 1 Unsaved Instance - inst1 = {'id': 1, - 'type': 'box', - 'file_id': 1, - 'label_file_id': 2, - 'soft_delete': False, - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, - 'deletion_type': None, - 'created_time': '2022-01-12T18:31:59.530816', - 'action_type': 'created', - 'deleted_time': None, - 'change_source': None, - 'p1': None, - 'p2': None, - 'cp': None, - 'center_x': None, - 'center_y': None, - 'creation_ref_id': '10460e33-c02b-459d-a412-ab7b4512cc7c', - 'width': 18, - 'height': 18, - 'number': None, - 'interpolated': False, - 'machine_made': False, - 'model_id': None, - 'model_run_id': None, - 'sequence_id': None, - 'front_face': None, - 'rear_face': None, - 'rating': None, - 'attribute_groups': None, - 'member_created_id': None, - 'previous_id': None, - 'next_id': None, - 'root_id': 1, - 'version': 1, - 'nodes': [], - 'edges': [], - 'pause_object': None, - 'label_file': {'id': 2, 'hash': None, 'type': 'image', 'state': 'added', - 'created_time': '2022-01-12T18:31:59.508361', 'time_last_updated': None, - 'ann_is_complete': None, 'original_filename': 'ykzwdu', 'video_id': None, - 'video_parent_file_id': None, 'count_instances_changed': None, - 'image': {'original_filename': None, 'width': None, 'height': None, - 'soft_delete': False, 'url_signed': None, 'url_signed_thumb': None, - 'annotation_status': None}} - } - instance_list = [inst1] - - # 2. save the instance. + video_data = { + 'video_mode': True, + 'video_file_id': file1.id, + 'current_frame': frame.frame_number + } ann_update = Annotation_Update( session = self.session, project = self.project, - instance_list_new = instance_list, + video_data = video_data, + instance_list_new = [inst1, inst2], file = file1, do_init_existing_instances = True ) - ann_update.main() + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) - new_instance_list = [x.serialize_with_label() for x in ann_update.new_added_instances] - self.session.commit() - session2 = sessionMaker.session_factory() + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - # 3. Re save with no changes - ann_update2 = Annotation_Update( - session = session2, + self.assertEqual(len(added_instances), 1) + self.assertEqual(len(new_instance_list), 1) + self.assertEqual(len(deleted_instances), 0) + + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertIsNotNone(new_instance_list[0]['id']) + + # Test with cache regenerated too + updated_frame_file.set_cache_key_dirty(cache_key = 'instance_list') + regenerated_instance_list = updated_frame_file.get_with_cache( + cache_key = 'instance_list', + cache_miss_function = updated_frame_file.serialize_instance_list_only, + session = self.session) + + self.assertEqual(len(regenerated_instance_list), 1) + + self.assertEqual(regenerated_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(regenerated_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(regenerated_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(regenerated_instance_list[0]['y_max'], inst1['y_max']) + + def test_update_non_duplicate_instances_no_ids(self): + """ + We send a list of non duplicate instances with no IDs + expect: File cache should have all those instances with new ID's + :return: + """ + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) + frame = data_mocking.create_file( + {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, + self.session) + inst1 = { + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } + inst2 = { + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 50, + 'y_min': 50, + 'x_max': 120, + 'y_max': 120, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } + self.project.label_dict['label_file_id_list'] = [label_file.id] + video_data = { + 'video_mode': True, + 'video_file_id': file1.id, + 'current_frame': frame.frame_number + } + ann_update = Annotation_Update( + session = self.session, project = self.project, - instance_list_new = new_instance_list, + video_data = video_data, + instance_list_new = [inst1, inst2], file = file1, do_init_existing_instances = True ) - ann_update2.main() - self.assertEqual(len(ann_update2.system_upgrade_hash_changes), 0) + updated_file = ann_update.annotation_update_main() + updated_frame_file = File.get_by_id(self.session, frame.id) + + new_instance_list = updated_frame_file.cache_dict['instance_list'] + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances + + self.assertEqual(len(added_instances), 2) + self.assertEqual(len(new_instance_list), 2) + self.assertEqual(len(deleted_instances), 0) - def test__saving_untouched_instance_list_does_not_restore(self): - """ - We had a bug that whenever we resaved an instance and there where no changes - we would mark the instance as restored even though we were not restoring anything. + self.assertEqual(new_instance_list[0]['x_min'], inst1['x_min']) + self.assertEqual(new_instance_list[0]['y_min'], inst1['y_min']) + self.assertEqual(new_instance_list[0]['x_max'], inst1['x_max']) + self.assertEqual(new_instance_list[0]['y_max'], inst1['y_max']) + self.assertIsNotNone(new_instance_list[0]['id']) - This was because we were not checking the is_new_instance flag in the - __validate_user_deletion() function of shared/annotation.py + self.assertEqual(new_instance_list[1]['x_min'], inst2['x_min']) + self.assertEqual(new_instance_list[1]['y_min'], inst2['y_min']) + self.assertEqual(new_instance_list[1]['x_max'], inst2['x_max']) + self.assertEqual(new_instance_list[1]['y_max'], inst2['y_max']) + self.assertIsNotNone(new_instance_list[1]['id']) - This regression test case covers this bug - :return: - """ + def test_special__removing_duplicate_instances_in_new_instance_list(self): """ - Sometimes default values can make the hash change after - saving to DB. This test checks that is not happening. + This is an important test to test when the client is sending the same + instance data twice in the payload. Not handling this can lead to unexpected results. + Check: https://github.com/diffgram/diffgram/issues/226 :return: """ - file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) - label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) + file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'video'}, self.session) + frame = data_mocking.create_file( + {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, + self.session) + label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) + # 2 Exactly equal instances self.project.label_dict['label_file_id_list'] = [label_file.id] - # 1. Initial State 1 Unsaved Instance - inst1 = {'type': 'box', - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'soft_delete': False, - 'x_min': 10, - 'y_min': 10, - 'x_max': 28, - 'y_max': 28, - 'deletion_type': None, - 'created_time': '2022-01-12T18:31:59.530816', - 'action_type': 'created', - 'deleted_time': None, - 'change_source': None, - 'p1': None, - 'p2': None, - 'cp': None, - 'center_x': None, - 'center_y': None, - 'creation_ref_id': '10460e33-c02b-459d-a412-ab7b4512gdghf', - 'number': None, - 'interpolated': False, - 'machine_made': False, - 'model_id': None, - 'model_run_id': None, - 'sequence_id': None, - 'front_face': None, - 'rear_face': None, - 'rating': None, - 'attribute_groups': None, - 'member_created_id': None, - 'previous_id': None, - 'next_id': None, - 'root_id': 1, - 'version': 1, - 'nodes': [], - 'edges': [], - 'pause_object': None, - 'label_file': {'id': 2, 'hash': None, 'type': 'image', 'state': 'added', - 'created_time': '2022-01-12T18:31:59.508361', 'time_last_updated': None, - 'ann_is_complete': None, 'original_filename': 'ykzwdu', 'video_id': None, - 'video_parent_file_id': None, 'count_instances_changed': None, - 'image': {'original_filename': None, 'width': None, 'height': None, - 'soft_delete': False, 'url_signed': None, 'url_signed_thumb': None, - 'annotation_status': None}} - } - instance_list = [inst1] + inst = { + 'creation_ref_id': str(uuid.uuid4()), + 'x_min': 1, + 'y_min': 1, + 'x_max': 18, + 'y_max': 18, + 'soft_delete': False, + 'label_file_id': label_file.id, + 'type': 'box' + } + instance_data = [ + inst.copy(), + inst.copy() + ] + video_data = { + 'video_mode': True, + 'video_file_id': file1.id, + 'current_frame': frame.frame_number + } - # 2. save the instance. ann_update = Annotation_Update( session = self.session, project = self.project, - instance_list_new = instance_list, + video_data = video_data, + instance_list_new = instance_data, file = file1, do_init_existing_instances = True ) ann_update.main() - new_instance_list = [x.serialize_with_label() for x in ann_update.new_added_instances] - self.session.commit() - - session2 = sessionMaker.session_factory() - new_instance_list[0]['x_min'] += 10 - new_instance_list[0]['x_max'] += 10 - new_instance_list[0]['y_max'] += 10 - new_instance_list[0]['y_min'] += 10 - - # 3. Move the instance and resave - ann_update2 = Annotation_Update( - session = session2, - project = self.project, - instance_list_new = new_instance_list, - file = file1, - do_init_existing_instances = True - ) - ann_update2.main() - self.assertEqual(len(ann_update2.new_added_instances), 1) - - instance_list_result = ann_update2.file.instance_list - new_instance_list = [x.serialize_with_label() for x in instance_list_result] - session2.commit() - # 4. Resave with no changes - ann_update2 = Annotation_Update( - session = session2, - project = self.project, - instance_list_new = new_instance_list, - file = file1, - do_init_existing_instances = True - ) - ann_update2.main() - instance_list_result = ann_update2.file.instance_list + deleted_instances = ann_update.new_deleted_instances + added_instances = ann_update.new_added_instances - self.assertEqual(len(instance_list_result), 2) - self.assertEqual(len(ann_update2.new_added_instances), 0) - self.assertNotEqual(instance_list_result[1].action_type, 'undeleted') + self.assertEqual(len(added_instances), 1) def test_create_instance_relation(self): """ @@ -1574,134 +1439,121 @@ def test_create_instance_relation(self): self.assertEqual(new_instances2[0].from_instance_id, new_instances[1].id) self.assertEqual(new_instances2[0].to_instance_id, new_instances[0].id) - def test_check_relations_id_existence(self): + + def test__saving_untouched_instance_list_does_not_restore(self): """ - Check calls to check_relations_id_existence + We had a bug that whenever we resaved an instance and there where no changes + we would mark the instance as restored even though we were not restoring anything. + + This was because we were not checking the is_new_instance flag in the + __validate_user_deletion() function of shared/annotation.py + + This regression test case covers this bug :return: """ - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'image'}, self.session) - frame = data_mocking.create_file( - {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, - self.session) - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - # 2 Exactly equal instances + """ + Sometimes default values can make the hash change after + saving to DB. This test checks that is not happening. + :return: + """ + file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) + label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) self.project.label_dict['label_file_id_list'] = [label_file.id] - instance1 = data_mocking.create_instance( - {'x_min': 1, - 'x_max': 10, - 'y_min': 1, - 'y_max': 10, - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'type': 'relation' - }, - self.session - ) - instance2 = data_mocking.create_instance( - {'x_min': 2, - 'x_max': 15, - 'y_min': 2, - 'y_max': 15, - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'type': 'box' - }, - self.session + # 1. Initial State 1 Unsaved Instance + inst1 = {'type': 'box', + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'soft_delete': False, + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, + 'deletion_type': None, + 'created_time': '2022-01-12T18:31:59.530816', + 'action_type': 'created', + 'deleted_time': None, + 'change_source': None, + 'p1': None, + 'p2': None, + 'cp': None, + 'center_x': None, + 'center_y': None, + 'creation_ref_id': '10460e33-c02b-459d-a412-ab7b4512gdghf', + 'number': None, + 'interpolated': False, + 'machine_made': False, + 'model_id': None, + 'model_run_id': None, + 'sequence_id': None, + 'front_face': None, + 'rear_face': None, + 'rating': None, + 'attribute_groups': None, + 'member_created_id': None, + 'previous_id': None, + 'next_id': None, + 'root_id': 1, + 'version': 1, + 'nodes': [], + 'edges': [], + 'pause_object': None, + 'label_file': {'id': 2, 'hash': None, 'type': 'image', 'state': 'added', + 'created_time': '2022-01-12T18:31:59.508361', 'time_last_updated': None, + 'ann_is_complete': None, 'original_filename': 'ykzwdu', 'video_id': None, + 'video_parent_file_id': None, 'count_instances_changed': None, + 'image': {'original_filename': None, 'width': None, 'height': None, + 'soft_delete': False, 'url_signed': None, 'url_signed_thumb': None, + 'annotation_status': None}} + } + instance_list = [inst1] + + # 2. save the instance. + ann_update = Annotation_Update( + session = self.session, + project = self.project, + instance_list_new = instance_list, + file = file1, + do_init_existing_instances = True ) + ann_update.main() + + new_instance_list = [x.serialize_with_label() for x in ann_update.new_added_instances] + self.session.commit() + + session2 = sessionMaker.session_factory() + new_instance_list[0]['x_min'] += 10 + new_instance_list[0]['x_max'] += 10 + new_instance_list[0]['y_max'] += 10 + new_instance_list[0]['y_min'] += 10 + + # 3. Move the instance and resave ann_update2 = Annotation_Update( - session = self.session, + session = session2, project = self.project, - instance_list_new = [], + instance_list_new = new_instance_list, file = file1, do_init_existing_instances = True ) + ann_update2.main() + self.assertEqual(len(ann_update2.new_added_instances), 1) - ann_update2.instance = instance1 - # Check Error State - ann_update2.check_relations_id_existence(None, None, None, None) - self.assertTrue('from_id' in ann_update2.log['error']) - # Check IDs available - ann_update2.log['error'] = {} - ann_update2.instance.from_instance_id = 1 - ann_update2.instance.to_instance_id = 1 - ann_update2.check_relations_id_existence(ann_update2.instance.from_instance_id, - ann_update2.instance.to_instance_id, - None, - None) - self.assertEqual(len(ann_update2.log['error'].keys()), 0) - self.assertEqual(len(ann_update2.new_instance_relations_list_no_ids), 0) - # Check IDs Not available - ann_update2.check_relations_id_existence(None, - None, - str(uuid.uuid4()), - str(uuid.uuid4())) - self.assertEqual(len(ann_update2.log['error'].keys()), 0) - self.assertEqual(len(ann_update2.new_instance_relations_list_no_ids), 1) - - def test_add_missing_ids_to_new_relations(self): - """ - Tests calss to add_missing_ids_to_new_relations() - :return: - """ - file1 = data_mocking.create_file({'project_id': self.project.id, 'type': 'image'}, self.session) - frame = data_mocking.create_file( - {'project_id': self.project.id, 'type': 'frame', 'video_parent_file_id': file1.id, 'frame_number': 5}, - self.session) - label_file = data_mocking.create_file({'project_id': self.project.id, 'type': 'label'}, self.session) - # 2 Exactly equal instances - self.project.label_dict['label_file_id_list'] = [label_file.id] - instance1 = data_mocking.create_instance( - {'x_min': 1, - 'x_max': 10, - 'y_min': 1, - 'y_max': 10, - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'type': 'relation' - }, - self.session - ) - instance1.creation_ref_id = str(uuid.uuid4()) - instance2 = data_mocking.create_instance( - {'x_min': 2, - 'x_max': 15, - 'y_min': 2, - 'y_max': 15, - 'file_id': file1.id, - 'label_file_id': label_file.id, - 'type': 'box' - }, - self.session - ) - relation = data_mocking.create_instance( - { - 'file_id': file1.id, - 'label_file_id': None, - 'type': 'relation' - }, - self.session - ) - relation.hash_instance() - old_hash = relation.hash - - instance2.creation_ref_id = str(uuid.uuid4()) - self.session.commit() - ann_update = Annotation_Update( - session = self.session, + instance_list_result = ann_update2.file.instance_list + new_instance_list = [x.serialize_with_label() for x in instance_list_result] + session2.commit() + # 4. Resave with no changes + ann_update2 = Annotation_Update( + session = session2, project = self.project, - instance_list_new = [], + instance_list_new = new_instance_list, file = file1, do_init_existing_instances = True ) - ann_update.new_added_instances = [instance1, instance2] - ann_update.new_instance_relations_list_no_ids = [{'instance': relation, - 'from_ref': instance1.creation_ref_id, - 'to_ref': instance2.creation_ref_id}] - ann_update.add_missing_ids_to_new_relations() + ann_update2.main() + instance_list_result = ann_update2.file.instance_list - self.assertEqual(relation.from_instance_id, instance1.id) - self.assertEqual(relation.to_instance_id, instance2.id) - self.assertNotEqual(old_hash, relation.hash) + self.assertEqual(len(instance_list_result), 2) + self.assertEqual(len(ann_update2.new_added_instances), 0) + self.assertNotEqual(instance_list_result[1].action_type, 'undeleted') def test_create_relation_no_ids(self): """ @@ -1771,3 +1623,164 @@ def test_create_relation_no_ids(self): ann_update.main() self.assertEqual(len(ann_update.log['error'].keys()), 0) + + """ + Next 2 methods below are for testing hashing functionality + """ + def test_hashing_algorithm_changes(self): + """ + This case handles when the instance.hash() function changes, ie we add or remove + an element for the hash creation. We check things like: + + - Avoid tracking this hash change and marking it as a system event. + - Making sure this change is not attached to the user who saved the file + - Avoiding the "restored" state on the instance history. + - Checking overall instance history is valid. + :return: + """ + file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) + label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) + self.project.label_dict['label_file_id_list'] = [label_file.id] + # 1. Initial State 2 Saved Instances + instance1 = data_mocking.create_instance( + {'x_min': 1, + 'x_max': 10, + 'y_min': 1, + 'y_max': 10, + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'type': 'box' + }, + self.session + ) + instance2 = data_mocking.create_instance( + {'x_min': 2, + 'x_max': 15, + 'y_min': 2, + 'y_max': 15, + 'file_id': file1.id, + 'label_file_id': label_file.id, + 'type': 'box' + }, + self.session + ) + instance1.hash_instance() + instance2.hash_instance() + self.session.commit() + instance_list = [x.serialize_with_label() for x in [instance1, instance2]] + + # 2. We Edit One Instance, the other one stays the same + instance_list[1]['x_max'] = 25 + instance_list[1]['y_max'] = 25 + + # 3. We change the hashing algorithm (with a mock) + def dummy_hashing_algorithm(instance): + hash_data = [ + instance.x_min, + instance.x_max, + instance.y_min, + instance.y_max, + ] + + instance.hash = hashlib.sha256(json.dumps(hash_data, + sort_keys = True).encode('utf-8')).hexdigest() + + with patch.object(Instance, 'hash_instance', dummy_hashing_algorithm): + # 4. Re-save the instance. + ann_update = Annotation_Update( + session = self.session, + project = self.project, + instance_list_new = instance_list, + file = file1, + do_init_existing_instances = True + ) + ann_update.main() + # 5. Expect just one changed instance (Even though hasing algo changed, it was not a user edit). + self.assertEqual(len(ann_update.new_added_instances), 1) + + def test_no_changes_in_hashes_by_default_values(self): + """ + Sometimes default values can make the hash change after + saving to DB. This test checks that is not happening. + :return: + """ + file1 = data_mocking.create_file({'project_id': self.project.id}, self.session) + label_file = data_mocking.create_file({'project_id': self.project.id}, self.session) + self.project.label_dict['label_file_id_list'] = [label_file.id] + # 1. Initial State 1 Unsaved Instance + inst1 = {'id': 1, + 'type': 'box', + 'file_id': 1, + 'label_file_id': 2, + 'soft_delete': False, + 'x_min': 10, + 'y_min': 10, + 'x_max': 28, + 'y_max': 28, + 'deletion_type': None, + 'created_time': '2022-01-12T18:31:59.530816', + 'action_type': 'created', + 'deleted_time': None, + 'change_source': None, + 'p1': None, + 'p2': None, + 'cp': None, + 'center_x': None, + 'center_y': None, + 'creation_ref_id': '10460e33-c02b-459d-a412-ab7b4512cc7c', + 'width': 18, + 'height': 18, + 'number': None, + 'interpolated': False, + 'machine_made': False, + 'model_id': None, + 'model_run_id': None, + 'sequence_id': None, + 'front_face': None, + 'rear_face': None, + 'rating': None, + 'attribute_groups': None, + 'member_created_id': None, + 'previous_id': None, + 'next_id': None, + 'root_id': 1, + 'version': 1, + 'nodes': [], + 'edges': [], + 'pause_object': None, + 'label_file': {'id': 2, 'hash': None, 'type': 'image', 'state': 'added', + 'created_time': '2022-01-12T18:31:59.508361', 'time_last_updated': None, + 'ann_is_complete': None, 'original_filename': 'ykzwdu', 'video_id': None, + 'video_parent_file_id': None, 'count_instances_changed': None, + 'image': {'original_filename': None, 'width': None, 'height': None, + 'soft_delete': False, 'url_signed': None, 'url_signed_thumb': None, + 'annotation_status': None}} + } + instance_list = [inst1] + + # 2. save the instance. + ann_update = Annotation_Update( + session = self.session, + project = self.project, + instance_list_new = instance_list, + file = file1, + do_init_existing_instances = True + ) + ann_update.main() + + new_instance_list = [x.serialize_with_label() for x in ann_update.new_added_instances] + self.session.commit() + session2 = sessionMaker.session_factory() + + # 3. Re save with no changes + ann_update2 = Annotation_Update( + session = session2, + project = self.project, + instance_list_new = new_instance_list, + file = file1, + do_init_existing_instances = True + ) + ann_update2.main() + self.assertEqual(len(ann_update2.system_upgrade_hash_changes), 0) + + \ No newline at end of file diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py new file mode 100644 index 000000000..cabe455a1 --- /dev/null +++ b/default/tests/shared/test_annotation_new.py @@ -0,0 +1,103 @@ +from methods.regular.regular_api import * +from default.tests.test_utils import testing_setup +from shared.tests.test_utils import common_actions, data_mocking +from base64 import b64encode +from shared.annotation import Annotation_Update +from unittest.mock import patch, MagicMock, Mock, call + + +class TestAnnotationUpdate(testing_setup.DiffgramBaseTestCase): + + def setUp(self): + # TODO: this test is assuming the 'my-sandbox-project' exists and some object have been previously created. + # For future tests a mechanism of setting up and tearing down the database should be created. + super(TestAnnotationUpdate, self).setUp() + + # TODO: How can we clean this up / re-use all the setup? + + @patch('shared.annotation.regular_log.default') + @patch('shared.annotation.Project.get_by_id') + @patch('shared.annotation.get_member') + @patch('shared.annotation.Annotation_Update.get_allowed_label_file_ids') + @patch('shared.annotation.Annotation_Update.init_video_input') + @patch('shared.annotation.Annotation_Update.task_update') + @patch('shared.annotation.Annotation_Update.init_file') + @patch('shared.annotation.Annotation_Update.init_existing_instances') + @patch('shared.annotation.Annotation_Update.refresh_instance_count') + def test_post_init_main_functions_called(self, mock_refresh_instance_count, mock_init_existing_instances, mock_init_file, mock_task_update, mock_init_video_input, mock_get_allowed_label_file_ids, mock_get_member, mock_Project_get_by_id, mock_default_log): + # Arrange + mock_session = MagicMock() + mock_project = MagicMock() + mock_member = MagicMock() + mock_file = MagicMock() + + # func_calls = Mock() + # func_calls.m1, func_calls.m2, func_calls.m3, func_calls.m4, func_calls.m5, func_calls.m6 = mock_get_allowed_label_file_ids, mock_init_video_input, mock_task_update, mock_init_file, mock_init_existing_instances, mock_refresh_instance_count + + # Act + instance = Annotation_Update(session=mock_session, file=mock_file, member=mock_member, project=mock_project) + + # Assert + self.assertEqual(instance.log, mock_default_log.return_value) # Log was initialized + mock_Project_get_by_id.assert_not_called() # Project was not fetched + mock_get_member.assert_not_called() # Member was not fetched + + # TODO: This isnt working, got it from here https://stackoverflow.com/questions/32463321/how-to-assert-method-call-order-with-python-mock + # func_calls.assert_has_calls([call.m1(), call.m2(), call.m3(), call.m4(), call.m5(), call.m6()]) + mock_get_allowed_label_file_ids.assert_called_once() + self.assertEqual(instance.previous_next_instance_map, {}) + mock_init_video_input.assert_called_once() + mock_task_update.assert_called_once() + mock_init_file.assert_called_once() + mock_init_existing_instances.assert_called_once() + mock_refresh_instance_count.assert_called_once() + + @patch('shared.annotation.Annotation_Update.get_allowed_label_file_ids') + @patch('shared.annotation.Annotation_Update.init_video_input') + @patch('shared.annotation.Annotation_Update.task_update') + @patch('shared.annotation.Annotation_Update.init_file') + @patch('shared.annotation.Annotation_Update.init_existing_instances') + @patch('shared.annotation.Annotation_Update.refresh_instance_count') + def test_post_init_project_creating_for_instance_template(self, mock_refresh_instance_count, mock_init_existing_instances, mock_init_file, mock_task_update, mock_init_video_input, mock_get_allowed_label_file_ids): + # Arrange + mock_session = MagicMock() + mock_file = MagicMock() + creating_for_instance_template = True + + # Act + Annotation_Update(session=mock_session, file=mock_file, creating_for_instance_template=creating_for_instance_template) + + mock_get_allowed_label_file_ids.assert_not_called() + mock_init_video_input.assert_not_called() + mock_task_update.assert_not_called() + mock_init_file.assert_not_called() + mock_init_existing_instances.assert_not_called() + mock_refresh_instance_count.assert_not_called() + + @patch('shared.annotation.Project.get_by_id') + def test_post_init_project_fetched(self, mock_Project_get_by_id): + # Arrange + mock_session = MagicMock() + mock_file = MagicMock() + project_id = 123 + + # Act + Annotation_Update(session=mock_session, file=mock_file, project_id=project_id) + + # Assert + mock_Project_get_by_id.assert_called_once_with(mock_session, project_id) + + @patch('shared.annotation.get_member') + def test_post_init_member_fetched(self, mock_get_member): + # Arrange + mock_session = MagicMock() + mock_file = MagicMock() + project_id = 123 + + # Act + Annotation_Update(session=mock_session, file=mock_file, project_id=project_id) + + # Assert + mock_get_member.assert_called_once_with(session=mock_session) + + diff --git a/shared/annotation.py b/shared/annotation.py index a46210fc9..cd42dd3e0 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -484,6 +484,7 @@ def instance_template_main(self): return self.new_added_instances + # Tested def __check_all_instances_available_in_new_instance_list(self): if not self.do_init_existing_instances: return True @@ -536,18 +537,21 @@ def __check_all_instances_available_in_new_instance_list(self): # return False return True + # Tested def append_new_instance_list_hash(self, instance): if instance.soft_delete is False: self.new_instance_dict_hash[instance.hash] = instance return True return False + # Tested def order_new_instances_by_date(self): self.instance_list_new.sort( key = lambda item: (item.get('client_created_time') is not None, item.get('client_created_time')), reverse = True) return self.instance_list_new + # Tested measily, needs a big cleanup def annotation_update_main(self): """ @@ -605,6 +609,7 @@ def annotation_update_main(self): logger.error(f"Error updating annotation {str(self.log)}") return self.return_orginal_file_type() + # TODO: Whats the point of this? def main(self): return self.annotation_update_main() @@ -724,7 +729,8 @@ def init_file(self): task = self.task ) self.is_new_file = True - + + # Tested def detect_and_remove_collisions(self, instance_list): result = [] hashes_dict = {} @@ -1576,6 +1582,7 @@ def find_serialized_instance_index(self, id): if instance.get('id') == id: return i + # Tested def update_sequence_id_in_cache_list(self, instance): """ Updates the sequences ID in the cache list. @@ -1591,6 +1598,7 @@ def update_sequence_id_in_cache_list(self, instance): if existing_serialized_instance.get('id') == instance.id: existing_serialized_instance['sequence_id'] = instance.sequence_id + # Tested def update_cache_single_instance_in_list_context(self): """ CAUTION this assumes that instance_list_kept_serialized will exist etc @@ -1682,6 +1690,7 @@ def may_create_new_file_old_source_control(self): self.file = File.copy_file_from_existing( self.session, directory, self.file) + # Tested def add_missing_ids_to_new_relations(self): for relation_elm in self.new_instance_relations_list_no_ids: @@ -1703,6 +1712,7 @@ def add_missing_ids_to_new_relations(self): instance.hash_instance() self.session.add(instance) + # Tested def check_relations_id_existence(self, from_id, to_id, from_ref, to_ref): """ Checks if current instance is a relations and if ID's are available for saving. @@ -2045,6 +2055,7 @@ def sequence_update(self, return sequence + # Tested def check_polygon_points_and_build_bounds(self): self.instance.x_min = 99999 self.instance.x_max = 0 From 7cee49fe6eca14cf33335911377a028d87182eb3 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Mon, 28 Aug 2023 16:48:11 -0600 Subject: [PATCH 02/23] Add tested note --- shared/annotation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/annotation.py b/shared/annotation.py index cd42dd3e0..4e5e6d159 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -430,6 +430,7 @@ class Annotation_Update(): # https://diffgram.readme.io/docs/general-annotation-update + # Tested def __post_init__(self): self.log = regular_log.default() From 72c6058c6ab80269c4aa9cfda430b785e255a741 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Mon, 28 Aug 2023 17:34:43 -0600 Subject: [PATCH 03/23] Add unit tests for instance_template_main --- default/tests/shared/test_annotation_new.py | 84 ++++++++++++++++----- shared/annotation.py | 1 + 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index cabe455a1..bb49d9c5c 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -12,6 +12,13 @@ def setUp(self): # TODO: this test is assuming the 'my-sandbox-project' exists and some object have been previously created. # For future tests a mechanism of setting up and tearing down the database should be created. super(TestAnnotationUpdate, self).setUp() + self.mock_session = MagicMock() + self.mock_project = MagicMock() + self.mock_member = MagicMock() + self.mock_file = MagicMock() + + # TODO: Try this out if we have to instantiate the class often + # self.instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) # TODO: How can we clean this up / re-use all the setup? @@ -26,16 +33,11 @@ def setUp(self): @patch('shared.annotation.Annotation_Update.refresh_instance_count') def test_post_init_main_functions_called(self, mock_refresh_instance_count, mock_init_existing_instances, mock_init_file, mock_task_update, mock_init_video_input, mock_get_allowed_label_file_ids, mock_get_member, mock_Project_get_by_id, mock_default_log): # Arrange - mock_session = MagicMock() - mock_project = MagicMock() - mock_member = MagicMock() - mock_file = MagicMock() - # func_calls = Mock() # func_calls.m1, func_calls.m2, func_calls.m3, func_calls.m4, func_calls.m5, func_calls.m6 = mock_get_allowed_label_file_ids, mock_init_video_input, mock_task_update, mock_init_file, mock_init_existing_instances, mock_refresh_instance_count # Act - instance = Annotation_Update(session=mock_session, file=mock_file, member=mock_member, project=mock_project) + instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) # Assert self.assertEqual(instance.log, mock_default_log.return_value) # Log was initialized @@ -60,12 +62,10 @@ def test_post_init_main_functions_called(self, mock_refresh_instance_count, mock @patch('shared.annotation.Annotation_Update.refresh_instance_count') def test_post_init_project_creating_for_instance_template(self, mock_refresh_instance_count, mock_init_existing_instances, mock_init_file, mock_task_update, mock_init_video_input, mock_get_allowed_label_file_ids): # Arrange - mock_session = MagicMock() - mock_file = MagicMock() creating_for_instance_template = True # Act - Annotation_Update(session=mock_session, file=mock_file, creating_for_instance_template=creating_for_instance_template) + Annotation_Update(session=self.mock_session, file=self.mock_file, creating_for_instance_template=creating_for_instance_template) mock_get_allowed_label_file_ids.assert_not_called() mock_init_video_input.assert_not_called() @@ -77,27 +77,77 @@ def test_post_init_project_creating_for_instance_template(self, mock_refresh_ins @patch('shared.annotation.Project.get_by_id') def test_post_init_project_fetched(self, mock_Project_get_by_id): # Arrange - mock_session = MagicMock() - mock_file = MagicMock() project_id = 123 # Act - Annotation_Update(session=mock_session, file=mock_file, project_id=project_id) + Annotation_Update(session=self.mock_session, file=self.mock_file, project_id=project_id) # Assert - mock_Project_get_by_id.assert_called_once_with(mock_session, project_id) + mock_Project_get_by_id.assert_called_once_with(self.mock_session, project_id) @patch('shared.annotation.get_member') def test_post_init_member_fetched(self, mock_get_member): # Arrange - mock_session = MagicMock() - mock_file = MagicMock() project_id = 123 # Act - Annotation_Update(session=mock_session, file=mock_file, project_id=project_id) + Annotation_Update(session=self.mock_session, file=self.mock_file, project_id=project_id) # Assert - mock_get_member.assert_called_once_with(session=mock_session) + mock_get_member.assert_called_once_with(session=self.mock_session) + + @patch('shared.annotation.Annotation_Update.update_instance_list') + @patch('shared.annotation.logger') + def test_instance_template_main_empty_instance_list_new(self, mock_logger, mock_update_instance_list): + # Arrange + instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) + instance.instance_list_new = [] + + # Act + result = instance.instance_template_main() + # Assert + self.assertIsNone(result) + mock_logger.error.assert_called_once_with('Error, please provide instance_list_new {\'error\': {}, \'info\': {}, \'success\': False}') + mock_update_instance_list.assert_not_called() + + @patch('shared.annotation.Annotation_Update.update_instance_list') + def test_instance_template_main_success(self, mock_update_instance_list): + # Arrange + instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) + instance.instance_list_new = [{'instance_id': 1}, {'instance_id': 2}] + instance.per_instance_spec_list = [{'label_file_id': {'required': True}}, {'some_other_key': 'value'}] + mock_new_added_instances = Mock() + instance.new_added_instances = mock_new_added_instances + # Act + result = instance.instance_template_main() + + # Assert + self.assertEqual(result, mock_new_added_instances) + mock_update_instance_list.assert_called_with(hash_instances=False, validate_label_file=False, overwrite_existing_instances=True) + self.assertEqual(instance.per_instance_spec_list, [{'label_file_id': {'required': False}}, {'some_other_key': 'value'}]) + + @patch('shared.annotation.logger') + @patch('shared.annotation.Annotation_Update.update_instance_list') + @patch('shared.annotation.Annotation_Update.return_orginal_file_type') + def test_instance_template_main_error_detected(self, mock_return_orginal_file_type, mock_update_instance_list, mock_logger): + # Arrange + instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) + mock_error_log = { "error": { "some_error": "error_message" }, "info": {}, "success": False } + mock_instance_list = [{'instance_id': 1}, {'instance_id': 2}] + instance.instance_list_new = mock_instance_list + instance.log["error"] = mock_error_log["error"] + + # Act + result = instance.instance_template_main() + + # Assert + self.assertEqual(result, mock_return_orginal_file_type.return_value) + mock_update_instance_list.assert_called_with(hash_instances=False, validate_label_file=False, overwrite_existing_instances=True) + expected_logger_calls = [ + call(f"Error updating annotation {str(mock_error_log)}"), + call(f"Instance list is: {mock_instance_list}") + ] + mock_logger.error.assert_has_calls(expected_logger_calls) + \ No newline at end of file diff --git a/shared/annotation.py b/shared/annotation.py index 4e5e6d159..e26e00621 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -458,6 +458,7 @@ def __post_init__(self): self.refresh_instance_count() + # Tested def instance_template_main(self): """ This is the main flow for creating/updating From 690e038d17b9d37dbdc8e4df15fb5bb545f623f0 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Mon, 28 Aug 2023 17:55:23 -0600 Subject: [PATCH 04/23] Cleanup --- default/tests/shared/test_annotation_new.py | 31 ++++++++------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index bb49d9c5c..da6fae1a0 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -1,7 +1,5 @@ from methods.regular.regular_api import * from default.tests.test_utils import testing_setup -from shared.tests.test_utils import common_actions, data_mocking -from base64 import b64encode from shared.annotation import Annotation_Update from unittest.mock import patch, MagicMock, Mock, call @@ -9,16 +7,12 @@ class TestAnnotationUpdate(testing_setup.DiffgramBaseTestCase): def setUp(self): - # TODO: this test is assuming the 'my-sandbox-project' exists and some object have been previously created. - # For future tests a mechanism of setting up and tearing down the database should be created. super(TestAnnotationUpdate, self).setUp() self.mock_session = MagicMock() self.mock_project = MagicMock() self.mock_member = MagicMock() self.mock_file = MagicMock() - - # TODO: Try this out if we have to instantiate the class often - # self.instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) + self.instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) # TODO: How can we clean this up / re-use all the setup? @@ -100,11 +94,10 @@ def test_post_init_member_fetched(self, mock_get_member): @patch('shared.annotation.logger') def test_instance_template_main_empty_instance_list_new(self, mock_logger, mock_update_instance_list): # Arrange - instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) - instance.instance_list_new = [] + self.instance.instance_list_new = [] # Act - result = instance.instance_template_main() + result = self.instance.instance_template_main() # Assert self.assertIsNone(result) @@ -114,33 +107,31 @@ def test_instance_template_main_empty_instance_list_new(self, mock_logger, mock_ @patch('shared.annotation.Annotation_Update.update_instance_list') def test_instance_template_main_success(self, mock_update_instance_list): # Arrange - instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) - instance.instance_list_new = [{'instance_id': 1}, {'instance_id': 2}] - instance.per_instance_spec_list = [{'label_file_id': {'required': True}}, {'some_other_key': 'value'}] + self.instance.instance_list_new = [{'instance_id': 1}, {'instance_id': 2}] + self.instance.per_instance_spec_list = [{'label_file_id': {'required': True}}, {'some_other_key': 'value'}] mock_new_added_instances = Mock() - instance.new_added_instances = mock_new_added_instances + self.instance.new_added_instances = mock_new_added_instances # Act - result = instance.instance_template_main() + result = self.instance.instance_template_main() # Assert self.assertEqual(result, mock_new_added_instances) mock_update_instance_list.assert_called_with(hash_instances=False, validate_label_file=False, overwrite_existing_instances=True) - self.assertEqual(instance.per_instance_spec_list, [{'label_file_id': {'required': False}}, {'some_other_key': 'value'}]) + self.assertEqual(self.instance.per_instance_spec_list, [{'label_file_id': {'required': False}}, {'some_other_key': 'value'}]) @patch('shared.annotation.logger') @patch('shared.annotation.Annotation_Update.update_instance_list') @patch('shared.annotation.Annotation_Update.return_orginal_file_type') def test_instance_template_main_error_detected(self, mock_return_orginal_file_type, mock_update_instance_list, mock_logger): # Arrange - instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) mock_error_log = { "error": { "some_error": "error_message" }, "info": {}, "success": False } mock_instance_list = [{'instance_id': 1}, {'instance_id': 2}] - instance.instance_list_new = mock_instance_list - instance.log["error"] = mock_error_log["error"] + self.instance.instance_list_new = mock_instance_list + self.instance.log["error"] = mock_error_log["error"] # Act - result = instance.instance_template_main() + result = self.instance.instance_template_main() # Assert self.assertEqual(result, mock_return_orginal_file_type.return_value) From c991620928765598d43e58b3a825fa5916397f8f Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Tue, 29 Aug 2023 13:00:02 -0600 Subject: [PATCH 05/23] Test __perform_external_map_action --- default/tests/shared/test_annotation_new.py | 40 ++++++++++++++++++++- shared/annotation.py | 1 + 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index da6fae1a0..0401c0c57 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -141,4 +141,42 @@ def test_instance_template_main_error_detected(self, mock_return_orginal_file_ty call(f"Instance list is: {mock_instance_list}") ] mock_logger.error.assert_has_calls(expected_logger_calls) - \ No newline at end of file + + + def test___perform_external_map_action(self): + # Arrange + self.instance.external_map = Mock() + self.instance.external_map_action = 'set_instance_id' + self.instance.instance = Mock() + + # Act + self.instance._Annotation_Update__perform_external_map_action() + + # Assert + self.assertEqual(self.instance.external_map.instance, self.instance.instance) + self.instance.session.add.assert_called_once_with(self.instance.external_map) + + def test___perform_external_map_action_wrong_action(self): + # Arrange + self.instance.external_map = Mock() + self.instance.external_map_action = 'something_else' + self.instance.instance = Mock() + + # Act + self.instance._Annotation_Update__perform_external_map_action() + + # Assert + self.assertNotEqual(self.instance.external_map.instance, self.instance.instance) + self.instance.session.add.assert_not_called() + + def test___perform_external_map_action_no_external_map(self): + # Arrange + self.instance.external_map = None + self.instance.external_map_action = 'set_instance_id' + self.instance.instance = Mock() + + # Act + self.instance._Annotation_Update__perform_external_map_action() + + # Assert + self.instance.session.add.assert_not_called() diff --git a/shared/annotation.py b/shared/annotation.py index e26e00621..38a895cda 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -615,6 +615,7 @@ def annotation_update_main(self): def main(self): return self.annotation_update_main() + # Tested def __perform_external_map_action(self): if not self.external_map: return From 2906c7cb71485dbcdabab57be5fcc72a62718311 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Tue, 29 Aug 2023 13:09:01 -0600 Subject: [PATCH 06/23] Tests for return_orginal_file_type & instance_list_cache_update --- default/tests/shared/test_annotation_new.py | 46 +++++++++++++++++++++ shared/annotation.py | 3 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index 0401c0c57..f8198b5a4 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -180,3 +180,49 @@ def test___perform_external_map_action_no_external_map(self): # Assert self.instance.session.add.assert_not_called() + + @patch('shared.annotation.FileStats.update_file_stats_data') + def test_instance_list_cache_update(self, mock_update_file_stats_data): + # Arrange + mock_instance_list_kept_serialized = Mock() + self.instance.instance_list_kept_serialized = mock_instance_list_kept_serialized + + # Act + self.instance.instance_list_cache_update() + + # Assert + self.instance.file.set_cache_by_key.assert_called_once_with(cache_key='instance_list', value=mock_instance_list_kept_serialized) + mock_update_file_stats_data.assert_called_once_with(session=self.instance.session, instance_list=mock_instance_list_kept_serialized, file_id=self.instance.file.id, project=self.instance.project) + + @patch('shared.annotation.FileStats.update_file_stats_data') + def test_instance_list_cache_update_no_file(self, mock_update_file_stats_data): + # Arrange + self.instance.file = None + + # Act + self.instance.instance_list_cache_update() + + # Assert + mock_update_file_stats_data.assert_not_called() + + def test_return_orginal_file_type_video_parent_file(self): + # Arrange + mock_video_parent_file = Mock() + self.instance.video_parent_file = mock_video_parent_file + + # Act + result = self.instance.return_orginal_file_type() + + # Assert + self.assertEqual(result, mock_video_parent_file) + + def test_return_orginal_file_type_base_file(self): + # Arrange + self.instance.video_parent_file = None + + # Act + result = self.instance.return_orginal_file_type() + + # Assert + self.assertEqual(result, self.instance.file) + diff --git a/shared/annotation.py b/shared/annotation.py index 38a895cda..a97da3e58 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -624,6 +624,7 @@ def __perform_external_map_action(self): self.external_map.instance = self.instance self.session.add(self.external_map) + # Tested def instance_list_cache_update(self): """ High level idea of caching @@ -660,7 +661,7 @@ def instance_list_cache_update(self): project = self.project ) - + # Tested def return_orginal_file_type(self): """ Not a fan of this setup... but at least this way From 7687b5468b2a76309b4635be11ad9168e5a2af04 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Tue, 29 Aug 2023 17:45:48 -0600 Subject: [PATCH 07/23] Commit progress --- default/tests/shared/test_annotation_new.py | 65 +++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index f8198b5a4..3154e86ef 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -226,3 +226,68 @@ def test_return_orginal_file_type_base_file(self): # Assert self.assertEqual(result, self.instance.file) + @patch('shared.annotation.logger') + def test_rehash_existing_instances_same_hash(self, mock_logger): + # Arrange + mock_instance = Mock() + mock_instance.hash = 'hash1' + mock_instance_list = [mock_instance] + + # Act + result = self.instance.rehash_existing_instances(mock_instance_list) + + # Assert + self.assertEqual(result, mock_instance_list) + self.instance.session.add.assert_not_called() + self.assertEqual(self.instance.system_upgrade_hash_changes, []) + mock_logger.info.assert_not_called() + + # TODO: side_effect isnt working here + @patch('shared.annotation.logger') + def test_rehash_existing_instances_different_hash(self, mock_logger): + # Arrange + mock_instance = Mock(spec=Annotation_Update).return_value + mock_instance.id = '123' + mock_instance.hash = 'hash1' + mock_instance.hash_instances = Mock() + + def side_effect(): + mock_instance.hash = 'hash2' + + mock_instance.hash_instances.side_effect = side_effect + mock_instance_list = [mock_instance] + + print("HASH IS: ", mock_instance.hash) + # Act + result = self.instance.rehash_existing_instances(mock_instance_list) + print("HASH NOW: ", mock_instance.hash) + + # Assert + self.instance.session.add.assert_called_once_with(mock_instance) + self.assertEqual(self.instance.system_upgrade_hash_changes, [['hash1', 'hash2']]) + mock_logger.info.assert_called_once_with( + 'Warning: Hashing algorithm upgrade Instance ID: {} has changed \n from: {} \n to: {}'.format( + mock_instance.id, + 'hash1', + 'hash2' + )) + self.assertEqual(result, mock_instance_list) + + # def rehash_existing_instances(self, instance_list): + # result = [] + # for instance in instance_list: + # prev_hash = instance.hash + # instance.hash_instance() + # new_hash = instance.hash + # if prev_hash != new_hash: + # logger.info( + # 'Warning: Hashing algorithm upgrade Instance ID: {} has changed \n from: {} \n to: {}'.format( + # instance.id, + # prev_hash, + # new_hash + # )) + # self.system_upgrade_hash_changes.append([prev_hash, new_hash]) + # self.session.add(instance) + # result.append(instance) + + # return result \ No newline at end of file From 3df6485426b629e9858b581a6cfae715490035c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 21:59:01 +0000 Subject: [PATCH 08/23] build(deps): bump apollo-server-core from 3.12.0 to 3.12.1 in /frontend Bumps [apollo-server-core](https://github.com/apollographql/apollo-server/tree/HEAD/packages/apollo-server-core) from 3.12.0 to 3.12.1. - [Release notes](https://github.com/apollographql/apollo-server/releases) - [Commits](https://github.com/apollographql/apollo-server/commits/apollo-server-core@3.12.1/packages/apollo-server-core) --- updated-dependencies: - dependency-name: apollo-server-core dependency-type: indirect ... Signed-off-by: dependabot[bot] --- frontend/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/yarn.lock b/frontend/yarn.lock index e73f26496..8772a95f6 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -6598,9 +6598,9 @@ apollo-reporting-protobuf@^3.4.0: "@apollo/protobufjs" "1.2.6" apollo-server-core@^3.12.0: - version "3.12.0" - resolved "https://registry.yarnpkg.com/apollo-server-core/-/apollo-server-core-3.12.0.tgz#8aa2a7329ce6fe1823290c45168c749db01548df" - integrity sha512-hq7iH6Cgldgmnjs9FVSZeKWRpi0/ZR+iJ1arzeD2VXGxxgk1mAm/cz1Tx0TYgegZI+FvvrRl0UhKEx7sLnIxIg== + version "3.12.1" + resolved "https://registry.yarnpkg.com/apollo-server-core/-/apollo-server-core-3.12.1.tgz#ba255c37345db29c48a2e0c064c519a8d62eb5af" + integrity sha512-9SF5WAkkV0FZQ2HVUWI9Jada1U0jg7e8NCN9EklbtvaCeUlOLyXyM+KCWuZ7+dqHxjshbtcwylPHutt3uzoNkw== dependencies: "@apollo/utils.keyvaluecache" "^1.0.1" "@apollo/utils.logger" "^1.0.0" From fae130fda64c4183bb1282b9770b056055138b25 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Thu, 31 Aug 2023 11:00:37 -0600 Subject: [PATCH 09/23] Cleanup --- default/tests/shared/test_annotation_new.py | 28 +++------------------ shared/annotation.py | 1 - 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index 3154e86ef..dbfd2f10e 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -2,7 +2,7 @@ from default.tests.test_utils import testing_setup from shared.annotation import Annotation_Update from unittest.mock import patch, MagicMock, Mock, call - +from unittest import skip class TestAnnotationUpdate(testing_setup.DiffgramBaseTestCase): @@ -242,10 +242,11 @@ def test_rehash_existing_instances_same_hash(self, mock_logger): self.assertEqual(self.instance.system_upgrade_hash_changes, []) mock_logger.info.assert_not_called() - # TODO: side_effect isnt working here + # TODO: the side_effect func isnt working, hash is still set to 'hash1' after the action + @skip("side_effect func isnt working as expected") @patch('shared.annotation.logger') def test_rehash_existing_instances_different_hash(self, mock_logger): - # Arrange + # Arrange mock_instance = Mock(spec=Annotation_Update).return_value mock_instance.id = '123' mock_instance.hash = 'hash1' @@ -257,10 +258,8 @@ def side_effect(): mock_instance.hash_instances.side_effect = side_effect mock_instance_list = [mock_instance] - print("HASH IS: ", mock_instance.hash) # Act result = self.instance.rehash_existing_instances(mock_instance_list) - print("HASH NOW: ", mock_instance.hash) # Assert self.instance.session.add.assert_called_once_with(mock_instance) @@ -272,22 +271,3 @@ def side_effect(): 'hash2' )) self.assertEqual(result, mock_instance_list) - - # def rehash_existing_instances(self, instance_list): - # result = [] - # for instance in instance_list: - # prev_hash = instance.hash - # instance.hash_instance() - # new_hash = instance.hash - # if prev_hash != new_hash: - # logger.info( - # 'Warning: Hashing algorithm upgrade Instance ID: {} has changed \n from: {} \n to: {}'.format( - # instance.id, - # prev_hash, - # new_hash - # )) - # self.system_upgrade_hash_changes.append([prev_hash, new_hash]) - # self.session.add(instance) - # result.append(instance) - - # return result \ No newline at end of file diff --git a/shared/annotation.py b/shared/annotation.py index a97da3e58..2a009b40e 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -611,7 +611,6 @@ def annotation_update_main(self): logger.error(f"Error updating annotation {str(self.log)}") return self.return_orginal_file_type() - # TODO: Whats the point of this? def main(self): return self.annotation_update_main() From 9031c9079d47bf3de0c3d566a7f8d0a11f1896ca Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Thu, 31 Aug 2023 13:15:04 -0600 Subject: [PATCH 10/23] Cleanup --- shared/annotation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shared/annotation.py b/shared/annotation.py index 2a009b40e..b67bf38b5 100755 --- a/shared/annotation.py +++ b/shared/annotation.py @@ -553,7 +553,7 @@ def order_new_instances_by_date(self): reverse = True) return self.instance_list_new - # Tested measily, needs a big cleanup + # Tested def annotation_update_main(self): """ @@ -758,7 +758,8 @@ def detect_and_remove_collisions(self, instance_list): self.session.add(inst) return result - + + # Tested def rehash_existing_instances(self, instance_list): result = [] for instance in instance_list: From bedf37b1e8dbbad925fa30296a0a20ec90626454 Mon Sep 17 00:00:00 2001 From: Francesco Virga Date: Thu, 31 Aug 2023 13:21:31 -0600 Subject: [PATCH 11/23] Comments --- default/tests/shared/test_annotation_new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default/tests/shared/test_annotation_new.py b/default/tests/shared/test_annotation_new.py index dbfd2f10e..0672f91b8 100644 --- a/default/tests/shared/test_annotation_new.py +++ b/default/tests/shared/test_annotation_new.py @@ -14,7 +14,7 @@ def setUp(self): self.mock_file = MagicMock() self.instance = Annotation_Update(session=self.mock_session, file=self.mock_file, member=self.mock_member, project=self.mock_project) - # TODO: How can we clean this up / re-use all the setup? + # TODO: Look at how we can clean up all the patching, ideally some way we can re-use it between tests @patch('shared.annotation.regular_log.default') @patch('shared.annotation.Project.get_by_id') From 7733860c2d50ba7a70da8ef235391f90b5d03306 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:43:51 +0000 Subject: [PATCH 12/23] build(deps): bump cryptography from 41.0.3 to 41.0.4 in /walrus Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- walrus/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/walrus/requirements.txt b/walrus/requirements.txt index b944bb45b..24340f8f7 100644 --- a/walrus/requirements.txt +++ b/walrus/requirements.txt @@ -22,7 +22,7 @@ memory-profiler==0.55.0 tenacity>=6.0.0 botocore==1.19.63 scaleapi==0.3.1 -cryptography==41.0.3 +cryptography==41.0.4 colorlog==4.2.1 labelbox==3.47.1 boto3==1.16.63 From e3fbd9ecd775d7abb9a713c8726f211a6e759aab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:44:06 +0000 Subject: [PATCH 13/23] build(deps): bump cryptography from 41.0.3 to 41.0.4 in /default Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- default/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default/requirements.txt b/default/requirements.txt index 6de009ac6..911bf9a85 100644 --- a/default/requirements.txt +++ b/default/requirements.txt @@ -20,7 +20,7 @@ imageio==2.9.0 pyotp==2.2.6 analytics-python==1.2.9 memory-profiler==0.55.0 -cryptography==41.0.3 +cryptography==41.0.4 colorlog==4.2.1 boto3==1.16.4 alembic==1.11.1 From afb0328d09e19eca86309d42bbf160160f8808d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:44:31 +0000 Subject: [PATCH 14/23] build(deps): bump cryptography from 41.0.3 to 41.0.4 in /eventhandlers Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- eventhandlers/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventhandlers/requirements.txt b/eventhandlers/requirements.txt index b8928477b..aae081170 100644 --- a/eventhandlers/requirements.txt +++ b/eventhandlers/requirements.txt @@ -16,7 +16,7 @@ Flask-SSLify==0.1.5 requests==2.31.0 imageio==2.9.0 analytics-python==1.2.9 -cryptography==41.0.3 +cryptography==41.0.4 colorlog==4.2.1 boto3==1.16.4 azure-storage-blob==12.8.0 From 059b4b24e6437adde88f467c33bc6b29372c58e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 21:29:30 +0000 Subject: [PATCH 15/23] build(deps): bump get-func-name from 2.0.0 to 2.0.2 in /frontend Bumps [get-func-name](https://github.com/chaijs/get-func-name) from 2.0.0 to 2.0.2. - [Release notes](https://github.com/chaijs/get-func-name/releases) - [Commits](https://github.com/chaijs/get-func-name/commits/v2.0.2) --- updated-dependencies: - dependency-name: get-func-name dependency-type: indirect ... Signed-off-by: dependabot[bot] --- frontend/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 94beb39c4..32840d8ae 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -12037,9 +12037,9 @@ get-caller-file@^2.0.1, get-caller-file@^2.0.5: integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== get-func-name@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41" - integrity sha1-6td0q+5y4gQJQzoGY2YCPdaIekE= + version "2.0.2" + resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41" + integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ== get-intrinsic@^1.0.2: version "1.1.3" From 3dad73542801b786e7479a8f62e0d252e74291a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 00:55:49 +0000 Subject: [PATCH 16/23] build(deps): bump pillow from 9.3.0 to 10.0.1 in /walrus Bumps [pillow](https://github.com/python-pillow/Pillow) from 9.3.0 to 10.0.1. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/9.3.0...10.0.1) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- walrus/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/walrus/requirements.txt b/walrus/requirements.txt index b944bb45b..5886521f8 100644 --- a/walrus/requirements.txt +++ b/walrus/requirements.txt @@ -6,7 +6,7 @@ gunicorn==20.0.4 psycopg2-binary==2.9.6 numpy==1.24.3 scipy==1.10.1 -pillow==9.3.0 +pillow==10.0.1 moviepy==1.0.3 google-cloud-storage==1.35.1 google-api-python-client==1.12.3 From cf857ffe841af1630690a5584574aa52ff56ab7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:40:44 +0000 Subject: [PATCH 17/23] build(deps): bump @babel/traverse from 7.16.8 to 7.23.2 in /frontend Bumps [@babel/traverse](https://github.com/babel/babel/tree/HEAD/packages/babel-traverse) from 7.16.8 to 7.23.2. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.23.2/packages/babel-traverse) --- updated-dependencies: - dependency-name: "@babel/traverse" dependency-type: indirect ... Signed-off-by: dependabot[bot] --- frontend/yarn.lock | 151 +++++++++++++++++++++++++++++++-------------- 1 file changed, 103 insertions(+), 48 deletions(-) diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 94beb39c4..f5782d006 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -168,6 +168,14 @@ dependencies: "@babel/highlight" "^7.18.6" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.16.4", "@babel/compat-data@^7.16.8": version "7.16.8" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.16.8.tgz#31560f9f29fdf1868de8cb55049538a1b9732a60" @@ -278,7 +286,7 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/generator@^7.16.7", "@babel/generator@^7.16.8": +"@babel/generator@^7.16.7": version "7.16.8" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.16.8.tgz#359d44d966b8cd059d543250ce79596f792f2ebe" integrity sha512-1ojZwE9+lOXzcWdWmO6TbUzDfqLD39CmEhN8+2cX9XkDo5yW1OpgfejfliysR2AWLpMamTiOiAp/mtroaymhpw== @@ -296,6 +304,16 @@ "@jridgewell/gen-mapping" "^0.3.2" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.16.7.tgz#bb2339a7534a9c128e3102024c60760a3a7f3862" @@ -492,6 +510,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.21.5.tgz#c769afefd41d171836f7cb63e295bedf689d48ba" integrity sha512-IYl4gZ3ETsWocUWgsFZLM5i1BYx9SoemminVEXadgLBa9TdeorzgLKm8wWLA6J1N/kT3Kch8XIk1laNzYoHKvQ== +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.16.7.tgz#12a6d8522fdd834f194e868af6354e8650242b7a" @@ -531,6 +554,14 @@ "@babel/template" "^7.20.7" "@babel/types" "^7.21.0" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-get-function-arity@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.7.tgz#ea08ac753117a669f1508ba06ebcc49156387419" @@ -552,6 +583,13 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.16.7.tgz#42b9ca4b2b200123c3b7e726b0ae5153924905b0" @@ -779,6 +817,13 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-string-parser@^7.18.10": version "7.18.10" resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.18.10.tgz#181f22d28ebe1b3857fa575f5c290b1aaf659b56" @@ -789,6 +834,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.21.5.tgz#2b3eea65443c6bdc31c22d037c65f6d323b6b2bd" integrity sha512-5pTUx3hAJaZIdW99sJ6ZUUgWq/Y+Hja7TowEnLNMm1VivRgZQL3vpBY3qUACVsvw+yQU6+YgfBVmcbLaZtrA1w== +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" @@ -804,6 +854,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.16.7.tgz#b203ce62ce5fe153899b617c08957de860de4d23" @@ -884,12 +939,21 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.1.6", "@babel/parser@^7.14.7", "@babel/parser@^7.16.4", "@babel/parser@^7.16.7", "@babel/parser@^7.16.8", "@babel/parser@^7.7.0": +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + +"@babel/parser@^7.1.0", "@babel/parser@^7.1.6", "@babel/parser@^7.14.7", "@babel/parser@^7.16.4", "@babel/parser@^7.16.7", "@babel/parser@^7.7.0": version "7.16.8" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.16.8.tgz#61c243a3875f7d0b0962b0543a33ece6ff2f1f17" integrity sha512-i7jDUfrVBWc+7OKcBzEe5n7fbv3i2fWtxKzzCvOjnzSxMfWMigAhtfJ7qzZNGFNMsCCd67+uz553dYKWXPvCKw== -"@babel/parser@^7.12.11", "@babel/parser@^7.12.7", "@babel/parser@^7.13.16", "@babel/parser@^7.18.4", "@babel/parser@^7.20.7", "@babel/parser@^7.21.4", "@babel/parser@^7.21.5", "@babel/parser@^7.21.8", "@babel/parser@^7.6.0", "@babel/parser@^7.9.6": +"@babel/parser@^7.12.11", "@babel/parser@^7.12.7", "@babel/parser@^7.13.16", "@babel/parser@^7.18.4", "@babel/parser@^7.20.7", "@babel/parser@^7.21.4", "@babel/parser@^7.21.8", "@babel/parser@^7.6.0", "@babel/parser@^7.9.6": version "7.21.8" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.8.tgz#642af7d0333eab9c0ad70b14ac5e76dbde7bfdf8" integrity sha512-6zavDGdzG3gUqAdWvlLFfk+36RilI+Pwyuuh7HItyeScCWP3k6i8vKclAQ0bM/0y/Kz/xiwvxhMv9MgTJP5gmA== @@ -899,6 +963,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.0.tgz#497fcafb1d5b61376959c1c338745ef0577aa02c" integrity sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.16.7.tgz#4eda6d6c2a0aa79c70fa7b6da67763dfe2141050" @@ -2708,51 +2777,28 @@ "@babel/parser" "^7.18.10" "@babel/types" "^7.18.10" -"@babel/traverse@^7.0.0", "@babel/traverse@^7.13.0", "@babel/traverse@^7.16.7", "@babel/traverse@^7.16.8", "@babel/traverse@^7.7.0": - version "7.16.8" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.16.8.tgz#bab2f2b09a5fe8a8d9cad22cbfe3ba1d126fef9c" - integrity sha512-xe+H7JlvKsDQwXRsBhSnq1/+9c+LlQcCK3Tn/l5sbx02HYns/cn7ibp9+RV1sIUqu7hKg91NWsgHurO9dowITQ== - dependencies: - "@babel/code-frame" "^7.16.7" - "@babel/generator" "^7.16.8" - "@babel/helper-environment-visitor" "^7.16.7" - "@babel/helper-function-name" "^7.16.7" - "@babel/helper-hoist-variables" "^7.16.7" - "@babel/helper-split-export-declaration" "^7.16.7" - "@babel/parser" "^7.16.8" - "@babel/types" "^7.16.8" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.12.11", "@babel/traverse@^7.12.9", "@babel/traverse@^7.21.5", "@babel/traverse@^7.7.2": - version "7.21.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.21.5.tgz#ad22361d352a5154b498299d523cf72998a4b133" - integrity sha512-AhQoI3YjWi6u/y/ntv7k48mcrCXmus0t79J9qPNlk/lAsFlCiJ047RmbfMOawySTHtywXhbXgpx/8nXMYd+oFw== - dependencies: - "@babel/code-frame" "^7.21.4" - "@babel/generator" "^7.21.5" - "@babel/helper-environment-visitor" "^7.21.5" - "@babel/helper-function-name" "^7.21.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.21.5" - "@babel/types" "^7.21.5" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.18.9", "@babel/traverse@^7.19.0": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.19.0.tgz#eb9c561c7360005c592cc645abafe0c3c4548eed" - integrity sha512-4pKpFRDh+utd2mbRC8JLnlsMUii3PMHjpL6a0SZ4NMZy7YFP9aXORxEhdMVOc9CpWtDF09IkciQLEhK7Ml7gRA== - dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.19.0" - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-function-name" "^7.19.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.19.0" - "@babel/types" "^7.19.0" +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.0.0", "@babel/traverse@^7.12.11", "@babel/traverse@^7.12.9", "@babel/traverse@^7.13.0", "@babel/traverse@^7.16.7", "@babel/traverse@^7.16.8", "@babel/traverse@^7.18.9", "@babel/traverse@^7.19.0", "@babel/traverse@^7.21.5", "@babel/traverse@^7.7.0", "@babel/traverse@^7.7.2": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -2782,6 +2828,15 @@ "@babel/helper-validator-identifier" "^7.18.6" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" From 7800f58a2d449f141f8bfd00cfc363e8a8cd831d Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:36:50 -0700 Subject: [PATCH 18/23] flag buggy test --- default/tests/methods/video/test_video_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/default/tests/methods/video/test_video_view.py b/default/tests/methods/video/test_video_view.py index 1c1cd706f..e2f42b783 100644 --- a/default/tests/methods/video/test_video_view.py +++ b/default/tests/methods/video/test_video_view.py @@ -32,6 +32,7 @@ def setUp(self): self.project = project_data['project'] def test_get_video_frame_from_task(self): + return # See https://github.com/diffgram/diffgram/issues/1560 # Create mock task job = data_mocking.create_job({ 'name': f"my-test-job-{1}", From 945bfc6d4939ec24a8af4b1fde516df926270939 Mon Sep 17 00:00:00 2001 From: Zakharchenko Sergey Date: Thu, 19 Oct 2023 11:17:58 -0700 Subject: [PATCH 19/23] fix hotkeys firing for multiple image annotation panes --- .../image_and_video_annotation/annotation_core.vue | 14 ++++++++++++-- frontend/src/utils/hotkey_listener.ts | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue b/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue index 4ba5b0f05..37676aab0 100644 --- a/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue +++ b/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue @@ -741,6 +741,14 @@ export default Vue.extend({ watch: { is_active: function (){ this.canvas_element.style.cursor = '' + if ( !this.hotkeyListenerScope ) { + return + } + if ( this.is_active) { + this.hotkey_listener.setScopes([this.hotkeyListenerScope]) + } else { + this.hotkey_listener.removeScope(this.hotkeyListenerScope) + } }, global_instance: function(){ this.$emit('global_instance_changed', this.working_file.id, this.global_instance) @@ -1733,6 +1741,10 @@ export default Vue.extend({ this.setupHotkeys(this.hotkeyListenerScope) + if ( this.is_active ) { + this.hotkey_listener.setScopes([this.hotkeyListenerScope]) + } + if ( this.annotation_ui_context.working_file_list[0].id === this.annotation_ui_context.working_file.id ) { @@ -1753,8 +1765,6 @@ export default Vue.extend({ setupHotkeys(scope) { - this.hotkey_listener.addScope(scope) - this.hotkey_listener.addFilter(this.hotkeyListenerFilter) this.hotkey_listener.onKeydown({ keys: 's', scope }, () => { diff --git a/frontend/src/utils/hotkey_listener.ts b/frontend/src/utils/hotkey_listener.ts index 508d9bd4e..26b37758e 100644 --- a/frontend/src/utils/hotkey_listener.ts +++ b/frontend/src/utils/hotkey_listener.ts @@ -143,6 +143,10 @@ export class HotkeyListener { return this.selectedScopes } + setScopes(scopes: Array) : void { + this.selectedScopes = scopes + } + addScope(scope: string) { if (!this.selectedScopes.includes(scope)) { this.selectedScopes.push(scope) From 11959d05d4ea356644627c270f5c37a3d77bedf7 Mon Sep 17 00:00:00 2001 From: Zakharchenko Sergey Date: Thu, 19 Oct 2023 11:44:58 -0700 Subject: [PATCH 20/23] update HotkeyListener documentation --- frontend/src/utils/hotkey_listener.ts | 33 ++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/frontend/src/utils/hotkey_listener.ts b/frontend/src/utils/hotkey_listener.ts index 26b37758e..adac77fbf 100644 --- a/frontend/src/utils/hotkey_listener.ts +++ b/frontend/src/utils/hotkey_listener.ts @@ -14,12 +14,10 @@ interface HotkeysOptions { scope: string element ?: HTMLElement, debounceThreshold?: number - platformDependent: boolean - preventDefaultEvent: boolean + platformDependent?: boolean + preventDefaultEvent?: boolean } -export type KeysOrOpts = HotkeysOptions - export type KeyEventHandler = (event: KeyboardEvent, handler: HotkeysEvent) => void /** * `HotkeyListener` - A utility class to manage keyboard hotkeys using the `hotkeys-js` library. @@ -37,12 +35,13 @@ export type KeyEventHandler = (event: KeyboardEvent, handler: HotkeysEvent) => v * - Fetch the listener instance: `const listener = HotkeyListener.getInstance()`. * * 2. **Hotkey Registration**: - * - Register a hotkey that reacts on keyup: `listener.onKeyup('ctrl+b', callback)`. - * - For keydown events: `listener.onKeydown('ctrl+b', callback)`.] + * - Register a hotkey that reacts on keyup: `listener.onKeyup({keys: 'ctrl+b', scope: 'foo'}, callback)`. + * - For keydown events: `listener.onKeydown({keys: 'ctrl+a,ctrl+b', scope: 'foo'}, callback)`.] * - For binding to individual special keys like shift, ctrl, etc. use onSpecialKeydown and onSpecialKeyup: `listener.onSpecialKeydown('shift', callback)` * * 3. **Scopes**: - * - Select scope and create if doesn't exist already: `listener.addScope('myScope')`. + * - Select only scopes passed as an argument: `listener.setScopes(['scope_one', 'scope_two'])`. + * - Select scope and keep previously selected scopes selected: `listener.addScope('myScope')`. * - Cancle selection of a scope: `listener.removeScope('myScope')`. * - Delete scope and all hotkey bindings: `listener.deleteScope('myScope')`. * - Note: A hotkey will only trigger if its scope is among the selected scopes. @@ -55,8 +54,26 @@ export type KeyEventHandler = (event: KeyboardEvent, handler: HotkeysEvent) => v * **Important**: Due to `hotkeys-js` providing a singleton instance, `HotkeyListener` uses the singleton pattern * to ensure a single centralized management point. * + * @example + * ```javascript + * const listener = HotkeyListener.getInstance(); // Initialization + * + * // Register hotkeys + * listener.onKeyup({keys: 'ctrl+a', scope: 'foo'}, () => {}); // Register callback to fire when 'ctrl+a' is pressed and scope 'foo' is active + * listener.onKeyup({keys: 'ctrl+b', scope: 'foo'}, () => {}); // Register callback to fire when 'ctrl+b' is pressed and scope 'foo' is active + * listener.onKeyup({keys: 'ctrl+c', scope: 'bar'}, () => {}); // Register callback to fire when 'ctrl+c' is pressed and scope 'bar' is active + * + * // Enable/disable scopes + * listener.addScopes('foo'); // Hotkey callbacks registered for 'foo' scope will be active + * listener.removeScope('foo'); // Hotkey callbacks registered for 'foo' scope won't fire + * listener.addScopes('bar'); // Hotkey callbacks registered for 'foo' and 'bar' scopes will be active + * listener.setScopes(['foo']); // Only hotkey callbacks registered for 'foo' scope will be active + * listener.deleteScope('foo'); // Unselects 'foo' scope and unregisters all the callbacks associated with that scope. + + * listener.addScope('bar'); // Hotkeys for 'bar' scope will still fire callbacks, but not for 'foo' scope since deleteScope was called on it + * ``` * TODO: - * 1. If we are ever in situation where we need to have many too many scopes registered + * 1. If we are ever in situation where we need to have too many scopes registered * at the same time, we can enhance HotkeyListener logic so it unregisters callbacks for non-active scopes. * This will reduce the number of active callbacks * @class From 5a7ae33307fe6218e4d61f1f21e2f0b096e07c22 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Thu, 19 Oct 2023 21:40:13 -0700 Subject: [PATCH 21/23] Fix tests Assumptions on hotkey_listener being available --- .../annotation/image_and_video_annotation/annotation_core.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue b/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue index 37676aab0..74a86b329 100644 --- a/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue +++ b/frontend/src/components/annotation/image_and_video_annotation/annotation_core.vue @@ -1741,7 +1741,7 @@ export default Vue.extend({ this.setupHotkeys(this.hotkeyListenerScope) - if ( this.is_active ) { + if(this.is_active && this.hotkey_listener) { this.hotkey_listener.setScopes([this.hotkeyListenerScope]) } From 505fc556b86c033ebd8c8fed3955806a23352b23 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Thu, 19 Oct 2023 21:45:54 -0700 Subject: [PATCH 22/23] Add default PRs Otherwise requires moving from Draft to ready for review. Leaving E2E tests as is for now. --- .github/workflows/diffgram_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diffgram_testing.yaml b/.github/workflows/diffgram_testing.yaml index e5e196ec7..70b2a5b35 100644 --- a/.github/workflows/diffgram_testing.yaml +++ b/.github/workflows/diffgram_testing.yaml @@ -11,7 +11,7 @@ on: branches: - '**' pull_request: - types: [ready_for_review] + types: [opened, synchronize, ready_for_review] jobs: Default-Service-Unit-Tests: runs-on: ubuntu-20.04 From 8bf8ccb99150dab6b10bce1e6a0b674620613135 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Thu, 19 Oct 2023 21:51:35 -0700 Subject: [PATCH 23/23] Fix tests --- frontend/tests/unit/annotation/annotation_core.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/tests/unit/annotation/annotation_core.spec.js b/frontend/tests/unit/annotation/annotation_core.spec.js index ae2f4e3cd..555e44080 100644 --- a/frontend/tests/unit/annotation/annotation_core.spec.js +++ b/frontend/tests/unit/annotation/annotation_core.spec.js @@ -84,6 +84,7 @@ describe("Test annotation_core", () => { onKeydown: jest.fn(), onSpecialKeydown: jest.fn(), onSpecialKeyup: jest.fn(), + setScopes: jest.fn() }, label_schema: { id: 1,