feathr-ai · xiaoyongzhu · Dec 1, 2022 · Nov 10, 2022 · Nov 12, 2022 · Nov 12, 2022
diff --git a/FeathrRegistry.Dockerfile b/FeathrRegistry.Dockerfile
@@ -11,7 +11,7 @@ RUN npm install && npm run build
 FROM python:3.9
 
 ## Install dependencies
-RUN apt-get update -y && apt-get install -y nginx
+RUN apt-get update -y && apt-get install -y nginx freetds-dev
 COPY ./registry /usr/src/registry
 WORKDIR /usr/src/registry/sql-registry
 RUN pip install -r requirements.txt

diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py
@@ -244,6 +244,24 @@ def list_registered_features(self, project_name: str = None) -> List[str]:
         `project_name` must not be None or empty string because it violates the RBAC policy
         """
         return self.registry.list_registered_features(project_name)
+
+    def delete_project(self, project_name: str):
+        """
+        Deletes given feature for 
+        """
+        return self.registry.delete_project(project_name)
+
+    def delete_anchored_feature(self, project_name: str, anchor_name: str, feature_name: str):
+        """
+        Deletes anchored feature associated with project and anchor
+        """
+        return self.registry.delete_anchored_feature(project_name, anchor_name, feature_name)
+
+    def delete_derived_feature(self, project_name: str, feature_name: str):
+        """
+        Deletes derived feature 
+        """
+        return self.registry.delete_derived_feature(project_name, feature_name)
 
     def _get_registry_client(self):
         """

diff --git a/feathr_project/feathr/registry/_feathr_registry_client.py b/feathr_project/feathr/registry/_feathr_registry_client.py
@@ -137,6 +137,29 @@ def list_registered_features(self, project_name: str) -> List[str]:
             "qualifiedName": r["attributes"]["qualifiedName"],
         } for r in resp]
 
+    def delete_project(self, project_name: str):
+        """
+        Deletes project
+        """
+        r = self._delete(f"/projects/{project_name}")
+        return r
+
+    def delete_anchored_feature(self, project_name: str, anchor_name: str, feature_name: str):
+        """
+        Deletes anchored feature
+        """
+        qualified_name = f"{project_name}__{anchor_name}__{feature_name}"
+        r = self._delete(f"/features/{qualified_name}")
+        return r
+
+    def delete_derived_feature(self, project_name: str, feature_name: str):
+        """
+        Deletes derived feature
+        """
+        qualified_name = f"{project_name}__{feature_name}"
+        r = self._delete(f"/features/{qualified_name}")
+        return r
+
     def get_features_from_registry(self, project_name: str) -> Tuple[List[FeatureAnchor], List[DerivedFeature]]:
         """
         [Sync Features from registry to local workspace, given a project_name, will write project's features from registry to to user's local workspace]
@@ -187,6 +210,10 @@ def _create_derived_feature(self, s: DerivedFeature) -> UUID:
     def _get(self, path: str) -> dict:
         logging.debug("PATH: ", path)
         return check(requests.get(f"{self.endpoint}{path}", headers=self._get_auth_header())).json()
+
+    def _delete(self, path: str) -> dict:
+        logging.debug("PATH: ", path)
+        return check(requests.delete(f"{self.endpoint}{path}", headers=self._get_auth_header())).json()
 
     def _post(self, path: str, body: dict) -> dict:
         logging.debug("PATH: ", path)

diff --git a/feathr_project/feathr/registry/feature_registry.py b/feathr_project/feathr/registry/feature_registry.py
@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
+from re import L
 
 from typing import Any, Dict, List, Optional, Tuple
 from feathr.definition.feature_derivations import DerivedFeature
@@ -28,6 +29,27 @@ def list_registered_features(self, project_name: str) -> List[str]:
         """
         pass
 
+    @abstractmethod
+    def delete_project(self, project_name: str):
+        """
+        Deletes project given project name. Project name must not be empty.
+        """
+        pass
+
+    @abstractmethod
+    def delete_anchored_feature(self, project_name: str, anchor_name: str, feature_name: str):
+        """
+        Deletes anchor feature
+        """
+        pass
+
+    @abstractmethod
+    def delete_derived_feature(self, project_name: str, feature_name: str):
+        """
+        Deletes derived feature
+        """
+        pass
+
     @abstractmethod
     def get_features_from_registry(self, project_name: str) -> Tuple[List[FeatureAnchor], List[DerivedFeature]]:
         """[Sync Features from registry to local workspace, given a project_name, will write project's features from registry to to user's local workspace]

diff --git a/registry/access_control/api.py b/registry/access_control/api.py
@@ -25,6 +25,11 @@ async def get_project(project: str, access: UserAccess  = Depends(project_read_a
                             headers=get_api_header(access.user_name)).content.decode('utf-8')
     return json.loads(response)
 
+@router.delete("/projects/{project}", name="Delete project [Write Access Required]")
+def delete_project(project: str, access: UserAccess = Depends(project_write_access)):
+    response = requests.delete(url=f"{registry_url}/projects/{project}",
+                            headers=get_api_header(access.user_name)).content.decode('utf-8')
+    return json.loads(response)
 
 @router.get("/projects/{project}/datasources", name="Get data sources of my project [Read Access Required]")
 def get_project_datasources(project: str, access: UserAccess = Depends(project_read_access)) -> list:
@@ -58,6 +63,11 @@ def get_feature(feature: str, requestor: User = Depends(get_user)) -> dict:
         feature_qualifiedName, requestor, AccessType.READ)
     return ret
 
+@router.delete("/features/{feature}", name="Deletes a single feature by feature ID [Write Access Required]")
+def delete_feature(feature: str, access: UserAccess = Depends(project_write_access)) -> str:
+    response = requests.delete(url=f"{registry_url}/features/{feature}",
+                               headers=get_api_header(access.user_name)).content.decode('utf-8')
+    return json.loads(response)
 
 @router.get("/features/{feature}/lineage", name="Get Feature Lineage [Read Access Required]")
 def get_feature_lineage(feature: str, requestor: User = Depends(get_user)) -> dict:

diff --git a/registry/purview-registry/api-spec.md b/registry/purview-registry/api-spec.md
@@ -287,6 +287,9 @@ Get everything defined in the project
 
 Response Type: [`EntitiesAndRelationships`](#entitiesandrelationships)
 
+### `DELETE /projects/{project}`
+Deletes project
+
 ### `GET /projects/{project}/datasources`
 Get all sources defined in the project.
 
@@ -320,6 +323,9 @@ Response Type: Object
 | entity          | [`Entity`](#entity)   |                             |
 | referredEntities| `map<string, object>` | For compatibility, not used |
 
+### `DELETE /features/{feature}`
+Deletes feature
+
 ### `POST /projects`
 Create new project
 

diff --git a/registry/purview-registry/main.py b/registry/purview-registry/main.py
@@ -56,6 +56,14 @@ def get_projects_ids() -> dict:
 def get_projects(project: str) -> dict:
     return to_camel(registry.get_project(project).to_dict())
 
+@router.delete("/projects/{project}",tags=["Project"])
+def delete_project(project: str) -> str:
+    project_entity = registry.get_project(project)
+    if project_entity is None:
+        raise HTTPException(
+            status_code=404, details=f"Project {project} not found"
+        )
+    return registry.delete_project(project, project_entity)
 
 @router.get("/projects/{project}/datasources",tags=["Project"])
 def get_project_datasources(project: str) -> list:
@@ -90,6 +98,13 @@ def get_feature(feature: str) -> dict:
             status_code=404, detail=f"Feature {feature} not found")
     return to_camel(e.to_dict())
 
+@router.delete("/features/{feature}", tags=["Feature"])
+def delete_feature(feature: str) -> str:
+    e = registry.get_entity(feature)
+    if e.entity_type not in [EntityType.DerivedFeature, EntityType.AnchorFeature]:
+        raise HTTPException(
+            status_code=404, detail=f"Feature {feature} not found")
+    return registry.delete_feature(feature)
 
 @router.get("/features/{feature}/lineage",tags=["Feature"])
 def get_feature_lineage(feature: str) -> dict:

diff --git a/registry/purview-registry/registry/interface.py b/registry/purview-registry/registry/interface.py
@@ -92,3 +92,11 @@ def create_project_anchor_feature(self, project_id: UUID, anchor_id: UUID, defin
     @abstractmethod
     def create_project_derived_feature(self, project_id: UUID, definition: DerivedFeatureDef) -> UUID:
         pass
+
+    @abstractmethod
+    def delete_feature(self, id: Union[str,UUID]) -> str:
+        pass
+
+    @abstractmethod
+    def delete_project(self, project_id: Union[str, UUID], project: EntitiesAndRelations) -> str:
+        pass
diff --git a/registry/purview-registry/registry/purview_registry.py b/registry/purview-registry/registry/purview_registry.py
@@ -192,6 +192,55 @@ def get_lineage(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations:
             upstream_entities + downstream_entities,
             upstream_edges + downstream_edges)
 
+    def delete_feature(self, id_or_name: Union[str, UUID]) -> str:
+        """
+        Deletes either AnchorFeature or DerivedFeature recursively
+        """
+        id = self.get_entity_id(id_or_name)
+        visited = []
+        children = []
+        visited.append(id)
+        children.append(id)
+        while children:
+            child = children.pop(0)
+            downstream_entities, _ = self._bfs(child, RelationshipType.Produces)
+            neighbors = self.get_all_neighbours(child)
+            downstream_guids = [x.id for x in downstream_entities]
+            edge_guids = [str(x.id) for x in neighbors]
+            ## Delete all edges associated with entity
+            self.purview_client.delete_entity(edge_guids)
+            ## Delete current entity
+            self.purview_client.delete_entity(str(child))
+            for downstream_entity in downstream_guids:
+                if downstream_entity not in visited:
+                    visited.append(downstream_entity)
+                    children.append(downstream_entity)
+        return str(id)
+
+    def delete_project(self, project_id: str, project: EntitiesAndRelations) -> str:
+        """
+        Deletes project by deleting all child components first
+        """
+        project_id = self.get_entity_id(project_id)
+        entity_mappings = project.entities
+        entity_mappings.pop(project_id)
+        for entity_id, entity in entity_mappings.items():
+            if entity.entity_type in (EntityType.AnchorFeature, EntityType.DerivedFeature):
+                self.delete_feature(entity_id)
+            else:
+                neighbors = self.get_all_neighbours(entity_id)
+                edge_guids = [str(x.id) for x in neighbors]
+                ## Delete all edges associated with entity
+                self.purview_client.delete_entity(edge_guids)
+                ## Delete entity
+                self.purview_client.delete_entity(entity_id)
+        ## Finally delete project
+        neighbors = self.get_all_neighbours(project_id)
+        edge_guids = [str(x.id) for x in neighbors]
+        self.purview_client.delete_entity(edge_guids)
+        self.purview_client.delete_entity(project_id)
+        return project_id
+
     def _get_edges(self, ids: list[UUID]) -> list[Edge]:
         all_edges = set()
         for id in ids:

diff --git a/registry/sql-registry/api-spec.md b/registry/sql-registry/api-spec.md
@@ -285,6 +285,9 @@ Response Type: `dict`
 ### `GET /projects/{project}`
 Get everything defined in the project
 
+### `DELETE /projects/{project}`
+Deletes project
+
 Response Type: [`EntitiesAndRelationships`](#entitiesandrelationships)
 
 ### `GET /projects/{project}/datasources`
@@ -320,6 +323,9 @@ Response Type: Object
 | entity          | [`Entity`](#entity)   |                             |
 | referredEntities| `map<string, object>` | For compatibility, not used |
 
+### `DELETE /features/{feature}`
+Deletes feature from project
+
 ### `POST /projects`
 Create new project
 

diff --git a/registry/sql-registry/main.py b/registry/sql-registry/main.py
@@ -86,6 +86,14 @@ def get_projects_ids() -> dict:
 def get_projects(project: str) -> dict:
     return registry.get_project(project).to_dict()
 
+@router.delete("/projects/{project}")
+def delete_project(project: str) -> str:
+    project_entity = registry.get_project(project)
+    if project_entity is None:
+        raise HTTPException(
+            status_code=404, details=f"Project {project} not found"
+        )
+    return registry.delete_project(project, project_entity)
 
 @router.get("/projects/{project}/datasources")
 def get_project_datasources(project: str) -> list:
@@ -135,13 +143,19 @@ def get_feature(feature: str) -> dict:
             status_code=404, detail=f"Feature {feature} not found")
     return e.to_dict()
 
+@router.delete("/features/{feature}")
+def delete_feature(feature: str) -> str:
+    e = registry.get_entity(feature)
+    if e.entity_type not in [EntityType.DerivedFeature, EntityType.AnchorFeature]:
+        raise HTTPException(
+            status_code=404, detail=f"Feature {feature} not found")
+    return registry.delete_feature(feature)
 
 @router.get("/features/{feature}/lineage")
 def get_feature_lineage(feature: str) -> dict:
     lineage = registry.get_lineage(feature)
     return lineage.to_dict()
 
-
 @router.post("/projects")
 def new_project(definition: dict) -> dict:
     id = registry.create_project(ProjectDef(**to_snake(definition)))

diff --git a/registry/sql-registry/registry/db_registry.py b/registry/sql-registry/registry/db_registry.py
@@ -105,6 +105,49 @@ def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations:
             df.attributes.input_features = features
         all_edges = self._get_edges(ids)
         return EntitiesAndRelations([project] + children, list(edges.union(all_edges)))
+
+    def delete_feature(self, id: Union[str,UUID]):
+        """
+        Deletes Feature. An AnchorFeature or DerivedFeature can produce additional downstream features.
+        Hence, to delete all downstream features we use BFS.
+        """
+        id = self.get_entity_id(id)
+        visited = []
+        children = []
+        visited.append(id)
+        children.append(id)
+        with self.conn.transaction() as c:
+            while children:
+                child = children.pop(0)
+                downstream_entities, _ = self._bfs(child, RelationshipType.Produces)
+                downstream_entity_keys = [x.id for x in downstream_entities]
+                self._delete_all_entity_edges(c, child)
+                self._delete_entity(c, child)
+                for downstream_entity in downstream_entity_keys:
+                    if downstream_entity not in visited:
+                        visited.append(downstream_entity)
+                        children.append(downstream_entity)
+        return str(id)
+
+    def delete_project(self, project_id: str, project: EntitiesAndRelations):
+        """
+        Deletes Project by first deleting all children entities and then finally deleting project
+        """
+        project_id = self.get_entity_id(project_id)
+        entity_mappings = project.entities
+        entity_mappings.pop(project_id)
+        with self.conn.transaction() as c:
+            ## Delete Children Entities first. Check for Feature Entitites to ensure they are recursively deleted
+            for entity_id, entity in entity_mappings.items():
+                if entity.entity_type in (EntityType.AnchorFeature, EntityType.DerivedFeature):
+                    self.delete_feature(entity_id)
+                else:
+                    self._delete_all_entity_edges(c, entity_id)
+                    self._delete_entity(c, entity_id)
+            ## Finally delete project
+            self._delete_all_entity_edges(c, project_id)
+            self._delete_entity(c, project_id)
+        return project_id
 
     def search_entity(self,
                       keyword: str,
@@ -386,6 +429,20 @@ def _create_edge(self, cursor, from_id: UUID, to_id: UUID, type: RelationshipTyp
             "to_id": str(to_id),
             "type": type.name
         })
+
+    def _delete_all_entity_edges(self, cursor, entity_id: UUID):
+        """
+        Deletes all edges associated with an entity
+        """
+        sql = fr'''DELETE FROM edges WHERE from_id = %s OR to_id = %s'''
+        cursor.execute(sql, (str(entity_id), str(entity_id)))
+
+    def _delete_entity(self, cursor, entity_id: UUID):
+        """
+        Deletes entity from entities table
+        """
+        sql = fr'''DELETE FROM entities WHERE entity_id = %s'''
+        cursor.execute(sql, str(entity_id))
 
     def _fill_entity(self, e: Entity) -> Entity:
         """

diff --git a/registry/sql-registry/registry/interface.py b/registry/sql-registry/registry/interface.py
@@ -111,3 +111,17 @@ def create_project_derived_feature(self, project_id: UUID, definition: DerivedFe
         Create a new derived feature under the project
         """
         pass
+
+    @abstractmethod
+    def delete_feature(self, id: Union[str,UUID]) -> str:
+        """
+        Deletes feature
+        """
+        pass
+
+    @abstractmethod
+    def delete_project(self, project_id: str, project: EntitiesAndRelations) -> str:
+        """
+        Deletes project
+        """
+        pass