feathr-ai · enya-yx · Jan 10, 2023 · Jan 3, 2023 · Jan 3, 2023 · Jan 3, 2023
diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml
@@ -257,6 +257,39 @@ jobs:
         run: |
           # skip cloud related tests
           pytest --cov-report term-missing --cov=feathr_project/feathr/spark_provider feathr_project/test/test_local_spark_e2e.py --cov-config=.github/workflows/.coveragerc_local
+
+  registry_test:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'registry test'))
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: "8"
+          distribution: "temurin"
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Feathr Package
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f ./registry/test/requirements.txt ]; then pip install -r ./registry/test/requirements.txt; fi
+          if [ -f ./registry/purview-registry/requirements.txt ]; then pip install -r ./registry/purview-registry/requirements.txt; fi
+          if [ -f ./registry/sql-registry/requirements.txt ]; then pip install -r ./registry/sql-registry/requirements.txt; fi
+      - name: Run Registry Test Cases
+        env:
+          AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
+          AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
+          AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}}
+          PURVIEW_NAME: ${{secrets.PURVIEW_NAME}}
+          CONNECTION_STR: ${{secrets.CONNECTION_STR}}
+        run: |
+          pytest --cov-report term-missing --cov=registry/sql-registry/registry --cov-config=registry/test/.coveragerc registry/test/test_sql_registry.py
+          pytest --cov-report term-missing --cov=registry/purview-registry/registry --cov-config=registry/test/.coveragerc registry/test/test_purview_registry.py
 
   failure_notification:
     # If any failure, warning message will be sent

diff --git a/registry/purview-registry/main.py b/registry/purview-registry/main.py
@@ -1,7 +1,7 @@
 import os
 import traceback
 from re import sub
-from typing import Optional
+from typing import Optional, Dict, List
 from uuid import UUID
 from fastapi import APIRouter, FastAPI, HTTPException
 from fastapi.responses import JSONResponse
@@ -46,7 +46,7 @@ def to_camel(s):
                    )
 
 
-def exc_to_content(e: Exception) -> dict:
+def exc_to_content(e: Exception) -> Dict:
     content={"message": str(e)}
     if os.environ.get("REGISTRY_DEBUGGING"):
         content["traceback"] = "".join(traceback.TracebackException.from_exception(e).format())
@@ -97,19 +97,19 @@ async def index_error_handler(_, exc: IndexError):
     )
 
 @router.get("/projects",tags=["Project"])
-def get_projects() -> list[str]:
+def get_projects() -> List[str]:
     return registry.get_projects()
 
 @router.get("/projects-ids")
-def get_projects_ids() -> dict:
+def get_projects_ids() -> Dict:
     return registry.get_projects_ids()
 
 @router.get("/projects/{project}",tags=["Project"])
-def get_projects(project: str) -> dict:
+def get_projects(project: str) -> Dict:
     return to_camel(registry.get_project(project).to_dict())
 
 @router.get("/dependent/{entity}")
-def get_dependent_entities(entity: str) -> list:
+def get_dependent_entities(entity: str) -> List:
     entity_id = registry.get_entity_id(entity)
     downstream_entities = registry.get_dependent_entities(entity_id)
     return list([e.to_dict() for e in downstream_entities])
@@ -126,15 +126,15 @@ def delete_entity(entity: str):
     registry.delete_entity(entity_id)
 
 @router.get("/projects/{project}/datasources",tags=["Project"])
-def get_project_datasources(project: str) -> list:
+def get_project_datasources(project: str) -> List:
     p = registry.get_entity(project,True)
     source_ids = [s.id for s in p.attributes.sources]
     sources = registry.get_entities(source_ids)
     return list([to_camel(e.to_dict()) for e in sources])
 
 
 @router.get("/projects/{project}/datasources/{datasource}",tags=["Project"])
-def get_datasource(project: str, datasource: str) -> dict:
+def get_datasource(project: str, datasource: str) -> Dict:
     p = registry.get_entity(project,True)
     for s in p.attributes.sources:
         if str(s.id) == datasource:
@@ -145,55 +145,55 @@ def get_datasource(project: str, datasource: str) -> dict:
 
 
 @router.get("/projects/{project}/features",tags=["Project"])
-def get_project_features(project: str, keyword: Optional[str] = None) -> list:
+def get_project_features(project: str, keyword: Optional[str] = None) -> List:
     atlasEntities = registry.get_project_features(project, keywords=keyword)
     return list([to_camel(e.to_dict()) for e in atlasEntities])
 
 
 @router.get("/features/{feature}",tags=["Feature"])
-def get_feature(feature: str) -> dict:
+def get_feature(feature: str) -> Dict:
     e = registry.get_entity(feature,True)
     if e.entity_type not in [EntityType.DerivedFeature, EntityType.AnchorFeature]:
         raise HTTPException(
             status_code=404, detail=f"Feature {feature} not found")
     return to_camel(e.to_dict())
 
 @router.get("/features/{feature}/lineage",tags=["Feature"])
-def get_feature_lineage(feature: str) -> dict:
+def get_feature_lineage(feature: str) -> Dict:
     lineage = registry.get_lineage(feature)
     return to_camel(lineage.to_dict())
 
 
 @router.post("/projects",tags=["Project"])
-def new_project(definition: dict) -> UUID:
+def new_project(definition: Dict) -> UUID:
     id = registry.create_project(ProjectDef(**to_snake(definition)))
     return {"guid": str(id)}
 
 
 @router.post("/projects/{project}/datasources",tags=["Project"])
-def new_project_datasource(project: str, definition: dict) -> UUID:
+def new_project_datasource(project: str, definition: Dict) -> UUID:
     project_id = registry.get_entity_id(project)
     id = registry.create_project_datasource(project_id, SourceDef(**to_snake(definition)))
     return {"guid": str(id)}
 
 
 @router.post("/projects/{project}/anchors",tags=["Project"])
-def new_project_anchor(project: str, definition: dict) -> UUID:
+def new_project_anchor(project: str, definition: Dict) -> UUID:
     project_id = registry.get_entity_id(project)
     id = registry.create_project_anchor(project_id, AnchorDef(**to_snake(definition)))
     return {"guid": str(id)}
 
 
 @router.post("/projects/{project}/anchors/{anchor}/features",tags=["Project"])
-def new_project_anchor_feature(project: str, anchor: str, definition: dict) -> UUID:
+def new_project_anchor_feature(project: str, anchor: str, definition: Dict) -> UUID:
     project_id = registry.get_entity_id(project)
     anchor_id = registry.get_entity_id(anchor)
     id = registry.create_project_anchor_feature(project_id, anchor_id, AnchorFeatureDef(**to_snake(definition)))
     return {"guid": str(id)}
 
 
 @router.post("/projects/{project}/derivedfeatures",tags=["Project"])
-def new_project_derived_feature(project: str, definition: dict) -> UUID:
+def new_project_derived_feature(project: str, definition: Dict) -> UUID:
     project_id = registry.get_entity_id(project)
     id = registry.create_project_derived_feature(project_id, DerivedFeatureDef(**to_snake(definition)))
     return {"guid": str(id)}

diff --git a/registry/purview-registry/registry/interface.py b/registry/purview-registry/registry/interface.py
@@ -1,19 +1,19 @@
 from abc import ABC, abstractclassmethod, abstractmethod
-from typing import Union
+from typing import Union, List, Dict
 from uuid import UUID
 from registry.models import *
 
 
 class Registry(ABC):
     @abstractmethod
-    def get_projects(self) -> list[str]:
+    def get_projects(self) -> List[str]:
         """
         Returns the names of all projects
         """
         pass
 
     @abstractmethod
-    def get_projects_ids(self) -> dict:
+    def get_projects_ids(self) -> Dict:
         """
         Returns the ids to names mapping of all projects
         """
@@ -27,7 +27,7 @@ def get_entity(self, id_or_name: Union[str, UUID],recursive = False) -> Entity:
         pass
 
     @abstractmethod
-    def get_entities(self, ids: list[UUID]) -> list[Entity]:
+    def get_entities(self, ids: List[UUID]) -> List[Entity]:
         """
         Get list of entities by their ids
         """
@@ -41,7 +41,7 @@ def get_entity_id(self, id_or_name: Union[str, UUID]) -> UUID:
         pass
 
     @abstractmethod
-    def get_neighbors(self, id_or_name: Union[str, UUID], relationship: RelationshipType) -> list[Edge]:
+    def get_neighbors(self, id_or_name: Union[str, UUID], relationship: RelationshipType) -> List[Edge]:
         """
         Get list of edges with specified type that connect to this entity.
         The edge contains fromId and toId so we can follow to the entity it connects to
@@ -66,8 +66,8 @@ def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations:
     @abstractmethod
     def search_entity(self,
                       keyword: str,
-                      type: list[EntityType],
-                      project: Optional[Union[str, UUID]] = None) -> list[EntityRef]:
+                      type: List[EntityType],
+                      project: Optional[Union[str, UUID]] = None) -> List[EntityRef]:
         """
         Search entities with specified type that also match the keyword in a project
         """
@@ -94,7 +94,7 @@ def create_project_derived_feature(self, project_id: UUID, definition: DerivedFe
         pass
 
     @abstractmethod
-    def get_dependent_entities(self, entity_id: Union[str, UUID]) -> list[Entity]:
+    def get_dependent_entities(self, entity_id: Union[str, UUID]) -> List[Entity]:
         """
         Given entity id, returns list of all entities that are downstream/dependent on given entity
         """