diff --git a/registry/.dockerignore b/registry/.dockerignore new file mode 100644 index 000000000..bc0ed1f7a --- /dev/null +++ b/registry/.dockerignore @@ -0,0 +1,3 @@ +__pycache__ +.env +.vscode diff --git a/registry/.env b/registry/.env new file mode 100644 index 000000000..08ff9370c --- /dev/null +++ b/registry/.env @@ -0,0 +1,5 @@ +AAD_INSTANCE=https://login.microsoftonline.com +API_CLIENT_ID=e1475341-ff84-4b6e-a187-349e35551cff +API_CLIENT_SECRET=cMZ8Q~d.gI6A-j-EMFbbmz.xy6jXOxUKH4dLYdhH +SWAGGER_UI_CLIENT_ID=e1335db3-a7eb-42b0-976c-82c9b5bdf0fb +AAD_TENANT_ID=72f988bf-86f1-41af-91ab-2d7cd011db47 \ No newline at end of file diff --git a/registry/.gitignore b/registry/.gitignore new file mode 100644 index 000000000..ed2a6faed --- /dev/null +++ b/registry/.gitignore @@ -0,0 +1,4 @@ +__pycache__ +.env +.vscode +.idea diff --git a/registry/Dockerfile b/registry/Dockerfile new file mode 100644 index 000000000..d2647021d --- /dev/null +++ b/registry/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.9 + +COPY ./ /usr/src + +WORKDIR /usr/src +RUN pip install -r requirements.txt + +# Start web server +CMD [ "uvicorn","main:app","--host", "0.0.0.0", "--port", "80" ] diff --git a/registry/__init__.py b/registry/__init__.py new file mode 100644 index 000000000..017991d93 --- /dev/null +++ b/registry/__init__.py @@ -0,0 +1,4 @@ +from common.database import DbConnection, connect +from common.registry_models import * +from sql_registry import registry +from access_control import * diff --git a/registry/access_control/README.md b/registry/access_control/README.md new file mode 100644 index 000000000..f89532233 --- /dev/null +++ b/registry/access_control/README.md @@ -0,0 +1,25 @@ +# Access Control Notes (WIP) + +A Global Switch `RBAC_ENABLED` is required to set as `True` to turn on the access control protection feature. + +Please note that in this version, only access control management API and UI experience are included. Supported scenarios status are tracked below: + +- General Fundation: + - [x] Access Control Abstrct Class + - [x] API Spec Contents for Access Control Management APIs + - [ ] API Sepc Contents for Registry API Access Control +- SQL Implementaion: + - [x] `userroles` table CUD through FastAPI + - [x] `userroles` table schema & test data + - [x] Enable/Disable with `RBAC_ENABLE` configuration + - [ ] Initialize default Admin role for project creator: After `create_project` API is ready +- UI Experience + - [x] Hidden page `../management` for global admin to make CUD requests to `userroles` table + - [x] Use id token in Management API Request headers to identify requestor + - [ ] Protect SQL Registry API with Access Control: After `create_project` API is ready +- Future Enhancements: + - [ ] Functional in Feathr Client + - [ ] Support Security Group scenario in Access Control + - [ ] Support AAD Groups + - [ ] Support Other OAuth Providers + \ No newline at end of file diff --git a/registry/access_control/__init__.py b/registry/access_control/__init__.py new file mode 100644 index 000000000..c6e5cf2e5 --- /dev/null +++ b/registry/access_control/__init__.py @@ -0,0 +1,8 @@ +__all__ = ["auth", "models", "interface", "db_rbac"] + + +from access_control.auth import * +from access_control.interface import RBAC +from access_control.models import * +from access_control.db_rbac import DbRBAC +from common.database import DbConnection, connect diff --git a/registry/access_control/access-control-gateway-sepc.md b/registry/access_control/access-control-gateway-sepc.md new file mode 100644 index 000000000..9b3adb5a3 --- /dev/null +++ b/registry/access_control/access-control-gateway-sepc.md @@ -0,0 +1,31 @@ +# Feathr Registry Access Control Gateway Specifications + +## Registry API with Access Control Gateway +**Access Control Gateway** is an access control **Plugin** component of feature registry API. It can work with different type of backend registry. When user enables this component, registry requests will be validated in a gateway as below flow chart: + +```mermaid +flowchart TD + A[Get Registry API Request] --> B{Is Id Token Valid?}; + B -- No --> D[Return 401]; + B -- Yes --> C{Have Permission?}; + C -- No --> F[Return 403]; + C -- Yes --> E[Call Registry API*]; + E --> G{API Service Avaiable?} + G -- No --> I[Return 503] + G -- Yes --> H[Return API Results] +``` +If Access control component is NOT enabled, the flow will start from __Call Registry API*__ + +## Acess Control Rules +- For all **get** requests, check **read** permission for certain project. +- For all **post** request, check **write** permission for certain project. +- For all **access control management** request, check **manage** permission for certian project. +- In case of feature level query, will verify the parent project access of the feature. +- Registry API calls and returns will be transparently transfered. +- A header `x-access-control-enabled` will be added for API calls protected by access control gateway + +## Management Rules +### Initialize `userroles` table +In current version, user needs to mannually initialze `userroles` table admins in SQL table. +When `create_registry` and `create_project` API is enabled, default admin role will be assgined to the creator. +Admin roles can add or delete roles in management UI page or thorugh management API. \ No newline at end of file diff --git a/registry/access_control/access-control-gateway-spec.md b/registry/access_control/access-control-gateway-spec.md new file mode 100644 index 000000000..f5c3eab33 --- /dev/null +++ b/registry/access_control/access-control-gateway-spec.md @@ -0,0 +1,36 @@ +# Feathr Registry Access Control Gateway Specifications + +## Registry API with Access Control Gateway + +**Access Control Gateway** is an access control **Plugin** component of feature registry API. It can work with different type of backend registry. When user enables this component, registry requests will be validated in a gateway as below flow chart: + +```mermaid +flowchart TD + A[Get Registry API Request] --> B{Is Id Token Valid?}; + B -- No --> D[Return 401]; + B -- Yes --> C{Have Permission?}; + C -- No --> F[Return 403]; + C -- Yes --> E[Call Registry API*]; + E --> G{API Service Avaiable?} + G -- No --> I[Return 503] + G -- Yes --> H[Return API Results] +``` + +If Access control component is NOT enabled, the flow will start from **Call Registry API*** + +## Acess Control Rules + +- For all **get** requests, check **read** permission for certain project. +- For all **post** request, check **write** permission for certain project. +- For all **access control management** request, check **manage** permission for certian project. +- In case of feature level query, will verify the parent project access of the feature. +- Registry API calls and returns will be transparently transfered. +- A header `x-access-control-enabled` will be added for API calls protected by access control gateway + +## Management Rules + +### Initialize `userroles` table + +In current version, user needs to mannually initialze `userroles` table admins in SQL table. +When `create_registry` and `create_project` API is enabled, default admin role will be assgined to the creator. +Admin roles can add or delete roles in management UI page or thorugh management API. diff --git a/registry/access_control/access.py b/registry/access_control/access.py new file mode 100644 index 000000000..c8bc49ac8 --- /dev/null +++ b/registry/access_control/access.py @@ -0,0 +1,39 @@ +from typing import Any +from xmlrpc.client import Boolean + +from fastapi import Depends, HTTPException, status +from access_control.db_rbac import DbRBAC + +from access_control.models import SUPER_ADMIN_SCOPE, AccessType, RoleType, User +from access_control.authorize import authorize + +rbac = DbRBAC() + +class ForbiddenAccess(HTTPException): + def __init__(self, detail: Any = None) -> None: + super().__init__(status_code=status.HTTP_403_FORBIDDEN, detail=detail, headers={"WWW-Authenticate": "Bearer"}) + + +def get_user(user: User = Depends(authorize)) -> User: + return user + +def project_read_access(project: str, user: User = Depends(authorize)) -> User: + return _project_access(project, user, AccessType.READ) + +def project_write_access(project: str, user: User = Depends(authorize)) -> User: + return _project_access(project, user, AccessType.WRITE) + +def project_manage_access(project: str, user: User = Depends(authorize)) -> User: + return _project_access(project, user, AccessType.MANAGE) + +def _project_access(project:str, user: User, access: str): + if rbac.validate_project_access_users(project, user.preferred_username, access): + return user + else: + raise ForbiddenAccess(f"{access} privileges for project {project} required for user {user.preferred_username}") + +def global_admin_access(user: User = Depends(authorize)): + if user.preferred_username in rbac.global_admin: + return user + else: + raise ForbiddenAccess('Admin privileges required') \ No newline at end of file diff --git a/registry/access_control/auth.py b/registry/access_control/auth.py new file mode 100644 index 000000000..630032728 --- /dev/null +++ b/registry/access_control/auth.py @@ -0,0 +1,49 @@ +import json +import requests +import jwt +from jwt.algorithms import RSAAlgorithm + +BEARER_TOKEN = "BEARER " + + +class AuthProvider(): + """This is the abstract class to decode JWT ID token. + Sample Usage with Azure ID token: + jwks_uri = "https://login.microsoftonline.com/common/discovery/v2.0/keys" + client_id = {Your Client Id} + auth = rbac.AuthProvider(jwks_uri, client_id) + """ + + def __init__(self, jwks_uri, client_id): + """Args: + - client Id: used as audience ("aud") + - jwks_uri: used to get public key pool + """ + self.client_id = client_id + self.cert_set = self.get_certs(jwks_uri) + + def get_certs(self, jwks_uri: str): + """Get certs from jwks uri""" + certs = requests.get(jwks_uri).json() + return {cert['kid']: cert for cert in certs['keys']} + + def get_public_key(self, token: str): + """ Get public key based on token kid""" + header_data = jwt.get_unverified_header(token) + kid = header_data['kid'] + return RSAAlgorithm.from_jwk(json.dumps(self.cert_set[kid])) + + def decode_token(self, bearer_token: str): + """ Decode ID token with RA256 Algorithm + Sample Usage with Azure ID token: + decoded = auth.decode_token(token) + username = decoded.get('preferred_username').lower() + """ + # TODO: Process Bearer Token more elegantly + token = bearer_token[len(BEARER_TOKEN):] + return jwt.decode(token, self.get_public_key(token), algorithms=[ + "RS256"], audience=self.client_id) + +def AzureADAuth(client_id: str): + jwks_uri = "https://login.microsoftonline.com/common/discovery/v2.0/keys" + auth = AuthProvider(jwks_uri, client_id) diff --git a/registry/access_control/authorize.py b/registry/access_control/authorize.py new file mode 100644 index 000000000..3a59e2f32 --- /dev/null +++ b/registry/access_control/authorize.py @@ -0,0 +1,114 @@ +import base64 +import logging +import requests +import rsa +from typing import Any, Dict, Mapping, Optional, Union + +from fastapi import HTTPException, Request, status +from fastapi.security import OAuth2AuthorizationCodeBearer +import jwt +from jwt.exceptions import ExpiredSignatureError, PyJWKError + +import access_control.config as config +from access_control.models import User + + +log = logging.getLogger() + +log.info(config.AAD_INSTANCE) + +BEARER_TOKEN = "BEARER " + +class InvalidAuthorization(HTTPException): + def __init__(self, detail: Any = None) -> None: + super().__init__(status_code=status.HTTP_401_UNAUTHORIZED, detail=detail, headers={"WWW-Authenticate": "Bearer"}) + + +class AzureADAuth(OAuth2AuthorizationCodeBearer): + # cached AAD jwt keys + aad_jwt_keys_cache: dict = {} + + def __init__(self, aad_instance: str = config.AAD_INSTANCE, aad_tenant: str = config.AAD_TENANT_ID): + self.base_auth_url: str = f"{aad_instance}/{aad_tenant}" + + async def __call__(self, request: Request) -> User: + # token: str = await super(AzureADAuthorization, self).__call__(request) or '' + bearer_token: str = request.headers.get("authorization") + token = bearer_token[len(BEARER_TOKEN):] + decoded_token = self._decode_token(token) + return self._get_user_from_token(decoded_token) + + @staticmethod + def _get_user_from_token(decoded_token: Mapping) -> User: + try: + user_id = decoded_token['oid'] + except Exception as e: + logging.debug(e) + raise InvalidAuthorization(detail='Unable to extract user details from token') + + return User( + id=user_id, + name=decoded_token.get('name', ''), + preferred_username=decoded_token.get('preferred_username', ''), + roles=decoded_token.get('roles', []) + ) + + @staticmethod + def _get_key_id(token: str) -> Optional[str]: + headers = jwt.get_unverified_header(token) + return headers['kid'] if headers and 'kid' in headers else None + + @staticmethod + def _ensure_b64padding(key: str) -> str: + """ + The base64 encoded keys are not always correctly padded, so pad with the right number of = + """ + key = key.encode('utf-8') + missing_padding = len(key) % 4 + for _ in range(missing_padding): + key = key + b'=' + return key + + def _cache_aad_keys(self) -> None: + """ + Cache all AAD JWT keys - so we don't have to make a web call each auth request + """ + response = requests.get(f"{self.base_auth_url}/v2.0/.well-known/openid-configuration") + aad_metadata = response.json() if response.ok else None + jwks_uri = aad_metadata['jwks_uri'] if aad_metadata and 'jwks_uri' in aad_metadata else None + if jwks_uri: + response = requests.get(jwks_uri) + keys = response.json() if response.ok else None + if keys and 'keys' in keys: + for key in keys['keys']: + n = int.from_bytes(base64.urlsafe_b64decode(self._ensure_b64padding(key['n'])), "big") + e = int.from_bytes(base64.urlsafe_b64decode(self._ensure_b64padding(key['e'])), "big") + pub_key = rsa.PublicKey(n, e) + # Cache the PEM formatted public key. + AzureADAuth.aad_jwt_keys_cache[key['kid']] = pub_key.save_pkcs1() + + def _get_token_key(self, key_id: str) -> str: + if key_id not in AzureADAuth.aad_jwt_keys_cache: + self._cache_aad_keys() + return AzureADAuth.aad_jwt_keys_cache[key_id] + + def _decode_token(self, token: str) -> Mapping: + key_id = self._get_key_id(token) + if not key_id: + raise InvalidAuthorization('The token does not contain kid') + key = self._get_token_key(key_id) + try: + decode = jwt.decode(token, key=key, algorithms=['RS256'], audience=config.API_AUDIENCE) + return decode + except ExpiredSignatureError as e: + logging.debug(f'The token signature has expired: {e}') + raise InvalidAuthorization('The token signature has expired') + except PyJWKError as e: + logging.debug(f'Invalid token: {e}') + raise InvalidAuthorization('The token is invalid') + except Exception as e: + logging.debug(f'Unexpected error: {e}') + raise InvalidAuthorization('Unable to decode token') + + +authorize = AzureADAuth() \ No newline at end of file diff --git a/registry/access_control/config.py b/registry/access_control/config.py new file mode 100644 index 000000000..016b066dc --- /dev/null +++ b/registry/access_control/config.py @@ -0,0 +1,18 @@ +from starlette.config import Config + + +config = Config(".env") + +API_PREFIX: str = "/api" +VERSION: str = "0.1.0" +PROJECT_NAME: str = "FastAPI with AAD Authentication" +DEBUG: bool = config("DEBUG", cast=bool, default=False) + +# Authentication +API_CLIENT_ID: str = config("API_CLIENT_ID", default="e1475341-ff84-4b6e-a187-349e35551cff") +API_CLIENT_SECRET: str = config("API_CLIENT_SECRET", default="cMZ8Q~d.gI6A-j-EMFbbmz.xy6jXOxUKH4dLYdhH") +SWAGGER_UI_CLIENT_ID: str = config("SWAGGER_UI_CLIENT_ID", default=" e1335db3-a7eb-42b0-976c-82c9b5bdf0fb") +AAD_TENANT_ID: str = config("AAD_TENANT_ID", default="72f988bf-86f1-41af-91ab-2d7cd011db47") + +AAD_INSTANCE: str = config("AAD_INSTANCE", default="https://login.microsoftonline.com") +API_AUDIENCE: str = config("API_AUDIENCE", default="db8dc4b0-202e-450c-b38d-7396ad9631a5") \ No newline at end of file diff --git a/registry/access_control/db_rbac.py b/registry/access_control/db_rbac.py new file mode 100644 index 000000000..30df73af7 --- /dev/null +++ b/registry/access_control/db_rbac.py @@ -0,0 +1,114 @@ +from distutils.log import warn +import logging +import os +from xmlrpc.client import Boolean +from access_control.interface import RBAC +from access_control.models import Access, AccessType, RoleAccessMapping, User, UserRole, RoleType, SUPER_ADMIN_SCOPE +from common.database import connect + + +class DbRBAC(RBAC): + def __init__(self): + self.conn = connect() + # cached user role records + self.userroles = self._get_userroles() + self.global_admin = self._get_global_admin_users() + + def refresh_cache(self): + self.userroles = self._get_userroles() + self.global_admin = self._get_global_admin_users() + + def _get_userroles(self) -> list[UserRole]: + """query all the active user role records in SQL table + """ + rows = self.conn.query( + fr"""select record_id, project_name, user_name, role_name, create_by, create_reason, create_time, delete_by, delete_reason, delete_time + from userroles + where delete_reason is null""") + ret = [] + for row in rows: + r = UserRole(**row) + ret.append(UserRole(**row)) + logging.info(f"{ret.__len__} user roles are get.") + return ret + + def _get_global_admin_users(self) -> list[str]: + return [u.user_name for u in self.userroles if (u.project_name == SUPER_ADMIN_SCOPE and u.role_name == RoleType.ADMIN)] + + def validate_project_access_users(self, project:str, user:str, access:str = AccessType.READ) -> Boolean: + for u in self.userroles: + if (u.user_name == user and u.project_name in [project, SUPER_ADMIN_SCOPE] and (access in u.access)): + return True + return False + + + + def get_userroles_by_user(self, user_name: str, role_name: str = None) -> list[UserRole]: + """query the active user role of certain user + """ + query = fr"""select record_id, project_name, user_name, role_name, create_by, create_reason, create_time, delete_by, delete_reason, delete_time + from userroles + where delete_reason is null and user_name ='{user_name}'""" + if role_name: + query += fr"and role_name = '{role_name}'" + rows = self.conn.query(query) + ret = [] + for row in rows: + ret.append(UserRole(**row)) + return ret + + def get_userroles_by_project(self, project_name: str, role_name: str = None) -> list[UserRole]: + """query the active user role of certain project. + """ + query = fr"""select record_id, project_name, user_name, role_name, create_by, create_reason, create_time, delete_reason, delete_time + from userroles + where delete_reason is null and project_name ='{project_name}'""" + if role_name: + query += fr"and role_name = '{role_name}'" + rows = self.conn.query(query) + ret = [] + for row in rows: + ret.append(UserRole(**row)) + return ret + + def add_userrole(self, project_name: str, user_name: str, role_name: str, create_reason: str, by:str): + """insert new user role relationship into sql table + """ + # check if record already exist + for u in self.userroles: + if u.project_name == project_name and u.user_name == user_name and u.role_name == role_name: + warn(f"User {user_name} already have {role_name} role of {project_name}.") + return True + + # insert new record + query = f"""insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) + values ('{project_name}','{user_name}','{role_name}','{by}' ,'{create_reason}', getutcdate())""" + self.conn.update(query) + self.refresh_cache() + return + + def delete_userrole(self, project_name: str, user_name: str, role_name: str, delete_reason: str, by:str): + """mark existing user role relationship as deleted with reason + """ + query = fr"""UPDATE userroles SET + [delete_by] = '{by}', + [delete_reason] = '{delete_reason}', + [delete_time] = getutcdate() + WHERE [user_name] = '{user_name}' and [project_name] = '{project_name}' and [role_name] = '{role_name}' + and [delete_time] is null""" + self.conn.update(query) + self.refresh_cache() + return + + def init_userrole(self, creator_name: str, project_name: str): + """initialize user role relationship when a new project is created + TODO: Add init user role to every new project call + """ + create_by = "system" + create_reason = "creator of project, get admin by default." + query = fr"""insert into userroles (project_name, user_name, role_name, create_reason, create_time) + values ('{project_name}','{creator_name}','{RoleType.ADMIN}','{create_by}','{create_reason}', getutcdate())""" + return self.conn.update(query) + + def validate_access(self, user: User, project: str ,access: Access): + return \ No newline at end of file diff --git a/registry/access_control/interface.py b/registry/access_control/interface.py new file mode 100644 index 000000000..ede9d7306 --- /dev/null +++ b/registry/access_control/interface.py @@ -0,0 +1,46 @@ +from abc import ABC, abstractmethod +from access_control.models import Access, User, UserRole + + +class RBAC(ABC): + @abstractmethod + def _get_userroles(self) -> list[UserRole]: + """Get List of All User Role Records + """ + pass + + @abstractmethod + def add_userrole(self, userrole: UserRole): + """Add a Role to a User + """ + pass + + @abstractmethod + def delete_userrole(self, userrole: UserRole): + """Delete a Role of a User + """ + pass + + @abstractmethod + def init_userrole(self, creator_name: str, project_name: str): + """Default User Role Relationship when a new project is created + """ + pass + + @abstractmethod + def get_userroles_by_user(self, user_name: str) -> list[UserRole]: + """Get List of All User Role Records for a User + """ + pass + + @abstractmethod + def get_userroles_by_project(self, project_name: str) -> list[UserRole]: + """Get List of All User Role Records for a Project + """ + pass + + @abstractmethod + def validate_access(self, user: User, project: str ,access: Access): + """Validate if a Role has certian access + """ + pass \ No newline at end of file diff --git a/registry/access_control/main.py b/registry/access_control/main.py new file mode 100644 index 000000000..3770d8cc1 --- /dev/null +++ b/registry/access_control/main.py @@ -0,0 +1,133 @@ +import os +from typing import Optional +from fastapi import APIRouter, FastAPI, HTTPException, Request +from fastapi.security import OAuth2PasswordBearer +from starlette.middleware.cors import CORSMiddleware +from registry import * +from registry.db_registry import DbRegistry +from registry.models import EntityType +from rbac import * +from rbac.db_rbac import DbRBAC +from pydantic import BaseSettings + +rp = "/" +try: + rp = os.environ["API_BASE"] + if rp[0] != '/': + rp = '/' + rp +except: + pass +print("Using API BASE: ", rp) + +registry = DbRegistry() +app = FastAPI() +router = APIRouter() +rbac_enabled = str(os.environ.get("RBAC_ENABLED")).lower() == 'true' +rbac = DbRBAC() + +# Enables CORS +app.add_middleware(CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + +# Enable Auth +app.add_middleware(AuthMiddleware) + + +@router.get("/projects") +def get_projects() -> list[str]: + return registry.get_projects() + + +@router.get("/projects/{project}") +def get_projects(project: str) -> dict: + return registry.get_project(project).to_dict() + + +@router.get("/projects/{project}/datasources") +def get_project_datasources(project: str) -> list: + p = registry.get_entity(project) + source_ids = [s.id for s in p.attributes.sources] + sources = registry.get_entities(source_ids) + return list([e.to_dict() for e in sources]) + + +@router.get("/projects/{project}/features") +def get_project_features(project: str, keyword: Optional[str] = None) -> list: + if keyword is None: + p = registry.get_entity(project) + feature_ids = [s.id for s in p.attributes.anchor_features] + \ + [s.id for s in p.attributes.derived_features] + features = registry.get_entities(feature_ids) + return list([e.to_dict() for e in features]) + else: + efs = registry.search_entity( + keyword, [EntityType.AnchorFeature, EntityType.DerivedFeature]) + feature_ids = [ef.id for ef in efs] + features = registry.get_entities(feature_ids) + return list([e.to_dict() for e in features]) + + +@router.get("/features/{feature}") +def get_feature(feature: str) -> dict: + e = registry.get_entity(feature) + if e.entity_type not in [EntityType.DerivedFeature, EntityType.AnchorFeature]: + raise HTTPException( + status_code=404, detail=f"Feature {feature} not found") + return e + + +@router.get("/features/{feature}/lineage") +def get_feature_lineage(feature: str) -> dict: + lineage = registry.get_lineage(feature) + return lineage.to_dict() + + +@router.get("/userroles") +def get_userroles(req: Request) -> list: + if rbac_enabled: + if rbac.is_global_admin(req.headers.get("authorization")): + userroles = rbac.get_userroles() + return list([r.to_dict() for r in userroles]) + else: + raise HTTPException( + status_code=403, detail=f"Only `Global Admin` have access to this content.") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +@router.post("/users/{user}/userroles/add") +def add_userrole(project: str, user: str, role: str, reason: str, req: Request): + if rbac_enabled: + if rbac.is_project_admin(req.headers.get("authorization"), project): + return rbac.add_userrole(project, user, role, reason, rbac.requestor) + else: + raise HTTPException( + status_code=403, detail=f"Only `Project Admin` can add userroles") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +@router.delete("/users/{user}/userroles/delete") +def delete_userrole(project: str, user: str, role: str, reason: str, req: Request): + if rbac_enabled: + if rbac.is_project_admin(req.headers.get("authorization"), project): + rbac.delete_userrole(project, user, role, reason, rbac.requestor) + else: + raise HTTPException( + status_code=403, detail=f"Only `Project Admin` can delete userroles") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +app.include_router(prefix=rp, router=router) diff --git a/registry/access_control/models.py b/registry/access_control/models.py new file mode 100644 index 000000000..cc4b3150a --- /dev/null +++ b/registry/access_control/models.py @@ -0,0 +1,90 @@ +from typing import List, Optional +from pydantic import BaseModel +from datetime import datetime +from enum import Enum +from numpy import number + +class User(BaseModel): + id: str + name: str + preferred_username: str + roles: List[str] + +SUPER_ADMIN_SCOPE = "global" + +class AccessType(str, Enum): + READ = "read", + WRITE = "write", + MANAGE = "manage", + +class RoleType(str, Enum): + ADMIN = "admin", + CONSUMER = "consumer", + PRODUCER = "producer", + DEFAULT = "default", + + +RoleAccessMapping = { + RoleType.ADMIN: ["read", "write", "manage"], + RoleType.CONSUMER: ["read"], + RoleType.PRODUCER: ["read", "write"], + RoleType.DEFAULT: [] +} + + +class UserRole(): + def __init__(self, + record_id: number, + project_name: str, + user_name: str, + role_name: str, + create_by: str, + create_reason: str, + create_time: datetime, + delete_by: Optional[str] = None, + delete_reason: Optional[str] = None, + delete_time: Optional[datetime] = None, + **kwargs): + self.record_id = record_id + self.project_name = project_name.lower() + self.user_name = user_name.lower() + self.role_name = role_name.lower() + self.create_by = create_by.lower() + self.create_reason = create_reason + self.create_time = create_time + self.delete_by = delete_by + self.delete_reason = delete_reason + self.delete_time = delete_time + self.access = RoleAccessMapping[RoleType(self.role_name)] + + def to_dict(self) -> dict: + return { + "id": str(self.record_id), + "scope": self.project_name, + "userName": self.user_name, + "roleName": str(self.role_name), + "createBy": self.create_by, + "createReason": self.create_reason, + "createTime": str(self.create_time), + "deleteBy": str(self.delete_by), + "deleteReason": self.delete_reason, + "deleteTime": str(self.delete_time), + "access": self.access + } + + +class Access(): + def __init__(self, + record_id: number, + project_name: str, + access_name: str) -> None: + self.record_id = record_id + self.project_name = project_name + self.access_name = access_name + + def to_dict(self) -> dict: + return { + "record_id": str(self.record_id), + "project_name": self.project_name, + "access": self.access_name, + } \ No newline at end of file diff --git a/registry/access_control/requirements.txt b/registry/access_control/requirements.txt new file mode 100644 index 000000000..77c6d123b --- /dev/null +++ b/registry/access_control/requirements.txt @@ -0,0 +1,4 @@ +pymssql +fastapi +uvicorn +pyjwt \ No newline at end of file diff --git a/registry/api.py b/registry/api.py new file mode 100644 index 000000000..f61019d10 --- /dev/null +++ b/registry/api.py @@ -0,0 +1,42 @@ +from http.client import OK +from typing import List, Optional + +from fastapi import APIRouter, Body, Depends, status + +from access_control.access import global_admin_access, get_user, project_read_access +from access_control.models import User +from access_control.db_rbac import DbRBAC + +router = APIRouter() +rbac = DbRBAC() + +# no authentication needed +# even anonymous users can check project names +@router.get('/projects', status_code=status.HTTP_200_OK, name="Get a list of Project Names [NO AUTH REQUIRED]") +async def get_projects(requestor: User = Depends(get_user)): + return ['project 1','Project 2'] + +# requires user to be authenticated, only returns items for users with project read access +# that the user is authenticated is verified by Depends(get_user) +@router.get('/projects/{project}', status_code=status.HTTP_200_OK, name="Get My Project [Read Access Required]") +async def get_project(project: str, requestor: User = Depends(project_read_access)): + return OK + +@router.get("/projects/{project}/features") +def get_project_features(project: str, keyword: Optional[str] = None, requestor: User = Depends(project_read_access)) -> list: + return OK + +@router.get("/userroles") +def get_userroles(requestor: User = Depends(global_admin_access)) -> list: + return list([r.to_dict() for r in rbac.userroles]) + + +@router.post("/users/{user}/userroles/add") +def add_userrole(project: str, user: str, role: str, reason: str, requestor: User = Depends(global_admin_access)): + return rbac.add_userrole(project, user, role, reason, requestor.preferred_username) + + +@router.delete("/users/{user}/userroles/delete") +def delete_userrole(project: str, user: str, role: str, reason: str, requestor: User = Depends(global_admin_access)): + rbac.delete_userrole(project, user, role, reason, requestor.preferred_username) + return OK \ No newline at end of file diff --git a/registry/common/database.py b/registry/common/database.py new file mode 100644 index 000000000..4e0453778 --- /dev/null +++ b/registry/common/database.py @@ -0,0 +1,151 @@ +from abc import ABC, abstractmethod +from contextlib import contextmanager +import logging +import threading +import os +import pymssql + + +providers = [] + +class DbConnection(ABC): + @abstractmethod + def query(self, sql: str, *args, **kwargs) -> list[dict]: + pass + + @abstractmethod + def update(self, sql: str, *args, **kwargs): + pass + +def quote(id): + if isinstance(id, str): + return f"'{id}'" + else: + return ",".join([f"'{i}'" for i in id]) + + +def parse_conn_str(s: str) -> dict: + """ + TODO: Not a sound and safe implementation, but useful enough in this case + as the connection string is provided by users themselves. + """ + parts = dict([p.strip().split("=", 1) + for p in s.split(";") if len(p.strip()) > 0]) + server = parts["Server"].split(":")[1].split(",")[0] + return { + "host": server, + "database": parts["Initial Catalog"], + "user": parts["User ID"], + "password": parts["Password"], + # "charset": "utf-8", ## For unknown reason this causes connection failure + } + + +class MssqlConnection(DbConnection): + @staticmethod + def connect(autocommit = True): + conn_str = os.environ["CONNECTION_STR"] + if "Server=" not in conn_str: + logging.debug("`CONNECTION_STR` is not in ADO connection string format") + return None + params = parse_conn_str(conn_str) + if not autocommit: + params["autocommit"] = False + return MssqlConnection(params) + + def __init__(self, params): + self.params = params + self.make_connection() + self.mutex = threading.Lock() + + def make_connection(self): + self.conn = pymssql.connect(**self.params) + + def query(self, sql: str, *args, **kwargs) -> list[dict]: + """ + Make SQL query and return result + """ + logging.debug(f"SQL: `{sql}`") + # NOTE: Only one cursor is allowed at the same time + retry = 0 + while True: + try: + with self.mutex: + c = self.conn.cursor(as_dict=True) + c.execute(sql, *args, **kwargs) + return c.fetchall() + except pymssql.OperationalError: + logging.warning("Database error, retrying...") + # Reconnect + self.make_connection() + retry += 1 + if retry >= 3: + # Stop retrying + raise + pass + + def update(self, sql: str, *args, **kwargs): + retry = 0 + while True: + try: + with self.mutex: + c = self.conn.cursor(as_dict=True) + c.execute(sql, *args, **kwargs) + self.conn.commit() + return True + except pymssql.OperationalError: + logging.warning("Database error, retrying...") + # Reconnect + self.make_connection() + retry += 1 + if retry >= 3: + # Stop retrying + raise + pass + + @contextmanager + def transaction(self): + """ + Start a transaction so we can run multiple SQL in one batch. + User should use `with` with the returned value, look into db_registry.py for more real usage. + + NOTE: `self.query` and `self.execute` will use a different MSSQL connection so any change made + in this transaction will *not* be visible in these calls. + + The minimal implementation could look like this if the underlying engine doesn't support transaction. + ``` + @contextmanager + def transaction(self): + try: + c = self.create_or_get_connection(...) + yield c + finally: + c.close(...) + ``` + """ + conn = None + cursor = None + try: + # As one MssqlConnection has only one connection, we need to create a new one to disable `autocommit` + conn = MssqlConnection.connect(autocommit=False).conn + cursor = conn.cursor(as_dict=True) + yield cursor + except Exception as e: + logging.warning(f"Exception: {e}") + if conn: + conn.rollback() + raise e + finally: + if conn: + conn.commit() + + +providers.append(MssqlConnection) + + +def connect(*args, **kargs): + for p in providers: + ret = p.connect(*args, **kargs) + if ret is not None: + return ret + raise RuntimeError("Cannot connect to database") \ No newline at end of file diff --git a/registry/common/registry_models.py b/registry/common/registry_models.py new file mode 100644 index 000000000..3c08d2692 --- /dev/null +++ b/registry/common/registry_models.py @@ -0,0 +1,728 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional, Union +from uuid import UUID +import json +import re + + +def _to_snake(d, level: int = 0): + """ + Convert `string`, `list[string]`, or all keys in a `dict` into snake case + The maximum length of input string or list is 100, or it will be truncated before being processed, for dict, the exception will be thrown if it has more than 100 keys. + the maximum nested level is 10, otherwise the exception will be thrown + """ + if level >= 10: + raise ValueError("Too many nested levels") + if isinstance(d, str): + d = d[:100] + return re.sub(r'([A-Z]\w+$)', r'_\1', d).lower() + if isinstance(d, list): + d = d[:100] + return [_to_snake(i, level + 1) if isinstance(i, (dict, list)) else i for i in d] + if len(d) > 100: + raise ValueError("Dict has too many keys") + return {_to_snake(a, level + 1): _to_snake(b, level + 1) if isinstance(b, (dict, list)) else b for a, b in d.items()} + + +def _to_type(value, type): + """ + Convert `value` into `type`, + or `list[type]` if `value` is a list + NOTE: This is **not** a generic implementation, only for objects in this module + """ + if isinstance(value, type): + return value + if isinstance(value, list): + return list([_to_type(v, type) for v in value]) + if isinstance(value, dict): + if hasattr(type, "new"): + try: + # The convention is to use `new` method to create the object from a dict + return type.new(**_to_snake(value)) + except TypeError: + pass + return type(**_to_snake(value)) + if issubclass(type, Enum): + try: + n = int(value) + return type(n) + except ValueError: + pass + if hasattr(type, "new"): + try: + # As well as Enum types, some of them have alias that cannot be handled by default Enum constructor + return type.new(value) + except KeyError: + pass + return type[value] + return type(value) + + +def _to_uuid(value): + return _to_type(value, UUID) + + +class ValueType(Enum): + UNSPECIFIED = 0 + BOOLEAN = 1 + INT = 2 + LONG = 3 + FLOAT = 4 + DOUBLE = 5 + STRING = 6 + BYTES = 7 + + +class VectorType(Enum): + TENSOR = 0 + + +class TensorCategory(Enum): + DENSE = 0 + SPARSE = 1 + + +class EntityType(Enum): + Project = 1 + Source = 2 + Anchor = 3 + AnchorFeature = 4 + DerivedFeature = 5 + + @staticmethod + def new(v): + return { + "feathr_workspace_v1": EntityType.Project, + "feathr_source_v1": EntityType.Source, + "feathr_anchor_v1": EntityType.Anchor, + "feathr_anchor_feature_v1": EntityType.AnchorFeature, + "feathr_derived_feature_v1": EntityType.DerivedFeature, + }[v] + + def __str__(self): + return { + EntityType.Project: "feathr_workspace_v1", + EntityType.Source: "feathr_source_v1", + EntityType.Anchor: "feathr_anchor_v1", + EntityType.AnchorFeature: "feathr_anchor_feature_v1", + EntityType.DerivedFeature: "feathr_derived_feature_v1", + }[self] + + +class RelationshipType(Enum): + Contains = 1 + BelongsTo = 2 + Consumes = 3 + Produces = 4 + + +class ToDict(ABC): + """ + This ABC is used to convert object to dict, then JSON. + """ + @abstractmethod + def to_dict(self) -> dict: + pass + + def to_json(self, indent=None) -> str: + return json.dumps(self.to_dict(), indent=indent) + + +class FeatureType(ToDict): + def __init__(self, + type: Union[str, VectorType], + tensor_category: Union[str, TensorCategory], + dimension_type: list[Union[str, ValueType]], + val_type: Union[str, ValueType]): + self.type = _to_type(type, VectorType) + self.tensor_category = _to_type(tensor_category, TensorCategory) + self.dimension_type = _to_type(dimension_type, ValueType) + self.val_type = _to_type(val_type, ValueType) + + def to_dict(self) -> dict: + return { + "type": self.type.name, + "tensorCategory": self.tensor_category.name, + "dimensionType": [t.name for t in self.dimension_type], + "valType": self.val_type.name, + } + + +class TypedKey(ToDict): + def __init__(self, + key_column: str, + key_column_type: ValueType, + full_name: Optional[str] = None, + description: Optional[str] = None, + key_column_alias: Optional[str] = None): + self.key_column = key_column + self.key_column_type = _to_type(key_column_type, ValueType) + self.full_name = full_name + self.description = description + self.key_column_alias = key_column_alias + + def to_dict(self) -> dict: + ret = { + "key_column": self.key_column, + "key_column_type": self.key_column_type.name, + } + if self.full_name is not None: + ret["full_name"] = self.full_name + if self.description is not None: + ret["description"] = self.full_name + if self.key_column_alias is not None: + ret["key_column_alias"] = self.key_column_alias + return ret + + +class Transformation(ToDict): + @staticmethod + def new(**kwargs): + if "transform_expr" in kwargs: + return ExpressionTransformation(**kwargs) + elif "def_expr" in kwargs: + return WindowAggregationTransformation(**kwargs) + elif "name" in kwargs: + return UdfTransformation(**kwargs) + else: + raise ValueError(kwargs) + + +class ExpressionTransformation(Transformation): + def __init__(self, transform_expr: str): + self.transform_expr = transform_expr + + def to_dict(self) -> dict: + return { + "transform_expr": self.transform_expr + } + + +class WindowAggregationTransformation(Transformation): + def __init__(self, + def_expr: str, + agg_func: Optional[str] = None, + window: Optional[str] = None, + group_by: Optional[str] = None, + filter: Optional[str] = None, + limit: Optional[int] = None): + self.def_expr = def_expr + self.agg_func = agg_func + self.window = window + self.group_by = group_by + self.filter = filter + self.limit = limit + + def to_dict(self) -> dict: + ret = { + "def_expr": self.def_expr, + } + if self.agg_func is not None: + ret["agg_func"] = self.agg_func + if self.window is not None: + ret["window"] = self.window + if self.group_by is not None: + ret["group_by"] = self.group_by + if self.filter is not None: + ret["filter"] = self.filter + if self.limit is not None: + ret["limit"] = self.limit + return ret + + +class UdfTransformation(Transformation): + def __init__(self, name: str): + self.name = name + + def to_dict(self) -> dict: + return { + "name": self.name + } + + +class EntityRef(ToDict): + def __init__(self, + id: UUID, + type: Union[str, EntityType], + qualified_name: Optional[str] = None, + uniq_attr: dict = {}): + self.id = id + self.type = _to_type(type, EntityType) + if qualified_name is not None: + self.uniq_attr = {"qualifiedName": qualified_name} + else: + self.uniq_attr = uniq_attr + + @property + def entity_type(self) -> EntityType: + return self.type + + @property + def qualified_name(self) -> EntityType: + return self.uniq_attr['qualifiedName'] + + def get_ref(self): + return self + + def to_dict(self) -> dict: + return { + "guid": str(self.id), + "typeName": str(self.type), + "uniqueAttributes": self.uniq_attr, + } + + +class Attributes(ToDict): + @staticmethod + def new(entity_type: Union[str, EntityType], **kwargs): + return { + EntityType.Project: ProjectAttributes, + EntityType.Source: SourceAttributes, + EntityType.Anchor: AnchorAttributes, + EntityType.AnchorFeature: AnchorFeatureAttributes, + EntityType.DerivedFeature: DerivedFeatureAttributes, + }[_to_type(entity_type, EntityType)](**kwargs) + + +class Entity(ToDict): + def __init__(self, + entity_id: Union[str, UUID], + qualified_name: str, + entity_type: Union[str, EntityType], + attributes: Union[dict, Attributes], + **kwargs): + self.id = _to_uuid(entity_id) + self.qualified_name = qualified_name + self.entity_type = _to_type(entity_type, EntityType) + if isinstance(attributes, Attributes): + self.attributes = attributes + else: + self.attributes = Attributes.new( + entity_type, **_to_snake(attributes)) + + def get_ref(self) -> EntityRef: + return EntityRef(self.id, + self.attributes.entity_type, + self.qualified_name) + + def to_dict(self) -> dict: + return { + "guid": str(self.id), + "lastModifiedTS": "1", + "status": "ACTIVE", + "displayText": self.attributes.name, + "typeName": str(self.attributes.entity_type), + "attributes": self.attributes.to_dict(), + } + + +class ProjectAttributes(Attributes): + def __init__(self, + name: str, + children: list[Union[dict, Entity]] = [], + tags: dict = {}, + **kwargs): + self.name = name + self.tags = tags + self._children = [] + if len(children) > 0: + self.children = children + + @property + def entity_type(self) -> EntityType: + return EntityType.Project + + @property + def children(self): + return self._children + + @children.setter + def children(self, v: list[Union[dict, Entity]]): + for f in v: + if isinstance(f, Entity): + self._children.append(f) + elif isinstance(f, dict): + self._children.append(_to_type(f, Entity)) + else: + raise TypeError(f) + + @property + def sources(self): + return [ + e for e in self.children if e.entity_type == EntityType.Source] + + @property + def anchors(self): + return [ + e for e in self.children if e.entity_type == EntityType.Anchor] + + @property + def anchor_features(self): + return [ + e for e in self.children if e.entity_type == EntityType.AnchorFeature] + + @property + def derived_features(self): + return [ + e for e in self.children if e.entity_type == EntityType.DerivedFeature] + + def to_dict(self) -> dict: + return { + "qualifiedName": self.name, + "name": self.name, + "sources": list([e.get_ref().to_dict() for e in self.sources]), + "anchors": list([e.get_ref().to_dict() for e in self.anchors]), + "anchor_features": list([e.get_ref().to_dict() for e in self.anchor_features]), + "derived_features": list([e.get_ref().to_dict() for e in self.derived_features]), + "tags": self.tags, + } + + +class SourceAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + type: str, + path: str, + preprocessing: Optional[str] = None, + event_timestamp_column: Optional[str] = None, + timestamp_format: Optional[str] = None, + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.type = type + self.path = path + self.preprocessing = preprocessing + self.event_timestamp_column = event_timestamp_column + self.timestamp_format = timestamp_format + self.tags = tags + + @property + def entity_type(self) -> EntityType: + return EntityType.Source + + def to_dict(self) -> dict: + ret = { + "qualifiedName": self.qualified_name, + "name": self.name, + "type": self.type, + "path": self.path, + "tags": self.tags, + } + if self.preprocessing is not None: + ret["preprocessing"] = self.preprocessing + if self.event_timestamp_column is not None: + ret["eventTimestampColumn"] = self.event_timestamp_column + if self.timestamp_format is not None: + ret["timestampFormat"] = self.timestamp_format + return ret + + +class AnchorAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + # source: Optional[Union[dict, EntityRef, Entity]] = None, + # features: list[Union[dict, EntityRef, Entity]] = [], + tags: dict = {}, + **kwargs): + self.qualified_name = qualified_name + self.name = name + self._source = None + self._features = [] + # if source is not None: + # self._source = source.get_ref() + # if len(features)>0: + # self._set_feature(features) + self.tags = tags + + @property + def entity_type(self) -> EntityType: + return EntityType.Anchor + + @property + def source(self) -> EntityRef: + return self._source + + @source.setter + def source(self, s): + if isinstance(s, Entity): + self._source = s.get_ref() + elif isinstance(s, EntityRef): + self._source = s + elif isinstance(s, dict): + self._source = _to_type(s, Entity).get_ref() + else: + raise TypeError(s) + + @property + def features(self): + return self._features + + @features.setter + def features(self, features): + self._features = [] + for f in features: + if isinstance(f, Entity): + self._features.append(f.get_ref()) + elif isinstance(f, EntityRef): + self._features.append(f) + elif isinstance(f, dict): + self._features.append(_to_type(f, Entity).get_ref()) + else: + raise TypeError(f) + + def to_dict(self) -> dict: + ret = { + "qualifiedName": self.qualified_name, + "name": self.name, + "features": list([e.get_ref().to_dict() for e in self.features]), + "tags": self.tags, + } + if self.source is not None: + ret["source"] = self.source.get_ref().to_dict() + return ret + + +class AnchorFeatureAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.type = _to_type(type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self.tags = tags + + @property + def entity_type(self) -> EntityType: + return EntityType.AnchorFeature + + def to_dict(self) -> dict: + return { + "qualifiedName": self.qualified_name, + "name": self.name, + "type": self.type.to_dict(), + "transformation": self.transformation.to_dict(), + "key": list([k.to_dict() for k in self.key]), + "tags": self.tags, + } + + +class DerivedFeatureAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + # input_anchor_features: list[Union[dict, EntityRef, Entity]] = [], + # input_derived_features: list[Union[dict, EntityRef, Entity]] = [], + tags: dict = {}, + **kwargs): + self.qualified_name = qualified_name + self.name = name + self.type = _to_type(type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self._input_anchor_features = [] + self._input_derived_features = [] + self.tags = tags + # self._set_input_anchor_features(input_anchor_features) + # self._set_input_derived_features(input_derived_features) + + @property + def entity_type(self) -> EntityType: + return EntityType.DerivedFeature + + @property + def input_features(self): + return self._input_anchor_features + self._input_derived_features + + @input_features.setter + def input_features(self, v: Union[dict, Entity]): + self._input_anchor_features = [] + self._input_derived_features = [] + for f in v: + e = None + if isinstance(f, Entity): + e = f + elif isinstance(f, dict): + e = _to_type(f, Entity) + else: + raise TypeError(f) + + if e.entity_type == EntityType.AnchorFeature: + self._input_anchor_features.append(e) + elif e.entity_type == EntityType.DerivedFeature: + self._input_derived_features.append(e) + else: + pass + + @property + def input_anchor_features(self): + return self._input_anchor_features + + # @input_anchor_features.setter + # def input_anchor_features(self, v): + # self._input_anchor_features = [] + # for f in v: + # if isinstance(f, Entity): + # self._input_anchor_features.append(f.get_ref()) + # elif isinstance(f, EntityRef): + # self._input_anchor_features.append(f) + # elif isinstance(f, dict): + # self._input_anchor_features.append( + # to_type(f, Entity).get_ref()) + # else: + # raise TypeError(f) + + @property + def input_derived_features(self): + return self._input_derived_features + + # @input_derived_features.setter + # def input_derived_features(self, v): + # self._input_derived_features = [] + # for f in v: + # if isinstance(f, Entity): + # self._input_derived_features.append(f.get_ref()) + # elif isinstance(f, EntityRef): + # self._input_derived_features.append(f) + # elif isinstance(f, dict): + # self._input_derived_features.append( + # to_type(f, Entity).get_ref()) + # else: + # raise TypeError(f) + + def to_dict(self) -> dict: + return { + "qualifiedName": self.qualified_name, + "name": self.name, + "type": self.type.to_dict(), + "transformation": self.transformation.to_dict(), + "key": list([k.to_dict() for k in self.key]), + "input_anchor_features": [e.get_ref().to_dict() for e in self.input_anchor_features], + "input_derived_features": [e.get_ref().to_dict() for e in self.input_derived_features], + "tags": self.tags, + } + + +class Edge(ToDict): + def __init__(self, + edge_id: Union[str, UUID], + from_id: Union[str, UUID], + to_id: Union[str, UUID], + conn_type: Union[str, RelationshipType]): + self.id = _to_uuid(edge_id) + self.from_id = _to_uuid(from_id) + self.to_id = _to_uuid(to_id) + self.conn_type = _to_type(conn_type, RelationshipType) + + def __eq__(self, o: object) -> bool: + # Edge ID is kinda useless + return self.from_id == o.from_id and self.to_id == o.to_id and self.conn_type == o.conn_type + + def __hash__(self) -> int: + return hash((self.from_id, self.to_id, self.conn_type)) + + def to_dict(self) -> dict: + return { + "relationshipId": str(self.id), + "fromEntityId": str(self.from_id), + "toEntityId": str(self.to_id), + "relationshipType": self.conn_type.name, + } + + +class EntitiesAndRelations(ToDict): + def __init__(self, entities: list[Entity], edges: list[Edge]): + self.entities = dict([(e.id, e) for e in entities]) + self.edges = set(edges) + + def to_dict(self) -> dict: + return { + "guidEntityMap": dict([(str(id), self.entities[id].to_dict()) for id in self.entities]), + "relations": list([e.to_dict() for e in self.edges]), + } + + +class ProjectDef: + def __init__(self, qualified_name: str, tags: dict = {}): + self.qualified_name = qualified_name + self.name = qualified_name + self.tags = tags + + +class SourceDef: + def __init__(self, + qualified_name: str, + name: str, + path: str, + type: str, + preprocessing: Optional[str] = None, + event_timestamp_column: Optional[str] = None, + timestamp_format: Optional[str] = None, + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.path = path + self.type = type + self.preprocessing = preprocessing + self.event_timestamp_column = event_timestamp_column + self.timestamp_format = timestamp_format + self.tags = tags + + +class AnchorDef: + def __init__(self, + qualified_name: str, + name: str, + source_id: Union[str, UUID], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.source_id = _to_uuid(source_id) + self.tags = tags + + +class AnchorFeatureDef: + def __init__(self, + qualified_name: str, + name: str, + feature_type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.feature_type = _to_type(feature_type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self.tags = tags + + +class DerivedFeatureDef: + def __init__(self, + qualified_name: str, + name: str, + feature_type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + input_anchor_features: list[Union[str, UUID]], + input_derived_features: list[Union[str, UUID]], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.feature_type = _to_type(feature_type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self.input_anchor_features = _to_uuid(input_anchor_features) + self.input_derived_features = _to_uuid(input_derived_features) + self.tags = tags diff --git a/registry/main.py b/registry/main.py new file mode 100644 index 000000000..d070ed7db --- /dev/null +++ b/registry/main.py @@ -0,0 +1,43 @@ +import os +import uvicorn +from fastapi import FastAPI +from starlette.middleware.cors import CORSMiddleware +from api import router as api_router + +rp = "/api/v1" + +# Need to be deleted +os.environ["API_BASE"] = "api/v1" +os.environ["CONNECTION_STR"] = "Server=tcp:feathrtestsql4.database.windows.net,1433;Initial Catalog=testsql;Persist Security Info=False;User ID=feathr@feathrtestsql4;Password=Password01!;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30" +os.environ["AZURE_CLIENT_ID"] = "db8dc4b0-202e-450c-b38d-7396ad9631a5" +os.environ["RBAC_ENABLED"] = "True" + +def get_application() -> FastAPI: + application = FastAPI( + title="access control", + debug=True, + version="0.1.0", + swagger_ui_oauth2_redirect_url='/oauth2-redirect', + swagger_ui_init_oauth={ + "usePkceWithAuthorizationCodeGrant": True, + "clientId": "e1475341-ff84-4b6e-a187-349e35551cff", + "scopes": [f'api://e1475341-ff84-4b6e-a187-349e35551cff/access_as_user'] + } + ) + # Enables CORS + application.add_middleware(CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + application.include_router(prefix = rp, router = api_router) + return application + + +app = get_application() + + +if __name__ == "__main__": + uvicorn.run("main:app", host="localhost", port=8000, reload=True) \ No newline at end of file diff --git a/registry/requirements.txt b/registry/requirements.txt new file mode 100644 index 000000000..77c6d123b --- /dev/null +++ b/registry/requirements.txt @@ -0,0 +1,4 @@ +pymssql +fastapi +uvicorn +pyjwt \ No newline at end of file diff --git a/registry/sql-registry/README.md b/registry/sql-registry/README.md index f06ca7def..95ce14ab2 100644 --- a/registry/sql-registry/README.md +++ b/registry/sql-registry/README.md @@ -2,4 +2,30 @@ This is the reference implementation of [the Feathr API spec](./api-spec.md), base on SQL databases instead of PurView. -Please note that this implementation uses iterations of `select` to retrieve graph lineages, this approach is very inefficient and should **not** be considered as production-ready. We only suggest to use this implementation for testing/researching purposes. \ No newline at end of file +Please note that this implementation uses iterations of `select` to retrieve graph lineages, this approach is very inefficient and should **not** be considered as production-ready. We only suggest to use this implementation for testing/researching purposes. + +## Access Control Notes (WIP) + +A Global Switch `RBAC_ENABLED` is required to set as `True` to turn on the access control protection feature. + +Please note that in this version, only access control management API and UI experience are included. Supported scenarios status are tracked below: + +- General Fundation: + - [x] Access Control Abstrct Class + - [x] API Spec Contents for Access Control Management APIs + - [ ] API Sepc Contents for Registry API Access Control +- SQL Implementaion: + - [x] `userroles` table CUD through FastAPI + - [x] `userroles` table schema & test data + - [x] Enable/Disable with `RBAC_ENABLE` configuration + - [ ] Initialize default Admin role for project creator: After `create_project` API is ready +- UI Experience + - [x] Hidden page `../management` for global admin to make CUD requests to `userroles` table + - [x] Use id token in Management API Request headers to identify requestor + - [ ] Protect SQL Registry API with Access Control: After `create_project` API is ready +- Future Enhancements: + - [ ] Functional in Feathr Client + - [ ] Support Security Group scenario in Access Control + - [ ] Support AAD Groups + - [ ] Support Other OAuth Providers + \ No newline at end of file diff --git a/registry/sql-registry/api-spec.md b/registry/sql-registry/api-spec.md index 1b14cae8b..1fcaa7391 100644 --- a/registry/sql-registry/api-spec.md +++ b/registry/sql-registry/api-spec.md @@ -364,3 +364,66 @@ Create new derived feature in the project | Field | Type | |-------|------| | guid | Guid | + + +## Access Control + +### Access Role Mapping + +| Pre-Defined Role | Access | +| ------------------ | ---------------------------------- | +| Admin | Read, Write, Manage | +| Contributor | Read, Write | +| Viewer | Read Only | +| Default | NA (configurable) | +| Global Admin | Admin Access to All Projects | +| Global Contributor | Contributor Access to All Projects | +| Global Viewer | Viewer Access to All Projects | + ++ Rules + + Project creator will defaultly get `Admin` access to the Project + + When a project is shared, the `Read` access is granted. + + In this version, customized role is NOT supported. + +### UserRoles + +Type: Object + +| Field | Type | +| ------------ | --------- | +| scope | `string` | +| userName | `string` | +| roleName | `string` | +| createBy | `string` | +| createDate | `datetime`| +| createReason | `string` | +| deleteDate | `datetime`| +| deleteReason | `string` | +| deleteBy | `string` | +| access | `list` | + ++ Notes: + + For SQL Access Control Implementation, a IDENTITY(1,1) id auto generated to provide a view of record insert sequence. + +## Access Control APIs + +### `GET /userroles` + +List all user roles + ++ Response Type: [`array`](#userroles) ++ Required Access: Manage + +### `POST /users/{user}/userroles/add` + +Create a new user role + ++ Request Type: [`UserRoles`](#userroles) ++ Required Access: Manage + +### `DELETE /users/{user}/userroles/delete` + +Remove a user role assignment + ++ Request Type: [`UserRoles`](#userroles) ++ Required Access: Manage diff --git a/registry/sql-registry/main.py b/registry/sql-registry/main.py index a40fae89c..d8804cc07 100644 --- a/registry/sql-registry/main.py +++ b/registry/sql-registry/main.py @@ -1,10 +1,12 @@ import os from typing import Optional -from fastapi import APIRouter, FastAPI, HTTPException +from fastapi import APIRouter, FastAPI, HTTPException, Request from starlette.middleware.cors import CORSMiddleware from registry import * from registry.db_registry import DbRegistry from registry.models import EntityType +from rbac import * +from rbac.db_rbac import DbRBAC rp = "/" try: @@ -18,14 +20,17 @@ registry = DbRegistry() app = FastAPI() router = APIRouter() +rbac_enabled = str(os.environ.get("RBAC_ENABLED")).lower() == 'true' +rbac = DbRBAC() # Enables CORS app.add_middleware(CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + @router.get("/projects") def get_projects() -> list[str]: @@ -54,7 +59,8 @@ def get_project_features(project: str, keyword: Optional[str] = None) -> list: features = registry.get_entities(feature_ids) return list([e.to_dict() for e in features]) else: - efs = registry.search_entity(keyword, [EntityType.AnchorFeature, EntityType.DerivedFeature]) + efs = registry.search_entity( + keyword, [EntityType.AnchorFeature, EntityType.DerivedFeature]) feature_ids = [ef.id for ef in efs] features = registry.get_entities(feature_ids) return list([e.to_dict() for e in features]) @@ -64,7 +70,8 @@ def get_project_features(project: str, keyword: Optional[str] = None) -> list: def get_feature(feature: str) -> dict: e = registry.get_entity(feature) if e.entity_type not in [EntityType.DerivedFeature, EntityType.AnchorFeature]: - raise HTTPException(status_code=404, detail=f"Feature {feature} not found") + raise HTTPException( + status_code=404, detail=f"Feature {feature} not found") return e @@ -74,4 +81,47 @@ def get_feature_lineage(feature: str) -> dict: return lineage.to_dict() -app.include_router(prefix = rp, router=router) +@router.get("/userroles") +def get_userroles(req: Request) -> list: + if rbac_enabled: + if rbac.is_global_admin(req.headers.get("authorization")): + userroles = rbac.get_userroles() + return list([r.to_dict() for r in userroles]) + else: + raise HTTPException( + status_code=403, detail=f"Only `Global Admin` have access to this content.") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +@router.post("/users/{user}/userroles/add") +def add_userrole(project: str, user: str, role: str, reason: str, req:Request): + if rbac_enabled: + if rbac.is_project_admin(req.headers.get("authorization"), project): + return rbac.add_userrole(project, user, role, reason, rbac.requestor) + else: + raise HTTPException( + status_code=403, detail=f"Only `Project Admin` can add userroles") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +@router.delete("/users/{user}/userroles/delete") +def delete_userrole(project: str, user: str, role: str, reason: str, req:Request): + if rbac_enabled: + if rbac.is_project_admin(req.headers.get("authorization"), project): + rbac.delete_userrole(project, user, role, reason, rbac.requestor) + else: + raise HTTPException( + status_code=403, detail=f"Only `Project Admin` can delete userroles") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +app.include_router(prefix=rp, router=router) diff --git a/registry/sql-registry/rbac/__init__.py b/registry/sql-registry/rbac/__init__.py new file mode 100644 index 000000000..203359df0 --- /dev/null +++ b/registry/sql-registry/rbac/__init__.py @@ -0,0 +1,7 @@ +__all__ = ["auth", "models", "interface", "db_rbac"] + +from rbac.auth import * +from rbac.interface import RBAC +from rbac.models import * +from registry.database import DbConnection, connect +from rbac.db_rbac import DbRBAC diff --git a/registry/sql-registry/rbac/auth.py b/registry/sql-registry/rbac/auth.py new file mode 100644 index 000000000..bf564301f --- /dev/null +++ b/registry/sql-registry/rbac/auth.py @@ -0,0 +1,44 @@ +import json +import requests +import jwt +from jwt.algorithms import RSAAlgorithm + +BEARER_TOKEN = "BEARER " + +class AuthProvider(): + """This is the abstract class to decode JWT ID token. + Sample Usage with Azure ID token: + jwks_uri = "https://login.microsoftonline.com/common/discovery/v2.0/keys" + client_id = {Your Client Id} + auth = rbac.AuthProvider(jwks_uri, client_id) + """ + + def __init__(self, jwks_uri, client_id): + """Args: + - client Id: used as audience ("aud") + - jwks_uri: used to get public key pool + """ + self.client_id = client_id + self.cert_set = self.get_certs(jwks_uri) + + def get_certs(self, jwks_uri: str): + """Get certs from jwks uri""" + certs = requests.get(jwks_uri).json() + return {cert['kid']: cert for cert in certs['keys']} + + def get_public_key(self, token: str): + """ Get public key based on token kid""" + header_data = jwt.get_unverified_header(token) + kid = header_data['kid'] + return RSAAlgorithm.from_jwk(json.dumps(self.cert_set[kid])) + + def decode_token(self, bearer_token: str): + """ Decode ID token with RA256 Algorithm + Sample Usage with Azure ID token: + decoded = auth.decode_token(token) + username = decoded.get('preferred_username').lower() + """ + # TODO: Process Bearer Token more elegantly + token = bearer_token[len(BEARER_TOKEN):] + return jwt.decode(token, self.get_public_key(token), algorithms=[ + "RS256"], audience=self.client_id) diff --git a/registry/sql-registry/rbac/db_rbac.py b/registry/sql-registry/rbac/db_rbac.py new file mode 100644 index 000000000..a735642ce --- /dev/null +++ b/registry/sql-registry/rbac/db_rbac.py @@ -0,0 +1,119 @@ +from distutils.log import warn +from logging import log +import os +from rbac import RBAC, AuthProvider +from rbac.models import Access, RoleAccessMapping, UserRole, RoleType, SUPER_ADMIN_SCOPE +from registry import connect + + +class DbRBAC(RBAC): + def __init__(self): + self.conn = connect() + + def get_userroles(self) -> list[UserRole]: + """query all the active user role records in SQL table + """ + rows = self.conn.query( + fr"""select record_id, project_name, user_name, role_name, create_by, create_reason, create_time, delete_by, delete_reason, delete_time + from userroles + where delete_reason is null""") + ret = [] + for row in rows: + r = UserRole(**row) + ret.append(UserRole(**row)) + return ret + + def get_userroles_by_user(self, user_name: str, role_name: str = None) -> list[UserRole]: + """query the active user role of certain user + """ + query = fr"""select record_id, project_name, user_name, role_name, create_by, create_reason, create_time, delete_by, delete_reason, delete_time + from userroles + where delete_reason is null and user_name ='{user_name}'""" + if role_name: + query += fr"and role_name = '{role_name}'" + rows = self.conn.query(query) + ret = [] + for row in rows: + ret.append(UserRole(**row)) + return ret + + def get_userroles_by_project(self, project_name: str, role_name: str = None) -> list[UserRole]: + """query the active user role of certain project. + """ + query = fr"""select record_id, project_name, user_name, role_name, create_by, create_reason, create_time, delete_reason, delete_time + from userroles + where delete_reason is null and project_name ='{project_name}'""" + if role_name: + query += fr"and role_name = '{role_name}'" + rows = self.conn.query(query) + ret = [] + for row in rows: + ret.append(UserRole(**row)) + return ret + + def add_userrole(self, project_name: str, user_name: str, role_name: str, create_reason: str, by:str): + """insert new user role relationship into sql table + """ + # check if record already exist + query = fr"""select * from userroles where delete_reason is null + and project_name ='{project_name}' and user_name = '{user_name}' and role_name = '{role_name}'""" + if len(self.conn.query(query)) > 0: + warn(f"User {user_name} already have {role_name} role of {project_name}.") + return True + + # insert new record + query = f"""insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) + values ('{project_name}','{user_name}','{role_name}','{by}' ,'{create_reason}', getutcdate())""" + return self.conn.update(query) + + def delete_userrole(self, project_name: str, user_name: str, role_name: str, delete_reason: str, by:str): + """mark existing user role relationship as deleted with reason + """ + query = fr"""UPDATE userroles SET + [delete_by] = '{by}', + [delete_reason] = '{delete_reason}', + [delete_time] = getutcdate() + WHERE [user_name] = '{user_name}' and [project_name] = '{project_name}' and [role_name] = '{role_name}' + and [delete_time] is null""" + return self.conn.update(query) + + def init_userrole(self, creator_name: str, project_name: str): + """initialize user role relationship when a new project is created + TODO: Add init user role to every new project call + """ + create_by = "system" + create_reason = "creator of project, get admin by default." + query = fr"""insert into userroles (project_name, user_name, role_name, create_reason, create_time) + values ('{project_name}','{creator_name}','{RoleType.ADMIN}','{create_by}','{create_reason}', getutcdate())""" + return self.conn.update(query) + + def validate_access(self, userrole: UserRole, access: Access): + if (userrole.project_name == SUPER_ADMIN_SCOPE or userrole.project_name == access.project_name) and access.access_name in RoleAccessMapping[userrole.role_name]: + return True + else: + return False + + def get_aad_user_name(self, token: str): + # TODO: extend this function when more login account types are suported + jwks_uri = "https://login.microsoftonline.com/common/discovery/v2.0/keys" + client_id = os.environ.get("AZURE_CLIENT_ID") + if client_id: + auth = AuthProvider(jwks_uri, client_id) + decoded = auth.decode_token(token) + return decoded.get("preferred_username") + else: + warn("`AZURE_CLIENT_ID` is not set, cannot decode AAD token.") + return None + + def is_project_admin(self, token: str, project_name: str): + """To protect management APIs, only admin can modify RBAC records + """ + self.requestor = self.get_aad_user_name(token) + userroles = self.get_userroles_by_user(self.requestor, RoleType.ADMIN) + for userrole in userroles: + if userrole.project_name == project_name or userrole.project_name == SUPER_ADMIN_SCOPE: + return True + return False + + def is_global_admin(self, token: str): + return self.is_project_admin(token, SUPER_ADMIN_SCOPE) diff --git a/registry/sql-registry/rbac/interface.py b/registry/sql-registry/rbac/interface.py new file mode 100644 index 000000000..355a20701 --- /dev/null +++ b/registry/sql-registry/rbac/interface.py @@ -0,0 +1,48 @@ +from abc import ABC, abstractmethod +from registry.database import DbConnection +from rbac.models import Access, UserRole +from uuid import UUID + + +class RBAC(ABC): + @abstractmethod + def get_userroles(self) -> list[UserRole]: + """Get List of All User Role Records + """ + pass + + @abstractmethod + def add_userrole(self, userrole: UserRole): + """Add a Role to a User + """ + pass + + @abstractmethod + def delete_userrole(self, userrole: UserRole): + """Delete a Role of a User + """ + pass + + @abstractmethod + def init_userrole(self, creator_name: str, project_name: str): + """Default User Role Relationship when a new project is created + """ + pass + + @abstractmethod + def get_userroles_by_user(self, user_name: str) -> list[UserRole]: + """Get List of All User Role Records for a User + """ + pass + + @abstractmethod + def get_userroles_by_project(self, project_name: str) -> list[UserRole]: + """Get List of All User Role Records for a Project + """ + pass + + @abstractmethod + def validate_access(self, userrole: UserRole, access: Access): + """Validate if a Role has certian access + """ + pass diff --git a/registry/sql-registry/rbac/models.py b/registry/sql-registry/rbac/models.py new file mode 100644 index 000000000..e4b8527ed --- /dev/null +++ b/registry/sql-registry/rbac/models.py @@ -0,0 +1,80 @@ +from datetime import datetime +from enum import Enum +from typing import Optional +from numpy import number +from registry.models import ToDict + +SUPER_ADMIN_SCOPE = "global" + + +class RoleType(str, Enum): + ADMIN = "admin", + CONSUMER = "consumer", + PRODUCER = "producer" + DEFAULT = "default" + + +RoleAccessMapping = { + RoleType.ADMIN: ["read", "write", "manage"], + RoleType.CONSUMER: ["read"], + RoleType.PRODUCER: ["read", "write"], + RoleType.DEFAULT: [] +} + + +class UserRole(ToDict): + def __init__(self, + record_id: number, + project_name: str, + user_name: str, + role_name: str, + create_by: str, + create_reason: str, + create_time: datetime, + delete_by: Optional[str] = None, + delete_reason: Optional[str] = None, + delete_time: Optional[datetime] = None, + **kwargs): + self.record_id = record_id + self.project_name = project_name.lower() + self.user_name = user_name.lower() + self.role_name = role_name.lower() + self.create_by = create_by.lower() + self.create_reason = create_reason + self.create_time = create_time + self.delete_by = delete_by + self.delete_reason = delete_reason + self.delete_time = delete_time + self.access = RoleAccessMapping[RoleType(self.role_name)] + + def to_dict(self) -> dict: + return { + "id": str(self.record_id), + "scope": self.project_name, + "userName": self.user_name, + "roleName": str(self.role_name), + "createBy": self.create_by, + "createReason": self.create_reason, + "createTime": str(self.create_time), + "deleteBy": str(self.delete_by), + "deleteReason": self.delete_reason, + "deleteTime": str(self.delete_time), + "access": self.access + } + + +class Access(ToDict): + def __init__(self, + record_id: number, + project_name: str, + access_name: str) -> None: + self.record_id = record_id + self.project_name = project_name + self.access_name = access_name + + def to_dict(self) -> dict: + return { + "record_id": str(self.record_id), + "project_name": self.project_name, + "access": self.access_name, + } diff --git a/registry/sql-registry/registry/database.py b/registry/sql-registry/registry/database.py index d82568972..77d18cf90 100644 --- a/registry/sql-registry/registry/database.py +++ b/registry/sql-registry/registry/database.py @@ -7,11 +7,13 @@ providers = [] + class DbConnection(ABC): @abstractmethod - def execute(self, sql: str, *args, **kwargs) -> list[dict]: + def query(self, sql: str, *args, **kwargs) -> list[dict]: pass + def quote(id): if isinstance(id, str): return f"'{id}'" @@ -37,6 +39,8 @@ def parse_conn_str(s: str) -> dict: class MssqlConnection(DbConnection): + os.environ["CONNECTION_STR"] = "Server=tcp:feathrtestsql4.database.windows.net,1433;Initial Catalog=testsql;Persist Security Info=False;User ID=feathr@feathrtestsql4;Password=Password01!;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30" + @staticmethod def connect(*args, **kwargs): conn_str = os.environ["CONNECTION_STR"] @@ -49,11 +53,11 @@ def __init__(self, params): self.params = params self.make_connection() self.mutex = threading.Lock() - + def make_connection(self): self.conn = pymssql.connect(**self.params) - def execute(self, sql: str, *args, **kwargs) -> list[dict]: + def query(self, sql: str, *args, **kwargs) -> list[dict]: debug(f"SQL: `{sql}`") # NOTE: Only one cursor is allowed at the same time retry = 0 @@ -73,6 +77,25 @@ def execute(self, sql: str, *args, **kwargs) -> list[dict]: raise pass + def update(self, sql: str, *args, **kwargs): + retry = 0 + while True: + try: + with self.mutex: + c = self.conn.cursor(as_dict=True) + c.execute(sql, *args, **kwargs) + self.conn.commit() + return True + except pymssql.OperationalError: + warn("Database error, retrying...") + # Reconnect + self.make_connection() + retry += 1 + if retry >= 3: + # Stop retrying + raise + pass + providers.append(MssqlConnection) @@ -82,4 +105,4 @@ def connect(): ret = p.connect() if ret is not None: return ret - raise RuntimeError("Cannot connect to database") \ No newline at end of file + raise RuntimeError("Cannot connect to database") diff --git a/registry/sql-registry/registry/db_registry.py b/registry/sql-registry/registry/db_registry.py index f5456c5e5..74ccf2713 100644 --- a/registry/sql-registry/registry/db_registry.py +++ b/registry/sql-registry/registry/db_registry.py @@ -18,7 +18,7 @@ def __init__(self): self.conn = connect() def get_projects(self) -> list[str]: - ret = self.conn.execute( + ret = self.conn.query( f"select qualified_name from entities where entity_type='{EntityType.Project}'") return list([r["qualified_name"] for r in ret]) @@ -35,12 +35,12 @@ def get_entity_id(self, id_or_name: Union[str, UUID]) -> UUID: except ValueError: pass # It is a name - ret = self.conn.execute( + ret = self.conn.query( f"select entity_id from entities where qualified_name='{id_or_name}'") return ret[0]["entity_id"] def get_neighbors(self, id_or_name: Union[str, UUID], relationship: RelationshipType) -> list[Edge]: - rows = self.conn.execute(fr''' + rows = self.conn.query(fr''' select edge_id, from_id, to_id, conn_type from edges where from_id = '{self.get_entity_id(id_or_name)}' @@ -78,7 +78,8 @@ def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: edges = edges.union(conn) features = list([child_map[id] for id in feature_ids]) anchor.attributes.features = features - source_id = self.get_neighbors(anchor.id, RelationshipType.Consumes)[0].to_id + source_id = self.get_neighbors( + anchor.id, RelationshipType.Consumes)[0].to_id anchor.attributes.source = child_map[source_id] for df in project.attributes.derived_features: conn = self.get_neighbors(anchor.id, RelationshipType.Consumes) @@ -88,7 +89,7 @@ def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: df.attributes.input_features = features all_edges = self._get_edges(ids) return EntitiesAndRelations([project] + children, list(edges.union(all_edges))) - + def _fill_entity(self, e: Entity) -> Entity: """ Entities in the DB contains only attributes belong to itself, but the returned @@ -105,7 +106,8 @@ def _fill_entity(self, e: Entity) -> Entity: feature_ids = [e.to_id for e in conn] features = self._get_entities(feature_ids) e.attributes.features = features - source_id = self.get_neighbors(e.id, RelationshipType.Consumes)[0].to_id + source_id = self.get_neighbors( + e.id, RelationshipType.Consumes)[0].to_id source = self.get_entity(source_id) e.attributes.source = source return e @@ -116,21 +118,21 @@ def _fill_entity(self, e: Entity) -> Entity: e.attributes.input_features = features return e return e - + def _get_edges(self, ids: list[UUID], types: list[RelationshipType] = []) -> list[Edge]: sql = fr"""select edge_id, from_id, to_id, conn_type from edges where from_id in ({quote(ids)}) and to_id in ({quote(ids)})""" - if len(types)>0: + if len(types) > 0: sql = fr"""select edge_id, from_id, to_id, conn_type from edges where conn_type in ({quote(types)}) and from_id in ({quote(ids)}) and to_id in ({quote(ids)})""" - rows = self.conn.execute(sql) + rows = self.conn.query(sql) return list([_to_type(row, Edge) for row in rows]) - + def _get_entity(self, id_or_name: Union[str, UUID]) -> Entity: - row = self.conn.execute(fr''' + row = self.conn.query(fr''' select entity_id, qualified_name, entity_type, attributes from entities where entity_id = '{self.get_entity_id(id_or_name)}' @@ -139,7 +141,7 @@ def _get_entity(self, id_or_name: Union[str, UUID]) -> Entity: return _to_type(row, Entity) def _get_entities(self, ids: list[UUID]) -> list[Entity]: - rows = self.conn.execute(fr''' + rows = self.conn.query(fr''' select entity_id, qualified_name, entity_type, attributes from entities where entity_id in ({quote(ids)}) @@ -154,7 +156,7 @@ def _bfs(self, id: UUID, conn_type: RelationshipType) -> Tuple[list[Entity], lis """ Breadth first traversal Starts from `id`, follow edges with `conn_type` only. - + WARN: There is no depth limit. """ connections = [] @@ -180,7 +182,7 @@ def _bfs_step(self, ids: list[UUID], conn_type: RelationshipType) -> set[dict]: """ ids = list([id["to_id"] for id in ids]) sql = fr"""select edge_id, from_id, to_id, conn_type from edges where conn_type = '{conn_type.name}' and from_id in ({quote(ids)})""" - return self.conn.execute(sql) + return self.conn.query(sql) def search_entity(self, keyword: str, @@ -190,5 +192,5 @@ def search_entity(self, """ types = ",".join([quote(str(t)) for t in type]) sql = fr'''select entity_id as id, qualified_name, entity_type as type from entities where qualified_name like %s and entity_type in ({types})''' - rows = self.conn.execute(sql, ('%' + keyword + '%', )) + rows = self.conn.query(sql, ('%' + keyword + '%', )) return list([EntityRef(**row) for row in rows]) diff --git a/registry/sql-registry/requirements.txt b/registry/sql-registry/requirements.txt index c6d61de98..77c6d123b 100644 --- a/registry/sql-registry/requirements.txt +++ b/registry/sql-registry/requirements.txt @@ -1,3 +1,4 @@ pymssql fastapi -uvicorn \ No newline at end of file +uvicorn +pyjwt \ No newline at end of file diff --git a/registry/sql-registry/scripts/schema.sql b/registry/sql-registry/scripts/schema.sql index d7258d577..b7b343196 100644 --- a/registry/sql-registry/scripts/schema.sql +++ b/registry/sql-registry/scripts/schema.sql @@ -12,4 +12,32 @@ create table edges from_id varchar(50) not null, to_id varchar(50) not null, conn_type varchar(20) not null, +) + +create table userroles +( + record_id int IDENTITY(1,1), + project_name varchar(50) not null, + user_name varchar(50) not null, + role_name varchar(50) not null, + create_by varchar(50) not null, + create_reason varchar(50) not null, + create_time datetime not null, + delete_by varchar(50), + delete_reason varchar(50), + delete_time datetime, +) + +create table access +( + record_id uniqueidentifier NOT NULL DEFAULT newid(), + project_name varchar(50) not null, + access_name varchar(50) not null, +) + +create table roleaccess +( + record_id uniqueidentifier NOT NULL DEFAULT newid(), + role_name varchar(50) not null, + access_name varchar(50) not null, ) \ No newline at end of file diff --git a/registry/sql-registry/scripts/test_data.sql b/registry/sql-registry/scripts/test_data.sql index a248d56fe..d1a2ec664 100644 --- a/registry/sql-registry/scripts/test_data.sql +++ b/registry/sql-registry/scripts/test_data.sql @@ -90,3 +90,22 @@ insert into edges (edge_id, from_id, to_id, conn_type) values ('23a05c64-b204-4c insert into edges (edge_id, from_id, to_id, conn_type) values ('3721c600-6423-4a38-a9b2-d85b426b6eaa', '226b42ee-0c34-4329-b935-744aecc63fb4', '103baca1-377a-4ddf-8429-5da91026c269', 'Consumes'); insert into edges (edge_id, from_id, to_id, conn_type) values ('5f9b86fe-bdc9-4a76-b07c-876b9d7c1ee1', '103baca1-377a-4ddf-8429-5da91026c269', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Produces'); +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('global', 'abc@microsoft.com','admin', 'test_data@microsoft.com', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('global', 'efg@microsoft.com','admin', 'test_data@microsoft.com', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('feathr_ci_registry_12_33_182947', 'efg@microsoft.com','admin','test_data@microsoft.com', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('feathr_ci_registry_12_33_182947', 'hij@microsoft.com','consumer', 'test_data@microsoft.com', 'test data', getutcdate()) + +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'read') +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'write') +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'manage') +insert into access (project_name, access_name) values ('global', 'read') +insert into access (project_name, access_name) values ('global', 'write') +insert into access (project_name, access_name) values ('global', 'manage') + +insert into roleaccess (role_name, access_name) values ('admin', 'read') +insert into roleaccess (role_name, access_name) values ('admin', 'write') +insert into roleaccess (role_name, access_name) values ('admin', 'manage') +insert into roleaccess (role_name, access_name) values ('contributor', 'read') +insert into roleaccess (role_name, access_name) values ('contributor', 'write') +insert into roleaccess (role_name, access_name) values ('viewer', 'read') +insert into roleaccess (role_name, access_name) values ('default', '') \ No newline at end of file diff --git a/registry/sql_registry/.dockerignore b/registry/sql_registry/.dockerignore new file mode 100644 index 000000000..bc0ed1f7a --- /dev/null +++ b/registry/sql_registry/.dockerignore @@ -0,0 +1,3 @@ +__pycache__ +.env +.vscode diff --git a/registry/sql_registry/.gitignore b/registry/sql_registry/.gitignore new file mode 100644 index 000000000..ed2a6faed --- /dev/null +++ b/registry/sql_registry/.gitignore @@ -0,0 +1,4 @@ +__pycache__ +.env +.vscode +.idea diff --git a/registry/sql_registry/Dockerfile b/registry/sql_registry/Dockerfile new file mode 100644 index 000000000..d2647021d --- /dev/null +++ b/registry/sql_registry/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.9 + +COPY ./ /usr/src + +WORKDIR /usr/src +RUN pip install -r requirements.txt + +# Start web server +CMD [ "uvicorn","main:app","--host", "0.0.0.0", "--port", "80" ] diff --git a/registry/sql_registry/README.md b/registry/sql_registry/README.md new file mode 100644 index 000000000..f06ca7def --- /dev/null +++ b/registry/sql_registry/README.md @@ -0,0 +1,5 @@ +# SQL-Based Registry for Feathr + +This is the reference implementation of [the Feathr API spec](./api-spec.md), base on SQL databases instead of PurView. + +Please note that this implementation uses iterations of `select` to retrieve graph lineages, this approach is very inefficient and should **not** be considered as production-ready. We only suggest to use this implementation for testing/researching purposes. \ No newline at end of file diff --git a/registry/sql_registry/__init__.py b/registry/sql_registry/__init__.py new file mode 100644 index 000000000..524ca287d --- /dev/null +++ b/registry/sql_registry/__init__.py @@ -0,0 +1,6 @@ +__all__ = ["interface", "models", "database", "db_registry"] + +from sql_registry.registry.models import * +from sql_registry.registry.interface import Registry +from sql_registry.registry.database import DbConnection, connect +from sql_registry.registry.db_registry import DbRegistry diff --git a/registry/sql_registry/api-spec.md b/registry/sql_registry/api-spec.md new file mode 100644 index 000000000..83e1f95c3 --- /dev/null +++ b/registry/sql_registry/api-spec.md @@ -0,0 +1,425 @@ +# Feathr Registry API Specifications + +## Data Models + +### EntityType +Type: Enum + +| Value | +|-----------------------------| +| `feathr_workspace_v1` | +| `feathr_source_v1` | +| `feathr_anchor_v1` | +| `feathr_anchor_feature_v1` | +| `feathr_derived_feature_v1` | + +### ValueType +Type: Enum + +| Value | +|---------------| +| `UNSPECIFIED` | +| `BOOL` | +| `INT32` | +| `INT64` | +| `FLOAT` | +| `DOUBLE` | +| `STRING` | +| `BYTES` | + +### VectorType +Type: Enum + +| Value | +|----------| +| `TENSOR` | + +### TensorCategory +Type: Enum + +| Value | +|----------| +| `DENSE` | +| `SPARSE` | + +### FeatureType +Type: Object + +| Field | Type | +|----------------|-------------------------------------| +| type | [`VectorType`](#valuetype) | +| tensorCategory | [`TensorCategory`](#tensorcategory) | +| dimensionType | [`array`](#valuetype) | +| valType | [`ValueType`](#valuetype) | + +### TypedKey +Type: Object + +| Field | Type | +|------------------|-----------------------------| +| key_column | `string` | +| key_column_type | [`ValueType`](#valuetype) | +| full_name | `string`, optional | +| description | `string`, optional | +| key_column_alias | `string`, optional | + +### ExpressionTransformation +Type: Object + +| Field | Type | +|----------------|----------| +| transform_expr | `string` | + +### WindowAggregationTransformation +Type: Object + +| Field | Type | +|----------|--------------------| +| def_expr | `string` | +| agg_func | `string`, optional | +| window | `string`, optional | +| group_by | `string`, optional | +| filter | `string`, optional | +| limit | `number`, optional | + +### UdfTransformation +Type: Object + +| Field | Type | +|-------|----------| +| name | `string` | + +### EntityReference +Type: Object + +| Field | Type | Comments | +|------------------|-----------------------------|--------------------------------------| +| guid | `Guid` | | +| typeName | [`EntityType`](#entitytype) | | +| uniqueAttributes | `map` | Contains `qualifiedName` only so far | + +### ProjectAttributes +Type: Object + +| Field | Type | +|------------------|----------------------------------------------| +| qualifiedName | `string` | +| name | `string` | +| anchors | [`array`](#entityreference) | +| sources | [`array`](#entityreference) | +| anchor_features | [`array`](#entityreference) | +| derived_features | [`array`](#entityreference) | +| tags | `map` | + +### SourceAttributes +Type: Object + +| Field | Type | +|----------------------|-----------------------| +| qualifiedName | `string` | +| name | `string` | +| path | `string` | +| preprocessing | `string`, optional | +| eventTimestampColumn | `string`, optional | +| timestampFormat | `string`, optional | +| type | `string` | +| tags | `map` | + +### AnchorAttributes +Type: Object + +| Field | Type | +|---------------|----------------------------------------------| +| qualifiedName | `string` | +| name | `string` | +| features | [`array`](#entityreference) | +| source | [`EntityReference`](#entityreference) | +| tags | `map` | + +### AnchorFeatureAttributes +Type: Object + +| Field | Type | +|----------------|--------------------------------| +| qualifiedName | `string` | +| name | `string` | +| type | [`FeatureType`](#featuretype) | +| transformation | [`ExpressionTransformation`](#expressiontransformation)
`or` [`WindowAggregationTransformation`](#windowaggregationtransformation)
`or` [`UdfTransformation`](#udftransformation) | +| key | [`array`](#typedkey) | +| tags | `map` | + +### DerivedFeatureAttributes +Type: Object + +| Field | Type | +|------------------------|--------------------------------| +| qualifiedName | `string` | +| name | `string` | +| type | [`FeatureType`](#featuretype) | +| transformation | [`ExpressionTransformation`](#expressiontransformation)
`or` [`WindowAggregationTransformation`](#windowaggregationtransformation)
`or` [`UdfTransformation`](#udftransformation) | +| key | [`array`](#typedkey) | +| input_anchor_features | [`array`](#entityreference) | +| input_derived_features | [`array`](#entityreference) | +| tags | `map` | + +### EntityStatus +Type: Enum + +| Value | +|----------| +| `ACTIVE` | + +### Entity +Type: Object + +| Field | Type | +|----------------|---------------------------------| +| guid | `Guid` | +| lastModifiedTS | `string` | +| status | [`EntityStatus`](#entitystatus) | +| displayText | `string` | +| typeName | [`EntityType`](#entitytype) | +| attributes | [`ProjectAttributes`](#projectattributes)
`or` [`SourceAttributes`](#sourceattributes)
`or` [`AnchorAttributes`](#anchorattributes)
`or` [`AnchorFeatureAttributes`](#anchorfeatureattributes)
`or` [`DerivedFeatureAttributes`](#derivedfeatureattributes) | + +### RelationshipType +Type: Enum + +| Value | +|-------------| +| `BelongsTo` | +| `Contains` | +| `Produces` | +| `Consumes` | + +### Relationship +Type: Object + +| Field | Type | +|------------------|-----------------------------------------| +| relationshipId | `Guid` | +| relationshipType | [`RelationshipType`](#relationshiptype) | +| fromEntityId | `Guid` | +| toEntityId | `Guid` | + +### ProjectDefinition +Type: Object + +| Field | Type | +|----------------------|-----------------------| +| qualifiedName | `string` | +| tags | `map` | + + +### SourceDefinition +Type: Object + +| Field | Type | +|----------------------|-----------------------| +| qualifiedName | `string` | +| name | `string` | +| path | `string` | +| preprocessing | `string`, optional | +| eventTimestampColumn | `string`, optional | +| timestampFormat | `string`, optional | +| type | `string` | +| tags | `map` | + +### AnchorDefinition +Type: Object + +| Field | Type | +|----------------------|-----------------------| +| qualifiedName | `string` | +| name | `string` | +| source_id | `Guid` | +| tags | `map` | + +### AnchorFeatureDefinition +Type: Object + +| Field | Type | +|----------------|--------------------------------| +| qualifiedName | `string` | +| name | `string` | +| featureType | [`FeatureType`](#featuretype) | +| transformation | [`ExpressionTransformation`](#expressiontransformation)
`or` [`WindowAggregationTransformation`](#windowaggregationtransformation)
`or` [`UdfTransformation`](#udftransformation) | +| key | [`array`](#typedkey) | +| tags | `map` | + +### DerivedFeatureDefinition +Type: Object + +| Field | Type | +|------------------------|--------------------------------| +| qualifiedName | `string` | +| name | `string` | +| featureType | [`FeatureType`](#featuretype) | +| transformation | [`ExpressionTransformation`](#expressiontransformation)
`or` [`WindowAggregationTransformation`](#windowaggregationtransformation)
`or` [`UdfTransformation`](#udftransformation) | +| key | [`array`](#typedkey) | +| input_anchor_features | `array` | +| input_derived_features | `array` | +| tags | `map` | + + +### EntitiesAndRelationships +Type: Object + +| Field | Type | +|---------------|----------------------------------------| +| guidEntityMap | [`map`](#entity) | +| relations | [`array`](#relationship) | + + +## Feathr Registry API + +### `GET /projects` +List **names** of all projects. + +Response Type: `array` + +### `GET /projects/{project}` +Get everything defined in the project + +Response Type: [`EntitiesAndRelationships`](#entitiesandrelationships) + +### `GET /projects/{project}/datasources` +Get all sources defined in the project. + +Response Type: [`array`](#entity) + +### `GET /projects/{project}/features` +Get all anchor features and derived features in the project, or only features meet the search criteria in the project. + +Query Parameters: + +| Field | Type | +|---------|--------| +| keyword | string | +| size | number | +| offset | number | + + +Response Type: Object + +| Field | Type | +|----------|----------------------------| +| features | [`array`](#entity) | + +### `GET /features/:feature` +Get feature details. + +Response Type: Object + +| Field | Type | Comments | +|-----------------|-----------------------|-----------------------------| +| entity | [`Entity`](#entity) | | +| referredEntities| `map` | For compatibility, not used | + +### `POST /projects` +Create new project + ++ Request Type: [`ProjectDefinition`](#projectdefinition) ++ Response Type: Object + +| Field | Type | +|-------|------| +| guid | Guid | + +### `POST /projects/{project}/datasources` +Create new source in the project + ++ Request Type: [`SourceDefinition`](#sourcedefinition) ++ Response Type: Object + +| Field | Type | +|-------|------| +| guid | Guid | + +### `POST /projects/{project}/anchors` +Create new anchor in the project + ++ Request Type: [`AnchorDefinition`](#anchordefinition) ++ Response Type: Object + +| Field | Type | +|-------|------| +| guid | Guid | + +### `POST /projects/{project}/anchors/{anchor}/features` +Create new anchor feature in the project under specified anchor + ++ Request Type: [`AnchorFeatureDefinition`](#anchorfeaturedefinition) ++ Response Type: Object + +| Field | Type | +|-------|------| +| guid | Guid | + +### `POST /projects/{project}/derivedfeatures` +Create new derived feature in the project + ++ Request Type: [`DerivedFeatureDefinition`](#derivedfeaturedefinition) ++ Response Type: Object + +| Field | Type | +|-------|------| +| guid | Guid | + + +## Access Control + +### Access Role Mapping + +| Pre-Defined Role | Access | +| ------------------ | ---------------------------------- | +| Admin | Read, Write, Management | +| Contributor | Read, Write | +| Viewer | Read Only | +| Default | NA (configurable) | +| Global Admin | Admin Access to All Projects | +| Global Contributor | Contributor Access to All Projects | +| Global Viewer | Viewer Access to All Projects | + ++ Rules + + Project creator will defaultly get `Admin` access to the Project + + When a project is shared, the `Read` access is granted. + + In this version, customized role is NOT supported. + +### UserRoles + +Type: Object + +| Field | Type | +| ------------ | --------- | +| id | `string` | +| scope | `string` | +| userName | `string` | +| roleName | `string` | +| createDate | `datetime`| +| createReason | `string` | +| deleteDate | `datetime`| +| deleteReason | `string` | +| access | `list` | + +## Access Control APIs + +### `GET/userroles` + +List all user roles + ++ Response Type: [`array`](#userroles) ++ Required Access: Management + +### `POST /users/{user}/userroles/add` + +Create a new user role + ++ Request Type: [`UserRoles`](#userroles) ++ Required Access: Management + +### `POST /users/{user}/userroles/delete` + +Remove a user role assginment + ++ Request Type: [`UserRoles`](#userroles) ++ Required Access: Management \ No newline at end of file diff --git a/registry/sql_registry/main.py b/registry/sql_registry/main.py new file mode 100644 index 000000000..e1e3eda5c --- /dev/null +++ b/registry/sql_registry/main.py @@ -0,0 +1,145 @@ +import os +from typing import Optional +from fastapi import APIRouter, FastAPI, HTTPException +from starlette.middleware.cors import CORSMiddleware +from registry import * +from registry.db_registry import DbRegistry +from registry.models import EntityType +from rbac import * +from rbac.db_rbac import DbRBAC + +rp = "/" +os.environ["API_BASE"] = "api/v1" +try: + rp = os.environ["API_BASE"] + if rp[0] != '/': + rp = '/' + rp +except: + pass +print("Using API BASE: ", rp) + +registry = DbRegistry() +rbac = DbRBAC() +app = FastAPI() +router = APIRouter() + +# Enables CORS +app.add_middleware(CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + +@router.get("/projects") +def get_projects() -> list[str]: + return registry.get_projects() + + +@router.get("/projects/{project}") +def get_projects(project: str) -> dict: + return registry.get_project(project).to_dict() + + +@router.get("/projects/{project}/datasources") +def get_project_datasources(project: str) -> list: + p = registry.get_entity(project) + source_ids = [s.id for s in p.attributes.sources] + sources = registry.get_entities(source_ids) + return list([e.to_dict() for e in sources]) + + +@router.get("/projects/{project}/features") +def get_project_features(project: str, keyword: Optional[str] = None) -> list: + if keyword is None: + p = registry.get_entity(project) + feature_ids = [s.id for s in p.attributes.anchor_features] + \ + [s.id for s in p.attributes.derived_features] + features = registry.get_entities(feature_ids) + return list([e.to_dict() for e in features]) + else: + efs = registry.search_entity( + keyword, [EntityType.AnchorFeature, EntityType.DerivedFeature]) + feature_ids = [ef.id for ef in efs] + features = registry.get_entities(feature_ids) + return list([e.to_dict() for e in features]) + + +@router.get("/features/{feature}") +def get_feature(feature: str) -> dict: + e = registry.get_entity(feature) + if e.entity_type not in [EntityType.DerivedFeature, EntityType.AnchorFeature]: + raise HTTPException( + status_code=404, detail=f"Feature {feature} not found") + return e + + +@router.get("/features/{feature}/lineage") +def get_feature_lineage(feature: str) -> dict: + lineage = registry.get_lineage(feature) + return lineage.to_dict() + + +@router.get("/userroles") +def get_userroles(code: str) -> list: + if rbac.is_global_admin(code): + userroles = rbac.get_userroles() + return list([r.to_dict() for r in userroles]) + else: + raise HTTPException( +<<<<<<< Updated upstream:registry/sql-registry/main.py + status_code=403, detail=f"Only Global Admin have access to this content.") + + +@router.post("/users/{user}/userroles/add") +def add_userrole(project: str, user: str, role: str, reason: str, code: str): + if rbac.is_project_admin(code, project): + return rbac.add_userrole(project, user, role, reason) + else: + raise HTTPException( + status_code=403, detail=f"Only Project Admin can add userroles") + + +@router.post("/users/{user}/userroles/delete") +def delete_userrole(project: str, user: str, role: str, reason: str, code: str): + if rbac.is_project_admin(code, project): + rbac.delete_userrole(project, user, role, reason) + else: + raise HTTPException( + status_code=403, detail=f"Only Project Admin can delete userroles") +======= + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +@router.post("/users/{user}/userroles/add") +def add_userrole(project: str, user: str, role: str, reason: str, req: Request): + if rbac_enabled: + if rbac.is_project_admin(req.headers.get("authorization"), project): + return rbac.add_userrole(project, user, role, reason, rbac.requestor) + else: + raise HTTPException( + status_code=403, detail=f"Only `Project Admin` can add userroles") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) + + +@router.delete("/users/{user}/userroles/delete") +def delete_userrole(project: str, user: str, role: str, reason: str, req: Request): + if rbac_enabled: + if rbac.is_project_admin(req.headers.get("authorization"), project): + rbac.delete_userrole(project, user, role, reason, rbac.requestor) + else: + raise HTTPException( + status_code=403, detail=f"Only `Project Admin` can delete userroles") + else: + raise HTTPException( + status_code=503, detail=f"Registry access control is not enabled. Please Set `RBAC_ENABLED` to true." + ) +>>>>>>> Stashed changes:registry/sql_registry/main.py + + +app.include_router(prefix=rp, router=router) diff --git a/registry/sql_registry/registry/__init__.py b/registry/sql_registry/registry/__init__.py new file mode 100644 index 000000000..524ca287d --- /dev/null +++ b/registry/sql_registry/registry/__init__.py @@ -0,0 +1,6 @@ +__all__ = ["interface", "models", "database", "db_registry"] + +from sql_registry.registry.models import * +from sql_registry.registry.interface import Registry +from sql_registry.registry.database import DbConnection, connect +from sql_registry.registry.db_registry import DbRegistry diff --git a/registry/sql_registry/registry/database.py b/registry/sql_registry/registry/database.py new file mode 100644 index 000000000..38aa33425 --- /dev/null +++ b/registry/sql_registry/registry/database.py @@ -0,0 +1,106 @@ +from abc import ABC, abstractmethod +import threading +from distutils.log import debug, warn +import os +import pymssql + + +providers = [] + + +class DbConnection(ABC): + @abstractmethod + def query(self, sql: str, *args, **kwargs) -> list[dict]: + pass + + +def quote(id): + if isinstance(id, str): + return f"'{id}'" + else: + return ",".join([f"'{i}'" for i in id]) + + +def parse_conn_str(s: str) -> dict: + """ + TODO: Not a sound and safe implementation, but useful enough in this case + as the connection string is provided by users themselves. + """ + parts = dict([p.strip().split("=", 1) + for p in s.split(";") if len(p.strip()) > 0]) + server = parts["Server"].split(":")[1].split(",")[0] + return { + "host": server, + "database": parts["Initial Catalog"], + "user": parts["User ID"], + "password": parts["Password"], + # "charset": "utf-8", ## For unknown reason this causes connection failure + } + + +class MssqlConnection(DbConnection): + @staticmethod + def connect(*args, **kwargs): + conn_str = "Server=tcp:feathrtestsql4.database.windows.net,1433;Initial Catalog=testsql;Persist Security Info=False;User ID=feathr@feathrtestsql4;Password=Password01!;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + if "Server=" not in conn_str: + debug("`CONNECTION_STR` is not in ADO connection string format") + return None + return MssqlConnection(parse_conn_str(conn_str)) + + def __init__(self, params): + self.params = params + self.make_connection() + self.mutex = threading.Lock() + + def make_connection(self): + self.conn = pymssql.connect(**self.params) + + def query(self, sql: str, *args, **kwargs) -> list[dict]: + debug(f"SQL: `{sql}`") + # NOTE: Only one cursor is allowed at the same time + retry = 0 + while True: + try: + with self.mutex: + c = self.conn.cursor(as_dict=True) + c.execute(sql, *args, **kwargs) + return c.fetchall() + except pymssql.OperationalError: + warn("Database error, retrying...") + # Reconnect + self.make_connection() + retry += 1 + if retry >= 3: + # Stop retrying + raise + pass + + def update(self, sql: str, *args, **kwargs): + retry = 0 + while True: + try: + with self.mutex: + c = self.conn.cursor(as_dict=True) + c.execute(sql, *args, **kwargs) + self.conn.commit() + return True + except pymssql.OperationalError: + warn("Database error, retrying...") + # Reconnect + self.make_connection() + retry += 1 + if retry >= 3: + # Stop retrying + raise + pass + + +providers.append(MssqlConnection) + + +def connect(): + for p in providers: + ret = p.connect() + if ret is not None: + return ret + raise RuntimeError("Cannot connect to database") diff --git a/registry/sql_registry/registry/db_registry.py b/registry/sql_registry/registry/db_registry.py new file mode 100644 index 000000000..b1d3aad37 --- /dev/null +++ b/registry/sql_registry/registry/db_registry.py @@ -0,0 +1,196 @@ +from typing import Optional, Tuple, Union +from uuid import UUID +from sql_registry.registry import Registry +from sql_registry.registry import connect +from sql_registry.registry.models import Edge, EntitiesAndRelations, Entity, EntityRef, EntityType, RelationshipType, _to_type, _to_uuid +import json + + +def quote(id): + if isinstance(id, str): + return f"'{id}'" + else: + return ",".join([f"'{i}'" for i in id]) + + +class DbRegistry(Registry): + def __init__(self): + self.conn = connect() + + def get_projects(self) -> list[str]: + ret = self.conn.query( + f"select qualified_name from entities where entity_type='{EntityType.Project}'") + return list([r["qualified_name"] for r in ret]) + + def get_entity(self, id_or_name: Union[str, UUID]) -> Entity: + return self._fill_entity(self._get_entity(id_or_name)) + + def get_entities(self, ids: list[UUID]) -> list[Entity]: + return list([self._fill_entity(e) for e in self._get_entities(ids)]) + + def get_entity_id(self, id_or_name: Union[str, UUID]) -> UUID: + try: + id = _to_uuid(id_or_name) + return id + except ValueError: + pass + # It is a name + ret = self.conn.query( + f"select entity_id from entities where qualified_name='{id_or_name}'") + return ret[0]["entity_id"] + + def get_neighbors(self, id_or_name: Union[str, UUID], relationship: RelationshipType) -> list[Edge]: + rows = self.conn.query(fr''' + select edge_id, from_id, to_id, conn_type + from edges + where from_id = '{self.get_entity_id(id_or_name)}' + and conn_type = '{relationship.name}' + ''') + return list([Edge(**row) for row in rows]) + + def get_lineage(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: + """ + Get feature lineage on both upstream and downstream + Returns [entity_id:entity] map and list of edges have been traversed. + """ + id = self.get_entity_id(id_or_name) + upstream_entities, upstream_edges = self._bfs( + id, RelationshipType.Consumes) + downstream_entities, downstream_edges = self._bfs( + id, RelationshipType.Produces) + return EntitiesAndRelations( + upstream_entities + downstream_entities, + upstream_edges + downstream_edges) + + def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: + """ + This function returns not only the project itself, but also everything in the project + """ + project = self._get_entity(id_or_name) + edges = set(self.get_neighbors(id_or_name, RelationshipType.Contains)) + ids = list([e.to_id for e in edges]) + children = self._get_entities(ids) + child_map = dict([(e.id, e) for e in children]) + project.attributes.children = children + for anchor in project.attributes.anchors: + conn = self.get_neighbors(anchor.id, RelationshipType.Contains) + feature_ids = [e.to_id for e in conn] + edges = edges.union(conn) + features = list([child_map[id] for id in feature_ids]) + anchor.attributes.features = features + source_id = self.get_neighbors( + anchor.id, RelationshipType.Consumes)[0].to_id + anchor.attributes.source = child_map[source_id] + for df in project.attributes.derived_features: + conn = self.get_neighbors(anchor.id, RelationshipType.Consumes) + input_ids = [e.to_id for e in conn] + edges = edges.union(conn) + features = list([child_map[id] for id in input_ids]) + df.attributes.input_features = features + all_edges = self._get_edges(ids) + return EntitiesAndRelations([project] + children, list(edges.union(all_edges))) + + def _fill_entity(self, e: Entity) -> Entity: + """ + Entities in the DB contains only attributes belong to itself, but the returned + data model contains connections/contents, so we need to fill this gap + """ + if e.entity_type == EntityType.Project: + edges = self.get_neighbors(e.id, RelationshipType.Contains) + ids = list([e.to_id for e in edges]) + children = self._get_entities(ids) + e.attributes.children = children + return e + if e.entity_type == EntityType.Anchor: + conn = self.get_neighbors(e.id, RelationshipType.Contains) + feature_ids = [e.to_id for e in conn] + features = self._get_entities(feature_ids) + e.attributes.features = features + source_id = self.get_neighbors( + e.id, RelationshipType.Consumes)[0].to_id + source = self.get_entity(source_id) + e.attributes.source = source + return e + if e.entity_type == EntityType.DerivedFeature: + conn = self.get_neighbors(e.id, RelationshipType.Consumes) + feature_ids = [e.to_id for e in conn] + features = self._get_entities(feature_ids) + e.attributes.input_features = features + return e + return e + + def _get_edges(self, ids: list[UUID], types: list[RelationshipType] = []) -> list[Edge]: + sql = fr"""select edge_id, from_id, to_id, conn_type from edges + where from_id in ({quote(ids)}) + and to_id in ({quote(ids)})""" + if len(types) > 0: + sql = fr"""select edge_id, from_id, to_id, conn_type from edges + where conn_type in ({quote(types)}) + and from_id in ({quote(ids)}) + and to_id in ({quote(ids)})""" + rows = self.conn.query(sql) + return list([_to_type(row, Edge) for row in rows]) + + def _get_entity(self, id_or_name: Union[str, UUID]) -> Entity: + row = self.conn.query(fr''' + select entity_id, qualified_name, entity_type, attributes + from entities + where entity_id = '{self.get_entity_id(id_or_name)}' + ''')[0] + row["attributes"] = json.loads(row["attributes"]) + return _to_type(row, Entity) + + def _get_entities(self, ids: list[UUID]) -> list[Entity]: + rows = self.conn.query(fr''' + select entity_id, qualified_name, entity_type, attributes + from entities + where entity_id in ({quote(ids)}) + ''') + ret = [] + for row in rows: + row["attributes"] = json.loads(row["attributes"]) + ret.append(Entity(**row)) + return ret + + def _bfs(self, id: UUID, conn_type: RelationshipType) -> Tuple[list[Entity], list[Edge]]: + """ + Breadth first traversal + Starts from `id`, follow edges with `conn_type` only. + + WARN: There is no depth limit. + """ + connections = [] + to_ids = [{ + "to_id": id, + }] + # BFS over SQL + while len(to_ids) != 0: + to_ids = self._bfs_step(to_ids, conn_type) + connections.extend(to_ids) + ids = set([id]) + for r in connections: + ids.add(r["from_id"]) + ids.add(r["to_id"]) + entities = self.get_entities(ids) + edges = list([Edge(**c) for c in connections]) + return (entities, edges) + + def _bfs_step(self, ids: list[UUID], conn_type: RelationshipType) -> set[dict]: + """ + One step of the BFS process + Returns all edges that connect to node ids the next step + """ + ids = list([id["to_id"] for id in ids]) + sql = fr"""select edge_id, from_id, to_id, conn_type from edges where conn_type = '{conn_type.name}' and from_id in ({quote(ids)})""" + return self.conn.query(sql) + + def search_entity(self, + keyword: str, + type: list[EntityType]) -> list[EntityRef]: + """ + WARN: This search function is implemented via `like` operator, which could be extremely slow. + """ + types = ",".join([quote(str(t)) for t in type]) + sql = fr'''select entity_id as id, qualified_name, entity_type as type from entities where qualified_name like %s and entity_type in ({types})''' + rows = self.conn.query(sql, ('%' + keyword + '%', )) + return list([EntityRef(**row) for row in rows]) diff --git a/registry/sql_registry/registry/interface.py b/registry/sql_registry/registry/interface.py new file mode 100644 index 000000000..cd0ece3a7 --- /dev/null +++ b/registry/sql_registry/registry/interface.py @@ -0,0 +1,69 @@ +from abc import ABC, abstractmethod +from typing import Union +from uuid import UUID +from sql_registry.registry.database import DbConnection + +from sql_registry.registry.models import * + + +class Registry(ABC): + @abstractmethod + def get_projects(self) -> list[str]: + """ + Returns the names of all projects + """ + pass + + @abstractmethod + def get_entity(self, id_or_name: Union[str, UUID]) -> Entity: + """ + Get one entity by its id or qualified name + """ + pass + + @abstractmethod + def get_entities(self, ids: list[UUID]) -> list[Entity]: + """ + Get list of entities by their ids + """ + pass + + @abstractmethod + def get_entity_id(self, id_or_name: Union[str, UUID]) -> UUID: + """ + Get entity id by its name + """ + pass + + @abstractmethod + def get_neighbors(self, id_or_name: Union[str, UUID], relationship: RelationshipType) -> list[Edge]: + """ + Get list of edges with specified type that connect to this entity. + The edge contains fromId and toId so we can follow to the entity it connects to + """ + pass + + @abstractmethod + def get_lineage(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: + """ + Get all the upstream and downstream entities of an entity, along with all edges connect them. + Only meaningful to features and data sources. + """ + pass + + @abstractmethod + def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: + """ + Get a project and everything inside of it, both entities and edges + """ + pass + + @abstractmethod + def search_entity(self, + keyword: str, + type: list[EntityType], + project: Optional[Union[str, UUID]] = None) -> list[EntityRef]: + """ + Search entities with specified type that also match the keyword in a project + """ + pass diff --git a/registry/sql_registry/registry/models.py b/registry/sql_registry/registry/models.py new file mode 100644 index 000000000..3c08d2692 --- /dev/null +++ b/registry/sql_registry/registry/models.py @@ -0,0 +1,728 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional, Union +from uuid import UUID +import json +import re + + +def _to_snake(d, level: int = 0): + """ + Convert `string`, `list[string]`, or all keys in a `dict` into snake case + The maximum length of input string or list is 100, or it will be truncated before being processed, for dict, the exception will be thrown if it has more than 100 keys. + the maximum nested level is 10, otherwise the exception will be thrown + """ + if level >= 10: + raise ValueError("Too many nested levels") + if isinstance(d, str): + d = d[:100] + return re.sub(r'([A-Z]\w+$)', r'_\1', d).lower() + if isinstance(d, list): + d = d[:100] + return [_to_snake(i, level + 1) if isinstance(i, (dict, list)) else i for i in d] + if len(d) > 100: + raise ValueError("Dict has too many keys") + return {_to_snake(a, level + 1): _to_snake(b, level + 1) if isinstance(b, (dict, list)) else b for a, b in d.items()} + + +def _to_type(value, type): + """ + Convert `value` into `type`, + or `list[type]` if `value` is a list + NOTE: This is **not** a generic implementation, only for objects in this module + """ + if isinstance(value, type): + return value + if isinstance(value, list): + return list([_to_type(v, type) for v in value]) + if isinstance(value, dict): + if hasattr(type, "new"): + try: + # The convention is to use `new` method to create the object from a dict + return type.new(**_to_snake(value)) + except TypeError: + pass + return type(**_to_snake(value)) + if issubclass(type, Enum): + try: + n = int(value) + return type(n) + except ValueError: + pass + if hasattr(type, "new"): + try: + # As well as Enum types, some of them have alias that cannot be handled by default Enum constructor + return type.new(value) + except KeyError: + pass + return type[value] + return type(value) + + +def _to_uuid(value): + return _to_type(value, UUID) + + +class ValueType(Enum): + UNSPECIFIED = 0 + BOOLEAN = 1 + INT = 2 + LONG = 3 + FLOAT = 4 + DOUBLE = 5 + STRING = 6 + BYTES = 7 + + +class VectorType(Enum): + TENSOR = 0 + + +class TensorCategory(Enum): + DENSE = 0 + SPARSE = 1 + + +class EntityType(Enum): + Project = 1 + Source = 2 + Anchor = 3 + AnchorFeature = 4 + DerivedFeature = 5 + + @staticmethod + def new(v): + return { + "feathr_workspace_v1": EntityType.Project, + "feathr_source_v1": EntityType.Source, + "feathr_anchor_v1": EntityType.Anchor, + "feathr_anchor_feature_v1": EntityType.AnchorFeature, + "feathr_derived_feature_v1": EntityType.DerivedFeature, + }[v] + + def __str__(self): + return { + EntityType.Project: "feathr_workspace_v1", + EntityType.Source: "feathr_source_v1", + EntityType.Anchor: "feathr_anchor_v1", + EntityType.AnchorFeature: "feathr_anchor_feature_v1", + EntityType.DerivedFeature: "feathr_derived_feature_v1", + }[self] + + +class RelationshipType(Enum): + Contains = 1 + BelongsTo = 2 + Consumes = 3 + Produces = 4 + + +class ToDict(ABC): + """ + This ABC is used to convert object to dict, then JSON. + """ + @abstractmethod + def to_dict(self) -> dict: + pass + + def to_json(self, indent=None) -> str: + return json.dumps(self.to_dict(), indent=indent) + + +class FeatureType(ToDict): + def __init__(self, + type: Union[str, VectorType], + tensor_category: Union[str, TensorCategory], + dimension_type: list[Union[str, ValueType]], + val_type: Union[str, ValueType]): + self.type = _to_type(type, VectorType) + self.tensor_category = _to_type(tensor_category, TensorCategory) + self.dimension_type = _to_type(dimension_type, ValueType) + self.val_type = _to_type(val_type, ValueType) + + def to_dict(self) -> dict: + return { + "type": self.type.name, + "tensorCategory": self.tensor_category.name, + "dimensionType": [t.name for t in self.dimension_type], + "valType": self.val_type.name, + } + + +class TypedKey(ToDict): + def __init__(self, + key_column: str, + key_column_type: ValueType, + full_name: Optional[str] = None, + description: Optional[str] = None, + key_column_alias: Optional[str] = None): + self.key_column = key_column + self.key_column_type = _to_type(key_column_type, ValueType) + self.full_name = full_name + self.description = description + self.key_column_alias = key_column_alias + + def to_dict(self) -> dict: + ret = { + "key_column": self.key_column, + "key_column_type": self.key_column_type.name, + } + if self.full_name is not None: + ret["full_name"] = self.full_name + if self.description is not None: + ret["description"] = self.full_name + if self.key_column_alias is not None: + ret["key_column_alias"] = self.key_column_alias + return ret + + +class Transformation(ToDict): + @staticmethod + def new(**kwargs): + if "transform_expr" in kwargs: + return ExpressionTransformation(**kwargs) + elif "def_expr" in kwargs: + return WindowAggregationTransformation(**kwargs) + elif "name" in kwargs: + return UdfTransformation(**kwargs) + else: + raise ValueError(kwargs) + + +class ExpressionTransformation(Transformation): + def __init__(self, transform_expr: str): + self.transform_expr = transform_expr + + def to_dict(self) -> dict: + return { + "transform_expr": self.transform_expr + } + + +class WindowAggregationTransformation(Transformation): + def __init__(self, + def_expr: str, + agg_func: Optional[str] = None, + window: Optional[str] = None, + group_by: Optional[str] = None, + filter: Optional[str] = None, + limit: Optional[int] = None): + self.def_expr = def_expr + self.agg_func = agg_func + self.window = window + self.group_by = group_by + self.filter = filter + self.limit = limit + + def to_dict(self) -> dict: + ret = { + "def_expr": self.def_expr, + } + if self.agg_func is not None: + ret["agg_func"] = self.agg_func + if self.window is not None: + ret["window"] = self.window + if self.group_by is not None: + ret["group_by"] = self.group_by + if self.filter is not None: + ret["filter"] = self.filter + if self.limit is not None: + ret["limit"] = self.limit + return ret + + +class UdfTransformation(Transformation): + def __init__(self, name: str): + self.name = name + + def to_dict(self) -> dict: + return { + "name": self.name + } + + +class EntityRef(ToDict): + def __init__(self, + id: UUID, + type: Union[str, EntityType], + qualified_name: Optional[str] = None, + uniq_attr: dict = {}): + self.id = id + self.type = _to_type(type, EntityType) + if qualified_name is not None: + self.uniq_attr = {"qualifiedName": qualified_name} + else: + self.uniq_attr = uniq_attr + + @property + def entity_type(self) -> EntityType: + return self.type + + @property + def qualified_name(self) -> EntityType: + return self.uniq_attr['qualifiedName'] + + def get_ref(self): + return self + + def to_dict(self) -> dict: + return { + "guid": str(self.id), + "typeName": str(self.type), + "uniqueAttributes": self.uniq_attr, + } + + +class Attributes(ToDict): + @staticmethod + def new(entity_type: Union[str, EntityType], **kwargs): + return { + EntityType.Project: ProjectAttributes, + EntityType.Source: SourceAttributes, + EntityType.Anchor: AnchorAttributes, + EntityType.AnchorFeature: AnchorFeatureAttributes, + EntityType.DerivedFeature: DerivedFeatureAttributes, + }[_to_type(entity_type, EntityType)](**kwargs) + + +class Entity(ToDict): + def __init__(self, + entity_id: Union[str, UUID], + qualified_name: str, + entity_type: Union[str, EntityType], + attributes: Union[dict, Attributes], + **kwargs): + self.id = _to_uuid(entity_id) + self.qualified_name = qualified_name + self.entity_type = _to_type(entity_type, EntityType) + if isinstance(attributes, Attributes): + self.attributes = attributes + else: + self.attributes = Attributes.new( + entity_type, **_to_snake(attributes)) + + def get_ref(self) -> EntityRef: + return EntityRef(self.id, + self.attributes.entity_type, + self.qualified_name) + + def to_dict(self) -> dict: + return { + "guid": str(self.id), + "lastModifiedTS": "1", + "status": "ACTIVE", + "displayText": self.attributes.name, + "typeName": str(self.attributes.entity_type), + "attributes": self.attributes.to_dict(), + } + + +class ProjectAttributes(Attributes): + def __init__(self, + name: str, + children: list[Union[dict, Entity]] = [], + tags: dict = {}, + **kwargs): + self.name = name + self.tags = tags + self._children = [] + if len(children) > 0: + self.children = children + + @property + def entity_type(self) -> EntityType: + return EntityType.Project + + @property + def children(self): + return self._children + + @children.setter + def children(self, v: list[Union[dict, Entity]]): + for f in v: + if isinstance(f, Entity): + self._children.append(f) + elif isinstance(f, dict): + self._children.append(_to_type(f, Entity)) + else: + raise TypeError(f) + + @property + def sources(self): + return [ + e for e in self.children if e.entity_type == EntityType.Source] + + @property + def anchors(self): + return [ + e for e in self.children if e.entity_type == EntityType.Anchor] + + @property + def anchor_features(self): + return [ + e for e in self.children if e.entity_type == EntityType.AnchorFeature] + + @property + def derived_features(self): + return [ + e for e in self.children if e.entity_type == EntityType.DerivedFeature] + + def to_dict(self) -> dict: + return { + "qualifiedName": self.name, + "name": self.name, + "sources": list([e.get_ref().to_dict() for e in self.sources]), + "anchors": list([e.get_ref().to_dict() for e in self.anchors]), + "anchor_features": list([e.get_ref().to_dict() for e in self.anchor_features]), + "derived_features": list([e.get_ref().to_dict() for e in self.derived_features]), + "tags": self.tags, + } + + +class SourceAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + type: str, + path: str, + preprocessing: Optional[str] = None, + event_timestamp_column: Optional[str] = None, + timestamp_format: Optional[str] = None, + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.type = type + self.path = path + self.preprocessing = preprocessing + self.event_timestamp_column = event_timestamp_column + self.timestamp_format = timestamp_format + self.tags = tags + + @property + def entity_type(self) -> EntityType: + return EntityType.Source + + def to_dict(self) -> dict: + ret = { + "qualifiedName": self.qualified_name, + "name": self.name, + "type": self.type, + "path": self.path, + "tags": self.tags, + } + if self.preprocessing is not None: + ret["preprocessing"] = self.preprocessing + if self.event_timestamp_column is not None: + ret["eventTimestampColumn"] = self.event_timestamp_column + if self.timestamp_format is not None: + ret["timestampFormat"] = self.timestamp_format + return ret + + +class AnchorAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + # source: Optional[Union[dict, EntityRef, Entity]] = None, + # features: list[Union[dict, EntityRef, Entity]] = [], + tags: dict = {}, + **kwargs): + self.qualified_name = qualified_name + self.name = name + self._source = None + self._features = [] + # if source is not None: + # self._source = source.get_ref() + # if len(features)>0: + # self._set_feature(features) + self.tags = tags + + @property + def entity_type(self) -> EntityType: + return EntityType.Anchor + + @property + def source(self) -> EntityRef: + return self._source + + @source.setter + def source(self, s): + if isinstance(s, Entity): + self._source = s.get_ref() + elif isinstance(s, EntityRef): + self._source = s + elif isinstance(s, dict): + self._source = _to_type(s, Entity).get_ref() + else: + raise TypeError(s) + + @property + def features(self): + return self._features + + @features.setter + def features(self, features): + self._features = [] + for f in features: + if isinstance(f, Entity): + self._features.append(f.get_ref()) + elif isinstance(f, EntityRef): + self._features.append(f) + elif isinstance(f, dict): + self._features.append(_to_type(f, Entity).get_ref()) + else: + raise TypeError(f) + + def to_dict(self) -> dict: + ret = { + "qualifiedName": self.qualified_name, + "name": self.name, + "features": list([e.get_ref().to_dict() for e in self.features]), + "tags": self.tags, + } + if self.source is not None: + ret["source"] = self.source.get_ref().to_dict() + return ret + + +class AnchorFeatureAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.type = _to_type(type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self.tags = tags + + @property + def entity_type(self) -> EntityType: + return EntityType.AnchorFeature + + def to_dict(self) -> dict: + return { + "qualifiedName": self.qualified_name, + "name": self.name, + "type": self.type.to_dict(), + "transformation": self.transformation.to_dict(), + "key": list([k.to_dict() for k in self.key]), + "tags": self.tags, + } + + +class DerivedFeatureAttributes(Attributes): + def __init__(self, + qualified_name: str, + name: str, + type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + # input_anchor_features: list[Union[dict, EntityRef, Entity]] = [], + # input_derived_features: list[Union[dict, EntityRef, Entity]] = [], + tags: dict = {}, + **kwargs): + self.qualified_name = qualified_name + self.name = name + self.type = _to_type(type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self._input_anchor_features = [] + self._input_derived_features = [] + self.tags = tags + # self._set_input_anchor_features(input_anchor_features) + # self._set_input_derived_features(input_derived_features) + + @property + def entity_type(self) -> EntityType: + return EntityType.DerivedFeature + + @property + def input_features(self): + return self._input_anchor_features + self._input_derived_features + + @input_features.setter + def input_features(self, v: Union[dict, Entity]): + self._input_anchor_features = [] + self._input_derived_features = [] + for f in v: + e = None + if isinstance(f, Entity): + e = f + elif isinstance(f, dict): + e = _to_type(f, Entity) + else: + raise TypeError(f) + + if e.entity_type == EntityType.AnchorFeature: + self._input_anchor_features.append(e) + elif e.entity_type == EntityType.DerivedFeature: + self._input_derived_features.append(e) + else: + pass + + @property + def input_anchor_features(self): + return self._input_anchor_features + + # @input_anchor_features.setter + # def input_anchor_features(self, v): + # self._input_anchor_features = [] + # for f in v: + # if isinstance(f, Entity): + # self._input_anchor_features.append(f.get_ref()) + # elif isinstance(f, EntityRef): + # self._input_anchor_features.append(f) + # elif isinstance(f, dict): + # self._input_anchor_features.append( + # to_type(f, Entity).get_ref()) + # else: + # raise TypeError(f) + + @property + def input_derived_features(self): + return self._input_derived_features + + # @input_derived_features.setter + # def input_derived_features(self, v): + # self._input_derived_features = [] + # for f in v: + # if isinstance(f, Entity): + # self._input_derived_features.append(f.get_ref()) + # elif isinstance(f, EntityRef): + # self._input_derived_features.append(f) + # elif isinstance(f, dict): + # self._input_derived_features.append( + # to_type(f, Entity).get_ref()) + # else: + # raise TypeError(f) + + def to_dict(self) -> dict: + return { + "qualifiedName": self.qualified_name, + "name": self.name, + "type": self.type.to_dict(), + "transformation": self.transformation.to_dict(), + "key": list([k.to_dict() for k in self.key]), + "input_anchor_features": [e.get_ref().to_dict() for e in self.input_anchor_features], + "input_derived_features": [e.get_ref().to_dict() for e in self.input_derived_features], + "tags": self.tags, + } + + +class Edge(ToDict): + def __init__(self, + edge_id: Union[str, UUID], + from_id: Union[str, UUID], + to_id: Union[str, UUID], + conn_type: Union[str, RelationshipType]): + self.id = _to_uuid(edge_id) + self.from_id = _to_uuid(from_id) + self.to_id = _to_uuid(to_id) + self.conn_type = _to_type(conn_type, RelationshipType) + + def __eq__(self, o: object) -> bool: + # Edge ID is kinda useless + return self.from_id == o.from_id and self.to_id == o.to_id and self.conn_type == o.conn_type + + def __hash__(self) -> int: + return hash((self.from_id, self.to_id, self.conn_type)) + + def to_dict(self) -> dict: + return { + "relationshipId": str(self.id), + "fromEntityId": str(self.from_id), + "toEntityId": str(self.to_id), + "relationshipType": self.conn_type.name, + } + + +class EntitiesAndRelations(ToDict): + def __init__(self, entities: list[Entity], edges: list[Edge]): + self.entities = dict([(e.id, e) for e in entities]) + self.edges = set(edges) + + def to_dict(self) -> dict: + return { + "guidEntityMap": dict([(str(id), self.entities[id].to_dict()) for id in self.entities]), + "relations": list([e.to_dict() for e in self.edges]), + } + + +class ProjectDef: + def __init__(self, qualified_name: str, tags: dict = {}): + self.qualified_name = qualified_name + self.name = qualified_name + self.tags = tags + + +class SourceDef: + def __init__(self, + qualified_name: str, + name: str, + path: str, + type: str, + preprocessing: Optional[str] = None, + event_timestamp_column: Optional[str] = None, + timestamp_format: Optional[str] = None, + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.path = path + self.type = type + self.preprocessing = preprocessing + self.event_timestamp_column = event_timestamp_column + self.timestamp_format = timestamp_format + self.tags = tags + + +class AnchorDef: + def __init__(self, + qualified_name: str, + name: str, + source_id: Union[str, UUID], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.source_id = _to_uuid(source_id) + self.tags = tags + + +class AnchorFeatureDef: + def __init__(self, + qualified_name: str, + name: str, + feature_type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.feature_type = _to_type(feature_type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self.tags = tags + + +class DerivedFeatureDef: + def __init__(self, + qualified_name: str, + name: str, + feature_type: Union[dict, FeatureType], + transformation: Union[dict, Transformation], + key: list[Union[dict, TypedKey]], + input_anchor_features: list[Union[str, UUID]], + input_derived_features: list[Union[str, UUID]], + tags: dict = {}): + self.qualified_name = qualified_name + self.name = name + self.feature_type = _to_type(feature_type, FeatureType) + self.transformation = _to_type(transformation, Transformation) + self.key = _to_type(key, TypedKey) + self.input_anchor_features = _to_uuid(input_anchor_features) + self.input_derived_features = _to_uuid(input_derived_features) + self.tags = tags diff --git a/registry/sql_registry/requirements.txt b/registry/sql_registry/requirements.txt new file mode 100644 index 000000000..c6d61de98 --- /dev/null +++ b/registry/sql_registry/requirements.txt @@ -0,0 +1,3 @@ +pymssql +fastapi +uvicorn \ No newline at end of file diff --git a/registry/sql_registry/scripts/schema.sql b/registry/sql_registry/scripts/schema.sql new file mode 100644 index 000000000..e94b83dfc --- /dev/null +++ b/registry/sql_registry/scripts/schema.sql @@ -0,0 +1,41 @@ +create table entities +( + entity_id varchar(50) not null primary key, + qualified_name varchar(200) not null, + entity_type varchar(100) not null, + attributes NVARCHAR(MAX) not null, +) + +create table edges +( + edge_id varchar(50) not null primary key, + from_id varchar(50) not null, + to_id varchar(50) not null, + conn_type varchar(20) not null, +) + +create table userroles +( + record_id int IDENTITY(1,1), + project_name varchar(50) not null, + user_name varchar(50) not null, + role_name varchar(50) not null, + create_reason varchar(50) not null, + create_time datetime not null, + delete_reason varchar(50), + delete_time datetime, +) + +create table access +( + record_id int IDENTITY(1,1), + project_name varchar(50) not null, + access_name varchar(50) not null, +) + +create table roleaccess +( + record_id int IDENTITY(1,1), + role_name varchar(50) not null, + access_name varchar(50) not null, +) \ No newline at end of file diff --git a/registry/sql_registry/scripts/test_data.sql b/registry/sql_registry/scripts/test_data.sql new file mode 100644 index 000000000..185f3d752 --- /dev/null +++ b/registry/sql_registry/scripts/test_data.sql @@ -0,0 +1,111 @@ +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'feathr_ci_registry_12_33_182947__PASSTHROUGH', 'feathr_source_v1', '{"path": "PASSTHROUGH", "qualifiedName": "feathr_ci_registry_12_33_182947__PASSTHROUGH", "name": "PASSTHROUGH", "type": "PASSTHROUGH"}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('dc24b1d5-206d-40db-b10a-606dd16a0297', 'feathr_ci_registry_12_33_182947__request_features__f_is_long_trip_distance', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_is_long_trip_distance", "name": "f_is_long_trip_distance", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "BOOLEAN"}, "transformation": {"transform_expr": "cast_float(trip_distance)>30"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('c626c41c-d6c2-4b16-a267-6cdeea497c52', 'feathr_ci_registry_12_33_182947__f_trip_time_rounded', 'feathr_derived_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded", "name": "f_trip_time_rounded", "input_derived_features": [], "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "f_trip_time_duration % 10"}, "input_anchor_features": [{"guid": "103baca1-377a-4ddf-8429-5da91026c269", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration"}}], "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('537bc481-aa15-4a3b-be4e-2042da6f5a09', 'feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_max_fare', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_max_fare", "name": "f_location_max_fare", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"filter": null, "agg_func": "MAX", "limit": null, "group_by": null, "window": "90d", "def_expr": "cast_float(fare_amount)"}, "key": [{"full_name": "nyc_taxi.location_id", "key_column": "DOLocationID", "description": "location id in NYC", "key_column_alias": "DOLocationID", "key_column_type": "2"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('479c6306-5fdb-4e06-9008-c18f68db52a4', 'feathr_ci_registry_12_33_182947__f_trip_time_rounded_plus', 'feathr_derived_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded_plus", "name": "f_trip_time_rounded_plus", "input_derived_features": [{"guid": "c626c41c-d6c2-4b16-a267-6cdeea497c52", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded"}}], "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "f_trip_time_rounded + 100"}, "input_anchor_features": [], "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'feathr_ci_registry_12_33_182947__nycTaxiBatchSource', 'feathr_source_v1', '{"timestamp_format": "yyyy-MM-dd HH:mm:ss", "path": "wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", "event_timestamp_column": "lpep_dropoff_datetime", "preprocessing": " def add_new_dropoff_and_fare_amount_column(df: DataFrame):\n df = df.withColumn(\"new_lpep_dropoff_datetime\", col(\"lpep_dropoff_datetime\"))\n df = df.withColumn(\"new_fare_amount\", col(\"fare_amount\") + 1000000)\n return df\n", "qualifiedName": "feathr_ci_registry_12_33_182947__nycTaxiBatchSource", "name": "nycTaxiBatchSource", "type": "wasbs", "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_avg_fare', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_avg_fare", "name": "f_location_avg_fare", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"filter": null, "agg_func": "AVG", "limit": null, "group_by": null, "window": "90d", "def_expr": "cast_float(fare_amount)"}, "key": [{"full_name": "nyc_taxi.location_id", "key_column": "DOLocationID", "description": "location id in NYC", "key_column_alias": "DOLocationID", "key_column_type": "2"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('cd7306a7-c458-45e8-a00b-44a2f2117135', 'feathr_ci_registry_12_33_182947', 'feathr_workspace_v1', '{"anchor_features": [{"guid": "a5c47bd8-3729-45fa-8701-b8b76ada150a", "typeName": "feathr_anchor_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures"}}, {"guid": "260325a5-27f9-40d1-8697-c727feb1dbdc", "typeName": "feathr_anchor_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features"}}], "derived_features": [{"guid": "226b42ee-0c34-4329-b935-744aecc63fb4", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_distance"}}, {"guid": "c626c41c-d6c2-4b16-a267-6cdeea497c52", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded"}}, {"guid": "479c6306-5fdb-4e06-9008-c18f68db52a4", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded_plus"}}], "qualifiedName": "feathr_ci_registry_12_33_182947", "name": "feathr_ci_registry_12_33_182947", "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'feathr_ci_registry_12_33_182947__request_features__f_trip_distance', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_distance", "name": "f_trip_distance", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"transform_expr": "trip_distance"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}], "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('103baca1-377a-4ddf-8429-5da91026c269', 'feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration", "name": "f_trip_time_duration", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "(to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime))/60"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('a5c47bd8-3729-45fa-8701-b8b76ada150a', 'feathr_ci_registry_12_33_182947__aggregationFeatures', 'feathr_anchor_v1', '{"features": [{"guid": "2a052ccd-3e31-46a7-bffb-2ab1302b1b00", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_avg_fare"}}, {"guid": "537bc481-aa15-4a3b-be4e-2042da6f5a09", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_max_fare"}}], "qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures", "name": "aggregationFeatures", "source": {"guid": "c4a0ae0f-09cc-43bf-94e9-21ff178fbda6", "typeName": "feathr_source_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__nycTaxiBatchSource"}}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('260325a5-27f9-40d1-8697-c727feb1dbdc', 'feathr_ci_registry_12_33_182947__request_features', 'feathr_anchor_v1', '{"features": [{"guid": "5316c516-77f9-4be4-a7ec-8bf6e893e2aa", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_distance"}}, {"guid": "103baca1-377a-4ddf-8429-5da91026c269", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration"}}, {"guid": "dc24b1d5-206d-40db-b10a-606dd16a0297", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_is_long_trip_distance"}}, {"guid": "2380fe5b-ce2a-401e-98bf-af8b98460f67", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_day_of_week"}}], "qualifiedName": "feathr_ci_registry_12_33_182947__request_features", "name": "request_features", "source": {"guid": "a4cfbc03-c65d-4f32-be3d-1d11247c9cdd", "typeName": "feathr_source_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__PASSTHROUGH"}}, "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('2380fe5b-ce2a-401e-98bf-af8b98460f67', 'feathr_ci_registry_12_33_182947__request_features__f_day_of_week', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_day_of_week", "name": "f_day_of_week", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "dayofweek(lpep_dropoff_datetime)"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('226b42ee-0c34-4329-b935-744aecc63fb4', 'feathr_ci_registry_12_33_182947__f_trip_time_distance', 'feathr_derived_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_distance", "name": "f_trip_time_distance", "input_derived_features": [], "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"transform_expr": "f_trip_distance * f_trip_time_duration"}, "input_anchor_features": [{"guid": "5316c516-77f9-4be4-a7ec-8bf6e893e2aa", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_distance"}}, {"guid": "103baca1-377a-4ddf-8429-5da91026c269", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration"}}], "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); + +insert into edges (edge_id, from_id, to_id, conn_type) values ('455f7195-8463-4c60-9cf0-65bd9db0ae0a', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a2777fd2-1136-40d0-8686-47b5b5fed1ef', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('ca88290d-03d1-4641-bf36-cbf3280b3e9d', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('1da26ec9-6608-4971-ac04-8ed170543325', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('9ae3a7c0-0163-4170-b0cf-81705e5b6aca', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('50068cf8-7f5e-482a-a018-68bf27f89f6d', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('898a10fd-0315-4fcb-9803-0144047033c7', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('08540367-3d1f-4d57-8af1-36b3a11762ed', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('d05ca3c6-2610-4352-9be7-c4d8dd6ab6b6', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('ed6ca745-ee85-403c-b1bf-a3f1a0463132', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('55d85311-8e42-4273-a538-ef4dc33b1570', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('215cdc37-678a-4c56-a390-76d6471fa629', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('4166aca1-e0f4-4883-b2ff-0051ee80a830', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e1174ff1-0b1a-4b03-a1f5-b8b923a131ba', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b314c95b-e4ea-4a39-9584-9b7eb0a6b8d2', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('7a37d438-8c33-4cc8-a0bb-9db3963f073a', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('31265953-4820-470f-8cfc-38efefec9fa7', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('58c448cf-87bd-4f36-92d5-e6f6de48569d', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c86e4e49-88c9-4ac8-a1b8-ff473e1dc588', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '103baca1-377a-4ddf-8429-5da91026c269', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('3fddedcf-c590-43b9-aaf4-4c4ce6600f2e', '103baca1-377a-4ddf-8429-5da91026c269', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('be429eb2-758d-4783-b166-cfcc7d2fb4f2', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('bd66f19f-3508-4b13-ba40-598dd4abbd0d', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('178c7c56-0f25-4048-be18-435ab5a169f4', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c8dc4c34-7950-4a04-a4bd-829a4c20ab4e', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('92f143c0-59a8-461b-b9cc-179f3403dd38', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e9a6f066-f4cc-4de4-bb26-2bc9522760a4', '226b42ee-0c34-4329-b935-744aecc63fb4', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('805b2c95-fc22-48b1-81cc-f86e3f2c2956', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e54835e6-b399-4d49-8ab8-6b452cbc00ca', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('02a774bb-0a1c-4874-b9e2-bbd4806c2f3e', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('17c74f58-18bc-4c4a-974b-2732a84576c3', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c8dded57-22f8-46d0-a29c-3e7c072ebee4', '260325a5-27f9-40d1-8697-c727feb1dbdc', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('7e2995b8-2a4f-49c1-b72c-80c8c6d1e060', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('3ba9b460-5878-43c1-bb00-c88916bf5e6b', '260325a5-27f9-40d1-8697-c727feb1dbdc', '103baca1-377a-4ddf-8429-5da91026c269', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a4bea335-611d-49cc-9e94-b18896103d40', '103baca1-377a-4ddf-8429-5da91026c269', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('f0d65e5b-9762-4c65-9333-83aa6c1beb75', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('cd72ee26-b867-4321-8566-7daa488f7a61', 'dc24b1d5-206d-40db-b10a-606dd16a0297', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('bc211906-6d48-4649-8e49-518cd89c61f8', '260325a5-27f9-40d1-8697-c727feb1dbdc', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('aeafba5d-b352-4c36-8fd5-d135cea70605', '2380fe5b-ce2a-401e-98bf-af8b98460f67', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('2b6126be-63fd-4140-8891-e0b5db880573', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('89a348be-ae8d-40ad-a1cd-12d00b981b2a', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('7e6219a9-e433-4706-9145-f791a58ef7c3', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a447062c-c75e-48b2-bbe3-5d9eab770c26', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('faf70a4b-2358-4881-952b-aae3cec55053', '103baca1-377a-4ddf-8429-5da91026c269', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b6647b84-5043-4ac2-8b31-3db7c2d6cf32', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '103baca1-377a-4ddf-8429-5da91026c269', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b63cb71d-71c1-49a1-93bd-ea63cb4ab4e7', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c582313e-9840-4ae6-827c-d2a01363ee6b', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a2d2ca10-11eb-4e6f-9a5c-b8196a56d10a', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('50701b58-58ef-4b6e-ac00-5f69d8ceebf6', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('6d010032-02fb-4e94-aef7-85e56d5cf99c', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('421b8c1f-01b9-49c3-8820-ee05313d103a', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b6b8af3a-2531-46bc-9b1e-acf3a4c51396', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('323d044f-bbbd-45fc-a93c-40ee0f17ab87', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('cbe5c0d4-621b-4e6b-a6c1-76c8ca6105f2', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('bedbe18b-a7c3-40d3-b230-22f9ac5c6c76', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('84902105-164c-4fc3-9690-e638f16c3075', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', '103baca1-377a-4ddf-8429-5da91026c269', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('44668672-1520-4371-98d2-3ed48bddf9ea', '103baca1-377a-4ddf-8429-5da91026c269', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e8fe5609-53e7-4793-a39d-2ce75c630cd9', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('591285ff-53dd-4e27-a30d-754fe97ea3be', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('df901204-59c1-4f3e-87f0-8ac07e90bc39', '226b42ee-0c34-4329-b935-744aecc63fb4', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('23a05c64-b204-4c48-b906-9fabb6ea298b', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('3721c600-6423-4a38-a9b2-d85b426b6eaa', '226b42ee-0c34-4329-b935-744aecc63fb4', '103baca1-377a-4ddf-8429-5da91026c269', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('5f9b86fe-bdc9-4a76-b07c-876b9d7c1ee1', '103baca1-377a-4ddf-8429-5da91026c269', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Produces'); + +insert into userroles (project_name, user_name, role_name, create_reason, create_time) values ('global', 'abc@microsoft.com','admin', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_reason, create_time) values ('global', 'efg@microsoft.com','admin', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_reason, create_time) values ('feathr_ci_registry_12_33_182947', 'efg@microsoft.com','admin', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_reason, create_time) values ('feathr_ci_registry_12_33_182947', 'hij@microsoft.com','consumer', 'test data', getutcdate()) + +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'read') +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'write') +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'manage') +insert into access (project_name, access_name) values ('global', 'read') +insert into access (project_name, access_name) values ('global', 'write') +insert into access (project_name, access_name) values ('global', 'manage') + +insert into roleaccess (role_name, access_name) values ('admin', 'read') +insert into roleaccess (role_name, access_name) values ('admin', 'write') +insert into roleaccess (role_name, access_name) values ('admin', 'manage') +insert into roleaccess (role_name, access_name) values ('contributor', 'read') +insert into roleaccess (role_name, access_name) values ('contributor', 'write') +insert into roleaccess (role_name, access_name) values ('viewer', 'read') +insert into roleaccess (role_name, access_name) values ('default', '') \ No newline at end of file diff --git a/registry/sql_registry/test/test_basic.py b/registry/sql_registry/test/test_basic.py new file mode 100644 index 000000000..22e343aba --- /dev/null +++ b/registry/sql_registry/test/test_basic.py @@ -0,0 +1,17 @@ +import registry +r=registry.DbRegistry() + +l=r.get_lineage('226b42ee-0c34-4329-b935-744aecc63fb4').to_dict() +assert(len(l["guidEntityMap"]) == 4) + +af1=r.get_entity('2380fe5b-ce2a-401e-98bf-af8b98460f67') +af2=r.get_entity('feathr_ci_registry_12_33_182947__request_features__f_day_of_week') +assert(af1.to_dict()==af2.to_dict()) +df1=r.get_entity('226b42ee-0c34-4329-b935-744aecc63fb4') +df2=r.get_entity('feathr_ci_registry_12_33_182947__f_trip_time_distance') +assert(df1.to_dict()==df2.to_dict()) + +p=r.get_project('feathr_ci_registry_12_33_182947') +assert(len(p.to_dict()['guidEntityMap'])==14) + +es=r.search_entity("time", [registry.EntityType.DerivedFeature]) diff --git a/registry/sql_schema/access-control-schema.sql b/registry/sql_schema/access-control-schema.sql new file mode 100644 index 000000000..ddda87dd6 --- /dev/null +++ b/registry/sql_schema/access-control-schema.sql @@ -0,0 +1,27 @@ +create table userroles +( + record_id int IDENTITY(1,1), + project_name varchar(50) not null, + user_name varchar(50) not null, + role_name varchar(50) not null, + create_by varchar(50) not null, + create_reason varchar(50) not null, + create_time datetime not null, + delete_by varchar(50), + delete_reason varchar(50), + delete_time datetime, +) + +create table access +( + record_id uniqueidentifier NOT NULL DEFAULT newid(), + project_name varchar(50) not null, + access_name varchar(50) not null, +) + +create table roleaccess +( + record_id uniqueidentifier NOT NULL DEFAULT newid(), + role_name varchar(50) not null, + access_name varchar(50) not null, +) \ No newline at end of file diff --git a/registry/sql_schema/access-control-test-data.sql b/registry/sql_schema/access-control-test-data.sql new file mode 100644 index 000000000..a63a7fdd3 --- /dev/null +++ b/registry/sql_schema/access-control-test-data.sql @@ -0,0 +1,19 @@ +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('global', 'abc@microsoft.com','admin', 'test_data@microsoft.com', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('global', 'efg@microsoft.com','admin', 'test_data@microsoft.com', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('feathr_ci_registry_12_33_182947', 'efg@microsoft.com','admin','test_data@microsoft.com', 'test data', getutcdate()) +insert into userroles (project_name, user_name, role_name, create_by, create_reason, create_time) values ('feathr_ci_registry_12_33_182947', 'hij@microsoft.com','consumer', 'test_data@microsoft.com', 'test data', getutcdate()) + +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'read') +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'write') +insert into access (project_name, access_name) values ('feathr_ci_registry_12_33_182947', 'manage') +insert into access (project_name, access_name) values ('global', 'read') +insert into access (project_name, access_name) values ('global', 'write') +insert into access (project_name, access_name) values ('global', 'manage') + +insert into roleaccess (role_name, access_name) values ('admin', 'read') +insert into roleaccess (role_name, access_name) values ('admin', 'write') +insert into roleaccess (role_name, access_name) values ('admin', 'manage') +insert into roleaccess (role_name, access_name) values ('contributor', 'read') +insert into roleaccess (role_name, access_name) values ('contributor', 'write') +insert into roleaccess (role_name, access_name) values ('viewer', 'read') +insert into roleaccess (role_name, access_name) values ('default', '') \ No newline at end of file diff --git a/registry/sql_schema/registry-schema.sql b/registry/sql_schema/registry-schema.sql new file mode 100644 index 000000000..d7258d577 --- /dev/null +++ b/registry/sql_schema/registry-schema.sql @@ -0,0 +1,15 @@ +create table entities +( + entity_id varchar(50) not null primary key, + qualified_name varchar(200) not null, + entity_type varchar(100) not null, + attributes NVARCHAR(MAX) not null, +) + +create table edges +( + edge_id varchar(50) not null primary key, + from_id varchar(50) not null, + to_id varchar(50) not null, + conn_type varchar(20) not null, +) \ No newline at end of file diff --git a/registry/sql_schema/registry-test-data.sql b/registry/sql_schema/registry-test-data.sql new file mode 100644 index 000000000..9b3894e94 --- /dev/null +++ b/registry/sql_schema/registry-test-data.sql @@ -0,0 +1,91 @@ +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'feathr_ci_registry_12_33_182947__PASSTHROUGH', 'feathr_source_v1', '{"path": "PASSTHROUGH", "qualifiedName": "feathr_ci_registry_12_33_182947__PASSTHROUGH", "name": "PASSTHROUGH", "type": "PASSTHROUGH"}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('dc24b1d5-206d-40db-b10a-606dd16a0297', 'feathr_ci_registry_12_33_182947__request_features__f_is_long_trip_distance', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_is_long_trip_distance", "name": "f_is_long_trip_distance", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "BOOLEAN"}, "transformation": {"transform_expr": "cast_float(trip_distance)>30"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('c626c41c-d6c2-4b16-a267-6cdeea497c52', 'feathr_ci_registry_12_33_182947__f_trip_time_rounded', 'feathr_derived_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded", "name": "f_trip_time_rounded", "input_derived_features": [], "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "f_trip_time_duration % 10"}, "input_anchor_features": [{"guid": "103baca1-377a-4ddf-8429-5da91026c269", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration"}}], "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('537bc481-aa15-4a3b-be4e-2042da6f5a09', 'feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_max_fare', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_max_fare", "name": "f_location_max_fare", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"filter": null, "agg_func": "MAX", "limit": null, "group_by": null, "window": "90d", "def_expr": "cast_float(fare_amount)"}, "key": [{"full_name": "nyc_taxi.location_id", "key_column": "DOLocationID", "description": "location id in NYC", "key_column_alias": "DOLocationID", "key_column_type": "2"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('479c6306-5fdb-4e06-9008-c18f68db52a4', 'feathr_ci_registry_12_33_182947__f_trip_time_rounded_plus', 'feathr_derived_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded_plus", "name": "f_trip_time_rounded_plus", "input_derived_features": [{"guid": "c626c41c-d6c2-4b16-a267-6cdeea497c52", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded"}}], "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "f_trip_time_rounded + 100"}, "input_anchor_features": [], "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'feathr_ci_registry_12_33_182947__nycTaxiBatchSource', 'feathr_source_v1', '{"timestamp_format": "yyyy-MM-dd HH:mm:ss", "path": "wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", "event_timestamp_column": "lpep_dropoff_datetime", "preprocessing": " def add_new_dropoff_and_fare_amount_column(df: DataFrame):\n df = df.withColumn(\"new_lpep_dropoff_datetime\", col(\"lpep_dropoff_datetime\"))\n df = df.withColumn(\"new_fare_amount\", col(\"fare_amount\") + 1000000)\n return df\n", "qualifiedName": "feathr_ci_registry_12_33_182947__nycTaxiBatchSource", "name": "nycTaxiBatchSource", "type": "wasbs", "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_avg_fare', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_avg_fare", "name": "f_location_avg_fare", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"filter": null, "agg_func": "AVG", "limit": null, "group_by": null, "window": "90d", "def_expr": "cast_float(fare_amount)"}, "key": [{"full_name": "nyc_taxi.location_id", "key_column": "DOLocationID", "description": "location id in NYC", "key_column_alias": "DOLocationID", "key_column_type": "2"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('cd7306a7-c458-45e8-a00b-44a2f2117135', 'feathr_ci_registry_12_33_182947', 'feathr_workspace_v1', '{"anchor_features": [{"guid": "a5c47bd8-3729-45fa-8701-b8b76ada150a", "typeName": "feathr_anchor_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures"}}, {"guid": "260325a5-27f9-40d1-8697-c727feb1dbdc", "typeName": "feathr_anchor_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features"}}], "derived_features": [{"guid": "226b42ee-0c34-4329-b935-744aecc63fb4", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_distance"}}, {"guid": "c626c41c-d6c2-4b16-a267-6cdeea497c52", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded"}}, {"guid": "479c6306-5fdb-4e06-9008-c18f68db52a4", "typeName": "feathr_derived_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_rounded_plus"}}], "qualifiedName": "feathr_ci_registry_12_33_182947", "name": "feathr_ci_registry_12_33_182947", "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'feathr_ci_registry_12_33_182947__request_features__f_trip_distance', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_distance", "name": "f_trip_distance", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"transform_expr": "trip_distance"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}], "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('103baca1-377a-4ddf-8429-5da91026c269', 'feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration", "name": "f_trip_time_duration", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "(to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime))/60"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('a5c47bd8-3729-45fa-8701-b8b76ada150a', 'feathr_ci_registry_12_33_182947__aggregationFeatures', 'feathr_anchor_v1', '{"features": [{"guid": "2a052ccd-3e31-46a7-bffb-2ab1302b1b00", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_avg_fare"}}, {"guid": "537bc481-aa15-4a3b-be4e-2042da6f5a09", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures__f_location_max_fare"}}], "qualifiedName": "feathr_ci_registry_12_33_182947__aggregationFeatures", "name": "aggregationFeatures", "source": {"guid": "c4a0ae0f-09cc-43bf-94e9-21ff178fbda6", "typeName": "feathr_source_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__nycTaxiBatchSource"}}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('260325a5-27f9-40d1-8697-c727feb1dbdc', 'feathr_ci_registry_12_33_182947__request_features', 'feathr_anchor_v1', '{"features": [{"guid": "5316c516-77f9-4be4-a7ec-8bf6e893e2aa", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_distance"}}, {"guid": "103baca1-377a-4ddf-8429-5da91026c269", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration"}}, {"guid": "dc24b1d5-206d-40db-b10a-606dd16a0297", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_is_long_trip_distance"}}, {"guid": "2380fe5b-ce2a-401e-98bf-af8b98460f67", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_day_of_week"}}], "qualifiedName": "feathr_ci_registry_12_33_182947__request_features", "name": "request_features", "source": {"guid": "a4cfbc03-c65d-4f32-be3d-1d11247c9cdd", "typeName": "feathr_source_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__PASSTHROUGH"}}, "tags": {"for_test_purpose": "true"}}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('2380fe5b-ce2a-401e-98bf-af8b98460f67', 'feathr_ci_registry_12_33_182947__request_features__f_day_of_week', 'feathr_anchor_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_day_of_week", "name": "f_day_of_week", "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "INT"}, "transformation": {"transform_expr": "dayofweek(lpep_dropoff_datetime)"}, "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); +insert into entities (entity_id, qualified_name, entity_type, attributes) +values('226b42ee-0c34-4329-b935-744aecc63fb4', 'feathr_ci_registry_12_33_182947__f_trip_time_distance', 'feathr_derived_feature_v1', '{"qualifiedName": "feathr_ci_registry_12_33_182947__f_trip_time_distance", "name": "f_trip_time_distance", "input_derived_features": [], "type": {"type": "TENSOR", "tensorCategory": "DENSE", "dimensionType": [], "valType": "FLOAT"}, "transformation": {"transform_expr": "f_trip_distance * f_trip_time_duration"}, "input_anchor_features": [{"guid": "5316c516-77f9-4be4-a7ec-8bf6e893e2aa", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_distance"}}, {"guid": "103baca1-377a-4ddf-8429-5da91026c269", "typeName": "feathr_anchor_feature_v1", "uniqueAttributes": {"qualifiedName": "feathr_ci_registry_12_33_182947__request_features__f_trip_time_duration"}}], "key": [{"full_name": "feathr.dummy_typedkey", "key_column": "NOT_NEEDED", "description": "A dummy typed key for passthrough/request feature.", "key_column_alias": "NOT_NEEDED", "key_column_type": "UNSPECIFIED"}]}'); + +insert into edges (edge_id, from_id, to_id, conn_type) values ('455f7195-8463-4c60-9cf0-65bd9db0ae0a', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a2777fd2-1136-40d0-8686-47b5b5fed1ef', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('ca88290d-03d1-4641-bf36-cbf3280b3e9d', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('1da26ec9-6608-4971-ac04-8ed170543325', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('9ae3a7c0-0163-4170-b0cf-81705e5b6aca', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('50068cf8-7f5e-482a-a018-68bf27f89f6d', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('898a10fd-0315-4fcb-9803-0144047033c7', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('08540367-3d1f-4d57-8af1-36b3a11762ed', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('d05ca3c6-2610-4352-9be7-c4d8dd6ab6b6', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('ed6ca745-ee85-403c-b1bf-a3f1a0463132', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('55d85311-8e42-4273-a538-ef4dc33b1570', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('215cdc37-678a-4c56-a390-76d6471fa629', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('4166aca1-e0f4-4883-b2ff-0051ee80a830', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e1174ff1-0b1a-4b03-a1f5-b8b923a131ba', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b314c95b-e4ea-4a39-9584-9b7eb0a6b8d2', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('7a37d438-8c33-4cc8-a0bb-9db3963f073a', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('31265953-4820-470f-8cfc-38efefec9fa7', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('58c448cf-87bd-4f36-92d5-e6f6de48569d', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c86e4e49-88c9-4ac8-a1b8-ff473e1dc588', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '103baca1-377a-4ddf-8429-5da91026c269', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('3fddedcf-c590-43b9-aaf4-4c4ce6600f2e', '103baca1-377a-4ddf-8429-5da91026c269', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('be429eb2-758d-4783-b166-cfcc7d2fb4f2', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('bd66f19f-3508-4b13-ba40-598dd4abbd0d', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('178c7c56-0f25-4048-be18-435ab5a169f4', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c8dc4c34-7950-4a04-a4bd-829a4c20ab4e', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('92f143c0-59a8-461b-b9cc-179f3403dd38', 'cd7306a7-c458-45e8-a00b-44a2f2117135', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e9a6f066-f4cc-4de4-bb26-2bc9522760a4', '226b42ee-0c34-4329-b935-744aecc63fb4', 'cd7306a7-c458-45e8-a00b-44a2f2117135', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('805b2c95-fc22-48b1-81cc-f86e3f2c2956', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e54835e6-b399-4d49-8ab8-6b452cbc00ca', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('02a774bb-0a1c-4874-b9e2-bbd4806c2f3e', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('17c74f58-18bc-4c4a-974b-2732a84576c3', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c8dded57-22f8-46d0-a29c-3e7c072ebee4', '260325a5-27f9-40d1-8697-c727feb1dbdc', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('7e2995b8-2a4f-49c1-b72c-80c8c6d1e060', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('3ba9b460-5878-43c1-bb00-c88916bf5e6b', '260325a5-27f9-40d1-8697-c727feb1dbdc', '103baca1-377a-4ddf-8429-5da91026c269', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a4bea335-611d-49cc-9e94-b18896103d40', '103baca1-377a-4ddf-8429-5da91026c269', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('f0d65e5b-9762-4c65-9333-83aa6c1beb75', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('cd72ee26-b867-4321-8566-7daa488f7a61', 'dc24b1d5-206d-40db-b10a-606dd16a0297', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('bc211906-6d48-4649-8e49-518cd89c61f8', '260325a5-27f9-40d1-8697-c727feb1dbdc', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'Contains'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('aeafba5d-b352-4c36-8fd5-d135cea70605', '2380fe5b-ce2a-401e-98bf-af8b98460f67', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'BelongsTo'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('2b6126be-63fd-4140-8891-e0b5db880573', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('89a348be-ae8d-40ad-a1cd-12d00b981b2a', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '260325a5-27f9-40d1-8697-c727feb1dbdc', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('7e6219a9-e433-4706-9145-f791a58ef7c3', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a447062c-c75e-48b2-bbe3-5d9eab770c26', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('faf70a4b-2358-4881-952b-aae3cec55053', '103baca1-377a-4ddf-8429-5da91026c269', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b6647b84-5043-4ac2-8b31-3db7c2d6cf32', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '103baca1-377a-4ddf-8429-5da91026c269', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b63cb71d-71c1-49a1-93bd-ea63cb4ab4e7', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('c582313e-9840-4ae6-827c-d2a01363ee6b', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'dc24b1d5-206d-40db-b10a-606dd16a0297', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('a2d2ca10-11eb-4e6f-9a5c-b8196a56d10a', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('50701b58-58ef-4b6e-ac00-5f69d8ceebf6', 'a4cfbc03-c65d-4f32-be3d-1d11247c9cdd', '2380fe5b-ce2a-401e-98bf-af8b98460f67', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('6d010032-02fb-4e94-aef7-85e56d5cf99c', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('421b8c1f-01b9-49c3-8820-ee05313d103a', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'a5c47bd8-3729-45fa-8701-b8b76ada150a', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('b6b8af3a-2531-46bc-9b1e-acf3a4c51396', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('323d044f-bbbd-45fc-a93c-40ee0f17ab87', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', '2a052ccd-3e31-46a7-bffb-2ab1302b1b00', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('cbe5c0d4-621b-4e6b-a6c1-76c8ca6105f2', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('bedbe18b-a7c3-40d3-b230-22f9ac5c6c76', 'c4a0ae0f-09cc-43bf-94e9-21ff178fbda6', '537bc481-aa15-4a3b-be4e-2042da6f5a09', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('84902105-164c-4fc3-9690-e638f16c3075', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', '103baca1-377a-4ddf-8429-5da91026c269', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('44668672-1520-4371-98d2-3ed48bddf9ea', '103baca1-377a-4ddf-8429-5da91026c269', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('e8fe5609-53e7-4793-a39d-2ce75c630cd9', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('591285ff-53dd-4e27-a30d-754fe97ea3be', 'c626c41c-d6c2-4b16-a267-6cdeea497c52', '479c6306-5fdb-4e06-9008-c18f68db52a4', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('df901204-59c1-4f3e-87f0-8ac07e90bc39', '226b42ee-0c34-4329-b935-744aecc63fb4', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('23a05c64-b204-4c48-b906-9fabb6ea298b', '5316c516-77f9-4be4-a7ec-8bf6e893e2aa', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Produces'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('3721c600-6423-4a38-a9b2-d85b426b6eaa', '226b42ee-0c34-4329-b935-744aecc63fb4', '103baca1-377a-4ddf-8429-5da91026c269', 'Consumes'); +insert into edges (edge_id, from_id, to_id, conn_type) values ('5f9b86fe-bdc9-4a76-b07c-876b9d7c1ee1', '103baca1-377a-4ddf-8429-5da91026c269', '226b42ee-0c34-4329-b935-744aecc63fb4', 'Produces'); \ No newline at end of file diff --git a/ui/.env b/ui/.env index 0a9fb0b61..010363dec 100644 --- a/ui/.env +++ b/ui/.env @@ -1,3 +1,3 @@ REACT_APP_AAD_APP_CLIENT_ID=db8dc4b0-202e-450c-b38d-7396ad9631a5 REACT_APP_AAD_APP_AUTHORITY=https://login.microsoftonline.com/common -REACT_APP_API_ENDPOINT=https://feathr-registry.azurewebsites.net +REACT_APP_API_ENDPOINT=http://localhost:8000 diff --git a/ui/src/api/api.tsx b/ui/src/api/api.tsx index ee509411f..96ae256b9 100644 --- a/ui/src/api/api.tsx +++ b/ui/src/api/api.tsx @@ -1,175 +1,203 @@ import Axios from "axios"; -import { DataSource, Feature, FeatureLineage, UserRole, Role } from "../models/model"; -import { InteractionRequiredAuthError, PublicClientApplication } from "@azure/msal-browser"; -import mockUserRole from "./mock/userrole.json"; +import { DataSource,Feature,FeatureLineage,UserRole,Role,} from "../models/model"; +import { PublicClientApplication,} from "@azure/msal-browser"; import { getMsalConfig } from "../utils/utils"; const API_ENDPOINT = process.env.REACT_APP_API_ENDPOINT + "/api/v1"; const msalInstance = getMsalConfig(); export const fetchDataSources = async (project: string) => { - const token = await getIdToken(msalInstance); - return Axios - .get(`${ API_ENDPOINT }/projects/${ project }/datasources?code=${ token }`, - { headers: {} }) - .then((response) => { - return response.data; - }) + const axios = await authAxios(msalInstance); + return axios.get( + `${API_ENDPOINT}/projects/${project}/datasources`, + { headers: {} } + ).then((response) => { + return response.data; + }); }; export const fetchProjects = async () => { - const token = await getIdToken(msalInstance); - return Axios - .get<[]>(`${ API_ENDPOINT }/projects?code=${ token }`, - { - headers: {} - }) - .then((response) => { - return response.data; - }) + const axios = await authAxios(msalInstance); + return axios.get<[]>(`${API_ENDPOINT}/projects`, { + headers: {}, + }).then((response) => { + return response.data; + }); }; -export const fetchFeatures = async (project: string, page: number, limit: number, keyword: string) => { - const token = await getIdToken(msalInstance); - return Axios - .get(`${ API_ENDPOINT }/projects/${ project }/features?code=${ token }`, - { - params: { 'keyword': keyword, 'page': page, 'limit': limit }, - headers: {} - }) - .then((response) => { - return response.data; - }) +export const fetchFeatures = async ( + project: string, + page: number, + limit: number, + keyword: string +) => { + const axios = await authAxios(msalInstance); + return axios.get( + `${API_ENDPOINT}/projects/${project}/features`, + { + params: { keyword: keyword, page: page, limit: limit }, + headers: {}, + } + ).then((response) => { + return response.data; + }); }; export const fetchFeature = async (project: string, featureId: string) => { const token = await getIdToken(msalInstance); - return Axios - .get(`${ API_ENDPOINT }/features/${ featureId }?code=${ token }`, {}) - .then((response) => { - return response.data; - }) + return Axios.get( + `${API_ENDPOINT}/features/${featureId}?code=${token}`, + {} + ).then((response) => { + return response.data; + }); }; export const fetchProjectLineages = async (project: string) => { const token = await getIdToken(msalInstance); - return Axios - .get(`${ API_ENDPOINT }/projects/${ project }?code=${ token }`, {}) - .then((response) => { - return response.data; - }) + return Axios.get( + `${API_ENDPOINT}/projects/${project}?code=${token}`, + {} + ).then((response) => { + return response.data; + }); }; export const fetchFeatureLineages = async (project: string) => { const token = await getIdToken(msalInstance); - return Axios - .get(`${ API_ENDPOINT }/features/lineage/${ project }?code=${ token }`, {}) - .then((response) => { - return response.data; - }) + return Axios.get( + `${API_ENDPOINT}/features/lineage/${project}?code=${token}`, + {} + ).then((response) => { + return response.data; + }); }; // Following are place-holder code export const createFeature = async (feature: Feature) => { const token = await getIdToken(msalInstance); - return Axios - .post(`${ API_ENDPOINT }/features?code=${ token }`, feature, - { - headers: { "Content-Type": "application/json;" }, - params: {}, - }).then((response) => { + return Axios.post(`${API_ENDPOINT}/features?code=${token}`, feature, { + headers: { "Content-Type": "application/json;" }, + params: {}, + }) + .then((response) => { return response; - }).catch((error) => { + }) + .catch((error) => { return error.response; }); -} +}; export const updateFeature = async (feature: Feature, id: string) => { const token = await getIdToken(msalInstance); feature.guid = id; - return await Axios.put(`${ API_ENDPOINT }/features/${ id }?code=${ token }`, feature, + return await Axios.put( + `${API_ENDPOINT}/features/${id}?code=${token}`, + feature, { headers: { "Content-Type": "application/json;" }, params: {}, - }).then((response) => { - return response - }).catch((error) => { - return error.response - }); + } + ) + .then((response) => { + return response; + }) + .catch((error) => { + return error.response; + }); }; export const deleteFeature = async (qualifiedName: string) => { const token = await getIdToken(msalInstance); - return await Axios - .delete(`${ API_ENDPOINT }/features/${ qualifiedName }?code=${ token }`, - { - headers: { "Content-Type": "application/json;" }, - params: {}, - }).then((response) => { - return response - }).catch((error) => { - return error.response + return await Axios.delete( + `${API_ENDPOINT}/features/${qualifiedName}?code=${token}`, + { + headers: { "Content-Type": "application/json;" }, + params: {}, + } + ) + .then((response) => { + return response; + }) + .catch((error) => { + return error.response; }); }; export const listUserRole = async () => { - let data:UserRole[] = mockUserRole - return data -}; - -export const getUserRole = async (userName: string) => { - const token = await getIdToken(msalInstance); - return await Axios - .get(`${ API_ENDPOINT }/user/${userName}/userroles?code=${ token }`, {}) - .then((response) => { + const axios = await authAxios(msalInstance); + return await axios.get( + `${API_ENDPOINT}/userroles`, + {} + ).then((response) => { return response.data; - }) -} + }); +}; export const addUserRole = async (role: Role) => { - const token = await getIdToken(msalInstance); - return await Axios - .post(`${ API_ENDPOINT }/user/${role.userName}/userroles/new?code=${ token }`, role, - { - headers: { "Content-Type": "application/json;" }, - params: {}, - }).then((response) => { + const axios = await authAxios(msalInstance); + return await axios.post( + `${API_ENDPOINT}/users/${role.userName}/userroles/add`, + role, + { + params: { + project: role.scope, + role: role.roleName, + reason: role.reason, + }, + } + ) + .then((response) => { return response; - }).catch((error) => { + }) + .catch((error) => { return error.response; }); -} +}; -export const deleteUserRole = async (role: Role) => { - const token = await getIdToken(msalInstance); - return await Axios - .post(`${ API_ENDPOINT }/user/${role.userName}/userroles/delete?code=${ token }`, role, - { - headers: { "Content-Type": "application/json;" }, - params: {}, - }).then((response) => { +export const deleteUserRole = async (userrole: UserRole) => { + const axios = await authAxios(msalInstance); + const reason = "Delete from management UI."; + return await axios.delete( + `${API_ENDPOINT}/users/${userrole.userName}/userroles/delete`, + { + params: { + project: userrole.scope, + role: userrole.roleName, + reason: reason, + }, + } + ) + .then((response) => { return response; - }).catch((error) => { + }) + .catch((error) => { return error.response; }); -} +}; -export const getIdToken = async( msalInstance: PublicClientApplication ): Promise => { +export const getIdToken = async ( + msalInstance: PublicClientApplication +): Promise => { const activeAccount = msalInstance.getActiveAccount(); // This will only return a non-null value if you have logic somewhere else that calls the setActiveAccount API const accounts = msalInstance.getAllAccounts(); const request = { - scopes: ["User.Read"], - account: activeAccount || accounts[0] + scopes: ["User.Read"], + account: activeAccount || accounts[0], }; // Silently acquire an token for a given set of scopes. Will use cached token if available, otherwise will attempt to acquire a new token from the network via refresh token. - await msalInstance.acquireTokenSilent(request).then(response => { - return response.idToken - }).catch(error => { - if (error instanceof InteractionRequiredAuthError) { - msalInstance.acquireTokenPopup(request).then(response => { - return response.idToken - }); - } - }) - return "" -} \ No newline at end of file + // A known issue may cause token expire: https://github.com/AzureAD/microsoft-authentication-library-for-js/issues/4206 + const authResult = await msalInstance.acquireTokenSilent(request); + return authResult.idToken; +}; + +export const authAxios = async (msalInstance: PublicClientApplication) =>{ + const token = await getIdToken(msalInstance); + return Axios.create({ + headers: { + 'Authorization': "Bearer "+ token, + 'Content-Type': 'application/json', + }, + baseURL: API_ENDPOINT, + }); +}; \ No newline at end of file diff --git a/ui/src/api/mock/userrole.json b/ui/src/api/mock/userrole.json deleted file mode 100644 index 20535b8ec..000000000 --- a/ui/src/api/mock/userrole.json +++ /dev/null @@ -1,63 +0,0 @@ -[ - { - "id": 1, - "scope": "Global", - "userName": "edwinc@microsoft.com", - "roleName": "Admin", - "permissions": [ - "Read", - "Write", - "Management" - ], - "createReason": "Resource Owner", - "createTime": "2022/5/15" - }, - { - "id": 1, - "scope": "Global", - "userName": "yuqwe@microsoft.com", - "roleName": "Admin", - "permissions": [ - "Read", - "Write", - "Management" - ], - "createReason": "Test Purpose", - "createTime": "2022/5/16" - }, - { - "id": 2, - "scope": "Project A: Frontend Datasets", - "userName": "blairch@microsoft.com", - "roleName": "Producer", - "permissions": [ - "Read", - "Write" - ], - "createReason": "Project Owner", - "createTime": "2022/5/16" - }, - { - "id": 3, - "scope": "Project B: Backend Datasets", - "userName": "xuchen@microsoft.com", - "roleName": "Producer", - "permissions": [ - "Read", - "Write" - ], - "createReason": "Project Owner", - "createTime": "2022/5/16" - }, - { - "id": 4, - "scope": "Project B: Backend Datasets", - "userName": "yihgu@microsoft.com", - "roleName": "Consumer", - "permissions": [ - "Read" - ], - "createReason": "Data Engineering", - "createTime": "2022/5/17" - } -] \ No newline at end of file diff --git a/ui/src/components/roleManagementForm.tsx b/ui/src/components/roleManagementForm.tsx index fa8a6a4ac..4d7949862 100644 --- a/ui/src/components/roleManagementForm.tsx +++ b/ui/src/components/roleManagementForm.tsx @@ -11,9 +11,9 @@ type RoleManagementFormProps = { userRole?: UserRole; }; -const Admin = "Admin" -const Producer = "Producer" -const Consumer = "Consumer" +const Admin = "admin" +const Producer = "producer" +const Consumer = "consumer" const RoleManagementForm: React.FC = ({ editMode, userRole }) => { const [fireRedirect] = useState(false); @@ -63,7 +63,7 @@ const RoleManagementForm: React.FC = ({ editMode, userR - + diff --git a/ui/src/components/userRoles.tsx b/ui/src/components/userRoles.tsx index 2fe18f005..0f890d775 100644 --- a/ui/src/components/userRoles.tsx +++ b/ui/src/components/userRoles.tsx @@ -1,148 +1,168 @@ -import React, { useCallback, useEffect, useState } from 'react'; -import { useNavigate } from 'react-router-dom'; -import { Button, Modal, PageHeader, Row, Space, Table, Tag } from "antd"; +import React, { useCallback, useEffect, useState } from "react"; +import { useNavigate } from "react-router-dom"; +import { + Button, + Menu, + message, + PageHeader, + Popconfirm, + Row, + Space, + Table, + Tag, +} from "antd"; import { UserRole } from "../models/model"; -import { listUserRole } from "../api"; +import { deleteUserRole, listUserRole } from "../api"; const UserRoles: React.FC = () => { - const navigate = useNavigate(); - const [visible, setVisible] = React.useState(false); - const [confirmLoading, setConfirmLoading] = React.useState(false); - const [modalText, setModalText] = React.useState('Content of the modal'); + const navigate = useNavigate(); - const showModal = () => { - setVisible(true); - setModalText("This Role Assignment will be deleted."); - }; - const handleOk = () => { - setModalText('The modal will be closed after two seconds'); - setConfirmLoading(true); - setTimeout(() => { - setVisible(false); - setConfirmLoading(false); - }, 2000); - }; + const onDelete = async (row: UserRole) => { + console.log( + `The [${row.roleName}] Role of [${row.userName}] user role delete request is sent.` + ); + const res = await deleteUserRole(row); + if (res.status === 200) { + message.success(`Role ${row.roleName} of user ${row.userName} deleted`); + } else { + message.error("Failed to delete userrole."); + } + setLoading(false); + fetchData(); + }; - const handleCancel = () => { - console.log('Clicked cancel button'); - setVisible(false); - }; - const columns = [ - { - title:
Scope
, - dataIndex: 'scope', - key: 'scope', - align: 'center' as 'center', - }, - { - title:
User
, - dataIndex: 'userName', - key: 'userName', - align: 'center' as 'center', - }, - { - title:
Role
, - dataIndex: 'roleName', - key: 'roleName', - align: 'center' as 'center', - }, - { - title:
Permissions
, - key: 'permissions', - dataIndex: 'permissions', - render: (tags: any[]) => ( - <> - {tags.map(tag => { - let color = tag.length > 5 ? 'red' : 'green'; - if (tag === 'Write') color = 'blue' - return ( - - {tag.toUpperCase()} - - ); - })} - - ), - }, - { - title:
Create Reason
, - dataIndex: 'createReason', - key: 'createReason', - align: 'center' as 'center', - }, - { - title:
Create Time
, - dataIndex: 'createTime', - key: 'createTime', - align: 'center' as 'center', - }, - { - title: 'Action', - key: 'action', - render: () => ( - - - -

{modalText}

- - - ), - }, - ]; - const [page, setPage] = useState(1); - const [, setLoading] = useState(false); - const [tableData, setTableData] = useState(); + const columns = [ + { + title:
Scope (Project/Global)
, + dataIndex: "scope", + key: "scope", + align: "center" as "center", + }, + { + title:
Role
, + dataIndex: "roleName", + key: "roleName", + align: "center" as "center", + }, + { + title:
User
, + dataIndex: "userName", + key: "userName", + align: "center" as "center", + }, + { + title:
Permissions
, + key: "access", + dataIndex: "access", + render: (tags: any[]) => ( + <> + {tags.map((tag) => { + let color = tag.length > 5 ? "red" : "green"; + if (tag === "write") color = "blue"; + return ( + + {tag.toUpperCase()} + + ); + })} + + ), + }, + { + title:
Create By
, + dataIndex: "createBy", + key: "createBy", + align: "center" as "center", + }, + { + title:
Create Reason
, + dataIndex: "createReason", + key: "createReason", + align: "center" as "center", + }, + { + title:
Create Time
, + dataIndex: "createTime", + key: "createTime", + align: "center" as "center", + }, + { + title: "Action", + key: "action", + render: (userName: string, row: UserRole) => ( + + + + { + onDelete(row); + }} + > + Delete + + + + + ), + }, + ]; + const [page, setPage] = useState(1); + const [, setLoading] = useState(false); + const [tableData, setTableData] = useState(); - const fetchData = useCallback(async () => { - setLoading(true); - const result = await listUserRole(); - console.log(result); - setPage(page); - setTableData(result); - setLoading(false); - }, [page]) + const fetchData = useCallback(async () => { + setLoading(true); + const result = await listUserRole(); + console.log(result); + setPage(page); + setTableData(result); + setLoading(false); + }, [page]); - const onClickRoleAssign = () => { - navigate('/role-management'); - return; - } + const onClickRoleAssign = () => { + navigate("/role-management"); + return; + }; - useEffect(() => { - fetchData() - }, [fetchData]) + useEffect(() => { + fetchData(); + }, [fetchData]); - return ( -
- - -
- <> -

- Below is the mock data for now. Will connect with Management APIs. -

- -
- - - - - - ; - - ); -} + return ( +
+ + +
+ <> +

+ This page is protected by Feathr RBAC. Only Global Admin can see + the content details. （Work In Progress） +

+ +
+ + + + + +
; + + ); +}; export default UserRoles; diff --git a/ui/src/models/model.ts b/ui/src/models/model.ts index c89c31b26..2bf5467d5 100644 --- a/ui/src/models/model.ts +++ b/ui/src/models/model.ts @@ -15,31 +15,31 @@ export interface FeatureAttributes { key: FeatureKey[]; window: string; _input_anchor_features: Feature[]; - _input_derived_features: Feature[] + _input_derived_features: Feature[]; } export interface FeatureType { - type: string, - tensor_category: string, - dimension_type: string[], - val_type: string + type: string; + tensor_category: string; + dimension_type: string[]; + val_type: string; } export interface FeatureTransformation { - transform_expr: string, - filter: string, - agg_func: string, - limit: string, - group_by: string, - window: string, - def_expr: string + transform_expr: string; + filter: string; + agg_func: string; + limit: string; + group_by: string; + window: string; + def_expr: string; } export interface FeatureKey { - full_name: string, - key_column: string, - description: string, - key_column_alias: string, - key_column_type: string + full_name: string; + key_column: string; + description: string; + key_column_alias: string; + key_column_type: string; } export interface DataSource { @@ -52,7 +52,6 @@ export interface DataSourceAttributes { path: string; } - export interface FeatureLineage { guidEntityMap: any; relations: any; @@ -67,6 +66,7 @@ export interface UserRole { createReason: string; deleteTime?: any; deleteReason?: any; + access?: string[]; } export interface Role {