From bf84b36239fdf8524960d5aeddd7aa899c05391d Mon Sep 17 00:00:00 2001 From: enya-yx Date: Tue, 27 Sep 2022 17:31:19 +0800 Subject: [PATCH 1/4] remove unused credentials and deprcated purview settings --- docs/how-to-guides/azure-deployment-cli.md | 4 ---- .../feathr-configuration-and-env.md | 6 +++--- docs/quickstart_synapse.md | 6 ------ docs/samples/customer360/Customer360.ipynb | 18 ++++++++---------- ...databricks_quickstart_nyc_taxi_driver.ipynb | 8 +------- docs/samples/product_recommendation_demo.ipynb | 2 +- .../product_recommendation_demo_advanced.ipynb | 2 +- 7 files changed, 14 insertions(+), 32 deletions(-) diff --git a/docs/how-to-guides/azure-deployment-cli.md b/docs/how-to-guides/azure-deployment-cli.md index 3762f7b3f..7c6973700 100644 --- a/docs/how-to-guides/azure-deployment-cli.md +++ b/docs/how-to-guides/azure-deployment-cli.md @@ -112,10 +112,6 @@ az group create -l $location -n $resource_group_name sp_password=$(az ad sp create-for-rbac --name $service_principal_name --role Contributor --scopes /subscriptions/$subscription_id/resourceGroups/$resource_group_name --query "[password]" --output tsv) sp_appid=$(az ad sp list --display-name $service_principal_name --query "[].{appId:appId}" --output tsv) sp_tenantid=$(az ad sp list --display-name $service_principal_name --query "[].{appOwnerOrganizationId:appOwnerOrganizationId}" --output tsv) -echo "AZURE_CLIENT_ID: $sp_appid" -echo "AZURE_TENANT_ID: $sp_tenantid" -echo "AZURE_CLIENT_SECRET: $sp_password" -This will give three variables: AZURE_CLIENT_ID, AZURE_TENANT_ID and AZURE_CLIENT_SECRET. You will need them later. ``` Note: **You should save AZURE_CLIENT_SECRET because you will only see it once here** diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index fd32fb2f6..75b567418 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -77,9 +77,9 @@ Feathr will get the configurations in the following order: | ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. | Required if using Redis as online store. | | REDIS_PASSWORD | Password for the Redis cluster. | Required if using Redis as online store. | | FEATURE_REGISTRY__API_ENDPOINT | Specifies registry endpoint. | Required if using registry service. | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__DELIMITER | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Depracated) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__DELIMITER (Depracated) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Depracated)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| # Explanation for selected configurations diff --git a/docs/quickstart_synapse.md b/docs/quickstart_synapse.md index 5dee17931..0a66a96bb 100644 --- a/docs/quickstart_synapse.md +++ b/docs/quickstart_synapse.md @@ -61,9 +61,6 @@ project_config: # Redis password for your online store - "REDIS_PASSWORD" # Client IDs and Client Secret for the service principal. Read the getting started docs on how to get those information. - - "AZURE_CLIENT_ID" - - "AZURE_TENANT_ID" - - "AZURE_CLIENT_SECRET" offline_store: --- @@ -98,9 +95,6 @@ These values can also be retrieved by using cloud key value store, such as [Azur ```python import os os.environ['REDIS_PASSWORD'] = '' -os.environ['AZURE_CLIENT_ID'] = '' -os.environ['AZURE_TENANT_ID'] = '' -os.environ['AZURE_CLIENT_SECRET'] = '' ``` Please refer to [A note on using azure key vault to store credentials](https://github.com/feathr-ai/feathr/blob/41e7496b38c43af6d7f8f1de842f657b27840f6d/docs/how-to-guides/feathr-configuration-and-env.md#a-note-on-using-azure-key-vault-to-store-credentials) for more details. diff --git a/docs/samples/customer360/Customer360.ipynb b/docs/samples/customer360/Customer360.ipynb index 4b202e13a..07b5d07ed 100644 --- a/docs/samples/customer360/Customer360.ipynb +++ b/docs/samples/customer360/Customer360.ipynb @@ -192,9 +192,6 @@ " project_name: 'customer360'\n", " required_environment_variables:\n", " - 'REDIS_PASSWORD'\n", - " - 'AZURE_CLIENT_ID'\n", - " - 'AZURE_TENANT_ID'\n", - " - 'AZURE_CLIENT_SECRET'\n", " - 'ADLS_ACCOUNT'\n", " - 'ADLS_KEY'\n", " - 'WASB_ACCOUNT'\n", @@ -239,10 +236,7 @@ " port: 6380\n", " ssl_enabled: True\n", "feature_registry:\n", - " purview:\n", - " type_system_initialization: true\n", - " purview_name: ''\n", - " delimiter: '__'\n", + " api_endpoint: \"https://.azurewebsites.net/api/v1\"\n", "\"\"\"\n", "# write this configuration string to a temporary location and load it to Feathr\n", "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", @@ -331,9 +325,6 @@ "source": [ "import os\n", "os.environ['REDIS_PASSWORD'] = ''\n", - "os.environ['AZURE_CLIENT_ID'] = ''\n", - "os.environ['AZURE_TENANT_ID'] = ''\n", - "os.environ['AZURE_CLIENT_SECRET'] = ''\n", "os.environ['ADLS_ACCOUNT'] = ''\n", "os.environ['ADLS_KEY'] = ''\n", "os.environ['WASB_ACCOUNT'] = \"\"\n", @@ -601,6 +592,13 @@ "And then we need to build those features so that it can be consumed later. Note that we have to build both the \"anchor\" and the \"derived\" features (which is not anchored to a source)." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb index 82aaf3832..52790f884 100644 --- a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb +++ b/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb @@ -332,9 +332,6 @@ " project_name: 'feathr_getting_started2'\n", " required_environment_variables:\n", " - 'REDIS_PASSWORD'\n", - " - 'AZURE_CLIENT_ID'\n", - " - 'AZURE_TENANT_ID'\n", - " - 'AZURE_CLIENT_SECRET'\n", "offline_store:\n", " adls:\n", " adls_enabled: true\n", @@ -364,10 +361,7 @@ " port: 6380\n", " ssl_enabled: True\n", "feature_registry:\n", - " purview:\n", - " type_system_initialization: true\n", - " purview_name: ''\n", - " delimiter: '__'\n", + " api_endpoint: \"https://.azurewebsites.net/api/v1\"\n", "\"\"\"\n", "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", "with open(tmp.name, \"w\") as text_file:\n", diff --git a/docs/samples/product_recommendation_demo.ipynb b/docs/samples/product_recommendation_demo.ipynb index aa7699eb5..470155e63 100644 --- a/docs/samples/product_recommendation_demo.ipynb +++ b/docs/samples/product_recommendation_demo.ipynb @@ -260,7 +260,7 @@ "\n", "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", "\n", - "To run this notebook, for Azure users, you need AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET and REDIS_PASSWORD.\n", + "To run this notebook, for Azure users, you need REDIS_PASSWORD.\n", "To run this notebook, for Databricks useres, you need DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD." ] }, diff --git a/docs/samples/product_recommendation_demo_advanced.ipynb b/docs/samples/product_recommendation_demo_advanced.ipynb index fff2a1cd5..1248cfb85 100644 --- a/docs/samples/product_recommendation_demo_advanced.ipynb +++ b/docs/samples/product_recommendation_demo_advanced.ipynb @@ -389,7 +389,7 @@ "\n", "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", "\n", - "To run this notebook, for Azure users, you need AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET and REDIS_PASSWORD.\n", + "To run this notebook, for Azure users, you need REDIS_PASSWORD.\n", "To run this notebook, for Databricks useres, you need DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD." ] }, From 538e4a57e71a0fabbb8728f690bee68e2d2a66ef Mon Sep 17 00:00:00 2001 From: enya-yx Date: Tue, 27 Sep 2022 17:36:19 +0800 Subject: [PATCH 2/4] fix typo --- docs/how-to-guides/feathr-configuration-and-env.md | 6 +++--- docs/samples/customer360/Customer360.ipynb | 7 ------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 75b567418..6e2a69032 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -77,9 +77,9 @@ Feathr will get the configurations in the following order: | ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. | Required if using Redis as online store. | | REDIS_PASSWORD | Password for the Redis cluster. | Required if using Redis as online store. | | FEATURE_REGISTRY__API_ENDPOINT | Specifies registry endpoint. | Required if using registry service. | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Depracated) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__DELIMITER (Depracated) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Depracated)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Deprecated) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__DELIMITER (Deprecated) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Deprecated)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| # Explanation for selected configurations diff --git a/docs/samples/customer360/Customer360.ipynb b/docs/samples/customer360/Customer360.ipynb index 07b5d07ed..db042011b 100644 --- a/docs/samples/customer360/Customer360.ipynb +++ b/docs/samples/customer360/Customer360.ipynb @@ -592,13 +592,6 @@ "And then we need to build those features so that it can be consumed later. Note that we have to build both the \"anchor\" and the \"derived\" features (which is not anchored to a source)." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, From 84bc0713a94ed5bd859ef8ccd7b54ec4ec3af03e Mon Sep 17 00:00:00 2001 From: enya-yx Date: Fri, 30 Sep 2022 16:23:02 +0800 Subject: [PATCH 3/4] quick fix --- docs/how-to-guides/azure-deployment-cli.md | 4 ++++ docs/how-to-guides/feathr-configuration-and-env.md | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/how-to-guides/azure-deployment-cli.md b/docs/how-to-guides/azure-deployment-cli.md index 7c6973700..ff1043eae 100644 --- a/docs/how-to-guides/azure-deployment-cli.md +++ b/docs/how-to-guides/azure-deployment-cli.md @@ -113,6 +113,10 @@ sp_password=$(az ad sp create-for-rbac --name $service_principal_name --role Con sp_appid=$(az ad sp list --display-name $service_principal_name --query "[].{appId:appId}" --output tsv) sp_tenantid=$(az ad sp list --display-name $service_principal_name --query "[].{appOwnerOrganizationId:appOwnerOrganizationId}" --output tsv) ``` +echo "AZURE_CLIENT_ID: $sp_appid" +echo "AZURE_TENANT_ID: $sp_tenantid" +echo "AZURE_CLIENT_SECRET: $sp_password" +This will give three variables: AZURE_CLIENT_ID, AZURE_TENANT_ID and AZURE_CLIENT_SECRET. You will need them later. Note: **You should save AZURE_CLIENT_SECRET because you will only see it once here** diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 6e2a69032..e4d393a43 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -77,9 +77,9 @@ Feathr will get the configurations in the following order: | ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. | Required if using Redis as online store. | | REDIS_PASSWORD | Password for the Redis cluster. | Required if using Redis as online store. | | FEATURE_REGISTRY__API_ENDPOINT | Specifies registry endpoint. | Required if using registry service. | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Deprecated) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__DELIMITER (Deprecated) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Deprecated)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Deprecated Soon) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__DELIMITER (Deprecated Soon) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Deprecated Soon)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| # Explanation for selected configurations From 1afb3bce4b91d8819bfe367c9d425d759c090925 Mon Sep 17 00:00:00 2001 From: enya-yx Date: Fri, 30 Sep 2022 16:26:46 +0800 Subject: [PATCH 4/4] quick fix --- docs/how-to-guides/azure-deployment-cli.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/how-to-guides/azure-deployment-cli.md b/docs/how-to-guides/azure-deployment-cli.md index ff1043eae..70067b148 100644 --- a/docs/how-to-guides/azure-deployment-cli.md +++ b/docs/how-to-guides/azure-deployment-cli.md @@ -112,12 +112,11 @@ az group create -l $location -n $resource_group_name sp_password=$(az ad sp create-for-rbac --name $service_principal_name --role Contributor --scopes /subscriptions/$subscription_id/resourceGroups/$resource_group_name --query "[password]" --output tsv) sp_appid=$(az ad sp list --display-name $service_principal_name --query "[].{appId:appId}" --output tsv) sp_tenantid=$(az ad sp list --display-name $service_principal_name --query "[].{appOwnerOrganizationId:appOwnerOrganizationId}" --output tsv) -``` echo "AZURE_CLIENT_ID: $sp_appid" echo "AZURE_TENANT_ID: $sp_tenantid" echo "AZURE_CLIENT_SECRET: $sp_password" This will give three variables: AZURE_CLIENT_ID, AZURE_TENANT_ID and AZURE_CLIENT_SECRET. You will need them later. - +``` Note: **You should save AZURE_CLIENT_SECRET because you will only see it once here** ## Create a storage account