From 70a296ee8da6faf622c7c8ebe57e9a7a334bcdaf Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 8 Jun 2022 14:04:12 -0700 Subject: [PATCH 01/18] optimize environment variable behavior --- feathr_project/feathr/client.py | 76 ++++++++-------- .../feathr/utils/_envvariableutil.py | 91 ++++++++++++------- .../test/prep_azure_kafka_test_data.py | 3 +- 3 files changed, 95 insertions(+), 75 deletions(-) diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py index 8314bdba2..b141e8f5c 100644 --- a/feathr_project/feathr/client.py +++ b/feathr_project/feathr/client.py @@ -90,7 +90,7 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir self.logger = logging.getLogger(__name__) # Redis key separator self._KEY_SEPARATOR = ':' - envutils = _EnvVaraibleUtil(config_path) + self.envutils = _EnvVaraibleUtil(config_path) if local_workspace_dir: self.local_workspace_dir = local_workspace_dir else: @@ -98,32 +98,30 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir tem_dir_obj = tempfile.TemporaryDirectory() self.local_workspace_dir = tem_dir_obj.name - self.envutils = envutils - if not os.path.exists(config_path): self.logger.warning('Configuration path does not exist, you need to set the environment variables explicitly. For all the environment variables, please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml') # Load all configs from yaml at initialization # DO NOT load any configs from yaml during runtime. - self.project_name = envutils.get_environment_variable_with_default( + self.project_name = self.envutils.get_environment_variable_with_default( 'project_config', 'project_name') # Redis configs - self.redis_host = envutils.get_environment_variable_with_default( + self.redis_host = self.envutils.get_environment_variable_with_default( 'online_store', 'redis', 'host') - self.redis_port = envutils.get_environment_variable_with_default( + self.redis_port = self.envutils.get_environment_variable_with_default( 'online_store', 'redis', 'port') - self.redis_ssl_enabled = envutils.get_environment_variable_with_default( + self.redis_ssl_enabled = self.envutils.get_environment_variable_with_default( 'online_store', 'redis', 'ssl_enabled') # S3 configs - self.s3_endpoint = envutils.get_environment_variable_with_default( + self.s3_endpoint = self.envutils.get_environment_variable_with_default( 'offline_store', 's3', 's3_endpoint') # spark configs - self.output_num_parts = envutils.get_environment_variable_with_default( + self.output_num_parts = self.envutils.get_environment_variable_with_default( 'spark_config', 'spark_result_output_parts') - self.spark_runtime = envutils.get_environment_variable_with_default( + self.spark_runtime = self.envutils.get_environment_variable_with_default( 'spark_config', 'spark_cluster') self.credential = credential @@ -135,22 +133,22 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir # Spark job submission. The feathr jar hosted in cloud saves the time users needed to upload the jar from # their local env. self._FEATHR_JOB_JAR_PATH = \ - envutils.get_environment_variable_with_default( + self.envutils.get_environment_variable_with_default( 'spark_config', 'azure_synapse', 'feathr_runtime_location') if self.credential is None: self.credential = DefaultAzureCredential(exclude_interactive_browser_credential=False) self.feathr_spark_laucher = _FeathrSynapseJobLauncher( - synapse_dev_url=envutils.get_environment_variable_with_default( + synapse_dev_url=self.envutils.get_environment_variable_with_default( 'spark_config', 'azure_synapse', 'dev_url'), - pool_name=envutils.get_environment_variable_with_default( + pool_name=self.envutils.get_environment_variable_with_default( 'spark_config', 'azure_synapse', 'pool_name'), - datalake_dir=envutils.get_environment_variable_with_default( + datalake_dir=self.envutils.get_environment_variable_with_default( 'spark_config', 'azure_synapse', 'workspace_dir'), - executor_size=envutils.get_environment_variable_with_default( + executor_size=self.envutils.get_environment_variable_with_default( 'spark_config', 'azure_synapse', 'executor_size'), - executors=envutils.get_environment_variable_with_default( + executors=self.envutils.get_environment_variable_with_default( 'spark_config', 'azure_synapse', 'executor_num'), credential=self.credential ) @@ -159,17 +157,17 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir # Spark job submission. The feathr jar hosted in cloud saves the time users needed to upload the jar from # their local env. self._FEATHR_JOB_JAR_PATH = \ - envutils.get_environment_variable_with_default( + self.envutils.get_environment_variable_with_default( 'spark_config', 'databricks', 'feathr_runtime_location') self.feathr_spark_laucher = _FeathrDatabricksJobLauncher( - workspace_instance_url=envutils.get_environment_variable_with_default( + workspace_instance_url=self.envutils.get_environment_variable_with_default( 'spark_config', 'databricks', 'workspace_instance_url'), - token_value=_EnvVaraibleUtil.get_environment_variable( + token_value=self.envutils.get_environment_variable( 'DATABRICKS_WORKSPACE_TOKEN_VALUE'), - config_template=envutils.get_environment_variable_with_default( + config_template=self.envutils.get_environment_variable_with_default( 'spark_config', 'databricks', 'config_template'), - databricks_work_dir=envutils.get_environment_variable_with_default( + databricks_work_dir=self.envutils.get_environment_variable_with_default( 'spark_config', 'databricks', 'work_dir') ) @@ -177,9 +175,9 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir # initialize registry - self.registry_delimiter = envutils.get_environment_variable_with_default( + self.registry_delimiter = self.envutils.get_environment_variable_with_default( 'feature_registry', 'purview', 'delimiter') - self.azure_purview_name = envutils.get_environment_variable_with_default( + self.azure_purview_name = self.envutils.get_environment_variable_with_default( 'feature_registry', 'purview', 'purview_name') # initialize the registry no matter whether we set purview name or not, given some of the methods are used there. self.registry = _FeatureRegistry(self.project_name, self.azure_purview_name, self.registry_delimiter, project_registry_tag, config_path = config_path, credential=self.credential) @@ -382,7 +380,7 @@ def _construct_redis_client(self): """Constructs the Redis client. The host, port, credential and other parameters can be set via environment parameters. """ - password = _EnvVaraibleUtil.get_environment_variable(REDIS_PASSWORD) + password = self.envutils.get_environment_variable(REDIS_PASSWORD) host = self.redis_host port = self.redis_port ssl_enabled = self.redis_ssl_enabled @@ -576,7 +574,7 @@ def _materialize_features_with_config(self, feature_gen_conf_path: str = 'featur Job configurations and job arguments (or sometimes called job parameters) have quite some overlaps (i.e. you can achieve the same goal by either using the job arguments/parameters vs. job configurations). But the job tags should just be used for metadata purpose. ''' optional_params = [] - if _EnvVaraibleUtil.get_environment_variable('KAFKA_SASL_JAAS_CONFIG'): + if self.envutils.get_environment_variable('KAFKA_SASL_JAAS_CONFIG'): optional_params = optional_params + ['--kafka-config', self._get_kafka_config_str()] return self.feathr_spark_laucher.submit_feathr_job( job_name=self.project_name + '_feathr_feature_materialization_job', @@ -612,7 +610,7 @@ def wait_job_to_finish(self, timeout_sec: int = 300): def _getRedisConfigStr(self): """Construct the Redis config string. The host, port, credential and other parameters can be set via environment variables.""" - password = _EnvVaraibleUtil.get_environment_variable(REDIS_PASSWORD) + password = self.envutils.get_environment_variable(REDIS_PASSWORD) host = self.redis_host port = self.redis_port ssl_enabled = self.redis_ssl_enabled @@ -630,8 +628,8 @@ def _get_s3_config_str(self): endpoint = self.s3_endpoint # if s3 endpoint is set in the feathr_config, then we need other environment variables # keys can't be only accessed through environment - access_key = _EnvVaraibleUtil.get_environment_variable('S3_ACCESS_KEY') - secret_key = _EnvVaraibleUtil.get_environment_variable('S3_SECRET_KEY') + access_key = self.envutils.get_environment_variable('S3_ACCESS_KEY') + secret_key = self.envutils.get_environment_variable('S3_SECRET_KEY') # HOCCON format will be parsed by the Feathr job config_str = """ S3_ENDPOINT: {S3_ENDPOINT} @@ -643,10 +641,10 @@ def _get_s3_config_str(self): def _get_adls_config_str(self): """Construct the ADLS config string for abfs(s). The Account, access key and other parameters can be set via environment variables.""" - account = _EnvVaraibleUtil.get_environment_variable('ADLS_ACCOUNT') + account = self.envutils.get_environment_variable('ADLS_ACCOUNT') # if ADLS Account is set in the feathr_config, then we need other environment variables # keys can't be only accessed through environment - key = _EnvVaraibleUtil.get_environment_variable('ADLS_KEY') + key = self.envutils.get_environment_variable('ADLS_KEY') # HOCCON format will be parsed by the Feathr job config_str = """ ADLS_ACCOUNT: {ADLS_ACCOUNT} @@ -657,10 +655,10 @@ def _get_adls_config_str(self): def _get_blob_config_str(self): """Construct the Blob config string for wasb(s). The Account, access key and other parameters can be set via environment variables.""" - account = _EnvVaraibleUtil.get_environment_variable('BLOB_ACCOUNT') + account = self.envutils.get_environment_variable('BLOB_ACCOUNT') # if BLOB Account is set in the feathr_config, then we need other environment variables # keys can't be only accessed through environment - key = _EnvVaraibleUtil.get_environment_variable('BLOB_KEY') + key = self.envutils.get_environment_variable('BLOB_KEY') # HOCCON format will be parsed by the Feathr job config_str = """ BLOB_ACCOUNT: {BLOB_ACCOUNT} @@ -671,12 +669,12 @@ def _get_blob_config_str(self): def _get_sql_config_str(self): """Construct the SQL config string for jdbc. The dbtable (query), user, password and other parameters can be set via environment variables.""" - table = _EnvVaraibleUtil.get_environment_variable('JDBC_TABLE') - user = _EnvVaraibleUtil.get_environment_variable('JDBC_USER') - password = _EnvVaraibleUtil.get_environment_variable('JDBC_PASSWORD') - driver = _EnvVaraibleUtil.get_environment_variable('JDBC_DRIVER') - auth_flag = _EnvVaraibleUtil.get_environment_variable('JDBC_AUTH_FLAG') - token = _EnvVaraibleUtil.get_environment_variable('JDBC_TOKEN') + table = self.envutils.get_environment_variable('JDBC_TABLE') + user = self.envutils.get_environment_variable('JDBC_USER') + password = self.envutils.get_environment_variable('JDBC_PASSWORD') + driver = self.envutils.get_environment_variable('JDBC_DRIVER') + auth_flag = self.envutils.get_environment_variable('JDBC_AUTH_FLAG') + token = self.envutils.get_environment_variable('JDBC_TOKEN') # HOCCON format will be parsed by the Feathr job config_str = """ JDBC_TABLE: {JDBC_TABLE} @@ -707,7 +705,7 @@ def _get_snowflake_config_str(self): def _get_kafka_config_str(self): """Construct the Kafka config string. The endpoint, access key, secret key, and other parameters can be set via environment variables.""" - sasl = _EnvVaraibleUtil.get_environment_variable('KAFKA_SASL_JAAS_CONFIG') + sasl = self.envutils.get_environment_variable('KAFKA_SASL_JAAS_CONFIG') # HOCCON format will be parsed by the Feathr job config_str = """ KAFKA_SASL_JAAS_CONFIG: "{sasl}" diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index c4038736b..c5a06625f 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -8,14 +8,17 @@ class _EnvVaraibleUtil(object): def __init__(self, config_path): self.config_path = config_path self.akv_client = None + self.akv_name = os.environ.get("FEATHR_KEY_VAULT_NAME") def get_environment_variable_with_default(self, *args): """Gets the environment variable for the variable key. Args: *args: list of keys in feathr_config.yaml file Return: - A environment variable for the variable key. If it's not set in the environment, then a default is retrieved - from the feathr_config.yaml file with the same config key. + A environment variable for the variable key. It will retrieve the value of the environment variables in the following order: + If the key is set in the envrionment variable, Feathr will use the value of that environment variable + If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. + If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault """ # if envs exist, just return the existing env variable without reading the file @@ -24,51 +27,69 @@ def get_environment_variable_with_default(self, *args): # make it work for lower case and upper case. env_variable = os.environ.get( env_keyword, os.environ.get(upper_env_keyword)) + + # If the key is set in the envrionment variable, Feathr will use the value of that environment variable if env_variable: return env_variable - # if the config path doesn't exist, just return - try: - assert os.path.exists(os.path.abspath(self.config_path)) - except: - logger.info("{} is not set and configuration file {} cannot be found. One of those should be set." , env_keyword, self.config_path) - - with open(os.path.abspath(self.config_path), 'r') as stream: - try: - yaml_config = yaml.safe_load(stream) - # concat all layers - # check in environment variable - yaml_layer = yaml_config + # If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. + if os.path.exists(os.path.abspath(self.config_path)): + with open(os.path.abspath(self.config_path), 'r') as stream: + try: + yaml_config = yaml.safe_load(stream) + # concat all layers and check in environment variable + yaml_layer = yaml_config - # resolve one layer after another - for arg in args: - yaml_layer = yaml_layer[arg] - return yaml_layer - except KeyError as exc: - logger.info(exc) - return "" - except yaml.YAMLError as exc: - logger.info(exc) + # resolve one layer after another + for arg in args: + yaml_layer = yaml_layer[arg] + return yaml_layer + except KeyError as exc: + logger.info("{} not found in the config file, loading it in key vault.", env_keyword) + # if not found in the config file, use key vault to get the value + return self._get_environment_variable_from_key_vault(env_keyword) if self.akv_name else "" + except yaml.YAMLError as exc: + logger.info(exc) + # If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault + if self.akv_name: + return self._get_environment_variable_from_key_vault(env_keyword) + + # print out warning message if cannot find the env variable in all the resources + logger.warning('Environment variable {} not found.', env_keyword) - @staticmethod - def get_environment_variable(variable_key): + def get_environment_variable(self, variable_key): """Gets the environment variable for the variable key. + Args: variable_key: environment variable key that is used to retrieve the environment variable Return: - A environment variable for the variable key. + A environment variable for the variable key. It will retrieve the value of the environment variables in the following order: + If the key is set in the envrionment variable, Feathr will use the value of that environment variable + If it's not available in the environment variable file, Feathr will try to reterive the value from key vault Raises: ValueError: If the environment variable is not set for this key, an exception is thrown. """ password = os.environ.get(variable_key) - if not password: - logger.info(variable_key + - ' is not set in the environment variables.') - akv_name = os.environ.get("KEY_VAULT_NAME") - if akv_name: - logger.info('Fetching the value {} from Key Vault.', variable_key) - akv_client = AzureKeyVaultClient(akv_name) - password = akv_client.get_akv_secret(variable_key) - return password + if password: + return password + + # If it's not available in the environment variable file, Feathr will try to reterive the value from key vault + logger.info(variable_key + ' is not set in the environment variables.') + + if self.akv_name: + return self._get_environment_variable_from_key_vault(variable_key) + + + def _get_environment_variable_from_key_vault(self, variable_key): + """Gets the secrets from a key valut + + Args: + variable_key: environment variable key that is used to retrieve the environment variable + Return: + value from the key vault + """ + logger.info('Fetching the value {} from Key Vault {}.', variable_key, self.akv_name) + akv_client = AzureKeyVaultClient(self.akv_name) + return akv_client.get_akv_secret(variable_key) \ No newline at end of file diff --git a/feathr_project/test/prep_azure_kafka_test_data.py b/feathr_project/test/prep_azure_kafka_test_data.py index 7d4a195d3..674040be8 100644 --- a/feathr_project/test/prep_azure_kafka_test_data.py +++ b/feathr_project/test/prep_azure_kafka_test_data.py @@ -40,7 +40,8 @@ def send_avro_record_to_kafka(topic, record): bytes_writer = io.BytesIO() encoder = BinaryEncoder(bytes_writer) writer.write(record, encoder) - sasl = _EnvVaraibleUtil.get_environment_variable('KAFKA_SASL_JAAS_CONFIG') + envutils = _EnvVaraibleUtil() + sasl = envutils.get_environment_variable('KAFKA_SASL_JAAS_CONFIG') conf = { 'bootstrap.servers': KAFKA_BROKER, 'security.protocol': 'SASL_SSL', From bbf489222b82f24223a2ad25b07f9860f65ba904 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 8 Jun 2022 14:40:24 -0700 Subject: [PATCH 02/18] Create feathr-configuration-and-env.md --- .../feathr-configuration-and-env.md | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/how-to-guides/feathr-configuration-and-env.md diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md new file mode 100644 index 000000000..6f54d7b7c --- /dev/null +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -0,0 +1,68 @@ +--- +layout: default +title: Configuration and environment varialbles in Feathr +parent: Feathr How-to Guides +--- + +# Configuration and environment varialbles in Feathr + + +This file contains the configurations that are used by Feathr +All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of this config file. +For example, `feathr_runtime_location` for databricks can be overwritten by setting this environment variable: +SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION +Another example would be overwriting Redis host with this config: `ONLINE_STORE__REDIS__HOST` +For example if you want to override this setting in a shell environment: +export ONLINE_STORE__REDIS__HOST=feathrazure.redis.cache.windows.net + +# A list of environment variables that Feathr uses + +|Environment Variable | Description | +| ------------------------------- | --------------------------------------------------------------------------- | +| REDIS_PASSWORD | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | +| AZURE_CLIENT_ID | Azure SQL DB, Azure Synapse Dedicated SQL Pools, Azure SQL in VM, Snowflake | +| AZURE_TENANT_ID | Kafka, EventHub | +| AZURE_CLIENT_SECRET | Azure Cache for Redis | +| ADLS_ACCOUNT | Azure Purview | +| WASB_ACCOUNT | Azure Synapse Spark Pools, Databricks | +| WASB_KEY | Azure Machine Learning, Jupyter Notebook, Databricks Notebook | +| S3_ACCESS_KEY | Parquet, ORC, Avro, JSON, Delta Lake | +| S3_SECRET_KEY | Azure Key Vault | +| JDBC_TABLE | Azure Key Vault | + +| JDBC_USER | Azure Key Vault | +| JDBC_PASSWORD | Azure Key Vault | +| KAFKA_SASL_JAAS_CONFIG | Azure Key Vault | +|OFFLINE_STORE__ADLS__ADLS_ENABLED| | +|OFFLINE_STORE__WASB__WASB_ENABLED| | +|OFFLINE_STORE__S3__S3_ENABLED| | +|OFFLINE_STORE__S3__S3_ENDPOINT| | +|OFFLINE_STORE__SNOWFLAKE__URL| | +|OFFLINE_STORE__SNOWFLAKE__USER| | +|OFFLINE_STORE__SNOWFLAKE__ROLE| | +|OFFLINE_STORE__JDBC__JDBC_ENABLED| | +|OFFLINE_STORE__JDBC__JDBC_DATABASE| | +|OFFLINE_STORE__JDBC__JDBC_TABLE| | +|SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL| | +|SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME| | +|SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR| | +|SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE| | +|SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM| | +|SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION| | +|SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL| | +|SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE| | +|SPARK_CONFIG__DATABRICKS__WORK_DIR| | +|SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION| | +|ONLINE_STORE__REDIS__HOST| | +|ONLINE_STORE__REDIS__PORT| | +|ONLINE_STORE__REDIS__SSL_ENABLED| | +|FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME| | +|FEATURE_REGISTRY__PURVIEW__DELIMITER| | +|FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION| | +# Default behaviors + +Feathr will get the required configuration in the following order: + +1. If the key is set in the envrionment variable, Feathr will use the value of that environment variable +2. If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. +3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault. Note that usually a key vault is case sensitive. \ No newline at end of file From 036ea1b8cc03bb138552a26c45f02b96d6f1e705 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 8 Jun 2022 14:49:00 -0700 Subject: [PATCH 03/18] Update feathr-configuration-and-env.md --- .../feathr-configuration-and-env.md | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 6f54d7b7c..2e9bfe283 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -29,36 +29,40 @@ export ONLINE_STORE__REDIS__HOST=feathrazure.redis.cache.windows.net | S3_ACCESS_KEY | Parquet, ORC, Avro, JSON, Delta Lake | | S3_SECRET_KEY | Azure Key Vault | | JDBC_TABLE | Azure Key Vault | - | JDBC_USER | Azure Key Vault | | JDBC_PASSWORD | Azure Key Vault | | KAFKA_SASL_JAAS_CONFIG | Azure Key Vault | +|PROJECT_CONFIG__PROJECT_NAME| | +|PROJECT_CONFIG__REQUIRED_ENVIRONMENT_VARIABLES| | +|PROJECT_CONFIG__OPTIONAL_ENVIRONMENT_VARIABLES| | |OFFLINE_STORE__ADLS__ADLS_ENABLED| | |OFFLINE_STORE__WASB__WASB_ENABLED| | |OFFLINE_STORE__S3__S3_ENABLED| | -|OFFLINE_STORE__S3__S3_ENDPOINT| | +|OFFLINE_STORE__S3__S3_ENDPOINT| S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | |OFFLINE_STORE__SNOWFLAKE__URL| | |OFFLINE_STORE__SNOWFLAKE__USER| | |OFFLINE_STORE__SNOWFLAKE__ROLE| | |OFFLINE_STORE__JDBC__JDBC_ENABLED| | |OFFLINE_STORE__JDBC__JDBC_DATABASE| | |OFFLINE_STORE__JDBC__JDBC_TABLE| | -|SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL| | -|SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME| | -|SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR| | +|SPARK_CONFIG__SPARK_CLUSTER| choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | +|SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS| configure number of parts for the spark output for feature generation job | +|SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL| dev URL to the synapse cluster. Usually it's `https://yourclustername.dev.azuresynapse.net` | +|SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME| name of the sparkpool that you are going to use | +|SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR| workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here | |SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE| | |SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM| | |SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION| | -|SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL| | -|SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE| | -|SPARK_CONFIG__DATABRICKS__WORK_DIR| | -|SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION| | -|ONLINE_STORE__REDIS__HOST| | +|SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL| workspace instance | +|SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE| config string including run time information, spark version, machine size, etc. | +|SPARK_CONFIG__DATABRICKS__WORK_DIR| workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | +|SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION| Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with dbfs:/. this is the default location so end users don't have to compile the runtime again. | +|ONLINE_STORE__REDIS__HOST| Redis configs to access Redis cluster | |ONLINE_STORE__REDIS__PORT| | |ONLINE_STORE__REDIS__SSL_ENABLED| | -|FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME| | -|FEATURE_REGISTRY__PURVIEW__DELIMITER| | -|FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION| | +|FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME| configure the name of the purview endpoint | +|FEATURE_REGISTRY__PURVIEW__DELIMITER| delimiter indicates that how the project/workspace name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | +|FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION| controls whether the type system will be initialized or not. Usually this is only required to be executed once. | # Default behaviors Feathr will get the required configuration in the following order: From 8b209fc6dbf42cdc096efc361635ed1cd2244fcc Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 8 Jun 2022 17:27:53 -0700 Subject: [PATCH 04/18] Update feathr-configuration-and-env.md --- .../feathr-configuration-and-env.md | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 2e9bfe283..ad2837278 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -4,9 +4,19 @@ title: Configuration and environment varialbles in Feathr parent: Feathr How-to Guides --- -# Configuration and environment varialbles in Feathr +# Default behaviors + +Feathr will get the configurations in the following order: + +1. If the key is set in the envrionment variable, Feathr will use the value of that environment variable +2. If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. +3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault. Note that usually a key vault is case sensitive. +# Configuration and environment varialbles in Feathr + +Feathr uses a YAML file and a few environment variables to allow end users to have more flexibility. +See the example of the following configurations in [this file](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml): This file contains the configurations that are used by Feathr All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of this config file. For example, `feathr_runtime_location` for databricks can be overwritten by setting this environment variable: @@ -15,10 +25,14 @@ Another example would be overwriting Redis host with this config: `ONLINE_STORE_ For example if you want to override this setting in a shell environment: export ONLINE_STORE__REDIS__HOST=feathrazure.redis.cache.windows.net + # A list of environment variables that Feathr uses + + |Environment Variable | Description | | ------------------------------- | --------------------------------------------------------------------------- | +| FEATHR_KEY_VAULT_NAME | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | | REDIS_PASSWORD | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | | AZURE_CLIENT_ID | Azure SQL DB, Azure Synapse Dedicated SQL Pools, Azure SQL in VM, Snowflake | | AZURE_TENANT_ID | Kafka, EventHub | @@ -56,17 +70,10 @@ export ONLINE_STORE__REDIS__HOST=feathrazure.redis.cache.windows.net |SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL| workspace instance | |SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE| config string including run time information, spark version, machine size, etc. | |SPARK_CONFIG__DATABRICKS__WORK_DIR| workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | -|SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION| Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with dbfs:/. this is the default location so end users don't have to compile the runtime again. | +|SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION| Feathr Job configuration. Support local paths, path start with `http(s)://`, and paths start with dbfs:/. this is the default location so end users don't have to compile the runtime again. | |ONLINE_STORE__REDIS__HOST| Redis configs to access Redis cluster | |ONLINE_STORE__REDIS__PORT| | |ONLINE_STORE__REDIS__SSL_ENABLED| | |FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME| configure the name of the purview endpoint | |FEATURE_REGISTRY__PURVIEW__DELIMITER| delimiter indicates that how the project/workspace name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | |FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION| controls whether the type system will be initialized or not. Usually this is only required to be executed once. | -# Default behaviors - -Feathr will get the required configuration in the following order: - -1. If the key is set in the envrionment variable, Feathr will use the value of that environment variable -2. If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. -3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault. Note that usually a key vault is case sensitive. \ No newline at end of file From 2b2d31eae43aef2b4c312b16d3d05034122b49d7 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 10:38:21 -0700 Subject: [PATCH 05/18] update log --- feathr_project/feathr/spark_provider/_databricks_submission.py | 2 +- feathr_project/feathr/spark_provider/_synapse_submission.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/feathr_project/feathr/spark_provider/_databricks_submission.py b/feathr_project/feathr/spark_provider/_databricks_submission.py index 3eca8a3a1..f6dc82991 100644 --- a/feathr_project/feathr/spark_provider/_databricks_submission.py +++ b/feathr_project/feathr/spark_provider/_databricks_submission.py @@ -85,7 +85,7 @@ def upload_or_get_cloud_path(self, local_path_or_http_path: str): elif src_parse_result.scheme.startswith('dbfs'): # passed a cloud path logger.info( - 'Skipping file {} as the file starts with dbfs:/', local_path_or_http_path) + 'Skip uploading file {} as the file starts with dbfs:/', local_path_or_http_path) returned_path = local_path_or_http_path elif src_parse_result.scheme.startswith(('wasb','s3','gs')): # if the path starts with a location that's not a local path diff --git a/feathr_project/feathr/spark_provider/_synapse_submission.py b/feathr_project/feathr/spark_provider/_synapse_submission.py index adfa8e973..83527e50f 100644 --- a/feathr_project/feathr/spark_provider/_synapse_submission.py +++ b/feathr_project/feathr/spark_provider/_synapse_submission.py @@ -340,7 +340,7 @@ def upload_file_to_workdir(self, src_file_path: str) -> str: logger.info("{} is downloaded and then uploaded to location: {}", src_file_path, returned_path) elif src_parse_result.scheme.startswith('abfs') or src_parse_result.scheme.startswith('wasb'): # passed a cloud path - logger.info("Skipping file {} as it's already in the cloud", src_file_path) + logger.info("Skip uploading file {} as it's already in the cloud", src_file_path) returned_path = src_file_path else: # else it should be a local file path or dir From 7295c9c521ee13117ec2bb8dec4f3c0cd137b986 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 14:28:17 -0700 Subject: [PATCH 06/18] update docs --- .../feathr-configuration-and-env.md | 127 +++++++++--------- .../feathr/utils/_envvariableutil.py | 6 +- .../feathr_user_workspace/feathr_config.yaml | 5 +- 3 files changed, 72 insertions(+), 66 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index ad2837278..02e52fc98 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -4,76 +4,77 @@ title: Configuration and environment varialbles in Feathr parent: Feathr How-to Guides --- -# Default behaviors +# Configuration and environment varialbles in Feathr -Feathr will get the configurations in the following order: +Feathr uses a YAML file and a few environment variables to allow end users to have more flexibility. See the example of the following configurations in [this file](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml). -1. If the key is set in the envrionment variable, Feathr will use the value of that environment variable -2. If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. -3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault. Note that usually a key vault is case sensitive. +In that YAML file, it contains the configurations that are used by Feathr. All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of this config file. For example, `feathr_runtime_location` for databricks can be overwritten by setting this environment variable: `SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION`. For example, you can set it in python: +```python +os.environ['SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION'] = "https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-LATEST.jar" +``` -# Configuration and environment varialbles in Feathr +or in shell environment: -Feathr uses a YAML file and a few environment variables to allow end users to have more flexibility. -See the example of the following configurations in [this file](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml): -This file contains the configurations that are used by Feathr -All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of this config file. -For example, `feathr_runtime_location` for databricks can be overwritten by setting this environment variable: -SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION -Another example would be overwriting Redis host with this config: `ONLINE_STORE__REDIS__HOST` -For example if you want to override this setting in a shell environment: -export ONLINE_STORE__REDIS__HOST=feathrazure.redis.cache.windows.net +```bash +export SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION=https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-LATEST.jar +``` +This allows end users to store the configuration in a secure way, say in Kubernetes secrets, key vault, etc. All the configurations available for end users to configure are listed below. -# A list of environment variables that Feathr uses +# Default behaviors +Feathr will get the configurations in the following order: -|Environment Variable | Description | -| ------------------------------- | --------------------------------------------------------------------------- | -| FEATHR_KEY_VAULT_NAME | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | -| REDIS_PASSWORD | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | -| AZURE_CLIENT_ID | Azure SQL DB, Azure Synapse Dedicated SQL Pools, Azure SQL in VM, Snowflake | -| AZURE_TENANT_ID | Kafka, EventHub | -| AZURE_CLIENT_SECRET | Azure Cache for Redis | -| ADLS_ACCOUNT | Azure Purview | -| WASB_ACCOUNT | Azure Synapse Spark Pools, Databricks | -| WASB_KEY | Azure Machine Learning, Jupyter Notebook, Databricks Notebook | -| S3_ACCESS_KEY | Parquet, ORC, Avro, JSON, Delta Lake | -| S3_SECRET_KEY | Azure Key Vault | -| JDBC_TABLE | Azure Key Vault | -| JDBC_USER | Azure Key Vault | -| JDBC_PASSWORD | Azure Key Vault | -| KAFKA_SASL_JAAS_CONFIG | Azure Key Vault | -|PROJECT_CONFIG__PROJECT_NAME| | -|PROJECT_CONFIG__REQUIRED_ENVIRONMENT_VARIABLES| | -|PROJECT_CONFIG__OPTIONAL_ENVIRONMENT_VARIABLES| | -|OFFLINE_STORE__ADLS__ADLS_ENABLED| | -|OFFLINE_STORE__WASB__WASB_ENABLED| | -|OFFLINE_STORE__S3__S3_ENABLED| | -|OFFLINE_STORE__S3__S3_ENDPOINT| S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | -|OFFLINE_STORE__SNOWFLAKE__URL| | -|OFFLINE_STORE__SNOWFLAKE__USER| | -|OFFLINE_STORE__SNOWFLAKE__ROLE| | -|OFFLINE_STORE__JDBC__JDBC_ENABLED| | -|OFFLINE_STORE__JDBC__JDBC_DATABASE| | -|OFFLINE_STORE__JDBC__JDBC_TABLE| | -|SPARK_CONFIG__SPARK_CLUSTER| choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | -|SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS| configure number of parts for the spark output for feature generation job | -|SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL| dev URL to the synapse cluster. Usually it's `https://yourclustername.dev.azuresynapse.net` | -|SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME| name of the sparkpool that you are going to use | -|SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR| workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here | -|SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE| | -|SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM| | -|SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION| | -|SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL| workspace instance | -|SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE| config string including run time information, spark version, machine size, etc. | -|SPARK_CONFIG__DATABRICKS__WORK_DIR| workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | -|SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION| Feathr Job configuration. Support local paths, path start with `http(s)://`, and paths start with dbfs:/. this is the default location so end users don't have to compile the runtime again. | -|ONLINE_STORE__REDIS__HOST| Redis configs to access Redis cluster | -|ONLINE_STORE__REDIS__PORT| | -|ONLINE_STORE__REDIS__SSL_ENABLED| | -|FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME| configure the name of the purview endpoint | -|FEATURE_REGISTRY__PURVIEW__DELIMITER| delimiter indicates that how the project/workspace name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | -|FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION| controls whether the type system will be initialized or not. Usually this is only required to be executed once. | +1. If the key is set in the envrionment variable, Feathr will use the value of that environment variable +2. If it's not set in the environment, then a value is retrieved from from the feathr_config.yaml file with the same config key. +3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from a key vault service. Currently only Azure Key Vault is supported. + +# A list of environment variables that Feathr uses +| Environment Variable | Description | +| ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. Optional | +| REDIS_PASSWORD | Password for the Redis cluster. Required if you are using Redis. | +| AZURE_CLIENT_ID | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| AZURE_TENANT_ID |Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. EventHub | +| AZURE_CLIENT_SECRET | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| ADLS_ACCOUNT | ADLS account that you connect to. Required if using ADLS as an offline store. | +| ADLS_KEY | ADLS key that you connect to. Required if using ADLS as an offline store. | +| WASB_ACCOUNT | Azure BLOB Storage account that you connect to. Required if using Azure BLOB Storage as an offline store. | +| WASB_KEY | Azure BLOB Storage key that you connect to. Required if using Azure BLOB Storage as an offline store. | +| S3_ACCESS_KEY | Parquet, ORC, Avro, JSON, Delta Lake | +| S3_SECRET_KEY | Azure Key Vault | +| JDBC_TABLE | Azure Key Vault | +| JDBC_USER | Azure Key Vault | +| JDBC_PASSWORD | Azure Key Vault | +| KAFKA_SASL_JAAS_CONFIG | Azure Key Vault | +| PROJECT_CONFIG__PROJECT_NAME | | +| OFFLINE_STORE__ADLS__ADLS_ENABLED | | +| OFFLINE_STORE__WASB__WASB_ENABLED | | +| OFFLINE_STORE__S3__S3_ENABLED | | +| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | +| OFFLINE_STORE__SNOWFLAKE__URL | | +| OFFLINE_STORE__SNOWFLAKE__USER | | +| OFFLINE_STORE__SNOWFLAKE__ROLE | | +| OFFLINE_STORE__JDBC__JDBC_ENABLED | | +| OFFLINE_STORE__JDBC__JDBC_DATABASE | | +| OFFLINE_STORE__JDBC__JDBC_TABLE | | +| SPARK_CONFIG__SPARK_CLUSTER | choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | +| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | configure number of parts for the spark output for feature generation job | +| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | dev URL to the synapse cluster. Usually it's `https://yourclustername.dev.azuresynapse.net` | +| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the sparkpool that you are going to use | +| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here | +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | | +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | | +| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | | +| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | workspace instance | +| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | config string including run time information, spark version, machine size, etc. | +| SPARK_CONFIG__DATABRICKS__WORK_DIR | workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | +| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr Job configuration. Support local paths, path start with `http(s)://`, and paths start with dbfs:/. this is the default location so end users don't have to compile the runtime again. | +| ONLINE_STORE__REDIS__HOST | Redis configs to access Redis cluster | +| ONLINE_STORE__REDIS__PORT | | +| ONLINE_STORE__REDIS__SSL_ENABLED | | +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | configure the name of the purview endpoint | +| FEATURE_REGISTRY__PURVIEW__DELIMITER | delimiter indicates that how the project/workspace name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | controls whether the type system will be initialized or not. Usually this is only required to be executed once. | diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index c5a06625f..77be6fe67 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -7,8 +7,10 @@ class _EnvVaraibleUtil(object): def __init__(self, config_path): self.config_path = config_path - self.akv_client = None - self.akv_name = os.environ.get("FEATHR_KEY_VAULT_NAME") + # Set to none first to avoid invalid reference + self.akv_name = None + self.akv_name = self.get_environment_variable_with_default( 'secrets', 'azure_key_vault', 'name') + def get_environment_variable_with_default(self, *args): """Gets the environment variable for the variable key. diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml index ad26efe4e..c307d268a 100644 --- a/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml +++ b/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml @@ -117,4 +117,7 @@ feature_registry: # controls whether the type system will be initialized or not. Usually this is only required to be executed once. type_system_initialization: false - \ No newline at end of file + +secrets: + azure_key_vault: + name: feathrazuretest3-kv \ No newline at end of file From 27a7144aedddce316b6033de2c0254a671720c95 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 14:58:02 -0700 Subject: [PATCH 07/18] Update feathr-configuration-and-env.md --- .../feathr-configuration-and-env.md | 98 ++++++++++--------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 02e52fc98..0456a1f83 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -32,49 +32,55 @@ Feathr will get the configurations in the following order: 3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from a key vault service. Currently only Azure Key Vault is supported. # A list of environment variables that Feathr uses -| Environment Variable | Description | -| ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. Optional | -| REDIS_PASSWORD | Password for the Redis cluster. Required if you are using Redis. | -| AZURE_CLIENT_ID | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| AZURE_TENANT_ID |Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. EventHub | -| AZURE_CLIENT_SECRET | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| ADLS_ACCOUNT | ADLS account that you connect to. Required if using ADLS as an offline store. | -| ADLS_KEY | ADLS key that you connect to. Required if using ADLS as an offline store. | -| WASB_ACCOUNT | Azure BLOB Storage account that you connect to. Required if using Azure BLOB Storage as an offline store. | -| WASB_KEY | Azure BLOB Storage key that you connect to. Required if using Azure BLOB Storage as an offline store. | -| S3_ACCESS_KEY | Parquet, ORC, Avro, JSON, Delta Lake | -| S3_SECRET_KEY | Azure Key Vault | -| JDBC_TABLE | Azure Key Vault | -| JDBC_USER | Azure Key Vault | -| JDBC_PASSWORD | Azure Key Vault | -| KAFKA_SASL_JAAS_CONFIG | Azure Key Vault | -| PROJECT_CONFIG__PROJECT_NAME | | -| OFFLINE_STORE__ADLS__ADLS_ENABLED | | -| OFFLINE_STORE__WASB__WASB_ENABLED | | -| OFFLINE_STORE__S3__S3_ENABLED | | -| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | -| OFFLINE_STORE__SNOWFLAKE__URL | | -| OFFLINE_STORE__SNOWFLAKE__USER | | -| OFFLINE_STORE__SNOWFLAKE__ROLE | | -| OFFLINE_STORE__JDBC__JDBC_ENABLED | | -| OFFLINE_STORE__JDBC__JDBC_DATABASE | | -| OFFLINE_STORE__JDBC__JDBC_TABLE | | -| SPARK_CONFIG__SPARK_CLUSTER | choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | -| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | configure number of parts for the spark output for feature generation job | -| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | dev URL to the synapse cluster. Usually it's `https://yourclustername.dev.azuresynapse.net` | -| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the sparkpool that you are going to use | -| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here | -| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | | -| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | | -| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | | -| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | workspace instance | -| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | config string including run time information, spark version, machine size, etc. | -| SPARK_CONFIG__DATABRICKS__WORK_DIR | workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | -| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr Job configuration. Support local paths, path start with `http(s)://`, and paths start with dbfs:/. this is the default location so end users don't have to compile the runtime again. | -| ONLINE_STORE__REDIS__HOST | Redis configs to access Redis cluster | -| ONLINE_STORE__REDIS__PORT | | -| ONLINE_STORE__REDIS__SSL_ENABLED | | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | configure the name of the purview endpoint | -| FEATURE_REGISTRY__PURVIEW__DELIMITER | delimiter indicates that how the project/workspace name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | controls whether the type system will be initialized or not. Usually this is only required to be executed once. | +| Environment Variable | Description | +| -------------- | -------------- | +| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. Optional | +| AZURE_CLIENT_ID | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| AZURE_TENANT_ID |Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| AZURE_CLIENT_SECRET | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. | +| ADLS_ACCOUNT | ADLS account that you connect to. Required if using ADLS as an offline store. | +| ADLS_KEY | ADLS key that you connect to. Required if using ADLS as an offline store. | +| OFFLINE_STORE__WASB__WASB_ENABLED | Whether to enable Azure BLOB storage as offline store or not. | +| WASB_ACCOUNT | Azure BLOB Storage account that you connect to. Required if using Azure BLOB Storage as an offline store. | +| WASB_KEY | Azure BLOB Storage key that you connect to. Required if using Azure BLOB Storage as an offline store. | +| S3_ACCESS_KEY | AWS S3 access key for the S3 account. Required if using AWS S3 Storage as an offline store. | +| S3_SECRET_KEY | AWS S3 secret key for the S3 account. Required if using AWS S3 Storage as an offline store. | +| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. | +| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | +| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. | +| OFFLINE_STORE__JDBC__JDBC_DATABASE | If using JDBC endpoint as offline store, this config specifies the JDBC database to read from | +| OFFLINE_STORE__JDBC__JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from. Same as `JDBC_TABLE`. | +| JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from | +| JDBC_USER | If using JDBC endpoint as offline store, this config specifies the JDBC user | +| JDBC_PASSWORD | If using JDBC endpoint as offline store, this config specifies the JDBC password | +| KAFKA_SASL_JAAS_CONFIG | If using EventHub as a streaming input source, this configures the KAFKA stream. If using EventHub, read [here](https://github.com/Azure/azure-event-hubs-for-kafka#updating-your-kafka-client-configuration) for how to get this string from the existing string in Azure Portal. | +| PROJECT_CONFIG__PROJECT_NAME | Configures the project name. | +| OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. Required if using Snowflake as an offline store. | +| SPARK_CONFIG__SPARK_CLUSTER | Choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | +| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | +| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | +| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the sparkpool that you are going to use | +| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here | +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | Specifies the executor size for the Azure Synapse cluster. Currently the options are `Small`, `Medium`, `Large`. | +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Sepcifies the number of executors for the Azure Synapse cluster | +| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | +| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | Workspace instance URL for your databricks cluster. Will be something like this: `https://adb-6885802458123232.12.azuredatabricks.net/` | +| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#explanation-for-selected-configurations) for more details. | +| SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | +| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | +| ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. Required if you are using Redis. | +| ONLINE_STORE__REDIS__PORT | Redis port number to access Redis cluster. Required if you are using Redis. | +| ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. Required if you are using Redis. | +| REDIS_PASSWORD | Password for the Redis cluster. Required if you are using Redis. | +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | Configure the name of the purview endpoint. Required if using Purview as the endpoint. | +| FEATURE_REGISTRY__PURVIEW__DELIMITER | Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | + +# Explanation for selected configurations +# SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE +```json +{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","node_type_id":"Standard_D3_v2","num_workers":2,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"}},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} +``` \ No newline at end of file From f70cfacf111e6c7aff166bfabc212ca1858beaab Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 15:13:02 -0700 Subject: [PATCH 08/18] Update feathr-configuration-and-env.md --- .../feathr-configuration-and-env.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 0456a1f83..4827947b3 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -81,6 +81,22 @@ Feathr will get the configurations in the following order: # Explanation for selected configurations # SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE + +Essentially it's a compact JSON string represents the important configurations that you can configure for the databricks cluster that you use. There are parts that marked as "FEATHR_FILL_IN" that Feathr will fill in, but all the other parts are customizable. + +Essentially, the config template represents what is going to be submitted to a databricks cluster, and you can see the structure of this configuration template by visiting the [Databricks job runs API](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runs-submit): + +The most important and useful part would be the `new_cluster` section. For example, you can change`spark_version`, `node_type_id`, `num_workers`, etc. based on your environment. + ```json {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","node_type_id":"Standard_D3_v2","num_workers":2,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"}},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} -``` \ No newline at end of file +``` + +Another use case is to use `instance_pool_id`, where instead of creating the Spark cluster from scratch every time, you can reuse a pool to run the job to make the run time shorter: + +```json +{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":2,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} + +``` + +Other advanced settings includes `idempotency_token` to guarantee the idempotency of job run requests, etc. \ No newline at end of file From 4e9ca4b996b5112b5840248b906e57b9458c1516 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 15:19:18 -0700 Subject: [PATCH 09/18] Update feathr-configuration-and-env.md --- .../feathr-configuration-and-env.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 4827947b3..4e000e508 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -35,9 +35,9 @@ Feathr will get the configurations in the following order: | Environment Variable | Description | | -------------- | -------------- | | SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. Optional | -| AZURE_CLIENT_ID | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| AZURE_TENANT_ID |Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| AZURE_CLIENT_SECRET | Authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| AZURE_CLIENT_ID | Client ID for authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| AZURE_TENANT_ID |Client ID for authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | +| AZURE_CLIENT_SECRET | Client ID for authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | | OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. | | ADLS_ACCOUNT | ADLS account that you connect to. Required if using ADLS as an offline store. | | ADLS_KEY | ADLS key that you connect to. Required if using ADLS as an offline store. | @@ -68,7 +68,7 @@ Feathr will get the configurations in the following order: | SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Sepcifies the number of executors for the Azure Synapse cluster | | SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | | SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | Workspace instance URL for your databricks cluster. Will be something like this: `https://adb-6885802458123232.12.azuredatabricks.net/` | -| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#explanation-for-selected-configurations) for more details. | +| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#sparkconfigdatabricksconfigtemplate) for more details. | | SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | | SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | | ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. Required if you are using Redis. | @@ -76,11 +76,11 @@ Feathr will get the configurations in the following order: | ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. Required if you are using Redis. | | REDIS_PASSWORD | Password for the Redis cluster. Required if you are using Redis. | | FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | Configure the name of the purview endpoint. Required if using Purview as the endpoint. | -| FEATURE_REGISTRY__PURVIEW__DELIMITER | Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' | +| FEATURE_REGISTRY__PURVIEW__DELIMITER | See [here](#featureregistrypurviewdelimiter) for more details. | | FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | # Explanation for selected configurations -# SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE +## SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE Essentially it's a compact JSON string represents the important configurations that you can configure for the databricks cluster that you use. There are parts that marked as "FEATHR_FILL_IN" that Feathr will fill in, but all the other parts are customizable. @@ -99,4 +99,8 @@ Another use case is to use `instance_pool_id`, where instead of creating the Spa ``` -Other advanced settings includes `idempotency_token` to guarantee the idempotency of job run requests, etc. \ No newline at end of file +Other advanced settings includes `idempotency_token` to guarantee the idempotency of job run requests, etc. + + +## FEATURE_REGISTRY__PURVIEW__DELIMITER +Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' \ No newline at end of file From ba5cf7abe9e5634aa8eb9e22bbf0ea5ef8d73aae Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 15:20:19 -0700 Subject: [PATCH 10/18] Update feathr-configuration-and-env.md --- docs/how-to-guides/feathr-configuration-and-env.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 4e000e508..ae2ab50c9 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -103,4 +103,4 @@ Other advanced settings includes `idempotency_token` to guarantee the idempotenc ## FEATURE_REGISTRY__PURVIEW__DELIMITER -Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a globally unique name called 'foo__taxi_driver__f_daily_trips' \ No newline at end of file +Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a global unique name called 'foo__taxi_driver__f_daily_trips' \ No newline at end of file From 854639127441f76967654aa316ff9401f1ed53a7 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 15:59:57 -0700 Subject: [PATCH 11/18] Adding tests --- .../feathr-configuration-and-env.md | 1 + feathr_project/test/test_secrets_read.py | 64 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 feathr_project/test/test_secrets_read.py diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index ae2ab50c9..69ac1cda8 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -59,6 +59,7 @@ Feathr will get the configurations in the following order: | OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. Required if using Snowflake as an offline store. | | OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. Required if using Snowflake as an offline store. | | OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. Required if using Snowflake as an offline store. | +|JDBC_SF_PASSWORD| Configurations for Snowflake password| | SPARK_CONFIG__SPARK_CLUSTER | Choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | | SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | | SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py new file mode 100644 index 000000000..7c5026bb9 --- /dev/null +++ b/feathr_project/test/test_secrets_read.py @@ -0,0 +1,64 @@ +import os +from datetime import datetime +from pathlib import Path +from unittest import result + +from click.testing import CliRunner +from feathr import (BOOLEAN, FLOAT, INT32, FeatureQuery, ObservationSettings, + SparkExecutionConfiguration, TypedKey, ValueType) +from feathr.client import FeathrClient +from feathr.utils.job_utils import get_result_df + +from test_fixture import basic_test_setup +from feathr.constants import OUTPUT_FORMAT + + +# test parquet file read/write without an extension name +def test_feathr_get_secrets_from_key_vault(): + """ + Test if the program can read the key vault secrets as expected + """ + # delete ONLINE_STORE__REDIS__HOST to test get_environment_variable_with_default() + del os.environ['ONLINE_STORE__REDIS__HOST'] + # TODO: need to test get_environment_variable() as well + os.environ['SECRETS__AZURE_KEY_VAULT__NAME'] = 'feathrazuretest3-kv' + + # the config below doesn't have `ONLINE_STORE__REDIS__HOST` for testing purpose + yaml_config = """ + project_config: + project_name: 'feathr_awe_demo_get_features' + offline_store: + s3: + s3_enabled: true + s3_endpoint: 's3.amazonaws.com' + snowflake: + url: "dqllago-ol19457.snowflakecomputing.com" + user: "feathrintegration" + role: "ACCOUNTADMIN" + spark_config: + spark_cluster: 'databricks' + spark_result_output_parts: '1' + databricks: + workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' + workspace_token_value: '' + config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":2,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' + work_dir: 'dbfs:/feathr_getting_started' + feathr_runtime_location: '' + online_store: + redis: + port: 6380 + ssl_enabled: True + feature_registry: + purview: + type_system_initialization: false + purview_name: 'feathrazuretest3-purview1' + delimiter: '__' + """ + + with open("/tmp/feathr_config.yaml", "w") as text_file: + text_file.write(yaml_config) + + client = FeathrClient(config_path="/tmp/feathr_config.yaml") + # `redis_host` should be there since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault + assert client.redis_host is not None + \ No newline at end of file From b9d3b3ba422b31abfa58184de0c8349f1994b5ff Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 16:02:17 -0700 Subject: [PATCH 12/18] Update test_secrets_read.py --- feathr_project/test/test_secrets_read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index 7c5026bb9..07639fcb8 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -26,7 +26,7 @@ def test_feathr_get_secrets_from_key_vault(): # the config below doesn't have `ONLINE_STORE__REDIS__HOST` for testing purpose yaml_config = """ project_config: - project_name: 'feathr_awe_demo_get_features' + project_name: 'project_feathr_integration_test' offline_store: s3: s3_enabled: true From e6e2cbab75812f71f6c3cd294ae84e0884b55081 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 16:25:38 -0700 Subject: [PATCH 13/18] fix test --- feathr_project/feathr/secrets/akv_client.py | 4 +++- .../feathr/utils/_envvariableutil.py | 22 +++++-------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/feathr_project/feathr/secrets/akv_client.py b/feathr_project/feathr/secrets/akv_client.py index 84c0ff379..4fb7e9d23 100644 --- a/feathr_project/feathr/secrets/akv_client.py +++ b/feathr_project/feathr/secrets/akv_client.py @@ -15,8 +15,10 @@ def get_akv_secret(self, secret_name: str): credential=DefaultAzureCredential() ) try: + # replace '_' with '-' since Azure Key Vault doesn't support it + variable_replaced = secret_name.replace('_','-') + logger.info('Fetching the value {} from Key Vault {}.', variable_replaced, self.akv_name) secret = self.secret_client.get_secret(secret_name) - logger.debug(f"Secret: {secret_name} is retrieved from Key Vault {self.akv_name}.") return secret.value except ResourceNotFoundError as e: logger.error(f"Secret: {secret_name} cannot be found in Key Vault {self.akv_name}.") \ No newline at end of file diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 77be6fe67..cb14d3ed6 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -10,7 +10,7 @@ def __init__(self, config_path): # Set to none first to avoid invalid reference self.akv_name = None self.akv_name = self.get_environment_variable_with_default( 'secrets', 'azure_key_vault', 'name') - + self.akv_client = AzureKeyVaultClient(self.akv_name) if self.akv_name else None def get_environment_variable_with_default(self, *args): """Gets the environment variable for the variable key. @@ -49,12 +49,12 @@ def get_environment_variable_with_default(self, *args): except KeyError as exc: logger.info("{} not found in the config file, loading it in key vault.", env_keyword) # if not found in the config file, use key vault to get the value - return self._get_environment_variable_from_key_vault(env_keyword) if self.akv_name else "" + return self.akv_client.get_akv_secret(env_keyword) if self.akv_name else "" except yaml.YAMLError as exc: logger.info(exc) # If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault if self.akv_name: - return self._get_environment_variable_from_key_vault(env_keyword) + return self.akv_client.get_akv_secret(env_keyword) # print out warning message if cannot find the env variable in all the resources logger.warning('Environment variable {} not found.', env_keyword) @@ -81,17 +81,5 @@ def get_environment_variable(self, variable_key): logger.info(variable_key + ' is not set in the environment variables.') if self.akv_name: - return self._get_environment_variable_from_key_vault(variable_key) - - - def _get_environment_variable_from_key_vault(self, variable_key): - """Gets the secrets from a key valut - - Args: - variable_key: environment variable key that is used to retrieve the environment variable - Return: - value from the key vault - """ - logger.info('Fetching the value {} from Key Vault {}.', variable_key, self.akv_name) - akv_client = AzureKeyVaultClient(self.akv_name) - return akv_client.get_akv_secret(variable_key) \ No newline at end of file + return self.akv_client.get_akv_secret(variable_key) + \ No newline at end of file From 9fad4c6619c09d50821a5771b84b44a30f1aae79 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 17:00:10 -0700 Subject: [PATCH 14/18] Update test_secrets_read.py --- feathr_project/test/test_secrets_read.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index 07639fcb8..4106a41ff 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -18,8 +18,6 @@ def test_feathr_get_secrets_from_key_vault(): """ Test if the program can read the key vault secrets as expected """ - # delete ONLINE_STORE__REDIS__HOST to test get_environment_variable_with_default() - del os.environ['ONLINE_STORE__REDIS__HOST'] # TODO: need to test get_environment_variable() as well os.environ['SECRETS__AZURE_KEY_VAULT__NAME'] = 'feathrazuretest3-kv' From 38aa25899e39ca82284337594d353641cfe89363 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 19:13:42 -0700 Subject: [PATCH 15/18] fix test --- feathr_project/feathr/secrets/akv_client.py | 10 ++++++---- feathr_project/feathr/utils/_envvariableutil.py | 6 +++--- feathr_project/test/test_secrets_read.py | 3 +-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/feathr_project/feathr/secrets/akv_client.py b/feathr_project/feathr/secrets/akv_client.py index 4fb7e9d23..2313002f2 100644 --- a/feathr_project/feathr/secrets/akv_client.py +++ b/feathr_project/feathr/secrets/akv_client.py @@ -16,9 +16,11 @@ def get_akv_secret(self, secret_name: str): ) try: # replace '_' with '-' since Azure Key Vault doesn't support it - variable_replaced = secret_name.replace('_','-') - logger.info('Fetching the value {} from Key Vault {}.', variable_replaced, self.akv_name) - secret = self.secret_client.get_secret(secret_name) + variable_replaced = secret_name.replace('_','-') #.upper() + logger.info('Fetching the secret {} from Key Vault {}.', variable_replaced, self.akv_name) + secret = self.secret_client.get_secret(variable_replaced) + logger.info('Secret {} fetched from Key Vault {}.', variable_replaced, self.akv_name) return secret.value except ResourceNotFoundError as e: - logger.error(f"Secret: {secret_name} cannot be found in Key Vault {self.akv_name}.") \ No newline at end of file + logger.error(f"Secret {secret_name} cannot be found in Key Vault {self.akv_name}.") + raise \ No newline at end of file diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index cb14d3ed6..48229c49c 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -47,11 +47,11 @@ def get_environment_variable_with_default(self, *args): yaml_layer = yaml_layer[arg] return yaml_layer except KeyError as exc: - logger.info("{} not found in the config file, loading it in key vault.", env_keyword) + logger.warning("{} not found in the config file, loading it in key vault.", env_keyword) # if not found in the config file, use key vault to get the value - return self.akv_client.get_akv_secret(env_keyword) if self.akv_name else "" + # return self.akv_client.get_akv_secret(env_keyword) if self.akv_name else "" except yaml.YAMLError as exc: - logger.info(exc) + logger.warning(exc) # If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault if self.akv_name: return self.akv_client.get_akv_secret(env_keyword) diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index 4106a41ff..e1d14dcc6 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -12,7 +12,6 @@ from test_fixture import basic_test_setup from feathr.constants import OUTPUT_FORMAT - # test parquet file read/write without an extension name def test_feathr_get_secrets_from_key_vault(): """ @@ -59,4 +58,4 @@ def test_feathr_get_secrets_from_key_vault(): client = FeathrClient(config_path="/tmp/feathr_config.yaml") # `redis_host` should be there since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault assert client.redis_host is not None - \ No newline at end of file + From a58e0faad2823aa0d41457f23aee483cbe9333f7 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 19:23:14 -0700 Subject: [PATCH 16/18] Update feathr-configuration-and-env.md --- docs/how-to-guides/feathr-configuration-and-env.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 69ac1cda8..5f33de0d2 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -1,6 +1,6 @@ --- layout: default -title: Configuration and environment varialbles in Feathr +title: Configuration, environment varialbles, and store secrets in a secure way parent: Feathr How-to Guides --- @@ -104,4 +104,11 @@ Other advanced settings includes `idempotency_token` to guarantee the idempotenc ## FEATURE_REGISTRY__PURVIEW__DELIMITER -Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a global unique name called 'foo__taxi_driver__f_daily_trips' \ No newline at end of file +Delimiter indicates that how the project name, feature names etc. are delimited. By default it will be '__'. this is for global reference (mainly for feature sharing). For exmaple, when we setup a project called foo, and we have an anchor called 'taxi_driver' and the feature name is called 'f_daily_trips'. the feature will have a global unique name called 'foo__taxi_driver__f_daily_trips' + + +# A note on using Azure Key Vault to store credentials + +Feathr has native integrations with Azure Key Vault to make it more secure to access resources. However, Azure Key Vault doesn't support the secret name to have underscore `_` in the secret name. Feathr will automatically convert underscore `_` to dash `-`. For example, Feathr will look for `ONLINE-STORE--REDIS--HOST` in Azure Key Vault if the actual environment variable is `ONLINE_STORE__REDIS__HOST`. + +Azure Key Vault is not case sensitive, so `online_store__redis__host` and `ONLINE_STORE__REDIS__HOST` will result in the same request to Azure Key Vault and yield the same result. \ No newline at end of file From befba3995d95631c3bae6f4b1631bc2dcab9fc20 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 9 Jun 2022 20:43:25 -0700 Subject: [PATCH 17/18] fix comments and test --- feathr_project/feathr/secrets/akv_client.py | 7 ++++++- feathr_project/feathr/utils/_envvariableutil.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/feathr_project/feathr/secrets/akv_client.py b/feathr_project/feathr/secrets/akv_client.py index 2313002f2..cdec01e12 100644 --- a/feathr_project/feathr/secrets/akv_client.py +++ b/feathr_project/feathr/secrets/akv_client.py @@ -8,7 +8,12 @@ def __init__(self, akv_name: str): self.akv_name = akv_name self.secret_client = None - def get_akv_secret(self, secret_name: str): + def get_feathr_akv_secret(self, secret_name: str): + """Get Feathr Secrets from Azure Key Vault. Note that this function will replace '_' in `secret_name` with '-' since Azure Key Vault doesn't support it + + Returns: + _type_: _description_ + """ if self.secret_client is None: self.secret_client = SecretClient( vault_url = f"https://{self.akv_name}.vault.azure.net", diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 48229c49c..676644eb6 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -2,7 +2,7 @@ import yaml from loguru import logger from feathr.secrets.akv_client import AzureKeyVaultClient - +from azure.core.exceptions import ResourceNotFoundError class _EnvVaraibleUtil(object): def __init__(self, config_path): @@ -54,10 +54,12 @@ def get_environment_variable_with_default(self, *args): logger.warning(exc) # If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault if self.akv_name: - return self.akv_client.get_akv_secret(env_keyword) - - # print out warning message if cannot find the env variable in all the resources - logger.warning('Environment variable {} not found.', env_keyword) + try: + return self.akv_client.get_feathr_akv_secret(env_keyword) + except ResourceNotFoundError: + # print out warning message if cannot find the env variable in all the resources + logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', env_keyword) + return "" def get_environment_variable(self, variable_key): """Gets the environment variable for the variable key. @@ -81,5 +83,5 @@ def get_environment_variable(self, variable_key): logger.info(variable_key + ' is not set in the environment variables.') if self.akv_name: - return self.akv_client.get_akv_secret(variable_key) + return self.akv_client.get_feathr_akv_secret(variable_key) \ No newline at end of file From 4d35b88c8eafd35b77c0af8d1311836ffcfec5d2 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 11 Jun 2022 12:01:51 -0700 Subject: [PATCH 18/18] fix comments --- .../concepts/feathr-concepts-for-beginners.md | 4 +- .../feathr-configuration-and-env.md | 100 +++++++++--------- .../feathr/utils/_envvariableutil.py | 21 ++-- 3 files changed, 62 insertions(+), 63 deletions(-) diff --git a/docs/concepts/feathr-concepts-for-beginners.md b/docs/concepts/feathr-concepts-for-beginners.md index 7bb2f0530..d095b259b 100644 --- a/docs/concepts/feathr-concepts-for-beginners.md +++ b/docs/concepts/feathr-concepts-for-beginners.md @@ -107,9 +107,9 @@ client.get_offline_features(observation_settings=settings, ## What is "materialization" in Feathr? -You are very likely to train a machine learning model with the features that you just queried (with `get_offline_features()`). After you have trained a machine learning model, say a fraud detection model, you are likely to put the machine learning model into an online envrionment and do online inference. +You are very likely to train a machine learning model with the features that you just queried (with `get_offline_features()`). After you have trained a machine learning model, say a fraud detection model, you are likely to put the machine learning model into an online environment and do online inference. -In that case, you will need to retrieve the features (for example the user historical spending) in real time, since the fraud detection model is very time sensitive. Usually some key-value store is used for that scenario (for example Redis), and Feathr will help you to materialize features in the online environment for faster inference. That is why you will see something like below, where you specify Redis as the online storage you want to use, and retrieve features from online envrionment using `get_online_features()` from there: +In that case, you will need to retrieve the features (for example the user historical spending) in real time, since the fraud detection model is very time sensitive. Usually some key-value store is used for that scenario (for example Redis), and Feathr will help you to materialize features in the online environment for faster inference. That is why you will see something like below, where you specify Redis as the online storage you want to use, and retrieve features from online environment using `get_online_features()` from there: ```python redisSink = RedisSink(table_name="nycTaxiDemoFeature") diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 5f33de0d2..382932ea1 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -1,10 +1,10 @@ --- layout: default -title: Configuration, environment varialbles, and store secrets in a secure way +title: Configuration, environment variables, and store secrets in a secure way parent: Feathr How-to Guides --- -# Configuration and environment varialbles in Feathr +# Configuration and environment variables in Feathr Feathr uses a YAML file and a few environment variables to allow end users to have more flexibility. See the example of the following configurations in [this file](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml). @@ -27,58 +27,58 @@ This allows end users to store the configuration in a secure way, say in Kuberne Feathr will get the configurations in the following order: -1. If the key is set in the envrionment variable, Feathr will use the value of that environment variable -2. If it's not set in the environment, then a value is retrieved from from the feathr_config.yaml file with the same config key. +1. If the key is set in the environment variable, Feathr will use the value of that environment variable +2. If it's not set in the environment, then a value is retrieved from the feathr_config.yaml file with the same config key. 3. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from a key vault service. Currently only Azure Key Vault is supported. # A list of environment variables that Feathr uses -| Environment Variable | Description | -| -------------- | -------------- | -| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. Optional | -| AZURE_CLIENT_ID | Client ID for authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| AZURE_TENANT_ID |Client ID for authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| AZURE_CLIENT_SECRET | Client ID for authentication into Azure Services. This is required if you are using Service Principal to login with Feathr. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | -| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. | -| ADLS_ACCOUNT | ADLS account that you connect to. Required if using ADLS as an offline store. | -| ADLS_KEY | ADLS key that you connect to. Required if using ADLS as an offline store. | +| Environment Variable | Description | Required? | +| -------------- | -------------- | -------------- | +| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. | Optional | +| AZURE_CLIENT_ID | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | +| AZURE_TENANT_ID |Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. |This is required if you are using Service Principal to login with Feathr. | +| AZURE_CLIENT_SECRET | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. |This is required if you are using Service Principal to login with Feathr. | +| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. |Optional | +| ADLS_ACCOUNT | ADLS account that you connect to. |Required if using ADLS as an offline store. | +| ADLS_KEY | ADLS key that you connect to. |Required if using ADLS as an offline store. | | OFFLINE_STORE__WASB__WASB_ENABLED | Whether to enable Azure BLOB storage as offline store or not. | -| WASB_ACCOUNT | Azure BLOB Storage account that you connect to. Required if using Azure BLOB Storage as an offline store. | -| WASB_KEY | Azure BLOB Storage key that you connect to. Required if using Azure BLOB Storage as an offline store. | -| S3_ACCESS_KEY | AWS S3 access key for the S3 account. Required if using AWS S3 Storage as an offline store. | -| S3_SECRET_KEY | AWS S3 secret key for the S3 account. Required if using AWS S3 Storage as an offline store. | -| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. | -| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | -| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. | -| OFFLINE_STORE__JDBC__JDBC_DATABASE | If using JDBC endpoint as offline store, this config specifies the JDBC database to read from | -| OFFLINE_STORE__JDBC__JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from. Same as `JDBC_TABLE`. | -| JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from | -| JDBC_USER | If using JDBC endpoint as offline store, this config specifies the JDBC user | -| JDBC_PASSWORD | If using JDBC endpoint as offline store, this config specifies the JDBC password | -| KAFKA_SASL_JAAS_CONFIG | If using EventHub as a streaming input source, this configures the KAFKA stream. If using EventHub, read [here](https://github.com/Azure/azure-event-hubs-for-kafka#updating-your-kafka-client-configuration) for how to get this string from the existing string in Azure Portal. | -| PROJECT_CONFIG__PROJECT_NAME | Configures the project name. | -| OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. Required if using Snowflake as an offline store. | -| OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. Required if using Snowflake as an offline store. | -| OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. Required if using Snowflake as an offline store. | -|JDBC_SF_PASSWORD| Configurations for Snowflake password| -| SPARK_CONFIG__SPARK_CLUSTER | Choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | -| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | -| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | -| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the sparkpool that you are going to use | -| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here | -| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | Specifies the executor size for the Azure Synapse cluster. Currently the options are `Small`, `Medium`, `Large`. | -| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Sepcifies the number of executors for the Azure Synapse cluster | -| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | -| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | Workspace instance URL for your databricks cluster. Will be something like this: `https://adb-6885802458123232.12.azuredatabricks.net/` | -| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#sparkconfigdatabricksconfigtemplate) for more details. | -| SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | -| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | -| ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. Required if you are using Redis. | -| ONLINE_STORE__REDIS__PORT | Redis port number to access Redis cluster. Required if you are using Redis. | -| ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. Required if you are using Redis. | -| REDIS_PASSWORD | Password for the Redis cluster. Required if you are using Redis. | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | Configure the name of the purview endpoint. Required if using Purview as the endpoint. | -| FEATURE_REGISTRY__PURVIEW__DELIMITER | See [here](#featureregistrypurviewdelimiter) for more details. | -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | +| WASB_ACCOUNT | Azure BLOB Storage account that you connect to.| Required if using Azure BLOB Storage as an offline store. | +| WASB_KEY | Azure BLOB Storage key that you connect to. |Required if using Azure BLOB Storage as an offline store. | +| S3_ACCESS_KEY | AWS S3 access key for the S3 account. |Required if using AWS S3 Storage as an offline store. | +| S3_SECRET_KEY | AWS S3 secret key for the S3 account. |Required if using AWS S3 Storage as an offline store. | +| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. |Optional | +| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. |Required if using AWS S3 Storage as an offline store. | +| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. |Optional | +| OFFLINE_STORE__JDBC__JDBC_DATABASE | If using JDBC endpoint as offline store, this config specifies the JDBC database to read from. | Required if using JDBC sources as offline store | +| OFFLINE_STORE__JDBC__JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from. Same as `JDBC_TABLE`. |Required if using JDBC sources as offline store | +| JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from |Required if using JDBC sources as offline store | +| JDBC_USER | If using JDBC endpoint as offline store, this config specifies the JDBC user |Required if using JDBC sources as offline store | +| JDBC_PASSWORD | If using JDBC endpoint as offline store, this config specifies the JDBC password |Required if using JDBC sources as offline store | +| KAFKA_SASL_JAAS_CONFIG | If using EventHub as a streaming input source, this configures the KAFKA stream. If using EventHub, read [here](https://github.com/Azure/azure-event-hubs-for-kafka#updating-your-kafka-client-configuration) for how to get this string from the existing string in Azure Portal. | Required if using Kafka/EventHub as streaming source input.| +| PROJECT_CONFIG__PROJECT_NAME | Configures the project name. | Required| +| OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. |Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. |Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. |Required if using Snowflake as an offline store. | +|JDBC_SF_PASSWORD| Configurations for Snowflake password|Required if using Snowflake as an offline store. | +| SPARK_CONFIG__SPARK_CLUSTER | Choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | Required| +| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | Required| +| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | Required if using Azure Synapse| +| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the sparkpool that you are going to use |Required if using Azure Synapse| +| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here |Required if using Azure Synapse| +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | Specifies the executor size for the Azure Synapse cluster. Currently the options are `Small`, `Medium`, `Large`. |Required if using Azure Synapse| +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Sepcifies the number of executors for the Azure Synapse cluster |Required if using Azure Synapse| +| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). |Required if using Azure Synapse| +| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | Workspace instance URL for your databricks cluster. Will be something like this: `https://adb-6885802458123232.12.azuredatabricks.net/` |Required if using Databricks| +| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#sparkconfigdatabricksconfigtemplate) for more details. |Required if using Databricks| +| SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. |Required if using Databricks| +| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). |Required if using Databricks| +| ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. |Required if using Redis as online store. | +| ONLINE_STORE__REDIS__PORT | Redis port number to access Redis cluster. |Required if using Redis as online store. | +| ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. |Required if using Redis as online store. | +| REDIS_PASSWORD | Password for the Redis cluster. |Required if using Redis as online store. | +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | Configure the name of the purview endpoint. |Required if using Purview as the endpoint. | +| FEATURE_REGISTRY__PURVIEW__DELIMITER | See [here](#featureregistrypurviewdelimiter) for more details. | Required| +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required| # Explanation for selected configurations ## SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 676644eb6..4f93c6249 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -18,8 +18,8 @@ def get_environment_variable_with_default(self, *args): *args: list of keys in feathr_config.yaml file Return: A environment variable for the variable key. It will retrieve the value of the environment variables in the following order: - If the key is set in the envrionment variable, Feathr will use the value of that environment variable - If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. + If the key is set in the environment variable, Feathr will use the value of that environment variable + If it's not set in the environment, then a default is retrieved from the feathr_config.yaml file with the same config key. If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault """ @@ -30,11 +30,11 @@ def get_environment_variable_with_default(self, *args): env_variable = os.environ.get( env_keyword, os.environ.get(upper_env_keyword)) - # If the key is set in the envrionment variable, Feathr will use the value of that environment variable + # If the key is set in the environment variable, Feathr will use the value of that environment variable if env_variable: return env_variable - # If it's not set in the environment, then a default is retrieved from from the feathr_config.yaml file with the same config key. + # If it's not set in the environment, then a default is retrieved from the feathr_config.yaml file with the same config key. if os.path.exists(os.path.abspath(self.config_path)): with open(os.path.abspath(self.config_path), 'r') as stream: try: @@ -47,11 +47,10 @@ def get_environment_variable_with_default(self, *args): yaml_layer = yaml_layer[arg] return yaml_layer except KeyError as exc: - logger.warning("{} not found in the config file, loading it in key vault.", env_keyword) - # if not found in the config file, use key vault to get the value - # return self.akv_client.get_akv_secret(env_keyword) if self.akv_name else "" + logger.info("{} not found in the config file.", env_keyword) except yaml.YAMLError as exc: logger.warning(exc) + # If it's not available in the feathr_config.yaml file, Feathr will try to reterive the value from key vault if self.akv_name: try: @@ -69,15 +68,15 @@ def get_environment_variable(self, variable_key): variable_key: environment variable key that is used to retrieve the environment variable Return: A environment variable for the variable key. It will retrieve the value of the environment variables in the following order: - If the key is set in the envrionment variable, Feathr will use the value of that environment variable + If the key is set in the environment variable, Feathr will use the value of that environment variable If it's not available in the environment variable file, Feathr will try to reterive the value from key vault Raises: ValueError: If the environment variable is not set for this key, an exception is thrown. """ - password = os.environ.get(variable_key) + env_var_value = os.environ.get(variable_key) - if password: - return password + if env_var_value: + return env_var_value # If it's not available in the environment variable file, Feathr will try to reterive the value from key vault logger.info(variable_key + ' is not set in the environment variables.')