diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 27b98b1086e..dc4784b3025 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -18,11 +18,10 @@ import click import pkg_resources -import toml import yaml -from feast import config as feast_config from feast.client import Client +from feast.config import Config from feast.feature_set import FeatureSet from feast.loaders.yaml import yaml_loader @@ -64,14 +63,7 @@ def version(client_only: bool, **kwargs): } if not client_only: - feast_client = Client( - core_url=feast_config.get_config_property_or_fail( - "core_url", force_config=kwargs - ), - serving_url=feast_config.get_config_property_or_fail( - "serving_url", force_config=kwargs - ), - ) + feast_client = Client(**kwargs) feast_versions_dict.update(feast_client.version()) print(json.dumps(feast_versions_dict)) @@ -94,13 +86,8 @@ def config_list(): """ List Feast properties for the currently active configuration """ - try: - feast_config_string = toml.dumps(feast_config._get_or_create_config()) - if not feast_config_string.strip(): - print("Configuration has not been set") - else: - print(feast_config_string.replace('""', "").strip()) + print(Config()) except Exception as e: _logger.error("Error occurred when reading Feast configuration file") _logger.exception(e) @@ -115,7 +102,9 @@ def config_set(prop, value): Set a Feast properties for the currently active configuration """ try: - feast_config.set_property(prop.strip(), value.strip()) + conf = Config() + conf.set(option=prop.strip(), value=value.strip()) + conf.save() except Exception as e: _logger.error("Error in reading config file") _logger.exception(e) @@ -135,9 +124,7 @@ def feature_set_list(): """ List all feature sets """ - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client + feast_client = Client() # type: Client table = [] for fs in feast_client.list_feature_sets(): @@ -161,11 +148,7 @@ def feature_set_create(filename): """ feature_sets = [FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)] - - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client - + feast_client = Client() # type: Client feast_client.apply(feature_sets) @@ -176,10 +159,7 @@ def feature_set_describe(name: str, version: int): """ Describe a feature set """ - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client - + feast_client = Client() # type: Client fs = feast_client.get_feature_set(name=name, version=version) if not fs: print( @@ -204,9 +184,7 @@ def project_create(name: str): """ Create a project """ - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client + feast_client = Client() # type: Client feast_client.create_project(name) @@ -216,9 +194,7 @@ def project_archive(name: str): """ Archive a project """ - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client + feast_client = Client() # type: Client feast_client.archive_project(name) @@ -227,9 +203,7 @@ def project_list(): """ List all projects """ - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client + feast_client = Client() # type: Client table = [] for project in feast_client.list_projects(): @@ -265,10 +239,7 @@ def ingest(name, version, filename, file_type): Ingest feature data into a feature set """ - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client - + feast_client = Client() # type: Client feature_set = feast_client.get_feature_set(name=name, version=version) feature_set.ingest_file(file_path=filename) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index ffdb71743d0..2a0b636b373 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import logging import os import shutil @@ -26,6 +27,15 @@ import pyarrow as pa import pyarrow.parquet as pq +from feast.config import Config +from feast.constants import ( + CONFIG_CORE_SECURE_KEY, + CONFIG_CORE_URL_KEY, + CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, + CONFIG_PROJECT_KEY, + CONFIG_SERVING_SECURE_KEY, + CONFIG_SERVING_URL_KEY, +) from feast.core.CoreService_pb2 import ( ApplyFeatureSetRequest, ApplyFeatureSetResponse, @@ -63,14 +73,6 @@ _logger = logging.getLogger(__name__) -GRPC_CONNECTION_TIMEOUT_DEFAULT = 3 # type: int -GRPC_CONNECTION_TIMEOUT_APPLY = 600 # type: int -FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" -FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" -FEAST_PROJECT_ENV_KEY = "FEAST_PROJECT" -FEAST_CORE_SECURE_ENV_KEY = "FEAST_CORE_SECURE" -FEAST_SERVING_SECURE_ENV_KEY = "FEAST_SERVING_SECURE" -BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300 CPU_COUNT = os.cpu_count() # type: int @@ -79,14 +81,7 @@ class Client: Feast Client: Used for creating, managing, and retrieving features. """ - def __init__( - self, - core_url: str = None, - serving_url: str = None, - project: str = None, - core_secure: bool = None, - serving_secure: bool = None, - ): + def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): """ The Feast Client should be initialized with at least one service url @@ -96,12 +91,15 @@ def __init__( project: Sets the active project. This field is optional. core_secure: Use client-side SSL/TLS for Core gRPC API serving_secure: Use client-side SSL/TLS for Serving gRPC API + options: Configuration options to initialize client with + **kwargs: Additional keyword arguments that will be used as + configuration options along with "options" """ - self._core_url: str = core_url - self._serving_url: str = serving_url - self._project: str = project - self._core_secure: bool = core_secure - self._serving_secure: bool = serving_secure + + if options is None: + options = dict() + self._config = Config(options={**options, **kwargs}) + self.__core_channel: grpc.Channel = None self.__serving_channel: grpc.Channel = None self._core_service_stub: CoreServiceStub = None @@ -115,12 +113,7 @@ def core_url(self) -> str: Returns: Feast Core URL string """ - - if self._core_url is not None: - return self._core_url - if os.getenv(FEAST_CORE_URL_ENV_KEY) is not None: - return os.getenv(FEAST_CORE_URL_ENV_KEY) - return "" + return self._config.get(CONFIG_CORE_URL_KEY) @core_url.setter def core_url(self, value: str): @@ -130,7 +123,7 @@ def core_url(self, value: str): Args: value: Feast Core URL """ - self._core_url = value + self._config.set(CONFIG_CORE_URL_KEY, value) @property def serving_url(self) -> str: @@ -140,11 +133,7 @@ def serving_url(self) -> str: Returns: Feast Serving URL string """ - if self._serving_url is not None: - return self._serving_url - if os.getenv(FEAST_SERVING_URL_ENV_KEY) is not None: - return os.getenv(FEAST_SERVING_URL_ENV_KEY) - return "" + return self._config.get(CONFIG_SERVING_URL_KEY) @serving_url.setter def serving_url(self, value: str): @@ -154,7 +143,7 @@ def serving_url(self, value: str): Args: value: Feast Serving URL """ - self._serving_url = value + self._config.set(CONFIG_SERVING_URL_KEY, value) @property def core_secure(self) -> bool: @@ -164,10 +153,7 @@ def core_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - - if self._core_secure is not None: - return self._core_secure - return os.getenv(FEAST_CORE_SECURE_ENV_KEY, "").lower() == "true" + return self._config.getboolean(CONFIG_CORE_SECURE_KEY) @core_secure.setter def core_secure(self, value: bool): @@ -177,7 +163,7 @@ def core_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._core_secure = value + self._config.set(CONFIG_CORE_SECURE_KEY, value) @property def serving_secure(self) -> bool: @@ -187,10 +173,7 @@ def serving_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - - if self._serving_secure is not None: - return self._serving_secure - return os.getenv(FEAST_SERVING_SECURE_ENV_KEY, "").lower() == "true" + return self._config.getboolean(CONFIG_SERVING_SECURE_KEY) @serving_secure.setter def serving_secure(self, value: bool): @@ -200,7 +183,7 @@ def serving_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._serving_secure = value + self._config.set(CONFIG_SERVING_SECURE_KEY, value) def version(self): """ @@ -211,14 +194,16 @@ def version(self): if self.serving_url: self._connect_serving() serving_version = self._serving_service_stub.GetFeastServingInfo( - GetFeastServingInfoRequest(), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + GetFeastServingInfoRequest(), + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ).version result["serving"] = {"url": self.serving_url, "version": serving_version} if self.core_url: self._connect_core() core_version = self._core_service_stub.GetFeastCoreVersion( - GetFeastCoreVersionRequest(), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + GetFeastCoreVersionRequest(), + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ).version result["core"] = {"url": self.core_url, "version": core_version} @@ -247,7 +232,7 @@ def _connect_core(self, skip_if_connected: bool = True): try: grpc.channel_ready_future(self.__core_channel).result( - timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY) ) except grpc.FutureTimeoutError: raise ConnectionError( @@ -281,7 +266,7 @@ def _connect_serving(self, skip_if_connected=True): try: grpc.channel_ready_future(self.__serving_channel).result( - timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY) ) except grpc.FutureTimeoutError: raise ConnectionError( @@ -299,11 +284,7 @@ def project(self) -> Union[str, None]: Returns: Project name """ - if self._project is not None: - return self._project - if os.getenv(FEAST_PROJECT_ENV_KEY) is not None: - return os.getenv(FEAST_PROJECT_ENV_KEY) - return None + return self._config.get(CONFIG_PROJECT_KEY) def set_project(self, project: str): """ @@ -312,7 +293,7 @@ def set_project(self, project: str): Args: project: Project to set as active """ - self._project = project + self._config.set(CONFIG_PROJECT_KEY, project) def list_projects(self) -> List[str]: """ @@ -324,7 +305,8 @@ def list_projects(self) -> List[str]: """ self._connect_core() response = self._core_service_stub.ListProjects( - ListProjectsRequest(), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + ListProjectsRequest(), + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ListProjectsResponse return list(response.projects) @@ -338,7 +320,8 @@ def create_project(self, project: str): self._connect_core() self._core_service_stub.CreateProject( - CreateProjectRequest(name=project), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + CreateProjectRequest(name=project), + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: CreateProjectResponse def archive_project(self, project): @@ -353,7 +336,8 @@ def archive_project(self, project): self._connect_core() self._core_service_stub.ArchiveProject( - ArchiveProjectRequest(name=project), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + ArchiveProjectRequest(name=project), + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ArchiveProjectResponse if self._project == project: @@ -402,7 +386,7 @@ def _apply_feature_set(self, feature_set: FeatureSet): try: apply_fs_response = self._core_service_stub.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set_proto), - timeout=GRPC_CONNECTION_TIMEOUT_APPLY, + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ApplyFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) @@ -573,7 +557,8 @@ def get_batch_features( # Retrieve serving information to determine store type and # staging location serving_info = self._serving_service_stub.GetFeastServingInfo( - GetFeastServingInfoRequest(), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + GetFeastServingInfoRequest(), + timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: GetFeastServingInfoResponse if serving_info.type != FeastServingType.FEAST_SERVING_TYPE_BATCH: diff --git a/sdk/python/feast/config.py b/sdk/python/feast/config.py index 061bf24c3b7..fd35b6e5d87 100644 --- a/sdk/python/feast/config.py +++ b/sdk/python/feast/config.py @@ -13,152 +13,212 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import logging import os -import sys +from configparser import ConfigParser, NoOptionError from os.path import expanduser, join -from typing import Dict -from urllib.parse import ParseResult, urlparse +from typing import Dict, Optional -import toml +from feast.constants import ( + CONFIG_FEAST_ENV_VAR_PREFIX, + CONFIG_FILE_DEFAULT_DIRECTORY, + CONFIG_FILE_NAME, + CONFIG_FILE_SECTION, + FEAST_CONFIG_FILE_ENV_KEY, +) +from feast.constants import FEAST_DEFAULT_OPTIONS as DEFAULTS _logger = logging.getLogger(__name__) -feast_configuration_properties = {"core_url": "URL", "serving_url": "URL"} -CONFIGURATION_FILE_DIR = os.environ.get("FEAST_CONFIG", ".feast") -CONFIGURATION_FILE_NAME = "config.toml" +def _init_config(path: str): + """ + Returns a ConfigParser that reads in a feast configuration file. If the + file does not exist it will be created. + Args: + path: Optional path to initialize as Feast configuration -def _get_or_create_config() -> Dict: - """Get user configuration file or create it and return""" + Returns: ConfigParser of the Feast configuration file, with defaults + preloaded - user_config_file_dir, user_config_file_path = _get_config_file_locations() - user_config_file_dir = user_config_file_dir.rstrip("/") + "/" - if not os.path.exists(os.path.dirname(user_config_file_dir)): - os.makedirs(os.path.dirname(user_config_file_dir)) + """ + # Create the configuration file directory if needed + config_dir = os.path.dirname(path) + config_dir = config_dir.rstrip("/") + "/" - if not os.path.isfile(user_config_file_path): - _save_config(user_config_file_path, _props_to_dict()) + if not os.path.exists(os.path.dirname(config_dir)): + os.makedirs(os.path.dirname(config_dir)) - try: - return toml.load(user_config_file_path) - except FileNotFoundError: - _logger.error( - "Could not find Feast configuration file " + user_config_file_path - ) - sys.exit(1) - except toml.decoder.TomlDecodeError: - _logger.error( - "Could not decode Feast configuration file " + user_config_file_path - ) - sys.exit(1) - except Exception as e: - _logger.error(e) - sys.exit(1) + # Create the configuration file itself + config = ConfigParser(defaults=DEFAULTS) + if os.path.exists(path): + config.read(path) + # Store all configuration in a single section + if not config.has_section(CONFIG_FILE_SECTION): + config.add_section(CONFIG_FILE_SECTION) -def set_property(prop: str, value: str): - """ - Sets a single property in the Feast users local configuration file + # Save the current configuration + config.write(open(path, "w")) - Args: - prop: Feast property name - value: Feast property value + return config + + +def _get_feast_env_vars(): """ - if _is_valid_property(prop, value): - active_feast_config = _get_or_create_config() - active_feast_config[prop] = value - _, user_config_file_path = _get_config_file_locations() - _save_config(user_config_file_path, active_feast_config) - print("Updated property [%s]" % prop) - else: - _logger.error("Invalid property selected") - sys.exit(1) - - -def get_config_property_or_fail(prop: str, force_config: Dict[str, str] = None) -> str: + Get environmental variables that start with FEAST_ + Returns: Dict of Feast environmental variables (stripped of prefix) """ - Gets a single property in the users configuration + feast_env_vars = {} + for key in os.environ.keys(): + if key.upper().startswith(CONFIG_FEAST_ENV_VAR_PREFIX): + feast_env_vars[key[len(CONFIG_FEAST_ENV_VAR_PREFIX) :]] = os.environ[key] + return feast_env_vars - Args: - prop: Property to retrieve - force_config: Configuration dictionary containing properties that should - be overridden. This will take precedence over file based properties. - Returns: - Returns a string property +class Config: + """ + Maintains and provides access to Feast configuration + + Configuration is stored as key/value pairs. The user can specify options + through either input arguments to this class, environmental variables, or + by setting the config in a configuration file + """ - if ( - isinstance(force_config, dict) - and prop in force_config - and force_config[prop] is not None + + def __init__( + self, options: Optional[Dict[str, str]] = None, path: Optional[str] = None, ): - return force_config[prop] + """ + Configuration options are returned as follows (higher replaces lower) + 1. Initialized options ("options" argument) + 2. Environmental variables (reloaded on every "get") + 3. Configuration file options (loaded once) + 4. Default options (loaded once from memory) + + Args: + options: (optional) A list of initialized/hardcoded options. + path: (optional) File path to configuration file + """ + if not path: + path = join( + expanduser("~"), + os.environ.get( + FEAST_CONFIG_FILE_ENV_KEY, CONFIG_FILE_DEFAULT_DIRECTORY, + ), + CONFIG_FILE_NAME, + ) + + config = _init_config(path) + + self._options = {} + if options and isinstance(options, dict): + self._options = options + + self._config = config # type: ConfigParser + self._path = path # type: str + + def get(self, option): + """ + Returns a single configuration option as a string + + Args: + option: Name of the option + + Returns: String option that is returned + + """ + return self._config.get( + CONFIG_FILE_SECTION, + option, + vars={**_get_feast_env_vars(), **self._options}, + ) - active_feast_config = _get_or_create_config() - if _is_valid_property(prop, active_feast_config[prop]): - return active_feast_config[prop] - _logger.error("Could not load Feast property from configuration: %s" % prop) - sys.exit(1) + def getboolean(self, option): + """ + Returns a single configuration option as a boolean + Args: + option: Name of the option -def _props_to_dict() -> Dict[str, str]: - """Create empty dictionary of all Feast properties""" - prop_dict = {} - for prop in feast_configuration_properties: - prop_dict[prop] = "" - return prop_dict + Returns: Boolean option value that is returned + """ + return self._config.getboolean( + CONFIG_FILE_SECTION, + option, + vars={**_get_feast_env_vars(), **self._options}, + ) -def _is_valid_property(prop: str, value: str) -> bool: - """ - Validates both a Feast property as well as value + def getint(self, option): + """ + Returns a single configuration option as an integer - Args: - prop: Feast property name - value: Feast property value + Args: + option: Name of the option - Returns: - Returns True if property and value are valid - """ - if prop not in feast_configuration_properties: - _logger.error("You are trying to set an invalid property") - sys.exit(1) + Returns: Integer option value that is returned - prop_type = feast_configuration_properties[prop] + """ + return self._config.getint( + CONFIG_FILE_SECTION, + option, + vars={**_get_feast_env_vars(), **self._options}, + ) - if prop_type == "URL": - if "//" not in value: - value = "%s%s" % ("grpc://", value) - parsed_value = urlparse(value) # type: ParseResult - if parsed_value.netloc: - return True + def getfloat(self, option): + """ + Returns a single configuration option as an integer - _logger.error("The property you are trying to set could not be identified") - sys.exit(1) + Args: + option: Name of the option + Returns: Float option value that is returned -def _save_config(user_config_file_path: str, config_string: Dict[str, str]): - """ - Saves Feast configuration + """ + return self._config.getfloat( + CONFIG_FILE_SECTION, + option, + vars={**_get_feast_env_vars(), **self._options}, + ) - Args: - user_config_file_path: Local file system path to save configuration - config_string: Contents in dictionary format to save to path - """ - try: - with open(user_config_file_path, "w+") as f: - toml.dump(config_string, f) - except Exception as e: - _logger.error("Could not update configuration file for Feast") - print(e) - sys.exit(1) - - -def _get_config_file_locations() -> (str, str): - """Gets the local user configuration directory and file path""" - user_config_file_dir = join(expanduser("~"), CONFIGURATION_FILE_DIR) - user_config_file_path = join(user_config_file_dir, CONFIGURATION_FILE_NAME) - return user_config_file_dir, user_config_file_path + def set(self, option, value): + """ + Sets a configuration option. Must be serializable to string + Args: + option: Option name to use as key + value: Value to store under option + """ + self._config.set(CONFIG_FILE_SECTION, option, value=str(value)) + + def exists(self, option): + """ + Tests whether a specific option is available + + Args: + option: Name of the option to check + + Returns: Boolean true/false whether the option is set + + """ + try: + self.get(option=option) + return True + except NoOptionError: + return False + + def save(self): + """ + Save the current configuration to disk. This does not include + environmental variables or initialized options + """ + self._config.write(open(self._path, "w")) + + def __str__(self): + result = "" + for section_name in self._config.sections(): + result += "\n[" + section_name + "]\n" + for name, value in self._config.items(section_name): + result += name + " = " + value + "\n" + return result diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 9b001ac4067..c4bde75404a 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -14,4 +14,35 @@ # limitations under the License. # -DATETIME_COLUMN = "datetime" # type: str +# General constants +DATETIME_COLUMN = "datetime" +FEAST_CONFIG_FILE_ENV_KEY = "FEAST_CONFIG" +CONFIG_FEAST_ENV_VAR_PREFIX = "FEAST_" +CONFIG_FILE_DEFAULT_DIRECTORY = ".feast" +CONFIG_FILE_NAME = "config" +CONFIG_FILE_SECTION = "general" + + +# Feast configuration options +CONFIG_CORE_URL_KEY = "core_url" +CONFIG_SERVING_URL_KEY = "serving_url" +CONFIG_PROJECT_KEY = "project" +CONFIG_CORE_SECURE_KEY = "core_secure" +CONFIG_SERVING_SECURE_KEY = "serving_secure" +CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY = "grpc_connection_timeout_default" +CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY = "grpc_connection_timeout_apply_key" +CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY = ( + "batch_feature_request_wait_time_seconds" +) + +# Configuration option default values +FEAST_DEFAULT_OPTIONS = { + CONFIG_PROJECT_KEY: "default", + CONFIG_CORE_URL_KEY: "localhost:6565", + CONFIG_CORE_SECURE_KEY: "False", + CONFIG_SERVING_URL_KEY: "localhost:6565", + CONFIG_SERVING_SECURE_KEY: "False", + CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY: "3", + CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY: "600", + CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY: "600", +} diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 1478256c066..b41500125cd 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pkgutil import tempfile from concurrent import futures diff --git a/sdk/python/tests/test_config.py b/sdk/python/tests/test_config.py new file mode 100644 index 00000000000..9ed34a736a2 --- /dev/null +++ b/sdk/python/tests/test_config.py @@ -0,0 +1,139 @@ +# Copyright 2020 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from tempfile import mkstemp + +import pytest + +from feast.config import Config + + +class TestConfig: + @pytest.fixture + def normal_config(self): + fd, path = mkstemp() + return Config(path=path) + + def test_init_config_file_with_path(self): + configuration_string = "[general]\nCORE_URL = grpc://127.0.0.1:6565" + + fd, path = mkstemp() + with open(fd, "w") as f: + f.write(configuration_string) + config = Config(path=path) + assert config.get("core_url") == "grpc://127.0.0.1:6565" + + def test_load_environmental_variable(self, normal_config): + import os + + serving_url = "http://196.25.1.1" + os.environ["FEAST_SERVING_URL"] = serving_url + assert normal_config.get("SERVING_URL") == serving_url + del os.environ["FEAST_SERVING_URL"] + + def test_env_var_not_case_sensitive(self, normal_config): + import os + + serving_url = "http://196.25.1.1" + os.environ["FEAST_SerVING_url"] = serving_url + assert normal_config.get("SERVING_URL") == serving_url + + def test_force_options(self): + fd, path = mkstemp() + options = {"feast_config_1": "one", "random_config_two": 2} + config = Config(options, path) + assert config.get("feast_config_1") == "one" + + def test_init_options_precedence(self): + """ + Init options > env var > file options > default options + """ + fd, path = mkstemp() + os.environ["FEAST_CORE_URL"] = "env" + options = {"core_url": "init", "serving_url": "init"} + configuration_string = "[general]\nCORE_URL = file\n" + with open(fd, "w") as f: + f.write(configuration_string) + config = Config(options, path) + assert config.get("core_url") == "init" + del os.environ["FEAST_CORE_URL"] + + def test_env_var_precedence(self): + """ + Env vars > file options > default options + """ + fd, path = mkstemp() + os.environ["FEAST_CORE_URL"] = "env" + configuration_string = "[general]\nCORE_URL = file\n" + with open(fd, "w") as f: + f.write(configuration_string) + config = Config(path=path) + assert config.get("CORE_URL") == "env" + + del os.environ["FEAST_CORE_URL"] + + def test_file_option_precedence(self): + """ + file options > default options + """ + fd, path = mkstemp() + configuration_string = "[general]\nCORE_URL = file\n" + with open(fd, "w") as f: + f.write(configuration_string) + config = Config(path=path) + assert config.get("CORE_URL") == "file" + + def test_default_options(self): + """ + default options + """ + fd, path = mkstemp() + config = Config(path=path) + assert config.get("CORE_URL") == "localhost:6565" + + def test_type_casting(self): + """ + Test type casting of strings to other types + """ + fd, path = mkstemp() + os.environ["FEAST_INT_VAR"] = "1" + os.environ["FEAST_FLOAT_VAR"] = "1.0" + os.environ["FEAST_BOOLEAN_VAR"] = "True" + config = Config(path=path) + + assert config.getint("INT_VAR") == 1 + assert config.getfloat("FLOAT_VAR") == 1.0 + assert config.getboolean("BOOLEAN_VAR") is True + + def test_set_value(self): + """ + Test type casting of strings to other types + """ + fd, path = mkstemp() + config = Config(path=path) + config.set("my_val", 1) + + assert config.getint("my_val") == 1 + + def test_exists(self): + """ + Test type casting of strings to other types + """ + fd, path = mkstemp() + config = Config(path=path) + config.set("my_val_exist", 1) + + assert config.exists("my_val_exist") is True + assert config.exists("my_val_not_exist") is False