Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
9e0e7c7
Add support for DynamoDB and S3 registry
leonid133 Apr 27, 2021
44aadd8
rcu and wcu as a parameter of dynamodb online store
leonid133 Apr 27, 2021
6383791
fix linter
leonid133 May 4, 2021
73ff67a
aws dependency to extras
leonid133 May 18, 2021
aa6d0da
FEAST_S3_ENDPOINT_URL
leonid133 May 18, 2021
0a87050
tests
leonid133 May 18, 2021
3b8bb31
merge from master
leonid133 May 18, 2021
00e8675
fix signature, after merge
leonid133 May 18, 2021
6a99cd9
aws default region name configurable
leonid133 May 18, 2021
32dc799
merge from master
leonid133 Jun 11, 2021
db616c4
add offlinestore config type to test
leonid133 Jun 11, 2021
8dcbd5a
review changes
leonid133 Jun 11, 2021
fee93dd
merge from master
leonid133 Jun 18, 2021
2bbe268
Merge branch 'master' of https://github.com/feast-dev/feast into feat…
leonid133 Jun 18, 2021
5d33a79
Merge branch 'master' of https://github.com/feast-dev/feast into feat…
leonid133 Jun 18, 2021
24c44ee
merge latest from master
leonid133 Jun 23, 2021
7b99cde
review requested changes
leonid133 Jun 23, 2021
3a985b0
integration test for Dynamo
leonid133 Jun 23, 2021
6973581
change the rest of table_name to table_instance (where table_name is …
leonid133 Jun 28, 2021
e928424
fix DynamoDBOnlineStore commit
leonid133 Jun 28, 2021
59d7e4c
move client to _initialize_dynamodb
leonid133 Jun 28, 2021
594b932
rename document_id to entity_id and Row to entity_id
leonid133 Jun 28, 2021
15a787c
The default value is None
leonid133 Jun 28, 2021
7eaa654
Remove Datastore from the docstring.
leonid133 Jun 28, 2021
1468117
get rid of the return call from S3RegistryStore
leonid133 Jun 28, 2021
5dbe429
merge two exceptions
leonid133 Jun 29, 2021
986d45e
For ci requirement
leonid133 Jun 29, 2021
79d85c7
remove configuration from test
leonid133 Jun 29, 2021
f50b2fb
feast-integration-tests for tests
leonid133 Jun 29, 2021
509c521
change test path
leonid133 Jun 29, 2021
cd67973
add fixture feature_store_with_s3_registry to test
leonid133 Jun 29, 2021
5466d20
merge from master
leonid133 Jun 29, 2021
3d1b78c
region required
leonid133 Jun 29, 2021
ff8d635
Merge branch 'master' of https://github.com/feast-dev/feast into feat…
leonid133 Jun 29, 2021
57a607c
Address the rest of the comments
Jul 2, 2021
e9422ea
Merge branch 'master' into feature/online_dynamodb
Jul 2, 2021
3cd9597
Update to_table to to_arrow
Jul 2, 2021
124b337
Merge branch 'master' into feature/online_dynamodb
Jul 3, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add offlinestore config type to test
Signed-off-by: lblokhin <lenin133@yandex.ru>
  • Loading branch information
leonid133 committed Jun 11, 2021
commit db616c46a3882327990811dbcd00df0bec6dbfe5
7 changes: 4 additions & 3 deletions sdk/python/feast/infra/aws_dynamodb_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from feast.entity import Entity
from feast.feature_view import FeatureView
from feast.infra.key_encoding_utils import serialize_entity_key
from feast.infra.offline_stores.helpers import get_offline_store_from_sources
from feast.infra.offline_stores.helpers import get_offline_store_from_config
from feast.infra.provider import (
Provider,
RetrievalJob,
Expand Down Expand Up @@ -48,6 +48,8 @@ def __init__(self, config: RepoConfig):
else:
self.region_name = os.environ.get("AWS_DEFAULT_REGION", "us-west-2")
Comment thread
leonid133 marked this conversation as resolved.
Outdated
os.environ["AWS_DEFAULT_REGION"] = self.region_name
assert config.offline_store is not None
self.offline_store = get_offline_store_from_config(config.offline_store)

def _initialize_dynamodb(self):
return boto3.resource("dynamodb", region_name=self.region_name)
Expand Down Expand Up @@ -184,8 +186,7 @@ def materialize_single_feature_view(
start_date = utils.make_tzaware(start_date)
end_date = utils.make_tzaware(end_date)

offline_store = get_offline_store_from_sources([feature_view.input])
table = offline_store.pull_latest_from_table_or_query(
table = self.offline_store.pull_latest_from_table_or_query(
data_source=feature_view.input,
join_key_columns=join_key_columns,
feature_name_columns=feature_name_columns,
Expand Down
2 changes: 2 additions & 0 deletions sdk/python/feast/repo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ def _validate_offline_store_config(cls, values):
if "type" not in values["offline_store"]:
if values["provider"] == "local" or values["provider"] == "redis":
values["offline_store"]["type"] = "file"
elif values["provider"] == "aws_dynamodb":
values["offline_store"]["type"] = "bigquery"
elif values["provider"] == "gcp":
values["offline_store"]["type"] = "bigquery"

Expand Down
10 changes: 6 additions & 4 deletions sdk/python/tests/test_cli_aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@ def test_basic() -> None:
repo_config.write_text(
dedent(
f"""
project: {project_id}
registry: {data_path / "registry.db"}
provider: aws_dynamodb
"""
project: {project_id}
registry: {data_path / "registry.db"}
provider: aws_dynamodb
offline_store:
type: file
"""
)
)

Expand Down