Skip to content

Add support for Redis and Redis Cluster #1511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 22 commits into from
Jun 9, 2021
Merged

Add support for Redis and Redis Cluster #1511

Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
d97aaea
Add support for Redis as online store
qooba Apr 25, 2021
ae9ed8e
Add support for Redis as online store
qooba Apr 25, 2021
e165f00
Add redis provider
qooba Apr 26, 2021
df8f3cb
Add redis online provider
qooba Apr 27, 2021
6bea80d
Add redis online provider
qooba Apr 27, 2021
5745ff7
Add redis online provider
qooba Apr 27, 2021
a3c4f3b
Add redis online provider
qooba Apr 28, 2021
e434e59
Add redis online provider - integration tests
qooba May 6, 2021
8ccfc6b
Add redis online provider - integration tests
qooba May 6, 2021
be9a17f
Add redis provider
qooba May 12, 2021
7606fe7
Add redis provider
qooba May 12, 2021
2a95e3e
correct redis provider
qooba May 24, 2021
435c856
correct redis provider
qooba May 26, 2021
d329439
correct redis provider test
qooba May 26, 2021
b5269e7
correct redis provider
qooba May 30, 2021
f602b9a
correct redis provider
qooba May 30, 2021
bbde0b0
change redis connection string
qooba Jun 2, 2021
d86f23c
change redis connection string
qooba Jun 2, 2021
b0c8101
fix redis materialize
qooba Jun 7, 2021
52232cc
Rename connection string and set defaults
woop Jun 8, 2021
53f8f72
Fix argument in redis test
woop Jun 9, 2021
e12ecff
Fix broken connection string
woop Jun 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add redis online provider 
Signed-off-by: qooba <dev@qooba.net>
  • Loading branch information
@qooba @woop
qooba authored and woop committed Jun 9, 2021
commit 6bea80d1a36ba9e2c91d5cade1030a7021a08b0f
24 changes: 14 additions & 10 deletions sdk/python/feast/infra/redis_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ def teardown_infra(
) -> None:
# according to the repos_operations.py we can delete the whole project
client = self._get_client()
#keys = client.keys("{project}:*")
#client.unlink(*keys)
keys = client.keys("*{project}:*")
client.unlink(*keys)

def online_write_batch(
self,
Expand Down Expand Up @@ -108,15 +108,19 @@ def online_read(
ts_key = f"_ts:{feature_view}"
hset_keys.append(ts_key)
values = client.hmget(redis_key_bin, hset_keys)

requested_features.append(ts_key)
res_val = dict(zip(requested_features, values))
res_ts = res_val.pop(ts_key)

res_ts = Timestamp()
ts_val = res_val.pop(ts_key)
if ts_val:
res_ts.ParseFromString(ts_val)

res = {}
for feature_name, val_bin in res_val.items():
val = ValueProto()
val.FromString(val_bin)
if val_bin:
val.ParseFromString(val_bin)
res[feature_name] = val

if not res:
Expand Down Expand Up @@ -235,10 +239,10 @@ def get_historical_features(

def _redis_key(project: str, entity_key: EntityKeyProto) -> str:
redis_key = RedisKeyProto(
project=project,
entity_names=entity_key.join_keys,
entity_values=entity_key.entity_values,
)
project=project,
entity_names=entity_key.join_keys,
entity_values=entity_key.entity_values,
)
#key = _mmh3(serialize_entity_key(entity_key))
return redis_key.SerializeToString()

Expand All @@ -249,7 +253,7 @@ def _mmh3(key: str) -> str:
https://stackoverflow.com/questions/13141787/convert-decimal-int-to-little-endian-string-x-x
"""
key_hash = mmh3.hash(key,signed=False)
bytes.fromhex(struct.pack('<Q', key_hash).hex().rstrip('0'))
return bytes.fromhex(struct.pack('<Q', key_hash).hex().rstrip('0'))