Skip to content

Read and write path for Datastore and SQLite #1376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 6 commits into from
Mar 12, 2021
Merged

Read and write path for Datastore and SQLite #1376

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
221e01e
local read-write path
oavdeev Mar 9, 2021
f0c9a6a
commas
oavdeev Mar 11, 2021
d196a5e
bump python version
oavdeev Mar 11, 2021
2820f4d
fix dockerfile
oavdeev Mar 11, 2021
c99f12f
fixing tests
oavdeev Mar 11, 2021
c2ee468
don't use fancy sqlite upsert as it is not in python 3.7.5
oavdeev Mar 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
don't use fancy sqlite upsert as it is not in python 3.7.5 
Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>
  • Loading branch information
@oavdeev
oavdeev committed Mar 11, 2021
commit c2ee4684d995f2a45b0be95b6e128140b7493cc6
37 changes: 20 additions & 17 deletions sdk/python/feast/infra/local_sqlite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import os
import sqlite3
import sys
from datetime import datetime
from typing import Dict, List, Optional, Tuple

Expand All @@ -21,11 +20,6 @@ class LocalSqlite(Provider):

def __init__(self, config: LocalOnlineStoreConfig):
self._db_path = config.path
sqlite_ver = tuple(map(int, sqlite3.sqlite_version.split(".")))
if not sqlite_ver >= (3, 24, 0):
raise Exception(
f"SQLite version >= 3.24.0 reqiured (got {sqlite3.sqlite_version}, Python {sys.version}"
)

def _get_conn(self):
return sqlite3.connect(
Expand Down Expand Up @@ -66,21 +60,15 @@ def online_write_batch(
for entity_key, values, timestamp in data:
for feature_name, val in values.items():
entity_key_bin = serialize_entity_key(entity_key)

conn.execute(
f"""INSERT INTO {_table_id(project, table)}
(entity_key, feature_name, value, event_ts, created_ts)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT (entity_key, feature_name) DO UPDATE
f"""
UPDATE {_table_id(project, table)}
SET value = ?, event_ts = ?, created_ts = ?
WHERE event_ts < ? OR (event_ts = ? AND created_ts < ?)
WHERE (event_ts < ? OR (event_ts = ? AND created_ts < ?))
AND (entity_key = ? AND feature_name = ?)
""",
(
# INSERT
entity_key_bin,
feature_name,
val.SerializeToString(),
timestamp,
created_ts,
# SET
val.SerializeToString(),
timestamp,
Expand All @@ -89,6 +77,21 @@ def online_write_batch(
timestamp,
timestamp,
created_ts,
entity_key_bin,
feature_name,
),
)

conn.execute(
f"""INSERT OR IGNORE INTO {_table_id(project, table)}
(entity_key, feature_name, value, event_ts, created_ts)
VALUES (?, ?, ?, ?, ?)""",
(
entity_key_bin,
feature_name,
val.SerializeToString(),
timestamp,
created_ts,
),
)

Expand Down
2 changes: 1 addition & 1 deletion sdk/python/requirements-ci.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
numpy==1.19
cryptography==3.3.2
flake8
black==19.10b0
Expand All @@ -18,7 +19,6 @@ pytest==6.0.0
pytest-lazy-fixture==0.6.3
pytest-timeout==1.4.2
pytest-ordering==0.6.*
numpy==1.19
pytest-mock==1.10.4
PyYAML==5.3.1
great-expectations==0.13.2
Expand Down