Skip to content

Commit 09f525c

Browse files
felixwang9817felixwang9817
authored
Upload docker image to ECR during feast apply (feast-dev#1877)
* Upload docker image to ECR during feast apply Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Change dockerhub reference and add error checking around imports Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Move logic to update_infra and add logging and error checking Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Remove feature server logic from Provider ABC Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Rename repository Signed-off-by: Felix Wang <wangfelix98@gmail.com>
1 parent a435283 commit 09f525c

6 files changed

Lines changed: 86 additions & 4 deletions

File tree

‎sdk/python/feast/constants.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,5 @@
1616

1717
# Maximum interval(secs) to wait between retries for retry function
1818
MAX_WAIT_INTERVAL: str = "60"
19+
20+
AWS_LAMBDA_FEATURE_SERVER_IMAGE = "feastdev/feature-server"

‎sdk/python/feast/errors.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,14 @@ def __init__(
210210
)
211211

212212

213+
class DockerDaemonNotRunning(Exception):
214+
def __init__(self):
215+
super().__init__(
216+
"The Docker Python sdk cannot connect to the Docker daemon. Please make sure you have"
217+
"the docker daemon installed, and that it is running."
218+
)
219+
220+
213221
class RegistryInferenceFailure(Exception):
214222
def __init__(self, repo_obj_type: str, specific_issue: str):
215223
super().__init__(

‎sdk/python/feast/infra/aws.py‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
from tempfile import TemporaryFile
66
from urllib.parse import urlparse
77

8+
from colorama import Fore, Style
9+
10+
import feast
11+
from feast.constants import AWS_LAMBDA_FEATURE_SERVER_IMAGE
812
from feast.errors import S3RegistryBucketForbiddenAccess, S3RegistryBucketNotExist
913
from feast.infra.passthrough_provider import PassthroughProvider
1014
from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
@@ -13,11 +17,66 @@
1317

1418

1519
class AwsProvider(PassthroughProvider):
16-
"""
17-
This class only exists for backwards compatibility.
18-
"""
20+
def _upload_docker_image(self) -> None:
21+
import base64
22+
23+
try:
24+
import boto3
25+
except ImportError as e:
26+
from feast.errors import FeastExtrasDependencyImportError
27+
28+
raise FeastExtrasDependencyImportError("aws", str(e))
29+
30+
try:
31+
import docker
32+
from docker.errors import APIError
33+
except ImportError as e:
34+
from feast.errors import FeastExtrasDependencyImportError
1935

20-
pass
36+
raise FeastExtrasDependencyImportError("docker", str(e))
37+
38+
try:
39+
docker_client = docker.from_env()
40+
except APIError:
41+
from feast.errors import DockerDaemonNotRunning
42+
43+
raise DockerDaemonNotRunning()
44+
45+
print(
46+
f"Pulling remote image {Style.BRIGHT + Fore.GREEN}{AWS_LAMBDA_FEATURE_SERVER_IMAGE}{Style.RESET_ALL}:"
47+
)
48+
docker_client.images.pull(AWS_LAMBDA_FEATURE_SERVER_IMAGE)
49+
50+
version = ".".join(feast.__version__.split(".")[:3])
51+
repository_name = f"feast-python-server-{version}"
52+
ecr_client = boto3.client("ecr")
53+
try:
54+
print(
55+
f"Creating remote ECR repository {Style.BRIGHT + Fore.GREEN}{repository_name}{Style.RESET_ALL}:"
56+
)
57+
response = ecr_client.create_repository(repositoryName=repository_name)
58+
repository_uri = response["repository"]["repositoryUri"]
59+
except ecr_client.exceptions.RepositoryAlreadyExistsException:
60+
response = ecr_client.describe_repositories(
61+
repositoryNames=[repository_name]
62+
)
63+
repository_uri = response["repositories"][0]["repositoryUri"]
64+
65+
auth_token = ecr_client.get_authorization_token()["authorizationData"][0][
66+
"authorizationToken"
67+
]
68+
username, password = base64.b64decode(auth_token).decode("utf-8").split(":")
69+
70+
ecr_address = repository_uri.split("/")[0]
71+
docker_client.login(username=username, password=password, registry=ecr_address)
72+
73+
image = docker_client.images.get(AWS_LAMBDA_FEATURE_SERVER_IMAGE)
74+
image_remote_name = f"{repository_uri}:{version}"
75+
print(
76+
f"Pushing local image to remote {Style.BRIGHT + Fore.GREEN}{image_remote_name}{Style.RESET_ALL}:"
77+
)
78+
image.tag(image_remote_name)
79+
docker_client.api.push(repository_uri, tag=version)
2180

2281

2382
class S3RegistryStore(RegistryStore):

‎sdk/python/feast/infra/passthrough_provider.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ def update_infra(
5252
partial=partial,
5353
)
5454

55+
if self.repo_config.feature_server and self.repo_config.feature_server.enabled:
56+
self._upload_docker_image()
57+
5558
def teardown_infra(
5659
self,
5760
project: str,
@@ -147,3 +150,7 @@ def get_historical_features(
147150
full_feature_names=full_feature_names,
148151
)
149152
return job
153+
154+
def _upload_docker_image(self) -> None:
155+
"""Upload the docker image for the feature server to the cloud."""
156+
pass

‎sdk/python/feast/infra/provider.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,11 @@ def online_read(
147147
def get_provider(config: RepoConfig, repo_path: Path) -> Provider:
148148
if "." not in config.provider:
149149
if config.provider in {"gcp", "aws", "local"}:
150+
if config.provider == "aws":
151+
from feast.infra.aws import AwsProvider
152+
153+
return AwsProvider(config)
154+
150155
from feast.infra.passthrough_provider import PassthroughProvider
151156

152157
return PassthroughProvider(config)

‎sdk/python/setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878

7979
AWS_REQUIRED = [
8080
"boto3==1.17.*",
81+
"docker>=5.0.2",
8182
]
8283

8384
CI_REQUIRED = [

0 commit comments

Comments
 (0)