Skip to content

Commit 021b232

Browse files
committed
aws ci/cd
Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>
1 parent 39efe3c commit 021b232

10 files changed

Lines changed: 266 additions & 14 deletions

File tree

infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,6 @@ spec:
7171
{{- end }}
7272
{{- end }}
7373

74-
command:
75-
- python
76-
- "-m"
77-
- "feast.cli"
78-
- server
7974
ports:
8075
- name: http
8176
containerPort: {{ .Values.service.http.targetPort }}

infra/scripts/codebuild_runner.py

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
#!/usr/bin/env python
2+
3+
# This is a thin wrapper for AWS Codebuild API to kick off a build, wait for it to finish,
4+
# and tail build logs while it is running.
5+
6+
import os
7+
import json
8+
from typing import Dict, Any, List, Optional, AsyncGenerator
9+
from datetime import datetime
10+
import asyncio
11+
import sys
12+
import argparse
13+
import boto3
14+
15+
16+
class LogTailer:
17+
""" A simple cloudwatch log tailer. """
18+
19+
_next_token: Optional[str]
20+
21+
def __init__(self, client, log_group: str, log_stream: str):
22+
self._client = client
23+
self._next_token = None
24+
self._log_group = log_group
25+
self._log_stream = log_stream
26+
27+
def _get_log_events_args(self) -> Dict[str, Any]:
28+
res = dict(
29+
logGroupName=self._log_group,
30+
logStreamName=self._log_stream,
31+
limit=100,
32+
startFromHead=True,
33+
)
34+
if self._next_token:
35+
res["nextToken"] = self._next_token
36+
return res
37+
38+
async def tail_chunk(self) -> List[Dict[str, str]]:
39+
max_sleep = 5.0
40+
SLEEP_TIME = 0.5
41+
42+
while max_sleep > 0:
43+
resp = self._client.get_log_events(**self._get_log_events_args())
44+
events = resp["events"]
45+
self._next_token = resp.get("nextForwardToken")
46+
if events:
47+
return events
48+
else:
49+
max_sleep -= SLEEP_TIME
50+
await asyncio.sleep(SLEEP_TIME)
51+
else:
52+
return []
53+
54+
async def read_all_chunks(self) -> AsyncGenerator[List[Dict[str, str]], None]:
55+
while True:
56+
resp = self._client.get_log_events(**self._get_log_events_args())
57+
events = resp["events"]
58+
self._next_token = resp.get("nextForwardToken")
59+
if events:
60+
yield events
61+
else:
62+
return
63+
64+
65+
async def _wait_build_state(
66+
client, build_id, desired_phase: Optional[str], desired_states: List[str]
67+
) -> Dict[str, Any]:
68+
""" Wait until the build is in one of the desired states, or in the desired phase. """
69+
while True:
70+
resp = client.batch_get_builds(ids=[build_id])
71+
assert len(resp["builds"]) == 1
72+
build = resp["builds"][0]
73+
if build["buildStatus"] in desired_states:
74+
return build
75+
for phase in build["phases"]:
76+
if desired_phase and (phase["phaseType"] == desired_phase):
77+
return build
78+
79+
await asyncio.sleep(2)
80+
81+
82+
def print_log_event(event) -> None:
83+
print(
84+
str(datetime.fromtimestamp(event["timestamp"] / 1000.0)),
85+
event["message"],
86+
end="",
87+
)
88+
89+
90+
async def main() -> None:
91+
parser = argparse.ArgumentParser(description="Process some integers.")
92+
parser.add_argument(
93+
"--project-name", default="feast-ci-project", type=str, help="Project name"
94+
)
95+
parser.add_argument(
96+
"--source-location",
97+
type=str,
98+
help="Source location, e.g. https://github.com/feast/feast.git",
99+
)
100+
parser.add_argument(
101+
"--source-version", type=str, help="Source version, e.g. master"
102+
)
103+
parser.add_argument(
104+
"--location-from-prow", action='store_true', help="Infer source location and version from prow environment variables"
105+
)
106+
args = parser.parse_args()
107+
108+
if args.location_from_prow:
109+
job_spec = json.loads(os.getenv('JOB_SPEC', ''))
110+
source_location = job_spec['refs']['repo_link']
111+
source_version = source_version_from_prow_job_spec(job_spec)
112+
else:
113+
source_location = args.source_location
114+
source_version = args.source_version
115+
116+
await run_build(
117+
project_name=args.project_name,
118+
source_location=source_location,
119+
source_version=source_version,
120+
)
121+
122+
def source_version_from_prow_job_spec(job_spec: Dict[str, Any]) -> str:
123+
pull = job_spec['refs']['pulls'][0]
124+
return f'refs/pull/{pull["number"]}/head^{{{pull["sha"]}}}'
125+
126+
async def run_build(project_name: str, source_version: str, source_location: str):
127+
print(f"Building {project_name} at {source_version}", file=sys.stderr)
128+
logs_client = boto3.client("logs", region_name="us-west-2")
129+
codebuild_client = boto3.client("codebuild", region_name="us-west-2")
130+
131+
print("Submitting the build..", file=sys.stderr)
132+
build_resp = codebuild_client.start_build(
133+
projectName=project_name,
134+
sourceLocationOverride=source_location,
135+
sourceVersion=source_version,
136+
)
137+
138+
build_id = build_resp["build"]["id"]
139+
140+
try:
141+
print(
142+
"Waiting for the INSTALL phase to start before tailing the log",
143+
file=sys.stderr,
144+
)
145+
build = await _wait_build_state(
146+
codebuild_client,
147+
build_id,
148+
desired_phase="INSTALL",
149+
desired_states=["SUCCEEDED", "FAILED", "STOPPED", "TIMED_OUT", "FAULT"],
150+
)
151+
152+
if build["buildStatus"] != "IN_PROGRESS":
153+
print(
154+
f"Build failed before install phase: {build['buildStatus']}",
155+
file=sys.stderr,
156+
)
157+
sys.exit(1)
158+
159+
log_tailer = LogTailer(
160+
logs_client,
161+
log_stream=build["logs"]["streamName"],
162+
log_group=build["logs"]["groupName"],
163+
)
164+
165+
waiter_task = asyncio.create_task(
166+
_wait_build_state(
167+
codebuild_client,
168+
build_id,
169+
desired_phase=None,
170+
desired_states=["SUCCEEDED", "FAILED", "STOPPED", "TIMED_OUT", "FAULT"],
171+
)
172+
)
173+
174+
while not waiter_task.done():
175+
events = await log_tailer.tail_chunk()
176+
for event in events:
177+
print_log_event(event)
178+
179+
build_status = waiter_task.result()["buildStatus"]
180+
if build_status == "SUCCEEDED":
181+
print(f"Build {build_status}", file=sys.stderr)
182+
else:
183+
print(f"Build {build_status}", file=sys.stderr)
184+
sys.exit(1)
185+
except KeyboardInterrupt:
186+
print(f"Stopping build {build_id}", file=sys.stderr)
187+
codebuild_client.stop_build(id=build_id)
188+
189+
190+
if __name__ == "__main__":
191+
asyncio.run(main())

infra/scripts/setup-e2e-env-aws.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
make compile-protos-python
4+
5+
python -m pip install --upgrade pip setuptools wheel
6+
7+
python -m pip install -qr sdk/python/requirements-dev.txt
8+
python -m pip install -qr tests/requirements.txt
9+
10+
# Using mvn -q to make it less verbose. This step happens after docker containers were
11+
# succesfully built so it should be unlikely to fail.
12+
echo "########## Building ingestion jar"
13+
TIMEFORMAT='########## took %R seconds'
14+
time mvn -q --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean package
Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
11
#!/usr/bin/env bash
22

3-
aws sts get-caller-identity
3+
set -euo pipefail
4+
5+
pip install "s3fs" "boto3" "urllib3>=1.25.4"
6+
7+
export DISABLE_FEAST_SERVICE_FIXTURES=1
8+
export DISABLE_SERVICE_FIXTURES=1
9+
10+
PYTHONPATH=sdk/python pytest tests/e2e/ \
11+
--core-url cicd-feast-core:6565 \
12+
--serving-url cicd-feast-online-serving:6566 \
13+
--env aws \
14+
--emr-cluster-id $CLUSTER_ID \
15+
--staging-path $STAGING_PATH \
16+
--redis-url $NODE_IP:32379 \
17+
--emr-region us-west-2 \
18+
--kafka-brokers $NODE_IP:30092 \
19+
-m "not bq"

sdk/python/feast/pyspark/launchers/aws/emr_utils.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,11 @@ def _cancel_job(emr_client, job: EmrJobRef):
321321
else:
322322
step_id = job.step_id
323323

324-
emr_client.cancel_steps(ClusterId=job.cluster_id, StepIds=[step_id])
324+
emr_client.cancel_steps(
325+
ClusterId=job.cluster_id,
326+
StepIds=[step_id],
327+
StepCancellationOption="TERMINATE_PROCESS",
328+
)
325329

326330
_wait_for_job_state(
327331
emr_client, EmrJobRef(job.cluster_id, step_id), TERMINAL_STEP_STATES, 180

spark/ingestion/pom.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,10 @@
299299
<pattern>com.google.protobuf</pattern>
300300
<shadedPattern>com.google.protobuf.vendor</shadedPattern>
301301
</relocation>
302+
<relocation>
303+
<pattern>org.apache.kafka</pattern>
304+
<shadedPattern>org.apache.kafka.vendor</shadedPattern>
305+
</relocation>
302306
</relocations>
303307
<filters>
304308
<filter>

tests/e2e/conftest.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ def pytest_addoption(parser):
1414
parser.addoption("--staging-path", action="store")
1515
parser.addoption("--dataproc-cluster-name", action="store")
1616
parser.addoption("--dataproc-region", action="store")
17+
parser.addoption("--emr-cluster-id", action="store")
18+
parser.addoption("--emr-region", action="store")
1719
parser.addoption("--dataproc-project", action="store")
1820
parser.addoption("--ingestion-jar", action="store")
1921
parser.addoption("--redis-url", action="store", default="localhost:6379")

tests/e2e/fixtures/client.py

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import uuid
44
from typing import Optional, Tuple
55

6-
import pyspark
76
import pytest
87
from pytest_redis.executor import RedisExecutor
98

@@ -29,7 +28,9 @@ def feast_client(
2928
)
3029

3130
if pytestconfig.getoption("env") == "local":
32-
c = Client(
31+
import pyspark
32+
33+
return Client(
3334
core_url=f"{feast_core[0]}:{feast_core[1]}",
3435
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
3536
spark_launcher="standalone",
@@ -62,6 +63,22 @@ def feast_client(
6263
),
6364
**job_service_env,
6465
)
66+
elif pytestconfig.getoption("env") == "aws":
67+
return Client(
68+
core_url=f"{feast_core[0]}:{feast_core[1]}",
69+
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
70+
spark_launcher="emr",
71+
emr_cluster_id=pytestconfig.getoption("emr_cluster_id"),
72+
emr_region=pytestconfig.getoption("emr_region"),
73+
spark_staging_location=os.path.join(local_staging_path, "emr"),
74+
emr_log_location=os.path.join(local_staging_path, "emr_logs"),
75+
spark_ingestion_jar=ingestion_job_jar,
76+
redis_host=pytestconfig.getoption("redis_url").split(":")[0],
77+
redis_port=pytestconfig.getoption("redis_url").split(":")[1],
78+
historical_feature_output_location=os.path.join(
79+
local_staging_path, "historical_output"
80+
),
81+
)
6582
else:
6683
raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")
6784

tests/e2e/test_historical_features.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,18 @@ def read_parquet(uri):
2121
return pd.read_parquet(parsed_uri.path)
2222
elif parsed_uri.scheme == "gs":
2323
fs = gcsfs.GCSFileSystem()
24-
files = ["gs://" + path for path in gcsfs.GCSFileSystem().glob(uri + "/part-*")]
24+
files = ["gs://" + path for path in fs.glob(uri + "/part-*")]
25+
ds = parquet.ParquetDataset(files, filesystem=fs)
26+
return ds.read().to_pandas()
27+
elif parsed_uri.scheme == "s3":
28+
import s3fs
29+
30+
fs = s3fs.S3FileSystem()
31+
files = ["s3://" + path for path in fs.glob(uri + "/part-*")]
2532
ds = parquet.ParquetDataset(files, filesystem=fs)
2633
return ds.read().to_pandas()
2734
else:
28-
raise ValueError("Unsupported scheme")
35+
raise ValueError(f"Unsupported URL scheme {uri}")
2936

3037

3138
def generate_data():

tests/e2e/test_online_features.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,11 +133,11 @@ def test_streaming_ingestion(
133133
job = feast_client.start_stream_to_online_ingestion(feature_table)
134134

135135
wait_retry_backoff(
136-
lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 60
136+
lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 120
137137
)
138138

139139
wait_retry_backoff(
140-
lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 60
140+
lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 120
141141
)
142142

143143
try:
@@ -183,7 +183,9 @@ def ingest_and_verify(
183183
original.event_timestamp.max().to_pydatetime() + timedelta(seconds=1),
184184
)
185185

186-
wait_retry_backoff(lambda: (None, job.get_status() == SparkJobStatus.COMPLETED), 60)
186+
wait_retry_backoff(
187+
lambda: (None, job.get_status() == SparkJobStatus.COMPLETED), 180
188+
)
187189

188190
features = feast_client.get_online_features(
189191
[f"{feature_table.name}:unique_drivers"],

0 commit comments

Comments
 (0)