Skip to content

Commit 2830854

Browse files
woopwoop
authored
Add Feast CI image and infra housekeeping (feast-dev#558)
1 parent 18235a9 commit 2830854

28 files changed

+151
-83
lines changed

‎.prow/config.yaml‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ presubmits:
6767
spec:
6868
containers:
6969
- image: maven:3.6-jdk-11
70-
command: [".prow/scripts/test-core-ingestion.sh"]
70+
command: ["infra/scripts/test-core-ingestion.sh"]
7171
resources:
7272
requests:
7373
cpu: "2000m"
@@ -81,7 +81,7 @@ presubmits:
8181
spec:
8282
containers:
8383
- image: maven:3.6-jdk-8
84-
command: [".prow/scripts/test-core-ingestion.sh"]
84+
command: ["infra/scripts/test-core-ingestion.sh"]
8585
resources:
8686
requests:
8787
cpu: "2000m"
@@ -95,7 +95,7 @@ presubmits:
9595
spec:
9696
containers:
9797
- image: maven:3.6-jdk-11
98-
command: [".prow/scripts/test-serving.sh"]
98+
command: ["infra/scripts/test-serving.sh"]
9999
skip_branches:
100100
- ^v0\.(3|4)-branch$
101101

@@ -105,7 +105,7 @@ presubmits:
105105
spec:
106106
containers:
107107
- image: maven:3.6-jdk-8
108-
command: [".prow/scripts/test-serving.sh"]
108+
command: ["infra/scripts/test-serving.sh"]
109109
branches:
110110
- ^v0\.(3|4)-branch$
111111

@@ -115,7 +115,7 @@ presubmits:
115115
spec:
116116
containers:
117117
- image: maven:3.6-jdk-11
118-
command: [".prow/scripts/test-java-sdk.sh"]
118+
command: ["infra/scripts/test-java-sdk.sh"]
119119
skip_branches:
120120
- ^v0\.(3|4)-branch$
121121

@@ -125,7 +125,7 @@ presubmits:
125125
spec:
126126
containers:
127127
- image: maven:3.6-jdk-8
128-
command: [".prow/scripts/test-java-sdk.sh"]
128+
command: ["infra/scripts/test-java-sdk.sh"]
129129
branches:
130130
- ^v0\.(3|4)-branch$
131131

@@ -135,23 +135,23 @@ presubmits:
135135
spec:
136136
containers:
137137
- image: python:3.7
138-
command: [".prow/scripts/test-python-sdk.sh"]
138+
command: ["infra/scripts/test-python-sdk.sh"]
139139

140140
- name: test-golang-sdk
141141
decorate: true
142142
always_run: true
143143
spec:
144144
containers:
145145
- image: golang:1.13
146-
command: [".prow/scripts/test-golang-sdk.sh"]
146+
command: ["infra/scripts/test-golang-sdk.sh"]
147147

148148
- name: test-end-to-end
149149
decorate: true
150150
always_run: true
151151
spec:
152152
containers:
153153
- image: maven:3.6-jdk-11
154-
command: [".prow/scripts/test-end-to-end.sh"]
154+
command: ["infra/scripts/test-end-to-end.sh"]
155155
resources:
156156
requests:
157157
cpu: "6"
@@ -165,7 +165,7 @@ presubmits:
165165
spec:
166166
containers:
167167
- image: maven:3.6-jdk-8
168-
command: [".prow/scripts/test-end-to-end.sh"]
168+
command: ["infra/scripts/test-end-to-end.sh"]
169169
resources:
170170
requests:
171171
cpu: "6"
@@ -183,7 +183,7 @@ presubmits:
183183
secretName: feast-service-account
184184
containers:
185185
- image: maven:3.6-jdk-11
186-
command: [".prow/scripts/test-end-to-end-batch.sh"]
186+
command: ["infra/scripts/test-end-to-end-batch.sh"]
187187
resources:
188188
requests:
189189
cpu: "6"
@@ -204,7 +204,7 @@ presubmits:
204204
secretName: feast-service-account
205205
containers:
206206
- image: maven:3.6-jdk-8
207-
command: [".prow/scripts/test-end-to-end-batch.sh"]
207+
command: ["infra/scripts/test-end-to-end-batch.sh"]
208208
resources:
209209
requests:
210210
cpu: "6"
@@ -226,7 +226,7 @@ postsubmits:
226226
- sh
227227
- -c
228228
- |
229-
.prow/scripts/publish-python-sdk.sh \
229+
infra/scripts/publish-python-sdk.sh \
230230
--directory-path sdk/python --repository pypi
231231
volumeMounts:
232232
- name: pypirc
@@ -250,7 +250,7 @@ postsubmits:
250250
command:
251251
- bash
252252
- -c
253-
- .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
253+
- infra/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
254254
volumeMounts:
255255
- name: gpg-keys
256256
mountPath: /etc/gpg
@@ -282,7 +282,7 @@ postsubmits:
282282
command:
283283
- bash
284284
- -c
285-
- .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
285+
- infra/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
286286
volumeMounts:
287287
- name: gpg-keys
288288
mountPath: /etc/gpg
@@ -311,19 +311,19 @@ postsubmits:
311311
- bash
312312
- -c
313313
- |
314-
.prow/scripts/download-maven-cache.sh \
314+
infra/scripts/download-maven-cache.sh \
315315
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
316316
--output-dir $PWD/
317317
318318
if [ $PULL_BASE_REF == "master" ]; then
319319
320-
.prow/scripts/publish-docker-image.sh \
320+
infra/scripts/publish-docker-image.sh \
321321
--repository gcr.io/kf-feast/feast-core \
322322
--tag dev \
323323
--file infra/docker/core/Dockerfile \
324324
--google-service-account-file /etc/gcloud/service-account.json
325325
326-
.prow/scripts/publish-docker-image.sh \
326+
infra/scripts/publish-docker-image.sh \
327327
--repository gcr.io/kf-feast/feast-serving \
328328
--tag dev \
329329
--file infra/docker/serving/Dockerfile \
@@ -337,13 +337,13 @@ postsubmits:
337337
338338
else
339339
340-
.prow/scripts/publish-docker-image.sh \
340+
infra/scripts/publish-docker-image.sh \
341341
--repository gcr.io/kf-feast/feast-core \
342342
--tag ${PULL_BASE_REF:1} \
343343
--file infra/docker/core/Dockerfile \
344344
--google-service-account-file /etc/gcloud/service-account.json
345345
346-
.prow/scripts/publish-docker-image.sh \
346+
infra/scripts/publish-docker-image.sh \
347347
--repository gcr.io/kf-feast/feast-serving \
348348
--tag ${PULL_BASE_REF:1} \
349349
--file infra/docker/serving/Dockerfile \
@@ -400,7 +400,7 @@ postsubmits:
400400
sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/Chart.yaml
401401
sed -i "/ tag: /c\ tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/values.yaml
402402
403-
.prow/scripts/sync-helm-charts.sh
403+
infra/scripts/sync-helm-charts.sh
404404
volumeMounts:
405405
- name: service-account
406406
mountPath: /etc/gcloud/service-account.json

‎Makefile‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,15 @@ protos: compile-protos-go compile-protos-python compile-protos-docs
3030

3131
build: protos build-java build-docker build-html
3232

33+
install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependencies
34+
3335
# Java
3436

37+
install-java-ci-dependencies:
38+
cd core; mvn dependency:go-offline
39+
cd serving; mvn dependency:go-offline
40+
cd ingestion; mvn dependency:go-offline
41+
3542
format-java:
3643
mvn spotless:apply
3744

@@ -70,6 +77,9 @@ lint-python:
7077

7178
# Go SDK
7279

80+
install-go-ci-dependencies:
81+
go get -u golang.org/x/lint/golint
82+
7383
compile-protos-go:
7484
@$(foreach dir,$(PROTO_TYPE_SUBDIRS), cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos/ feast/$(dir)/*.proto;)
7585

@@ -81,15 +91,34 @@ lint-go:
8191

8292
# Docker
8393

84-
build-docker:
85-
docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
86-
docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
87-
8894
build-push-docker:
8995
@$(MAKE) build-docker registry=$(REGISTRY) version=$(VERSION)
96+
@$(MAKE) push-core-docker registry=$(REGISTRY) version=$(VERSION)
97+
@$(MAKE) push-serving-docker registry=$(REGISTRY) version=$(VERSION)
98+
@$(MAKE) push-ci-docker registry=$(REGISTRY)
99+
100+
build-docker: build-core-docker build-serving-docker build-ci-docker
101+
102+
push-core-docker:
90103
docker push $(REGISTRY)/feast-core:$(VERSION)
104+
105+
push-serving-docker:
91106
docker push $(REGISTRY)/feast-serving:$(VERSION)
92107

108+
push-ci-docker:
109+
docker push $(REGISTRY)/feast-ci:latest
110+
111+
build-core-docker:
112+
docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
113+
114+
build-serving-docker:
115+
docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
116+
117+
build-ci-docker:
118+
docker build -t $(REGISTRY)/feast-ci:latest -f infra/docker/ci/Dockerfile .
119+
120+
121+
93122
# Documentation
94123

95124
install-dependencies-proto-docs:

‎infra/docker/ci/Dockerfile‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
FROM maven:3.6-jdk-11
2+
3+
ENV PYTHON_VERSION 3.7
4+
ENV GOLANG_VERSION 1.14.1
5+
6+
RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" \
7+
| tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
8+
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
9+
| apt-key --keyring /usr/share/keyrings/cloud.google.gpg \
10+
add - && apt-get update -y && apt-get install google-cloud-sdk -y
11+
12+
# Update dependencies
13+
RUN apt-get update
14+
15+
# Install Make and Python
16+
RUN apt-get install -y build-essential curl python${PYTHON_VERSION} \
17+
python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils && \
18+
update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \
19+
update-alternatives --set python /usr/bin/python${PYTHON_VERSION} && \
20+
curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
21+
python get-pip.py --force-reinstall && \
22+
rm get-pip.py
23+
24+
25+
# Install Go
26+
RUN curl -O https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz && \
27+
tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && chown -R root:root ./go && mv go /usr/local
28+
ENV GOPATH /go
29+
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
30+
31+
# Add contents of local Feast repository to image (execute from Feast root)
32+
COPY . /feast/
33+
34+
# Install all dependencies
35+
RUN cd /feast && make install-ci-dependencies

.prow/scripts/download-maven-cache.sh renamed to infra/scripts/download-maven-cache.sh

File renamed without changes.

.prow/scripts/install-google-cloud-sdk.sh renamed to infra/scripts/install-google-cloud-sdk.sh

File renamed without changes.

.prow/scripts/publish-docker-image.sh renamed to infra/scripts/publish-docker-image.sh

File renamed without changes.

.prow/scripts/publish-java-sdk.sh renamed to infra/scripts/publish-java-sdk.sh

File renamed without changes.

.prow/scripts/publish-python-sdk.sh renamed to infra/scripts/publish-python-sdk.sh

File renamed without changes.

.prow/scripts/sync-helm-charts.sh renamed to infra/scripts/sync-helm-charts.sh

File renamed without changes.

.prow/scripts/test-core-ingestion.sh renamed to infra/scripts/test-core-ingestion.sh

File renamed without changes.

0 commit comments

Comments
 (0)