Skip to content

Commit e161eee

Browse files
pyalexpyalex
committed
infra
fix simple client
1 parent 0632fdf commit e161eee

53 files changed

Lines changed: 1183 additions & 365 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/complete.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ jobs:
77
runs-on: [self-hosted]
88
strategy:
99
matrix:
10-
component: [core, serving, jupyter]
10+
component: [core, serving, jc, jupyter]
1111
env:
1212
GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }}
1313
REGISTRY: gcr.io/kf-feast

‎.github/workflows/master_only.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
runs-on: [self-hosted]
1212
strategy:
1313
matrix:
14-
component: [core, serving, jupyter, ci]
14+
component: [core, serving, jc, jupyter, ci]
1515
env:
1616
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar
1717
steps:

‎Makefile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,16 @@ build-push-docker:
117117
@$(MAKE) push-core-docker registry=$(REGISTRY) version=$(VERSION)
118118
@$(MAKE) push-serving-docker registry=$(REGISTRY) version=$(VERSION)
119119
@$(MAKE) push-ci-docker registry=$(REGISTRY) version=$(VERSION)
120+
@$(MAKE) push-jc-docker registry=$(REGISTRY) version=$(VERSION)
120121

121-
build-docker: build-core-docker build-serving-docker build-ci-docker
122+
build-docker: build-core-docker build-serving-docker build-ci-docker build-jc-docker
122123

123124
push-core-docker:
124125
docker push $(REGISTRY)/feast-core:$(VERSION)
125126

127+
push-jc-docker:
128+
docker push $(REGISTRY)/feast-jc:$(VERSION)
129+
126130
push-serving-docker:
127131
docker push $(REGISTRY)/feast-serving:$(VERSION)
128132

@@ -135,6 +139,9 @@ push-jupyter-docker:
135139
build-core-docker:
136140
docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
137141

142+
build-jc-docker:
143+
docker build -t $(REGISTRY)/feast-jc:$(VERSION) -f infra/docker/jc/Dockerfile .
144+
138145
build-serving-docker:
139146
docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
140147

‎common-test/src/main/java/feast/common/it/BaseIT.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
3636
import org.springframework.test.annotation.DirtiesContext;
3737
import org.springframework.test.context.ActiveProfiles;
38-
import org.springframework.test.context.ContextConfiguration;
3938
import org.springframework.test.context.DynamicPropertyRegistry;
4039
import org.springframework.test.context.DynamicPropertySource;
4140
import org.testcontainers.containers.KafkaContainer;
@@ -51,7 +50,6 @@
5150
@ActiveProfiles("it")
5251
@Testcontainers
5352
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
54-
@ContextConfiguration
5553
public class BaseIT {
5654

5755
@Container public static PostgreSQLContainer<?> postgreSQLContainer = new PostgreSQLContainer<>();
@@ -135,6 +133,10 @@ public static void cleanTables() throws SQLException {
135133
tableNames.add(rs.getString(1));
136134
}
137135

136+
if (tableNames.isEmpty()) {
137+
return;
138+
}
139+
138140
// retries are needed since truncate require exclusive lock
139141
// and that often leads to Deadlock
140142
// since SpringApp is still running in another thread

‎common-test/src/main/java/feast/common/it/ExternalApp.java‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,92 @@
1-
package feast.common.it;public class ExternalApp {
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.it;
18+
19+
import com.google.auto.value.AutoValue;
20+
import java.util.HashMap;
21+
import java.util.Map;
22+
import org.springframework.boot.Banner;
23+
import org.springframework.boot.SpringApplication;
24+
import org.springframework.boot.builder.SpringApplicationBuilder;
25+
import org.springframework.context.ConfigurableApplicationContext;
26+
import org.springframework.core.env.MapPropertySource;
27+
import org.springframework.core.env.StandardEnvironment;
28+
import org.springframework.util.SocketUtils;
29+
import org.testcontainers.containers.PostgreSQLContainer;
30+
31+
@AutoValue
32+
public abstract class ExternalApp {
33+
private ConfigurableApplicationContext appContext;
34+
35+
abstract PostgreSQLContainer<?> getPostgreSQL();
36+
37+
abstract String getName();
38+
39+
abstract int getGRPCPort();
40+
41+
abstract int getWebPort();
42+
43+
abstract Map<String, Object> getProperties();
44+
45+
abstract Class<?> getSpringApplication();
46+
47+
public static Builder builder() {
48+
return new AutoValue_ExternalApp.Builder()
49+
.setProperties(new HashMap<>())
50+
.setWebPort(SocketUtils.findAvailableTcpPort());
51+
}
52+
53+
@AutoValue.Builder
54+
public interface Builder {
55+
Builder setSpringApplication(Class<?> app);
56+
57+
Builder setName(String name);
58+
59+
Builder setPostgreSQL(PostgreSQLContainer<?> psql);
60+
61+
Builder setGRPCPort(int port);
62+
63+
Builder setWebPort(int port);
64+
65+
Builder setProperties(Map<String, Object> properties);
66+
67+
ExternalApp build();
68+
}
69+
70+
public void start() {
71+
HashMap<String, Object> properties = new HashMap<>(getProperties());
72+
properties.put("spring.datasource.url", getPostgreSQL().getJdbcUrl());
73+
properties.put("spring.datasource.username", getPostgreSQL().getUsername());
74+
properties.put("spring.datasource.password", getPostgreSQL().getPassword());
75+
properties.put("grpc.server.port", getGRPCPort());
76+
properties.put("server.port", getWebPort());
77+
78+
StandardEnvironment env = new StandardEnvironment();
79+
env.setDefaultProfiles(getName());
80+
env.getPropertySources().addFirst(new MapPropertySource("primary", properties));
81+
82+
appContext =
83+
new SpringApplicationBuilder(getSpringApplication())
84+
.environment(env)
85+
.bannerMode(Banner.Mode.OFF)
86+
.run();
87+
}
88+
89+
public void stop() {
90+
SpringApplication.exit(appContext);
91+
}
292
}

common-test/src/main/java/feast/common/it/SimpleAPIClient.java renamed to common-test/src/main/java/feast/common/it/SimpleCoreClient.java

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@
2222
import java.util.List;
2323
import java.util.Map;
2424

25-
public class SimpleAPIClient {
25+
public class SimpleCoreClient {
2626
private CoreServiceGrpc.CoreServiceBlockingStub stub;
2727

28-
public SimpleAPIClient(CoreServiceGrpc.CoreServiceBlockingStub stub) {
28+
public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) {
2929
this.stub = stub;
3030
}
3131

@@ -127,19 +127,6 @@ public void archiveProject(String name) {
127127
stub.archiveProject(CoreServiceProto.ArchiveProjectRequest.newBuilder().setName(name).build());
128128
}
129129

130-
public void restartIngestionJob(String jobId) {
131-
stub.restartIngestionJob(
132-
CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(jobId).build());
133-
}
134-
135-
public List<IngestionJobProto.IngestionJob> listIngestionJobs() {
136-
return stub.listIngestionJobs(
137-
CoreServiceProto.ListIngestionJobsRequest.newBuilder()
138-
.setFilter(CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder().build())
139-
.build())
140-
.getJobsList();
141-
}
142-
143130
public String getFeastCoreVersion() {
144131
return stub.getFeastCoreVersion(
145132
CoreServiceProto.GetFeastCoreVersionRequest.getDefaultInstance())

‎common-test/src/main/java/feast/common/it/SimpleJCClient.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.it;
18+
19+
import feast.proto.core.CoreServiceProto;
20+
import feast.proto.core.IngestionJobProto;
21+
import feast.proto.core.JobCoordinatorServiceGrpc;
22+
import java.util.List;
23+
24+
public class SimpleJCClient {
25+
private final JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub;
26+
27+
public SimpleJCClient(JobCoordinatorServiceGrpc.JobCoordinatorServiceBlockingStub stub) {
28+
this.stub = stub;
29+
}
30+
31+
public void restartIngestionJob(String jobId) {
32+
stub.restartIngestionJob(
33+
CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(jobId).build());
34+
}
35+
36+
public List<IngestionJobProto.IngestionJob> listIngestionJobs() {
37+
return stub.listIngestionJobs(
38+
CoreServiceProto.ListIngestionJobsRequest.newBuilder()
39+
.setFilter(CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder().build())
40+
.build())
41+
.getJobsList();
42+
}
43+
}

‎core/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -305,12 +305,6 @@
305305
<version>6.1.2.Final</version>
306306
</dependency>
307307

308-
<dependency>
309-
<groupId>dev.feast</groupId>
310-
<artifactId>feast-common-test</artifactId>
311-
<version>${project.version}</version>
312-
<scope>test</scope>
313-
</dependency>
314308
<dependency>
315309
<groupId>sh.ory.keto</groupId>
316310
<artifactId>keto-client</artifactId>

‎core/src/main/java/feast/core/grpc/CoreServiceImpl.java‎

Lines changed: 0 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ public class CoreServiceImpl extends CoreServiceImplBase {
4747

4848
private final FeastProperties feastProperties;
4949
private SpecService specService;
50-
// private JobService jobService;
5150
private StatsService statsService;
5251
private ProjectService projectService;
5352
private final AuthorizationService authorizationService;
@@ -57,12 +56,10 @@ public CoreServiceImpl(
5756
SpecService specService,
5857
ProjectService projectService,
5958
StatsService statsService,
60-
// JobService jobService,
6159
FeastProperties feastProperties,
6260
AuthorizationService authorizationService) {
6361
this.specService = specService;
6462
this.projectService = projectService;
65-
// this.jobService = jobService;
6663
this.feastProperties = feastProperties;
6764
this.statsService = statsService;
6865
this.authorizationService = authorizationService;
@@ -307,72 +304,4 @@ public void listProjects(
307304
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
308305
}
309306
}
310-
311-
// @Override
312-
// public void listIngestionJobs(
313-
// ListIngestionJobsRequest request,
314-
// StreamObserver<ListIngestionJobsResponse> responseObserver) {
315-
// try {
316-
// ListIngestionJobsResponse response = this.jobService.listJobs(request);
317-
// responseObserver.onNext(response);
318-
// responseObserver.onCompleted();
319-
// } catch (InvalidArgumentException e) {
320-
// log.error("Recieved an invalid request on calling listIngestionJobs method:", e);
321-
// responseObserver.onError(
322-
// Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException());
323-
// } catch (Exception e) {
324-
// log.error("Unexpected exception on calling listIngestionJobs method:", e);
325-
// responseObserver.onError(
326-
// Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
327-
// }
328-
// }
329-
//
330-
// @Override
331-
// public void restartIngestionJob(
332-
// RestartIngestionJobRequest request,
333-
// StreamObserver<RestartIngestionJobResponse> responseObserver) {
334-
// try {
335-
// RestartIngestionJobResponse response = this.jobService.restartJob(request);
336-
// responseObserver.onNext(response);
337-
// responseObserver.onCompleted();
338-
// } catch (NoSuchElementException e) {
339-
// log.error(
340-
// "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e);
341-
// responseObserver.onError(
342-
// Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException());
343-
// } catch (UnsupportedOperationException e) {
344-
// log.error("Recieved an unsupported request on calling restartIngestionJob method:", e);
345-
// responseObserver.onError(
346-
//
347-
// Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException());
348-
// } catch (Exception e) {
349-
// log.error("Unexpected exception on calling restartIngestionJob method:", e);
350-
// responseObserver.onError(
351-
// Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
352-
// }
353-
// }
354-
355-
// @Override
356-
// public void stopIngestionJob(
357-
// StopIngestionJobRequest request, StreamObserver<StopIngestionJobResponse>
358-
// responseObserver) {
359-
// try {
360-
// StopIngestionJobResponse response = this.jobService.stopJob(request);
361-
// responseObserver.onNext(response);
362-
// responseObserver.onCompleted();
363-
// } catch (NoSuchElementException e) {
364-
// log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e);
365-
// responseObserver.onError(
366-
// Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException());
367-
// } catch (UnsupportedOperationException e) {
368-
// log.error("Recieved an unsupported request on calling stopIngestionJob method:", e);
369-
// responseObserver.onError(
370-
//
371-
// Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException());
372-
// } catch (Exception e) {
373-
// log.error("Unexpected exception on calling stopIngestionJob method:", e);
374-
// responseObserver.onError(
375-
// Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
376-
// }
377-
// }
378307
}

0 commit comments

Comments
 (0)