Skip to content

Add job-controller to test containers #1006

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 1 commit into from
Sep 16, 2020
Merged

Add job-controller to test containers #1006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion serving/src/test/java/feast/serving/it/BaseAuthIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class BaseAuthIT {
static final String FEATURE_NAME = "feature_1";
static final String ENTITY_ID = "entity_id";
static final String PROJECT_NAME = "project_1";
static final int CORE_START_MAX_WAIT_TIME_IN_MINUTES = 3;
static final int SERVICE_START_MAX_WAIT_TIME_IN_MINUTES = 3;
static final String CLIENT_ID = "client_id";
static final String CLIENT_SECRET = "client_secret";
static final String TOKEN_URL = "http://localhost:4444/oauth2/token";
Expand All @@ -42,6 +42,8 @@ public class BaseAuthIT {

static final String CORE = "core_1";

static final String JOB_CONTROLLER = "jobcontroller_1";

static final String HYDRA = "hydra_1";
static final int HYDRA_PORT = 4445;

Expand All @@ -51,6 +53,8 @@ public class BaseAuthIT {

static final int FEAST_CORE_PORT = 6565;

static final int FEAST_JOB_CONTROLLER_PORT = 6570;

@DynamicPropertySource
static void properties(DynamicPropertyRegistry registry) {
registry.add("feast.stores[0].name", () -> "online");
Expand Down
11 changes: 8 additions & 3 deletions serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,18 @@ public class ServingServiceOauthAuthenticationIT extends BaseAuthIT {
public static DockerComposeContainer environment =
new DockerComposeContainer(
new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"),
new File("src/test/resources/docker-compose/docker-compose-it-core.yml"))
new File("src/test/resources/docker-compose/docker-compose-it.yml"))
.withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200))
.withExposedService(
CORE,
6565,
FEAST_CORE_PORT,
Wait.forLogMessage(".*gRPC Server started.*\\n", 1)
.withStartupTimeout(Duration.ofMinutes(CORE_START_MAX_WAIT_TIME_IN_MINUTES)));
.withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES)))
.withExposedService(
JOB_CONTROLLER,
FEAST_JOB_CONTROLLER_PORT,
Wait.forLogMessage(".*gRPC Server started.*\\n", 1)
.withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES)));

@BeforeAll
static void globalSetup() throws IOException, InitializationError, InterruptedException {
Expand Down
13 changes: 9 additions & 4 deletions serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,16 +72,21 @@ public class ServingServiceOauthAuthorizationIT extends BaseAuthIT {
public static DockerComposeContainer environment =
new DockerComposeContainer(
new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"),
new File("src/test/resources/docker-compose/docker-compose-it-core.yml"),
new File("src/test/resources/docker-compose/docker-compose-it.yml"),
new File("src/test/resources/docker-compose/docker-compose-it-keto.yml"))
.withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200))
.withExposedService(
CORE,
6565,
FEAST_CORE_PORT,
Wait.forLogMessage(".*gRPC Server started.*\\n", 1)
.withStartupTimeout(Duration.ofMinutes(CORE_START_MAX_WAIT_TIME_IN_MINUTES)))
.withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES)))
.withExposedService(
JOB_CONTROLLER,
FEAST_JOB_CONTROLLER_PORT,
Wait.forLogMessage(".*gRPC Server started.*\\n", 1)
.withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES)))
.withExposedService("adaptor_1", KETO_ADAPTOR_PORT)
.withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200));;
.withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200));

@DynamicPropertySource
static void initialize(DynamicPropertyRegistry registry) {
Expand Down
8 changes: 0 additions & 8 deletions serving/src/test/resources/docker-compose/core/application-it.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
feast:
jobs:
polling_interval_milliseconds: 30000
job_update_timeout_seconds: 240
active_runner: direct
runners:
- name: direct
type: DirectRunner
options: {}
stream:
type: kafka
options:
Expand Down
14 changes: 14 additions & 0 deletions ...docker-compose/docker-compose-it-core.yml → ...rces/docker-compose/docker-compose-it.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,20 @@ services:
- -jar
- /opt/feast/feast-core.jar
- --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml

jobcontroller:
image: gcr.io/kf-feast/feast-jobcontroller:latest
volumes:
- ./job-controller/application-it.yml:/etc/feast/application.yml
depends_on:
- kafka
ports:
- 6570:6570
command:
- java
- -jar
- /opt/feast/feast-job-controller.jar
- --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml

kafka:
image: confluentinc/cp-kafka:5.2.1
Expand Down
16 changes: 16 additions & 0 deletions serving/src/test/resources/docker-compose/job-controller/application-it.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
feast:
core-host: core
jobs:
enabled: true
polling_interval_milliseconds: 30000
job_update_timeout_seconds: 240
active_runner: direct
runners:
- name: direct
type: DirectRunner
options: {}
stream:
type: kafka
options:
topic: feast-features
bootstrapServers: "kafka:9092,localhost:9094"