Skip to content

Commit 11bc7ee

Browse files
cheswoop
ches
authored and
woop
committed
Clean up the Maven build (feast-dev#262)
* Clean up the Maven build Three main themes: - Fix core module dependency on ingestion, so that Maven resolves it as an inter-project dep without requiring it to be installed to (local) repository and mucking with the version. - Use dependencyManagement in parent POM for common dependencies and families -- version drift between modules on things like gRPC and Google Cloud deps will bring more pain in the long run than any short- term convenience of letting them differ between modules. This makes it easy to control that consistently. - Avoid (re)defining properties in submodule POMs -- similar reasoning as dependencies: drift leads to suffering. If you build only a subset of modules at a time, resolved values may differ based on how Maven orders the build. Also avoid repeating things that inherit from the parent POM, like module groupId. * Enable -Xlint warnings, fix trivial deprecations * Make sdk/java a child module of the main build Updating the protobuf/protoc version everywhere to what the SDK used, because it requires new APIs. * Don't recompile protos when they haven't changed Checks mtime versus target/ directory. Saves considerable build time. * Remove unused dependencies According to mvn dependency:analyze -DignoreNonCompile and running tests. Also added some undeclared direct dependencies, though not covering them comprehensively yet.
1 parent f985024 commit 11bc7ee

16 files changed

Lines changed: 401 additions & 529 deletions

File tree

‎core/pom.xml‎

Lines changed: 11 additions & 195 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2020
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
2121
<modelVersion>4.0.0</modelVersion>
22+
2223
<parent>
2324
<groupId>feast</groupId>
2425
<artifactId>feast-parent</artifactId>
2526
<version>${revision}</version>
2627
</parent>
27-
<artifactId>feast-core</artifactId>
28-
<packaging>jar</packaging>
28+
2929
<name>Feast Core</name>
30+
<artifactId>feast-core</artifactId>
3031

3132
<build>
3233
<plugins>
@@ -78,46 +79,32 @@
7879
</plugins>
7980
</build>
8081

81-
<repositories>
82-
<repository>
83-
<id>jcenter</id>
84-
<url>https://jcenter.bintray.com/</url>
85-
</repository>
86-
<repository>
87-
<id>jitpack.io</id>
88-
<url>https://jitpack.io</url>
89-
</repository>
90-
</repositories>
91-
9282
<dependencies>
9383
<dependency>
94-
<groupId>com.github.gojek.feast</groupId>
84+
<groupId>feast</groupId>
9585
<artifactId>feast-ingestion</artifactId>
96-
<version>9428b230a5</version>
86+
<version>${project.version}</version>
87+
</dependency>
88+
89+
<dependency>
90+
<groupId>javax.inject</groupId>
91+
<artifactId>javax.inject</artifactId>
92+
<version>1</version>
9793
</dependency>
9894
<!--compile "org.springframework.boot:spring-boot-starter-web:${springBootVersion}"-->
9995
<dependency>
10096
<groupId>org.springframework.boot</groupId>
10197
<artifactId>spring-boot-starter-web</artifactId>
102-
<version>${springBootVersion}</version>
103-
<exclusions>
104-
<exclusion>
105-
<groupId>org.springframework.boot</groupId>
106-
<artifactId>spring-boot-starter-logging</artifactId>
107-
</exclusion>
108-
</exclusions>
10998
</dependency>
11099
<!--compile 'org.springframework.boot:spring-boot-starter-log4j2'-->
111100
<dependency>
112101
<groupId>org.springframework.boot</groupId>
113102
<artifactId>spring-boot-starter-log4j2</artifactId>
114-
<version>${springBootVersion}</version>
115103
</dependency>
116104
<!--compile 'org.lognet:grpc-spring-boot-starter:2.4.1'-->
117105
<dependency>
118106
<groupId>org.lognet</groupId>
119107
<artifactId>grpc-spring-boot-starter</artifactId>
120-
<version>2.4.1</version>
121108
</dependency>
122109
<!--compile "org.springframework.boot:spring-boot-starter-data-jpa:${springBootVersion}"-->
123110
<dependency>
@@ -129,66 +116,42 @@
129116
<dependency>
130117
<groupId>org.springframework.boot</groupId>
131118
<artifactId>spring-boot-starter-actuator</artifactId>
132-
<version>${springBootVersion}</version>
133119
</dependency>
134120

135121
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor -->
136122
<dependency>
137123
<groupId>org.springframework.boot</groupId>
138124
<artifactId>spring-boot-configuration-processor</artifactId>
139-
<version>${springBootVersion}</version>
140125
</dependency>
141126

142127

143-
<!--compile "io.grpc:grpc-netty:${grpcVersion}"-->
144-
<dependency>
145-
<groupId>io.grpc</groupId>
146-
<artifactId>grpc-netty</artifactId>
147-
<version>${grpcVersion}</version>
148-
</dependency>
149128
<!--compile "io.grpc:grpc-services:${grpcVersion}"-->
150129
<dependency>
151130
<groupId>io.grpc</groupId>
152131
<artifactId>grpc-services</artifactId>
153-
<version>${grpcVersion}</version>
154132
</dependency>
155133
<!--compile "io.grpc:grpc-stub:${grpcVersion}"-->
156134
<dependency>
157135
<groupId>io.grpc</groupId>
158136
<artifactId>grpc-stub</artifactId>
159-
<version>${grpcVersion}</version>
160137
</dependency>
161138
<!--compile "com.google.protobuf:protobuf-java-util:${protobufVersion}"-->
162139
<dependency>
163140
<groupId>com.google.protobuf</groupId>
164141
<artifactId>protobuf-java-util</artifactId>
165-
<version>${protobufVersion}</version>
166142
</dependency>
167143

168144
<!--compile 'com.google.guava:guava:26.0-jre'-->
169145
<dependency>
170146
<groupId>com.google.guava</groupId>
171147
<artifactId>guava</artifactId>
172-
<version>26.0-jre</version>
173148
</dependency>
174149
<!--compile 'com.google.code.gson:gson:2.8.5'-->
175150
<dependency>
176151
<groupId>com.google.code.gson</groupId>
177152
<artifactId>gson</artifactId>
178153
<version>2.8.5</version>
179154
</dependency>
180-
<!--compile 'commons-codec:commons-codec:1.10'-->
181-
<dependency>
182-
<groupId>commons-codec</groupId>
183-
<artifactId>commons-codec</artifactId>
184-
<version>1.10</version>
185-
</dependency>
186-
<!--compile 'joda-time:joda-time:2.9.9'-->
187-
<dependency>
188-
<groupId>joda-time</groupId>
189-
<artifactId>joda-time</artifactId>
190-
<version>2.9.9</version>
191-
</dependency>
192155
<dependency>
193156
<groupId>com.google.api-client</groupId>
194157
<artifactId>google-api-client</artifactId>
@@ -200,153 +163,28 @@
200163
<version>v1b3-rev266-1.25.0</version>
201164
</dependency>
202165

203-
<!--compile 'org.jdbi:jdbi3:3.0.0-beta2'-->
204-
<dependency>
205-
<groupId>org.jdbi</groupId>
206-
<artifactId>jdbi3</artifactId>
207-
<version>3.0.0-beta2</version>
208-
</dependency>
209-
<!--compile 'org.jdbi:jdbi3-sqlobject:3.4.0'-->
210-
<dependency>
211-
<groupId>org.jdbi</groupId>
212-
<artifactId>jdbi3-sqlobject</artifactId>
213-
<version>3.4.0</version>
214-
</dependency>
215-
<!--compile 'org.postgresql:postgresql:42.2.5'-->
216-
<dependency>
217-
<groupId>org.postgresql</groupId>
218-
<artifactId>postgresql</artifactId>
219-
<version>42.2.5</version>
220-
</dependency>
221166
<!--compile 'org.hibernate:hibernate-core:5.3.6.Final'-->
222167
<dependency>
223168
<groupId>org.hibernate</groupId>
224169
<artifactId>hibernate-core</artifactId>
225170
<version>5.3.6.Final</version>
226171
</dependency>
227-
<!--compile 'com.google.cloud.bigtable:bigtable-hbase-1.x:1.5.0'-->
228-
<dependency>
229-
<groupId>com.google.cloud.bigtable</groupId>
230-
<artifactId>bigtable-hbase-2.x-shaded</artifactId>
231-
<version>1.5.0</version>
232-
</dependency>
233-
<!--compile 'com.google.cloud:google-cloud-bigquery:1.48.0'-->
234-
<dependency>
235-
<groupId>com.google.cloud</groupId>
236-
<artifactId>google-cloud-bigquery</artifactId>
237-
<version>1.48.0</version>
238-
</dependency>
239-
<dependency>
240-
<groupId>com.google.cloud</groupId>
241-
<artifactId>google-cloud-nio</artifactId>
242-
<version>0.83.0-alpha</version>
243-
</dependency>
244-
<dependency>
245-
<groupId>org.apache.flink</groupId>
246-
<artifactId>flink-clients_2.11</artifactId>
247-
<version>1.5.5</version>
248-
</dependency>
249-
250-
<!-- Jackson due to jinjava dependency problems -->
251-
<dependency>
252-
<groupId>com.fasterxml.jackson.core</groupId>
253-
<artifactId>jackson-databind</artifactId>
254-
<version>2.9.9.1</version>
255-
</dependency>
256-
<dependency>
257-
<groupId>com.fasterxml.jackson.core</groupId>
258-
<artifactId>jackson-core</artifactId>
259-
<version>2.9.9</version>
260-
</dependency>
261-
<dependency>
262-
<groupId>com.fasterxml.jackson.core</groupId>
263-
<artifactId>jackson-annotations</artifactId>
264-
<version>2.9.9</version>
265-
</dependency>
266-
<dependency>
267-
<groupId>com.fasterxml.jackson.dataformat</groupId>
268-
<artifactId>jackson-dataformat-yaml</artifactId>
269-
<version>2.9.9</version>
270-
</dependency>
271-
272-
<!--compile 'com.github.spullara.mustache.java:compiler:0.9.5'-->
273-
<dependency>
274-
<groupId>com.github.spullara.mustache.java</groupId>
275-
<artifactId>compiler</artifactId>
276-
<version>0.9.5</version>
277-
</dependency>
278-
<dependency>
279-
<groupId>com.hubspot.jinjava</groupId>
280-
<artifactId>jinjava</artifactId>
281-
<version>2.4.12</version>
282-
<!--exclude jackson-->
283-
<exclusions>
284-
<exclusion>
285-
<groupId>com.fasterxml.jackson.core</groupId>
286-
<artifactId>jackson-databind</artifactId>
287-
</exclusion>
288-
<exclusion>
289-
<groupId>com.fasterxml.jackson.core</groupId>
290-
<artifactId>jackson-core</artifactId>
291-
</exclusion>
292-
<exclusion>
293-
<groupId>com.fasterxml.jackson.core</groupId>
294-
<artifactId>jackson-annotations</artifactId>
295-
</exclusion>
296-
</exclusions>
297-
</dependency>
298-
<!--compile 'io.micrometer:micrometer-core:1.0.7'-->
299-
<dependency>
300-
<groupId>io.micrometer</groupId>
301-
<artifactId>micrometer-core</artifactId>
302-
<version>1.0.7</version>
303-
</dependency>
304-
<!--compile 'io.micrometer:micrometer-registry-prometheus:1.0.7'-->
305-
<dependency>
306-
<groupId>io.micrometer</groupId>
307-
<artifactId>micrometer-registry-statsd</artifactId>
308-
<version>1.0.7</version>
309-
</dependency>
310-
<dependency>
311-
<groupId>com.datadoghq</groupId>
312-
<artifactId>java-dogstatsd-client</artifactId>
313-
<version>2.6.1</version>
314-
</dependency>
315172

316173
<dependency>
317174
<groupId>org.apache.kafka</groupId>
318175
<artifactId>kafka-clients</artifactId>
319-
<version>2.3.0</version>
320-
</dependency>
321-
322-
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-exec -->
323-
<dependency>
324-
<groupId>org.apache.commons</groupId>
325-
<artifactId>commons-exec</artifactId>
326-
<version>1.3</version>
327176
</dependency>
328177

329178
<!--compileOnly 'org.projectlombok:lombok:1.18.2'-->
330179
<dependency>
331180
<groupId>org.projectlombok</groupId>
332181
<artifactId>lombok</artifactId>
333-
<version>1.18.2</version>
334-
<scope>provided</scope>
335182
</dependency>
336183

337-
<!--testCompile "io.grpc:grpc-testing:${grpcVersion}"-->
338-
<dependency>
339-
<groupId>io.grpc</groupId>
340-
<artifactId>grpc-testing</artifactId>
341-
<version>${grpcVersion}</version>
342-
<scope>test</scope>
343-
</dependency>
344184
<!--testCompile 'org.hamcrest:hamcrest-all:1.3'-->
345185
<dependency>
346186
<groupId>org.hamcrest</groupId>
347187
<artifactId>hamcrest-all</artifactId>
348-
<version>1.3</version>
349-
<scope>test</scope>
350188
</dependency>
351189

352190
<!--testCompile 'com.jayway.jsonpath:json-path-assert:2.2.0'-->
@@ -362,36 +200,14 @@
362200
<version>2.23.0</version>
363201
<scope>test</scope>
364202
</dependency>
365-
<dependency>
366-
<groupId>com.squareup.okhttp3</groupId>
367-
<artifactId>mockwebserver</artifactId>
368-
<version>3.11.0</version>
369-
<scope>test</scope>
370-
</dependency>
371203

372-
<dependency>
373-
<groupId>org.springframework</groupId>
374-
<artifactId>spring-test</artifactId>
375-
<version>5.0.8.RELEASE</version>
376-
<scope>test</scope>
377-
</dependency>
378204
<dependency>
379205
<groupId>org.springframework.boot</groupId>
380206
<artifactId>spring-boot-test</artifactId>
381-
<version>${springBootVersion}</version>
382-
<scope>test</scope>
383207
</dependency>
384208
<dependency>
385209
<groupId>org.springframework.boot</groupId>
386210
<artifactId>spring-boot-test-autoconfigure</artifactId>
387-
<version>${springBootVersion}</version>
388-
<scope>test</scope>
389-
</dependency>
390-
<dependency>
391-
<groupId>com.h2database</groupId>
392-
<artifactId>h2</artifactId>
393-
<version>1.4.198</version>
394-
<scope>test</scope>
395211
</dependency>
396212
</dependencies>
397213
</project>

‎core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
package feast.core.job.dataflow;
1919

20-
import static com.google.common.base.Preconditions.checkNotNull;
21-
2220
import com.google.api.services.dataflow.Dataflow;
2321
import com.google.api.services.dataflow.model.Job;
2422
import com.google.common.base.Strings;
@@ -35,7 +33,7 @@
3533
import feast.core.model.JobInfo;
3634
import feast.core.util.TypeConversion;
3735
import feast.ingestion.ImportJob;
38-
import feast.ingestion.options.ImportJobPipelineOptions;
36+
import feast.ingestion.options.ImportOptions;
3937
import java.io.IOException;
4038
import java.util.ArrayList;
4139
import java.util.Collections;
@@ -128,7 +126,7 @@ public void abortJob(String dataflowJobId) {
128126

129127
private String submitDataflowJob(String jobName, List<FeatureSetSpec> featureSets, Store sink, boolean update) {
130128
try {
131-
ImportJobPipelineOptions pipelineOptions = getPipelineOptions(jobName, featureSets, sink, update);
129+
ImportOptions pipelineOptions = getPipelineOptions(jobName, featureSets, sink, update);
132130
DataflowPipelineJob pipelineResult = runPipeline(pipelineOptions);
133131
String jobId = waitForJobToRun(pipelineResult);
134132
return jobId;
@@ -138,11 +136,10 @@ private String submitDataflowJob(String jobName, List<FeatureSetSpec> featureSet
138136
}
139137
}
140138

141-
private ImportJobPipelineOptions getPipelineOptions(String jobName, List<FeatureSetSpec> featureSets, Store sink,
139+
private ImportOptions getPipelineOptions(String jobName, List<FeatureSetSpec> featureSets, Store sink,
142140
boolean update) throws InvalidProtocolBufferException {
143141
String[] args = TypeConversion.convertMapToArgs(defaultOptions);
144-
ImportJobPipelineOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args)
145-
.as(ImportJobPipelineOptions.class);
142+
ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class);
146143
Printer printer = JsonFormat.printer();
147144
List<String> featureSetsJson = new ArrayList<>();
148145
for (FeatureSetSpec featureSet : featureSets) {
@@ -164,7 +161,7 @@ private ImportJobPipelineOptions getPipelineOptions(String jobName, List<Feature
164161
return pipelineOptions;
165162
}
166163

167-
public DataflowPipelineJob runPipeline(ImportJobPipelineOptions pipelineOptions)
164+
public DataflowPipelineJob runPipeline(ImportOptions pipelineOptions)
168165
throws IOException {
169166
return (DataflowPipelineJob) ImportJob
170167
.runPipeline(pipelineOptions);

0 commit comments

Comments
 (0)