Skip to content

Commit e47903f

Browse files
pyalexpyalex
authored
Batch Ingestion Job rewritten on Spark (#1020)
* test scala spark Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * offline batch ingestion in spark Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * clean up Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * deduplicate rows & use latest Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * clarify Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * validation & deadletter Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * tests on mapping & deadletter Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * scala styling Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * integration test stage Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * remove version from ingestion-spark pom Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * refactor job options Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * clean up dependencies + some api docs Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * extend mapping test Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * add shade plugin version & group Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * disable buildkit on docker build Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
1 parent 8bf55de commit e47903f

23 files changed

Lines changed: 2024 additions & 21 deletions

‎.github/workflows/complete.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ jobs:
1212
GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }}
1313
REGISTRY: gcr.io/kf-feast
1414
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
15-
DOCKER_BUILDKIT: '1'
1615
steps:
1716
- uses: actions/checkout@v2
1817
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master

‎.scalafmt.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
align.preset = more
2+
maxColumn = 100

‎pom.xml‎

Lines changed: 32 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
<module>common</module>
4040
<module>job-controller</module>
4141
<module>common-test</module>
42+
<module>spark/ingestion</module>
4243
</modules>
4344

4445
<properties>
@@ -85,6 +86,27 @@
8586
<org.hibernate.validator.version>6.1.2.Final</org.hibernate.validator.version>
8687
<auto.value.version>1.6.6</auto.value.version>
8788

89+
<license.content><![CDATA[
90+
/*
91+
* SPDX-License-Identifier: Apache-2.0
92+
* Copyright 2018-$YEAR The Feast Authors
93+
*
94+
* Licensed under the Apache License, Version 2.0 (the "License");
95+
* you may not use this file except in compliance with the License.
96+
* You may obtain a copy of the License at
97+
*
98+
* https://www.apache.org/licenses/LICENSE-2.0
99+
*
100+
* Unless required by applicable law or agreed to in writing, software
101+
* distributed under the License is distributed on an "AS IS" BASIS,
102+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
103+
* See the License for the specific language governing permissions and
104+
* limitations under the License.
105+
*/
106+
]]>
107+
</license.content>
108+
<parent.basedir>${maven.multiModuleProjectDirectory}</parent.basedir>
109+
88110
<skipUTs>false</skipUTs>
89111
<feast.auth.providers.http.client.package.name>feast.common.auth.providers.http.client</feast.auth.providers.http.client.package.name>
90112
</properties>
@@ -528,26 +550,7 @@
528550
<configuration>
529551
<java>
530552
<licenseHeader>
531-
<content>
532-
<![CDATA[
533-
/*
534-
* SPDX-License-Identifier: Apache-2.0
535-
* Copyright 2018-$YEAR The Feast Authors
536-
*
537-
* Licensed under the Apache License, Version 2.0 (the "License");
538-
* you may not use this file except in compliance with the License.
539-
* You may obtain a copy of the License at
540-
*
541-
* https://www.apache.org/licenses/LICENSE-2.0
542-
*
543-
* Unless required by applicable law or agreed to in writing, software
544-
* distributed under the License is distributed on an "AS IS" BASIS,
545-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
546-
* See the License for the specific language governing permissions and
547-
* limitations under the License.
548-
*/
549-
]]>
550-
</content>
553+
<content>${license.content}</content>
551554
</licenseHeader>
552555
<googleJavaFormat>
553556
<version>1.7</version>
@@ -558,6 +561,15 @@
558561
</excludes>
559562
<removeUnusedImports />
560563
</java>
564+
<scala>
565+
<licenseHeader>
566+
<content>${license.content}</content>
567+
</licenseHeader>
568+
<scalafmt>
569+
<version>2.7.2</version>
570+
<file>${parent.basedir}/.scalafmt.conf</file>
571+
</scalafmt>
572+
</scala>
561573
</configuration>
562574
<executions>
563575
<!-- Move check to fail faster, but after compilation. Default is verify phase -->

‎spark/ingestion/pom.xml‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright 2018 The Feast Authors
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License");
6+
~ you may not use this file except in compliance with the License.
7+
~ You may obtain a copy of the License at
8+
~
9+
~ https://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ See the License for the specific language governing permissions and
15+
~ limitations under the License.
16+
~
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<parent>
24+
<groupId>dev.feast</groupId>
25+
<artifactId>feast-parent</artifactId>
26+
<version>${revision}</version>
27+
<relativePath>../..</relativePath>
28+
</parent>
29+
30+
<name>Feast Spark Ingestion</name>
31+
<artifactId>feast-ingestion-spark</artifactId>
32+
33+
<properties>
34+
<scala.version>2.12</scala.version>
35+
<scala.fullVersion>${scala.version}.12</scala.fullVersion>
36+
<spark.version>2.4.7</spark.version>
37+
<scala-maven-plugin.version>4.4.0</scala-maven-plugin.version>
38+
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
39+
<project.version>0.7-SNAPSHOT</project.version>
40+
</properties>
41+
42+
43+
<dependencies>
44+
<dependency>
45+
<groupId>dev.feast</groupId>
46+
<artifactId>datatypes-java</artifactId>
47+
<version>${project.version}</version>
48+
<exclusions>
49+
<exclusion>
50+
<groupId>*</groupId>
51+
<artifactId>*</artifactId>
52+
</exclusion>
53+
</exclusions>
54+
</dependency>
55+
56+
<dependency>
57+
<groupId>com.google.protobuf</groupId>
58+
<artifactId>protobuf-java</artifactId>
59+
<version>3.12.2</version>
60+
</dependency>
61+
62+
<dependency>
63+
<groupId>org.scala-lang</groupId>
64+
<artifactId>scala-library</artifactId>
65+
<version>${scala.fullVersion}</version>
66+
</dependency>
67+
68+
<dependency>
69+
<groupId>org.scala-lang.modules</groupId>
70+
<artifactId>scala-collection-compat_${scala.version}</artifactId>
71+
<version>2.2.0</version>
72+
</dependency>
73+
74+
<dependency>
75+
<groupId>org.apache.spark</groupId>
76+
<artifactId>spark-core_${scala.version}</artifactId>
77+
<version>${spark.version}</version>
78+
<scope>provided</scope>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>org.apache.spark</groupId>
83+
<artifactId>spark-streaming_${scala.version}</artifactId>
84+
<version>${spark.version}</version>
85+
<scope>provided</scope>
86+
</dependency>
87+
88+
<dependency>
89+
<groupId>org.apache.spark</groupId>
90+
<artifactId>spark-sql_${scala.version}</artifactId>
91+
<version>${spark.version}</version>
92+
<scope>provided</scope>
93+
</dependency>
94+
95+
<dependency>
96+
<groupId>org.codehaus.janino</groupId>
97+
<artifactId>janino</artifactId>
98+
<version>3.0.16</version>
99+
</dependency>
100+
101+
<dependency>
102+
<groupId>org.apache.spark</groupId>
103+
<artifactId>spark-sql-kafka-0-10_${scala.version}</artifactId>
104+
<version>${spark.version}</version>
105+
<scope>provided</scope>
106+
</dependency>
107+
108+
<dependency>
109+
<groupId>com.github.scopt</groupId>
110+
<artifactId>scopt_${scala.version}</artifactId>
111+
<version>3.7.1</version>
112+
</dependency>
113+
114+
<dependency>
115+
<groupId>com.google.cloud.spark</groupId>
116+
<artifactId>spark-bigquery_${scala.version}</artifactId>
117+
<version>0.17.2</version>
118+
<scope>provided</scope>
119+
</dependency>
120+
121+
<dependency>
122+
<groupId>joda-time</groupId>
123+
<artifactId>joda-time</artifactId>
124+
<version>2.10.6</version>
125+
</dependency>
126+
127+
<dependency>
128+
<groupId>com.redislabs</groupId>
129+
<artifactId>spark-redis_${scala.version}</artifactId>
130+
<version>2.5.0</version>
131+
</dependency>
132+
133+
<dependency>
134+
<groupId>org.apache.arrow</groupId>
135+
<artifactId>arrow-vector</artifactId>
136+
<version>0.16.0</version>
137+
</dependency>
138+
139+
<dependency>
140+
<groupId>io.netty</groupId>
141+
<artifactId>netty-all</artifactId>
142+
<version>4.1.52.Final</version>
143+
</dependency>
144+
145+
<dependency>
146+
<groupId>org.json4s</groupId>
147+
<artifactId>json4s-ext_${scala.version}</artifactId>
148+
<version>3.7.0-M6</version>
149+
</dependency>
150+
151+
<dependency>
152+
<groupId>org.scalatest</groupId>
153+
<artifactId>scalatest_${scala.version}</artifactId>
154+
<version>3.2.2</version>
155+
<scope>test</scope>
156+
</dependency>
157+
158+
<dependency>
159+
<groupId>org.scalacheck</groupId>
160+
<artifactId>scalacheck_${scala.version}</artifactId>
161+
<version>1.14.3</version>
162+
<scope>test</scope>
163+
</dependency>
164+
165+
<dependency>
166+
<groupId>com.dimafeng</groupId>
167+
<artifactId>testcontainers-scala-scalatest_${scala.version}</artifactId>
168+
<version>0.38.3</version>
169+
<scope>test</scope>
170+
</dependency>
171+
172+
</dependencies>
173+
174+
175+
<build>
176+
<sourceDirectory>src/main/scala</sourceDirectory>
177+
<testSourceDirectory>src/test/scala</testSourceDirectory>
178+
<plugins>
179+
<plugin>
180+
<groupId>net.alchim31.maven</groupId>
181+
<artifactId>scala-maven-plugin</artifactId>
182+
<executions>
183+
<execution>
184+
<id>scala-compile-first</id>
185+
<phase>process-resources</phase>
186+
<goals>
187+
<goal>add-source</goal>
188+
<goal>compile</goal>
189+
</goals>
190+
</execution>
191+
<execution>
192+
<id>scala-test-compile</id>
193+
<phase>process-test-resources</phase>
194+
<goals>
195+
<goal>testCompile</goal>
196+
</goals>
197+
</execution>
198+
</executions>
199+
<configuration>
200+
<scalaVersion>${scala.fullVersion}</scalaVersion>
201+
</configuration>
202+
</plugin>
203+
<plugin>
204+
<groupId>org.scalatest</groupId>
205+
<artifactId>scalatest-maven-plugin</artifactId>
206+
<version>2.0.0</version>
207+
<configuration>
208+
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
209+
<junitxml>.</junitxml>
210+
<filereports>TestSuiteReport.txt</filereports>
211+
</configuration>
212+
<executions>
213+
<execution>
214+
<id>test</id>
215+
<phase>integration-test</phase>
216+
<goals>
217+
<goal>test</goal>
218+
</goals>
219+
</execution>
220+
</executions>
221+
</plugin>
222+
<plugin>
223+
<groupId>org.apache.maven.plugins</groupId>
224+
<artifactId>maven-assembly-plugin</artifactId>
225+
<configuration>
226+
<descriptorRefs>
227+
<descriptorRef>jar-with-dependencies</descriptorRef>
228+
</descriptorRefs>
229+
</configuration>
230+
<executions>
231+
<execution>
232+
<id>make-assembly</id>
233+
<phase>package</phase>
234+
<goals>
235+
<goal>single</goal>
236+
</goals>
237+
</execution>
238+
</executions>
239+
</plugin>
240+
<plugin>
241+
<groupId>org.apache.maven.plugins</groupId>
242+
<artifactId>maven-compiler-plugin</artifactId>
243+
<executions>
244+
<execution>
245+
<phase>compile</phase>
246+
<goals>
247+
<goal>compile</goal>
248+
</goals>
249+
</execution>
250+
</executions>
251+
</plugin>
252+
<plugin>
253+
<groupId>org.apache.maven.plugins</groupId>
254+
<artifactId>maven-shade-plugin</artifactId>
255+
<version>3.2.4</version>
256+
<executions>
257+
<execution>
258+
<phase>package</phase>
259+
<goals>
260+
<goal>shade</goal>
261+
</goals>
262+
<configuration>
263+
<relocations>
264+
<relocation>
265+
<pattern>com.google.protobuf</pattern>
266+
<shadedPattern>com.google.protobuf.vendor</shadedPattern>
267+
</relocation>
268+
</relocations>
269+
<filters>
270+
<filter>
271+
<artifact>*:*</artifact>
272+
<excludes>
273+
<exclude>META-INF/*.SF</exclude>
274+
<exclude>META-INF/*.DSA</exclude>
275+
<exclude>META-INF/*.RSA</exclude>
276+
</excludes>
277+
</filter>
278+
</filters>
279+
</configuration>
280+
</execution>
281+
</executions>
282+
</plugin>
283+
</plugins>
284+
<pluginManagement>
285+
<plugins>
286+
<plugin>
287+
<groupId>net.alchim31.maven</groupId>
288+
<artifactId>scala-maven-plugin</artifactId>
289+
<version>${scala-maven-plugin.version}</version>
290+
</plugin>
291+
<plugin>
292+
<groupId>org.apache.maven.plugins</groupId>
293+
<artifactId>maven-assembly-plugin</artifactId>
294+
<version>${maven-assembly-plugin.version}</version>
295+
</plugin>
296+
</plugins>
297+
</pluginManagement>
298+
</build>
299+
</project>

0 commit comments

Comments
 (0)