Skip to content

Commit fe99de7

Browse files
dirtysaltdirtysalt
authored
[Feature] simplify hudi scanner dep and upgrade hadoop 3.x (StarRocks#33755)
Signed-off-by: yanz <dirtysalt1987@gmail.com>
1 parent cb54513 commit fe99de7

1 file changed

Lines changed: 26 additions & 78 deletions

File tree

‎java-extensions/hudi-reader/pom.xml‎

Lines changed: 26 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,9 @@
1616
<maven.compiler.target>8</maven.compiler.target>
1717
<java-extensions.home>${basedir}/../</java-extensions.home>
1818
<slf4j.version>1.7.36</slf4j.version>
19-
<presto.hadoop.version>2.7.4-11</presto.hadoop.version>
20-
<presto.hive.version>3.0.0-8</presto.hive.version>
21-
<hadoop.version>2.7.4</hadoop.version>
19+
<hive-apache.version>3.1.2-13</hive-apache.version>
20+
<hadoop.version>3.3.6</hadoop.version>
2221
<hudi.version>0.12.2</hudi.version>
23-
<fasterxml.version>2.13.4</fasterxml.version>
24-
<fasterxml.jackson.databind.version>2.13.4.2</fasterxml.jackson.databind.version>
25-
<esotericsoftware.kryo.shaded.version>4.0.2</esotericsoftware.kryo.shaded.version>
2622
<luben.zstd.jni.version>1.5.4-2</luben.zstd.jni.version>
2723
</properties>
2824

@@ -33,78 +29,54 @@
3329
<version>1.0.0</version>
3430
</dependency>
3531

32+
<!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-common -->
3633
<dependency>
3734
<groupId>org.apache.hudi</groupId>
38-
<artifactId>hudi-presto-bundle</artifactId>
35+
<artifactId>hudi-common</artifactId>
3936
<version>${hudi.version}</version>
4037
<exclusions>
4138
<exclusion>
42-
<groupId>com.google.protobuf</groupId>
43-
<artifactId>protobuf-java</artifactId>
39+
<groupId>org.apache.hbase</groupId>
40+
<artifactId>hbase-client</artifactId>
4441
</exclusion>
4542
<exclusion>
46-
<groupId>commons-lang</groupId>
47-
<artifactId>commons-lang</artifactId>
48-
</exclusion>
49-
<exclusion>
50-
<groupId>org.apache.hudi</groupId>
51-
<artifactId>hudi-common</artifactId>
52-
</exclusion>
53-
<exclusion>
54-
<groupId>org.apache.hudi</groupId>
55-
<artifactId>hudi-hadoop-mr-bundle</artifactId>
56-
</exclusion>
57-
<exclusion>
58-
<groupId>org.apache.parquet</groupId>
59-
<artifactId>parquet-avro</artifactId>
60-
</exclusion>
61-
<exclusion>
62-
<groupId>org.apache.avro</groupId>
63-
<artifactId>avro</artifactId>
43+
<groupId>org.apache.hbase</groupId>
44+
<artifactId>hbase-server</artifactId>
6445
</exclusion>
6546
</exclusions>
6647
</dependency>
6748

49+
<!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-hadoop-mr -->
50+
<dependency>
51+
<groupId>org.apache.hudi</groupId>
52+
<artifactId>hudi-hadoop-mr</artifactId>
53+
<version>${hudi.version}</version>
54+
</dependency>
55+
6856
<dependency>
69-
<groupId>com.facebook.presto.hive</groupId>
57+
<groupId>io.trino.hive</groupId>
7058
<artifactId>hive-apache</artifactId>
71-
<version>${presto.hive.version}</version>
72-
<exclusions>
73-
<exclusion>
74-
<groupId>org.slf4j</groupId>
75-
<artifactId>slf4j-log4j12</artifactId>
76-
</exclusion>
77-
</exclusions>
59+
<version>${hive-apache.version}</version>
7860
</dependency>
7961

62+
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
8063
<dependency>
81-
<groupId>com.facebook.presto.hadoop</groupId>
82-
<artifactId>hadoop-apache2</artifactId>
83-
<version>${presto.hadoop.version}</version>
84-
<exclusions>
85-
<exclusion>
86-
<groupId>org.slf4j</groupId>
87-
<artifactId>slf4j-log4j12</artifactId>
88-
</exclusion>
89-
</exclusions>
64+
<groupId>org.apache.hadoop</groupId>
65+
<artifactId>hadoop-client</artifactId>
66+
<version>${hadoop.version}</version>
9067
</dependency>
9168

69+
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
9270
<dependency>
9371
<groupId>org.apache.hadoop</groupId>
94-
<artifactId>hadoop-aws</artifactId>
72+
<artifactId>hadoop-common</artifactId>
9573
<version>${hadoop.version}</version>
96-
<exclusions>
97-
<exclusion>
98-
<groupId>org.apache.hadoop</groupId>
99-
<artifactId>hadoop-common</artifactId>
100-
</exclusion>
101-
</exclusions>
10274
</dependency>
10375

10476
<dependency>
105-
<groupId>commons-lang</groupId>
106-
<artifactId>commons-lang</artifactId>
107-
<version>2.6</version>
77+
<groupId>org.apache.hadoop</groupId>
78+
<artifactId>hadoop-aws</artifactId>
79+
<version>${hadoop.version}</version>
10880
</dependency>
10981

11082
<dependency>
@@ -113,36 +85,12 @@
11385
<version>${slf4j.version}</version>
11486
</dependency>
11587

116-
<dependency>
117-
<groupId>com.fasterxml.jackson.core</groupId>
118-
<artifactId>jackson-annotations</artifactId>
119-
<version>${fasterxml.version}</version>
120-
</dependency>
121-
122-
<dependency>
123-
<groupId>com.fasterxml.jackson.core</groupId>
124-
<artifactId>jackson-core</artifactId>
125-
<version>${fasterxml.version}</version>
126-
</dependency>
127-
128-
<dependency>
129-
<groupId>com.fasterxml.jackson.core</groupId>
130-
<artifactId>jackson-databind</artifactId>
131-
<version>${fasterxml.jackson.databind.version}</version>
132-
</dependency>
133-
13488
<dependency>
13589
<groupId>com.github.luben</groupId>
13690
<artifactId>zstd-jni</artifactId>
13791
<version>${luben.zstd.jni.version}</version>
13892
</dependency>
13993

140-
<dependency>
141-
<groupId>com.esotericsoftware</groupId>
142-
<artifactId>kryo-shaded</artifactId>
143-
<version>${esotericsoftware.kryo.shaded.version}</version>
144-
</dependency>
145-
14694
<dependency>
14795
<groupId>org.apache.logging.log4j</groupId>
14896
<artifactId>log4j-api</artifactId>

0 commit comments

Comments
 (0)