Skip to content

Commit c4844eb

Browse files
author
Chen Zhiling
committed
Add BigQuery storage (#546)
1 parent 67dd123 commit c4844eb

15 files changed

Lines changed: 1626 additions & 0 deletions

‎storage/connectors/bigquery/pom.xml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3+
4+
<parent>
5+
<groupId>dev.feast</groupId>
6+
<artifactId>feast-storage-connectors</artifactId>
7+
<version>${revision}</version>
8+
</parent>
9+
10+
<modelVersion>4.0.0</modelVersion>
11+
<artifactId>feast-storage-connector-bigquery</artifactId>
12+
13+
<name>Feast Storage Connector for BigQuery</name>
14+
15+
<dependencies>
16+
<dependency>
17+
<groupId>io.pebbletemplates</groupId>
18+
<artifactId>pebble</artifactId>
19+
<version>3.1.0</version>
20+
</dependency>
21+
22+
<!-- Google Cloud -->
23+
<dependency>
24+
<groupId>com.google.cloud</groupId>
25+
<artifactId>google-cloud-bigquery</artifactId>
26+
</dependency>
27+
28+
<dependency>
29+
<groupId>com.google.cloud</groupId>
30+
<artifactId>google-cloud-storage</artifactId>
31+
</dependency>
32+
33+
<dependency>
34+
<groupId>com.google.auto.value</groupId>
35+
<artifactId>auto-value-annotations</artifactId>
36+
<version>1.6.6</version>
37+
</dependency>
38+
39+
<dependency>
40+
<groupId>com.google.auto.value</groupId>
41+
<artifactId>auto-value</artifactId>
42+
<version>1.6.6</version>
43+
<scope>provided</scope>
44+
</dependency>
45+
46+
<dependency>
47+
<groupId>junit</groupId>
48+
<artifactId>junit</artifactId>
49+
<version>4.12</version>
50+
<scope>test</scope>
51+
</dependency>
52+
53+
<dependency>
54+
<groupId>org.apache.beam</groupId>
55+
<artifactId>beam-runners-direct-java</artifactId>
56+
<version>${org.apache.beam.version}</version>
57+
<scope>test</scope>
58+
</dependency>
59+
60+
<dependency>
61+
<groupId>org.hamcrest</groupId>
62+
<artifactId>hamcrest-core</artifactId>
63+
<scope>test</scope>
64+
</dependency>
65+
<dependency>
66+
<groupId>org.hamcrest</groupId>
67+
<artifactId>hamcrest-library</artifactId>
68+
<scope>test</scope>
69+
</dependency>
70+
71+
<dependency>
72+
<groupId>org.apache.beam</groupId>
73+
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
74+
<version>2.16.0</version>
75+
<scope>compile</scope>
76+
</dependency>
77+
</dependencies>
78+
79+
</project>

‎storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.storage.connectors.bigquery.common;
18+
19+
import com.google.cloud.bigquery.StandardSQLTypeName;
20+
import feast.types.ValueProto;
21+
import java.util.HashMap;
22+
import java.util.Map;
23+
24+
public class TypeUtil {
25+
26+
private static final Map<ValueProto.ValueType.Enum, StandardSQLTypeName>
27+
VALUE_TYPE_TO_STANDARD_SQL_TYPE = new HashMap<>();
28+
29+
static {
30+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.BYTES, StandardSQLTypeName.BYTES);
31+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
32+
ValueProto.ValueType.Enum.STRING, StandardSQLTypeName.STRING);
33+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.INT32, StandardSQLTypeName.INT64);
34+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.INT64, StandardSQLTypeName.INT64);
35+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
36+
ValueProto.ValueType.Enum.DOUBLE, StandardSQLTypeName.FLOAT64);
37+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
38+
ValueProto.ValueType.Enum.FLOAT, StandardSQLTypeName.FLOAT64);
39+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.BOOL, StandardSQLTypeName.BOOL);
40+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
41+
ValueProto.ValueType.Enum.BYTES_LIST, StandardSQLTypeName.BYTES);
42+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
43+
ValueProto.ValueType.Enum.STRING_LIST, StandardSQLTypeName.STRING);
44+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
45+
ValueProto.ValueType.Enum.INT32_LIST, StandardSQLTypeName.INT64);
46+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
47+
ValueProto.ValueType.Enum.INT64_LIST, StandardSQLTypeName.INT64);
48+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
49+
ValueProto.ValueType.Enum.DOUBLE_LIST, StandardSQLTypeName.FLOAT64);
50+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
51+
ValueProto.ValueType.Enum.FLOAT_LIST, StandardSQLTypeName.FLOAT64);
52+
VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
53+
ValueProto.ValueType.Enum.BOOL_LIST, StandardSQLTypeName.BOOL);
54+
}
55+
56+
/**
57+
* Converts {@link feast.types.ValueProto.ValueType} to its corresponding {@link
58+
* StandardSQLTypeName}
59+
*
60+
* @param valueType value type to convert
61+
* @return {@link StandardSQLTypeName}
62+
*/
63+
public static StandardSQLTypeName toStandardSqlType(ValueProto.ValueType.Enum valueType) {
64+
return VALUE_TYPE_TO_STANDARD_SQL_TYPE.get(valueType);
65+
}
66+
}

0 commit comments

Comments
 (0)