row compression

make batch size parameter simplify creating jobId
feast-dev · feast-ci-bot · Jun 16, 2020 · Jun 15, 2020 · Jun 16, 2020 · 237703bb909ac49836a769dcafd70d9c573bdd0a
commit 237703bb909ac49836a769dcafd70d9c573bdd0a
diff --git a/.../connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java b/.../connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java
@@ -17,9 +17,12 @@
 package feast.storage.connectors.bigquery.common;
 
 import com.google.cloud.bigquery.StandardSQLTypeName;
+import com.google.protobuf.ByteString;
 import feast.proto.types.ValueProto;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 public class TypeUtil {
 
@@ -63,4 +66,114 @@ public class TypeUtil {
   public static StandardSQLTypeName toStandardSqlType(ValueProto.ValueType.Enum valueType) {
     return VALUE_TYPE_TO_STANDARD_SQL_TYPE.get(valueType);
   }
+
+  public static Object protoValueToObject(
+      ValueProto.Value value, ValueProto.Value.ValCase valCase) {
+    switch (valCase) {
+      case BYTES_VAL:
+        return value.getBytesVal().toByteArray();
+      case STRING_VAL:
+        return value.getStringVal();
+      case INT32_VAL:
+        return value.getInt32Val();
+      case INT64_VAL:
+        return value.getInt64Val();
+      case DOUBLE_VAL:
+        return value.getDoubleVal();
+      case FLOAT_VAL:
+        return value.getFloatVal();
+      case BOOL_VAL:
+        return value.getBoolVal();
+      case BYTES_LIST_VAL:
+        return value.getBytesListVal().getValList().stream()
+            .map(ByteString::toByteArray)
+            .collect(Collectors.toList());
+      case STRING_LIST_VAL:
+        return value.getStringListVal().getValList();
+      case INT32_LIST_VAL:
+        return value.getInt32ListVal().getValList();
+      case INT64_LIST_VAL:
+        return value.getInt64ListVal().getValList();
+      case DOUBLE_LIST_VAL:
+        return value.getDoubleListVal().getValList();
+      case FLOAT_LIST_VAL:
+        return value.getFloatListVal().getValList();
+      case BOOL_LIST_VAL:
+        return value.getBoolListVal().getValList();
+      case VAL_NOT_SET:
+        break;
+    }
+    return null;
+  }
+
+  public static ValueProto.Value objectToProtoValue(
+      Object value, ValueProto.Value.ValCase valCase) {
+    ValueProto.Value.Builder builder = ValueProto.Value.newBuilder();
+    switch (valCase) {
+      case BYTES_VAL:
+        return builder.setBytesVal(ByteString.copyFrom((byte[]) value)).build();
+      case STRING_VAL:
+        return builder.setStringVal((String) value).build();
+      case INT32_VAL:
+        return builder.setInt32Val((Integer) value).build();
+      case INT64_VAL:
+        return builder.setInt64Val((Long) value).build();
+      case DOUBLE_VAL:
+        return builder.setDoubleVal((Double) value).build();
+      case FLOAT_VAL:
+        return builder.setFloatVal((Float) value).build();
+      case BOOL_VAL:
+        return builder.setBoolVal((Boolean) value).build();
+      case BYTES_LIST_VAL:
+        return builder
+            .setBytesListVal(
+                ValueProto.BytesList.newBuilder()
+                    .addAllVal(
+                        ((List<byte[]>) value)
+                            .stream().map(ByteString::copyFrom).collect(Collectors.toList()))
+                    .build())
+            .build();
+      case STRING_LIST_VAL:
+        return builder
+            .setStringListVal(
+                ValueProto.StringList.newBuilder().addAllVal((List<String>) value).build())
+            .build();
+      case INT32_LIST_VAL:
+        return builder
+            .setInt32ListVal(
+                ValueProto.Int32List.newBuilder().addAllVal((List<Integer>) value).build())
+            .build();
+      case INT64_LIST_VAL:
+        return builder
+            .setInt64ListVal(
+                ValueProto.Int64List.newBuilder().addAllVal((List<Long>) value).build())
+            .build();
+      case DOUBLE_LIST_VAL:
+        return builder
+            .setDoubleListVal(
+                ValueProto.DoubleList.newBuilder().addAllVal((List<Double>) value).build())
+            .build();
+      case FLOAT_LIST_VAL:
+        return builder
+            .setFloatListVal(
+                ValueProto.FloatList.newBuilder().addAllVal((List<Float>) value).build())
+            .build();
+      case BOOL_LIST_VAL:
+        return builder
+            .setBoolListVal(
+                ValueProto.BoolList.newBuilder().addAllVal((List<Boolean>) value).build())
+            .build();
+      case VAL_NOT_SET:
+        break;
+    }
+    return null;
+  }
+
+  public static Object protoValueToObject(ValueProto.Value value) {
+    return protoValueToObject(value, value.getValCase());
+  }
+
+  public static Object getDefaultProtoValue(ValueProto.Value.ValCase valCase) {
+    return protoValueToObject(ValueProto.Value.getDefaultInstance(), valCase);
+  }
 }
diff --git a/...query/src/main/java/feast/storage/connectors/bigquery/compression/CompactFeatureRows.java b/...query/src/main/java/feast/storage/connectors/bigquery/compression/CompactFeatureRows.java
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.compression;
+
+import feast.proto.types.FeatureRowProto;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.SnappyCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupIntoBatches;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Collects batch of FeatureRow, converts them into columnar form:
+ *
+ * <p>List&lt;FeatureRow&gt; -&gt;
+ *
+ * <p>- Feature1 List&lt;Integer&gt; - all values from all rows in one list
+ *
+ * <p>- Feature2 List&lt;String&gt;
+ *
+ * <p>- ...
+ *
+ * <p>FeatureRowsBatch is in column form, hence it is better compressible. For compression we use
+ * Snappy, since it's already available in Beam as SnappyCoder.
+ */
+public class CompactFeatureRows
+    extends PTransform<
+        PCollection<KV<String, FeatureRowProto.FeatureRow>>,
+        PCollection<KV<String, FeatureRowsBatch>>> {
+  private final int batchSize;
+
+  public CompactFeatureRows(int batchSize) {
+    this.batchSize = batchSize;
+  }
+
+  @Override
+  public PCollection<KV<String, FeatureRowsBatch>> expand(
+      PCollection<KV<String, FeatureRowProto.FeatureRow>> input) {
+    return input
+        .apply(GroupIntoBatches.ofSize(batchSize))
+        .apply(ParDo.of(new FeatureRowToColumnar()))
+        .setCoder(
+            KvCoder.of(
+                StringUtf8Coder.of(), SnappyCoder.of(FeatureRowsBatch.FeatureRowsCoder.of())));
+  }
+
+  public static class FeatureRowToColumnar
+      extends DoFn<KV<String, Iterable<FeatureRowProto.FeatureRow>>, KV<String, FeatureRowsBatch>> {
+    @ProcessElement
+    public void process(ProcessContext c) {
+      c.output(KV.of(c.element().getKey(), new FeatureRowsBatch(c.element().getValue())));
+    }
+  }
+}