From ec0db7cb0996495b0033d009a699690b7e960f7b Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Wed, 18 Dec 2019 14:38:20 +0800 Subject: [PATCH 01/17] Update helm dependency before building (#373) To ensure requirements.lock is in sync with requirements.yaml https://github.com/helm/helm/issues/2033 --- .prow/scripts/sync-helm-charts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prow/scripts/sync-helm-charts.sh b/.prow/scripts/sync-helm-charts.sh index 88fc04f7d14..8c242aeae69 100755 --- a/.prow/scripts/sync-helm-charts.sh +++ b/.prow/scripts/sync-helm-charts.sh @@ -31,7 +31,7 @@ fi exit_code=0 for dir in "$repo_dir"/*; do - if helm dependency build "$dir"; then + if helm dep update "$dir" && helm dep build "$dir"; then helm package --destination "$sync_dir" "$dir" else log_error "Problem building dependencies. Skipping packaging of '$dir'." From decb44b0cbe4464c21a6758b6476f84d764c4008 Mon Sep 17 00:00:00 2001 From: Chen Zhiling Date: Sat, 21 Dec 2019 14:05:01 +0800 Subject: [PATCH 02/17] Make redis key creation more determinisitic (#380) * Make redis key creation more determinisitic * Sort entity names --- .../redis/FeatureRowToRedisMutationDoFn.java | 19 ++- .../FeatureRowToRedisMutationDoFnTest.java | 132 ++++++++++++++++++ .../serving/service/RedisServingService.java | 1 + 3 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java index 9bc503f9870..c453c5c9206 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java @@ -24,6 +24,9 @@ import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -42,17 +45,27 @@ public FeatureRowToRedisMutationDoFn(Map featureSetSpecs private RedisKey getKey(FeatureRow featureRow) { FeatureSetSpec featureSetSpec = featureSetSpecs.get(featureRow.getFeatureSet()); - Set entityNames = + List entityNames = featureSetSpec.getEntitiesList().stream() .map(EntitySpec::getName) - .collect(Collectors.toSet()); + .sorted() + .collect(Collectors.toList()); + Map entityFields = new HashMap<>(); Builder redisKeyBuilder = RedisKey.newBuilder().setFeatureSet(featureRow.getFeatureSet()); for (Field field : featureRow.getFieldsList()) { if (entityNames.contains(field.getName())) { - redisKeyBuilder.addEntities(field); + entityFields.putIfAbsent(field.getName(), + Field.newBuilder() + .setName(field.getName()) + .setValue(field.getValue()) + .build() + ); } } + for (String entityName : entityNames) { + redisKeyBuilder.addEntities(entityFields.get(entityName)); + } return redisKeyBuilder.build(); } diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java new file mode 100644 index 00000000000..6e0db2dd49c --- /dev/null +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java @@ -0,0 +1,132 @@ +package feast.store.serving.redis; + +import static org.junit.Assert.*; + +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.ingestion.transform.ValidateFeatureRows; +import feast.storage.RedisProto.RedisKey; +import feast.store.serving.redis.RedisCustomIO.Method; +import feast.store.serving.redis.RedisCustomIO.RedisMutation; +import feast.test.TestUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType.Enum; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +public class FeatureRowToRedisMutationDoFnTest { + + @Rule + public transient TestPipeline p = TestPipeline.create(); + + private FeatureSetSpec fs = FeatureSetSpec.newBuilder() + .setName("feature_set") + .setVersion(1) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_primary") + .setValueType(Enum.INT32) + .build()) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_secondary") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) + .build(); + + @Test + public void shouldConvertRowWithDuplicateEntitiesToValidKey() { + Map featureSetSpecs = new HashMap<>(); + featureSetSpecs.put("feature_set", fs); + + FeatureRow offendingRow = FeatureRow.newBuilder() + .setFeatureSet("feature_set") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields(Field.newBuilder().setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addFields(Field.newBuilder().setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(2))) + .addFields(Field.newBuilder().setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + PCollection output = p + .apply(Create.of(Collections.singletonList(offendingRow))) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs))); + + RedisKey expectedKey = RedisKey.newBuilder() + .setFeatureSet("feature_set") + .addEntities(Field.newBuilder().setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addEntities(Field.newBuilder().setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + PAssert.that(output).satisfies((SerializableFunction, Void>) input -> { + input.forEach(rm -> { + assert(Arrays.equals(rm.getKey(), expectedKey.toByteArray())); + assert(Arrays.equals(rm.getValue(), offendingRow.toByteArray())); + }); + return null; + }); + p.run(); + } + + @Test + public void shouldConvertRowWithOutOfOrderEntitiesToValidKey() { + Map featureSetSpecs = new HashMap<>(); + featureSetSpecs.put("feature_set", fs); + + FeatureRow offendingRow = FeatureRow.newBuilder() + .setFeatureSet("feature_set") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields(Field.newBuilder().setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .addFields(Field.newBuilder().setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .build(); + + PCollection output = p + .apply(Create.of(Collections.singletonList(offendingRow))) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs))); + + RedisKey expectedKey = RedisKey.newBuilder() + .setFeatureSet("feature_set") + .addEntities(Field.newBuilder().setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addEntities(Field.newBuilder().setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + PAssert.that(output).satisfies((SerializableFunction, Void>) input -> { + input.forEach(rm -> { + assert(Arrays.equals(rm.getKey(), expectedKey.toByteArray())); + assert(Arrays.equals(rm.getValue(), offendingRow.toByteArray())); + }); + return null; + }); + p.run(); + } + +} \ No newline at end of file diff --git a/serving/src/main/java/feast/serving/service/RedisServingService.java b/serving/src/main/java/feast/serving/service/RedisServingService.java index 7c0d65dc42c..9eaeb17dea3 100644 --- a/serving/src/main/java/feast/serving/service/RedisServingService.java +++ b/serving/src/main/java/feast/serving/service/RedisServingService.java @@ -170,6 +170,7 @@ private RedisKey makeRedisKey( String featureSet, List featureSetEntityNames, EntityRow entityRow) { RedisKey.Builder builder = RedisKey.newBuilder().setFeatureSet(featureSet); Map fieldsMap = entityRow.getFieldsMap(); + featureSetEntityNames.sort(String::compareTo); for (int i = 0; i < featureSetEntityNames.size(); i++) { String entityName = featureSetEntityNames.get(i); From 5801e58cce47fadbd6db175c925aefd435f74927 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com> Date: Mon, 23 Dec 2019 17:05:57 +0800 Subject: [PATCH 03/17] Remove alpha v1 from java package name (#387) --- .../main/java/com/gojek/feast/{v1alpha1 => }/FeastClient.java | 2 +- .../main/java/com/gojek/feast/{v1alpha1 => }/RequestUtil.java | 2 +- sdk/java/src/main/java/com/gojek/feast/{v1alpha1 => }/Row.java | 2 +- .../java/com/gojek/feast/{v1alpha1 => }/RequestUtilTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename sdk/java/src/main/java/com/gojek/feast/{v1alpha1 => }/FeastClient.java (99%) rename sdk/java/src/main/java/com/gojek/feast/{v1alpha1 => }/RequestUtil.java (98%) rename sdk/java/src/main/java/com/gojek/feast/{v1alpha1 => }/Row.java (99%) rename sdk/java/src/test/java/com/gojek/feast/{v1alpha1 => }/RequestUtilTest.java (99%) diff --git a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java similarity index 99% rename from sdk/java/src/main/java/com/gojek/feast/v1alpha1/FeastClient.java rename to sdk/java/src/main/java/com/gojek/feast/FeastClient.java index b7a3e78ab13..91ddd2a442c 100644 --- a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.gojek.feast.v1alpha1; +package com.gojek.feast; import feast.serving.ServingAPIProto.FeatureSetRequest; import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; diff --git a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/RequestUtil.java b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java similarity index 98% rename from sdk/java/src/main/java/com/gojek/feast/v1alpha1/RequestUtil.java rename to sdk/java/src/main/java/com/gojek/feast/RequestUtil.java index 72fbe289f2f..e80b40bad9c 100644 --- a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/RequestUtil.java +++ b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.gojek.feast.v1alpha1; +package com.gojek.feast; import feast.serving.ServingAPIProto.FeatureSetRequest; import java.util.ArrayList; diff --git a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java similarity index 99% rename from sdk/java/src/main/java/com/gojek/feast/v1alpha1/Row.java rename to sdk/java/src/main/java/com/gojek/feast/Row.java index 77f9f298873..9366fe1bb03 100644 --- a/sdk/java/src/main/java/com/gojek/feast/v1alpha1/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.gojek.feast.v1alpha1; +package com.gojek.feast; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; diff --git a/sdk/java/src/test/java/com/gojek/feast/v1alpha1/RequestUtilTest.java b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java similarity index 99% rename from sdk/java/src/test/java/com/gojek/feast/v1alpha1/RequestUtilTest.java rename to sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java index 5f87ba01535..21c8bde15ec 100644 --- a/sdk/java/src/test/java/com/gojek/feast/v1alpha1/RequestUtilTest.java +++ b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.gojek.feast.v1alpha1; +package com.gojek.feast; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; From 2fcddaa154d8fc75d442d71cb15bb7174db35d2a Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 26 Dec 2019 09:11:19 +0800 Subject: [PATCH 04/17] Always set destination table in BigQuery query config in Feast Batch Serving so it can handle large results (#392) * Update BQ query config to always set destination table, so that it can work with large results Refer to: https://cloud.google.com/bigquery/quotas#query_jobs, maximum reponse-size bullet point. * Replace prefix for temp table name * Set expiry on entity rows table * Include exception message in error description GRPC client such as Feast Python SDK will usually not show error cause only error description * Code cleanup * Update batch-retrieval e2e test. Output rows may not have the same order as requested entity rows --- .../service/BigQueryServingService.java | 46 +++++++++++++------ .../bigquery/BatchRetrievalQueryRunnable.java | 35 ++++++++++++-- .../store/bigquery/SubqueryCallable.java | 14 ++++++ tests/e2e/bq-batch-retrieval.py | 5 +- 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/BigQueryServingService.java b/serving/src/main/java/feast/serving/service/BigQueryServingService.java index 701e146ee5d..7a950e3c8a9 100644 --- a/serving/src/main/java/feast/serving/service/BigQueryServingService.java +++ b/serving/src/main/java/feast/serving/service/BigQueryServingService.java @@ -32,6 +32,7 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; import com.google.cloud.storage.Storage; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.serving.ServingAPIProto; @@ -56,10 +57,13 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import org.joda.time.Duration; import org.slf4j.Logger; public class BigQueryServingService implements ServingService { + // Default no of millis for which a temporary table should exist before it is deleted in BigQuery. + public static final long TEMP_TABLE_EXPIRY_DURATION_MS = Duration.standardDays(1).getMillis(); private static final Logger log = org.slf4j.LoggerFactory.getLogger(BigQueryServingService.class); private final BigQuery bigquery; @@ -182,15 +186,15 @@ private Table loadEntities(DatasetSource datasetSource) { switch (datasetSource.getDatasetSourceCase()) { case FILE_SOURCE: try { - String tableName = generateTemporaryTableName(); - log.info("Loading entity dataset to table {}.{}.{}", projectId, datasetId, tableName); - TableId tableId = TableId.of(projectId, datasetId, tableName); - // Currently only avro supported + // Currently only AVRO format is supported if (datasetSource.getFileSource().getDataFormat() != DataFormat.DATA_FORMAT_AVRO) { throw Status.INVALID_ARGUMENT - .withDescription("Invalid file format, only avro supported") + .withDescription("Invalid file format, only AVRO is supported.") .asRuntimeException(); } + + TableId tableId = TableId.of(projectId, datasetId, createTempTableName()); + log.info("Loading entity rows to: {}.{}.{}", projectId, datasetId, tableId.getTable()); LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.of( tableId, datasetSource.getFileSource().getFileUrisList(), FormatOptions.avro()); @@ -198,6 +202,13 @@ private Table loadEntities(DatasetSource datasetSource) { loadJobConfiguration.toBuilder().setUseAvroLogicalTypes(true).build(); Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); job.waitFor(); + TableInfo expiry = + bigquery + .getTable(tableId) + .toBuilder() + .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) + .build(); + bigquery.update(expiry); loadedEntityTable = bigquery.getTable(tableId); if (!loadedEntityTable.exists()) { throw new RuntimeException( @@ -207,7 +218,7 @@ private Table loadEntities(DatasetSource datasetSource) { } catch (Exception e) { log.error("Exception has occurred in loadEntities method: ", e); throw Status.INTERNAL - .withDescription("Failed to load entity dataset into store") + .withDescription("Failed to load entity dataset into store: " + e.toString()) .withCause(e) .asRuntimeException(); } @@ -219,20 +230,23 @@ private Table loadEntities(DatasetSource datasetSource) { } } - private String generateTemporaryTableName() { - String source = String.format("feastserving%d", System.currentTimeMillis()); - String guid = UUID.nameUUIDFromBytes(source.getBytes()).toString(); - String suffix = guid.substring(0, Math.min(guid.length(), 10)).replaceAll("-", ""); - return String.format("temp_%s", suffix); - } - private TableId generateUUIDs(Table loadedEntityTable) { try { String uuidQuery = createEntityTableUUIDQuery(generateFullTableName(loadedEntityTable.getTableId())); - QueryJobConfiguration queryJobConfig = QueryJobConfiguration.newBuilder(uuidQuery).build(); + QueryJobConfiguration queryJobConfig = + QueryJobConfiguration.newBuilder(uuidQuery) + .setDestinationTable(TableId.of(projectId, datasetId, createTempTableName())) + .build(); Job queryJob = bigquery.create(JobInfo.of(queryJobConfig)); queryJob.waitFor(); + TableInfo expiry = + bigquery + .getTable(queryJobConfig.getDestinationTable()) + .toBuilder() + .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) + .build(); + bigquery.update(expiry); queryJobConfig = queryJob.getConfiguration(); return queryJobConfig.getDestinationTable(); } catch (InterruptedException | BigQueryException e) { @@ -242,4 +256,8 @@ private TableId generateUUIDs(Table loadedEntityTable) { .asRuntimeException(); } } + + public static String createTempTableName() { + return "_" + UUID.randomUUID().toString().replace("-", ""); + } } diff --git a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java index 2d51547d0e7..47587e1d0ef 100644 --- a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java +++ b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java @@ -16,6 +16,8 @@ */ package feast.serving.store.bigquery; +import static feast.serving.service.BigQueryServingService.TEMP_TABLE_EXPIRY_DURATION_MS; +import static feast.serving.service.BigQueryServingService.createTempTableName; import static feast.serving.store.bigquery.QueryTemplater.createTimestampLimitQuery; import com.google.auto.value.AutoValue; @@ -27,6 +29,8 @@ import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TableResult; import com.google.cloud.storage.Blob; import com.google.cloud.storage.Storage; @@ -175,15 +179,17 @@ Job runBatchQuery(List featureSetQueries) ExecutorCompletionService executorCompletionService = new ExecutorCompletionService<>(executorService); - List featureSetInfos = new ArrayList<>(); for (int i = 0; i < featureSetQueries.size(); i++) { QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(featureSetQueries.get(i)).build(); + QueryJobConfiguration.newBuilder(featureSetQueries.get(i)) + .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) + .build(); Job subqueryJob = bigquery().create(JobInfo.of(queryJobConfig)); executorCompletionService.submit( SubqueryCallable.builder() + .setBigquery(bigquery()) .setFeatureSetInfo(featureSetInfos().get(i)) .setSubqueryJob(subqueryJob) .build()); @@ -191,7 +197,8 @@ Job runBatchQuery(List featureSetQueries) for (int i = 0; i < featureSetQueries.size(); i++) { try { - FeatureSetInfo featureSetInfo = executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS); + FeatureSetInfo featureSetInfo = + executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS); featureSetInfos.add(featureSetInfo); } catch (InterruptedException | ExecutionException | TimeoutException e) { jobService() @@ -214,9 +221,20 @@ Job runBatchQuery(List featureSetQueries) String joinQuery = QueryTemplater.createJoinQuery( featureSetInfos, entityTableColumnNames(), entityTableName()); - QueryJobConfiguration queryJobConfig = QueryJobConfiguration.newBuilder(joinQuery).build(); + QueryJobConfiguration queryJobConfig = + QueryJobConfiguration.newBuilder(joinQuery) + .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) + .build(); queryJob = bigquery().create(JobInfo.of(queryJobConfig)); queryJob.waitFor(); + TableInfo expiry = + bigquery() + .getTable(queryJobConfig.getDestinationTable()) + .toBuilder() + .setExpirationTime( + System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) + .build(); + bigquery().update(expiry); return queryJob; } @@ -248,10 +266,19 @@ private FieldValueList getTimestampLimits(String entityTableName) { QueryJobConfiguration getTimestampLimitsQuery = QueryJobConfiguration.newBuilder(createTimestampLimitQuery(entityTableName)) .setDefaultDataset(DatasetId.of(projectId(), datasetId())) + .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) .build(); try { Job job = bigquery().create(JobInfo.of(getTimestampLimitsQuery)); TableResult getTimestampLimitsQueryResult = job.waitFor().getQueryResults(); + TableInfo expiry = + bigquery() + .getTable(getTimestampLimitsQuery.getDestinationTable()) + .toBuilder() + .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) + .build(); + bigquery().update(expiry); + FieldValueList result = null; for (FieldValueList fields : getTimestampLimitsQueryResult.getValues()) { result = fields; diff --git a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java index 3c28194e7a3..b2a9009a749 100644 --- a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java +++ b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java @@ -16,13 +16,16 @@ */ package feast.serving.store.bigquery; +import static feast.serving.service.BigQueryServingService.TEMP_TABLE_EXPIRY_DURATION_MS; import static feast.serving.store.bigquery.QueryTemplater.generateFullTableName; import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; import feast.serving.store.bigquery.model.FeatureSetInfo; import java.util.concurrent.Callable; @@ -33,6 +36,8 @@ @AutoValue public abstract class SubqueryCallable implements Callable { + public abstract BigQuery bigquery(); + public abstract FeatureSetInfo featureSetInfo(); public abstract Job subqueryJob(); @@ -44,6 +49,8 @@ public static Builder builder() { @AutoValue.Builder public abstract static class Builder { + public abstract Builder setBigquery(BigQuery bigquery); + public abstract Builder setFeatureSetInfo(FeatureSetInfo featureSetInfo); public abstract Builder setSubqueryJob(Job subqueryJob); @@ -57,6 +64,13 @@ public FeatureSetInfo call() throws BigQueryException, InterruptedException { subqueryJob().waitFor(); subqueryConfig = subqueryJob().getConfiguration(); TableId destinationTable = subqueryConfig.getDestinationTable(); + TableInfo expiry = + bigquery() + .getTable(destinationTable) + .toBuilder() + .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) + .build(); + bigquery().update(expiry); String fullTablePath = generateFullTableName(destinationTable); return new FeatureSetInfo(featureSetInfo(), fullTablePath); diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq-batch-retrieval.py index 067dd14a2fb..2d6668eaa86 100644 --- a/tests/e2e/bq-batch-retrieval.py +++ b/tests/e2e/bq-batch-retrieval.py @@ -14,6 +14,7 @@ from feast.type_map import ValueType from google.protobuf.duration_pb2 import Duration +pd.set_option('display.max_columns', None) @pytest.fixture(scope="module") def core_url(pytestconfig): @@ -112,8 +113,8 @@ def test_additional_columns_in_entity_table(client): feature_retrieval_job = client.get_batch_features( entity_rows=entity_df, feature_ids=["additional_columns:1:feature_value"] ) - output = feature_retrieval_job.to_dataframe() - print(output.head()) + output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"]) + print(output.head(10)) assert np.allclose(output["additional_float_col"], entity_df["additional_float_col"]) assert output["additional_string_col"].to_list() == entity_df["additional_string_col"].to_list() From 77229eb91552bf40b25a73d5a7b21e01bf03aa01 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 22 Dec 2019 11:23:29 +0800 Subject: [PATCH 05/17] Rebasing changes (#355) --- .prow/config.yaml | 22 +- sdk/__init__.py | 0 sdk/python/feast/client.py | 195 +++++++---- sdk/python/feast/feature_set.py | 208 +++++++++++- sdk/python/feast/loaders/abstract_producer.py | 248 ++++++++++++++ sdk/python/feast/loaders/ingest.py | 305 ++++++------------ sdk/python/feast/type_map.py | 166 +++++++++- sdk/python/setup.py | 1 + sdk/python/tests/test_client.py | 34 +- 9 files changed, 887 insertions(+), 292 deletions(-) create mode 100644 sdk/__init__.py create mode 100644 sdk/python/feast/loaders/abstract_producer.py diff --git a/.prow/config.yaml b/.prow/config.yaml index 41f95180fbb..4b6e352a12f 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -145,18 +145,18 @@ presubmits: postsubmits: gojek/feast: - name: publish-python-sdk - decorate: true + decorate: true spec: containers: - image: python:3 command: - sh - - -c + - -c - | .prow/scripts/publish-python-sdk.sh \ --directory-path sdk/python --repository pypi volumeMounts: - - name: pypirc + - name: pypirc mountPath: /root/.pypirc subPath: .pypirc readOnly: true @@ -170,7 +170,7 @@ postsubmits: - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ - name: publish-docker-images - decorate: true + decorate: true spec: containers: - image: google/cloud-sdk:273.0.0 @@ -182,14 +182,14 @@ postsubmits: --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ --output-dir $PWD/ - if [ $PULL_BASE_REF == "master" ]; then - + if [ $PULL_BASE_REF == "master" ]; then + .prow/scripts/publish-docker-image.sh \ --repository gcr.io/kf-feast/feast-core \ --tag dev \ --file infra/docker/core/Dockerfile \ --google-service-account-file /etc/gcloud/service-account.json - + .prow/scripts/publish-docker-image.sh \ --repository gcr.io/kf-feast/feast-serving \ --tag dev \ @@ -203,13 +203,13 @@ postsubmits: docker push gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA} else - + .prow/scripts/publish-docker-image.sh \ --repository gcr.io/kf-feast/feast-core \ --tag ${PULL_BASE_REF:1} \ --file infra/docker/core/Dockerfile \ --google-service-account-file /etc/gcloud/service-account.json - + .prow/scripts/publish-docker-image.sh \ --repository gcr.io/kf-feast/feast-serving \ --tag ${PULL_BASE_REF:1} \ @@ -244,7 +244,7 @@ postsubmits: - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ - name: publish-helm-chart - decorate: true + decorate: true spec: containers: - image: google/cloud-sdk:273.0.0-slim @@ -253,7 +253,7 @@ postsubmits: - -c - | gcloud auth activate-service-account --key-file /etc/gcloud/service-account.json - + curl -s https://get.helm.sh/helm-v2.16.1-linux-amd64.tar.gz | tar -C /tmp -xz mv /tmp/linux-amd64/helm /usr/bin/helm helm init --client-only diff --git a/sdk/__init__.py b/sdk/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 3af2e12a91e..646de343d52 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -15,10 +15,11 @@ import logging import os -import sys +import time from collections import OrderedDict from typing import Dict, Union from typing import List + import grpc import pandas as pd import pyarrow as pa @@ -33,11 +34,13 @@ GetFeatureSetResponse, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.exceptions import format_grpc_exception +from feast.core.FeatureSet_pb2 import FeatureSetStatus from feast.feature_set import FeatureSet, Entity from feast.job import Job +from feast.loaders.abstract_producer import get_producer from feast.loaders.file import export_dataframe_to_staging_location -from feast.loaders.ingest import ingest_table_to_kafka +from feast.loaders.ingest import KAFKA_CHUNK_PRODUCTION_TIMEOUT +from feast.loaders.ingest import get_feature_row_chunks from feast.serving.ServingService_pb2 import GetFeastServingInfoResponse from feast.serving.ServingService_pb2 import ( GetOnlineFeaturesRequest, @@ -257,7 +260,7 @@ def _apply_feature_set(self, feature_set: FeatureSet): print(f"No change detected or applied: {feature_set.name}") # Deep copy from the returned feature set to the local feature set - feature_set.update_from_feature_set(applied_fs) + feature_set._update_from_feature_set(applied_fs) def list_feature_sets(self) -> List[FeatureSet]: """ @@ -470,35 +473,55 @@ def get_online_features( ) # type: GetOnlineFeaturesResponse def ingest( - self, - feature_set: Union[str, FeatureSet], - source: Union[pd.DataFrame, str], - version: int = None, - force_update: bool = False, - max_workers: int = CPU_COUNT, - disable_progress_bar: bool = False, - chunk_size: int = 5000, - timeout: int = None, - ): + self, + feature_set: Union[str, FeatureSet], + source: Union[pd.DataFrame, str], + chunk_size: int = 10000, + version: int = None, + force_update: bool = False, + max_workers: int = max(CPU_COUNT - 1, 1), + disable_progress_bar: bool = False, + timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT + ) -> None: """ Loads feature data into Feast for a specific feature set. Args: - feature_set: Name of feature set or a feature set object - source: Either a file path or Pandas Dataframe to ingest into Feast + feature_set (typing.Union[str, FeatureSet]): + Feature set object or the string name of the feature set + (without a version). + + source (typing.Union[pd.DataFrame, str]): + Either a file path or Pandas Dataframe to ingest into Feast Files that are currently supported: - * parquet - * csv - * json - version: Feature set version - force_update: Automatically update feature set based on source data - prior to ingesting. This will also register changes to Feast - max_workers: Number of worker processes to use to encode values - disable_progress_bar: Disable printing of progress statistics - chunk_size: Maximum amount of rows to load into memory and ingest at - a time - timeout: Seconds to wait before ingestion times out + * parquet + * csv + * json + + chunk_size (int): + Amount of rows to load and ingest at a time. + + version (int): + Feature set version. + + force_update (bool): + Automatically update feature set based on source data prior to + ingesting. This will also register changes to Feast. + + max_workers (int): + Number of worker processes to use to encode values. + + disable_progress_bar (bool): + Disable printing of progress statistics. + + timeout (int): + Timeout in seconds to wait for completion. + + Returns: + None: + None """ + if isinstance(feature_set, FeatureSet): name = feature_set.name if version is None: @@ -508,35 +531,69 @@ def ingest( else: raise Exception(f"Feature set name must be provided") - table = _read_table_from_source(source) + # Read table and get row count + tmp_table_name = _read_table_from_source( + source, chunk_size, max_workers + ) - # Update the feature set based on DataFrame schema - if force_update: - # Use a small as reference DataFrame to infer fields - ref_df = table.to_batches(max_chunksize=20)[0].to_pandas() + pq_file = pq.ParquetFile(tmp_table_name) - feature_set.infer_fields_from_df( - ref_df, discard_unused_fields=True, replace_existing_features=True + row_count = pq_file.metadata.num_rows + + # Update the feature set based on PyArrow table of first row group + if force_update: + feature_set.infer_fields_from_pa( + table=pq_file.read_row_group(0), + discard_unused_fields=True, + replace_existing_features=True ) self.apply(feature_set) feature_set = self.get_feature_set(name, version) - if feature_set.source.source_type == "Kafka": - ingest_table_to_kafka( - feature_set=feature_set, - table=table, - max_workers=max_workers, - disable_pbar=disable_progress_bar, - chunk_size=chunk_size, - timeout=timeout, - ) - else: - raise Exception( - f"Could not determine source type for feature set " - f'"{feature_set.name}" with source type ' - f'"{feature_set.source.source_type}"' - ) + try: + # Kafka configs + brokers = feature_set.get_kafka_source_brokers() + topic = feature_set.get_kafka_source_topic() + producer = get_producer(brokers, row_count, disable_progress_bar) + + # Loop optimization declarations + produce = producer.produce + flush = producer.flush + + # Transform and push data to Kafka + if feature_set.source.source_type == "Kafka": + for chunk in get_feature_row_chunks( + file=tmp_table_name, + row_groups=list(range(pq_file.num_row_groups)), + fs=feature_set, + max_workers=max_workers): + + # Push FeatureRow one chunk at a time to kafka + for serialized_row in chunk: + produce(topic=topic, value=serialized_row) + + # Force a flush after each chunk + flush(timeout=timeout) + + # Remove chunk from memory + del chunk + + else: + raise Exception( + f"Could not determine source type for feature set " + f'"{feature_set.name}" with source type ' + f'"{feature_set.source.source_type}"' + ) + + # Print ingestion statistics + producer.print_results() + finally: + # Remove parquet file(s) that were created earlier + print("Removing temporary file(s)...") + os.remove(tmp_table_name) + + return None def _build_feature_set_request(feature_ids: List[str]) -> List[FeatureSetRequest]: @@ -566,18 +623,38 @@ def _build_feature_set_request(feature_ids: List[str]) -> List[FeatureSetRequest return list(feature_set_request.values()) -def _read_table_from_source(source: Union[pd.DataFrame, str]) -> pa.lib.Table: +def _read_table_from_source( + source: Union[pd.DataFrame, str], + chunk_size: int, + max_workers: int +) -> str: """ Infers a data source type (path or Pandas Dataframe) and reads it in as a PyArrow Table. + The PyArrow Table that is read will be written to a parquet file with row + group size determined by the minimum of: + * (table.num_rows / max_workers) + * chunk_size + + The parquet file that is created will be passed as file path to the + multiprocessing pool workers. + Args: - source: Either a string path or Pandas Dataframe + source (Union[pd.DataFrame, str]): + Either a string path or Pandas DataFrame. + + chunk_size (int): + Number of worker processes to use to encode values. + + max_workers (int): + Amount of rows to load and ingest at a time. Returns: - PyArrow table + str: Path to parquet file that was created. """ - # Pandas dataframe detected + + # Pandas DataFrame detected if isinstance(source, pd.DataFrame): table = pa.Table.from_pandas(df=source) @@ -601,4 +678,14 @@ def _read_table_from_source(source: Union[pd.DataFrame, str]) -> pa.lib.Table: # Ensure that PyArrow table is initialised assert isinstance(table, pa.lib.Table) - return table + + # Write table as parquet file with a specified row_group_size + tmp_table_name = f"{int(time.time())}.parquet" + row_group_size = min(int(table.num_rows/max_workers), chunk_size) + pq.write_table(table=table, where=tmp_table_name, + row_group_size=row_group_size) + + # Remove table from memory + del table + + return tmp_table_name diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 893378e8fac..28381891f59 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -13,21 +13,27 @@ # limitations under the License. -import pandas as pd -from typing import List, Optional from collections import OrderedDict from typing import Dict -from feast.source import Source -from pandas.api.types import is_datetime64_ns_dtype +from typing import List, Optional + +import pandas as pd +import pyarrow as pa +from feast.core.FeatureSet_pb2 import FeatureSet as FeatureSetProto +from feast.core.FeatureSet_pb2 import FeatureSetMeta as FeatureSetMetaProto +from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto from feast.entity import Entity from feast.feature import Feature, Field -from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto -from google.protobuf.duration_pb2 import Duration +from feast.loaders import yaml as feast_yaml +from feast.source import Source +from feast.type_map import DATETIME_COLUMN +from feast.type_map import pa_to_feast_value_type from feast.type_map import python_type_to_feast_value_type -from google.protobuf.json_format import MessageToJson from google.protobuf import json_format -from feast.type_map import DATETIME_COLUMN -from feast.loaders import yaml as feast_yaml +from google.protobuf.duration_pb2 import Duration +from google.protobuf.json_format import MessageToJson +from pandas.api.types import is_datetime64_ns_dtype +from pyarrow.lib import TimestampType class FeatureSet: @@ -256,7 +262,6 @@ def infer_fields_from_df( rows_to_sample: int = 100, ): """ - Adds fields (Features or Entities) to a feature set based on the schema of a Datatframe. Only Pandas dataframes are supported. All columns are detected as features, so setting at least one entity manually is @@ -283,6 +288,7 @@ def infer_fields_from_df( must have consistent types, even values within list types must be homogeneous """ + if entities is None: entities = list() if features is None: @@ -373,7 +379,187 @@ def infer_fields_from_df( self._fields = new_fields print(output_log) - def update_from_feature_set(self, feature_set): + def infer_fields_from_pa( + self, table: pa.lib.Table, + entities: Optional[List[Entity]] = None, + features: Optional[List[Feature]] = None, + replace_existing_features: bool = False, + replace_existing_entities: bool = False, + discard_unused_fields: bool = False + ) -> None: + """ + Adds fields (Features or Entities) to a feature set based on the schema + of a PyArrow table. Only PyArrow tables are supported. All columns are + detected as features, so setting at least one entity manually is + advised. + + + Args: + table (pyarrow.lib.Table): + PyArrow table to read schema from. + + entities (Optional[List[Entity]]): + List of entities that will be set manually and not inferred. + These will take precedence over any existing entities or + entities found in the PyArrow table. + + features (Optional[List[Feature]]): + List of features that will be set manually and not inferred. + These will take precedence over any existing feature or features + found in the PyArrow table. + + replace_existing_features (bool): + Boolean flag. If true, will replace existing features in this + feature set with features found in dataframe. If false, will + skip conflicting features. + + replace_existing_entities (bool): + Boolean flag. If true, will replace existing entities in this + feature set with features found in dataframe. If false, will + skip conflicting entities. + + discard_unused_fields (bool): + Boolean flag. Setting this to True will discard any existing + fields that are not found in the dataset or provided by the + user. + + Returns: + None: + None + """ + if entities is None: + entities = list() + if features is None: + features = list() + + # Validate whether the datetime column exists with the right name + if DATETIME_COLUMN not in table.column_names: + raise Exception("No column 'datetime'") + + # Validate the date type for the datetime column + if not isinstance(table.column(DATETIME_COLUMN).type, TimestampType): + raise Exception( + "Column 'datetime' does not have the correct type: datetime64[ms]" + ) + + # Create dictionary of fields that will not be inferred (manually set) + provided_fields = OrderedDict() + + for field in entities + features: + if not isinstance(field, Field): + raise Exception(f"Invalid field object type provided {type(field)}") + if field.name not in provided_fields: + provided_fields[field.name] = field + else: + raise Exception(f"Duplicate field name detected {field.name}.") + + new_fields = self._fields.copy() + output_log = "" + + # Add in provided fields + for name, field in provided_fields.items(): + if name in new_fields.keys(): + upsert_message = "created" + else: + upsert_message = "updated (replacing an existing field)" + + output_log += ( + f"{type(field).__name__} {field.name}" + f"({field.dtype}) manually {upsert_message}.\n" + ) + new_fields[name] = field + + # Iterate over all of the column names and create features + for column in table.column_names: + column = column.strip() + + # Skip datetime column + if DATETIME_COLUMN in column: + continue + + # Skip user provided fields + if column in provided_fields.keys(): + continue + + # Only overwrite conflicting fields if replacement is allowed + if column in new_fields: + if ( + isinstance(self._fields[column], Feature) + and not replace_existing_features + ): + continue + + if ( + isinstance(self._fields[column], Entity) + and not replace_existing_entities + ): + continue + + # Store this fields as a feature + # TODO: (Minor) Change the parameter name from dtype to patype + new_fields[column] = Feature( + name=column, + dtype=self._infer_pa_column_type(table.column(column)) + ) + + output_log += f"{type(new_fields[column]).__name__} {new_fields[column].name} ({new_fields[column].dtype}) added from PyArrow Table.\n" + + # Discard unused fields from feature set + if discard_unused_fields: + keys_to_remove = [] + for key in new_fields.keys(): + if not (key in table.column_names or key in provided_fields.keys()): + output_log += f"{type(new_fields[key]).__name__} {new_fields[key].name} ({new_fields[key].dtype}) removed because it is unused.\n" + keys_to_remove.append(key) + for key in keys_to_remove: + del new_fields[key] + + # Update feature set + self._fields = new_fields + print(output_log) + + def _infer_pd_column_type(self, column, series, rows_to_sample): + dtype = None + sample_count = 0 + + # Loop over all rows for this column to infer types + for key, value in series.iteritems(): + sample_count += 1 + # Stop sampling at the row limit + if sample_count > rows_to_sample: + continue + + # Infer the specific type for this row + current_dtype = python_type_to_feast_value_type(name=column, value=value) + + # Make sure the type is consistent for column + if dtype: + if dtype != current_dtype: + raise ValueError( + f"Type mismatch detected in column {column}. Both " + f"the types {current_dtype} and {dtype} " + f"have been found." + ) + else: + # Store dtype in field to type map if it isnt already + dtype = current_dtype + + return dtype + + def _infer_pa_column_type(self, column: pa.lib.ChunkedArray): + """ + Infers the PyArrow column type. + + :param column: Column from a PyArrow table + :type column: pa.lib.ChunkedArray + :return: + :rtype: + """ + # Validates the column to ensure that value types are consistent + column.validate() + return pa_to_feast_value_type(column) + + def _update_from_feature_set(self, feature_set): """ Deep replaces one feature set with another diff --git a/sdk/python/feast/loaders/abstract_producer.py b/sdk/python/feast/loaders/abstract_producer.py new file mode 100644 index 00000000000..884ae49984c --- /dev/null +++ b/sdk/python/feast/loaders/abstract_producer.py @@ -0,0 +1,248 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +from tqdm import tqdm + + +class AbstractProducer: + """ + Abstract class for Kafka producers + """ + + def __init__( + self, + brokers: str, + row_count: int, + disable_progress_bar: bool + ): + self.brokers = brokers + self.row_count = row_count + self.error_count = 0 + self.last_exception = "" + + # Progress bar will always display average rate + self.pbar = tqdm( + total=row_count, + unit="rows", + smoothing=0, + disable=disable_progress_bar + ) + + def produce(self, topic: str, data: str): + message = "{} should implement a produce method".format( + self.__class__.__name__) + raise NotImplementedError(message) + + def flush(self, timeout: int): + message = "{} should implement a flush method".format( + self.__class__.__name__) + raise NotImplementedError(message) + + def _inc_pbar(self, meta): + self.pbar.update(1) + + def _set_error(self, exception: str): + self.error_count += 1 + self.last_exception = exception + + def print_results(self) -> None: + """ + Print ingestion statistics. + + Returns: + None: None + """ + # Refresh and close tqdm progress bar + self.pbar.refresh() + + self.pbar.close() + + print("Ingestion complete!") + + failed_message = ( + "" + if self.error_count == 0 + else f"\nFail: {self.error_count / self.row_count}" + ) + + last_exception_message = ( + "" + if self.last_exception == "" + else f"\nLast exception:\n{self.last_exception}" + ) + + print( + f"\nIngestion statistics:" + f"\nSuccess: {self.pbar.n}/{self.row_count}" + f"{failed_message}" + f"{last_exception_message}" + ) + return None + + +class ConfluentProducer(AbstractProducer): + """ + Concrete implementation of Confluent Kafka producer (confluent-kafka) + """ + + def __init__( + self, + brokers: str, + row_count: int, + disable_progress_bar: bool + ): + from confluent_kafka import Producer + self.producer = Producer({"bootstrap.servers": brokers}) + super().__init__(brokers, row_count, disable_progress_bar) + + def produce(self, topic: str, value: bytes) -> None: + """ + Generic produce that implements confluent-kafka's produce method to + push a byte encoded object into a Kafka topic. + + Args: + topic (str): Kafka topic. + value (bytes): Byte encoded object. + + Returns: + None: None. + """ + + try: + self.producer.produce( + topic, value=value, callback=self._delivery_callback) + # Serve delivery callback queue. + # NOTE: Since produce() is an asynchronous API this poll() call + # will most likely not serve the delivery callback for the + # last produce()d message. + self.producer.poll(0) + except Exception as ex: + self._set_error(str(ex)) + + return None + + def flush(self, timeout: Optional[int]): + """ + Generic flush that implements confluent-kafka's flush method. + + Args: + timeout (Optional[int]): Timeout in seconds to wait for completion. + + Returns: + int: Number of messages still in queue. + """ + return self.producer.flush(timeout=timeout) + + def _delivery_callback(self, err: str, msg) -> None: + """ + Optional per-message delivery callback (triggered by poll() or flush()) + when a message has been successfully delivered or permanently failed + delivery (after retries). + + Although the msg argument is not used, the current method signature is + required as specified in the confluent-kafka documentation. + + Args: + err (str): Error message. + msg (): Kafka message. + + Returns: + None + """ + if err: + self._set_error(err) + else: + self._inc_pbar(None) + + +class KafkaPythonProducer(AbstractProducer): + """ + Concrete implementation of Python Kafka producer (kafka-python) + """ + + def __init__( + self, + brokers: str, + row_count: int, + disable_progress_bar: bool + ): + from kafka import KafkaProducer + self.producer = KafkaProducer(bootstrap_servers=[brokers]) + super().__init__(brokers, row_count, disable_progress_bar) + + def produce(self, topic: str, value: bytes): + """ + Generic produce that implements kafka-python's send method to push a + byte encoded object into a Kafka topic. + + Args: + topic (str): Kafka topic. + value (bytes): Byte encoded object. + + Returns: + FutureRecordMetadata: resolves to RecordMetadata + + Raises: + KafkaTimeoutError: if unable to fetch topic metadata, or unable + to obtain memory buffer prior to configured max_block_ms + """ + return self.producer.send(topic, value=value).add_callback( + self._inc_pbar).add_errback(self._set_error) + + def flush(self, timeout: Optional[int]): + """ + Generic flush that implements kafka-python's flush method. + + Args: + timeout (Optional[int]): timeout in seconds to wait for completion. + + Returns: + None + + Raises: + KafkaTimeoutError: failure to flush buffered records within the + provided timeout + """ + return self.producer.flush(timeout=timeout) + + +def get_producer( + brokers: str, row_count: int, disable_progress_bar: bool +) -> Union[ConfluentProducer, KafkaPythonProducer]: + """ + Simple context helper function that returns a AbstractProducer object when + invoked. + + This helper function will try to import confluent-kafka as a producer first. + + This helper function will fallback to kafka-python if it fails to import + confluent-kafka. + + Args: + brokers (str): Kafka broker information with hostname and port. + row_count (int): Number of rows in table + + Returns: + Union[ConfluentProducer, KafkaPythonProducer]: + Concrete implementation of a Kafka producer. Ig can be: + * confluent-kafka producer + * kafka-python producer + """ + try: + return ConfluentProducer(brokers, row_count, disable_progress_bar) + except ImportError as e: + print("Unable to import confluent-kafka, falling back to kafka-python") + return KafkaPythonProducer(brokers, row_count, disable_progress_bar) diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 23ba2ecb3b4..527ab481fe0 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -1,18 +1,16 @@ import logging -import multiprocessing -import os -import time from functools import partial -from multiprocessing import Process, Queue, Pool -from typing import Iterable +from multiprocessing import Pool +from typing import Iterable, List + import pandas as pd -import pyarrow as pa +import pyarrow.parquet as pq +from feast.constants import DATETIME_COLUMN from feast.feature_set import FeatureSet -from feast.type_map import convert_dict_to_proto_values +from feast.type_map import pa_column_to_timestamp_proto_column, \ + pa_column_to_proto_column +from feast.types import Field_pb2 as FieldProto from feast.types.FeatureRow_pb2 import FeatureRow -from kafka import KafkaProducer -from tqdm import tqdm -from feast.constants import DATETIME_COLUMN _logger = logging.getLogger(__name__) @@ -21,221 +19,120 @@ FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" # type: str FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" # type: str BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300 -CPU_COUNT = os.cpu_count() # type: int KAFKA_CHUNK_PRODUCTION_TIMEOUT = 120 # type: int -def _kafka_feature_row_producer( - feature_row_queue: Queue, row_count: int, brokers, topic, ctx: dict, pbar: tqdm -): +def _encode_pa_tables( + file: str, + fs: FeatureSet, + row_group_idx: int, +) -> List[bytes]: """ - Pushes Feature Rows to Kafka. Reads rows from a queue. Function will run - until total row_count is reached. + Helper function to encode a PyArrow table(s) read from parquet file(s) into + FeatureRows. - Args: - feature_row_queue: Queue containing feature rows. - row_count: Total row count to process - brokers: Broker to push to - topic: Topic to push to - ctx: Context dict used to communicate with primary process - pbar: Progress bar object - """ - - # Callback for failed production to Kafka - def on_error(e): - # Save last exception - ctx["last_exception"] = e - - # Increment error count - if "error_count" in ctx: - ctx["error_count"] += 1 - else: - ctx["error_count"] = 1 - - # Callback for succeeded production to Kafka - def on_success(meta): - pbar.update() - - producer = KafkaProducer(bootstrap_servers=brokers) - processed_rows = 0 - - # Loop through feature rows until all rows are processed - while processed_rows < row_count: - # Wait if queue is empty - if feature_row_queue.empty(): - time.sleep(1) - producer.flush(timeout=KAFKA_CHUNK_PRODUCTION_TIMEOUT) - else: - while not feature_row_queue.empty(): - row = feature_row_queue.get() - if row is not None: - # Push row to Kafka - producer.send(topic, row.SerializeToString()).add_callback( - on_success - ).add_errback(on_error) - processed_rows += 1 - - # Force an occasional flush - if processed_rows % 10000 == 0: - producer.flush(timeout=KAFKA_CHUNK_PRODUCTION_TIMEOUT) - del row - pbar.refresh() - - # Ensure that all rows are pushed - producer.flush(timeout=KAFKA_CHUNK_PRODUCTION_TIMEOUT) - - # Using progress bar as counter is much faster than incrementing dict - ctx["success_count"] = pbar.n - pbar.close() - - -def _encode_pa_chunks( - tbl: pa.lib.Table, - fs: FeatureSet, - max_workers: int, - df_datetime_dtype: pd.DataFrame.dtypes, - chunk_size: int = 5000, -) -> Iterable[FeatureRow]: - """ - Generator function to encode rows in PyArrow table to FeatureRows by - breaking up the table into batches. + This function accepts a list of file directory pointing to many parquet + files. All parquet files must have the same schema. - Each batch will have its rows spread accross a pool of workers to be - transformed into FeatureRow objects. + Each parquet file will be read into as a table and encoded into FeatureRows + using a pool of max_workers workers. Args: - tbl: PyArrow table to be processed. - fs: FeatureSet describing PyArrow table. - max_workers: Maximum number of workers. - df_datetime_dtype: Pandas dtype of datetime column. - chunk_size: Maximum size of each chunk when PyArrow table is batched. - - Returns: - Iterable FeatureRow object. - """ - - pool = Pool(max_workers) - - # Create a partial function with static non-iterable arguments - func = partial( - convert_dict_to_proto_values, - df_datetime_dtype=df_datetime_dtype, - feature_set=fs, - ) - - for batch in tbl.to_batches(max_chunksize=chunk_size): - m_df = batch.to_pandas() - results = pool.map_async(func, m_df.to_dict("records")) - yield from results.get() + file (str): + File directory of all the parquet file to encode. + Parquet file must have more than one row group. - pool.close() - pool.join() - return + fs (feast.feature_set.FeatureSet): + FeatureSet describing parquet files. + row_group_idx(int): + Row group index to read and encode into byte like FeatureRow + protobuf objects. -def ingest_table_to_kafka( - feature_set: FeatureSet, - table: pa.lib.Table, - max_workers: int, - chunk_size: int = 5000, - disable_pbar: bool = False, - timeout: int = None, -) -> None: + Returns: + List[bytes]: + List of byte encoded FeatureRows from the parquet file. """ - Ingest a PyArrow Table to a Kafka topic based for a Feature Set + pq_file = pq.ParquetFile(file) + # Read parquet file as a PyArrow table + table = pq_file.read_row_group(row_group_idx) + + # Add datetime column + datetime_col = pa_column_to_timestamp_proto_column( + table.column(DATETIME_COLUMN)) + + # Preprocess the columns by converting all its values to Proto values + proto_columns = { + field_name: pa_column_to_proto_column(field.dtype, + table.column(field_name)) + for field_name, field in fs.fields.items() + } + + feature_set = f"{fs.name}:{fs.version}" + + # List to store result + feature_rows = [] + + # Loop optimization declaration(s) + field = FieldProto.Field + proto_items = proto_columns.items() + append = feature_rows.append + + # Iterate through the rows + for row_idx in range(table.num_rows): + feature_row = FeatureRow(event_timestamp=datetime_col[row_idx], + feature_set=feature_set) + # Loop optimization declaration + ext = feature_row.fields.extend + + # Insert field from each column + for k, v in proto_items: + ext([field(name=k, value=v[row_idx])]) + + # Append FeatureRow in byte string form + append(feature_row.SerializeToString()) + + return feature_rows + + +def get_feature_row_chunks( + file: str, + row_groups: List[int], + fs: FeatureSet, + max_workers: int +) -> Iterable[List[bytes]]: + """ + Iterator function to encode a PyArrow table read from a parquet file to + FeatureRow(s). Args: - feature_set: FeatureSet describing PyArrow table. - table: PyArrow table to be processed. - max_workers: Maximum number of workers. - chunk_size: Maximum size of each chunk when PyArrow table is batched. - disable_pbar: Flag to indicate if tqdm progress bar should be disabled. - timeout: Maximum time before method times out - """ + file (str): + File directory of the parquet file. The parquet file must have more + than one row group. - pbar = tqdm(unit="rows", total=table.num_rows, disable=disable_pbar) - - # Use a small DataFrame to validate feature set schema - ref_df = table.to_batches(max_chunksize=100)[0].to_pandas() - df_datetime_dtype = ref_df[DATETIME_COLUMN].dtype - - # Validate feature set schema - _validate_dataframe(ref_df, feature_set) - - # Create queue through which encoding and production will coordinate - row_queue = Queue() - - # Create a context object to send and receive information across processes - ctx = multiprocessing.Manager().dict( - {"success_count": 0, "error_count": 0, "last_exception": ""} - ) - - # Create producer to push feature rows to Kafka - ingestion_process = Process( - target=_kafka_feature_row_producer, - args=( - row_queue, - table.num_rows, - feature_set.get_kafka_source_brokers(), - feature_set.get_kafka_source_topic(), - ctx, - pbar, - ), - ) - - try: - # Start ingestion process - print( - f"\n(ingest table to kafka) Ingestion started for {feature_set.name}:{feature_set.version}" - ) - ingestion_process.start() - - # Iterate over chunks in the table and return feature rows - for row in _encode_pa_chunks( - tbl=table, - fs=feature_set, - max_workers=max_workers, - chunk_size=chunk_size, - df_datetime_dtype=df_datetime_dtype, - ): - # Push rows onto a queue for the production process to pick up - row_queue.put(row) - while row_queue.qsize() > chunk_size: - time.sleep(0.1) - row_queue.put(None) - except Exception as ex: - _logger.error(f"Exception occurred: {ex}") - finally: - # Wait for the Kafka production to complete - ingestion_process.join(timeout=timeout) - failed_message = ( - "" - if ctx["error_count"] == 0 - else f"\nFail: {ctx['error_count']}/{table.num_rows}" - ) + row_groups (List[int]): + Specific row group indexes to be read and transformed in the parquet + file. - last_exception_message = ( - "" - if ctx["last_exception"] == "" - else f"\nLast exception:\n{ctx['last_exception']}" - ) - print( - f"\nIngestion statistics:" - f"\nSuccess: {ctx['success_count']}/{table.num_rows}" - f"{failed_message}" - f"{last_exception_message}" - ) + fs (feast.feature_set.FeatureSet): + FeatureSet describing parquet files. + max_workers (int): + Maximum number of workers to spawn. -def _validate_dataframe(dataframe: pd.DataFrame, feature_set: FeatureSet): + Returns: + Iterable[List[bytes]]: + Iterable list of byte encoded FeatureRow(s). """ - Validates a Pandas dataframe based on a feature set - Args: - dataframe: Pandas dataframe - feature_set: Feature Set instance - """ + pool = Pool(max_workers) + func = partial(_encode_pa_tables, file, fs) + for chunk in pool.imap_unordered(func, row_groups): + yield chunk + return + +def validate_dataframe(dataframe: pd.DataFrame, feature_set: FeatureSet): if "datetime" not in dataframe.columns: raise ValueError( f'Dataframe does not contain entity "datetime" in columns {dataframe.columns}' diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 7573276d74a..ca13c2573bc 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -12,12 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime, timezone +from typing import List + import numpy as np import pandas as pd -from datetime import datetime, timezone -from feast.value_type import ValueType +import pyarrow as pa +from feast.constants import DATETIME_COLUMN +from feast.types import ( + FeatureRow_pb2 as FeatureRowProto, + Field_pb2 as FieldProto, +) from feast.types.Value_pb2 import ( Value as ProtoValue, + ValueType as ProtoValueType, Int64List, Int32List, BoolList, @@ -26,9 +34,9 @@ StringList, FloatList, ) -from feast.types import FeatureRow_pb2 as FeatureRowProto, Field_pb2 as FieldProto +from feast.value_type import ValueType from google.protobuf.timestamp_pb2 import Timestamp -from feast.constants import DATETIME_COLUMN +from pyarrow.lib import TimestampType def python_type_to_feast_value_type( @@ -104,9 +112,9 @@ def python_type_to_feast_value_type( return ValueType[common_item_value_type.name + "_LIST"] else: raise ValueError( - f"Value type for field {name} is {value.dtype.__str__()} " - f"but recursion is not allowed. Array types can only be one " - f"level deep." + f"Value type for field {name} is {value.dtype.__str__()} but " + f"recursion is not allowed. Array types can only be one level " + f"deep." ) return type_map[value.dtype.__str__()] @@ -160,7 +168,7 @@ def convert_series_to_proto_values(row: pd.Series): def convert_dict_to_proto_values( - row: dict, df_datetime_dtype: pd.DataFrame.dtypes, feature_set + row: dict, df_datetime_dtype: pd.DataFrame.dtypes, feature_set ) -> FeatureRowProto.FeatureRow: """ Encode a dictionary describing a feature row into a FeatureRows object. @@ -211,12 +219,14 @@ def _pd_datetime_to_timestamp_proto(dtype, value) -> Timestamp: # If timestamp does not contain timezone, we assume it is of local # timezone and adjust it to UTC local_timezone = datetime.now(timezone.utc).astimezone().tzinfo - value = value.tz_localize(local_timezone).tz_convert("UTC").tz_localize(None) + value = value.tz_localize(local_timezone).tz_convert("UTC").tz_localize( + None) return Timestamp(seconds=int(value.timestamp())) if dtype.__str__() == "datetime64[ns, UTC]": return Timestamp(seconds=int(value.timestamp())) else: - return Timestamp(seconds=np.datetime64(value).astype("int64") // 1000000) + return Timestamp( + seconds=np.datetime64(value).astype("int64") // 1000000) def _type_err(item, dtype): @@ -344,3 +354,139 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue: return ProtoValue(bool_val=value) raise Exception(f"Unsupported data type: ${str(type(value))}") + +def pa_to_feast_value_attr(pa_type: object): + """ + Returns the equivalent Feast ValueType string for the given pa.lib type. + + Args: + pa_type (object): + PyArrow type. + + Returns: + str: + Feast attribute name in Feast ValueType string-ed representation. + """ + # Mapping of PyArrow type to attribute name in Feast ValueType strings + type_map = { + "timestamp[ms]": "int64_val", + "int32": "int32_val", + "int64": "int64_val", + "double": "double_val", + "float": "float_val", + "string": "string_val", + "binary": "bytes_val", + "bool": "bool_val", + "list": "int32_list_val", + "list": "int64_list_val", + "list": "double_list_val", + "list": "float_list_val", + "list": "string_list_val", + "list": "bytes_list_val", + "list": "bool_list_val", + } + + return type_map[pa_type.__str__()] + + +def pa_to_value_type(pa_type: object): + """ + Returns the equivalent Feast ValueType for the given pa.lib type. + + Args: + pa_type (object): + PyArrow type. + + Returns: + feast.types.Value_pb2.ValueType: + Feast ValueType. + + """ + + # Mapping of PyArrow to attribute name in Feast ValueType + type_map = { + "timestamp[ms]": ProtoValueType.INT64, + "int32": ProtoValueType.INT32, + "int64": ProtoValueType.INT64, + "double": ProtoValueType.DOUBLE, + "float": ProtoValueType.FLOAT, + "string": ProtoValueType.STRING, + "binary": ProtoValueType.BYTES, + "bool": ProtoValueType.BOOL, + "list": ProtoValueType.INT32_LIST, + "list": ProtoValueType.INT64_LIST, + "list": ProtoValueType.DOUBLE_LIST, + "list": ProtoValueType.FLOAT_LIST, + "list": ProtoValueType.STRING_LIST, + "list": ProtoValueType.BYTES_LIST, + "list": ProtoValueType.BOOL_LIST, + } + return type_map[pa_type.__str__()] + + +def pa_to_feast_value_type( + value: object +) -> ValueType: + type_map = { + "timestamp[ms]": ValueType.INT64, + "int32": ValueType.INT32, + "int64": ValueType.INT64, + "double": ValueType.DOUBLE, + "float": ValueType.FLOAT, + "string": ValueType.STRING, + "binary": ValueType.BYTES, + "bool": ValueType.BOOL, + "list": ValueType.INT32_LIST, + "list": ValueType.INT64_LIST, + "list": ValueType.DOUBLE_LIST, + "list": ValueType.FLOAT_LIST, + "list": ValueType.STRING_LIST, + "list": ValueType.BYTES_LIST, + "list": ValueType.BOOL_LIST, + } + return type_map[value.type.__str__()] + + +def pa_column_to_timestamp_proto_column( + column: pa.lib.ChunkedArray +) -> Timestamp: + if not isinstance(column.type, TimestampType): + raise Exception("Only TimestampType columns are allowed") + + proto_column = [] + for val in column: + timestamp = Timestamp() + timestamp.FromMicroseconds( + micros=int(val.as_py().timestamp() * 1_000_000)) + proto_column.append(timestamp) + return proto_column + + +def pa_column_to_proto_column( + feast_value_type, + column: pa.lib.ChunkedArray +) -> List[ProtoValue]: + type_map = {ValueType.INT32: "int32_val", + ValueType.INT64: "int64_val", + ValueType.FLOAT: "float_val", + ValueType.DOUBLE: "double_val", + ValueType.STRING: "string_val", + ValueType.BYTES: "bytes_val", + ValueType.BOOL: "bool_val", + ValueType.BOOL_LIST: {"bool_list_val": BoolList}, + ValueType.BYTES_LIST: {"bytes_list_val": BytesList}, + ValueType.STRING_LIST: {"string_list_val": StringList}, + ValueType.FLOAT_LIST: {"float_list_val": FloatList}, + ValueType.DOUBLE_LIST: {"double_list_val": DoubleList}, + ValueType.INT32_LIST: {"int32_list_val": Int32List}, + ValueType.INT64_LIST: {"int64_list_val": Int64List}, } + + value = type_map[feast_value_type] + # Process list types + if type(value) == dict: + list_param_name = list(value.keys())[0] + return [ProtoValue( + **{list_param_name: value[list_param_name](val=x.as_py())}) + for x in column] + else: + return [ProtoValue(**{value: x.as_py()}) for x in column] diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 66cad904b01..9ac7225e80e 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -44,6 +44,7 @@ "pyarrow>=0.15.1", "numpy", "google", + "confluent_kafka" ] # README file from Feast repo root directory diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 2243ebfd1b3..f979c5a55df 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -381,10 +381,40 @@ def test_feature_set_ingest_success(self, dataframe, client, mocker): ) # Need to create a mock producer - with patch("feast.loaders.ingest.KafkaProducer") as mocked_queue: + with patch("feast.client.get_producer") as mocked_queue: # Ingest data into Feast client.ingest("driver-feature-set", dataframe) + @pytest.mark.parametrize("dataframe,exception", [(dataframes.GOOD, TimeoutError)]) + def test_feature_set_ingest_fail_if_pending( + self, dataframe, exception, client, mocker + ): + with pytest.raises(exception): + driver_fs = FeatureSet( + "driver-feature-set", + source=KafkaSource(brokers="kafka:9092", topic="test"), + ) + driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT)) + driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING)) + driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64)) + driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64)) + + # Register with Feast core + client.apply(driver_fs) + driver_fs = driver_fs.to_proto() + driver_fs.meta.status = FeatureSetStatus.STATUS_PENDING + + mocker.patch.object( + client._core_service_stub, + "GetFeatureSet", + return_value=GetFeatureSetResponse(feature_set=driver_fs), + ) + + # Need to create a mock producer + with patch("feast.client.get_producer") as mocked_queue: + # Ingest data into Feast + client.ingest("driver-feature-set", dataframe, timeout=1) + @pytest.mark.parametrize( "dataframe,exception", [ @@ -445,6 +475,6 @@ def test_feature_set_types_success(self, client, dataframe, mocker): ) # Need to create a mock producer - with patch("feast.loaders.ingest.KafkaProducer") as mocked_queue: + with patch("feast.client.get_producer") as mocked_queue: # Ingest data into Feast client.ingest(all_types_fs, dataframe) From 5601a09aa54197167a4c2270881c253b796f2b47 Mon Sep 17 00:00:00 2001 From: voonhous Date: Sun, 22 Dec 2019 16:11:29 +0800 Subject: [PATCH 06/17] Added support to accept local avro files, GCS avro files and GCS wildcard paths (#375) --- sdk/python/feast/client.py | 201 +++++++++++++++++++++------- sdk/python/feast/job.py | 93 +++++++++++-- sdk/python/feast/loaders/file.py | 216 ++++++++++++++++++++++++++----- tests/e2e/bq-batch-retrieval.py | 96 ++++++++++++++ tests/e2e/conftest.py | 1 + tests/e2e/requirements.txt | 1 + 6 files changed, 518 insertions(+), 90 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 646de343d52..f649320749b 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import json import logging import os import time from collections import OrderedDict from typing import Dict, Union from typing import List +from urllib.parse import urlparse +import fastavro import grpc import pandas as pd import pyarrow as pa @@ -38,7 +40,7 @@ from feast.feature_set import FeatureSet, Entity from feast.job import Job from feast.loaders.abstract_producer import get_producer -from feast.loaders.file import export_dataframe_to_staging_location +from feast.loaders.file import export_source_to_staging_location from feast.loaders.ingest import KAFKA_CHUNK_PRODUCTION_TIMEOUT from feast.loaders.ingest import get_feature_row_chunks from feast.serving.ServingService_pb2 import GetFeastServingInfoResponse @@ -322,22 +324,28 @@ def list_entities(self) -> Dict[str, Entity]: return entities_dict def get_batch_features( - self, feature_ids: List[str], entity_rows: pd.DataFrame + self, feature_ids: List[str], entity_rows: Union[pd.DataFrame, str] ) -> Job: """ Retrieves historical features from a Feast Serving deployment. Args: - feature_ids: List of feature ids that will be returned for each - entity. Each feature id should have the following format + feature_ids (List[str]): + List of feature ids that will be returned for each entity. + Each feature id should have the following format "feature_set_name:version:feature_name". - entity_rows: Pandas dataframe containing entities and a 'datetime' - column. Each entity in a feature set must be present as a column - in this dataframe. The datetime column must + + entity_rows (Union[pd.DataFrame, str]): + Pandas dataframe containing entities and a 'datetime' column. + Each entity in a feature set must be present as a column in this + dataframe. The datetime column must contain timestamps in + datetime64 format. Returns: - Returns a job object that can be used to monitor retrieval progress - asynchronously, and can be used to materialize the results + feast.job.Job: + Returns a job object that can be used to monitor retrieval + progress asynchronously, and can be used to materialize the + results. Examples: >>> from feast import Client @@ -360,21 +368,11 @@ def get_batch_features( fs_request = _build_feature_set_request(feature_ids) - # Validate entity rows based on entities in Feast Core - self._validate_entity_rows_for_batch_retrieval(entity_rows, fs_request) - - # Remove timezone from datetime column - if isinstance( - entity_rows["datetime"].dtype, pd.core.dtypes.dtypes.DatetimeTZDtype - ): - entity_rows["datetime"] = pd.DatetimeIndex( - entity_rows["datetime"] - ).tz_localize(None) - # Retrieve serving information to determine store type and # staging location serving_info = self._serving_service_stub.GetFeastServingInfo( - GetFeastServingInfoRequest(), timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT + GetFeastServingInfoRequest(), + timeout=GRPC_CONNECTION_TIMEOUT_DEFAULT ) # type: GetFeastServingInfoResponse if serving_info.type != FeastServingType.FEAST_SERVING_TYPE_BATCH: @@ -383,17 +381,50 @@ def get_batch_features( f"does not support batch retrieval " ) - # Export and upload entity row dataframe to staging location + if isinstance(entity_rows, pd.DataFrame): + # Pandas DataFrame detected + # Validate entity rows to based on entities in Feast Core + self._validate_dataframe_for_batch_retrieval( + entity_rows=entity_rows, + feature_sets_request=fs_request + ) + + # Remove timezone from datetime column + if isinstance( + entity_rows["datetime"].dtype, + pd.core.dtypes.dtypes.DatetimeTZDtype + ): + entity_rows["datetime"] = pd.DatetimeIndex( + entity_rows["datetime"] + ).tz_localize(None) + elif isinstance(entity_rows, str): + # String based source + if entity_rows.endswith((".avro", "*")): + # Validate Avro entity rows to based on entities in Feast Core + self._validate_avro_for_batch_retrieval( + source=entity_rows, + feature_sets_request=fs_request + ) + else: + raise Exception( + f"Only .avro and wildcard paths are accepted as entity_rows" + ) + else: + raise Exception(f"Only pandas.DataFrame and str types are allowed" + f" as entity_rows, but got {type(entity_rows)}.") + + # Export and upload entity row DataFrame to staging location # provided by Feast - staged_file = export_dataframe_to_staging_location( + staged_files = export_source_to_staging_location( entity_rows, serving_info.job_staging_location - ) # type: str + ) # type: List[str] request = GetBatchFeaturesRequest( feature_sets=fs_request, dataset_source=DatasetSource( file_source=DatasetSource.FileSource( - file_uris=[staged_file], data_format=DataFormat.DATA_FORMAT_AVRO + file_uris=staged_files, + data_format=DataFormat.DATA_FORMAT_AVRO ) ), ) @@ -402,28 +433,107 @@ def get_batch_features( response = self._serving_service_stub.GetBatchFeatures(request) return Job(response.job, self._serving_service_stub) - def _validate_entity_rows_for_batch_retrieval( - self, entity_rows, feature_sets_request + def _validate_dataframe_for_batch_retrieval( + self, entity_rows: pd.DataFrame, feature_sets_request + ): + """ + Validate whether an the entity rows in a DataFrame contains the correct + information for batch retrieval. + + Datetime column must be present in the DataFrame. + + Args: + entity_rows (pd.DataFrame): + Pandas DataFrame containing entities and datetime column. Each + entity in a feature set must be present as a column in this + DataFrame. + + feature_sets_request: + Feature sets that will be requested. + """ + + self._validate_columns( + columns=entity_rows.columns, + feature_sets_request=feature_sets_request, + datetime_field="datetime" + ) + + def _validate_avro_for_batch_retrieval( + self, source: str, feature_sets_request ): """ - Validate whether an entity_row dataframe contains the correct - information for batch retrieval + Validate whether the entity rows in an Avro source file contains the + correct information for batch retrieval. + + Only gs:// and local files (file://) uri schemes are allowed. + + Avro file must have a column named "event_timestamp". + + No checks will be done if a GCS path is provided. Args: - entity_rows: Pandas dataframe containing entities and datetime - column. Each entity in a feature set must be present as a - column in this dataframe. - feature_sets_request: Feature sets that will be requested + source (str): + File path to Avro. + + feature_sets_request: + Feature sets that will be requested. """ + p = urlparse(source) + if p.scheme == "gs": + # GCS path provided (Risk is delegated to user) + # No validation if GCS path is provided + return + elif p.scheme == "file" or not p.scheme: + # Local file (file://) provided + file_path = os.path.abspath(os.path.join(p.netloc, p.path)) + else: + raise Exception(f"Unsupported uri scheme provided {p.scheme}, only " + f"local files (file://), and gs:// schemes are " + f"allowed") + + with open(file_path, "rb") as f: + reader = fastavro.reader(f) + schema = json.loads(reader.metadata["avro.schema"]) + columns = [x["name"] for x in schema["fields"]] + self._validate_columns( + columns=columns, + feature_sets_request=feature_sets_request, + datetime_field="event_timestamp" + ) + + def _validate_columns( + self, columns: List[str], + feature_sets_request, + datetime_field: str + ) -> None: + """ + Check if the required column contains the correct values for batch + retrieval. + + Args: + columns (List[str]): + List of columns to validate against feature_sets_request. + + feature_sets_request (): + Feature sets that will be requested. + + datetime_field (str): + Name of the datetime field that must be enforced and present as + a column in the data source. + + Returns: + None: + None + """ # Ensure datetime column exists - if "datetime" not in entity_rows.columns: + if datetime_field not in columns: raise ValueError( - f'Entity rows does not contain "datetime" column in columns ' - f"{entity_rows.columns}" + f'Entity rows does not contain "{datetime_field}" column in ' + f'columns {columns}' ) - # Validate dataframe columns based on feature set entities + # Validate Avro columns based on feature set entities for feature_set in feature_sets_request: fs = self.get_feature_set( name=feature_set.name, version=feature_set.version @@ -434,10 +544,10 @@ def _validate_entity_rows_for_batch_retrieval( f"could not be found" ) for entity_type in fs.entities: - if entity_type.name not in entity_rows.columns: + if entity_type.name not in columns: raise ValueError( - f'Dataframe does not contain entity "{entity_type.name}"' - f' column in columns "{entity_rows.columns}"' + f'Input does not contain entity' + f' "{entity_type.name}" column in columns "{columns}"' ) def get_online_features( @@ -596,7 +706,9 @@ def ingest( return None -def _build_feature_set_request(feature_ids: List[str]) -> List[FeatureSetRequest]: +def _build_feature_set_request( + feature_ids: List[str] +) -> List[FeatureSetRequest]: """ Builds a list of FeatureSet objects from feature set ids in order to retrieve feature data from Feast Serving @@ -629,7 +741,7 @@ def _read_table_from_source( max_workers: int ) -> str: """ - Infers a data source type (path or Pandas Dataframe) and reads it in as + Infers a data source type (path or Pandas DataFrame) and reads it in as a PyArrow Table. The PyArrow Table that is read will be written to a parquet file with row @@ -674,7 +786,8 @@ def _read_table_from_source( else: table = pq.read_table(file_path) else: - raise ValueError(f"Unknown data source provided for ingestion: {source}") + raise ValueError( + f"Unknown data source provided for ingestion: {source}") # Ensure that PyArrow table is initialised assert isinstance(table, pa.lib.Table) diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index 4273f86ea84..26f6181ee2d 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -1,12 +1,11 @@ import tempfile import time from datetime import datetime, timedelta -from typing import List +from typing import Iterable from urllib.parse import urlparse import fastavro import pandas as pd -from fastavro import reader as fastavro_reader from google.cloud import storage from feast.serving.ServingService_pb2 import GetJobRequest @@ -62,15 +61,18 @@ def reload(self): """ self.job_proto = self.serving_stub.GetJob(GetJobRequest(job=self.job_proto)).job - def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): """ - Wait until job is done to get an iterable rows of result. - The row can only represent an Avro row in Feast 0.3. + Wait until job is done to get the file uri to Avro result files on + Google Cloud Storage. Args: - timeout_sec: max no of seconds to wait until job is done. If "timeout_sec" is exceeded, an exception will be raised. + timeout_sec (int): + Max no of seconds to wait until job is done. If "timeout_sec" + is exceeded, an exception will be raised. - Returns: Iterable of Avro rows + Returns: + str: Google Cloud Storage file uris of the returned Avro files. """ max_wait_datetime = datetime.now() + timedelta(seconds=timeout_sec) wait_duration_sec = 2 @@ -78,11 +80,13 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): while self.status != JOB_STATUS_DONE: if datetime.now() > max_wait_datetime: raise Exception( - "Timeout exceeded while waiting for result. Please retry this method or use a longer timeout value." + "Timeout exceeded while waiting for result. Please retry " + "this method or use a longer timeout value." ) self.reload() time.sleep(wait_duration_sec) + # Backoff the wait duration exponentially up till MAX_WAIT_INTERVAL_SEC wait_duration_sec = min(wait_duration_sec * 2, MAX_WAIT_INTERVAL_SEC) @@ -95,7 +99,22 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): "your Feast Serving deployment." ) - uris = [urlparse(uri) for uri in self.job_proto.file_uris] + return [urlparse(uri) for uri in self.job_proto.file_uris] + + def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + """ + Wait until job is done to get an iterable rows of result. The row can + only represent an Avro row in Feast 0.3. + + Args: + timeout_sec (int): + Max no of seconds to wait until job is done. If "timeout_sec" + is exceeded, an exception will be raised. + + Returns: + Iterable of Avro rows. + """ + uris = self.get_avro_files(timeout_sec) for file_uri in uris: if file_uri.scheme == "gs": file_obj = tempfile.TemporaryFile() @@ -113,16 +132,64 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): for record in avro_reader: yield record - def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + def to_dataframe( + self, + timeout_sec: int = DEFAULT_TIMEOUT_SEC + ) -> pd.DataFrame: """ - Wait until job is done to get an interable rows of result + Wait until a job is done to get an iterable rows of result. This method + will split the response into chunked DataFrame of a specified size to + to be yielded to the instance calling it. Args: - timeout_sec: max no of seconds to wait until job is done. If "timeout_sec" is exceeded, an exception will be raised. - Returns: pandas Dataframe of the feature values + max_chunk_size (int): + Maximum number of rows that the DataFrame should contain. + + timeout_sec (int): + Max no of seconds to wait until job is done. If "timeout_sec" + is exceeded, an exception will be raised. + + Returns: + pd.DataFrame: + Pandas DataFrame of the feature values. """ records = [r for r in self.result(timeout_sec=timeout_sec)] return pd.DataFrame.from_records(records) + def to_chunked_dataframe( + self, + max_chunk_size: int = -1, + timeout_sec: int = DEFAULT_TIMEOUT_SEC + ) -> pd.DataFrame: + """ + Wait until a job is done to get an iterable rows of result. This method + will split the response into chunked DataFrame of a specified size to + to be yielded to the instance calling it. + + Args: + max_chunk_size (int): + Maximum number of rows that the DataFrame should contain. + + timeout_sec (int): + Max no of seconds to wait until job is done. If "timeout_sec" + is exceeded, an exception will be raised. + + Returns: + pd.DataFrame: + Pandas DataFrame of the feature values. + """ + # Max chunk size defined by user + records = [] + for result in self.result(timeout_sec=timeout_sec): + result.append(records) + if len(records) == max_chunk_size: + df = pd.DataFrame.from_records(records) + records.clear() # Empty records array + yield df + + # Handle for last chunk that is < max_chunk_size + if not records: + yield pd.DataFrame.from_records(records) + def __iter__(self): return iter(self.result()) diff --git a/sdk/python/feast/loaders/file.py b/sdk/python/feast/loaders/file.py index 8dd6b503a74..108f2790dd8 100644 --- a/sdk/python/feast/loaders/file.py +++ b/sdk/python/feast/loaders/file.py @@ -1,70 +1,159 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re import shutil import tempfile -from typing import Optional -from urllib.parse import urlparse import uuid -import pandas as pd from datetime import datetime +from typing import List, Optional, Tuple, Union +from urllib.parse import urlparse, ParseResult + +import pandas as pd from google.cloud import storage from pandavro import to_avro -def export_dataframe_to_staging_location( - df: pd.DataFrame, staging_location_uri: str -) -> str: +def export_source_to_staging_location( + source: Union[pd.DataFrame, str], staging_location_uri: str +) -> List[str]: """ - Uploads a dataframe to a remote staging location + Uploads a DataFrame as an Avro file to a remote staging location. + + The local staging location specified in this function is used for E2E + tests, please do not use it. Args: - df: Pandas dataframe - staging_location_uri: Remote staging location where dataframe should be written + source (Union[pd.DataFrame, str]: + Source of data to be staged. Can be a pandas DataFrame or a file + path. + + Only three types of source are allowed: + * Pandas DataFrame + * Local Avro file + * GCS Avro file + + + staging_location_uri (str): + Remote staging location where DataFrame should be written. Examples: - gs://bucket/path/ - file:///data/subfolder/ + * gs://bucket/path/ + * file:///data/subfolder/ Returns: - Returns the full path to the file in the remote staging location + List[str]: + Returns a list containing the full path to the file(s) in the + remote staging location. """ - # Validate staging location uri = urlparse(staging_location_uri) + + # Prepare Avro file to be exported to staging location + if isinstance(source, pd.DataFrame): + # DataFrame provided as a source + if uri.scheme == "file": + uri_path = uri.path + else: + uri_path = None + + # Remote gs staging location provided by serving + dir_path, file_name, source_path = export_dataframe_to_local( + source, + uri_path + ) + elif urlparse(source).scheme in ["", "file"]: + # Local file provided as a source + dir_path = None + file_name = os.path.basename(source) + source_path = os.path.abspath(os.path.join( + urlparse(source).netloc, urlparse(source).path)) + elif urlparse(source).scheme == "gs": + # Google Cloud Storage path provided + input_source_uri = urlparse(source) + if "*" in source: + # Wildcard path + return _get_files( + bucket=input_source_uri.hostname, + uri=input_source_uri + ) + else: + return [source] + else: + raise Exception(f"Only string and DataFrame types are allowed as a " + f"source, {type(source)} was provided.") + + # Push data to required staging location if uri.scheme == "gs": - dir_path, file_name, source_path = export_dataframe_to_local(df) + # Staging location is a Google Cloud Storage path upload_file_to_gcs( - source_path, uri.hostname, str(uri.path).strip("/") + "/" + file_name + source_path, + uri.hostname, + str(uri.path).strip("/") + "/" + file_name ) - if len(str(dir_path)) < 5: - raise Exception(f"Export location {dir_path} dangerous. Stopping.") - shutil.rmtree(dir_path) elif uri.scheme == "file": - dir_path, file_name, source_path = export_dataframe_to_local(df, uri.path) + # Staging location is a file path + # Used for end-to-end test + pass else: raise Exception( - f"Staging location {staging_location_uri} does not have a valid URI. Only gs:// and file:// are supported" + f"Staging location {staging_location_uri} does not have a " + f"valid URI. Only gs:// and file:// uri scheme are supported." ) - return staging_location_uri.rstrip("/") + "/" + file_name + # Clean up, remove local staging file + if isinstance(source, pd.DataFrame) and len(str(dir_path)) > 4: + shutil.rmtree(dir_path) + + return [staging_location_uri.rstrip("/") + "/" + file_name] -def export_dataframe_to_local(df: pd.DataFrame, dir_path: Optional[str] = None): +def export_dataframe_to_local( + df: pd.DataFrame, + dir_path: Optional[str] = None +) -> Tuple[str, str, str]: """ - Exports a pandas dataframe to the local filesystem + Exports a pandas DataFrame to the local filesystem. Args: - df: Pandas dataframe to save - dir_path: (optional) Absolute directory path '/data/project/subfolder/' + df (pd.DataFrame): + Pandas DataFrame to save. + + dir_path (Optional[str]): + Absolute directory path '/data/project/subfolder/'. + + Returns: + Tuple[str, str, str]: + Tuple of directory path, file name and destination path. The + destination path can be obtained by concatenating the directory + path and file name. """ # Create local staging location if not provided if dir_path is None: dir_path = tempfile.mkdtemp() - file_name = f'{datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p")}_{str(uuid.uuid4())[:8]}.avro' + file_name = _get_file_name() dest_path = f"{dir_path}/{file_name}" # Temporarily rename datetime column to event_timestamp. Ideally we would # force the schema with our avro writer instead. - df.columns = ["event_timestamp" if col == "datetime" else col for col in df.columns] + df.columns = [ + "event_timestamp" + if col == "datetime" else col + for col in df.columns + ] try: # Export dataset to file in local path @@ -74,23 +163,84 @@ def export_dataframe_to_local(df: pd.DataFrame, dir_path: Optional[str] = None): finally: # Revert event_timestamp column to datetime df.columns = [ - "datetime" if col == "event_timestamp" else col for col in df.columns + "datetime" + if col == "event_timestamp" else col + for col in df.columns ] return dir_path, file_name, dest_path -def upload_file_to_gcs(local_path: str, bucket: str, remote_path: str): +def upload_file_to_gcs(local_path: str, bucket: str, remote_path: str) -> None: """ - Upload a file from the local file system to Google Cloud Storage (GCS) + Upload a file from the local file system to Google Cloud Storage (GCS). Args: - local_path: Local filesystem path of file to upload - bucket: GCS bucket to upload to - remote_path: Path within GCS bucket to upload file to, includes file name + local_path (str): + Local filesystem path of file to upload. + + bucket (str): + GCS bucket destination to upload to. + + remote_path (str): + Path within GCS bucket to upload file to, includes file name. + + Returns: + None: + None """ storage_client = storage.Client(project=None) bucket = storage_client.get_bucket(bucket) blob = bucket.blob(remote_path) blob.upload_from_filename(local_path) + + +def _get_files(bucket: str, uri: ParseResult) -> List[str]: + """ + List all available files within a Google storage bucket that matches a wild + card path. + + Args: + bucket (str): + Google Storage bucket to reference. + + uri (urllib.parse.ParseResult): + Wild card uri path containing the "*" character. + Example: + * gs://feast/staging_location/* + * gs://feast/staging_location/file_*.avro + + Returns: + List[str]: + List of all available files matching the wildcard path. + """ + + storage_client = storage.Client(project=None) + bucket = storage_client.get_bucket(bucket) + path = uri.path + + if "*" in path: + regex = re.compile(path.replace("*", ".*?").strip("/")) + blob_list = bucket.list_blobs( + prefix=path.strip("/").split("*")[0], + delimiter="/" + ) + # File path should not be in path (file path must be longer than path) + return [f"{uri.scheme}://{uri.hostname}/{file}" + for file in [x.name for x in blob_list] + if re.match(regex, file) and file not in path] + else: + raise Exception(f"{path} is not a wildcard path") + + +def _get_file_name() -> str: + """ + Create a random file name. + + Returns: + str: + Randomised file name. + """ + + return f'{datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p")}_{str(uuid.uuid4())[:8]}.avro' diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq-batch-retrieval.py index 2d6668eaa86..6b654f9a70e 100644 --- a/tests/e2e/bq-batch-retrieval.py +++ b/tests/e2e/bq-batch-retrieval.py @@ -2,6 +2,7 @@ import time from datetime import datetime from datetime import timedelta +from urllib.parse import urlparse import numpy as np import pandas as pd @@ -12,7 +13,9 @@ from feast.feature import Feature from feast.feature_set import FeatureSet from feast.type_map import ValueType +from google.cloud import storage from google.protobuf.duration_pb2 import Duration +from pandavro import to_avro pd.set_option('display.max_columns', None) @@ -31,6 +34,11 @@ def allow_dirty(pytestconfig): return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False +@pytest.fixture(scope="module") +def gcs_path(pytestconfig): + return pytestconfig.getoption("gcs_path") + + @pytest.fixture(scope="module") def client(core_url, serving_url, allow_dirty): # Get client for core and serving @@ -45,6 +53,94 @@ def client(core_url, serving_url, allow_dirty): return client +def test_get_batch_features_with_file(client): + file_fs1 = FeatureSet( + "file_feature_set", + features=[Feature("feature_value", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) + + client.apply(file_fs1) + file_fs1 = client.get_feature_set(name="file_feature_set", version=1) + + N_ROWS = 10 + time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) + features_1_df = pd.DataFrame( + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "feature_value": [f"{i}" for i in range(N_ROWS)], + } + ) + client.ingest(file_fs1, features_1_df) + + # Rename column (datetime -> event_timestamp) + features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"}) + + to_avro(df=features_1_df, file_path_or_buffer="file_feature_set.avro") + + feature_retrieval_job = client.get_batch_features( + entity_rows="file://file_feature_set.avro", feature_ids=["file_feature_set:1:feature_value"] + ) + + output = feature_retrieval_job.to_dataframe() + print(output.head()) + + assert output["entity_id"].to_list() == [int(i) for i in output["file_feature_set_v1_feature_value"].to_list()] + + +def test_get_batch_features_with_gs_path(client, gcs_path): + gcs_fs1 = FeatureSet( + "gcs_feature_set", + features=[Feature("feature_value", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) + + client.apply(gcs_fs1) + gcs_fs1 = client.get_feature_set(name="gcs_feature_set", version=1) + + N_ROWS = 10 + time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) + features_1_df = pd.DataFrame( + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "feature_value": [f"{i}" for i in range(N_ROWS)], + } + ) + client.ingest(gcs_fs1, features_1_df) + + # Rename column (datetime -> event_timestamp) + features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"}) + + # Output file to local + file_name = "gcs_feature_set.avro" + to_avro(df=features_1_df, file_path_or_buffer=file_name) + + uri = urlparse(gcs_path) + bucket = uri.hostname + ts = int(time.time()) + remote_path = str(uri.path).strip("/") + f"{ts}/{file_name}" + + # Upload file to gcs + storage_client = storage.Client(project=None) + bucket = storage_client.get_bucket(bucket) + blob = bucket.blob(remote_path) + blob.upload_from_filename(file_name) + + feature_retrieval_job = client.get_batch_features( + entity_rows=f"{gcs_path}{ts}/*", + feature_ids=["gcs_feature_set:1:feature_value"] + ) + + output = feature_retrieval_job.to_dataframe() + print(output.head()) + + assert output["entity_id"].to_list() == [int(i) for i in output["gcs_feature_set_v1_feature_value"].to_list()] + + def test_order_by_creation_time(client): proc_time_fs = FeatureSet( "processing_time", diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b37770a83f9..8ea472b6620 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -2,3 +2,4 @@ def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") parser.addoption("--serving_url", action="store", default="localhost:6566") parser.addoption("--allow_dirty", action="store", default="False") + parser.addoption("--gcs_path", action="store", default="gs://feast-templocation-kf-feast/") diff --git a/tests/e2e/requirements.txt b/tests/e2e/requirements.txt index 6b999421c04..0ba345a000f 100644 --- a/tests/e2e/requirements.txt +++ b/tests/e2e/requirements.txt @@ -1,6 +1,7 @@ mock==2.0.0 numpy==1.16.4 pandas==0.24.2 +pandavro==1.5.* pytest==5.2.1 pytest-benchmark==3.2.2 pytest-mock==1.10.4 From d93274fe558d92b637fd0099dd9ad5b3e9679046 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 3 Jan 2020 14:15:31 +0800 Subject: [PATCH 07/17] Remove references to v0.4 protos and add fix for test from v0.4 --- sdk/python/feast/client.py | 8 ++++++-- sdk/python/feast/feature_set.py | 2 -- sdk/python/setup.py | 2 +- sdk/python/tests/test_client.py | 30 ------------------------------ 4 files changed, 7 insertions(+), 35 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index f649320749b..0254f4187ba 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -17,6 +17,7 @@ import os import time from collections import OrderedDict +from math import ceil from typing import Dict, Union from typing import List from urllib.parse import urlparse @@ -36,7 +37,6 @@ GetFeatureSetResponse, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.core.FeatureSet_pb2 import FeatureSetStatus from feast.feature_set import FeatureSet, Entity from feast.job import Job from feast.loaders.abstract_producer import get_producer @@ -336,11 +336,15 @@ def get_batch_features( "feature_set_name:version:feature_name". entity_rows (Union[pd.DataFrame, str]): + Either: Pandas dataframe containing entities and a 'datetime' column. Each entity in a feature set must be present as a column in this dataframe. The datetime column must contain timestamps in datetime64 format. + Or: + A file path in AVRO format representing the entity rows. + Returns: feast.job.Job: Returns a job object that can be used to monitor retrieval @@ -794,7 +798,7 @@ def _read_table_from_source( # Write table as parquet file with a specified row_group_size tmp_table_name = f"{int(time.time())}.parquet" - row_group_size = min(int(table.num_rows/max_workers), chunk_size) + row_group_size = min(ceil(table.num_rows / max_workers), chunk_size) pq.write_table(table=table, where=tmp_table_name, row_group_size=row_group_size) diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 28381891f59..42979ea9116 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -19,8 +19,6 @@ import pandas as pd import pyarrow as pa -from feast.core.FeatureSet_pb2 import FeatureSet as FeatureSetProto -from feast.core.FeatureSet_pb2 import FeatureSetMeta as FeatureSetMetaProto from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto from feast.entity import Entity from feast.feature import Feature, Field diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 9ac7225e80e..420d1a39afc 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -32,7 +32,7 @@ "googleapis-common-protos==1.*", "google-cloud-bigquery-storage==0.7.*", "grpcio==1.*", - "pandas==0.*", + "pandas>=0.25.0", "pandavro==1.5.*", "protobuf>=3.10", "PyYAML==5.1.*", diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index f979c5a55df..9ef6e3d56fb 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -385,36 +385,6 @@ def test_feature_set_ingest_success(self, dataframe, client, mocker): # Ingest data into Feast client.ingest("driver-feature-set", dataframe) - @pytest.mark.parametrize("dataframe,exception", [(dataframes.GOOD, TimeoutError)]) - def test_feature_set_ingest_fail_if_pending( - self, dataframe, exception, client, mocker - ): - with pytest.raises(exception): - driver_fs = FeatureSet( - "driver-feature-set", - source=KafkaSource(brokers="kafka:9092", topic="test"), - ) - driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT)) - driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING)) - driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64)) - driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64)) - - # Register with Feast core - client.apply(driver_fs) - driver_fs = driver_fs.to_proto() - driver_fs.meta.status = FeatureSetStatus.STATUS_PENDING - - mocker.patch.object( - client._core_service_stub, - "GetFeatureSet", - return_value=GetFeatureSetResponse(feature_set=driver_fs), - ) - - # Need to create a mock producer - with patch("feast.client.get_producer") as mocked_queue: - # Ingest data into Feast - client.ingest("driver-feature-set", dataframe, timeout=1) - @pytest.mark.parametrize( "dataframe,exception", [ From cd7ab6d352e885f24b0205ef1525e89e57c6e7ec Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Tue, 7 Jan 2020 16:12:05 +0800 Subject: [PATCH 08/17] Add missing wait time after running feast apply (#412) Otherwise ingested data will not be processed because Beam job is not ready --- tests/e2e/bq-batch-retrieval.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq-batch-retrieval.py index 6b654f9a70e..d7a717bed3b 100644 --- a/tests/e2e/bq-batch-retrieval.py +++ b/tests/e2e/bq-batch-retrieval.py @@ -17,7 +17,12 @@ from google.protobuf.duration_pb2 import Duration from pandavro import to_avro -pd.set_option('display.max_columns', None) +pd.set_option("display.max_columns", None) + +# How long we should wait for the Beam job to be ready for ingestion after running `feast apply`. +# When using DirectRunner, 20 seconds is a reasonable time from past observations. +WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY = 20 + @pytest.fixture(scope="module") def core_url(pytestconfig): @@ -62,6 +67,7 @@ def test_get_batch_features_with_file(client): ) client.apply(file_fs1) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) file_fs1 = client.get_feature_set(name="file_feature_set", version=1) N_ROWS = 10 @@ -99,6 +105,7 @@ def test_get_batch_features_with_gs_path(client, gcs_path): ) client.apply(gcs_fs1) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) gcs_fs1 = client.get_feature_set(name="gcs_feature_set", version=1) N_ROWS = 10 @@ -149,7 +156,7 @@ def test_order_by_creation_time(client): max_age=Duration(seconds=100), ) client.apply(proc_time_fs) - time.sleep(10) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) proc_time_fs = client.get_feature_set(name="processing_time", version=1) time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -188,13 +195,17 @@ def test_additional_columns_in_entity_table(client): max_age=Duration(seconds=100), ) client.apply(add_cols_fs) - time.sleep(10) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) add_cols_fs = client.get_feature_set(name="additional_columns", version=1) N_ROWS = 10 time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) features_df = pd.DataFrame( - {"datetime": [time_offset] * N_ROWS, "entity_id": [i for i in range(N_ROWS)], "feature_value": ["abc"] * N_ROWS} + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "feature_value": ["abc"] * N_ROWS, + } ) client.ingest(add_cols_fs, features_df) @@ -225,7 +236,7 @@ def test_point_in_time_correctness_join(client): max_age=Duration(seconds=100), ) client.apply(historical_fs) - time.sleep(10) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) historical_fs = client.get_feature_set(name="historical", version=1) time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -271,11 +282,11 @@ def test_multiple_featureset_joins(client): ) client.apply(fs1) - time.sleep(10) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) fs1 = client.get_feature_set(name="feature_set_1", version=1) client.apply(fs2) - time.sleep(10) + time.sleep(WAIT_TIME_IN_SECONDS_FOR_FEAST_APPLY) fs2 = client.get_feature_set(name="feature_set_2", version=1) N_ROWS = 10 From 505bdc6485aca28342868c75f2a5027250b046b8 Mon Sep 17 00:00:00 2001 From: smadarasmi Date: Wed, 8 Jan 2020 09:28:04 +0700 Subject: [PATCH 09/17] Add Cassandra Store (#360) * create cassandra store for registration and ingestion * Downgraded Guava to 25 * Beam 2.16 uses Cassandra 3.4.0 (So we cannot use Cassandra 4.x which shades Guava) * Cassandra 3.4.0 uses Guava version 16.0 but has a compatibility check to use a different class when we use version > 19.0. * Guava version 26 (version previously used) has breaking change to method used in compatibility check in Cassandra's dependency, hence version 25 * Using Cassandra's internal field 'writetime' to handle out of order arrivals. When older records where the primary key already exist in Cassandra are ingested, they are set as tombstones in Cassandra and ignored on retrieval. * Aware that this way of handling out of order arrival is specific to Cassandra, but until we have a general way to handle out of order arrivals we need to do it this way * Cassandra's object mapper requires stating table's name along with @Table annotation * table_name is still part of CassandraConfig for use in serving module * if user registers CassandraConfig with a different table name other than "feature_store", this will throw an exception * add cassandra serving service * Abstracted OnlineServingService for common implementation of online serving stores * Complete tests remain in RedisServingServiceTest while Cassandra tests only contain basic tests for writes, and some other implementation specific to Cassandra * update documentation to reflect current API and add cassandra store to docs * add default expiration to cassandra config for when featureset does not have max age * docs update, spotless check, and bug fix on cassandra schema --- CONTRIBUTING.md | 99 ++++--- core/pom.xml | 2 +- .../core/config/FeatureStreamConfig.java | 2 +- .../core/job/dataflow/DataflowJobManager.java | 2 +- .../core/job/direct/DirectJobRegistry.java | 2 +- .../job/direct/DirectRunnerJobManager.java | 2 +- .../main/java/feast/core/log/AuditLogger.java | 2 +- .../core/service/JobCoordinatorService.java | 2 +- .../feast/core/service/JobStatusService.java | 2 +- .../java/feast/core/util/TypeConversion.java | 2 +- .../feast/core/validators/MatchersTest.java | 4 +- .../feast/charts/feast-serving/values.yaml | 9 + ingestion/pom.xml | 13 + .../transform/CassandraMutationMapper.java | 60 +++++ .../CassandraMutationMapperFactory.java | 42 +++ .../ingestion/transform/WriteToStore.java | 44 +++- .../java/feast/ingestion/utils/StoreUtil.java | 87 ++++++ .../java/feast/ingestion/utils/ValueUtil.java | 57 ++++ .../serving/cassandra/CassandraMutation.java | 121 +++++++++ .../FeatureRowToCassandraMutationDoFn.java | 85 ++++++ .../transform/CassandraWriteToStoreIT.java | 248 ++++++++++++++++++ .../ingestion/util/CassandraStoreUtilIT.java | 167 ++++++++++++ ...FeatureRowToCassandraMutationDoFnTest.java | 225 ++++++++++++++++ .../src/test/java/feast/test/TestUtil.java | 143 +++++++++- pom.xml | 2 +- protos/feast/core/Store.proto | 33 ++- serving/pom.xml | 21 +- serving/sample_cassandra_config.yml | 13 + .../java/feast/serving/FeastProperties.java | 85 ++++++ .../configuration/ServingServiceConfig.java | 59 +++++ .../service/CassandraServingService.java | 153 +++++++++++ .../serving/service/OnlineServingService.java | 236 +++++++++++++++++ .../serving/service/RedisServingService.java | 212 +++------------ .../java/feast/serving/util/ValueUtil.java | 53 ++++ serving/src/main/resources/application.yml | 20 +- .../CassandraServingServiceITTest.java | 244 +++++++++++++++++ .../service/CassandraServingServiceTest.java | 117 +++++++++ .../service/RedisServingServiceTest.java | 18 +- .../java/feast/serving/test/TestUtil.java | 81 ++++++ .../embedded-store/LoadCassandra.cql | 8 + 40 files changed, 2522 insertions(+), 255 deletions(-) create mode 100644 ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java create mode 100644 ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java create mode 100644 ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java create mode 100644 ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java create mode 100644 ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java create mode 100644 ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java create mode 100644 ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java create mode 100644 ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java create mode 100644 serving/sample_cassandra_config.yml create mode 100644 serving/src/main/java/feast/serving/service/CassandraServingService.java create mode 100644 serving/src/main/java/feast/serving/service/OnlineServingService.java create mode 100644 serving/src/main/java/feast/serving/util/ValueUtil.java create mode 100644 serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java create mode 100644 serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java create mode 100644 serving/src/test/java/feast/serving/test/TestUtil.java create mode 100644 serving/src/test/resources/embedded-store/LoadCassandra.cql diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06476c0156d..061031a4e85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,42 +54,18 @@ mvn --projects core spring-boot:run # If Feast Core starts successfully, verify the correct Stores are registered # correctly, for example by using grpc_cli. -grpc_cli call localhost:6565 GetStores '' +grpc_cli call localhost:6565 ListStores '' -# Should return something similar to the following. -# Note that you should change BigQuery projectId and datasetId accordingly -# in "$FEAST_HOME/core/src/main/resources/application.yml" - -store { - name: "SERVING" - type: REDIS - subscriptions { - name: "*" - version: ">0" - } - redis_config { - host: "localhost" - port: 6379 - } -} -store { - name: "WAREHOUSE" - type: BIGQUERY - subscriptions { - name: "*" - version: ">0" - } - bigquery_config { - project_id: "my-google-project-id" - dataset_id: "my-bigquery-dataset-id" - } +# Should return something similar to the following if you have not updated any stores +{ + "store": [] } ``` #### Starting Feast Serving -Feast Serving requires administrators to provide an **existing** store name in Feast. -An instance of Feast Serving can only retrieve features from a **single** store. +Feast Serving requires administrators to provide an **existing** store name in Feast. +An instance of Feast Serving can only retrieve features from a **single** store. > In order to retrieve features from multiple stores you must start **multiple** instances of Feast serving. If you start multiple Feast serving on a single host, make sure that they are listening on different ports. @@ -105,6 +81,69 @@ grpc_cli call localhost:6566 GetFeastServingType '' type: FEAST_SERVING_TYPE_ONLINE ``` +#### Updating a store + +Create a new Store by sending a request to Feast Core. + +``` +# Example of updating a redis store + +grpc_cli call localhost:6565 UpdateStore ' +store { + name: "SERVING" + type: REDIS + subscriptions { + name: "*" + version: ">0" + } + redis_config { + host: "localhost" + port: 6379 + } +} +' + +# Other supported stores examples (replacing redis_config): +# BigQuery +bigquery_config { + project_id: "my-google-project-id" + dataset_id: "my-bigquery-dataset-id" +} + +# Cassandra: two options in cassandra depending on replication strategy +# See details: https://docs.datastax.com/en/cassandra/3.0/cassandra/architecture/archDataDistributeReplication.html +# +# Please note that table name must be "feature_store" as is specified in the @Table annotation of the +# datastax object mapper + +# SimpleStrategy +cassandra_config { + bootstrap_hosts: "localhost" + port: 9042 + keyspace: "feast" + table_name: "feature_store" + replication_options { + class: "SimpleStrategy" + replication_factor: 1 + } +} + +# NetworkTopologyStrategy +cassandra_config { + bootstrap_hosts: "localhost" + port: 9042 + keyspace: "feast" + table_name: "feature_store" + replication_options { + class: "NetworkTopologyStrategy" + east: 2 + west: 2 + } +} + +# To check that the Stores has been updated correctly. +grpc_cli call localhost:6565 ListStores '' +``` #### Registering a FeatureSet diff --git a/core/pom.xml b/core/pom.xml index f6e4909260a..d5b09292e84 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -114,7 +114,7 @@ protobuf-java-util - + com.google.guava guava diff --git a/core/src/main/java/feast/core/config/FeatureStreamConfig.java b/core/src/main/java/feast/core/config/FeatureStreamConfig.java index ca8240d7805..6d9a30f9e93 100644 --- a/core/src/main/java/feast/core/config/FeatureStreamConfig.java +++ b/core/src/main/java/feast/core/config/FeatureStreamConfig.java @@ -66,7 +66,7 @@ public Source getDefaultSource(FeastProperties feastProperties) { } catch (InterruptedException | ExecutionException e) { if (e.getCause().getClass().equals(TopicExistsException.class)) { log.warn( - Strings.lenientFormat( + String.format( "Unable to create topic %s in the feature stream, topic already exists, using existing topic.", topicName)); } else { diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java index 4e4533c4c9a..f19cf1a6569 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java +++ b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java @@ -121,7 +121,7 @@ public void abortJob(String dataflowJobId) { } catch (Exception e) { log.error("Unable to drain job with id: {}, cause: {}", dataflowJobId, e.getMessage()); throw new RuntimeException( - Strings.lenientFormat("Unable to drain job with id: %s", dataflowJobId), e); + String.format("Unable to drain job with id: %s", dataflowJobId), e); } } diff --git a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java b/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java index f7ded9fec76..8f6c87053ff 100644 --- a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java +++ b/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java @@ -41,7 +41,7 @@ public DirectJobRegistry() { public void add(DirectJob job) { if (jobs.containsKey(job.getJobId())) { throw new IllegalArgumentException( - Strings.lenientFormat("Job with id %s already exists and is running", job.getJobId())); + String.format("Job with id %s already exists and is running", job.getJobId())); } jobs.put(job.getJobId(), job); } diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java index a09fd394952..85b8a95dd56 100644 --- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java +++ b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java @@ -149,7 +149,7 @@ public void abortJob(String extId) { job.abort(); } catch (IOException e) { throw new RuntimeException( - Strings.lenientFormat("Unable to abort DirectRunner job %s", extId), e); + String.format("Unable to abort DirectRunner job %s", extId), e); } jobs.remove(extId); } diff --git a/core/src/main/java/feast/core/log/AuditLogger.java b/core/src/main/java/feast/core/log/AuditLogger.java index 5349b5548b0..2c60307805c 100644 --- a/core/src/main/java/feast/core/log/AuditLogger.java +++ b/core/src/main/java/feast/core/log/AuditLogger.java @@ -44,7 +44,7 @@ public static void log( map.put("resource", resource.toString()); map.put("id", id); map.put("action", action.toString()); - map.put("detail", Strings.lenientFormat(detail, args)); + map.put("detail", String.format(detail, args)); ObjectMessage msg = new ObjectMessage(map); log.log(AUDIT_LEVEL, msg); diff --git a/core/src/main/java/feast/core/service/JobCoordinatorService.java b/core/src/main/java/feast/core/service/JobCoordinatorService.java index c56531a9da7..b5c9fc6c1bb 100644 --- a/core/src/main/java/feast/core/service/JobCoordinatorService.java +++ b/core/src/main/java/feast/core/service/JobCoordinatorService.java @@ -179,7 +179,7 @@ private JobInfo updateJob( public void abortJob(String id) { Optional jobOptional = jobInfoRepository.findById(id); if (!jobOptional.isPresent()) { - throw new RetrievalException(Strings.lenientFormat("Unable to retrieve job with id %s", id)); + throw new RetrievalException(String.format("Unable to retrieve job with id %s", id)); } JobInfo job = jobOptional.get(); if (JobStatus.getTerminalState().contains(job.getStatus())) { diff --git a/core/src/main/java/feast/core/service/JobStatusService.java b/core/src/main/java/feast/core/service/JobStatusService.java index db6cd41ee8b..26d81647faa 100644 --- a/core/src/main/java/feast/core/service/JobStatusService.java +++ b/core/src/main/java/feast/core/service/JobStatusService.java @@ -66,7 +66,7 @@ public class JobStatusService { // public JobDetail getJob(String id) { // Optional job = jobInfoRepository.findById(id); // if (!job.isPresent()) { - // throw new RetrievalException(Strings.lenientFormat("Unable to retrieve job with id %s", + // throw new RetrievalException(String.format("Unable to retrieve job with id %s", // id)); // } // JobDetail.Builder jobDetailBuilder = job.get().getJobDetail().toBuilder(); diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java index e01a5511359..a7dd2b0d2a3 100644 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ b/core/src/main/java/feast/core/util/TypeConversion.java @@ -85,7 +85,7 @@ public static String convertMapToJsonString(Map map) { public static String[] convertMapToArgs(Map map) { List args = new ArrayList<>(); for (Entry arg : map.entrySet()) { - args.add(Strings.lenientFormat("--%s=%s", arg.getKey(), arg.getValue())); + args.add(String.format("--%s=%s", arg.getKey(), arg.getValue())); } return args.toArray(new String[] {}); } diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java index 774e58c7a87..13c9e006a44 100644 --- a/core/src/test/java/feast/core/validators/MatchersTest.java +++ b/core/src/test/java/feast/core/validators/MatchersTest.java @@ -43,7 +43,7 @@ public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { exception.expect(IllegalArgumentException.class); exception.expectMessage( - Strings.lenientFormat( + String.format( "invalid value for field %s: %s", "someField", "argument must be in upper snake case, and cannot include any special characters.")); @@ -61,7 +61,7 @@ public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() { public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { exception.expect(IllegalArgumentException.class); exception.expectMessage( - Strings.lenientFormat( + String.format( "invalid value for field %s: %s", "someField", "argument must be in lower snake case, and cannot include any special characters.")); diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml index b312a40692c..c5ccf108698 100644 --- a/infra/charts/feast/charts/feast-serving/values.yaml +++ b/infra/charts/feast/charts/feast-serving/values.yaml @@ -56,6 +56,15 @@ application.yaml: config-path: /etc/feast/feast-serving/store.yaml redis-pool-max-size: 128 redis-pool-max-idle: 64 + cassandra-pool-core-local-connections: 1 + cassandra-pool-max-local-connections: 1 + cassandra-pool-core-remote-connections: 1 + cassandra-pool-max-remote-connections: 1 + cassandra-pool-max-requests-local-connection: 32768 + cassandra-pool-max-requests-remote-connection: 2048 + cassandra-pool-new-local-connection-threshold: 30000 + cassandra-pool-new-remote-connection-threshold: 400 + cassandra-pool-timeout-millis: 0 jobs: staging-location: "" store-type: "" diff --git a/ingestion/pom.xml b/ingestion/pom.xml index eb892335180..72ac60578bc 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -213,6 +213,12 @@ ${org.apache.beam.version} + + org.apache.beam + beam-sdks-java-io-cassandra + ${org.apache.beam.version} + + redis.clients jedis @@ -245,6 +251,13 @@ test + + org.cassandraunit + cassandra-unit-shaded + 3.11.2.0 + test + + com.google.guava guava diff --git a/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java new file mode 100644 index 00000000000..b1e5c4f0ce5 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform; + +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.mapping.Mapper.Option; +import feast.store.serving.cassandra.CassandraMutation; +import java.io.Serializable; +import java.util.Iterator; +import java.util.concurrent.Future; +import org.apache.beam.sdk.io.cassandra.Mapper; + +/** A {@link Mapper} that supports writing {@code CassandraMutation}s with the Beam Cassandra IO. */ +public class CassandraMutationMapper implements Mapper, Serializable { + + private com.datastax.driver.mapping.Mapper mapper; + + CassandraMutationMapper(com.datastax.driver.mapping.Mapper mapper) { + this.mapper = mapper; + } + + @Override + public Iterator map(ResultSet resultSet) { + throw new UnsupportedOperationException("Only supports write operations"); + } + + @Override + public Future deleteAsync(CassandraMutation entityClass) { + throw new UnsupportedOperationException("Only supports write operations"); + } + + /** + * Saves records to Cassandra with: - Cassandra's internal write time set to the timestamp of the + * record. Cassandra will not override an existing record with the same partition key if the write + * time is older - Expiration of the record + * + * @param entityClass Cassandra's object mapper + */ + @Override + public Future saveAsync(CassandraMutation entityClass) { + return mapper.saveAsync( + entityClass, + Option.timestamp(entityClass.getWriteTime()), + Option.ttl(entityClass.getTtl())); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java new file mode 100644 index 00000000000..9f344650995 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform; + +import com.datastax.driver.core.Session; +import com.datastax.driver.mapping.MappingManager; +import feast.store.serving.cassandra.CassandraMutation; +import org.apache.beam.sdk.io.cassandra.Mapper; +import org.apache.beam.sdk.transforms.SerializableFunction; + +public class CassandraMutationMapperFactory implements SerializableFunction { + + private transient MappingManager mappingManager; + private Class entityClass; + + public CassandraMutationMapperFactory(Class entityClass) { + this.entityClass = entityClass; + } + + @Override + public Mapper apply(Session session) { + if (mappingManager == null) { + this.mappingManager = new MappingManager(session); + } + + return new CassandraMutationMapper(mappingManager.mapper(entityClass)); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java index 6f697f1c6fd..c89b95bfb22 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java @@ -16,12 +16,14 @@ */ package feast.ingestion.transform; +import com.datastax.driver.core.Session; import com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors; import com.google.api.services.bigquery.model.TableRow; import com.google.auto.value.AutoValue; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.BigQueryConfig; +import feast.core.StoreProto.Store.CassandraConfig; import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; import feast.ingestion.options.ImportOptions; @@ -29,11 +31,16 @@ import feast.ingestion.values.FailedElement; import feast.store.serving.bigquery.FeatureRowToTableRow; import feast.store.serving.bigquery.GetTableDestination; +import feast.store.serving.cassandra.CassandraMutation; +import feast.store.serving.cassandra.FeatureRowToCassandraMutationDoFn; import feast.store.serving.redis.FeatureRowToRedisMutationDoFn; import feast.store.serving.redis.RedisCustomIO; import feast.types.FeatureRowProto.FeatureRow; import java.io.IOException; +import java.util.Arrays; import java.util.Map; +import org.apache.beam.sdk.io.cassandra.CassandraIO; +import org.apache.beam.sdk.io.cassandra.Mapper; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method; @@ -47,10 +54,10 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.sdk.values.ValueInSingleWindow; import org.slf4j.Logger; @AutoValue @@ -61,8 +68,8 @@ public abstract class WriteToStore extends PTransform, P public static final String METRIC_NAMESPACE = "WriteToStore"; public static final String ELEMENTS_WRITTEN_METRIC = "elements_written"; - private static final Counter elementsWritten = Metrics - .counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC); + private static final Counter elementsWritten = + Metrics.counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC); public abstract Store getStore(); @@ -146,16 +153,37 @@ public void processElement(ProcessContext context) { .build()); } break; + case CASSANDRA: + CassandraConfig cassandraConfig = getStore().getCassandraConfig(); + SerializableFunction mapperFactory = + new CassandraMutationMapperFactory(CassandraMutation.class); + input + .apply( + "Create CassandraMutation from FeatureRow", + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + getFeatureSetSpecs(), cassandraConfig.getDefaultTtl()))) + .apply( + CassandraIO.write() + .withHosts(Arrays.asList(cassandraConfig.getBootstrapHosts().split(","))) + .withPort(cassandraConfig.getPort()) + .withKeyspace(cassandraConfig.getKeyspace()) + .withEntity(CassandraMutation.class) + .withMapperFactoryFn(mapperFactory)); + break; default: log.error("Store type '{}' is not supported. No Feature Row will be written.", storeType); break; } - input.apply("IncrementWriteToStoreElementsWrittenCounter", - MapElements.into(TypeDescriptors.booleans()).via((FeatureRow row) -> { - elementsWritten.inc(); - return true; - })); + input.apply( + "IncrementWriteToStoreElementsWrittenCounter", + MapElements.into(TypeDescriptors.booleans()) + .via( + (FeatureRow row) -> { + elementsWritten.inc(); + return true; + })); return PDone.in(input.getPipeline()); } diff --git a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java index 5ceb8bd2f96..3478227165b 100644 --- a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java +++ b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java @@ -18,6 +18,14 @@ import static feast.types.ValueProto.ValueType; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.KeyspaceMetadata; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.schemabuilder.Create; +import com.datastax.driver.core.schemabuilder.KeyspaceOptions; +import com.datastax.driver.core.schemabuilder.SchemaBuilder; +import com.datastax.driver.mapping.MappingManager; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetId; @@ -39,13 +47,19 @@ import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.core.StoreProto.Store; +import feast.core.StoreProto.Store.CassandraConfig; import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; +import feast.store.serving.cassandra.CassandraMutation; import feast.types.ValueProto.ValueType.Enum; +import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import redis.clients.jedis.JedisPool; @@ -112,6 +126,9 @@ public static void setupStore(Store store, FeatureSetSpec featureSetSpec) { store.getBigqueryConfig().getDatasetId(), BigQueryOptions.getDefaultInstance().getService()); break; + case CASSANDRA: + StoreUtil.setupCassandra(store.getCassandraConfig()); + break; default: log.warn("Store type '{}' is unsupported", storeType); break; @@ -246,4 +263,74 @@ public static void checkRedisConnection(RedisConfig redisConfig) { } jedisPool.close(); } + + /** + * Ensures Cassandra is accessible, else throw a RuntimeException. Creates Cassandra keyspace and + * table if it does not already exist + * + * @param cassandraConfig Please refer to feast.core.Store proto + */ + public static void setupCassandra(CassandraConfig cassandraConfig) { + List contactPoints = + Arrays.stream(cassandraConfig.getBootstrapHosts().split(",")) + .map(host -> new InetSocketAddress(host, cassandraConfig.getPort())) + .collect(Collectors.toList()); + Cluster cluster = Cluster.builder().addContactPointsWithPorts(contactPoints).build(); + Session session; + + try { + String keyspace = cassandraConfig.getKeyspace(); + KeyspaceMetadata keyspaceMetadata = cluster.getMetadata().getKeyspace(keyspace); + if (keyspaceMetadata == null) { + log.info("Creating keyspace '{}'", keyspace); + Map replicationOptions = + cassandraConfig.getReplicationOptionsMap().entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + KeyspaceOptions createKeyspace = + SchemaBuilder.createKeyspace(keyspace) + .ifNotExists() + .with() + .replication(replicationOptions); + session = cluster.newSession(); + session.execute(createKeyspace); + } + + session = cluster.connect(keyspace); + // Currently no support for creating table from entity mapper: + // https://datastax-oss.atlassian.net/browse/JAVA-569 + Create createTable = + SchemaBuilder.createTable(keyspace, cassandraConfig.getTableName()) + .ifNotExists() + .addPartitionKey(CassandraMutation.ENTITIES, DataType.text()) + .addClusteringColumn(CassandraMutation.FEATURE, DataType.text()) + .addColumn(CassandraMutation.VALUE, DataType.blob()); + log.info("Create Cassandra table if not exists.."); + session.execute(createTable); + + validateCassandraTable(session); + + session.close(); + } catch (RuntimeException e) { + throw new RuntimeException( + String.format( + "Failed to connect to Cassandra at bootstrap hosts: '%s' port: '%s'. Please check that your Cassandra is running and accessible from Feast.", + contactPoints.stream() + .map(InetSocketAddress::getHostName) + .collect(Collectors.joining(",")), + cassandraConfig.getPort()), + e); + } + cluster.close(); + } + + private static void validateCassandraTable(Session session) { + try { + new MappingManager(session).mapper(CassandraMutation.class).getTableMetadata(); + } catch (RuntimeException e) { + throw new RuntimeException( + String.format( + "Table created does not match the datastax object mapper: %s", + CassandraMutation.class.getSimpleName())); + } + } } diff --git a/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java b/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java new file mode 100644 index 00000000000..87a327e7726 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.utils; + +import feast.types.ValueProto.Value; + +/** + * Utility class for converting {@link Value} of different types to a string for storing as key in + * data stores + */ +public class ValueUtil { + + public static String toString(Value value) { + String strValue; + switch (value.getValCase()) { + case BYTES_VAL: + strValue = value.getBytesVal().toString(); + break; + case STRING_VAL: + strValue = value.getStringVal(); + break; + case INT32_VAL: + strValue = String.valueOf(value.getInt32Val()); + break; + case INT64_VAL: + strValue = String.valueOf(value.getInt64Val()); + break; + case DOUBLE_VAL: + strValue = String.valueOf(value.getDoubleVal()); + break; + case FLOAT_VAL: + strValue = String.valueOf(value.getFloatVal()); + break; + case BOOL_VAL: + strValue = String.valueOf(value.getBoolVal()); + break; + default: + throw new IllegalArgumentException( + String.format("toString method not supported for type %s", value.getValCase())); + } + return strValue; + } +} diff --git a/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java b/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java new file mode 100644 index 00000000000..23bbb9c3b31 --- /dev/null +++ b/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.cassandra; + +import com.datastax.driver.mapping.annotations.ClusteringColumn; +import com.datastax.driver.mapping.annotations.Computed; +import com.datastax.driver.mapping.annotations.PartitionKey; +import com.datastax.driver.mapping.annotations.Table; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.ingestion.utils.ValueUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.coders.DefaultCoder; + +/** + * Cassandra's object mapper that handles basic CRUD operations in Cassandra tables More info: + * https://docs.datastax.com/en/developer/java-driver/3.1/manual/object_mapper/ + */ +@DefaultCoder(value = AvroCoder.class) +@Table(name = "feature_store") +public final class CassandraMutation implements Serializable { + + public static final String ENTITIES = "entities"; + public static final String FEATURE = "feature"; + public static final String VALUE = "value"; + + @PartitionKey private final String entities; + + @ClusteringColumn private final String feature; + + private final ByteBuffer value; + + @Computed(value = "writetime(value)") + private final long writeTime; + + @Computed(value = "ttl(value)") + private final int ttl; + + // NoArgs constructor is needed when using Beam's CassandraIO withEntity and specifying this + // class, + // it looks for an init() method + CassandraMutation() { + this.entities = null; + this.feature = null; + this.value = null; + this.writeTime = 0; + this.ttl = 0; + } + + CassandraMutation(String entities, String feature, ByteBuffer value, long writeTime, int ttl) { + this.entities = entities; + this.feature = feature; + this.value = value; + this.writeTime = writeTime; + this.ttl = ttl; + } + + public long getWriteTime() { + return writeTime; + } + + public int getTtl() { + return ttl; + } + + static String keyFromFeatureRow(FeatureSetSpec featureSetSpec, FeatureRow featureRow) { + Set entityNames = + featureSetSpec.getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toSet()); + List entities = new ArrayList<>(); + for (Field field : featureRow.getFieldsList()) { + if (entityNames.contains(field.getName())) { + entities.add(field); + } + } + return featureRow.getFeatureSet() + + ":" + + entities.stream() + .map(f -> f.getName() + "=" + ValueUtil.toString(f.getValue())) + .collect(Collectors.joining("|")); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (o instanceof CassandraMutation) { + CassandraMutation that = (CassandraMutation) o; + return this.entities.equals(that.entities) + && this.feature.equals(that.feature) + && this.value.equals(that.value) + && this.writeTime == that.writeTime + && this.ttl == that.ttl; + } + return false; + } +} diff --git a/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java new file mode 100644 index 00000000000..177a8703e68 --- /dev/null +++ b/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java @@ -0,0 +1,85 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.cassandra; + +import com.google.protobuf.Duration; +import com.google.protobuf.util.Timestamps; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.beam.sdk.transforms.DoFn; +import org.slf4j.Logger; + +public class FeatureRowToCassandraMutationDoFn extends DoFn { + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(FeatureRowToCassandraMutationDoFn.class); + private Map featureSetSpecs; + private Map maxAges; + + public FeatureRowToCassandraMutationDoFn(Map specs, Duration defaultTtl) { + this.featureSetSpecs = specs; + this.maxAges = new HashMap<>(); + for (FeatureSetSpec spec : specs.values()) { + String featureSetName = spec.getName() + ":" + spec.getVersion(); + if (spec.getMaxAge() != null && spec.getMaxAge().getSeconds() > 0) { + maxAges.put(featureSetName, Math.toIntExact(spec.getMaxAge().getSeconds())); + } else { + maxAges.put(featureSetName, Math.toIntExact(defaultTtl.getSeconds())); + } + } + } + + /** Output a Cassandra mutation object for every feature in the feature row. */ + @ProcessElement + public void processElement(ProcessContext context) { + FeatureRow featureRow = context.element(); + try { + FeatureSetSpec featureSetSpec = featureSetSpecs.get(featureRow.getFeatureSet()); + Set featureNames = + featureSetSpec.getFeaturesList().stream() + .map(FeatureSpec::getName) + .collect(Collectors.toSet()); + String key = CassandraMutation.keyFromFeatureRow(featureSetSpec, featureRow); + + Collection mutations = new ArrayList<>(); + for (Field field : featureRow.getFieldsList()) { + if (featureNames.contains(field.getName())) { + mutations.add( + new CassandraMutation( + key, + field.getName(), + ByteBuffer.wrap(field.getValue().toByteArray()), + Timestamps.toMicros(featureRow.getEventTimestamp()), + maxAges.get(featureRow.getFeatureSet()))); + } + } + + mutations.forEach(context::output); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } +} diff --git a/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java b/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java new file mode 100644 index 00000000000..da914ca1757 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java @@ -0,0 +1,248 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.google.protobuf.InvalidProtocolBufferException; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.StoreProto.Store; +import feast.core.StoreProto.Store.CassandraConfig; +import feast.core.StoreProto.Store.StoreType; +import feast.test.TestUtil; +import feast.test.TestUtil.LocalCassandra; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType.Enum; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.apache.thrift.transport.TTransportException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +public class CassandraWriteToStoreIT implements Serializable { + private FeatureSetSpec featureSetSpec; + private FeatureRow row; + + class FakeCassandraWriteToStore extends WriteToStore { + + private FeatureSetSpec featureSetSpec; + + FakeCassandraWriteToStore(FeatureSetSpec featureSetSpec) { + this.featureSetSpec = featureSetSpec; + } + + @Override + public Store getStore() { + return Store.newBuilder() + .setType(StoreType.CASSANDRA) + .setName("SERVING") + .setCassandraConfig(getCassandraConfig()) + .build(); + } + + @Override + public Map getFeatureSetSpecs() { + return new HashMap() { + { + put(featureSetSpec.getName() + ":" + featureSetSpec.getVersion(), featureSetSpec); + } + }; + } + } + + private static CassandraConfig getCassandraConfig() { + return CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setTableName("feature_store") + .setKeyspace("test") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "SimpleStrategy"); + put("replication_factor", "1"); + } + }) + .build(); + } + + @BeforeClass + public static void startServer() throws InterruptedException, IOException, TTransportException { + LocalCassandra.start(); + LocalCassandra.createKeyspaceAndTable(getCassandraConfig()); + } + + @Before + public void setUp() { + featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + 1, + 10, + new HashMap() { + { + put("entity1", Enum.INT64); + put("entity2", Enum.STRING); + } + }, + new HashMap() { + { + put("feature1", Enum.INT64); + put("feature2", Enum.INT64); + } + }); + row = + TestUtil.createFeatureRow( + featureSetSpec, + 100, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("entity2", TestUtil.strValue("a")); + put("feature1", TestUtil.intValue(1)); + put("feature2", TestUtil.intValue(2)); + } + }); + } + + @Rule public transient TestPipeline testPipeline = TestPipeline.create(); + + @AfterClass + public static void cleanUp() { + LocalCassandra.stop(); + } + + @Test + public void testWriteCassandra_happyPath() throws InvalidProtocolBufferException { + PCollection input = testPipeline.apply(Create.of(row)); + + input.apply(new FakeCassandraWriteToStore(featureSetSpec)); + + testPipeline.run(); + + ResultSet resultSet = LocalCassandra.getSession().execute("SELECT * FROM test.feature_store"); + List actualResults = getResults(resultSet); + + List expectedFields = + Arrays.asList( + Field.newBuilder().setName("feature1").setValue(TestUtil.intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(TestUtil.intValue(2)).build()); + + assertTrue(actualResults.containsAll(expectedFields)); + assertEquals(expectedFields.size(), actualResults.size()); + } + + @Test(timeout = 30000) + public void testWriteCassandra_shouldNotRetrieveExpiredValues() + throws InvalidProtocolBufferException { + // Set max age to 1 second + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + 1, + 1, + new HashMap() { + { + put("entity1", Enum.INT64); + put("entity2", Enum.STRING); + } + }, + new HashMap() { + { + put("feature1", Enum.INT64); + put("feature2", Enum.INT64); + } + }); + + PCollection input = testPipeline.apply(Create.of(row)); + + input.apply(new FakeCassandraWriteToStore(featureSetSpec)); + + testPipeline.run(); + + while (true) { + ResultSet resultSet = + LocalCassandra.getSession() + .execute("SELECT feature, value, ttl(value) as expiry FROM test.feature_store"); + List results = getResults(resultSet); + if (results.isEmpty()) break; + } + } + + @Test + public void testWriteCassandra_shouldNotOverrideNewerValues() + throws InvalidProtocolBufferException { + FeatureRow olderRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("entity2", TestUtil.strValue("a")); + put("feature1", TestUtil.intValue(3)); + put("feature2", TestUtil.intValue(4)); + } + }); + + PCollection input = testPipeline.apply(Create.of(row, olderRow)); + + input.apply(new FakeCassandraWriteToStore(featureSetSpec)); + + testPipeline.run(); + + ResultSet resultSet = LocalCassandra.getSession().execute("SELECT * FROM test.feature_store"); + List actualResults = getResults(resultSet); + + List expectedFields = + Arrays.asList( + Field.newBuilder().setName("feature1").setValue(TestUtil.intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(TestUtil.intValue(2)).build()); + + assertTrue(actualResults.containsAll(expectedFields)); + assertEquals(expectedFields.size(), actualResults.size()); + } + + private List getResults(ResultSet resultSet) throws InvalidProtocolBufferException { + List results = new ArrayList<>(); + while (!resultSet.isExhausted()) { + Row row = resultSet.one(); + results.add( + Field.newBuilder() + .setName(row.getString("feature")) + .setValue(Value.parseFrom(row.getBytes("value"))) + .build()); + } + return results; + } +} diff --git a/ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java b/ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java new file mode 100644 index 00000000000..8c6874ecac7 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/util/CassandraStoreUtilIT.java @@ -0,0 +1,167 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.util; + +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.TableMetadata; +import com.datastax.driver.core.schemabuilder.Create; +import com.datastax.driver.core.schemabuilder.SchemaBuilder; +import feast.core.StoreProto.Store.CassandraConfig; +import feast.ingestion.utils.StoreUtil; +import feast.store.serving.cassandra.CassandraMutation; +import feast.test.TestUtil.LocalCassandra; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.thrift.transport.TTransportException; +import org.junit.After; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CassandraStoreUtilIT { + + @BeforeClass + public static void startServer() throws InterruptedException, IOException, TTransportException { + LocalCassandra.start(); + } + + @After + public void teardown() { + LocalCassandra.stop(); + } + + @Test + public void setupCassandra_shouldCreateKeyspaceAndTable() { + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("feature_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "NetworkTopologyStrategy"); + put("dc1", "2"); + put("dc2", "3"); + } + }) + .build(); + StoreUtil.setupCassandra(config); + + Map actualReplication = + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getReplication(); + Map expectedReplication = + new HashMap() { + { + put("class", "org.apache.cassandra.locator.NetworkTopologyStrategy"); + put("dc1", "2"); + put("dc2", "3"); + } + }; + TableMetadata tableMetadata = + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store"); + + Assert.assertEquals(expectedReplication, actualReplication); + Assert.assertNotNull(tableMetadata); + } + + @Test + public void setupCassandra_shouldBeIdempotent_whenTableAlreadyExistsAndSchemaMatches() { + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("feature_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "SimpleStrategy"); + put("replication_factor", "2"); + } + }) + .build(); + + LocalCassandra.createKeyspaceAndTable(config); + + // Check table is created + Assert.assertNotNull( + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store")); + + StoreUtil.setupCassandra(config); + + Assert.assertNotNull( + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store")); + } + + @Test(expected = RuntimeException.class) + public void setupCassandra_shouldThrowException_whenTableNameDoesNotMatchObjectMapper() { + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("test_data_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "NetworkTopologyStrategy"); + put("dc1", "2"); + put("dc2", "3"); + } + }) + .build(); + StoreUtil.setupCassandra(config); + } + + @Test(expected = RuntimeException.class) + public void setupCassandra_shouldThrowException_whenTableSchemaDoesNotMatchObjectMapper() { + LocalCassandra.getSession() + .execute( + "CREATE KEYSPACE test " + + "WITH REPLICATION = {" + + "'class': 'SimpleStrategy', 'replication_factor': 2 }"); + + Create createTable = + SchemaBuilder.createTable("test", "feature_store") + .ifNotExists() + .addPartitionKey(CassandraMutation.ENTITIES, DataType.text()) + .addClusteringColumn( + "featureName", DataType.text()) // Column name does not match in CassandraMutation + .addColumn(CassandraMutation.VALUE, DataType.blob()); + LocalCassandra.getSession().execute(createTable); + + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("feature_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "SimpleStrategy"); + put("replication_factor", "2"); + } + }) + .build(); + + StoreUtil.setupCassandra(config); + } +} diff --git a/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java new file mode 100644 index 00000000000..412c6488f20 --- /dev/null +++ b/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.cassandra; + +import com.google.protobuf.Duration; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.test.TestUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType.Enum; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.HashMap; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +public class FeatureRowToCassandraMutationDoFnTest implements Serializable { + + @Rule public transient TestPipeline testPipeline = TestPipeline.create(); + + @Test + public void processElement_shouldCreateCassandraMutation_givenFeatureRow() { + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + 1, + 10, + new HashMap() { + { + put("entity1", Enum.INT64); + } + }, + new HashMap() { + { + put("feature1", Enum.STRING); + } + }); + FeatureRow featureRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("feature1", TestUtil.strValue("a")); + } + }); + + PCollection input = testPipeline.apply(Create.of(featureRow)); + + PCollection output = + input.apply( + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + new HashMap() { + { + put( + featureSetSpec.getName() + ":" + featureSetSpec.getVersion(), + featureSetSpec); + } + }, + Duration.newBuilder().setSeconds(0).build()))); + + CassandraMutation[] expected = + new CassandraMutation[] { + new CassandraMutation( + "fs:1:entity1=1", + "feature1", + ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()), + 10000000, + 10) + }; + + PAssert.that(output).containsInAnyOrder(expected); + + testPipeline.run(); + } + + @Test + public void + processElement_shouldCreateCassandraMutations_givenFeatureRowWithMultipleEntitiesAndFeatures() { + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + 1, + 10, + new HashMap() { + { + put("entity1", Enum.INT64); + put("entity2", Enum.STRING); + } + }, + new HashMap() { + { + put("feature1", Enum.STRING); + put("feature2", Enum.INT64); + } + }); + FeatureRow featureRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("entity2", TestUtil.strValue("b")); + put("feature1", TestUtil.strValue("a")); + put("feature2", TestUtil.intValue(2)); + } + }); + + PCollection input = testPipeline.apply(Create.of(featureRow)); + + PCollection output = + input.apply( + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + new HashMap() { + { + put( + featureSetSpec.getName() + ":" + featureSetSpec.getVersion(), + featureSetSpec); + } + }, + Duration.newBuilder().setSeconds(0).build()))); + + CassandraMutation[] expected = + new CassandraMutation[] { + new CassandraMutation( + "fs:1:entity1=1|entity2=b", + "feature1", + ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()), + 10000000, + 10), + new CassandraMutation( + "fs:1:entity1=1|entity2=b", + "feature2", + ByteBuffer.wrap(TestUtil.intValue(2).toByteArray()), + 10000000, + 10) + }; + + PAssert.that(output).containsInAnyOrder(expected); + + testPipeline.run(); + } + + @Test + public void processElement_shouldUseDefaultMaxAge_whenMissingMaxAge() { + Duration defaultTtl = Duration.newBuilder().setSeconds(500).build(); + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + 1, + 0, + new HashMap() { + { + put("entity1", Enum.INT64); + } + }, + new HashMap() { + { + put("feature1", Enum.STRING); + } + }); + FeatureRow featureRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("feature1", TestUtil.strValue("a")); + } + }); + + PCollection input = testPipeline.apply(Create.of(featureRow)); + + PCollection output = + input.apply( + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + new HashMap() { + { + put( + featureSetSpec.getName() + ":" + featureSetSpec.getVersion(), + featureSetSpec); + } + }, + defaultTtl))); + + CassandraMutation[] expected = + new CassandraMutation[] { + new CassandraMutation( + "fs:1:entity1=1", + "feature1", + ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()), + 10000000, + 500) + }; + + PAssert.that(output).containsInAnyOrder(expected); + + testPipeline.run(); + } +} diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java index ef41f3950a5..51d47127599 100644 --- a/ingestion/src/test/java/feast/test/TestUtil.java +++ b/ingestion/src/test/java/feast/test/TestUtil.java @@ -16,10 +16,17 @@ */ package feast.test; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; +import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.StoreProto.Store.CassandraConfig; import feast.ingestion.transform.WriteToStore; +import feast.ingestion.utils.StoreUtil; import feast.storage.RedisProto.RedisKey; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FeatureRowProto.FeatureRow.Builder; @@ -33,13 +40,18 @@ import feast.types.ValueProto.StringList; import feast.types.ValueProto.Value; import feast.types.ValueProto.ValueType; +import feast.types.ValueProto.ValueType.Enum; import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; import kafka.server.KafkaConfig; import kafka.server.KafkaServerStartable; import org.apache.beam.sdk.PipelineResult; @@ -51,8 +63,10 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.thrift.transport.TTransportException; import org.apache.zookeeper.server.ServerConfig; import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.cassandraunit.utils.EmbeddedCassandraServerHelper; import org.joda.time.Duration; import redis.embedded.RedisServer; @@ -81,6 +95,37 @@ public static void stop() { } } + public static class LocalCassandra { + + public static void start() throws InterruptedException, IOException, TTransportException { + EmbeddedCassandraServerHelper.startEmbeddedCassandra(); + } + + public static void createKeyspaceAndTable(CassandraConfig config) { + StoreUtil.setupCassandra(config); + } + + public static String getHost() { + return EmbeddedCassandraServerHelper.getHost(); + } + + public static int getPort() { + return EmbeddedCassandraServerHelper.getNativeTransportPort(); + } + + public static Cluster getCluster() { + return EmbeddedCassandraServerHelper.getCluster(); + } + + public static Session getSession() { + return EmbeddedCassandraServerHelper.getSession(); + } + + public static void stop() { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra(); + } + } + public static class LocalKafka { private static KafkaServerStartable server; @@ -163,6 +208,85 @@ public static void publishFeatureRowsToKafka( }); } + /** + * Create a Feature Set Spec. + * + * @param name name of the feature set + * @param version version of the feature set + * @param maxAgeSeconds max age + * @param entities entities provided as map of string to {@link Enum} + * @param features features provided as map of string to {@link Enum} + * @return {@link FeatureSetSpec} + */ + public static FeatureSetSpec createFeatureSetSpec( + String name, + int version, + int maxAgeSeconds, + Map entities, + Map features) { + FeatureSetSpec.Builder featureSetSpec = + FeatureSetSpec.newBuilder() + .setName(name) + .setVersion(version) + .setMaxAge(com.google.protobuf.Duration.newBuilder().setSeconds(maxAgeSeconds).build()); + + for (Entry entity : entities.entrySet()) { + featureSetSpec.addEntities( + EntitySpec.newBuilder().setName(entity.getKey()).setValueType(entity.getValue()).build()); + } + + for (Entry feature : features.entrySet()) { + featureSetSpec.addFeatures( + FeatureSpec.newBuilder() + .setName(feature.getKey()) + .setValueType(feature.getValue()) + .build()); + } + + return featureSetSpec.build(); + } + + /** + * Create a Feature Row. + * + * @param featureSetSpec {@link FeatureSetSpec} + * @param timestampSeconds timestamp given in seconds + * @param fields fields provided as a map name to {@link Value} + * @return {@link FeatureRow} + */ + public static FeatureRow createFeatureRow( + FeatureSetSpec featureSetSpec, long timestampSeconds, Map fields) { + List featureNames = + featureSetSpec.getFeaturesList().stream() + .map(FeatureSpec::getName) + .collect(Collectors.toList()); + List entityNames = + featureSetSpec.getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toList()); + List requiredFields = + Stream.concat(featureNames.stream(), entityNames.stream()).collect(Collectors.toList()); + + if (fields.keySet().containsAll(requiredFields)) { + FeatureRow.Builder featureRow = + FeatureRow.newBuilder() + .setFeatureSet(featureSetSpec.getName() + ":" + featureSetSpec.getVersion()) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(timestampSeconds).build()); + for (Entry field : fields.entrySet()) { + featureRow.addFields( + Field.newBuilder().setName(field.getKey()).setValue(field.getValue()).build()); + } + return featureRow.build(); + } else { + String missingFields = + requiredFields.stream() + .filter(f -> !fields.keySet().contains(f)) + .collect(Collectors.joining(",")); + throw new IllegalArgumentException( + "FeatureRow is missing some fields defined in FeatureSetSpec: " + missingFields); + } + } + /** * Create a Feature Row with random value according to the FeatureSetSpec * @@ -352,15 +476,16 @@ public static Field field(String name, Object value, ValueType.Enum valueType) { /** * This blocking method waits until an ImportJob pipeline has written all elements to the store. - *

- * The pipeline must be in the RUNNING state before calling this method. * - * @param pipelineResult result of running the Pipeline + *

The pipeline must be in the RUNNING state before calling this method. + * + * @param pipelineResult result of running the Pipeline * @param maxWaitDuration wait until this max amount of duration * @throws InterruptedException if the thread is interruped while waiting */ - public static void waitUntilAllElementsAreWrittenToStore(PipelineResult pipelineResult, - Duration maxWaitDuration, Duration checkInterval) throws InterruptedException { + public static void waitUntilAllElementsAreWrittenToStore( + PipelineResult pipelineResult, Duration maxWaitDuration, Duration checkInterval) + throws InterruptedException { if (pipelineResult.getState().isTerminal()) { return; } @@ -409,4 +534,12 @@ public static void waitUntilAllElementsAreWrittenToStore(PipelineResult pipeline } } } + + public static Value intValue(int val) { + return Value.newBuilder().setInt64Val(val).build(); + } + + public static Value strValue(String val) { + return Value.newBuilder().setStringVal(val).build(); + } } diff --git a/pom.xml b/pom.xml index edf2a0244e9..98586740678 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ com.google.guava guava - 26.0-jre + 25.0-jre com.google.protobuf diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto index e1b8c581a38..9e1dc33143e 100644 --- a/protos/feast/core/Store.proto +++ b/protos/feast/core/Store.proto @@ -17,6 +17,8 @@ syntax = "proto3"; package feast.core; +import "google/protobuf/duration.proto"; + option java_package = "feast.core"; option java_outer_classname = "StoreProto"; option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core"; @@ -103,7 +105,20 @@ message Store { // BIGQUERY = 2; - // Unsupported in Feast 0.3 + // Cassandra stores entities as a string partition key, feature as clustering column. + // NOTE: This store currently uses max_age defined in FeatureSet for ttl + // + // Columns: + // - entities: concatenated string of feature set name and all entities' keys and values + // entities concatenated format - [feature_set]:[entity_name1=entity_value1]|[entity_name2=entity_value2] + // TODO: string representation of float or double types may have different value in different runtime or platform + // - feature: clustering column where each feature is a column + // - value: byte array of Value (refer to feast.types.Value) + // + // Internal columns: + // - writeTime: timestamp of the written record. This is used to ensure that new records are not replaced + // by older ones + // - ttl: expiration time the record. Currently using max_age from feature set spec as ttl CASSANDRA = 3; } @@ -118,8 +133,22 @@ message Store { } message CassandraConfig { - string host = 1; + // - bootstrapHosts: [comma delimited value of hosts] + string bootstrap_hosts = 1; int32 port = 2; + string keyspace = 3; + + // Please note that table name must be "feature_store" as is specified in the @Table annotation of the + // datastax object mapper + string table_name = 4; + + // This specifies the replication strategy to use. Please refer to docs for more details: + // https://docs.datastax.com/en/dse/6.7/cql/cql/cql_reference/cql_commands/cqlCreateKeyspace.html#cqlCreateKeyspace__cqlCreateKeyspacereplicationmap-Pr3yUQ7t + map replication_options = 5; + + // Default expiration in seconds to use when FeatureSetSpec does not have max_age defined. + // Specify 0 for no default expiration + google.protobuf.Duration default_ttl = 6; } message Subscription { diff --git a/serving/pom.xml b/serving/pom.xml index ab9efaff26e..3d41c18b8dc 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -141,7 +141,19 @@ redis.clients jedis - + + + com.datastax.cassandra + cassandra-driver-core + 3.4.0 + + + io.netty + * + + + + com.google.guava guava @@ -227,6 +239,13 @@ spring-boot-starter-test test + + + org.cassandraunit + cassandra-unit-shaded + 3.11.2.0 + test + diff --git a/serving/sample_cassandra_config.yml b/serving/sample_cassandra_config.yml new file mode 100644 index 00000000000..ca3d4cbbcca --- /dev/null +++ b/serving/sample_cassandra_config.yml @@ -0,0 +1,13 @@ +name: serving +type: CASSANDRA +cassandra_config: + bootstrap_hosts: localhost + port: 9042 + keyspace: feast + table_name: feature_store + replication_options: + class: SimpleStrategy + replication_factor: 1 +subscriptions: + - name: "*" + version: ">0" diff --git a/serving/src/main/java/feast/serving/FeastProperties.java b/serving/src/main/java/feast/serving/FeastProperties.java index e511835b0aa..356f808127b 100644 --- a/serving/src/main/java/feast/serving/FeastProperties.java +++ b/serving/src/main/java/feast/serving/FeastProperties.java @@ -85,6 +85,15 @@ public static class StoreProperties { private String configPath; private int redisPoolMaxSize; private int redisPoolMaxIdle; + private int cassandraPoolCoreLocalConnections; + private int cassandraPoolMaxLocalConnections; + private int cassandraPoolCoreRemoteConnections; + private int cassandraPoolMaxRemoteConnections; + private int cassandraPoolMaxRequestsLocalConnection; + private int cassandraPoolMaxRequestsRemoteConnection; + private int cassandraPoolNewLocalConnectionThreshold; + private int cassandraPoolNewRemoteConnectionThreshold; + private int cassandraPoolTimeoutMillis; public String getConfigPath() { return this.configPath; @@ -98,6 +107,42 @@ public int getRedisPoolMaxIdle() { return this.redisPoolMaxIdle; } + public int getCassandraPoolCoreLocalConnections() { + return this.cassandraPoolCoreLocalConnections; + } + + public int getCassandraPoolMaxLocalConnections() { + return this.cassandraPoolMaxLocalConnections; + } + + public int getCassandraPoolCoreRemoteConnections() { + return this.cassandraPoolCoreRemoteConnections; + } + + public int getCassandraPoolMaxRemoteConnections() { + return this.cassandraPoolMaxRemoteConnections; + } + + public int getCassandraPoolMaxRequestsLocalConnection() { + return this.cassandraPoolMaxRequestsLocalConnection; + } + + public int getCassandraPoolMaxRequestsRemoteConnection() { + return this.cassandraPoolMaxRequestsRemoteConnection; + } + + public int getCassandraPoolNewLocalConnectionThreshold() { + return this.cassandraPoolNewLocalConnectionThreshold; + } + + public int getCassandraPoolNewRemoteConnectionThreshold() { + return this.cassandraPoolNewRemoteConnectionThreshold; + } + + public int getCassandraPoolTimeoutMillis() { + return this.cassandraPoolTimeoutMillis; + } + public void setConfigPath(String configPath) { this.configPath = configPath; } @@ -109,6 +154,46 @@ public void setRedisPoolMaxSize(int redisPoolMaxSize) { public void setRedisPoolMaxIdle(int redisPoolMaxIdle) { this.redisPoolMaxIdle = redisPoolMaxIdle; } + + public void setCassandraPoolCoreLocalConnections(int cassandraPoolCoreLocalConnections) { + this.cassandraPoolCoreLocalConnections = cassandraPoolCoreLocalConnections; + } + + public void setCassandraPoolMaxLocalConnections(int cassandraPoolMaxLocalConnections) { + this.cassandraPoolMaxLocalConnections = cassandraPoolMaxLocalConnections; + } + + public void setCassandraPoolCoreRemoteConnections(int cassandraPoolCoreRemoteConnections) { + this.cassandraPoolCoreRemoteConnections = cassandraPoolCoreRemoteConnections; + } + + public void setCassandraPoolMaxRemoteConnections(int cassandraPoolMaxRemoteConnections) { + this.cassandraPoolMaxRemoteConnections = cassandraPoolMaxRemoteConnections; + } + + public void setCassandraPoolMaxRequestsLocalConnection( + int cassandraPoolMaxRequestsLocalConnection) { + this.cassandraPoolMaxRequestsLocalConnection = cassandraPoolMaxRequestsLocalConnection; + } + + public void setCassandraPoolMaxRequestsRemoteConnection( + int cassandraPoolMaxRequestsRemoteConnection) { + this.cassandraPoolMaxRequestsRemoteConnection = cassandraPoolMaxRequestsRemoteConnection; + } + + public void setCassandraPoolNewLocalConnectionThreshold( + int cassandraPoolNewLocalConnectionThreshold) { + this.cassandraPoolNewLocalConnectionThreshold = cassandraPoolNewLocalConnectionThreshold; + } + + public void setCassandraPoolNewRemoteConnectionThreshold( + int cassandraPoolNewRemoteConnectionThreshold) { + this.cassandraPoolNewRemoteConnectionThreshold = cassandraPoolNewRemoteConnectionThreshold; + } + + public void setCassandraPoolTimeoutMillis(int cassandraPoolTimeoutMillis) { + this.cassandraPoolTimeoutMillis = cassandraPoolTimeoutMillis; + } } public static class JobProperties { diff --git a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java index 08b9655e3e1..81994941d52 100644 --- a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java +++ b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java @@ -16,6 +16,10 @@ */ package feast.serving.configuration; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.Session; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.storage.Storage; @@ -23,19 +27,26 @@ import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.BigQueryConfig; import feast.core.StoreProto.Store.Builder; +import feast.core.StoreProto.Store.CassandraConfig; import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; import feast.core.StoreProto.Store.Subscription; import feast.serving.FeastProperties; import feast.serving.FeastProperties.JobProperties; +import feast.serving.FeastProperties.StoreProperties; import feast.serving.service.BigQueryServingService; import feast.serving.service.CachedSpecService; +import feast.serving.service.CassandraServingService; import feast.serving.service.JobService; import feast.serving.service.NoopJobService; import feast.serving.service.RedisServingService; import feast.serving.service.ServingService; import io.opentracing.Tracer; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -76,6 +87,13 @@ private Store setStoreConfig(Store.Builder builder, Map options) .build(); return builder.setBigqueryConfig(bqConfig).build(); case CASSANDRA: + CassandraConfig cassandraConfig = + CassandraConfig.newBuilder() + .setBootstrapHosts(options.get("host")) + .setPort(Integer.parseInt(options.get("port"))) + .setKeyspace(options.get("keyspace")) + .build(); + return builder.setCassandraConfig(cassandraConfig).build(); default: throw new IllegalArgumentException( String.format( @@ -135,6 +153,47 @@ public ServingService servingService( storage); break; case CASSANDRA: + StoreProperties storeProperties = feastProperties.getStore(); + PoolingOptions poolingOptions = new PoolingOptions(); + poolingOptions.setCoreConnectionsPerHost( + HostDistance.LOCAL, storeProperties.getCassandraPoolCoreLocalConnections()); + poolingOptions.setCoreConnectionsPerHost( + HostDistance.REMOTE, storeProperties.getCassandraPoolCoreRemoteConnections()); + poolingOptions.setMaxConnectionsPerHost( + HostDistance.LOCAL, storeProperties.getCassandraPoolMaxLocalConnections()); + poolingOptions.setMaxConnectionsPerHost( + HostDistance.REMOTE, storeProperties.getCassandraPoolMaxRemoteConnections()); + poolingOptions.setMaxRequestsPerConnection( + HostDistance.LOCAL, storeProperties.getCassandraPoolMaxRequestsLocalConnection()); + poolingOptions.setMaxRequestsPerConnection( + HostDistance.REMOTE, storeProperties.getCassandraPoolMaxRequestsRemoteConnection()); + poolingOptions.setNewConnectionThreshold( + HostDistance.LOCAL, storeProperties.getCassandraPoolNewLocalConnectionThreshold()); + poolingOptions.setNewConnectionThreshold( + HostDistance.REMOTE, storeProperties.getCassandraPoolNewRemoteConnectionThreshold()); + poolingOptions.setPoolTimeoutMillis(storeProperties.getCassandraPoolTimeoutMillis()); + CassandraConfig cassandraConfig = store.getCassandraConfig(); + List contactPoints = + Arrays.stream(cassandraConfig.getBootstrapHosts().split(",")) + .map(h -> new InetSocketAddress(h, cassandraConfig.getPort())) + .collect(Collectors.toList()); + Cluster cluster = + Cluster.builder() + .addContactPointsWithPorts(contactPoints) + .withPoolingOptions(poolingOptions) + .build(); + // Session in Cassandra is thread-safe and maintains connections to cluster nodes internally + // Recommended to use one session per keyspace instead of open and close connection for each + // request + Session session = cluster.connect(); + servingService = + new CassandraServingService( + session, + cassandraConfig.getKeyspace(), + cassandraConfig.getTableName(), + specService, + tracer); + break; case UNRECOGNIZED: case INVALID: throw new IllegalArgumentException( diff --git a/serving/src/main/java/feast/serving/service/CassandraServingService.java b/serving/src/main/java/feast/serving/service/CassandraServingService.java new file mode 100644 index 00000000000..e9a7dff8ac3 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/CassandraServingService.java @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.querybuilder.QueryBuilder; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import feast.serving.ServingAPIProto.FeatureSetRequest; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.serving.util.ValueUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import io.opentracing.Scope; +import io.opentracing.Tracer; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class CassandraServingService extends OnlineServingService { + + private final Session session; + private final String keyspace; + private final String tableName; + private final Tracer tracer; + + public CassandraServingService( + Session session, + String keyspace, + String tableName, + CachedSpecService specService, + Tracer tracer) { + super(specService, tracer); + this.session = session; + this.keyspace = keyspace; + this.tableName = tableName; + this.tracer = tracer; + } + + @Override + List createLookupKeys( + List featureSetEntityNames, + List entityRows, + FeatureSetRequest featureSetRequest) { + try (Scope scope = tracer.buildSpan("Cassandra-makeCassandraKeys").startActive(true)) { + String featureSetId = + String.format("%s:%s", featureSetRequest.getName(), featureSetRequest.getVersion()); + return entityRows.stream() + .map(row -> createCassandraKey(featureSetId, featureSetEntityNames, row)) + .collect(Collectors.toList()); + } + } + + @Override + protected boolean isEmpty(ResultSet response) { + return response.isExhausted(); + } + + /** + * Send a list of get request as an mget + * + * @param keys list of string keys + * @return list of {@link FeatureRow} in primitive byte representation for each key + */ + @Override + protected List getAll(List keys) { + List results = new ArrayList<>(); + for (String key : keys) { + results.add( + session.execute( + QueryBuilder.select() + .column("entities") + .column("feature") + .column("value") + .writeTime("value") + .as("writetime") + .from(keyspace, tableName) + .where(QueryBuilder.eq("entities", key)))); + } + return results; + } + + @Override + FeatureRow parseResponse(ResultSet resultSet) { + List fields = new ArrayList<>(); + Instant instant = Instant.now(); + while (!resultSet.isExhausted()) { + Row row = resultSet.one(); + long microSeconds = row.getLong("writetime"); + instant = + Instant.ofEpochSecond( + TimeUnit.MICROSECONDS.toSeconds(microSeconds), + TimeUnit.MICROSECONDS.toNanos( + Math.floorMod(microSeconds, TimeUnit.SECONDS.toMicros(1)))); + try { + fields.add( + Field.newBuilder() + .setName(row.getString("feature")) + .setValue(Value.parseFrom(ByteBuffer.wrap(row.getBytes("value").array()))) + .build()); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } + } + return FeatureRow.newBuilder() + .addAllFields(fields) + .setEventTimestamp( + Timestamp.newBuilder() + .setSeconds(instant.getEpochSecond()) + .setNanos(instant.getNano()) + .build()) + .build(); + } + + /** + * Create cassandra keys + * + * @param featureSet featureSet reference of the feature. E.g. feature_set_1:1 + * @param featureSetEntityNames entity names that belong to the featureSet + * @param entityRow entityRow to build the key from + * @return String + */ + private static String createCassandraKey( + String featureSet, List featureSetEntityNames, EntityRow entityRow) { + Map fieldsMap = entityRow.getFieldsMap(); + List res = new ArrayList<>(); + for (String entityName : featureSetEntityNames) { + res.add(entityName + "=" + ValueUtil.toString(fieldsMap.get(entityName))); + } + return featureSet + ":" + String.join("|", res); + } +} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java new file mode 100644 index 00000000000..699d48c1214 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import static feast.serving.util.Metrics.missingKeyCount; +import static feast.serving.util.Metrics.requestCount; +import static feast.serving.util.Metrics.requestLatency; +import static feast.serving.util.Metrics.staleKeyCount; + +import com.google.common.collect.Maps; +import com.google.protobuf.Duration; +import com.google.protobuf.InvalidProtocolBufferException; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.serving.ServingAPIProto.FeastServingType; +import feast.serving.ServingAPIProto.FeatureSetRequest; +import feast.serving.ServingAPIProto.GetBatchFeaturesRequest; +import feast.serving.ServingAPIProto.GetBatchFeaturesResponse; +import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; +import feast.serving.ServingAPIProto.GetFeastServingInfoResponse; +import feast.serving.ServingAPIProto.GetJobRequest; +import feast.serving.ServingAPIProto.GetJobResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import io.grpc.Status; +import io.opentracing.Scope; +import io.opentracing.Tracer; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +abstract class OnlineServingService implements ServingService { + + private final CachedSpecService specService; + private final Tracer tracer; + + OnlineServingService(CachedSpecService specService, Tracer tracer) { + this.specService = specService; + this.tracer = tracer; + } + + @Override + public GetFeastServingInfoResponse getFeastServingInfo( + GetFeastServingInfoRequest getFeastServingInfoRequest) { + return GetFeastServingInfoResponse.newBuilder() + .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) + .build(); + } + + @Override + public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + @Override + public GetJobResponse getJob(GetJobRequest getJobRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + /** {@inheritDoc} */ + @Override + public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { + try (Scope scope = + tracer.buildSpan("OnlineServingService-getOnlineFeatures").startActive(true)) { + long startTime = System.currentTimeMillis(); + GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = + GetOnlineFeaturesResponse.newBuilder(); + + List entityRows = request.getEntityRowsList(); + Map> featureValuesMap = + entityRows.stream() + .collect(Collectors.toMap(er -> er, er -> Maps.newHashMap(er.getFieldsMap()))); + + List featureSetRequests = request.getFeatureSetsList(); + for (FeatureSetRequest featureSetRequest : featureSetRequests) { + + FeatureSetSpec featureSetSpec = + specService.getFeatureSet(featureSetRequest.getName(), featureSetRequest.getVersion()); + + List featureSetEntityNames = + featureSetSpec.getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toList()); + + Duration defaultMaxAge = featureSetSpec.getMaxAge(); + if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) { + featureSetRequest = featureSetRequest.toBuilder().setMaxAge(defaultMaxAge).build(); + } + + sendAndProcessMultiGet( + createLookupKeys(featureSetEntityNames, entityRows, featureSetRequest), + entityRows, + featureValuesMap, + featureSetRequest); + } + List fieldValues = + featureValuesMap.values().stream() + .map(m -> FieldValues.newBuilder().putAllFields(m).build()) + .collect(Collectors.toList()); + requestLatency.labels("getOnlineFeatures").observe(System.currentTimeMillis() - startTime); + return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); + } + } + + /** + * Create lookup keys for corresponding data stores + * + * @param featureSetEntityNames list of entity names + * @param entityRows list of {@link EntityRow} + * @param featureSetRequest {@link FeatureSetRequest} + * @return list of {@link LookupKeyType} + */ + abstract List createLookupKeys( + List featureSetEntityNames, + List entityRows, + FeatureSetRequest featureSetRequest); + + /** + * Checks whether the response is empty, i.e. feature does not exist in the store + * + * @param response {@link ResponseType} + * @return boolean + */ + protected abstract boolean isEmpty(ResponseType response); + + /** + * Send a list of get requests + * + * @param keys list of {@link LookupKeyType} + * @return list of {@link ResponseType} + */ + protected abstract List getAll(List keys); + + /** + * Parse response from data store to FeatureRow + * + * @param response {@link ResponseType} + * @return {@link FeatureRow} + */ + abstract FeatureRow parseResponse(ResponseType response) throws InvalidProtocolBufferException; + + private List getResponses(List keys) { + try (Scope scope = tracer.buildSpan("OnlineServingService-sendMultiGet").startActive(true)) { + long startTime = System.currentTimeMillis(); + try { + return getAll(keys); + } catch (Exception e) { + throw Status.NOT_FOUND + .withDescription("Unable to retrieve feature from online store") + .withCause(e) + .asRuntimeException(); + } finally { + requestLatency.labels("sendMultiGet").observe(System.currentTimeMillis() - startTime); + } + } + } + + private void sendAndProcessMultiGet( + List keys, + List entityRows, + Map> featureValuesMap, + FeatureSetRequest featureSetRequest) { + List responses = getResponses(keys); + + long startTime = System.currentTimeMillis(); + try (Scope scope = tracer.buildSpan("OnlineServingService-processResponse").startActive(true)) { + String featureSetId = + String.format("%s:%d", featureSetRequest.getName(), featureSetRequest.getVersion()); + Map nullValues = + featureSetRequest.getFeatureNamesList().stream() + .collect( + Collectors.toMap( + name -> featureSetId + ":" + name, name -> Value.newBuilder().build())); + for (int i = 0; i < responses.size(); i++) { + EntityRow entityRow = entityRows.get(i); + Map featureValues = featureValuesMap.get(entityRow); + try { + ResponseType response = responses.get(i); + if (isEmpty(response)) { + missingKeyCount.labels(featureSetRequest.getName()).inc(); + featureValues.putAll(nullValues); + continue; + } + + FeatureRow featureRow = parseResponse(response); + boolean stale = isStale(featureSetRequest, entityRow, featureRow); + if (stale) { + staleKeyCount.labels(featureSetRequest.getName()).inc(); + featureValues.putAll(nullValues); + continue; + } + + requestCount.labels(featureSetRequest.getName()).inc(); + featureRow.getFieldsList().stream() + .filter(f -> featureSetRequest.getFeatureNamesList().contains(f.getName())) + .forEach(f -> featureValues.put(featureSetId + ":" + f.getName(), f.getValue())); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } + } + } finally { + requestLatency.labels("processResponse").observe(System.currentTimeMillis() - startTime); + } + } + + private static boolean isStale( + FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { + if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) { + return false; + } + long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); + if (givenTimestamp == 0) { + givenTimestamp = System.currentTimeMillis() / 1000; + } + long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); + return timeDifference > featureSetRequest.getMaxAge().getSeconds(); + } +} diff --git a/serving/src/main/java/feast/serving/service/RedisServingService.java b/serving/src/main/java/feast/serving/service/RedisServingService.java index 9eaeb17dea3..df97c25820f 100644 --- a/serving/src/main/java/feast/serving/service/RedisServingService.java +++ b/serving/src/main/java/feast/serving/service/RedisServingService.java @@ -16,29 +16,10 @@ */ package feast.serving.service; -import static feast.serving.util.Metrics.missingKeyCount; -import static feast.serving.util.Metrics.requestCount; -import static feast.serving.util.Metrics.requestLatency; -import static feast.serving.util.Metrics.staleKeyCount; - -import com.google.common.collect.Maps; import com.google.protobuf.AbstractMessageLite; -import com.google.protobuf.Duration; import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.serving.ServingAPIProto.FeastServingType; import feast.serving.ServingAPIProto.FeatureSetRequest; -import feast.serving.ServingAPIProto.GetBatchFeaturesRequest; -import feast.serving.ServingAPIProto.GetBatchFeaturesResponse; -import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.serving.ServingAPIProto.GetJobRequest; -import feast.serving.ServingAPIProto.GetJobResponse; -import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.storage.RedisProto.RedisKey; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; @@ -53,88 +34,18 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -public class RedisServingService implements ServingService { +public class RedisServingService extends OnlineServingService { private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisServingService.class); private final JedisPool jedisPool; - private final CachedSpecService specService; private final Tracer tracer; public RedisServingService(JedisPool jedisPool, CachedSpecService specService, Tracer tracer) { + super(specService, tracer); this.jedisPool = jedisPool; - this.specService = specService; this.tracer = tracer; } - /** {@inheritDoc} */ - @Override - public GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest) { - return GetFeastServingInfoResponse.newBuilder() - .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) - .build(); - } - - /** {@inheritDoc} */ - @Override - public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { - try (Scope scope = tracer.buildSpan("Redis-getOnlineFeatures").startActive(true)) { - long startTime = System.currentTimeMillis(); - GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = - GetOnlineFeaturesResponse.newBuilder(); - - List entityRows = request.getEntityRowsList(); - Map> featureValuesMap = - entityRows.stream() - .collect(Collectors.toMap(er -> er, er -> Maps.newHashMap(er.getFieldsMap()))); - - List featureSetRequests = request.getFeatureSetsList(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - - FeatureSetSpec featureSetSpec = - specService.getFeatureSet(featureSetRequest.getName(), featureSetRequest.getVersion()); - - List featureSetEntityNames = - featureSetSpec.getEntitiesList().stream() - .map(EntitySpec::getName) - .collect(Collectors.toList()); - - Duration defaultMaxAge = featureSetSpec.getMaxAge(); - if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) { - featureSetRequest = featureSetRequest.toBuilder().setMaxAge(defaultMaxAge).build(); - } - - List redisKeys = - getRedisKeys(featureSetEntityNames, entityRows, featureSetRequest); - - try { - sendAndProcessMultiGet(redisKeys, entityRows, featureValuesMap, featureSetRequest); - } catch (InvalidProtocolBufferException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } - } - List fieldValues = - featureValuesMap.values().stream() - .map(m -> FieldValues.newBuilder().putAllFields(m).build()) - .collect(Collectors.toList()); - requestLatency.labels("getOnlineFeatures").observe(System.currentTimeMillis() - startTime); - return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); - } - } - - @Override - public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - - @Override - public GetJobResponse getJob(GetJobRequest getJobRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - /** * Build the redis keys for retrieval from the store. * @@ -143,7 +54,8 @@ public GetJobResponse getJob(GetJobRequest getJobRequest) { * @param featureSetRequest details of the requested featureSet * @return list of RedisKeys */ - private List getRedisKeys( + @Override + List createLookupKeys( List featureSetEntityNames, List entityRows, FeatureSetRequest featureSetRequest) { @@ -158,6 +70,33 @@ private List getRedisKeys( } } + @Override + protected boolean isEmpty(byte[] response) { + return response == null; + } + + /** + * Send a list of get request as an mget + * + * @param keys list of {@link RedisKey} + * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} + */ + @Override + protected List getAll(List keys) { + Jedis jedis = jedisPool.getResource(); + byte[][] binaryKeys = + keys.stream() + .map(AbstractMessageLite::toByteArray) + .collect(Collectors.toList()) + .toArray(new byte[0][0]); + return jedis.mget(binaryKeys); + } + + @Override + FeatureRow parseResponse(byte[] response) throws InvalidProtocolBufferException { + return FeatureRow.parseFrom(response); + } + /** * Create {@link RedisKey} * @@ -188,93 +127,4 @@ private RedisKey makeRedisKey( } return builder.build(); } - - private void sendAndProcessMultiGet( - List redisKeys, - List entityRows, - Map> featureValuesMap, - FeatureSetRequest featureSetRequest) - throws InvalidProtocolBufferException { - - List jedisResps = sendMultiGet(redisKeys); - long startTime = System.currentTimeMillis(); - try (Scope scope = tracer.buildSpan("Redis-processResponse").startActive(true)) { - String featureSetId = - String.format("%s:%d", featureSetRequest.getName(), featureSetRequest.getVersion()); - - Map nullValues = - featureSetRequest.getFeatureNamesList().stream() - .collect( - Collectors.toMap( - name -> featureSetId + ":" + name, name -> Value.newBuilder().build())); - - for (int i = 0; i < jedisResps.size(); i++) { - EntityRow entityRow = entityRows.get(i); - Map featureValues = featureValuesMap.get(entityRow); - - byte[] jedisResponse = jedisResps.get(i); - if (jedisResponse == null) { - missingKeyCount.labels(featureSetRequest.getName()).inc(); - featureValues.putAll(nullValues); - continue; - } - - FeatureRow featureRow = FeatureRow.parseFrom(jedisResponse); - - boolean stale = isStale(featureSetRequest, entityRow, featureRow); - if (stale) { - staleKeyCount.labels(featureSetRequest.getName()).inc(); - featureValues.putAll(nullValues); - continue; - } - - requestCount.labels(featureSetRequest.getName()).inc(); - featureRow.getFieldsList().stream() - .filter(f -> featureSetRequest.getFeatureNamesList().contains(f.getName())) - .forEach(f -> featureValues.put(featureSetId + ":" + f.getName(), f.getValue())); - } - } finally { - requestLatency.labels("processResponse").observe(System.currentTimeMillis() - startTime); - } - } - - private boolean isStale( - FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { - if (featureSetRequest.getMaxAge().equals(Duration.getDefaultInstance())) { - return false; - } - long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); - if (givenTimestamp == 0) { - givenTimestamp = System.currentTimeMillis() / 1000; - } - long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); - return timeDifference > featureSetRequest.getMaxAge().getSeconds(); - } - - /** - * Send a list of get request as an mget - * - * @param keys list of {@link RedisKey} - * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} - */ - private List sendMultiGet(List keys) { - try (Scope scope = tracer.buildSpan("Redis-sendMultiGet").startActive(true)) { - long startTime = System.currentTimeMillis(); - try (Jedis jedis = jedisPool.getResource()) { - byte[][] binaryKeys = - keys.stream() - .map(AbstractMessageLite::toByteArray) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - return jedis.mget(binaryKeys); - } catch (Exception e) { - throw Status.NOT_FOUND - .withDescription("Unable to retrieve feature from Redis") - .withCause(e) - .asRuntimeException(); - } finally { - requestLatency.labels("sendMultiGet").observe(System.currentTimeMillis() - startTime); - } - } - } } diff --git a/serving/src/main/java/feast/serving/util/ValueUtil.java b/serving/src/main/java/feast/serving/util/ValueUtil.java new file mode 100644 index 00000000000..e3ede6af984 --- /dev/null +++ b/serving/src/main/java/feast/serving/util/ValueUtil.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.util; + +import feast.types.ValueProto.Value; + +public class ValueUtil { + + public static String toString(Value value) { + String strValue; + switch (value.getValCase()) { + case BYTES_VAL: + strValue = value.getBytesVal().toString(); + break; + case STRING_VAL: + strValue = value.getStringVal(); + break; + case INT32_VAL: + strValue = String.valueOf(value.getInt32Val()); + break; + case INT64_VAL: + strValue = String.valueOf(value.getInt64Val()); + break; + case DOUBLE_VAL: + strValue = String.valueOf(value.getDoubleVal()); + break; + case FLOAT_VAL: + strValue = String.valueOf(value.getFloatVal()); + break; + case BOOL_VAL: + strValue = String.valueOf(value.getBoolVal()); + break; + default: + throw new IllegalArgumentException( + String.format("toString method not supported for type %s", value.getValCase())); + } + return strValue; + } +} diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 2daa83fbfb2..e18f49207b0 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -1,7 +1,7 @@ feast: # This value is retrieved from project.version properties in pom.xml # https://docs.spring.io/spring-boot/docs/current/reference/html/ - version: @project.version@ +# version: @project.version@ # GRPC service address for Feast Core # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) core-host: ${FEAST_CORE_HOST:localhost} @@ -24,6 +24,24 @@ feast: redis-pool-max-size: ${FEAST_REDIS_POOL_MAX_SIZE:128} # If serving redis, the redis pool max idle conns redis-pool-max-idle: ${FEAST_REDIS_POOL_MAX_IDLE:16} + # If serving cassandra, minimum connection for local host (one in same data center) + cassandra-pool-core-local-connections: ${FEAST_CASSANDRA_CORE_LOCAL_CONNECTIONS:1} + # If serving cassandra, maximum connection for local host (one in same data center) + cassandra-pool-max-local-connections: ${FEAST_CASSANDRA_MAX_LOCAL_CONNECTIONS:1} + # If serving cassandra, minimum connection for remote host (one in remote data center) + cassandra-pool-core-remote-connections: ${FEAST_CASSANDRA_CORE_REMOTE_CONNECTIONS:1} + # If serving cassandra, maximum connection for remote host (one in same data center) + cassandra-pool-max-remote-connections: ${FEAST_CASSANDRA_MAX_REMOTE_CONNECTIONS:1} + # If serving cassandra, maximum number of concurrent requests per local connection (one in same data center) + cassandra-pool-max-requests-local-connection: ${FEAST_CASSANDRA_MAX_REQUESTS_LOCAL_CONNECTION:32768} + # If serving cassandra, maximum number of concurrent requests per remote connection (one in remote data center) + cassandra-pool-max-requests-remote-connection: ${FEAST_CASSANDRA_MAX_REQUESTS_REMOTE_CONNECTION:2048} + # If serving cassandra, number of requests which trigger opening of new local connection (if it is available) + cassandra-pool-new-local-connection-threshold: ${FEAST_CASSANDRA_NEW_LOCAL_CONNECTION_THRESHOLD:30000} + # If serving cassandra, number of requests which trigger opening of new remote connection (if it is available) + cassandra-pool-new-remote-connection-threshold: ${FEAST_CASSANDRA_NEW_REMOTE_CONNECTION_THRESHOLD:400} + # If serving cassandra, number of milliseconds to wait to acquire connection (after that go to next available host in query plan) + cassandra-pool-timeout-millis: ${FEAST_CASSANDRA_POOL_TIMEOUT_MILLIS:0} jobs: # job-staging-location specifies the URI to store intermediate files for batch serving. diff --git a/serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java b/serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java new file mode 100644 index 00000000000..a1778251a3d --- /dev/null +++ b/serving/src/test/java/feast/serving/service/CassandraServingServiceITTest.java @@ -0,0 +1,244 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import static feast.serving.test.TestUtil.intValue; +import static feast.serving.test.TestUtil.responseToMapList; +import static feast.serving.test.TestUtil.strValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.querybuilder.Insert; +import com.datastax.driver.core.querybuilder.QueryBuilder; +import com.datastax.driver.core.utils.Bytes; +import com.google.common.collect.Lists; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.serving.ServingAPIProto.FeatureSetRequest; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.serving.test.TestUtil.LocalCassandra; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import io.opentracing.Tracer; +import io.opentracing.Tracer.SpanBuilder; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.thrift.transport.TTransportException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Mockito; + +public class CassandraServingServiceITTest { + + @Mock CachedSpecService specService; + + @Mock Tracer tracer; + + private CassandraServingService cassandraServingService; + private Session session; + + @BeforeClass + public static void startServer() throws InterruptedException, IOException, TTransportException { + LocalCassandra.start(); + LocalCassandra.createKeyspaceAndTable(); + } + + @Before + public void setup() { + initMocks(this); + FeatureSetSpec featureSetSpec = + FeatureSetSpec.newBuilder() + .addEntities(EntitySpec.newBuilder().setName("entity1")) + .addEntities(EntitySpec.newBuilder().setName("entity2")) + .build(); + + when(specService.getFeatureSet("featureSet", 1)).thenReturn(featureSetSpec); + when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); + + session = + new Cluster.Builder() + .addContactPoints(LocalCassandra.getHost()) + .withPort(LocalCassandra.getPort()) + .build() + .connect(); + + populateTable(session); + + cassandraServingService = + new CassandraServingService(session, "test", "feature_store", specService, tracer); + } + + private void populateTable(Session session) { + session.execute( + insertQuery( + "test", "feature_store", "featureSet:1:entity1=1|entity2=a", "feature1", intValue(1))); + session.execute( + insertQuery( + "test", "feature_store", "featureSet:1:entity1=1|entity2=a", "feature2", intValue(1))); + session.execute( + insertQuery( + "test", "feature_store", "featureSet:1:entity1=2|entity2=b", "feature1", intValue(1))); + session.execute( + insertQuery( + "test", "feature_store", "featureSet:1:entity1=2|entity2=b", "feature2", intValue(1))); + } + + @AfterClass + public static void cleanUp() { + LocalCassandra.stop(); + } + + @Test + public void shouldReturnResponseWithValuesIfKeysPresent() { + GetOnlineFeaturesRequest request = + GetOnlineFeaturesRequest.newBuilder() + .addFeatureSets( + FeatureSetRequest.newBuilder() + .setName("featureSet") + .setVersion(1) + .addAllFeatureNames(Lists.newArrayList("feature1", "feature2")) + .build()) + .addEntityRows( + EntityRow.newBuilder() + .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", intValue(1)) + .putFields("entity2", strValue("a"))) + .addEntityRows( + EntityRow.newBuilder() + .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", intValue(2)) + .putFields("entity2", strValue("b"))) + .build(); + + GetOnlineFeaturesResponse expected = + GetOnlineFeaturesResponse.newBuilder() + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(1)) + .putFields("entity2", strValue("a")) + .putFields("featureSet:1:feature1", intValue(1)) + .putFields("featureSet:1:feature2", intValue(1))) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(2)) + .putFields("entity2", strValue("b")) + .putFields("featureSet:1:feature1", intValue(1)) + .putFields("featureSet:1:feature2", intValue(1))) + .build(); + GetOnlineFeaturesResponse actual = cassandraServingService.getOnlineFeatures(request); + + assertThat( + responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + } + + @Test + public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + GetOnlineFeaturesRequest request = + GetOnlineFeaturesRequest.newBuilder() + .addFeatureSets( + FeatureSetRequest.newBuilder() + .setName("featureSet") + .setVersion(1) + .addAllFeatureNames(Lists.newArrayList("feature1", "feature2")) + .build()) + .addEntityRows( + EntityRow.newBuilder() + .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", intValue(1)) + .putFields("entity2", strValue("a"))) + // Non-existing entity keys + .addEntityRows( + EntityRow.newBuilder() + .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", intValue(55)) + .putFields("entity2", strValue("ff"))) + .build(); + + GetOnlineFeaturesResponse expected = + GetOnlineFeaturesResponse.newBuilder() + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(1)) + .putFields("entity2", strValue("a")) + .putFields("featureSet:1:feature1", intValue(1)) + .putFields("featureSet:1:feature2", intValue(1))) + // Missing keys will return empty values + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(55)) + .putFields("entity2", strValue("ff")) + .putFields("featureSet:1:feature1", Value.newBuilder().build()) + .putFields("featureSet:1:feature2", Value.newBuilder().build())) + .build(); + GetOnlineFeaturesResponse actual = cassandraServingService.getOnlineFeatures(request); + + assertThat( + responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + } + + // This test should fail if cassandra no longer stores write time as microseconds or if we change + // the way we parse microseconds to com.google.protobuf.Timestamp + @Test + public void shouldInsertAndParseWriteTimestampInMicroSeconds() + throws InvalidProtocolBufferException { + session.execute( + "INSERT INTO test.feature_store (entities, feature, value)\n" + + " VALUES ('ENT1', 'FEAT1'," + + Bytes.toHexString(Value.newBuilder().build().toByteArray()) + + ")\n" + + " USING TIMESTAMP 1574318287123456;"); + + ResultSet resultSet = + session.execute( + QueryBuilder.select() + .column("entities") + .column("feature") + .column("value") + .writeTime("value") + .as("writetime") + .from("test", "feature_store") + .where(QueryBuilder.eq("entities", "ENT1"))); + FeatureRow featureRow = cassandraServingService.parseResponse(resultSet); + + Assert.assertEquals( + Timestamp.newBuilder().setSeconds(1574318287).setNanos(123456000).build(), + featureRow.getEventTimestamp()); + } + + private Insert insertQuery( + String database, String table, String key, String featureName, Value value) { + return QueryBuilder.insertInto(database, table) + .value("entities", key) + .value("feature", featureName) + .value("value", ByteBuffer.wrap(value.toByteArray())); + } +} diff --git a/serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java b/serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java new file mode 100644 index 00000000000..f965b14a640 --- /dev/null +++ b/serving/src/test/java/feast/serving/service/CassandraServingServiceTest.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import static feast.serving.test.TestUtil.intValue; +import static feast.serving.test.TestUtil.strValue; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import com.datastax.driver.core.Session; +import feast.serving.ServingAPIProto.FeatureSetRequest; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import io.opentracing.Tracer; +import io.opentracing.Tracer.SpanBuilder; +import java.util.ArrayList; +import java.util.List; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Mockito; + +public class CassandraServingServiceTest { + + @Mock Session session; + + @Mock CachedSpecService specService; + + @Mock Tracer tracer; + + private CassandraServingService cassandraServingService; + + @Before + public void setUp() { + initMocks(this); + + when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); + + cassandraServingService = + new CassandraServingService(session, "test", "feature_store", specService, tracer); + } + + @Test + public void shouldConstructCassandraKeyCorrectly() { + List cassandraKeys = + cassandraServingService.createLookupKeys( + new ArrayList() { + { + add("entity1"); + add("entity2"); + } + }, + new ArrayList() { + { + add( + EntityRow.newBuilder() + .putFields("entity1", intValue(1)) + .putFields("entity2", strValue("a")) + .build()); + add( + EntityRow.newBuilder() + .putFields("entity1", intValue(2)) + .putFields("entity2", strValue("b")) + .build()); + } + }, + FeatureSetRequest.newBuilder().setName("featureSet").setVersion(1).build()); + + List expectedKeys = + new ArrayList() { + { + add("featureSet:1:entity1=1|entity2=a"); + add("featureSet:1:entity1=2|entity2=b"); + } + }; + + Assert.assertEquals(expectedKeys, cassandraKeys); + } + + @Test(expected = Exception.class) + public void shouldThrowExceptionWhenCannotConstructCassandraKey() { + List cassandraKeys = + cassandraServingService.createLookupKeys( + new ArrayList() { + { + add("entity1"); + add("entity2"); + } + }, + new ArrayList() { + { + add(EntityRow.newBuilder().putFields("entity1", intValue(1)).build()); + add( + EntityRow.newBuilder() + .putFields("entity1", intValue(2)) + .putFields("entity2", strValue("b")) + .build()); + } + }, + FeatureSetRequest.newBuilder().setName("featureSet").setVersion(1).build()); + } +} diff --git a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java b/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java index 890699db6d1..dd448fdf2b2 100644 --- a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java @@ -16,6 +16,9 @@ */ package feast.serving.service; +import static feast.serving.test.TestUtil.intValue; +import static feast.serving.test.TestUtil.responseToMapList; +import static feast.serving.test.TestUtil.strValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.mockito.Mockito.when; @@ -39,7 +42,6 @@ import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; @@ -525,20 +527,6 @@ public void shouldFilterOutUndesiredRows() { responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } - private List> responseToMapList(GetOnlineFeaturesResponse response) { - return response.getFieldValuesList().stream() - .map(FieldValues::getFieldsMap) - .collect(Collectors.toList()); - } - - private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .addEntities(EntitySpec.newBuilder().setName("entity1")) diff --git a/serving/src/test/java/feast/serving/test/TestUtil.java b/serving/src/test/java/feast/serving/test/TestUtil.java new file mode 100644 index 00000000000..9c533590719 --- /dev/null +++ b/serving/src/test/java/feast/serving/test/TestUtil.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.test; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.types.ValueProto.Value; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.thrift.transport.TTransportException; +import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.cassandraunit.utils.EmbeddedCassandraServerHelper; + +@SuppressWarnings("WeakerAccess") +public class TestUtil { + + public static class LocalCassandra { + + public static void start() throws InterruptedException, IOException, TTransportException { + EmbeddedCassandraServerHelper.startEmbeddedCassandra(); + } + + public static void createKeyspaceAndTable() { + new ClassPathCQLDataSet("embedded-store/LoadCassandra.cql", true, true) + .getCQLStatements() + .forEach(s -> LocalCassandra.getSession().execute(s)); + } + + public static String getHost() { + return EmbeddedCassandraServerHelper.getHost(); + } + + public static int getPort() { + return EmbeddedCassandraServerHelper.getNativeTransportPort(); + } + + public static Cluster getCluster() { + return EmbeddedCassandraServerHelper.getCluster(); + } + + public static Session getSession() { + return EmbeddedCassandraServerHelper.getSession(); + } + + public static void stop() { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra(); + } + } + + public static List> responseToMapList(GetOnlineFeaturesResponse response) { + return response.getFieldValuesList().stream() + .map(FieldValues::getFieldsMap) + .collect(Collectors.toList()); + } + + public static Value intValue(int val) { + return Value.newBuilder().setInt64Val(val).build(); + } + + public static Value strValue(String val) { + return Value.newBuilder().setStringVal(val).build(); + } +} diff --git a/serving/src/test/resources/embedded-store/LoadCassandra.cql b/serving/src/test/resources/embedded-store/LoadCassandra.cql new file mode 100644 index 00000000000..c80da294b71 --- /dev/null +++ b/serving/src/test/resources/embedded-store/LoadCassandra.cql @@ -0,0 +1,8 @@ +CREATE KEYSPACE test with replication = {'class':'SimpleStrategy','replication_factor':1}; + +CREATE TABLE test.feature_store( + entities text, + feature text, + value blob, + PRIMARY KEY (entities, feature) +) WITH CLUSTERING ORDER BY (feature DESC); \ No newline at end of file From 801408aa98a783398e8318586502348c3efe5184 Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Wed, 8 Jan 2020 16:31:40 +0700 Subject: [PATCH 10/17] Backport Sonatype publishing and datatypes module for v0.3.x (#407) * Use Nexus staging plugin for deployment (#394) * Use Nexus staging pluging for deployment * Fix Javadoc error * Hard coded parent version as variable substitution is not supported * Introduce datatypes/java module for proto generation Rather than the Maven protobuf plugin running on the same symlinked definitions in several Java modules, localize this process into one module that the others depend on. This provides a single module that can be depended on by third-party extensions with the bare minimum of dependencies. Also removes proto files that are no longer used. * Java SDK release script (#406) * Use back revision variable in pom.xml So user or CI system can easily override revision from external sources such as Git tag name * Add flatten maven plugin This plugin is useful during deployment so the final pom is resolved without parent dependency, i.e. we do not necessarily need to upload parent library * Increase versions for maven source,javadoc,spotless plugins So it has newer features and more fixes * Add gpg-plugin needed to sign releases * Use oss configure for flatten plugin, add developers info in pom.xml (required for releasing library * Add publish-java-sdk script * Add more logs to publish-java-sdk.sh * Add ProwJob publish-java-sdk * Use GPG_KEY_IMPORT_DIR variable * Update revision in pom.xml to 0.4.2-SNAPSHOT * Publish datatypes/java along with sdk/java Co-authored-by: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com> Co-authored-by: David Heryanto --- .gitignore | 6 + .prow/config.yaml | 29 ++++ .prow/scripts/publish-java-sdk.sh | 72 +++++++++ .prow/scripts/test-end-to-end-batch.sh | 4 +- .prow/scripts/test-end-to-end.sh | 7 +- Makefile | 2 +- core/pom.xml | 8 +- .../feast/core/config/MonitoringConfig.java | 2 +- .../java/feast/core/service/SpecService.java | 4 +- .../java/feast/core/util/PackageUtil.java | 3 +- core/src/main/proto/feast | 1 - core/src/main/proto/third_party | 1 - datatypes/java/README.md | 43 ++++++ datatypes/java/pom.xml | 72 +++++++++ {sdk => datatypes}/java/src/main/proto/feast | 0 datatypes/java/src/main/proto/third_party | 1 + infra/docker/core/Dockerfile | 2 +- infra/docker/serving/Dockerfile | 2 +- ingestion/pom.xml | 12 +- ingestion/src/main/proto/feast | 1 - .../feast_ingestion/types/CoalesceAccum.proto | 35 ----- .../feast_ingestion/types/CoalesceKey.proto | 25 --- ingestion/src/main/proto/third_party | 1 - ingestion/src/test/proto/DriverArea.proto | 10 -- ingestion/src/test/proto/Ping.proto | 12 -- pom.xml | 146 ++++++++++++++---- sdk/java/pom.xml | 14 +- serving/pom.xml | 12 +- serving/src/main/proto/feast | 1 - serving/src/main/proto/third_party | 1 - 30 files changed, 379 insertions(+), 150 deletions(-) create mode 100755 .prow/scripts/publish-java-sdk.sh delete mode 120000 core/src/main/proto/feast delete mode 120000 core/src/main/proto/third_party create mode 100644 datatypes/java/README.md create mode 100644 datatypes/java/pom.xml rename {sdk => datatypes}/java/src/main/proto/feast (100%) create mode 120000 datatypes/java/src/main/proto/third_party delete mode 120000 ingestion/src/main/proto/feast delete mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto delete mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto delete mode 120000 ingestion/src/main/proto/third_party delete mode 100644 ingestion/src/test/proto/DriverArea.proto delete mode 100644 ingestion/src/test/proto/Ping.proto delete mode 120000 serving/src/main/proto/feast delete mode 120000 serving/src/main/proto/third_party diff --git a/.gitignore b/.gitignore index 8aaa0b69200..a8c5e3fe0ba 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,9 @@ dmypy.json # Pyre type checker .pyre/ .vscode + +# .flattened-pom.xml is generated by flatten-maven-plugin. +# This pom should not be committed because it is only used during release / deployment. +.flattened-pom.xml + +sdk/python/docs/html diff --git a/.prow/config.yaml b/.prow/config.yaml index 4b6e352a12f..39b81d76fd6 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -169,6 +169,34 @@ postsubmits: # https://github.com/semver/semver/issues/232 - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + - name: publish-java-sdk + decorate: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: + - bash + - -c + - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1} + volumeMounts: + - name: gpg-keys + mountPath: /etc/gpg + readOnly: true + - name: maven-settings + mountPath: /root/.m2/settings.xml + subPath: settings.xml + readOnly: true + volumes: + - name: gpg-keys + secret: + secretName: gpg-keys + - name: maven-settings + secret: + secretName: maven-settings + branches: + # Filter on tags with semantic versioning, prefixed with "v" + - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + - name: publish-docker-images decorate: true spec: @@ -278,4 +306,5 @@ postsubmits: secret: secretName: feast-service-account branches: + # Filter on tags with semantic versioning, prefixed with "v" - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh new file mode 100755 index 00000000000..91123c8d4ee --- /dev/null +++ b/.prow/scripts/publish-java-sdk.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +GPG_KEY_IMPORT_DIR=/etc/gpg + +usage() +{ + echo "usage: publish-java-sdk.sh + + --revision Value for the revision e.g. '0.2.3' + --gpg-key-import-dir Directory containing existing GPG keys to import. + The directory should contain these 2 files: + - public-key + - private-key + The default value is '/etc/gpg' + + This script assumes the GPG private key is protected by a passphrase. + The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml + file, credentials to upload releases to Sonatype must also be provided. + + # Example settings: ~/.m2/settings.xml + + + + ossrh + SONATYPE_USER + SONATYPE_PASSWORD + + + + + ossrh + + GPG_PASSPHRASE + + + + +" +} + +while [ "$1" != "" ]; do + case "$1" in + --revision ) REVISION="$2"; shift;; + --gpg-key-import-dir ) GPG_KEY_IMPORT_DIR="$2"; shift;; + -h | --help ) usage; exit;; + * ) usage; exit 1 + esac + shift +done + +if [ -z $REVISION ]; then usage; exit 1; fi + +echo "============================================================" +echo "Checking Maven and GPG versions" +echo "============================================================" +mvn --version +echo "" +gpg --version + +echo "============================================================" +echo "Importing GPG keys" +echo "============================================================" +gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key +gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key + +echo "============================================================" +echo "Deploying Java SDK with revision: $REVISION" +echo "============================================================" +mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh index b370c5b045b..ba395fc6166 100755 --- a/.prow/scripts/test-end-to-end-batch.sh +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -138,7 +138,7 @@ management: enabled: false EOF -nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \ +nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 30 @@ -191,7 +191,7 @@ spring: web-environment: false EOF -nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.warehouse.application.yml \ &> /var/log/feast-serving-warehouse.log & sleep 15 diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 2c6f4a098f9..e8160444d39 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -75,6 +75,9 @@ Building jars for Feast # Build jars for Feast mvn --quiet --batch-mode --define skipTests=true clean package +ls -lh core/target/*jar +ls -lh serving/target/*jar + echo " ============================================================ Starting Feast Core @@ -121,7 +124,7 @@ management: enabled: false EOF -nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \ +nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 30 @@ -172,7 +175,7 @@ spring: web-environment: false EOF -nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.online.application.yml \ &> /var/log/feast-serving-online.log & sleep 15 diff --git a/Makefile b/Makefile index b7eb0edbade..9f0742b2e6c 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ build-cli: $(MAKE) -C cli build-all build-java: - mvn clean verify -Drevision=$(VERSION) + mvn clean verify build-docker: docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . diff --git a/core/pom.xml b/core/pom.xml index d5b09292e84..b56f74dea2e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - feast + dev.feast feast-parent ${revision} @@ -39,16 +39,12 @@ false - - org.xolstice.maven.plugins - protobuf-maven-plugin - - feast + dev.feast feast-ingestion ${project.version} diff --git a/core/src/main/java/feast/core/config/MonitoringConfig.java b/core/src/main/java/feast/core/config/MonitoringConfig.java index fd20bed1ee8..53c9562c47c 100644 --- a/core/src/main/java/feast/core/config/MonitoringConfig.java +++ b/core/src/main/java/feast/core/config/MonitoringConfig.java @@ -66,7 +66,7 @@ public FeastResourceCollector feastResourceCollector( /** * Register custom Prometheus collector that exports metrics about JVM resource usage. * - * @return @{link {@link JVMResourceCollector}} + * @return {@link JVMResourceCollector} */ @Bean public JVMResourceCollector jvmResourceCollector() { diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 4ea2d288f2e..99862fd5dc0 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -82,7 +82,7 @@ public SpecService( * required. If the version is provided then it will be used for the lookup. If the version is * omitted then the latest version will be returned. * - * @param GetFeatureSetRequest containing the name and version of the feature set + * @param request containing the name and version of the feature set * @return GetFeatureSetResponse containing a single feature set */ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) @@ -141,7 +141,7 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) * *

The version filter is optional; If not provided, this method will return all featureSet * versions of the featureSet name provided. Valid version filters should optionally contain a - * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1 + * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1 * * @param filter filter containing the desired featureSet name and version filter * @return ListFeatureSetsResponse with list of featureSets found matching the filter diff --git a/core/src/main/java/feast/core/util/PackageUtil.java b/core/src/main/java/feast/core/util/PackageUtil.java index ef27332ac0f..20b2310644b 100644 --- a/core/src/main/java/feast/core/util/PackageUtil.java +++ b/core/src/main/java/feast/core/util/PackageUtil.java @@ -49,8 +49,9 @@ public class PackageUtil { * handled by default in Apache Beam. * * 

-   * @code
+   * 
    * URL url = new URL("jar:file:/tmp/springexample/target/spring-example-1.0-SNAPSHOT.jar!/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar!/");
+   * 
    * String resolvedPath = resolveSpringBootPackageClasspath(url);
    * // resolvedPath should point to "/tmp/springexample/target/spring-example-1.0-SNAPSHOT/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar"
    * // Note that spring-example-1.0-SNAPSHOT.jar is extracted in the process.
diff --git a/core/src/main/proto/feast b/core/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/core/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/core/src/main/proto/third_party b/core/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/core/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
new file mode 100644
index 00000000000..a062144ff3f
--- /dev/null
+++ b/datatypes/java/README.md
@@ -0,0 +1,43 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+
+  dev.feast
+  datatypes-java
+  0.3.6-SNAPSHOT
+
+```
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Publishing
+----------
+
+TODO: this module should be published to Maven Central upon Feast releases—this
+needs to be set up in POM configuration and release automation.
diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
new file mode 100644
index 00000000000..a6dfa8e345a
--- /dev/null
+++ b/datatypes/java/pom.xml
@@ -0,0 +1,72 @@
+
+
+
+    4.0.0
+
+    Feast Data Types for Java
+    
+        Data types and service contracts used throughout Feast components and
+        their interchanges. These are generated from Protocol Buffers and gRPC
+        definitions included in the package.
+    
+    datatypes-java
+
+    
+      dev.feast
+      feast-parent
+      ${revision}
+      ../..
+    
+
+    
+      
+        
+          org.xolstice.maven.plugins
+          protobuf-maven-plugin
+          
+            true
+            
+                com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
+            
+            grpc-java
+            
+                io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
+            
+          
+          
+            
+              
+                compile
+                compile-custom
+                test-compile
+              
+            
+          
+        
+      
+    
+
+    
+      
+        io.grpc
+        grpc-services
+      
+    
+
diff --git a/sdk/java/src/main/proto/feast b/datatypes/java/src/main/proto/feast
similarity index 100%
rename from sdk/java/src/main/proto/feast
rename to datatypes/java/src/main/proto/feast
diff --git a/datatypes/java/src/main/proto/third_party b/datatypes/java/src/main/proto/third_party
new file mode 120000
index 00000000000..f015f8477d1
--- /dev/null
+++ b/datatypes/java/src/main/proto/third_party
@@ -0,0 +1 @@
+../../../../../protos/third_party
\ No newline at end of file
diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile
index c4cfe34b71b..91ef030dc90 100644
--- a/infra/docker/core/Dockerfile
+++ b/infra/docker/core/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \
+RUN mvn --also-make --projects core,ingestion \
   -DskipTests=true --batch-mode package
 #
 # Unpack the jar and copy the files into production Docker image
diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile
index 3517183d782..5605c8846de 100644
--- a/infra/docker/serving/Dockerfile
+++ b/infra/docker/serving/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects serving -Drevision=$REVISION \
+RUN mvn --also-make --projects serving \
   -DskipTests=true --batch-mode package
 
 # ============================================================
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index 72ac60578bc..35ca4eb6986 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -21,7 +21,7 @@
   4.0.0
 
   
-    feast
+    dev.feast
     feast-parent
     ${revision}
   
@@ -31,10 +31,6 @@
 
   
     
-      
-        org.xolstice.maven.plugins
-        protobuf-maven-plugin
-      
       
         org.apache.maven.plugins
         maven-shade-plugin
@@ -90,6 +86,12 @@
   
 
   
+    
+      dev.feast
+      datatypes-java
+      ${project.version}
+    
+
     
       org.glassfish
       javax.el
diff --git a/ingestion/src/main/proto/feast b/ingestion/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/ingestion/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
deleted file mode 100644
index cb64dd715f6..00000000000
--- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2018 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-import "google/protobuf/timestamp.proto";
-import "feast/types/Field.proto";
-
-option java_package = "feast_ingestion.types";
-option java_outer_classname = "CoalesceAccumProto";
-
-// Accumlator for merging feature rows.
-message CoalesceAccum {
-  string entityKey = 1;
-  google.protobuf.Timestamp eventTimestamp = 3;
-  string entityName = 4;
-
-  map features = 6;
-  // map of features to their counter values when they were last added to accumulator
-  map featureMarks = 7;
-  int64 counter = 8;
-}
\ No newline at end of file
diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
deleted file mode 100644
index 9730b49ec3b..00000000000
--- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2018 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-option java_package = "feast_ingestion.types";
-option java_outer_classname = "CoalesceKeyProto";
-
-message CoalesceKey {
-  string entityName = 1;
-  string entityKey = 2;
-}
\ No newline at end of file
diff --git a/ingestion/src/main/proto/third_party b/ingestion/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/ingestion/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/ingestion/src/test/proto/DriverArea.proto b/ingestion/src/test/proto/DriverArea.proto
deleted file mode 100644
index fee838b9e17..00000000000
--- a/ingestion/src/test/proto/DriverArea.proto
+++ /dev/null
@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package feast;
-
-option java_outer_classname = "DriverAreaProto";
-
-message DriverArea {
-  int32 driverId = 1;
-  int32 areaId = 2;
-}
\ No newline at end of file
diff --git a/ingestion/src/test/proto/Ping.proto b/ingestion/src/test/proto/Ping.proto
deleted file mode 100644
index b1069afa5bd..00000000000
--- a/ingestion/src/test/proto/Ping.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-syntax = "proto3";
-
-package feast;
-import "google/protobuf/timestamp.proto";
-
-option java_outer_classname = "PingProto";
-
-message Ping {
-  double lat = 1;
-  double lng = 2;
-  google.protobuf.Timestamp timestamp = 3;
-}
diff --git a/pom.xml b/pom.xml
index 98586740678..28011960cce 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,12 +22,13 @@
     Feature Store for Machine Learning
     ${github.url}
 
-    feast
+    dev.feast
     feast-parent
     ${revision}
     pom
 
     
+        datatypes/java
         ingestion
         core
         serving
@@ -35,7 +36,7 @@
     
 
     
-        0.3.2-SNAPSHOT
+        0.3.6-SNAPSHOT
         https://github.com/gojek/feast
 
         UTF-8
@@ -48,7 +49,6 @@
         2.16.0
         1.91.0
         0.8.0
-
         1.9.10
         1.3
         2.3.0
@@ -59,9 +59,18 @@
 
     
         Gojek
-        https://www.gojek.io/
+        https://www.gojek.com
     
 
+    
+        
+            Feast Authors
+            ${github.url}
+            Gojek
+            https://www.gojek.com
+        
+    
+
     
         
             Apache License, Version 2.0
@@ -82,15 +91,15 @@
         ${github.url}/issues
     
 
+    
     
-        
         
-            feast-snapshot
-            file:///tmp/snapshot
+            ossrh
+            https://oss.sonatype.org/content/repositories/snapshots
         
         
-            feast
-            file:///tmp/snapshot
+            ossrh
+            https://oss.sonatype.org/service/local/staging/deploy/maven2/
         
     
 
@@ -280,10 +289,36 @@
         
 
         
+            
+                org.apache.maven.plugins
+                maven-source-plugin
+                3.2.1
+                
+                    
+                        attach-sources
+                        
+                            jar-no-fork
+                        
+                    
+                
+            
+            
+                org.apache.maven.plugins
+                maven-javadoc-plugin
+                3.1.1
+                
+                    
+                        attach-javadocs
+                        
+                            jar
+                        
+                    
+                
+            
             
                 com.diffplug.spotless
                 spotless-maven-plugin
-                1.26.0
+                1.26.1
                 
                     
                         
@@ -398,6 +433,78 @@
                     true
                 
             
+            
+            
+                org.sonatype.plugins
+                nexus-staging-maven-plugin
+                1.6.8
+                true
+                
+                    ossrh
+                    https://oss.sonatype.org/
+                    
+                    true
+                
+            
+            
+            
+                org.codehaus.mojo
+                flatten-maven-plugin
+                1.1.0
+                
+                    oss
+                
+                
+                    
+                        flatten
+                        process-resources
+                        
+                            flatten
+                        
+                    
+                    
+                        flatten.clean
+                        clean
+                        
+                            clean
+                        
+                    
+                
+            
+            
+            
+                org.apache.maven.plugins
+                maven-gpg-plugin
+                1.6
+                
+                    
+                        sign-artifacts
+                        verify
+                        
+                            sign
+                        
+                        
+                        
+                            
+                                --pinentry-mode
+                                loopback
+                            
+                            
+                            
+                            ${gpg.passphrase}
+                        
+                    
+                
+            
         
 
         
@@ -436,25 +543,6 @@
                     org.xolstice.maven.plugins
                     protobuf-maven-plugin
                     0.6.1
-                    
-                        true
-                        
-                            com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
-                        
-                        grpc-java
-                        
-                            io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
-                        
-                    
-                    
-                        
-                            
-                                compile
-                                compile-custom
-                                test-compile
-                            
-                        
-                    
                 
             
         
diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml
index 2c8b1d837a3..e8a82a485fc 100644
--- a/sdk/java/pom.xml
+++ b/sdk/java/pom.xml
@@ -6,10 +6,10 @@
 
   Feast SDK for Java
   SDK for registering, storing, and retrieving features
-  feast-client
+  feast-sdk
 
   
-    feast
+    dev.feast
     feast-parent
     ${revision}
     ../..
@@ -21,6 +21,12 @@
   
 
   
+    
+      dev.feast
+      datatypes-java
+      ${project.version}
+    
+
     
     
       io.grpc
@@ -79,10 +85,6 @@
 
   
     
-      
-        org.xolstice.maven.plugins
-        protobuf-maven-plugin
-      
       
       
         org.apache.maven.plugins
diff --git a/serving/pom.xml b/serving/pom.xml
index 3d41c18b8dc..1e8d1b83a68 100644
--- a/serving/pom.xml
+++ b/serving/pom.xml
@@ -21,7 +21,7 @@
   4.0.0
 
   
-    feast
+    dev.feast
     feast-parent
     ${revision}
   
@@ -47,10 +47,6 @@
           false
         
       
-      
-        org.xolstice.maven.plugins
-        protobuf-maven-plugin
-      
       
         org.apache.maven.plugins
         maven-failsafe-plugin
@@ -74,6 +70,12 @@
   
 
   
+    
+      dev.feast
+      datatypes-java
+      ${project.version}
+    
+
     
     
       org.slf4j
diff --git a/serving/src/main/proto/feast b/serving/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/serving/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/serving/src/main/proto/third_party b/serving/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/serving/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file

From 784c7e0e5fc9ba44e60e32cdfaf2f49f6589c209 Mon Sep 17 00:00:00 2001
From: Ches Martin 
Date: Fri, 14 Feb 2020 02:33:36 +0700
Subject: [PATCH 11/17] Set version to 0.3.7-SNAPSHOT

v0.3.6 was tagged and 0.3.x development has moved forward. See #442
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 28011960cce..fb03c3045b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
     
 
     
-        0.3.6-SNAPSHOT
+        0.3.7-SNAPSHOT
         https://github.com/gojek/feast
 
         UTF-8

From be4751089439d46e8c644ebe14de88c43f86a156 Mon Sep 17 00:00:00 2001
From: David Heryanto 
Date: Mon, 9 Mar 2020 11:43:37 +0800
Subject: [PATCH 12/17] Relax OS check when starting e2e test (#523)

---
 .prow/scripts/test-end-to-end-batch.sh | 6 ------
 .prow/scripts/test-end-to-end.sh       | 6 ------
 2 files changed, 12 deletions(-)

diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh
index ba395fc6166..b7c1794691b 100755
--- a/.prow/scripts/test-end-to-end-batch.sh
+++ b/.prow/scripts/test-end-to-end-batch.sh
@@ -3,12 +3,6 @@
 set -e
 set -o pipefail
 
-if ! cat /etc/*release | grep -q stretch; then
-    echo ${BASH_SOURCE} only supports Debian stretch. 
-    echo Please change your operating system to use this script.
-    exit 1
-fi
-
 echo "
 This script will run end-to-end tests for Feast Core and Batch Serving.
 
diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh
index e8160444d39..951972f8e6b 100755
--- a/.prow/scripts/test-end-to-end.sh
+++ b/.prow/scripts/test-end-to-end.sh
@@ -3,12 +3,6 @@
 set -e
 set -o pipefail
 
-if ! cat /etc/*release | grep -q stretch; then
-    echo ${BASH_SOURCE} only supports Debian stretch. 
-    echo Please change your operating system to use this script.
-    exit 1
-fi
-
 echo "
 This script will run end-to-end tests for Feast Core and Online Serving.
 

From be4d1893674b984638c33141ba8fd875db5ac6b7 Mon Sep 17 00:00:00 2001
From: Ches Martin 
Date: Mon, 9 Mar 2020 13:51:37 +0700
Subject: [PATCH 13/17] v0.3 backport: Fail Spotless check before tests (#516)

* Fail formatting check before tests execute

By default, the spotless Maven plugin binds its check goal to the verify
phase (late in the lifecycle, after integration tests). Because we
currently only run `mvn test` for CI, it doesn't proceed as far as
verify so missed formatting is not caught by CI.

This binds the check to an earlier phase, in between test-compile and
test, so that it will fail before `mvn test` but not disrupt your dev
workflow of compiling main and test sources as you work. This strikes a
good compromise on failing fast for code standards without being _too_
nagging.

For the complete lifecycle reference, see:
https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html

* Apply Spotless formatting

Hopefully for last time as a bulk operation, after 636354092c.
---
 .../core/config/FeatureStreamConfig.java      |   1 -
 .../feast/core/dao/FeatureSetRepository.java  |   6 +-
 .../java/feast/core/grpc/CoreServiceImpl.java |   5 +-
 .../core/job/dataflow/DataflowJobManager.java |   1 -
 .../core/job/direct/DirectJobRegistry.java    |   1 -
 .../job/direct/DirectRunnerJobManager.java    |   4 +-
 .../main/java/feast/core/log/AuditLogger.java |   1 -
 .../java/feast/core/model/FeatureSet.java     |  25 +--
 .../core/service/JobCoordinatorService.java   |   1 -
 .../java/feast/core/service/SpecService.java  |  16 +-
 .../java/feast/core/util/PackageUtil.java     |   6 +-
 .../java/feast/core/util/TypeConversion.java  |   1 -
 .../feast/core/service/SpecServiceTest.java   |  30 ++-
 .../feast/core/validators/MatchersTest.java   |   1 -
 .../redis/FeatureRowToRedisMutationDoFn.java  |  11 +-
 .../java/feast/ingestion/ImportJobTest.java   |  87 ++++----
 .../FeatureRowToRedisMutationDoFnTest.java    | 211 +++++++++++-------
 pom.xml                                       |  10 +
 .../bigquery/BatchRetrievalQueryRunnable.java |   3 +-
 19 files changed, 231 insertions(+), 190 deletions(-)

diff --git a/core/src/main/java/feast/core/config/FeatureStreamConfig.java b/core/src/main/java/feast/core/config/FeatureStreamConfig.java
index 6d9a30f9e93..4f444b59e83 100644
--- a/core/src/main/java/feast/core/config/FeatureStreamConfig.java
+++ b/core/src/main/java/feast/core/config/FeatureStreamConfig.java
@@ -16,7 +16,6 @@
  */
 package feast.core.config;
 
-import com.google.common.base.Strings;
 import feast.core.SourceProto.KafkaSourceConfig;
 import feast.core.SourceProto.SourceType;
 import feast.core.config.FeastProperties.StreamProperties;
diff --git a/core/src/main/java/feast/core/dao/FeatureSetRepository.java b/core/src/main/java/feast/core/dao/FeatureSetRepository.java
index ca4d6b9d1cb..fd996b331c2 100644
--- a/core/src/main/java/feast/core/dao/FeatureSetRepository.java
+++ b/core/src/main/java/feast/core/dao/FeatureSetRepository.java
@@ -36,11 +36,11 @@ public interface FeatureSetRepository extends JpaRepository
   List findByName(String name);
 
   // find all versions of featureSets with names matching the regex
-  @Query(nativeQuery = true, value = "SELECT * FROM feature_sets "
-      + "WHERE name LIKE ?1 ORDER BY name ASC, version ASC")
+  @Query(
+      nativeQuery = true,
+      value = "SELECT * FROM feature_sets " + "WHERE name LIKE ?1 ORDER BY name ASC, version ASC")
   List findByNameWithWildcardOrderByNameAscVersionAsc(String name);
 
   // find all feature sets and order by name and version
   List findAllByOrderByNameAscVersionAsc();
-
 }
diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
index 6387fd806b4..5398a82e782 100644
--- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
+++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
@@ -49,11 +49,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.lognet.springboot.grpc.GRpcService;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.transaction.annotation.Transactional;
 
-/**
- * Implementation of the feast core GRPC service.
- */
+/** Implementation of the feast core GRPC service. */
 @Slf4j
 @GRpcService(interceptors = {MonitoringInterceptor.class})
 public class CoreServiceImpl extends CoreServiceImplBase {
diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
index f19cf1a6569..fefb1145f53 100644
--- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
+++ b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
@@ -20,7 +20,6 @@
 
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.Job;
-import com.google.common.base.Strings;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.util.JsonFormat;
 import com.google.protobuf.util.JsonFormat.Printer;
diff --git a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java b/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java
index 8f6c87053ff..94b3a8fd571 100644
--- a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java
+++ b/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java
@@ -16,7 +16,6 @@
  */
 package feast.core.job.direct;
 
-import com.google.common.base.Strings;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
index 85b8a95dd56..cf4c6213eed 100644
--- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
+++ b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
@@ -16,7 +16,6 @@
  */
 package feast.core.job.direct;
 
-import com.google.common.base.Strings;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.util.JsonFormat;
 import com.google.protobuf.util.JsonFormat.Printer;
@@ -148,8 +147,7 @@ public void abortJob(String extId) {
     try {
       job.abort();
     } catch (IOException e) {
-      throw new RuntimeException(
-          String.format("Unable to abort DirectRunner job %s", extId), e);
+      throw new RuntimeException(String.format("Unable to abort DirectRunner job %s", extId), e);
     }
     jobs.remove(extId);
   }
diff --git a/core/src/main/java/feast/core/log/AuditLogger.java b/core/src/main/java/feast/core/log/AuditLogger.java
index 2c60307805c..275aa74edfa 100644
--- a/core/src/main/java/feast/core/log/AuditLogger.java
+++ b/core/src/main/java/feast/core/log/AuditLogger.java
@@ -16,7 +16,6 @@
  */
 package feast.core.log;
 
-import com.google.common.base.Strings;
 import java.util.Date;
 import java.util.Map;
 import java.util.TreeMap;
diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java
index 8ba7162d2f2..755ef687e32 100644
--- a/core/src/main/java/feast/core/model/FeatureSet.java
+++ b/core/src/main/java/feast/core/model/FeatureSet.java
@@ -155,50 +155,49 @@ public FeatureSetSpec toProto() throws InvalidProtocolBufferException {
    * @return boolean denoting if the source or schema have changed.
    */
   public boolean equalTo(FeatureSet other) {
-    if(!name.equals(other.getName())){
+    if (!name.equals(other.getName())) {
       return false;
     }
 
-    if (!source.equalTo(other.getSource())){
+    if (!source.equalTo(other.getSource())) {
       return false;
     }
 
-    if (maxAgeSeconds != other.maxAgeSeconds){
+    if (maxAgeSeconds != other.maxAgeSeconds) {
       return false;
     }
 
     // Create a map of all fields in this feature set
     Map fields = new HashMap<>();
 
-    for (Field e : entities){
+    for (Field e : entities) {
       fields.putIfAbsent(e.getName(), e);
     }
 
-    for (Field f : features){
+    for (Field f : features) {
       fields.putIfAbsent(f.getName(), f);
     }
 
     // Ensure map size is consistent with existing fields
-    if (fields.size() != other.features.size() + other.entities.size())
-    {
+    if (fields.size() != other.features.size() + other.entities.size()) {
       return false;
     }
 
     // Ensure the other entities and fields exist in the field map
-    for (Field e : other.entities){
-      if(!fields.containsKey(e.getName())){
+    for (Field e : other.entities) {
+      if (!fields.containsKey(e.getName())) {
         return false;
       }
-      if (!e.equals(fields.get(e.getName()))){
+      if (!e.equals(fields.get(e.getName()))) {
         return false;
       }
     }
 
-    for (Field f : features){
-      if(!fields.containsKey(f.getName())){
+    for (Field f : features) {
+      if (!fields.containsKey(f.getName())) {
         return false;
       }
-      if (!f.equals(fields.get(f.getName()))){
+      if (!f.equals(fields.get(f.getName()))) {
         return false;
       }
     }
diff --git a/core/src/main/java/feast/core/service/JobCoordinatorService.java b/core/src/main/java/feast/core/service/JobCoordinatorService.java
index b5c9fc6c1bb..7f0d4cb79ca 100644
--- a/core/src/main/java/feast/core/service/JobCoordinatorService.java
+++ b/core/src/main/java/feast/core/service/JobCoordinatorService.java
@@ -16,7 +16,6 @@
  */
 package feast.core.service;
 
-import com.google.common.base.Strings;
 import feast.core.FeatureSetProto.FeatureSetSpec;
 import feast.core.SourceProto;
 import feast.core.StoreProto;
diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java
index 99862fd5dc0..ffa1aa08e81 100644
--- a/core/src/main/java/feast/core/service/SpecService.java
+++ b/core/src/main/java/feast/core/service/SpecService.java
@@ -110,8 +110,9 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
 
       if (featureSet == null) {
         throw io.grpc.Status.NOT_FOUND
-            .withDescription(String.format("Feature set with name \"%s\" could not be found.",
-                request.getName()))
+            .withDescription(
+                String.format(
+                    "Feature set with name \"%s\" could not be found.", request.getName()))
             .asRuntimeException();
       }
     } else {
@@ -121,13 +122,14 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
 
       if (featureSet == null) {
         throw io.grpc.Status.NOT_FOUND
-            .withDescription(String.format("Feature set with name \"%s\" and version \"%s\" could "
-                + "not be found.", request.getName(), request.getVersion()))
+            .withDescription(
+                String.format(
+                    "Feature set with name \"%s\" and version \"%s\" could " + "not be found.",
+                    request.getName(), request.getVersion()))
             .asRuntimeException();
       }
     }
 
-
     // Only a single item in list, return successfully
     return GetFeatureSetResponse.newBuilder().setFeatureSet(featureSet.toProto()).build();
   }
@@ -154,7 +156,9 @@ public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter fil
     if (name.equals("")) {
       featureSets = featureSetRepository.findAllByOrderByNameAscVersionAsc();
     } else {
-      featureSets = featureSetRepository.findByNameWithWildcardOrderByNameAscVersionAsc(name.replace('*', '%'));
+      featureSets =
+          featureSetRepository.findByNameWithWildcardOrderByNameAscVersionAsc(
+              name.replace('*', '%'));
       featureSets =
           featureSets.stream()
               .filter(getVersionFilter(filter.getFeatureSetVersion()))
diff --git a/core/src/main/java/feast/core/util/PackageUtil.java b/core/src/main/java/feast/core/util/PackageUtil.java
index 20b2310644b..99c5d73ba78 100644
--- a/core/src/main/java/feast/core/util/PackageUtil.java
+++ b/core/src/main/java/feast/core/util/PackageUtil.java
@@ -44,9 +44,9 @@ public class PackageUtil {
    * points to the resource location. Note that the extraction process can take several minutes to
    * complete.
    *
-   * 
One use case of this function is to detect the class path of resources to stage when
-   * using Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be
-   * handled by default in Apache Beam.
+   * 
One use case of this function is to detect the class path of resources to stage when using
+   * Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be handled by
+   * default in Apache Beam.
    *
    * 
    * 
diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java
index a7dd2b0d2a3..5fe69819476 100644
--- a/core/src/main/java/feast/core/util/TypeConversion.java
+++ b/core/src/main/java/feast/core/util/TypeConversion.java
@@ -16,7 +16,6 @@
  */
 package feast.core.util;
 
-import com.google.common.base.Strings;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import java.lang.reflect.Type;
diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java
index a11adf022b9..6d112676350 100644
--- a/core/src/test/java/feast/core/service/SpecServiceTest.java
+++ b/core/src/test/java/feast/core/service/SpecServiceTest.java
@@ -68,14 +68,11 @@
 
 public class SpecServiceTest {
 
-  @Mock
-  private FeatureSetRepository featureSetRepository;
+  @Mock private FeatureSetRepository featureSetRepository;
 
-  @Mock
-  private StoreRepository storeRepository;
+  @Mock private StoreRepository storeRepository;
 
-  @Rule
-  public final ExpectedException expectedException = ExpectedException.none();
+  @Rule public final ExpectedException expectedException = ExpectedException.none();
 
   private SpecService specService;
   private List featureSets;
@@ -102,11 +99,12 @@ public void setUp() {
     Field f3f1 = new Field("f3", "f3f1", Enum.INT64);
     Field f3f2 = new Field("f3", "f3f2", Enum.INT64);
     Field f3e1 = new Field("f3", "f3e1", Enum.STRING);
-    FeatureSet featureSet3v1 = new FeatureSet(
-        "f3", 1, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource);
+    FeatureSet featureSet3v1 =
+        new FeatureSet(
+            "f3", 1, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource);
 
-    featureSets = Arrays
-        .asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1);
+    featureSets =
+        Arrays.asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1);
     when(featureSetRepository.findAll()).thenReturn(featureSets);
     when(featureSetRepository.findAllByOrderByNameAscVersionAsc()).thenReturn(featureSets);
     when(featureSetRepository.findByName("f1")).thenReturn(featureSets.subList(0, 3));
@@ -347,7 +345,6 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
     assertThat(applyFeatureSetResponse.getFeatureSet(), equalTo(expected));
   }
 
-
   @Test
   public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
       throws InvalidProtocolBufferException {
@@ -355,20 +352,21 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
     Field f3f1 = new Field("f3", "f3f1", Enum.INT64);
     Field f3f2 = new Field("f3", "f3f2", Enum.INT64);
     Field f3e1 = new Field("f3", "f3e1", Enum.STRING);
-    FeatureSetProto.FeatureSetSpec incomingFeatureSet = (new FeatureSet(
-        "f3", 5, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource)).toProto();
+    FeatureSetProto.FeatureSetSpec incomingFeatureSet =
+        (new FeatureSet(
+                "f3", 5, 100L, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1), defaultSource))
+            .toProto();
 
     FeatureSetSpec expected = incomingFeatureSet;
     ApplyFeatureSetResponse applyFeatureSetResponse =
         specService.applyFeatureSet(incomingFeatureSet);
     assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.NO_CHANGE));
     assertThat(applyFeatureSetResponse.getFeatureSet().getMaxAge(), equalTo(expected.getMaxAge()));
-    assertThat(applyFeatureSetResponse.getFeatureSet().getEntities(0),
-        equalTo(expected.getEntities(0)));
+    assertThat(
+        applyFeatureSetResponse.getFeatureSet().getEntities(0), equalTo(expected.getEntities(0)));
     assertThat(applyFeatureSetResponse.getFeatureSet().getName(), equalTo(expected.getName()));
   }
 
-
   @Test
   public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException {
     when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0)));
diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java
index 13c9e006a44..3bf09dd474f 100644
--- a/core/src/test/java/feast/core/validators/MatchersTest.java
+++ b/core/src/test/java/feast/core/validators/MatchersTest.java
@@ -19,7 +19,6 @@
 import static feast.core.validators.Matchers.checkLowerSnakeCase;
 import static feast.core.validators.Matchers.checkUpperSnakeCase;
 
-import com.google.common.base.Strings;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java
index c453c5c9206..a7471443617 100644
--- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java
+++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java
@@ -24,11 +24,9 @@
 import feast.store.serving.redis.RedisCustomIO.RedisMutation;
 import feast.types.FeatureRowProto.FeatureRow;
 import feast.types.FieldProto.Field;
-import feast.types.ValueProto.Value;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.slf4j.Logger;
@@ -55,12 +53,9 @@ private RedisKey getKey(FeatureRow featureRow) {
     Builder redisKeyBuilder = RedisKey.newBuilder().setFeatureSet(featureRow.getFeatureSet());
     for (Field field : featureRow.getFieldsList()) {
       if (entityNames.contains(field.getName())) {
-        entityFields.putIfAbsent(field.getName(),
-            Field.newBuilder()
-                .setName(field.getName())
-                .setValue(field.getValue())
-                .build()
-        );
+        entityFields.putIfAbsent(
+            field.getName(),
+            Field.newBuilder().setName(field.getName()).setValue(field.getValue()).build());
       }
     }
     for (String entityName : entityNames) {
diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
index bd034341ec9..4a09bee82ff 100644
--- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
+++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
@@ -170,12 +170,14 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow()
     Map expected = new HashMap<>();
 
     LOGGER.info("Generating test data ...");
-    IntStream.range(0, IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE).forEach(i -> {
-      FeatureRow randomRow = TestUtil.createRandomFeatureRow(spec);
-      RedisKey redisKey = TestUtil.createRedisKey(spec, randomRow);
-      input.add(randomRow);
-      expected.put(redisKey, randomRow);
-    });
+    IntStream.range(0, IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE)
+        .forEach(
+            i -> {
+              FeatureRow randomRow = TestUtil.createRandomFeatureRow(spec);
+              RedisKey redisKey = TestUtil.createRedisKey(spec, randomRow);
+              input.add(randomRow);
+              expected.put(redisKey, randomRow);
+            });
 
     LOGGER.info("Starting Import Job with the following options: {}", options.toString());
     PipelineResult pipelineResult = ImportJob.runPipeline(options);
@@ -183,43 +185,50 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow()
     Assert.assertEquals(pipelineResult.getState(), State.RUNNING);
 
     LOGGER.info("Publishing {} Feature Row messages to Kafka ...", input.size());
-    TestUtil.publishFeatureRowsToKafka(KAFKA_BOOTSTRAP_SERVERS, KAFKA_TOPIC, input,
-        ByteArraySerializer.class, KAFKA_PUBLISH_TIMEOUT_SEC);
-    TestUtil.waitUntilAllElementsAreWrittenToStore(pipelineResult,
+    TestUtil.publishFeatureRowsToKafka(
+        KAFKA_BOOTSTRAP_SERVERS,
+        KAFKA_TOPIC,
+        input,
+        ByteArraySerializer.class,
+        KAFKA_PUBLISH_TIMEOUT_SEC);
+    TestUtil.waitUntilAllElementsAreWrittenToStore(
+        pipelineResult,
         Duration.standardSeconds(IMPORT_JOB_MAX_RUN_DURATION_SEC),
         Duration.standardSeconds(IMPORT_JOB_CHECK_INTERVAL_DURATION_SEC));
 
     LOGGER.info("Validating the actual values written to Redis ...");
     Jedis jedis = new Jedis(REDIS_HOST, REDIS_PORT);
-    expected.forEach((key, expectedValue) -> {
-
-      // Ensure ingested key exists.
-      byte[] actualByteValue = jedis.get(key.toByteArray());
-      if (actualByteValue == null) {
-        LOGGER.error("Key not found in Redis: " + key);
-        LOGGER.info("Redis INFO:");
-        LOGGER.info(jedis.info());
-        String randomKey = jedis.randomKey();
-        if (randomKey != null) {
-          LOGGER.info("Sample random key, value (for debugging purpose):");
-          LOGGER.info("Key: " + randomKey);
-          LOGGER.info("Value: " + jedis.get(randomKey));
-        }
-        Assert.fail("Missing key in Redis.");
-      }
-
-      // Ensure value is a valid serialized FeatureRow object.
-      FeatureRow actualValue = null;
-      try {
-        actualValue = FeatureRow.parseFrom(actualByteValue);
-      } catch (InvalidProtocolBufferException e) {
-        Assert.fail(String
-            .format("Actual Redis value cannot be parsed as FeatureRow, key: %s, value :%s",
-                key, new String(actualByteValue, StandardCharsets.UTF_8)));
-      }
-
-      // Ensure the retrieved FeatureRow is equal to the ingested FeatureRow.
-      Assert.assertEquals(expectedValue, actualValue);
-    });
+    expected.forEach(
+        (key, expectedValue) -> {
+
+          // Ensure ingested key exists.
+          byte[] actualByteValue = jedis.get(key.toByteArray());
+          if (actualByteValue == null) {
+            LOGGER.error("Key not found in Redis: " + key);
+            LOGGER.info("Redis INFO:");
+            LOGGER.info(jedis.info());
+            String randomKey = jedis.randomKey();
+            if (randomKey != null) {
+              LOGGER.info("Sample random key, value (for debugging purpose):");
+              LOGGER.info("Key: " + randomKey);
+              LOGGER.info("Value: " + jedis.get(randomKey));
+            }
+            Assert.fail("Missing key in Redis.");
+          }
+
+          // Ensure value is a valid serialized FeatureRow object.
+          FeatureRow actualValue = null;
+          try {
+            actualValue = FeatureRow.parseFrom(actualByteValue);
+          } catch (InvalidProtocolBufferException e) {
+            Assert.fail(
+                String.format(
+                    "Actual Redis value cannot be parsed as FeatureRow, key: %s, value :%s",
+                    key, new String(actualByteValue, StandardCharsets.UTF_8)));
+          }
+
+          // Ensure the retrieved FeatureRow is equal to the ingested FeatureRow.
+          Assert.assertEquals(expectedValue, actualValue);
+        });
   }
 }
diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java
index 6e0db2dd49c..f74f7667850 100644
--- a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java
+++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java
@@ -1,3 +1,19 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package feast.store.serving.redis;
 
 import static org.junit.Assert.*;
@@ -6,11 +22,8 @@
 import feast.core.FeatureSetProto.EntitySpec;
 import feast.core.FeatureSetProto.FeatureSetSpec;
 import feast.core.FeatureSetProto.FeatureSpec;
-import feast.ingestion.transform.ValidateFeatureRows;
 import feast.storage.RedisProto.RedisKey;
-import feast.store.serving.redis.RedisCustomIO.Method;
 import feast.store.serving.redis.RedisCustomIO.RedisMutation;
-import feast.test.TestUtil;
 import feast.types.FeatureRowProto.FeatureRow;
 import feast.types.FieldProto.Field;
 import feast.types.ValueProto.Value;
@@ -31,64 +44,77 @@
 
 public class FeatureRowToRedisMutationDoFnTest {
 
-  @Rule
-  public transient TestPipeline p = TestPipeline.create();
-
-  private FeatureSetSpec fs = FeatureSetSpec.newBuilder()
-      .setName("feature_set")
-      .setVersion(1)
-      .addEntities(
-          EntitySpec.newBuilder()
-              .setName("entity_id_primary")
-              .setValueType(Enum.INT32)
-              .build())
-      .addEntities(
-          EntitySpec.newBuilder()
-              .setName("entity_id_secondary")
-              .setValueType(Enum.STRING)
-              .build())
-      .addFeatures(
-          FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build())
-      .addFeatures(
-          FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build())
-      .build();
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private FeatureSetSpec fs =
+      FeatureSetSpec.newBuilder()
+          .setName("feature_set")
+          .setVersion(1)
+          .addEntities(
+              EntitySpec.newBuilder().setName("entity_id_primary").setValueType(Enum.INT32).build())
+          .addEntities(
+              EntitySpec.newBuilder()
+                  .setName("entity_id_secondary")
+                  .setValueType(Enum.STRING)
+                  .build())
+          .addFeatures(
+              FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build())
+          .addFeatures(
+              FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build())
+          .build();
 
   @Test
   public void shouldConvertRowWithDuplicateEntitiesToValidKey() {
     Map featureSetSpecs = new HashMap<>();
     featureSetSpecs.put("feature_set", fs);
 
-    FeatureRow offendingRow = FeatureRow.newBuilder()
-        .setFeatureSet("feature_set")
-        .setEventTimestamp(Timestamp.newBuilder().setSeconds(10))
-        .addFields(Field.newBuilder().setName("entity_id_primary")
-            .setValue(Value.newBuilder().setInt32Val(1)))
-        .addFields(Field.newBuilder().setName("entity_id_primary")
-            .setValue(Value.newBuilder().setInt32Val(2)))
-        .addFields(Field.newBuilder().setName("entity_id_secondary")
-            .setValue(Value.newBuilder().setStringVal("a")))
-        .build();
-
-    PCollection output = p
-        .apply(Create.of(Collections.singletonList(offendingRow)))
-        .setCoder(ProtoCoder.of(FeatureRow.class))
-        .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs)));
-
-    RedisKey expectedKey = RedisKey.newBuilder()
-        .setFeatureSet("feature_set")
-        .addEntities(Field.newBuilder().setName("entity_id_primary")
-            .setValue(Value.newBuilder().setInt32Val(1)))
-        .addEntities(Field.newBuilder().setName("entity_id_secondary")
-            .setValue(Value.newBuilder().setStringVal("a")))
-        .build();
-
-    PAssert.that(output).satisfies((SerializableFunction, Void>) input -> {
-      input.forEach(rm -> {
-        assert(Arrays.equals(rm.getKey(), expectedKey.toByteArray()));
-        assert(Arrays.equals(rm.getValue(), offendingRow.toByteArray()));
-      });
-      return null;
-    });
+    FeatureRow offendingRow =
+        FeatureRow.newBuilder()
+            .setFeatureSet("feature_set")
+            .setEventTimestamp(Timestamp.newBuilder().setSeconds(10))
+            .addFields(
+                Field.newBuilder()
+                    .setName("entity_id_primary")
+                    .setValue(Value.newBuilder().setInt32Val(1)))
+            .addFields(
+                Field.newBuilder()
+                    .setName("entity_id_primary")
+                    .setValue(Value.newBuilder().setInt32Val(2)))
+            .addFields(
+                Field.newBuilder()
+                    .setName("entity_id_secondary")
+                    .setValue(Value.newBuilder().setStringVal("a")))
+            .build();
+
+    PCollection output =
+        p.apply(Create.of(Collections.singletonList(offendingRow)))
+            .setCoder(ProtoCoder.of(FeatureRow.class))
+            .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs)));
+
+    RedisKey expectedKey =
+        RedisKey.newBuilder()
+            .setFeatureSet("feature_set")
+            .addEntities(
+                Field.newBuilder()
+                    .setName("entity_id_primary")
+                    .setValue(Value.newBuilder().setInt32Val(1)))
+            .addEntities(
+                Field.newBuilder()
+                    .setName("entity_id_secondary")
+                    .setValue(Value.newBuilder().setStringVal("a")))
+            .build();
+
+    PAssert.that(output)
+        .satisfies(
+            (SerializableFunction, Void>)
+                input -> {
+                  input.forEach(
+                      rm -> {
+                        assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray()));
+                        assert (Arrays.equals(rm.getValue(), offendingRow.toByteArray()));
+                      });
+                  return null;
+                });
     p.run();
   }
 
@@ -97,36 +123,49 @@ public void shouldConvertRowWithOutOfOrderEntitiesToValidKey() {
     Map featureSetSpecs = new HashMap<>();
     featureSetSpecs.put("feature_set", fs);
 
-    FeatureRow offendingRow = FeatureRow.newBuilder()
-        .setFeatureSet("feature_set")
-        .setEventTimestamp(Timestamp.newBuilder().setSeconds(10))
-        .addFields(Field.newBuilder().setName("entity_id_secondary")
-            .setValue(Value.newBuilder().setStringVal("a")))
-        .addFields(Field.newBuilder().setName("entity_id_primary")
-            .setValue(Value.newBuilder().setInt32Val(1)))
-        .build();
-
-    PCollection output = p
-        .apply(Create.of(Collections.singletonList(offendingRow)))
-        .setCoder(ProtoCoder.of(FeatureRow.class))
-        .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs)));
-
-    RedisKey expectedKey = RedisKey.newBuilder()
-        .setFeatureSet("feature_set")
-        .addEntities(Field.newBuilder().setName("entity_id_primary")
-            .setValue(Value.newBuilder().setInt32Val(1)))
-        .addEntities(Field.newBuilder().setName("entity_id_secondary")
-            .setValue(Value.newBuilder().setStringVal("a")))
-        .build();
-
-    PAssert.that(output).satisfies((SerializableFunction, Void>) input -> {
-      input.forEach(rm -> {
-        assert(Arrays.equals(rm.getKey(), expectedKey.toByteArray()));
-        assert(Arrays.equals(rm.getValue(), offendingRow.toByteArray()));
-      });
-      return null;
-    });
+    FeatureRow offendingRow =
+        FeatureRow.newBuilder()
+            .setFeatureSet("feature_set")
+            .setEventTimestamp(Timestamp.newBuilder().setSeconds(10))
+            .addFields(
+                Field.newBuilder()
+                    .setName("entity_id_secondary")
+                    .setValue(Value.newBuilder().setStringVal("a")))
+            .addFields(
+                Field.newBuilder()
+                    .setName("entity_id_primary")
+                    .setValue(Value.newBuilder().setInt32Val(1)))
+            .build();
+
+    PCollection output =
+        p.apply(Create.of(Collections.singletonList(offendingRow)))
+            .setCoder(ProtoCoder.of(FeatureRow.class))
+            .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSetSpecs)));
+
+    RedisKey expectedKey =
+        RedisKey.newBuilder()
+            .setFeatureSet("feature_set")
+            .addEntities(
+                Field.newBuilder()
+                    .setName("entity_id_primary")
+                    .setValue(Value.newBuilder().setInt32Val(1)))
+            .addEntities(
+                Field.newBuilder()
+                    .setName("entity_id_secondary")
+                    .setValue(Value.newBuilder().setStringVal("a")))
+            .build();
+
+    PAssert.that(output)
+        .satisfies(
+            (SerializableFunction, Void>)
+                input -> {
+                  input.forEach(
+                      rm -> {
+                        assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray()));
+                        assert (Arrays.equals(rm.getValue(), offendingRow.toByteArray()));
+                      });
+                  return null;
+                });
     p.run();
   }
-
-}
\ No newline at end of file
+}
diff --git a/pom.xml b/pom.xml
index fb03c3045b7..0e5ce843e2f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -350,6 +350,16 @@
                         
                     
                 
+                
+                  
+                  
+                      spotless-check
+                      process-test-classes
+                      
+                          check
+                      
+                  
+              
             
             
                 org.apache.maven.plugins
diff --git a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java
index 47587e1d0ef..43ac75d0b21 100644
--- a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java
+++ b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java
@@ -231,8 +231,7 @@ Job runBatchQuery(List featureSetQueries)
         bigquery()
             .getTable(queryJobConfig.getDestinationTable())
             .toBuilder()
-            .setExpirationTime(
-                System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
+            .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS)
             .build();
     bigquery().update(expiry);
 

From 4f91b224ccf76105e61e3d47434846ca4b75c95d Mon Sep 17 00:00:00 2001
From: Ches Martin 
Date: Mon, 9 Mar 2020 23:22:35 +0700
Subject: [PATCH 14/17] v0.3 backport: Remove unused ingestion deps (#521)

* Make dependency:analyze run clean on datatypes-java

* Remove stale dependencies from ingestion

Unused according to `mvn -pl ingestion dependency:analyze`, and tests.

We had a recent bump of hibernate-validator with a CVE fix (#421) that I
was looking to backport, and it turns out it's not used anymore anyway.
---
 datatypes/java/pom.xml | 38 +++++++++++++++++++++++++++++++++
 ingestion/pom.xml      | 48 ------------------------------------------
 pom.xml                | 19 +++++++++++++++++
 3 files changed, 57 insertions(+), 48 deletions(-)

diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
index a6dfa8e345a..415d204293a 100644
--- a/datatypes/java/pom.xml
+++ b/datatypes/java/pom.xml
@@ -37,6 +37,17 @@
 
     
       
+        
+          org.apache.maven.plugins
+          maven-dependency-plugin
+          
+            
+            
+              javax.annotation
+            
+          
+        
+
         
           org.xolstice.maven.plugins
           protobuf-maven-plugin
@@ -64,9 +75,36 @@
     
 
     
+      
+      
+        com.google.guava
+        guava
+      
+      
+        com.google.protobuf
+        protobuf-java
+      
+
+      
+        io.grpc
+        grpc-core
+      
+      
+        io.grpc
+        grpc-protobuf
+      
       
         io.grpc
         grpc-services
       
+      
+        io.grpc
+        grpc-stub
+      
+
+      
+          javax.annotation
+          javax.annotation-api
+      
     
 
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index 35ca4eb6986..b6f1ea371a4 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -92,24 +92,6 @@
       ${project.version}
     
 
-    
-      org.glassfish
-      javax.el
-      3.0.0
-    
-
-    
-      javax.validation
-      validation-api
-      2.0.1.Final
-    
-
-    
-      org.hibernate.validator
-      hibernate-validator
-      6.0.13.Final
-    
-
     
       com.google.auto.value
       auto-value-annotations
@@ -122,15 +104,6 @@
       provided
     
 
-    
-      io.grpc
-      grpc-stub
-    
-
-    
-      com.google.cloud
-      google-cloud-storage
-    
     
       com.google.cloud
       google-cloud-bigquery
@@ -150,27 +123,6 @@
       mockito-core
     
 
-    
-      com.fasterxml.jackson.core
-      jackson-annotations
-    
-    
-      com.fasterxml.jackson.core
-      jackson-core
-    
-    
-      com.fasterxml.jackson.core
-      jackson-databind
-    
-    
-      com.fasterxml.jackson.dataformat
-      jackson-dataformat-yaml
-    
-    
-      com.fasterxml.jackson.module
-      jackson-module-jsonSchema
-    
-
     
       com.google.protobuf
       protobuf-java
diff --git a/pom.xml b/pom.xml
index 0e5ce843e2f..affdab061ac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -141,6 +141,11 @@
             
 
             
+            
+                io.grpc
+                grpc-core
+                ${grpcVersion}
+            
             
                 io.grpc
                 grpc-netty
@@ -524,6 +529,20 @@
                     docker-maven-plugin
                     0.20.1
                 
+                
+                    org.apache.maven.plugins
+                    maven-dependency-plugin
+                    3.1.1
+                    
+                        
+                        
+                            org.apache.maven.shared
+                            maven-dependency-analyzer
+                            1.11.1
+                        
+                    
+                
                 
                     org.apache.maven.plugins
                     maven-javadoc-plugin

From 1a35764b48c49eb5c0ada0745020e5b1aea88e41 Mon Sep 17 00:00:00 2001
From: Khor Shu Heng 
Date: Mon, 27 Apr 2020 11:06:14 +0800
Subject: [PATCH 15/17] Move e2e test scripts from .prow to infra

---
 .prow/config.yaml                             | 30 +++++++++----------
 .../scripts/download-maven-cache.sh           |  0
 .../scripts/install-google-cloud-sdk.sh       |  0
 .../scripts/publish-docker-image.sh           |  0
 {.prow => infra}/scripts/publish-java-sdk.sh  |  0
 .../scripts/publish-python-sdk.sh             |  0
 {.prow => infra}/scripts/sync-helm-charts.sh  |  0
 .../scripts/test-core-ingestion.sh            |  2 +-
 .../scripts/test-end-to-end-batch.sh          |  2 +-
 {.prow => infra}/scripts/test-end-to-end.sh   |  2 +-
 {.prow => infra}/scripts/test-golang-sdk.sh   |  0
 {.prow => infra}/scripts/test-java-sdk.sh     |  0
 {.prow => infra}/scripts/test-python-sdk.sh   |  0
 {.prow => infra}/scripts/test-serving.sh      |  2 +-
 14 files changed, 19 insertions(+), 19 deletions(-)
 rename {.prow => infra}/scripts/download-maven-cache.sh (100%)
 rename {.prow => infra}/scripts/install-google-cloud-sdk.sh (100%)
 rename {.prow => infra}/scripts/publish-docker-image.sh (100%)
 rename {.prow => infra}/scripts/publish-java-sdk.sh (100%)
 rename {.prow => infra}/scripts/publish-python-sdk.sh (100%)
 rename {.prow => infra}/scripts/sync-helm-charts.sh (100%)
 rename {.prow => infra}/scripts/test-core-ingestion.sh (95%)
 rename {.prow => infra}/scripts/test-end-to-end-batch.sh (99%)
 rename {.prow => infra}/scripts/test-end-to-end.sh (99%)
 rename {.prow => infra}/scripts/test-golang-sdk.sh (100%)
 rename {.prow => infra}/scripts/test-java-sdk.sh (100%)
 rename {.prow => infra}/scripts/test-python-sdk.sh (100%)
 rename {.prow => infra}/scripts/test-serving.sh (90%)

diff --git a/.prow/config.yaml b/.prow/config.yaml
index 39b81d76fd6..b7eec63924c 100644
--- a/.prow/config.yaml
+++ b/.prow/config.yaml
@@ -67,7 +67,7 @@ presubmits:
     spec:
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/test-core-ingestion.sh"]
+        command: ["infra/scripts/test-core-ingestion.sh"]
         resources:
           requests:
             cpu: "1000m"
@@ -81,7 +81,7 @@ presubmits:
     spec:
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/test-serving.sh"]
+        command: ["infra/scripts/test-serving.sh"]
 
   - name: test-java-sdk
     decorate: true
@@ -89,7 +89,7 @@ presubmits:
     spec:
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/test-java-sdk.sh"]
+        command: ["infra/scripts/test-java-sdk.sh"]
 
   - name: test-python-sdk
     decorate: true
@@ -97,7 +97,7 @@ presubmits:
     spec:
       containers:
       - image: python:3.7
-        command: [".prow/scripts/test-python-sdk.sh"]
+        command: ["infra/scripts/test-python-sdk.sh"]
 
   - name: test-golang-sdk
     decorate: true
@@ -105,7 +105,7 @@ presubmits:
     spec:
       containers:
       - image: golang:1.13
-        command: [".prow/scripts/test-golang-sdk.sh"]
+        command: ["infra/scripts/test-golang-sdk.sh"]
 
   - name: test-end-to-end
     decorate: true
@@ -113,7 +113,7 @@ presubmits:
     spec:
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/test-end-to-end.sh"]
+        command: ["infra/scripts/test-end-to-end.sh"]
         resources:
           requests:
             cpu: "1000m"
@@ -131,7 +131,7 @@ presubmits:
           secretName: feast-service-account
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/test-end-to-end-batch.sh"]
+        command: ["infra/scripts/test-end-to-end-batch.sh"]
         resources:
           requests:
             cpu: "1000m"
@@ -153,7 +153,7 @@ postsubmits:
         - sh
         - -c
         - |
-          .prow/scripts/publish-python-sdk.sh \
+          infra/scripts/publish-python-sdk.sh \
             --directory-path sdk/python --repository pypi
         volumeMounts:
         - name: pypirc
@@ -177,7 +177,7 @@ postsubmits:
         command:
         - bash
         - -c
-        - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
+        - infra/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
         volumeMounts:
         - name: gpg-keys
           mountPath: /etc/gpg
@@ -206,19 +206,19 @@ postsubmits:
         - bash
         - -c
         - |
-          .prow/scripts/download-maven-cache.sh \
+          infra/scripts/download-maven-cache.sh \
             --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
             --output-dir $PWD/
 
           if [ $PULL_BASE_REF == "master" ]; then
 
-          .prow/scripts/publish-docker-image.sh \
+          infra/scripts/publish-docker-image.sh \
             --repository gcr.io/kf-feast/feast-core \
             --tag dev \
             --file infra/docker/core/Dockerfile \
             --google-service-account-file /etc/gcloud/service-account.json
 
-          .prow/scripts/publish-docker-image.sh \
+          infra/scripts/publish-docker-image.sh \
             --repository gcr.io/kf-feast/feast-serving \
             --tag dev \
             --file infra/docker/serving/Dockerfile \
@@ -232,13 +232,13 @@ postsubmits:
 
           else
 
-          .prow/scripts/publish-docker-image.sh \
+          infra/scripts/publish-docker-image.sh \
             --repository gcr.io/kf-feast/feast-core \
             --tag ${PULL_BASE_REF:1} \
             --file infra/docker/core/Dockerfile \
             --google-service-account-file /etc/gcloud/service-account.json
 
-          .prow/scripts/publish-docker-image.sh \
+          infra/scripts/publish-docker-image.sh \
             --repository gcr.io/kf-feast/feast-serving \
             --tag ${PULL_BASE_REF:1} \
             --file infra/docker/serving/Dockerfile \
@@ -295,7 +295,7 @@ postsubmits:
           sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/Chart.yaml
           sed -i "/  tag: /c\  tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/values.yaml
 
-          .prow/scripts/sync-helm-charts.sh
+          infra/scripts/sync-helm-charts.sh
         volumeMounts:
         - name: service-account
           mountPath: /etc/gcloud/service-account.json
diff --git a/.prow/scripts/download-maven-cache.sh b/infra/scripts/download-maven-cache.sh
similarity index 100%
rename from .prow/scripts/download-maven-cache.sh
rename to infra/scripts/download-maven-cache.sh
diff --git a/.prow/scripts/install-google-cloud-sdk.sh b/infra/scripts/install-google-cloud-sdk.sh
similarity index 100%
rename from .prow/scripts/install-google-cloud-sdk.sh
rename to infra/scripts/install-google-cloud-sdk.sh
diff --git a/.prow/scripts/publish-docker-image.sh b/infra/scripts/publish-docker-image.sh
similarity index 100%
rename from .prow/scripts/publish-docker-image.sh
rename to infra/scripts/publish-docker-image.sh
diff --git a/.prow/scripts/publish-java-sdk.sh b/infra/scripts/publish-java-sdk.sh
similarity index 100%
rename from .prow/scripts/publish-java-sdk.sh
rename to infra/scripts/publish-java-sdk.sh
diff --git a/.prow/scripts/publish-python-sdk.sh b/infra/scripts/publish-python-sdk.sh
similarity index 100%
rename from .prow/scripts/publish-python-sdk.sh
rename to infra/scripts/publish-python-sdk.sh
diff --git a/.prow/scripts/sync-helm-charts.sh b/infra/scripts/sync-helm-charts.sh
similarity index 100%
rename from .prow/scripts/sync-helm-charts.sh
rename to infra/scripts/sync-helm-charts.sh
diff --git a/.prow/scripts/test-core-ingestion.sh b/infra/scripts/test-core-ingestion.sh
similarity index 95%
rename from .prow/scripts/test-core-ingestion.sh
rename to infra/scripts/test-core-ingestion.sh
index 98a47ca68c9..b7ec1d814b8 100755
--- a/.prow/scripts/test-core-ingestion.sh
+++ b/infra/scripts/test-core-ingestion.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-.prow/scripts/download-maven-cache.sh \
+infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/
 
diff --git a/.prow/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh
similarity index 99%
rename from .prow/scripts/test-end-to-end-batch.sh
rename to infra/scripts/test-end-to-end-batch.sh
index b7c1794691b..bfd6545ee8e 100755
--- a/.prow/scripts/test-end-to-end-batch.sh
+++ b/infra/scripts/test-end-to-end-batch.sh
@@ -79,7 +79,7 @@ Building jars for Feast
 ============================================================
 "
 
-.prow/scripts/download-maven-cache.sh \
+infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/
 
diff --git a/.prow/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
similarity index 99%
rename from .prow/scripts/test-end-to-end.sh
rename to infra/scripts/test-end-to-end.sh
index 951972f8e6b..3f9f31e80cf 100755
--- a/.prow/scripts/test-end-to-end.sh
+++ b/infra/scripts/test-end-to-end.sh
@@ -62,7 +62,7 @@ Building jars for Feast
 ============================================================
 "
 
-.prow/scripts/download-maven-cache.sh \
+infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/
 
diff --git a/.prow/scripts/test-golang-sdk.sh b/infra/scripts/test-golang-sdk.sh
similarity index 100%
rename from .prow/scripts/test-golang-sdk.sh
rename to infra/scripts/test-golang-sdk.sh
diff --git a/.prow/scripts/test-java-sdk.sh b/infra/scripts/test-java-sdk.sh
similarity index 100%
rename from .prow/scripts/test-java-sdk.sh
rename to infra/scripts/test-java-sdk.sh
diff --git a/.prow/scripts/test-python-sdk.sh b/infra/scripts/test-python-sdk.sh
similarity index 100%
rename from .prow/scripts/test-python-sdk.sh
rename to infra/scripts/test-python-sdk.sh
diff --git a/.prow/scripts/test-serving.sh b/infra/scripts/test-serving.sh
similarity index 90%
rename from .prow/scripts/test-serving.sh
rename to infra/scripts/test-serving.sh
index b56001619b3..ce9dc0a8162 100755
--- a/.prow/scripts/test-serving.sh
+++ b/infra/scripts/test-serving.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-.prow/scripts/download-maven-cache.sh \
+infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/
 

From a68ddf266521e5a86149252d0880b500adb5c2fd Mon Sep 17 00:00:00 2001
From: Chen Zhiling 
Date: Wed, 25 Mar 2020 15:37:42 +0800
Subject: [PATCH 16/17] Add index for join table for jobs-featuresets relation
 (#566, #647)

---
 .../main/java/feast/core/model/JobInfo.java   | 22 +++++++------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/feast/core/model/JobInfo.java b/core/src/main/java/feast/core/model/JobInfo.java
index 74d3402af56..b7b8b565b15 100644
--- a/core/src/main/java/feast/core/model/JobInfo.java
+++ b/core/src/main/java/feast/core/model/JobInfo.java
@@ -17,18 +17,7 @@
 package feast.core.model;
 
 import java.util.List;
-import javax.persistence.CascadeType;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.EnumType;
-import javax.persistence.Enumerated;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.JoinTable;
-import javax.persistence.ManyToMany;
-import javax.persistence.ManyToOne;
-import javax.persistence.OneToMany;
-import javax.persistence.Table;
+import javax.persistence.*;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
@@ -66,8 +55,13 @@ public class JobInfo extends AbstractTimestampEntity {
   // FeatureSets populated by the job
   @ManyToMany
   @JoinTable(
-      joinColumns = {@JoinColumn(name = "job_id")},
-      inverseJoinColumns = {@JoinColumn(name = "feature_set_id")})
+      name = "jobs_feature_sets",
+      joinColumns = @JoinColumn(name = "job_id"),
+      inverseJoinColumns = @JoinColumn(name = "feature_set_id"),
+      indexes = {
+        @Index(name = "idx_jobs_feature_sets_job_id", columnList = "job_id"),
+        @Index(name = "idx_jobs_feature_sets_feature_set_id", columnList = "feature_set_id")
+      })
   private List featureSets;
 
   // Job Metrics

From 987cb9922f161b48c3cd611465b93fce3d9549d6 Mon Sep 17 00:00:00 2001
From: Willem Pienaar <6728866+woop@users.noreply.github.com>
Date: Fri, 1 May 2020 18:40:31 +0800
Subject: [PATCH 17/17] Release v0.3.7 (#670)

* Release v0.3.7

* Remove SNAPSHOT from e2e tests

* Find jar version from Maven
---
 CHANGELOG.md                           | 349 +++++++++++++++++++++++++
 infra/scripts/test-end-to-end-batch.sh |   7 +-
 infra/scripts/test-end-to-end.sh       |   7 +-
 pom.xml                                |   2 +-
 4 files changed, 360 insertions(+), 5 deletions(-)
 create mode 100644 CHANGELOG.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000000..5c9db512dd4
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,349 @@
+# Changelog
+
+## [v0.3.7](https://github.com/gojek/feast/tree/v0.3.7) (2020-05-01)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.4.7...v0.3.7)
+
+**Merged pull requests:**
+
+- Moved end-to-end test scripts from .prow to infra [\#657](https://github.com/gojek/feast/pull/657) ([khorshuheng](https://github.com/khorshuheng))
+- Backported \#566 & \#647 to v0.3 [\#654](https://github.com/gojek/feast/pull/654) ([ches](https://github.com/ches))
+
+## [v0.3.6](https://github.com/gojek/feast/tree/v0.3.6) (2020-01-03)
+
+**Merged pull requests:**
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.3.5...v0.3.6)
+
+- Add support for file paths for providing entity rows during batch retrieval [\#375](https://github.com/gojek/feast/pull/376) ([voonhous](https://github.com/voonhous))
+
+## [v0.3.5](https://github.com/gojek/feast/tree/v0.3.5) (2019-12-26)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.3.4...v0.3.5)
+
+**Merged pull requests:**
+
+- Always set destination table in BigQuery query config in Feast Batch Serving so it can handle large results [\#392](https://github.com/gojek/feast/pull/392) ([davidheryanto](https://github.com/davidheryanto))
+
+## [v0.3.4](https://github.com/gojek/feast/tree/v0.3.4) (2019-12-23)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.3.3...v0.3.4)
+
+**Merged pull requests:**
+
+- Make redis key creation more determinisitic [\#380](https://github.com/gojek/feast/pull/380) ([zhilingc](https://github.com/zhilingc))
+
+## [v0.3.3](https://github.com/gojek/feast/tree/v0.3.3) (2019-12-18)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.3.2...v0.3.3)
+
+**Implemented enhancements:**
+
+- Added Docker Compose for Feast [\#272](https://github.com/gojek/feast/issues/272)
+- Added ability to check import job status and cancel job through Python SDK [\#194](https://github.com/gojek/feast/issues/194)
+- Added basic customer transactions example [\#354](https://github.com/gojek/feast/pull/354) ([woop](https://github.com/woop))
+
+**Merged pull requests:**
+
+- Added Prow jobs to automate the release of Docker images and Python SDK [\#369](https://github.com/gojek/feast/pull/369) ([davidheryanto](https://github.com/davidheryanto))
+- Fixed installation link in README.md [\#368](https://github.com/gojek/feast/pull/368) ([Jeffwan](https://github.com/Jeffwan))
+- Fixed Java SDK tests not actually running \(missing dependencies\) [\#366](https://github.com/gojek/feast/pull/366) ([woop](https://github.com/woop))
+- Added more batch retrieval tests [\#357](https://github.com/gojek/feast/pull/357) ([zhilingc](https://github.com/zhilingc))
+- Python SDK and Feast Core Bug Fixes [\#353](https://github.com/gojek/feast/pull/353) ([woop](https://github.com/woop))
+- Updated buildFeatureSets method in Golang SDK [\#351](https://github.com/gojek/feast/pull/351) ([davidheryanto](https://github.com/davidheryanto))
+- Python SDK cleanup [\#348](https://github.com/gojek/feast/pull/348) ([woop](https://github.com/woop))
+- Broke up queries for point in time correctness joins [\#347](https://github.com/gojek/feast/pull/347) ([zhilingc](https://github.com/zhilingc))
+- Exports gRPC call metrics and Feast resource metrics in Core [\#345](https://github.com/gojek/feast/pull/345) ([davidheryanto](https://github.com/davidheryanto))
+- Fixed broken Google Group link on Community page [\#343](https://github.com/gojek/feast/pull/343) ([ches](https://github.com/ches))
+- Ensured ImportJobTest is not flaky by checking WriteToStore metric and requesting adequate resources for testing [\#332](https://github.com/gojek/feast/pull/332) ([davidheryanto](https://github.com/davidheryanto))
+- Added docker-compose file with Jupyter notebook [\#328](https://github.com/gojek/feast/pull/328) ([khorshuheng](https://github.com/khorshuheng))
+- Added minimal implementation of ingesting Parquet and CSV files [\#327](https://github.com/gojek/feast/pull/327) ([voonhous](https://github.com/voonhous))
+
+## [v0.3.2](https://github.com/gojek/feast/tree/v0.3.2) (2019-11-29)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.3.1...v0.3.2)
+
+**Merged pull requests:**
+
+- Fixed incorrect BigQuery schema creation from FeatureSetSpec [\#340](https://github.com/gojek/feast/pull/340) ([davidheryanto](https://github.com/davidheryanto))
+- Filtered out feature sets that dont share the same source [\#339](https://github.com/gojek/feast/pull/339) ([zhilingc](https://github.com/zhilingc))
+- Changed latency calculation method to not use Timer [\#338](https://github.com/gojek/feast/pull/338) ([zhilingc](https://github.com/zhilingc))
+- Moved Prometheus annotations to pod template for serving [\#336](https://github.com/gojek/feast/pull/336) ([zhilingc](https://github.com/zhilingc))
+- Removed metrics windowing, cleaned up step names for metrics writing [\#334](https://github.com/gojek/feast/pull/334) ([zhilingc](https://github.com/zhilingc))
+- Set BigQuery table time partition inside get table function [\#333](https://github.com/gojek/feast/pull/333) ([zhilingc](https://github.com/zhilingc))
+- Added unit test in Redis to return values with no max age set [\#329](https://github.com/gojek/feast/pull/329) ([smadarasmi](https://github.com/smadarasmi))
+- Consolidated jobs into single steps instead of branching out [\#326](https://github.com/gojek/feast/pull/326) ([zhilingc](https://github.com/zhilingc))
+- Pinned Python SDK to minor versions for dependencies [\#322](https://github.com/gojek/feast/pull/322) ([woop](https://github.com/woop))
+- Added Auto format to Google style with Spotless [\#317](https://github.com/gojek/feast/pull/317) ([ches](https://github.com/ches))
+
+## [v0.3.1](https://github.com/gojek/feast/tree/v0.3.1) (2019-11-25)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.3.0...v0.3.1)
+
+**Merged pull requests:**
+
+- Added Prometheus metrics to serving [\#316](https://github.com/gojek/feast/pull/316) ([zhilingc](https://github.com/zhilingc))
+- Changed default job metrics sink to Statsd [\#315](https://github.com/gojek/feast/pull/315) ([zhilingc](https://github.com/zhilingc))
+- Fixed module import error in Feast CLI [\#314](https://github.com/gojek/feast/pull/314) ([davidheryanto](https://github.com/davidheryanto))
+
+## [v0.3.0](https://github.com/gojek/feast/tree/v0.3.0) (2019-11-19)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.1.8...v0.3.0)
+
+**Summary:**
+
+* Introduced "Feature Sets" as a concept with a new [Feast Core API](https://github.com/gojek/feast/blob/v0.3.0/protos/feast/core/CoreService.proto), [Feast Serving API](https://github.com/gojek/feast/blob/v0.3.0/protos/feast/serving/ServingService.proto)
+* Upgraded [Python SDK](https://github.com/gojek/feast/tree/v0.3.0/sdk/python) to support new Feast API. Allows for management of Feast as a library or through the command line.
+* Implemented a [Golang SDK](https://github.com/gojek/feast/tree/v0.3.0/sdk/go) and [Java SDK](https://github.com/gojek/feast/tree/v0.3.0/sdk/java) to support the new Feast Core and Feast Serving APIs.
+* Added support for multi-feature set retrieval and joins.
+* Added point-in-time correct retrieval for both batch and online serving.
+* Added support for an external source in Kafka.
+* Added job management to Feast Core to manage ingestion/population jobs to remote Feast deployments
+* Added metric support through Prometheus
+
+**Merged pull requests:**
+
+- Regenerate go protos [\#313](https://github.com/gojek/feast/pull/313) ([zhilingc](https://github.com/zhilingc))
+- Bump chart version to 0.3.0 [\#311](https://github.com/gojek/feast/pull/311) ([zhilingc](https://github.com/zhilingc))
+- Refactored Core API: ListFeatureSets, ListStore, and GetFeatureSet [\#309](https://github.com/gojek/feast/pull/309) ([woop](https://github.com/woop))
+- Use Maven's --also-make by default [\#308](https://github.com/gojek/feast/pull/308) ([ches](https://github.com/ches))
+- Python SDK Ingestion and schema inference updates [\#307](https://github.com/gojek/feast/pull/307) ([woop](https://github.com/woop))
+- Batch ingestion fix [\#299](https://github.com/gojek/feast/pull/299) ([zhilingc](https://github.com/zhilingc))
+- Update values-demo.yaml to make Minikube installation simpler [\#298](https://github.com/gojek/feast/pull/298) ([woop](https://github.com/woop))
+- Fix bug in core not setting default Kafka source [\#297](https://github.com/gojek/feast/pull/297) ([woop](https://github.com/woop))
+- Replace Prometheus logging in ingestion with StatsD logging [\#293](https://github.com/gojek/feast/pull/293) ([woop](https://github.com/woop))
+- Feast Core: Stage files manually when launching Dataflow jobs [\#291](https://github.com/gojek/feast/pull/291) ([davidheryanto](https://github.com/davidheryanto))
+- Database tweaks [\#290](https://github.com/gojek/feast/pull/290) ([smadarasmi](https://github.com/smadarasmi))
+- Feast Helm charts and build script [\#289](https://github.com/gojek/feast/pull/289) ([davidheryanto](https://github.com/davidheryanto))
+- Fix max\_age changes not updating specs and add TQDM silencing flag [\#292](https://github.com/gojek/feast/pull/292) ([woop](https://github.com/woop))
+- Ingestion fixes [\#286](https://github.com/gojek/feast/pull/286) ([zhilingc](https://github.com/zhilingc))
+- Consolidate jobs [\#279](https://github.com/gojek/feast/pull/279) ([zhilingc](https://github.com/zhilingc))
+- Import Spring Boot's dependency BOM, fix spring-boot:run at parent project level [\#276](https://github.com/gojek/feast/pull/276) ([ches](https://github.com/ches))
+- Feast 0.3 Continuous Integration \(CI\) Update  [\#271](https://github.com/gojek/feast/pull/271) ([davidheryanto](https://github.com/davidheryanto))
+- Add batch feature retrieval to Python SDK [\#268](https://github.com/gojek/feast/pull/268) ([woop](https://github.com/woop))
+- Set Maven build requirements and some project POM metadata [\#267](https://github.com/gojek/feast/pull/267) ([ches](https://github.com/ches))
+- Python SDK enhancements [\#264](https://github.com/gojek/feast/pull/264) ([woop](https://github.com/woop))
+- Use a symlink for Java SDK's protos [\#263](https://github.com/gojek/feast/pull/263) ([ches](https://github.com/ches))
+- Clean up the Maven build [\#262](https://github.com/gojek/feast/pull/262) ([ches](https://github.com/ches))
+- Add golang SDK [\#261](https://github.com/gojek/feast/pull/261) ([zhilingc](https://github.com/zhilingc))
+- Move storage configuration to serving [\#254](https://github.com/gojek/feast/pull/254) ([zhilingc](https://github.com/zhilingc))
+- Serving API changes for 0.3 [\#253](https://github.com/gojek/feast/pull/253) ([zhilingc](https://github.com/zhilingc))
+
+## [v0.1.8](https://github.com/gojek/feast/tree/v0.1.8) (2019-10-30)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.1.2...v0.1.8)
+
+**Implemented enhancements:**
+
+- Feast cli config file should be settable by an env var [\#149](https://github.com/gojek/feast/issues/149)
+- Helm chart for deploying feast using Flink as runner [\#64](https://github.com/gojek/feast/issues/64)
+- Get ingestion metrics when running on Flink runner [\#63](https://github.com/gojek/feast/issues/63)
+- Move source types into their own package and discover them using java.util.ServiceLoader [\#61](https://github.com/gojek/feast/issues/61)
+- Change config to yaml [\#51](https://github.com/gojek/feast/issues/51)
+- Ability to pass runner option during ingestion job submission [\#50](https://github.com/gojek/feast/issues/50)
+
+**Fixed bugs:**
+
+- Fix Print Method in Feast CLI [\#211](https://github.com/gojek/feast/issues/211)
+- Dataflow monitoring by core is failing with incorrect job id [\#153](https://github.com/gojek/feast/issues/153)
+- Feast core crashes without logger set [\#150](https://github.com/gojek/feast/issues/150)
+
+**Merged pull requests:**
+
+- Remove redis transaction [\#280](https://github.com/gojek/feast/pull/280) ([pradithya](https://github.com/pradithya))
+- Fix tracing to continue from existing trace created by grpc client [\#245](https://github.com/gojek/feast/pull/245) ([pradithya](https://github.com/pradithya))
+
+## [v0.1.2](https://github.com/gojek/feast/tree/v0.1.2) (2019-08-23)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.1.1...v0.1.2)
+
+**Fixed bugs:**
+
+- Batch Import, feature with datetime format issue [\#203](https://github.com/gojek/feast/issues/203)
+- Serving not correctly reporting readiness check if there is no activity [\#190](https://github.com/gojek/feast/issues/190)
+- Serving stop periodically reloading feature specification after a while [\#188](https://github.com/gojek/feast/issues/188)
+
+**Merged pull requests:**
+
+- Add `romanwozniak` to prow owners config [\#216](https://github.com/gojek/feast/pull/216) ([romanwozniak](https://github.com/romanwozniak))
+- Implement filter for create dataset api [\#215](https://github.com/gojek/feast/pull/215) ([pradithya](https://github.com/pradithya))
+- expand raw column to accomodate more features ingested in one go [\#213](https://github.com/gojek/feast/pull/213) ([budi](https://github.com/budi))
+- update feast installation docs [\#210](https://github.com/gojek/feast/pull/210) ([budi](https://github.com/budi))
+- Add Prow job for unit testing Python SDK [\#209](https://github.com/gojek/feast/pull/209) ([davidheryanto](https://github.com/davidheryanto))
+- fix create\_dataset [\#208](https://github.com/gojek/feast/pull/208) ([budi](https://github.com/budi))
+- Update Feast installation doc [\#207](https://github.com/gojek/feast/pull/207) ([davidheryanto](https://github.com/davidheryanto))
+- Fix unit test cli in prow script not returning correct exit code [\#206](https://github.com/gojek/feast/pull/206) ([davidheryanto](https://github.com/davidheryanto))
+- Fix pytests and make TS conversion conditional [\#205](https://github.com/gojek/feast/pull/205) ([zhilingc](https://github.com/zhilingc))
+- Use full prow build id as dataset name during test [\#200](https://github.com/gojek/feast/pull/200) ([davidheryanto](https://github.com/davidheryanto))
+- Add Feast CLI / python SDK documentation [\#199](https://github.com/gojek/feast/pull/199) ([romanwozniak](https://github.com/romanwozniak))
+- Update library version to fix security vulnerabilities in dependencies [\#198](https://github.com/gojek/feast/pull/198) ([davidheryanto](https://github.com/davidheryanto))
+- Update Prow configuration for Feast CI [\#197](https://github.com/gojek/feast/pull/197) ([davidheryanto](https://github.com/davidheryanto))
+- \[budi\] update python sdk quickstart [\#196](https://github.com/gojek/feast/pull/196) ([budi](https://github.com/budi))
+- Readiness probe [\#191](https://github.com/gojek/feast/pull/191) ([pradithya](https://github.com/pradithya))
+- Fix periodic feature spec reload [\#189](https://github.com/gojek/feast/pull/189) ([pradithya](https://github.com/pradithya))
+- Fixed a typo in environment variable in installation [\#187](https://github.com/gojek/feast/pull/187) ([gauravkumar37](https://github.com/gauravkumar37))
+- Revert "Update Quickstart" [\#185](https://github.com/gojek/feast/pull/185) ([zhilingc](https://github.com/zhilingc))
+- Update Quickstart [\#184](https://github.com/gojek/feast/pull/184) ([pradithya](https://github.com/pradithya))
+- Continuous integration and deployment \(CI/CD\) update [\#183](https://github.com/gojek/feast/pull/183) ([davidheryanto](https://github.com/davidheryanto))
+- Remove feature specs being able to declare their serving or warehouse stores [\#159](https://github.com/gojek/feast/pull/159) ([tims](https://github.com/tims))
+
+## [v0.1.1](https://github.com/gojek/feast/tree/v0.1.1) (2019-04-18)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.1.0...v0.1.1)
+
+**Fixed bugs:**
+
+- Fix BigQuery query template to retrieve training data [\#182](https://github.com/gojek/feast/pull/182) ([davidheryanto](https://github.com/davidheryanto))
+
+**Merged pull requests:**
+
+- Add python init files [\#176](https://github.com/gojek/feast/pull/176) ([zhilingc](https://github.com/zhilingc))
+- Change pypi package from Feast to feast [\#173](https://github.com/gojek/feast/pull/173) ([zhilingc](https://github.com/zhilingc))
+
+## [v0.1.0](https://github.com/gojek/feast/tree/v0.1.0) (2019-04-09)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.0.2...v0.1.0)
+
+**Implemented enhancements:**
+
+- Removal of storing historical value of feature in serving storage [\#53](https://github.com/gojek/feast/issues/53)
+- Remove feature "granularity" and relegate to metadata [\#17](https://github.com/gojek/feast/issues/17)
+
+**Closed issues:**
+
+- Add ability to name an import job [\#167](https://github.com/gojek/feast/issues/167)
+- Ingestion retrying an invalid FeatureRow endlessly [\#163](https://github.com/gojek/feast/issues/163)
+- Ability to associate data ingested in Warehouse store to its ingestion job [\#145](https://github.com/gojek/feast/issues/145)
+- Missing \(Fixing\) unit test for FeatureRowKafkaIO [\#132](https://github.com/gojek/feast/issues/132)
+
+**Merged pull requests:**
+
+- Catch all kind of exception to avoid retrying [\#171](https://github.com/gojek/feast/pull/171) ([pradithya](https://github.com/pradithya))
+- Integration test [\#170](https://github.com/gojek/feast/pull/170) ([zhilingc](https://github.com/zhilingc))
+- Proto error [\#169](https://github.com/gojek/feast/pull/169) ([pradithya](https://github.com/pradithya))
+- Add --name flag to submit job [\#168](https://github.com/gojek/feast/pull/168) ([pradithya](https://github.com/pradithya))
+- Prevent throwing RuntimeException when invalid proto is received [\#166](https://github.com/gojek/feast/pull/166) ([pradithya](https://github.com/pradithya))
+- Add davidheryanto to OWNER file [\#165](https://github.com/gojek/feast/pull/165) ([pradithya](https://github.com/pradithya))
+- Check validity of event timestamp in ValidateFeatureRowDoFn [\#164](https://github.com/gojek/feast/pull/164) ([pradithya](https://github.com/pradithya))
+- Remove granularity [\#162](https://github.com/gojek/feast/pull/162) ([pradithya](https://github.com/pradithya))
+- Better Kafka test [\#160](https://github.com/gojek/feast/pull/160) ([tims](https://github.com/tims))
+- Simplify and document CLI building steps [\#158](https://github.com/gojek/feast/pull/158) ([thirteen37](https://github.com/thirteen37))
+- Fix link typo in README.md [\#156](https://github.com/gojek/feast/pull/156) ([pradithya](https://github.com/pradithya))
+- Add Feast admin quickstart guide [\#155](https://github.com/gojek/feast/pull/155) ([thirteen37](https://github.com/thirteen37))
+- Pass all specs to ingestion by file [\#154](https://github.com/gojek/feast/pull/154) ([tims](https://github.com/tims))
+- Preload spec in serving cache [\#152](https://github.com/gojek/feast/pull/152) ([pradithya](https://github.com/pradithya))
+- Add job identifier to FeatureRow  [\#147](https://github.com/gojek/feast/pull/147) ([mansiib](https://github.com/mansiib))
+- Fix unit tests [\#146](https://github.com/gojek/feast/pull/146) ([mansiib](https://github.com/mansiib))
+- Add thirteen37 to OWNERS [\#144](https://github.com/gojek/feast/pull/144) ([thirteen37](https://github.com/thirteen37))
+- Fix import spec created from Importer.from\_csv [\#143](https://github.com/gojek/feast/pull/143) ([pradithya](https://github.com/pradithya))
+- Regenerate go [\#142](https://github.com/gojek/feast/pull/142) ([zhilingc](https://github.com/zhilingc))
+- Flat JSON for pubsub and text files [\#141](https://github.com/gojek/feast/pull/141) ([tims](https://github.com/tims))
+- Add wait flag for jobs, fix go proto path for dataset service [\#138](https://github.com/gojek/feast/pull/138) ([zhilingc](https://github.com/zhilingc))
+- Fix Python SDK importer's ability to apply features [\#135](https://github.com/gojek/feast/pull/135) ([woop](https://github.com/woop))
+- Refactor stores [\#110](https://github.com/gojek/feast/pull/110) ([tims](https://github.com/tims))
+- Coalesce rows [\#89](https://github.com/gojek/feast/pull/89) ([tims](https://github.com/tims))
+- Remove historical feature in serving store [\#87](https://github.com/gojek/feast/pull/87) ([pradithya](https://github.com/pradithya))
+
+## [v0.0.2](https://github.com/gojek/feast/tree/v0.0.2) (2019-03-11)
+
+[Full Changelog](https://github.com/gojek/feast/compare/v0.0.1...v0.0.2)
+
+**Implemented enhancements:**
+
+- Coalesce FeatureRows for improved "latest" value consistency in serving stores [\#88](https://github.com/gojek/feast/issues/88)
+- Kafka source [\#22](https://github.com/gojek/feast/issues/22)
+
+**Closed issues:**
+
+- Preload Feast's spec in serving cache [\#151](https://github.com/gojek/feast/issues/151)
+- Feast csv data upload job [\#137](https://github.com/gojek/feast/issues/137)
+- Blocking call to start feast ingestion job [\#136](https://github.com/gojek/feast/issues/136)
+- Python SDK fails to apply feature when submitting job [\#134](https://github.com/gojek/feast/issues/134)
+- Default dump format should be changed for Python SDK [\#133](https://github.com/gojek/feast/issues/133)
+- Listing resources and finding out system state [\#131](https://github.com/gojek/feast/issues/131)
+- Reorganise ingestion store classes to match architecture  [\#109](https://github.com/gojek/feast/issues/109)
+
+## [v0.0.1](https://github.com/gojek/feast/tree/v0.0.1) (2019-02-11)
+
+[Full Changelog](https://github.com/gojek/feast/compare/ec9def2bbb06dc759538e4424caadd70f548ea64...v0.0.1)
+
+**Implemented enhancements:**
+
+- Spring boot CLI logs show up as JSON [\#104](https://github.com/gojek/feast/issues/104)
+- Allow for registering feature that doesn't have warehouse store [\#5](https://github.com/gojek/feast/issues/5)
+
+**Fixed bugs:**
+
+- Error when submitting large import spec [\#125](https://github.com/gojek/feast/issues/125)
+- Ingestion is not ignoring unknown feature in streaming source [\#99](https://github.com/gojek/feast/issues/99)
+- Vulnerability in dependency \(core - jackson-databind \)  [\#92](https://github.com/gojek/feast/issues/92)
+- TF file for cloud build trigger broken [\#72](https://github.com/gojek/feast/issues/72)
+- Job Execution Failure with NullPointerException [\#46](https://github.com/gojek/feast/issues/46)
+- Runtime Dependency Error After Upgrade to Beam 2.9.0 [\#44](https://github.com/gojek/feast/issues/44)
+- \[FlinkRunner\] Core should not follow remote flink runner job to completion [\#21](https://github.com/gojek/feast/issues/21)
+- Go packages in protos use incorrect repo [\#16](https://github.com/gojek/feast/issues/16)
+
+**Merged pull requests:**
+
+- Disable test during docker image creation [\#129](https://github.com/gojek/feast/pull/129) ([pradithya](https://github.com/pradithya))
+- Repackage helm chart [\#127](https://github.com/gojek/feast/pull/127) ([pradithya](https://github.com/pradithya))
+- Increase the column size for storing raw import spec [\#126](https://github.com/gojek/feast/pull/126) ([pradithya](https://github.com/pradithya))
+- Update Helm Charts \(Redis, Logging\) [\#123](https://github.com/gojek/feast/pull/123) ([woop](https://github.com/woop))
+- Added LOG\_TYPE environmental variable [\#120](https://github.com/gojek/feast/pull/120) ([woop](https://github.com/woop))
+- Fix missing Redis write [\#119](https://github.com/gojek/feast/pull/119) ([pradithya](https://github.com/pradithya))
+- add logging when error on request feature [\#117](https://github.com/gojek/feast/pull/117) ([pradithya](https://github.com/pradithya))
+- run yarn run build during generate-resource [\#115](https://github.com/gojek/feast/pull/115) ([pradithya](https://github.com/pradithya))
+- Add loadBalancerSourceRanges option for both serving and core [\#114](https://github.com/gojek/feast/pull/114) ([zhilingc](https://github.com/zhilingc))
+- Build master [\#112](https://github.com/gojek/feast/pull/112) ([pradithya](https://github.com/pradithya))
+- Cleanup warning while building proto files [\#108](https://github.com/gojek/feast/pull/108) ([pradithya](https://github.com/pradithya))
+- Embed ui build & packaging into core's build [\#106](https://github.com/gojek/feast/pull/106) ([pradithya](https://github.com/pradithya))
+- Add build badge to README [\#103](https://github.com/gojek/feast/pull/103) ([woop](https://github.com/woop))
+- Ignore features in FeatureRow if it's not requested in import spec [\#101](https://github.com/gojek/feast/pull/101) ([pradithya](https://github.com/pradithya))
+- Add override for serving service static ip [\#100](https://github.com/gojek/feast/pull/100) ([zhilingc](https://github.com/zhilingc))
+- Fix go test [\#97](https://github.com/gojek/feast/pull/97) ([zhilingc](https://github.com/zhilingc))
+- add missing copyright headers and fix test fail due to previous merge [\#95](https://github.com/gojek/feast/pull/95) ([tims](https://github.com/tims))
+- Allow submission of kafka jobs [\#94](https://github.com/gojek/feast/pull/94) ([zhilingc](https://github.com/zhilingc))
+- upgrade jackson databind for security vulnerability [\#93](https://github.com/gojek/feast/pull/93) ([tims](https://github.com/tims))
+- Version revert [\#91](https://github.com/gojek/feast/pull/91) ([zhilingc](https://github.com/zhilingc))
+- Fix validating feature row when the associated feature spec has no warehouse store [\#90](https://github.com/gojek/feast/pull/90) ([pradithya](https://github.com/pradithya))
+- Add get command [\#85](https://github.com/gojek/feast/pull/85) ([zhilingc](https://github.com/zhilingc))
+- Avoid error thrown when no storage for warehouse/serving is registered [\#83](https://github.com/gojek/feast/pull/83) ([pradithya](https://github.com/pradithya))
+- Fix jackson dependency issue [\#82](https://github.com/gojek/feast/pull/82) ([zhilingc](https://github.com/zhilingc))
+- Allow registration of feature without warehouse store [\#80](https://github.com/gojek/feast/pull/80) ([pradithya](https://github.com/pradithya))
+- Remove branch from cloud build trigger [\#79](https://github.com/gojek/feast/pull/79) ([woop](https://github.com/woop))
+- move read transforms into "source" package as FeatureSources [\#74](https://github.com/gojek/feast/pull/74) ([tims](https://github.com/tims))
+- Fix tag regex in tf file [\#73](https://github.com/gojek/feast/pull/73) ([zhilingc](https://github.com/zhilingc))
+- Update charts [\#71](https://github.com/gojek/feast/pull/71) ([mansiib](https://github.com/mansiib))
+- Deduplicate storage ids before we fetch them [\#68](https://github.com/gojek/feast/pull/68) ([tims](https://github.com/tims))
+- Check the size of result against deduplicated request [\#67](https://github.com/gojek/feast/pull/67) ([pradithya](https://github.com/pradithya))
+- Add ability to submit ingestion job using Flink [\#62](https://github.com/gojek/feast/pull/62) ([pradithya](https://github.com/pradithya))
+- Fix vulnerabilities for webpack-dev [\#59](https://github.com/gojek/feast/pull/59) ([budi](https://github.com/budi))
+- Build push [\#56](https://github.com/gojek/feast/pull/56) ([zhilingc](https://github.com/zhilingc))
+- Fix github vulnerability issue with webpack [\#54](https://github.com/gojek/feast/pull/54) ([budi](https://github.com/budi))
+- Only lookup storage specs that we actually need [\#52](https://github.com/gojek/feast/pull/52) ([tims](https://github.com/tims))
+- Link Python SDK RFC to PR and Issue [\#49](https://github.com/gojek/feast/pull/49) ([woop](https://github.com/woop))
+- Python SDK [\#47](https://github.com/gojek/feast/pull/47) ([zhilingc](https://github.com/zhilingc))
+- Update com.google.httpclient to be same as Beam's dependency [\#45](https://github.com/gojek/feast/pull/45) ([pradithya](https://github.com/pradithya))
+- Bump Beam SDK to 2.9.0 [\#43](https://github.com/gojek/feast/pull/43) ([pradithya](https://github.com/pradithya))
+- Add fix for tests failing in docker image [\#40](https://github.com/gojek/feast/pull/40) ([zhilingc](https://github.com/zhilingc))
+- Change error store to be part of configuration instead [\#39](https://github.com/gojek/feast/pull/39) ([zhilingc](https://github.com/zhilingc))
+- Fix location of Prow's Tide configuration [\#35](https://github.com/gojek/feast/pull/35) ([woop](https://github.com/woop))
+- Add testing folder for deploying test infrastructure and running tests [\#34](https://github.com/gojek/feast/pull/34) ([woop](https://github.com/woop))
+- skeleton contributing guide [\#33](https://github.com/gojek/feast/pull/33) ([tims](https://github.com/tims))
+- allow empty string to select a NoOp write transform [\#30](https://github.com/gojek/feast/pull/30) ([tims](https://github.com/tims))
+- Remove packaging ingestion as separate profile \(fix \#28\) [\#29](https://github.com/gojek/feast/pull/29) ([pradithya](https://github.com/pradithya))
+- Change gopath to point to gojek repo [\#26](https://github.com/gojek/feast/pull/26) ([zhilingc](https://github.com/zhilingc))
+- Fixes \#31 - errors during kafka deserializer \(passing\) test execution [\#25](https://github.com/gojek/feast/pull/25) ([baskaranz](https://github.com/baskaranz))
+- Kafka IO fixes [\#23](https://github.com/gojek/feast/pull/23) ([tims](https://github.com/tims))
+- KafkaIO implementation for feast [\#19](https://github.com/gojek/feast/pull/19) ([baskaranz](https://github.com/baskaranz))
+- Return same type string for warehouse and serving NoOp stores [\#18](https://github.com/gojek/feast/pull/18) ([tims](https://github.com/tims))
+- \#12: prefetch specs and validate on job expansion [\#15](https://github.com/gojek/feast/pull/15) ([tims](https://github.com/tims))
+- Added RFC for Feast Python SDK [\#14](https://github.com/gojek/feast/pull/14) ([woop](https://github.com/woop))
+- Add more validation in feature spec registration [\#11](https://github.com/gojek/feast/pull/11) ([pradithya](https://github.com/pradithya))
+- Added rfcs/ folder with readme and template [\#10](https://github.com/gojek/feast/pull/10) ([woop](https://github.com/woop))
+- Expose ui service rpc [\#9](https://github.com/gojek/feast/pull/9) ([pradithya](https://github.com/pradithya))
+- Add Feast overview to README [\#8](https://github.com/gojek/feast/pull/8) ([woop](https://github.com/woop))
+- Directory structure changes [\#7](https://github.com/gojek/feast/pull/7) ([zhilingc](https://github.com/zhilingc))
+- Change register to apply [\#4](https://github.com/gojek/feast/pull/4) ([zhilingc](https://github.com/zhilingc))
+- Empty response handling in serving api [\#3](https://github.com/gojek/feast/pull/3) ([pradithya](https://github.com/pradithya))
+- Proto file fixes [\#1](https://github.com/gojek/feast/pull/1) ([pradithya](https://github.com/pradithya))
diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh
index bfd6545ee8e..49293826940 100755
--- a/infra/scripts/test-end-to-end-batch.sh
+++ b/infra/scripts/test-end-to-end-batch.sh
@@ -79,6 +79,9 @@ Building jars for Feast
 ============================================================
 "
 
+FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+echo Building Jars for version: $FEAST_BUILD_VERSION
+
 infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/
@@ -132,7 +135,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-${FEAST_BUILD_VERSION}.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 30
@@ -185,7 +188,7 @@ spring:
     web-environment: false
 EOF
 
-nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-${FEAST_BUILD_VERSION}.jar \
   --spring.config.location=file:///tmp/serving.warehouse.application.yml \
   &> /var/log/feast-serving-warehouse.log &
 sleep 15
diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
index 3f9f31e80cf..8843e2fa13a 100755
--- a/infra/scripts/test-end-to-end.sh
+++ b/infra/scripts/test-end-to-end.sh
@@ -62,6 +62,9 @@ Building jars for Feast
 ============================================================
 "
 
+FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+echo Building Jars for version: $FEAST_BUILD_VERSION
+
 infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/
@@ -118,7 +121,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-$FEAST_BUILD_VERSION.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 30
@@ -169,7 +172,7 @@ spring:
     web-environment: false
 EOF
 
-nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-$FEAST_BUILD_VERSION.jar \
   --spring.config.location=file:///tmp/serving.online.application.yml \
   &> /var/log/feast-serving-online.log &
 sleep 15
diff --git a/pom.xml b/pom.xml
index affdab061ac..e352aff2ad7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
     
 
     
-        0.3.7-SNAPSHOT
+        0.3.7
         https://github.com/gojek/feast
 
         UTF-8