From 8c576e1604354dc8716bb7c34989f7136fec284b Mon Sep 17 00:00:00 2001 From: pyalex Date: Sat, 1 Aug 2020 13:40:43 +0800 Subject: [PATCH 1/2] add checkArgument --- protos/feast/core/Store.proto | 5 +++-- .../connectors/bigquery/writer/BigQueryFeatureSink.java | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto index 3b4394150db..780d7a7db8b 100644 --- a/protos/feast/core/Store.proto +++ b/protos/feast/core/Store.proto @@ -108,7 +108,7 @@ message Store { int32 initial_backoff_ms = 3; // Optional. Maximum total number of retries for connecting to Redis. Default to zero retries. int32 max_retries = 4; - // Optional. how often flush data to redis + // Optional. How often flush data to redis int32 flush_frequency_seconds = 5; } @@ -118,6 +118,7 @@ message Store { string staging_location = 3; int32 initial_retry_delay_seconds = 4; int32 total_timeout_seconds = 5; + // Required. Frequency of running BQ load job and flushing all collected rows to BQ table int32 write_triggering_frequency_seconds = 6; } @@ -131,7 +132,7 @@ message Store { string connection_string = 1; int32 initial_backoff_ms = 2; int32 max_retries = 3; - // Optional. how often flush data to redis + // Optional. How often flush data to redis int32 flush_frequency_seconds = 4; } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java index ed5a7b020ee..2d55b308dd0 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java @@ -16,6 +16,8 @@ */ package feast.storage.connectors.bigquery.writer; +import static com.google.common.base.Preconditions.checkArgument; + import com.google.api.services.bigquery.model.TableSchema; import com.google.auto.value.AutoValue; import com.google.cloud.bigquery.*; @@ -62,6 +64,12 @@ public abstract class BigQueryFeatureSink implements FeatureSink { * @return {@link BigQueryFeatureSink.Builder} */ public static FeatureSink fromConfig(BigQueryConfig config) { + checkArgument( + config.getWriteTriggeringFrequencySeconds() > 0, + "Invalid configuration: " + + "write_triggering_frequency_seconds in BigQueryConfig must be positive integer. " + + "Please fix that in your serving configuration."); + return BigQueryFeatureSink.builder() .setDatasetId(config.getDatasetId()) .setProjectId(config.getProjectId()) From 2d218acb283e72adccfdd5e3a65adbe055645cec Mon Sep 17 00:00:00 2001 From: pyalex Date: Sat, 1 Aug 2020 14:17:08 +0800 Subject: [PATCH 2/2] add default write_triggering_frequency_seconds --- infra/charts/feast/README.md | 2 ++ infra/charts/feast/README.md.gotmpl | 1 + infra/charts/feast/values-batch-serving.yaml | 1 + infra/docker-compose/serving/batch-serving.yml | 1 + 4 files changed, 5 insertions(+) diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index f26950bd6f6..a78ffe4b926 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -164,6 +164,7 @@ feast-batch-serving: staging_location: gs:///feast-staging-location initial_retry_delay_seconds: 3 total_timeout_seconds: 21600 + write_triggering_frequency_seconds: 600 subscriptions: - name: "*" project: "*" @@ -280,6 +281,7 @@ feast-batch-serving: staging_location: gs:///feast-staging-location initial_retry_delay_seconds: 3 total_timeout_seconds: 21600 + write_triggering_frequency_seconds: 600 subscriptions: - name: "*" project: "*" diff --git a/infra/charts/feast/README.md.gotmpl b/infra/charts/feast/README.md.gotmpl index 75fc6661c80..d412aa01415 100644 --- a/infra/charts/feast/README.md.gotmpl +++ b/infra/charts/feast/README.md.gotmpl @@ -137,6 +137,7 @@ feast-batch-serving: staging_location: gs:///feast-staging-location initial_retry_delay_seconds: 3 total_timeout_seconds: 21600 + write_triggering_frequency_seconds: 600 subscriptions: - name: "*" project: "*" diff --git a/infra/charts/feast/values-batch-serving.yaml b/infra/charts/feast/values-batch-serving.yaml index 3ee35be1061..afa99f6d69b 100644 --- a/infra/charts/feast/values-batch-serving.yaml +++ b/infra/charts/feast/values-batch-serving.yaml @@ -20,6 +20,7 @@ feast-batch-serving: staging_location: gs:///feast-staging-location initial_retry_delay_seconds: 3 total_timeout_seconds: 21600 + write_triggering_frequency_seconds: 600 subscriptions: - name: "*" project: "*" diff --git a/infra/docker-compose/serving/batch-serving.yml b/infra/docker-compose/serving/batch-serving.yml index c34aba277c7..3feb81c84e1 100644 --- a/infra/docker-compose/serving/batch-serving.yml +++ b/infra/docker-compose/serving/batch-serving.yml @@ -12,6 +12,7 @@ feast: staging_location: gs://gcs_bucket/prefix initial_retry_delay_seconds: 1 total_timeout_seconds: 21600 + write_triggering_frequency_seconds: 600 subscriptions: - name: "*" project: "*"