From 5a6edc4f17c68d1de5781460cd18940a6bf6750c Mon Sep 17 00:00:00 2001 From: Tim Sell <63295+tims@users.noreply.github.com> Date: Mon, 11 Feb 2019 23:08:10 +0800 Subject: [PATCH 01/25] Coalesce rows (#89) * ptransform to coalesce feature rows in batch and streaming * add coalesce feature rows transform, to build state of latest row per key in the global window * coalesce for serving on entity name and key only, preparing for removing history from serving stores * change to rounding granularities after feature row coalesce * rename importSpec.options to be importspec.sourceOptions and add jobOptions * only write out the latest row for bigtable and remove postgres feature store * do not write historical values to redis serving store * formatting and fix test * core importspec validator test * fix typo on option name * use Timestamps.compare * remove redudant proto message * default coalescerows to true * Update go protos and cli tests * Update python sdk for new import spec --- cli/feast/pkg/parse/yaml.go | 6 - cli/feast/pkg/parse/yaml_test.go | 55 +- .../main/java/feast/core/model/JobInfo.java | 11 +- .../feast/core/validators/SpecValidator.java | 34 +- .../core/job/ScheduledJobMonitorTest.java | 2 + .../java/feast/core/model/JobInfoTest.java | 11 +- .../service/JobManagementServiceTest.java | 3 + .../core/validators/SpecValidatorTest.java | 71 ++- ingestion/pom.xml | 12 - .../main/java/feast/ingestion/ImportJob.java | 77 +-- .../feast/ingestion/boot/ImportJobModule.java | 10 +- .../ingestion/config/ImportSpecSupplier.java | 6 +- .../feast/ingestion/metrics/FeastMetrics.java | 76 +-- .../metrics/FeastMetricsPubSubSink.java | 110 ---- ...ons.java => ImportJobPipelineOptions.java} | 22 +- .../feast/ingestion/options/JobOptions.java | 72 +++ .../transform/CoalesceFeatureRows.java | 280 +++++++++ .../transform/CoalescePFeatureRows.java | 47 ++ .../transform/ErrorsStoreTransform.java | 4 +- .../transform/fn/ConvertTypesDoFn.java | 5 +- .../feast/ingestion/values/PFeatureRows.java | 1 - .../bigquery/BigQueryFeatureSource.java | 2 +- .../source/csv/CsvFileFeatureSource.java | 11 +- .../source/json/JsonFileFeatureSource.java | 2 +- .../source/kafka/KafkaFeatureSource.java | 2 +- .../source/pubsub/PubSubFeatureSource.java | 2 +- .../bigtable/FeatureRowBigTableIO.java | 19 +- .../FeatureRowToBigTableMutationDoFn.java | 63 +- .../storage/postgres/FeatureRowJdbcIO.java | 199 ------ .../storage/postgres/PostgresOptions.java | 38 -- .../postgres/PostgresServingStore.java | 47 -- .../redis/FeatureRowToRedisMutationDoFn.java | 38 +- .../feast_ingestion/types/CoalesceAccum.proto | 37 ++ .../feast_ingestion/types/CoalesceKey.proto | 25 + .../proto/feast_ingestion/types/README.md | 1 + .../src/test/java/feast/FeastMatchers.java | 2 +- .../test/java/feast/NormalizeFeatureRows.java | 57 ++ ingestion/src/test/java/feast/TestHelper.java | 3 - .../feast/ingestion/ImportJobCSVTest.java | 177 +++++- .../config/ImportSpecSupplierTest.java | 23 +- .../ingestion/options/JobOptionsTest.java | 89 +++ .../transform/CoalesceFeatureRowsTest.java | 568 ++++++++++++++++++ .../transform/ErrorsStoreTransformTest.java | 6 +- .../storage/postgres/PostgresOptionsTest.java | 137 ----- .../redis/FeatureRowRedisIOWriteTest.java | 52 -- .../FeatureRowToRedisMutationDoFnTest.java | 7 +- .../storage/redis/RedisStoreOptionsTest.java | 2 +- ingestion/src/test/resources/logback-test.xml | 28 + protos/feast/specs/ImportSpec.proto | 12 +- .../go/feast/core/DatasetService.pb.go | 388 ++++++++++++ .../generated/go/feast/specs/ImportSpec.pb.go | 95 +-- .../generated/go/feast/storage/BigTable.pb.go | 34 +- protos/generated/go/feast/types/Feature.pb.go | 33 +- .../generated/go/feast/types/FeatureRow.pb.go | 54 +- sdk/python/feast/sdk/importer.py | 43 +- sdk/python/feast/specs/ImportSpec_pb2.py | 103 +++- sdk/python/tests/sdk/test_importer.py | 4 +- 57 files changed, 2218 insertions(+), 1100 deletions(-) delete mode 100644 ingestion/src/main/java/feast/ingestion/metrics/FeastMetricsPubSubSink.java rename ingestion/src/main/java/feast/ingestion/options/{ImportJobOptions.java => ImportJobPipelineOptions.java} (81%) create mode 100644 ingestion/src/main/java/feast/ingestion/options/JobOptions.java create mode 100644 ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java create mode 100644 ingestion/src/main/java/feast/ingestion/transform/CoalescePFeatureRows.java delete mode 100644 ingestion/src/main/java/feast/storage/postgres/FeatureRowJdbcIO.java delete mode 100644 ingestion/src/main/java/feast/storage/postgres/PostgresOptions.java delete mode 100644 ingestion/src/main/java/feast/storage/postgres/PostgresServingStore.java create mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto create mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto create mode 100644 ingestion/src/main/proto/feast_ingestion/types/README.md create mode 100644 ingestion/src/test/java/feast/NormalizeFeatureRows.java create mode 100644 ingestion/src/test/java/feast/ingestion/options/JobOptionsTest.java create mode 100644 ingestion/src/test/java/feast/ingestion/transform/CoalesceFeatureRowsTest.java delete mode 100644 ingestion/src/test/java/feast/storage/postgres/PostgresOptionsTest.java create mode 100644 ingestion/src/test/resources/logback-test.xml create mode 100644 protos/generated/go/feast/core/DatasetService.pb.go diff --git a/cli/feast/pkg/parse/yaml.go b/cli/feast/pkg/parse/yaml.go index 503f0acb494..f02698171e1 100644 --- a/cli/feast/pkg/parse/yaml.go +++ b/cli/feast/pkg/parse/yaml.go @@ -16,7 +16,6 @@ package parse import ( "encoding/json" - "fmt" "time" "github.com/golang/protobuf/ptypes/timestamp" @@ -98,11 +97,6 @@ func YamlToImportSpec(in []byte) (*specs.ImportSpec, error) { return nil, err } - // schema must be available for 'file' or 'bigquery' - if (ymlMap["type"] == "file" || ymlMap["type"] == "bigquery") && ymlMap["schema"] == nil { - return nil, fmt.Errorf("Schema must be specified for importing data from file or BigQuery") - } - // either timestampValue or timestampColumn var timestampValue *timestamp.Timestamp var timestampColumn string diff --git a/cli/feast/pkg/parse/yaml_test.go b/cli/feast/pkg/parse/yaml_test.go index fbb43ef8ee9..00331668ea3 100644 --- a/cli/feast/pkg/parse/yaml_test.go +++ b/cli/feast/pkg/parse/yaml_test.go @@ -243,7 +243,9 @@ func TestYamlToImportSpec(t *testing.T) { { name: "valid yaml", input: []byte(`type: file -options: +jobOptions: + coalesceRows.enabled: "true" +sourceOptions: format: csv path: jaeger_last_opportunity_sample.csv entities: @@ -258,7 +260,10 @@ schema: featureId: driver.none.last_opportunity`), expected: &specs.ImportSpec{ Type: "file", - Options: map[string]string{ + JobOptions: map[string]string{ + "coalesceRows.enabled": "true", + }, + SourceOptions: map[string]string{ "format": "csv", "path": "jaeger_last_opportunity_sample.csv", }, @@ -299,49 +304,3 @@ schema: }) } } - -func TestYamlToImportSpecNoSchema(t *testing.T) { - tt := []struct { - name string - input []byte - expected *specs.ImportSpec - err error - }{ - { - name: "valid yaml", - input: []byte(`type: pubsub -options: - topic: projects/your-gcp-project/topics/feast-test -entities: - - testentity`), - expected: &specs.ImportSpec{ - Type: "pubsub", - Options: map[string]string{ - "topic": "projects/your-gcp-project/topics/feast-test", - }, - Entities: []string{"testentity"}, - }, - err: nil, - }, - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - spec, err := YamlToImportSpec(tc.input) - if tc.err == nil { - if err != nil { - t.Error(err) - } else if !cmp.Equal(spec, tc.expected) { - t.Errorf("Expected %s, got %s", tc.expected, spec) - } - } else { - // we expect an error - if err == nil { - t.Error(err) - } else if err.Error() != tc.err.Error() { - t.Errorf("Expected error %v, got %v", err.Error(), tc.err.Error()) - } - } - }) - } -} diff --git a/core/src/main/java/feast/core/model/JobInfo.java b/core/src/main/java/feast/core/model/JobInfo.java index cf6d7a5b781..d622fdff53c 100644 --- a/core/src/main/java/feast/core/model/JobInfo.java +++ b/core/src/main/java/feast/core/model/JobInfo.java @@ -55,8 +55,12 @@ public class JobInfo extends AbstractTimestampEntity { private String runner; // Job options. Stored as a json string as it is specific to the runner. - @Column(name = "options") - private String options; + @Column(name = "source_options") + private String sourceOptions; + + // Job options. Stored as a json string as it is specific to the runner. + @Column(name = "job_options") + private String jobOptions; // Entities populated by the job @ManyToMany @@ -99,7 +103,8 @@ public JobInfo( this.extId = extId; this.type = importSpec.getType(); this.runner = runner; - this.options = TypeConversion.convertMapToJsonString(importSpec.getOptionsMap()); + this.sourceOptions = TypeConversion.convertMapToJsonString(importSpec.getSourceOptionsMap()); + this.jobOptions = TypeConversion.convertMapToJsonString(importSpec.getJobOptionsMap()); this.entities = new ArrayList<>(); for (String entity : importSpec.getEntitiesList()) { EntityInfo entityInfo = new EntityInfo(); diff --git a/core/src/main/java/feast/core/validators/SpecValidator.java b/core/src/main/java/feast/core/validators/SpecValidator.java index c8500ac331b..18095b25914 100644 --- a/core/src/main/java/feast/core/validators/SpecValidator.java +++ b/core/src/main/java/feast/core/validators/SpecValidator.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static feast.core.validators.Matchers.checkLowerSnakeCase; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import feast.core.dao.EntityInfoRepository; @@ -40,6 +41,8 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.StorageSpecProto.StorageSpec; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -256,6 +259,19 @@ public void validateImportSpec(ImportSpec spec) throws IllegalArgumentException entityInfoRepository.existsById(name), Strings.lenientFormat("Entity %s not registered", name)); } + Map jobOptions = spec.getJobOptionsMap(); + if (jobOptions.size() > 0) { + List opts = Lists.newArrayList( + "sample.limit", + "coalesceRows.enabled", + "coalesceRows.delaySeconds", + "coalesceRows.timeoutSeconds" + ); + for (String key : jobOptions.keySet()) { + Preconditions.checkArgument(opts.contains(key), + Strings.lenientFormat("Option %s is not a valid jobOption", key)); + } + } } catch (NullPointerException | IllegalArgumentException e) { throw new IllegalArgumentException( Strings.lenientFormat("Validation for import spec failed: %s", e.getMessage())); @@ -264,8 +280,8 @@ public void validateImportSpec(ImportSpec spec) throws IllegalArgumentException private void checkKafkaImportSpecOption(ImportSpec spec) { try { - String topics = spec.getOptionsOrDefault("topics", ""); - String server = spec.getOptionsOrDefault("server", ""); + String topics = spec.getSourceOptionsOrDefault("topics", ""); + String server = spec.getSourceOptionsOrDefault("server", ""); if (topics.equals("") && server.equals("")) { throw new IllegalArgumentException( "Kafka ingestion requires either topics or servers"); @@ -278,7 +294,8 @@ private void checkKafkaImportSpecOption(ImportSpec spec) { private void checkFileImportSpecOption(ImportSpec spec) throws IllegalArgumentException { try { - checkArgument(!spec.getOptionsOrDefault("path", "").equals(""), "File path cannot be empty"); + checkArgument(!spec.getSourceOptionsOrDefault("path", "").equals(""), + "File path cannot be empty"); } catch (NullPointerException | IllegalArgumentException e) { throw new IllegalArgumentException( Strings.lenientFormat("Invalid options: %s", e.getMessage())); @@ -287,8 +304,8 @@ private void checkFileImportSpecOption(ImportSpec spec) throws IllegalArgumentEx private void checkPubSubImportSpecOption(ImportSpec spec) throws IllegalArgumentException { try { - String topic = spec.getOptionsOrDefault("topic", ""); - String subscription = spec.getOptionsOrDefault("subscription", ""); + String topic = spec.getSourceOptionsOrDefault("topic", ""); + String subscription = spec.getSourceOptionsOrDefault("subscription", ""); if (topic.equals("") && subscription.equals("")) { throw new IllegalArgumentException( "Pubsub ingestion requires either topic or subscription"); @@ -301,11 +318,12 @@ private void checkPubSubImportSpecOption(ImportSpec spec) throws IllegalArgument private void checkBigqueryImportSpecOption(ImportSpec spec) throws IllegalArgumentException { try { - checkArgument(!spec.getOptionsOrThrow("project").equals(""), + checkArgument(!spec.getSourceOptionsOrThrow("project").equals(""), "Bigquery project cannot be empty"); - checkArgument(!spec.getOptionsOrThrow("dataset").equals(""), + checkArgument(!spec.getSourceOptionsOrThrow("dataset").equals(""), "Bigquery dataset cannot be empty"); - checkArgument(!spec.getOptionsOrThrow("table").equals(""), "Bigquery table cannot be empty"); + checkArgument(!spec.getSourceOptionsOrThrow("table").equals(""), + "Bigquery table cannot be empty"); } catch (NullPointerException | IllegalArgumentException e) { throw new IllegalArgumentException( Strings.lenientFormat("Invalid options: %s", e.getMessage())); diff --git a/core/src/test/java/feast/core/job/ScheduledJobMonitorTest.java b/core/src/test/java/feast/core/job/ScheduledJobMonitorTest.java index d352044d16e..1a05869657f 100644 --- a/core/src/test/java/feast/core/job/ScheduledJobMonitorTest.java +++ b/core/src/test/java/feast/core/job/ScheduledJobMonitorTest.java @@ -66,6 +66,7 @@ public void getJobStatus_shouldUpdateJobInfoForRunningJob() { "Streaming", "DataflowRunner", "", + "", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), @@ -104,6 +105,7 @@ public void getJobMetrics_shouldPushToStatsDMetricPusherAndSaveNewMetricToDb() { "Streaming", "DataflowRunner", "", + "", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), diff --git a/core/src/test/java/feast/core/model/JobInfoTest.java b/core/src/test/java/feast/core/model/JobInfoTest.java index a75050d6022..f47d8435f83 100644 --- a/core/src/test/java/feast/core/model/JobInfoTest.java +++ b/core/src/test/java/feast/core/model/JobInfoTest.java @@ -41,9 +41,8 @@ public void shouldInitialiseGivenJobIdAndSpec() throws InvalidProtocolBufferExce .build(); ImportSpecProto.ImportSpec importSpec = ImportSpecProto.ImportSpec.newBuilder() - .setType("file") - .putOptions("format", "csv") - .putOptions("path", "gs://some/path") + .setType("file.csv") + .putSourceOptions("path", "gs://some/path") .addEntities("entity") .setSchema(schema) .build(); @@ -52,9 +51,9 @@ public void shouldInitialiseGivenJobIdAndSpec() throws InvalidProtocolBufferExce JobInfo expected = new JobInfo(); expected.setId("fake-job-id"); expected.setExtId("fake-ext-id"); - expected.setType("file"); + expected.setType("file.csv"); expected.setRunner("DataflowRunner"); - expected.setOptions(TypeConversion.convertMapToJsonString(importSpec.getOptionsMap())); + expected.setSourceOptions(TypeConversion.convertMapToJsonString(importSpec.getSourceOptionsMap())); List entities = new ArrayList<>(); EntityInfo entityInfo = new EntityInfo(); @@ -75,7 +74,7 @@ public void shouldInitialiseGivenJobIdAndSpec() throws InvalidProtocolBufferExce assertThat(actual.getRunner(), equalTo(expected.getRunner())); assertThat(actual.getEntities(), equalTo(expected.getEntities())); assertThat(actual.getFeatures(), equalTo(expected.getFeatures())); - assertThat(actual.getOptions(), equalTo(expected.getOptions())); + assertThat(actual.getSourceOptions(), equalTo(expected.getSourceOptions())); assertThat(actual.getRaw(), equalTo(expected.getRaw())); } } \ No newline at end of file diff --git a/core/src/test/java/feast/core/service/JobManagementServiceTest.java b/core/src/test/java/feast/core/service/JobManagementServiceTest.java index 709cf365b77..1902ccc578d 100644 --- a/core/src/test/java/feast/core/service/JobManagementServiceTest.java +++ b/core/src/test/java/feast/core/service/JobManagementServiceTest.java @@ -70,6 +70,7 @@ public void shouldListAllJobDetails() { "", "", "", + "", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), @@ -84,6 +85,7 @@ public void shouldListAllJobDetails() { "", "", "", + "", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), @@ -121,6 +123,7 @@ public void shouldReturnDetailOfRequestedJobId() { "", "", "", + "", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), diff --git a/core/src/test/java/feast/core/validators/SpecValidatorTest.java b/core/src/test/java/feast/core/validators/SpecValidatorTest.java index 0c666a11a6d..052fa609508 100644 --- a/core/src/test/java/feast/core/validators/SpecValidatorTest.java +++ b/core/src/test/java/feast/core/validators/SpecValidatorTest.java @@ -709,7 +709,7 @@ public void fileImportSpecWithoutEntityIdColumnInSchemaShouldThrowIllegalArgumen ImportSpec input = ImportSpec.newBuilder() .setType("file.csv") - .putOptions("path", "gs://asdasd") + .putSourceOptions("path", "gs://asdasd") .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage( @@ -728,9 +728,9 @@ public void bigQueryImportSpecWithoutEntityIdColumnInSchemaShouldThrowIllegalArg ImportSpec input = ImportSpec.newBuilder() .setType("bigquery") - .putOptions("project", "my-google-project") - .putOptions("dataset", "feast") - .putOptions("table", "feast") + .putSourceOptions("project", "my-google-project") + .putSourceOptions("dataset", "feast") + .putSourceOptions("table", "feast") .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage( @@ -750,7 +750,7 @@ public void importSpecWithoutValidEntityShouldThrowIllegalArgumentException() { ImportSpec input = ImportSpec.newBuilder() .setType("pubsub") - .putOptions("topic", "my/pubsub/topic") + .putSourceOptions("topic", "my/pubsub/topic") .addEntities("someEntity") .build(); exception.expect(IllegalArgumentException.class); @@ -775,7 +775,7 @@ public void importSpecWithUnregisteredFeaturesShouldThrowIllegalArgumentExceptio ImportSpec input = ImportSpec.newBuilder() .setType("pubsub") - .putOptions("topic", "my/pubsub/topic") + .putSourceOptions("topic", "my/pubsub/topic") .setSchema(schema) .addEntities("someEntity") .build(); @@ -802,8 +802,63 @@ public void importSpecWithKafkaSourceAndCorrectOptionsShouldPassValidation() { ImportSpec input = ImportSpec.newBuilder() .setType("kafka") - .putOptions("topics", "my-kafka-topic") - .putOptions("server", "localhost:54321") + .putSourceOptions("topics", "my-kafka-topic") + .putSourceOptions("server", "localhost:54321") + .setSchema(schema) + .addEntities("someEntity") + .build(); + validator.validateImportSpec(input); + } + + @Test + public void importSpecWithCoalesceJobOptionsShouldPassValidation() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + when(featureInfoRepository.existsById("some_existing_feature")).thenReturn(true); + when(entityInfoRepository.existsById("someEntity")).thenReturn(true); + Schema schema = + Schema.newBuilder() + .addFields(Field.newBuilder().setFeatureId("some_existing_feature").build()) + .build(); + ImportSpec input = + ImportSpec.newBuilder() + .setType("kafka") + .putSourceOptions("topics", "my-kafka-topic") + .putSourceOptions("server", "localhost:54321") + .putJobOptions("coalesceRows.enabled", "true") + .putJobOptions("coalesceRows.delaySeconds", "10000") + .putJobOptions("coalesceRows.timeoutSeconds", "20000") + .putJobOptions("sample.limit", "1000") + .setSchema(schema) + .addEntities("someEntity") + .build(); + validator.validateImportSpec(input); + } + + @Test + public void importSpecWithLimitJobOptionsShouldPassValidation() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + when(featureInfoRepository.existsById("some_existing_feature")).thenReturn(true); + when(entityInfoRepository.existsById("someEntity")).thenReturn(true); + Schema schema = + Schema.newBuilder() + .addFields(Field.newBuilder().setFeatureId("some_existing_feature").build()) + .build(); + ImportSpec input = + ImportSpec.newBuilder() + .setType("kafka") + .putSourceOptions("topics", "my-kafka-topic") + .putSourceOptions("server", "localhost:54321") + .putJobOptions("sample.limit", "1000") .setSchema(schema) .addEntities("someEntity") .build(); diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 69823921a9c..3abce057dd1 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -226,12 +226,6 @@ ${org.apache.beam.version} - - org.apache.beam - beam-sdks-java-io-jdbc - ${org.apache.beam.version} - - org.apache.beam beam-sdks-java-io-kafka @@ -302,12 +296,6 @@ 5.87.0.RELEASE - - org.postgresql - postgresql - 42.2.5 - - com.github.kstyrc embedded-redis diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index f048e5a5854..32420698253 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -18,7 +18,6 @@ package feast.ingestion; import com.google.api.services.bigquery.model.TableRow; -import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -28,7 +27,9 @@ import feast.ingestion.boot.PipelineModule; import feast.ingestion.config.ImportSpecSupplier; import feast.ingestion.model.Specs; -import feast.ingestion.options.ImportJobOptions; +import feast.ingestion.options.ImportJobPipelineOptions; +import feast.ingestion.options.JobOptions; +import feast.ingestion.transform.CoalescePFeatureRows; import feast.ingestion.transform.ErrorsStoreTransform; import feast.ingestion.transform.ReadFeaturesTransform; import feast.ingestion.transform.ServingStoreTransform; @@ -39,11 +40,11 @@ import feast.ingestion.transform.fn.LoggerDoFn; import feast.ingestion.transform.fn.RoundEventTimestampsDoFn; import feast.ingestion.values.PFeatureRows; +import feast.options.OptionsParser; import feast.specs.ImportSpecProto.ImportSpec; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import java.util.Arrays; -import java.util.List; import java.util.Random; import lombok.extern.slf4j.Slf4j; import org.apache.beam.runners.dataflow.DataflowPipelineJob; @@ -55,7 +56,6 @@ import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.Sample; import org.apache.beam.sdk.transforms.windowing.AfterWatermark; @@ -63,7 +63,6 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollection.IsBounded; -import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.commons.codec.digest.DigestUtils; import org.joda.time.DateTime; @@ -82,7 +81,7 @@ public class ImportJob { private final WarehouseStoreTransform warehouseStoreTransform; private final ErrorsStoreTransform errorsStoreTransform; private final boolean dryRun; - private final ImportJobOptions options; + private final ImportJobPipelineOptions options; private final Specs specs; @Inject @@ -93,7 +92,7 @@ public ImportJob( ServingStoreTransform servingStoreTransform, WarehouseStoreTransform warehouseStoreTransform, ErrorsStoreTransform errorsStoreTransform, - ImportJobOptions options, + ImportJobPipelineOptions options, Specs specs) { this.pipeline = pipeline; this.importSpec = importSpec; @@ -112,8 +111,8 @@ public static void main(String[] args) { public static PipelineResult mainWithResult(String[] args) { log.info("Arguments: " + Arrays.toString(args)); - ImportJobOptions options = - PipelineOptionsFactory.fromArgs(args).withValidation().as(ImportJobOptions.class); + ImportJobPipelineOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(ImportJobPipelineOptions.class); if (options.getJobName().isEmpty()) { options.setJobName(generateName()); } @@ -142,17 +141,18 @@ public void expand() { TypeDescriptor.of(FeatureRowExtended.class), ProtoCoder.of(FeatureRowExtended.class)); coderRegistry.registerCoderForType(TypeDescriptor.of(TableRow.class), TableRowJsonCoder.of()); + JobOptions jobOptions = OptionsParser.parse(importSpec.getJobOptionsMap(), JobOptions.class); + try { log.info(JsonFormat.printer().print(importSpec)); } catch (InvalidProtocolBufferException e) { // pass } - specs.validate(); PCollection features = pipeline.apply("Read", readFeaturesTransform); - if (options.getLimit() != null && options.getLimit() > 0) { - features = features.apply(Sample.any(options.getLimit())); + if (jobOptions.getSampleLimit() > 0) { + features = features.apply(Sample.any(jobOptions.getSampleLimit())); } PCollection featuresExtended = @@ -161,32 +161,36 @@ public void expand() { PFeatureRows pFeatureRows = PFeatureRows.of(featuresExtended); pFeatureRows = pFeatureRows.applyDoFn("Convert feature types", new ConvertTypesDoFn(specs)); pFeatureRows = pFeatureRows.apply("Validate features", new ValidateTransform(specs)); - pFeatureRows = + + log.info( + "A sample of size 1 of incoming rows from MAIN and ERRORS will logged every 30 seconds for visibility"); + logNRows(pFeatureRows, "Output sample", 1, Duration.standardSeconds(30)); + + PFeatureRows warehouseRows = roundTimestamps("Round timestamps for warehouse", pFeatureRows); + PFeatureRows servingRows = pFeatureRows; + if (jobOptions.isCoalesceRowsEnabled()) { + // Should we merge and dedupe rows before writing to the serving store? + servingRows = servingRows.apply("Coalesce Rows", new CoalescePFeatureRows( + jobOptions.getCoalesceRowsDelaySeconds(), + jobOptions.getCoalesceRowsTimeoutSeconds())); + } + servingRows = roundTimestamps("Round timestamps for serving", servingRows); + + if (!dryRun) { + servingRows.apply("Write to Serving Stores", servingStoreTransform); + warehouseRows.apply("Write to Warehouse Stores", warehouseStoreTransform); + pFeatureRows.getErrors().apply("Write errors", errorsStoreTransform); + } + } + + public PFeatureRows roundTimestamps(String name, PFeatureRows pFeatureRows) { + return PFeatureRows.of( pFeatureRows .getMain() - .apply( - "Round event timestamps to granularity", + .apply(name, ParDo.of(new RoundEventTimestampsDoFn())), pFeatureRows.getErrors()); - - if (!dryRun) { - List> errors = Lists.newArrayList(); - pFeatureRows = pFeatureRows.apply("Write to Serving Stores", servingStoreTransform); - errors.add(pFeatureRows.getErrors()); - pFeatureRows = PFeatureRows.of(pFeatureRows.getMain()); - - log.info( - "A sample of any 2 rows from each of MAIN, RETRIES and ERRORS will logged for convenience"); - logNRows(pFeatureRows, "Output sample", 2); - - PFeatureRows.of(pFeatureRows.getMain()) - .apply("Write to Warehouse Stores", warehouseStoreTransform); - errors.add(pFeatureRows.getErrors()); - - PCollectionList.of(errors).apply("flatten errors", Flatten.pCollections()) - .apply("Write serving errors", errorsStoreTransform); - } } public PipelineResult run() { @@ -195,17 +199,16 @@ public PipelineResult run() { return result; } - public void logNRows(PFeatureRows pFeatureRows, String name, int limit) { + public void logNRows(PFeatureRows pFeatureRows, String name, long limit, Duration period) { PCollection main = pFeatureRows.getMain(); PCollection errors = pFeatureRows.getErrors(); if (main.isBounded().equals(IsBounded.UNBOUNDED)) { Window minuteWindow = - Window.into(FixedWindows.of(Duration.standardMinutes(1L))) + Window.into(FixedWindows.of(period)) .triggering(AfterWatermark.pastEndOfWindow()) .discardingFiredPanes() - .withAllowedLateness(Duration.standardMinutes(1)); - + .withAllowedLateness(Duration.ZERO); main = main.apply(minuteWindow); errors = errors.apply(minuteWindow); } diff --git a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java index ce3bb622072..7e9e235b5bc 100644 --- a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java +++ b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java @@ -21,7 +21,7 @@ import com.google.inject.Provides; import com.google.inject.Singleton; import feast.ingestion.model.Specs; -import feast.ingestion.options.ImportJobOptions; +import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.service.CoreSpecService; import feast.ingestion.service.FileSpecService; import feast.ingestion.service.SpecService; @@ -42,24 +42,24 @@ */ public class ImportJobModule extends AbstractModule { - private final ImportJobOptions options; + private final ImportJobPipelineOptions options; private ImportSpec importSpec; - public ImportJobModule(ImportJobOptions options, ImportSpec importSpec) { + public ImportJobModule(ImportJobPipelineOptions options, ImportSpec importSpec) { this.options = options; this.importSpec = importSpec; } @Override protected void configure() { - bind(ImportJobOptions.class).toInstance(options); + bind(ImportJobPipelineOptions.class).toInstance(options); bind(PipelineOptions.class).toInstance(options); bind(ImportSpec.class).toInstance(importSpec); } @Provides @Singleton - Builder provideSpecService(ImportJobOptions options) { + Builder provideSpecService(ImportJobPipelineOptions options) { if (options.getCoreApiUri() != null) { return new CoreSpecService.Builder(options.getCoreApiUri()); } else if (options.getCoreApiSpecPath() != null) { diff --git a/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java b/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java index 04d2e128597..ba1b431bfae 100644 --- a/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java +++ b/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java @@ -22,17 +22,17 @@ import java.nio.file.Paths; import java.util.Base64; import java.util.function.Supplier; -import feast.ingestion.options.ImportJobOptions; +import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.util.ProtoUtil; import feast.specs.ImportSpecProto.ImportSpec; public class ImportSpecSupplier implements Supplier { - private ImportJobOptions options; + private ImportJobPipelineOptions options; private ImportSpec importSpec; @Inject - public ImportSpecSupplier(ImportJobOptions options) { + public ImportSpecSupplier(ImportJobPipelineOptions options) { this.options = options; } diff --git a/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java b/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java index d6463e7e5c7..c357454dbdf 100644 --- a/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java +++ b/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java @@ -17,34 +17,22 @@ package feast.ingestion.metrics; -import feast.types.FeatureRowProto; -import lombok.AllArgsConstructor; -import org.apache.beam.sdk.metrics.MetricResult; -import org.apache.beam.sdk.metrics.Metrics; -import org.apache.beam.sdk.transforms.DoFn; -import org.joda.time.DateTime; -import feast.ingestion.model.Features; -import feast.ingestion.model.Values; import feast.ingestion.util.DateUtil; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import feast.types.FeatureRowProto; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; -import feast.types.GranularityProto.Granularity.Enum; - import java.time.Instant; -import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.transforms.DoFn; public class FeastMetrics { - public static final String METRICS_ENTITY_NAME = "metrics"; - public static final String METRICS_FEATURE_JOB_ID = "metrics.hour.job_id"; - public static final String METRICS_FEATURE_NAMESPACE = "metrics.hour.namespace"; - public static final String METRICS_FEATURE_STEP = "metrics.hour.step"; - public static final String METRICS_FEATURE_NAME = "metrics.hour.name"; - public static final String METRICS_FEATURE_ATTEMPTED = "metrics.hour.attempted"; + public static final String FEAST_NAMESPACE = "feast"; - private FeastMetrics() {} + private FeastMetrics() { + } private static void inc(String name) { Metrics.counter(FeastMetrics.FEAST_NAMESPACE, name).inc(); @@ -70,41 +58,9 @@ public static CalculateLagMetricFunc lagUpdateDoFn() { return new CalculateLagMetricFunc(); } - /** - * Create a feature row from Metrics. - * The granularity is unrelated to the metrics themselves, and simply indicates the - * granularity at which to store and overwrite the metrics downstream. - * - * @param counter - * @param jobName - * @param granularity - * @return - */ - public static FeatureRow makeFeatureRow( - MetricResult counter, String jobName, Granularity.Enum granularity) { - String jobId = jobName; - String namespace = counter.getName().getNamespace(); - String step = counter.getStep(); - String name = counter.getName().getName(); - Long attempted = counter.getAttempted(); - - String entityId = String.join(":", new String[] {jobId, namespace, step, name}); - - return FeatureRow.newBuilder() - .setEntityName(METRICS_ENTITY_NAME) - .setEntityKey(entityId) - .setGranularity(granularity) - .setEventTimestamp(DateUtil.toTimestamp(DateTime.now())) - .addFeatures(Features.of(METRICS_FEATURE_JOB_ID, Values.ofString(jobId))) - .addFeatures(Features.of(METRICS_FEATURE_NAMESPACE, Values.ofString(namespace))) - .addFeatures(Features.of(METRICS_FEATURE_STEP, Values.ofString(step))) - .addFeatures(Features.of(METRICS_FEATURE_NAME, Values.ofString(name))) - .addFeatures(Features.of(METRICS_FEATURE_ATTEMPTED, Values.ofInt64(attempted))) - .build(); - } - @AllArgsConstructor public static class IncrRowExtendedFunc extends DoFn { + private String suffix; @ProcessElement @@ -117,18 +73,20 @@ public void processElement( @AllArgsConstructor public static class CalculateLagMetricFunc extends DoFn { + @ProcessElement - public void processElement(@Element FeatureRowExtended element, OutputReceiver out) { + public void processElement(@Element FeatureRowExtended element, + OutputReceiver out) { FeatureRowProto.FeatureRow row = element.getRow(); com.google.protobuf.Timestamp eventTimestamp = row.getEventTimestamp(); Instant now = Instant.now(); com.google.protobuf.Timestamp roundedCurrentTimestamp = - DateUtil.roundToGranularity( - com.google.protobuf.Timestamp.newBuilder() - .setSeconds(now.getEpochSecond()) - .setNanos(now.getNano()) - .build(), - row.getGranularity()); + DateUtil.roundToGranularity( + com.google.protobuf.Timestamp.newBuilder() + .setSeconds(now.getEpochSecond()) + .setNanos(now.getNano()) + .build(), + row.getGranularity()); long lagSeconds = roundedCurrentTimestamp.getSeconds() - eventTimestamp.getSeconds(); FeastMetrics.update("row:lag", lagSeconds); out.output(element); diff --git a/ingestion/src/main/java/feast/ingestion/metrics/FeastMetricsPubSubSink.java b/ingestion/src/main/java/feast/ingestion/metrics/FeastMetricsPubSubSink.java deleted file mode 100644 index 06b35a93cb7..00000000000 --- a/ingestion/src/main/java/feast/ingestion/metrics/FeastMetricsPubSubSink.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.metrics; - -import com.google.auto.service.AutoService; -import com.google.common.base.Preconditions; -import com.google.protobuf.util.JsonFormat; -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; -import lombok.extern.slf4j.Slf4j; -import org.apache.beam.sdk.io.gcp.pubsub.FeastPubsubHelper; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; -import org.apache.beam.sdk.metrics.MetricQueryResults; -import org.apache.beam.sdk.metrics.MetricResult; -import org.apache.beam.sdk.metrics.MetricsSink; -import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import feast.ingestion.options.ImportJobOptions; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; - -/** - * MetricsSink class for dogfooding metrics back into feast as FeatureRows with Google Cloud PubSub. - */ -@Slf4j -public class FeastMetricsPubSubSink implements MetricsSink { - - private FeastPubsubHelper pubsubHelper; - private String jobName; - private String topicUrl; - private Granularity.Enum granularity; - - public FeastMetricsPubSubSink(PipelineOptions pipelineOptions) { - this.pubsubHelper = new FeastPubsubHelper(pipelineOptions); - this.jobName = pipelineOptions.getJobName(); - FeastMetricsPubSubSinkOptions sinkOptions = - pipelineOptions.as(FeastMetricsPubSubSinkOptions.class); - this.topicUrl = sinkOptions.getMetricsPubSubSinkTopicUrl(); - - String granularityString = - Optional.ofNullable(sinkOptions.getMetricsPubSubSinkGranularity()).orElse("HOUR"); - this.granularity = Granularity.Enum.valueOf(granularityString.toUpperCase()); - - Preconditions.checkNotNull(topicUrl, "FeastMetricsPubSubSink requires pubsub topic url"); - } - - /** - * Publish metrics as FeatureRows to PubSub - * - * @param metricQueryResults - * @throws Exception - */ - @Override - public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception { - try { - for (MetricResult counter : metricQueryResults.getCounters()) { - FeatureRow row = FeastMetrics.makeFeatureRow(counter, jobName, granularity); - log.info(JsonFormat.printer().print(row)); - publish(topicUrl, row); - } - } catch (Throwable e) { - log.error(e.getMessage(), e); - } - } - - void publish(String pubSubTopicUrl, FeatureRow row) throws IOException { - TopicPath topicPath = PubsubClient.topicPathFromPath(pubSubTopicUrl); - pubsubHelper.publish(topicPath, row); - } - - public interface FeastMetricsPubSubSinkOptions extends PipelineOptions { - @Description("PubSub topic to write stats to") - String getMetricsPubSubSinkTopicUrl(); - - void setMetricsPubSubSinkTopicUrl(String value); - - @Description( - "Granularity to be set on published metrics FeatureRows, this effects storage" - + " downstream,it does not indicate any aggregation as all metrics are totals.") - String getMetricsPubSubSinkGranularity(); - - void setMetricsPubSubSinkGranularity(String value); - } - - @AutoService(PipelineOptionsRegistrar.class) - public static class FeastMetricsPubSubSinkOptionsRegistrar implements PipelineOptionsRegistrar { - @Override - public Iterable> getPipelineOptions() { - return Collections.singleton(ImportJobOptions.class); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportJobOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java similarity index 81% rename from ingestion/src/main/java/feast/ingestion/options/ImportJobOptions.java rename to ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java index 6da7d0f726f..72e8b9a53a1 100644 --- a/ingestion/src/main/java/feast/ingestion/options/ImportJobOptions.java +++ b/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java @@ -26,7 +26,10 @@ import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.options.Validation.Required; -public interface ImportJobOptions extends PipelineOptions { +/** + * Options passed to Beam to influence the job's execution environment + */ +public interface ImportJobPipelineOptions extends PipelineOptions { @Description("Import spec yaml file path") @Required(groups = {"importSpec"}) @@ -59,19 +62,6 @@ public interface ImportJobOptions extends PipelineOptions { void setCoreApiSpecPath(String value); - @Override - @Description("The beam sink class to which the metrics will be pushed") - @Default.InstanceFactory(NoOpMetricsSink.class) - Class getMetricsSink(); - - @Override - void setMetricsSink(Class metricsSink); - - @Description("Filter a to a max number if rows from the read source") - Long getLimit(); - - void setLimit(Long value); - @Description( "Set an errors store type. One of: [stderr, stdout, file.json]. Note that you should not use " + "stderr/stdout in production unless your data volume is extremely small.") @@ -88,11 +78,11 @@ public interface ImportJobOptions extends PipelineOptions { void setErrorsStoreOptions(String value); @AutoService(PipelineOptionsRegistrar.class) - class ImportJobOptionsRegistrar implements PipelineOptionsRegistrar { + class ImportJobPipelineOptionsRegistrar implements PipelineOptionsRegistrar { @Override public Iterable> getPipelineOptions() { - return Collections.singleton(ImportJobOptions.class); + return Collections.singleton(ImportJobPipelineOptions.class); } } } diff --git a/ingestion/src/main/java/feast/ingestion/options/JobOptions.java b/ingestion/src/main/java/feast/ingestion/options/JobOptions.java new file mode 100644 index 00000000000..c316f749bce --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/JobOptions.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.options; + +import com.fasterxml.jackson.annotation.JsonProperty; +import feast.options.Options; + +/** + * JobOptions are options passed in via the import spec, they are options that dictate certain + * behaviour of the job, they differ from the PipelineOptions in ImportJobOptions which are used to + * influence the execution environment. + */ +public class JobOptions implements Options { + + private long sampleLimit; + private boolean coalesceRowsEnabled = true; + private long coalesceRowsDelaySeconds; + private long coalesceRowsTimeoutSeconds; + + + @JsonProperty(value = "sample.limit") + public long getSampleLimit() { + return sampleLimit; + } + + public void setSampleLimit(long sampleLimit) { + this.sampleLimit = sampleLimit; + } + + @JsonProperty(value = "coalesceRows.enabled") + public boolean isCoalesceRowsEnabled() { + return coalesceRowsEnabled; + } + + public void setCoalesceRowsEnabled(boolean coalesceRows) { + this.coalesceRowsEnabled = coalesceRows; + } + + @JsonProperty(value = "coalesceRows.delaySeconds") + public long getCoalesceRowsDelaySeconds() { + return coalesceRowsDelaySeconds; + } + + public void setCoalesceRowsDelaySeconds(long coalesceRowsDelaySeconds) { + this.coalesceRowsDelaySeconds = coalesceRowsDelaySeconds; + } + + @JsonProperty(value = "coalesceRows.timeoutSeconds") + public long getCoalesceRowsTimeoutSeconds() { + return coalesceRowsTimeoutSeconds; + } + + public void setCoalesceRowsTimeoutSeconds(long coalesceRowsTimeoutSeconds) { + this.coalesceRowsTimeoutSeconds = coalesceRowsTimeoutSeconds; + } + +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java b/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java new file mode 100644 index 00000000000..a7c53d21c65 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java @@ -0,0 +1,280 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.transform; + +import com.google.common.base.Preconditions; +import com.google.protobuf.util.Timestamps; +import feast.types.FeatureProto.Feature; +import feast.types.FeatureRowProto.FeatureRow; +import feast_ingestion.types.CoalesceAccumProto.CoalesceAccum; +import feast_ingestion.types.CoalesceKeyProto.CoalesceKey; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.StateSpec; +import org.apache.beam.sdk.state.StateSpecs; +import org.apache.beam.sdk.state.TimeDomain; +import org.apache.beam.sdk.state.Timer; +import org.apache.beam.sdk.state.TimerSpec; +import org.apache.beam.sdk.state.TimerSpecs; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.transforms.Values; +import org.apache.beam.sdk.transforms.WithKeys; +import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollection.IsBounded; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * Takes FeatureRow, and merges them if they have the same FeatureRowKey, so that the latest values + * will be emitted. It emits only once for batch. + * + * For streaming we emits after a delay of 10 seconds (event time) by default we keep the previous + * state around for merging with future events. These timeout after 30 minutes by default. + */ +public class CoalesceFeatureRows extends + PTransform, PCollection> { + + private static final SerializableFunction KEY_FUNCTION = (row) -> + CoalesceKey.newBuilder() + .setEntityName(row.getEntityName()) + .setEntityKey(row.getEntityKey()).build(); + + private static final Duration DEFAULT_DELAY = Duration.standardSeconds(10); + private static final Duration DEFAULT_TIMEOUT = Duration.ZERO; + + private Duration delay; + private Duration timeout; + + CoalesceFeatureRows() { + this(0, 0); + } + + public CoalesceFeatureRows(long delaySeconds, long timeoutSeconds) { + this(Duration.standardSeconds(delaySeconds), Duration.standardSeconds(timeoutSeconds)); + } + + public CoalesceFeatureRows(Duration delay, Duration timeout) { + this.delay = (delay.isEqual(Duration.ZERO)) ? DEFAULT_DELAY : delay; + this.timeout = (timeout.isEqual(Duration.ZERO)) ? DEFAULT_TIMEOUT : timeout; + } + + /** + * Return a FeatureRow of the new features accumulated since the given timestamp + */ + public static FeatureRow toFeatureRow(CoalesceAccum accum, long counter) { + Preconditions.checkArgument(counter <= + accum.getCounter(), "Accumulator has no features at or newer than the provided counter"); + FeatureRow.Builder builder = FeatureRow.newBuilder() + .setEntityName(accum.getEntityName()) + .setGranularity(accum.getGranularity()) + .setEntityKey(accum.getEntityKey()); + // This will be the latest timestamp + if (accum.hasEventTimestamp()) { + builder.setEventTimestamp(accum.getEventTimestamp()); + } + + Map features = accum.getFeaturesMap(); + if (counter <= 0) { + builder.addAllFeatures(features.values()); + } else { + List featureList = accum.getFeatureMarksMap().entrySet().stream() + .filter((e) -> e.getValue() > counter) + .map((e) -> features.get(e.getKey())) + .collect(Collectors.toList()); + builder.addAllFeatures(featureList); + + } + return builder.build(); + } + + public static FeatureRow combineFeatureRows(Iterable rows) { + return toFeatureRow(combineFeatureRows(CoalesceAccum.getDefaultInstance(), rows), 0); + } + + public static CoalesceAccum combineFeatureRows(CoalesceAccum seed, Iterable rows) { + CoalesceAccum.Builder accum = seed.toBuilder(); + Map features = new HashMap<>(); + Map featureMarks = new HashMap<>(); + long rowCount = seed.getCounter(); + for (FeatureRow row : rows) { + rowCount += 1; + if (Timestamps.compare(accum.getEventTimestamp(), row.getEventTimestamp()) + <= 0) { + // row has later timestamp than accum. + for (Feature feature : row.getFeaturesList()) { + features.put(feature.getId(), feature); + // These marks are used to determine which features are new when we convert an accum + // back into a FeatureRow. + featureMarks.put(feature.getId(), rowCount); + } + accum.setEntityName(row.getEntityName()); + accum.setEntityKey(row.getEntityKey()); + accum.setGranularity(row.getGranularity()); + if (row.hasEventTimestamp()) { + accum.setEventTimestamp(row.getEventTimestamp()); + } + } else { + for (Feature feature : row.getFeaturesList()) { + String featureId = feature.getId(); + // only insert an older feature if there was no newer one. + if (!features.containsKey(featureId)) { + features.put(featureId, feature); + featureMarks.put(feature.getId(), rowCount); + } + } + } + } + if (rowCount == seed.getCounter()) { + return seed; + } else { + return accum + .setCounter(rowCount) + .putAllFeatures(features) + .putAllFeatureMarks(featureMarks) + .build(); + } + } + + @Override + public PCollection expand(PCollection input) { + PCollection> kvs = input + .apply(WithKeys.of(KEY_FUNCTION).withKeyType(TypeDescriptor.of(CoalesceKey.class))) + .setCoder(KvCoder.of(ProtoCoder.of(CoalesceKey.class), ProtoCoder.of(FeatureRow.class))); + + if (kvs.isBounded().equals(IsBounded.UNBOUNDED)) { + return kvs.apply("Configure window", Window.>configure() + .withAllowedLateness(Duration.ZERO) + .discardingFiredPanes() + .triggering(AfterProcessingTime.pastFirstElementInPane())) + .apply(ParDo.of(new CombineStateDoFn(delay, timeout))) + .apply(Values.create()); + } else { + return kvs.apply(Combine.perKey(CoalesceFeatureRows::combineFeatureRows)) + .apply(Values.create()); + } + } + + + @Slf4j + @AllArgsConstructor + public static class CombineStateDoFn extends + DoFn, KV> { + + @StateId("lastKnownAccumValue") + private final StateSpec> lastKnownAccumValueSpecs = + StateSpecs.value(ProtoCoder.of(CoalesceAccum.class)); + @StateId("newElementsBag") + private final StateSpec> newElementsBag = + StateSpecs.bag(ProtoCoder.of(FeatureRow.class)); + @StateId("lastTimerTimestamp") + private final StateSpec> lastTimerTimestamp = StateSpecs.value(); + @TimerId("bufferTimer") + private final TimerSpec bufferTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME); + @TimerId("timeoutTimer") + private final TimerSpec timeoutTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME); + + private Duration delay; + private Duration timeout; + + @ProcessElement + public void processElement(ProcessContext context, + @StateId("newElementsBag") BagState newElementsBag, + @TimerId("bufferTimer") Timer bufferTimer, + @TimerId("timeoutTimer") Timer timeoutTimer, + @StateId("lastTimerTimestamp") ValueState lastTimerTimestampValue) { + newElementsBag.add(context.element().getValue()); + log.debug("Adding FeatureRow to state bag {}", context.element()); + + Instant lastTimerTimestamp = lastTimerTimestampValue.read(); + Instant contextTimestamp = context.timestamp(); + if (lastTimerTimestamp == null && timeout.isLongerThan(Duration.ZERO)) { + // We never timeout the state if a timeout is not set. + timeoutTimer.offset(timeout).setRelative(); + } + if (lastTimerTimestamp == null || lastTimerTimestamp.isBefore(contextTimestamp) + || lastTimerTimestamp.equals(contextTimestamp)) { + lastTimerTimestamp = context.timestamp().plus(delay); + log.debug("Setting timer for key {} to {}", context.element().getKey(), lastTimerTimestamp); + lastTimerTimestampValue.write(lastTimerTimestamp); + bufferTimer.offset(delay).setRelative(); + } + } + + @OnTimer("bufferTimer") + public void bufferOnTimer( + OnTimerContext context, OutputReceiver> out, + @StateId("newElementsBag") BagState newElementsBag, + @StateId("lastKnownAccumValue") ValueState lastKnownAccumValue) { + log.debug("bufferOnTimer triggered {}", context.timestamp()); + flush(out, newElementsBag, lastKnownAccumValue); + } + + @OnTimer("timeoutTimer") + public void timeoutOnTimer( + OnTimerContext context, OutputReceiver> out, + @StateId("newElementsBag") BagState newElementsBag, + @StateId("lastKnownAccumValue") ValueState lastKnownAccumValue) { + log.debug("timeoutOnTimer triggered {}", context.timestamp()); + flush(out, newElementsBag, lastKnownAccumValue); + newElementsBag.clear(); + lastKnownAccumValue.clear(); + } + + public void flush( + OutputReceiver> out, + @StateId("newElementsBag") BagState newElementsBag, + @StateId("lastKnownAccumValue") ValueState lastKnownAccumValue) { + log.debug("Flush triggered"); + Iterable rows = newElementsBag.read(); + if (!rows.iterator().hasNext()) { + log.debug("Flush with no new elements"); + return; + } + CoalesceAccum lastKnownAccum = lastKnownAccumValue.read(); + if (lastKnownAccum == null) { + lastKnownAccum = CoalesceAccum.getDefaultInstance(); + } + // Check if we have more than one value in our list. + CoalesceAccum accum = combineFeatureRows(lastKnownAccum, rows); + FeatureRow row = toFeatureRow(accum, lastKnownAccum.getCounter()); + log.debug("Timer fired and added FeatureRow to output {}", row); + // Clear the elements now that they have been processed + newElementsBag.clear(); + lastKnownAccumValue.write(accum); + + // Output the value stored in the the processed que which matches this timers time + out.output(KV.of(KEY_FUNCTION.apply(row), row)); + } + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/CoalescePFeatureRows.java b/ingestion/src/main/java/feast/ingestion/transform/CoalescePFeatureRows.java new file mode 100644 index 00000000000..cf7932f0155 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/CoalescePFeatureRows.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package feast.ingestion.transform; + +import feast.ingestion.values.PFeatureRows; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import feast.types.FeatureRowProto.FeatureRow; +import lombok.AllArgsConstructor; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; + +/** + * This class is a work around to make some refactoring easier. PFeatureRows should be deprecated. + */ +@AllArgsConstructor +public class CoalescePFeatureRows extends + PTransform { + + private long delaySeconds; + private long timeoutSeconds; + + @Override + public PFeatureRows expand(PFeatureRows input) { + PCollection output = input.getMain() + .apply(MapElements.into(TypeDescriptor.of(FeatureRow.class)) + .via(FeatureRowExtended::getRow)) + .apply(new CoalesceFeatureRows(delaySeconds, timeoutSeconds)) + .apply(new ToFeatureRowExtended()); + return PFeatureRows.of(output, input.getErrors()); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java index 97799a0ae86..7b8e41030e9 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java @@ -22,7 +22,7 @@ import com.google.inject.Inject; import feast.ingestion.model.Specs; -import feast.ingestion.options.ImportJobOptions; +import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; import feast.storage.ErrorsStore; @@ -44,7 +44,7 @@ public class ErrorsStoreTransform extends FeatureIO.Write { @Inject public ErrorsStoreTransform( - ImportJobOptions options, Specs specs, List errorsStores) { + ImportJobPipelineOptions options, Specs specs, List errorsStores) { this.specs = specs; this.errorsStores = errorsStores; this.errorsStoreType = options.getErrorsStoreType(); diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java index c26b25a81fb..cd19a913ff9 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java @@ -36,8 +36,11 @@ public void processElementImpl(ProcessContext context) { FeatureRow.Builder rowBuilder = FeatureRow.newBuilder(); rowBuilder .setEntityName(row.getEntityName()) - .setEventTimestamp(row.getEventTimestamp()) .setEntityKey(row.getEntityKey()); + if (row.hasEventTimestamp()) { + rowBuilder.setEventTimestamp(row.getEventTimestamp()); + } + for (Feature feature : row.getFeaturesList()) { String featureId = feature.getId(); diff --git a/ingestion/src/main/java/feast/ingestion/values/PFeatureRows.java b/ingestion/src/main/java/feast/ingestion/values/PFeatureRows.java index 3c46e92f158..86a710ddae6 100644 --- a/ingestion/src/main/java/feast/ingestion/values/PFeatureRows.java +++ b/ingestion/src/main/java/feast/ingestion/values/PFeatureRows.java @@ -14,7 +14,6 @@ * limitations under the License. * */ - package feast.ingestion.values; import feast.ingestion.transform.fn.BaseFeatureDoFn; diff --git a/ingestion/src/main/java/feast/source/bigquery/BigQueryFeatureSource.java b/ingestion/src/main/java/feast/source/bigquery/BigQueryFeatureSource.java index 9ce7f965f2c..b183ffbddc0 100644 --- a/ingestion/src/main/java/feast/source/bigquery/BigQueryFeatureSource.java +++ b/ingestion/src/main/java/feast/source/bigquery/BigQueryFeatureSource.java @@ -65,7 +65,7 @@ public class BigQueryFeatureSource extends FeatureSource { @Override public PCollection expand(PInput input) { BigQuerySourceOptions options = OptionsParser - .parse(importSpec.getOptionsMap(), BigQuerySourceOptions.class); + .parse(importSpec.getSourceOptionsMap(), BigQuerySourceOptions.class); List entities = importSpec.getEntitiesList(); Preconditions.checkArgument( diff --git a/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java b/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java index 837050fc680..f1a99dd017f 100644 --- a/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java +++ b/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java @@ -70,7 +70,7 @@ public class CsvFileFeatureSource extends FeatureSource { @Override public PCollection expand(PInput input) { CsvFileFeatureSourceOptions options = OptionsParser - .parse(importSpec.getOptionsMap(), CsvFileFeatureSourceOptions.class); + .parse(importSpec.getSourceOptionsMap(), CsvFileFeatureSourceOptions.class); List entities = importSpec.getEntitiesList(); Preconditions.checkArgument( entities.size() == 1, "exactly 1 entity must be set for CSV import"); @@ -94,6 +94,11 @@ public PCollection expand(PInput input) { schema.getFieldsList().size() > 0, "CSV import needs schema with a least one field specified"); + if (!Strings.isNullOrEmpty(timestampColumn)) { + Preconditions.checkArgument(fieldNames.contains(timestampColumn), + String.format("timestampColumn %s, does not match any field", timestampColumn)); + } + PCollection stringMaps = input.getPipeline().apply(CsvIO.read(path, fieldNames)); return stringMaps.apply( @@ -109,6 +114,9 @@ public void processElement(ProcessContext context) { for (Entry entry : stringMap.entrySet()) { String name = entry.getKey(); String value = entry.getValue(); + if (value.isEmpty()) { + continue; + } Field field = fields.get(name); // A feature can only be one of these things @@ -137,7 +145,6 @@ public void processElement(ProcessContext context) { } - public static class CsvFileFeatureSourceOptions implements Options { @NotEmpty diff --git a/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java b/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java index a38a2e0b8c1..08131c16f8b 100644 --- a/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java +++ b/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java @@ -56,7 +56,7 @@ public class JsonFileFeatureSource extends FeatureSource { @Override public PCollection expand(PInput input) { JsonFileFeatureSourceOptions options = OptionsParser - .parse(importSpec.getOptionsMap(), JsonFileFeatureSourceOptions.class); + .parse(importSpec.getSourceOptionsMap(), JsonFileFeatureSourceOptions.class); PCollection jsonLines = input.getPipeline().apply(TextIO.read().from(options.path)); return jsonLines.apply( ParDo.of( diff --git a/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java b/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java index d6d5e36ad01..9246c5a9fdc 100644 --- a/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java +++ b/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java @@ -59,7 +59,7 @@ public PCollection expand(PInput input) { checkArgument(importSpec.getType().equals(KAFKA_FEATURE_SOURCE_TYPE)); KafkaReadOptions options = - OptionsParser.parse(importSpec.getOptionsMap(), KafkaReadOptions.class); + OptionsParser.parse(importSpec.getSourceOptionsMap(), KafkaReadOptions.class); List topicsList = new ArrayList<>(Arrays.asList(options.topics.split(","))); diff --git a/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java b/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java index 925094828c6..c8bafbafc70 100644 --- a/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java +++ b/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java @@ -60,7 +60,7 @@ public class PubSubFeatureSource extends FeatureSource { public PCollection expand(PInput input) { checkArgument(importSpec.getType().equals(PUBSUB_FEATURE_SOURCE_TYPE)); PubSubReadOptions options = - OptionsParser.parse(importSpec.getOptionsMap(), PubSubReadOptions.class); + OptionsParser.parse(importSpec.getSourceOptionsMap(), PubSubReadOptions.class); PubsubIO.Read read = readProtos(); diff --git a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java b/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java index 67d1c2181c8..7437f59c55a 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java +++ b/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java @@ -19,20 +19,17 @@ import com.google.cloud.bigtable.beam.CloudBigtableConfiguration; import com.google.cloud.bigtable.beam.CloudBigtableIO; +import feast.ingestion.model.Specs; +import feast.ingestion.transform.FeatureIO; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collections; import lombok.extern.slf4j.Slf4j; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.hadoop.hbase.client.Mutation; -import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; -import feast.ingestion.transform.SplitFeatures.SingleOutputSplit; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @Slf4j public class FeatureRowBigTableIO { @@ -51,17 +48,9 @@ public Write(BigTableStoreOptions bigTableOptions, Specs specs) { public PDone expand(PCollection input) { log.info("Using BigTable options: " + bigTableOptions.toString()); - // we need a row per granularity, because they will have different keys in BigTable - PCollection features = - input - .apply( - "Split by granularity", - new SingleOutputSplit<>(FeatureSpec::getGranularity, specs)) - .setCoder(ProtoCoder.of(FeatureRowExtended.class)); - // entity name to mutation key value PCollection> mutations = - features.apply( + input.apply( "Map to BigTable mutations", ParDo.of(new FeatureRowToBigTableMutationDoFn(bigTableOptions.prefix, specs))); diff --git a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java b/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java index b36f84dcdda..bfd44ceda87 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java +++ b/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java @@ -18,26 +18,21 @@ package feast.storage.bigtable; import com.google.common.base.Charsets; -import com.google.common.collect.Lists; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.KV; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; import feast.SerializableCache; import feast.ingestion.model.Specs; import feast.ingestion.util.DateUtil; import feast.options.OptionsParser; -import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.BigTableProto.BigTableRowKey; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; -import feast.types.GranularityProto.Granularity.Enum; +import lombok.extern.slf4j.Slf4j; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; /** * DoFn for taking a feature row and making Bigtable mutations out of it. Also keys the mutations by @@ -47,6 +42,8 @@ public class FeatureRowToBigTableMutationDoFn extends DoFn> { + private static final String LATEST_KEY = "0"; + private final SerializableCache servingOptionsCache = SerializableCache.builder() .loadingFunction( @@ -64,33 +61,21 @@ public class FeatureRowToBigTableMutationDoFn } public static BigTableRowKey makeBigTableRowKey( - String entityKey, com.google.protobuf.Timestamp timestamp, Granularity.Enum granularity) { + String entityKey) { return BigTableRowKey.newBuilder() .setSha1Prefix(DigestUtils.sha1Hex(entityKey).substring(0, 7)) .setEntityKey(entityKey) - .setReversedMillis(String.valueOf(getReversedRoundedMillis(timestamp, granularity))) + .setReversedMillis(LATEST_KEY) .build(); } - public static long getReversedRoundedMillis( - com.google.protobuf.Timestamp timestamp, Granularity.Enum granularity) { - if (granularity == Granularity.Enum.NONE) { - return 0L; // We store it as zero instead of reversed Long.MAX_VALUE. - } - timestamp = DateUtil.roundToGranularity(timestamp, granularity); - return Long.MAX_VALUE - timestamp.getSeconds() * 1000; - } - @ProcessElement public void processElement(ProcessContext context) { FeatureRowExtended rowExtended = context.element(); FeatureRow row = rowExtended.getRow(); - EntitySpec entitySpec = specs.getEntitySpec(row.getEntityName()); - List mutations = makePut(rowExtended); - for (Put put : mutations) { - context.output(KV.of(getTableName(row), put)); - } + Put put = makePut(rowExtended); + context.output(KV.of(getTableName(row), put)); } private String getTableName(FeatureRow row) { @@ -107,21 +92,12 @@ private String getTableName(FeatureRow row) { *

bigtable row key = {sha1(row.key), row.key, row.timestamp} family = {feature.group, * granularity} qualifier = {feature.name} value = {feature.value} */ - public List makePut(FeatureRowExtended rowExtended) { + public Put makePut(FeatureRowExtended rowExtended) { FeatureRow row = rowExtended.getRow(); - Granularity.Enum granularity = row.getGranularity(); // We always additinally overwrite a None granularity row so that it is trivial to retrieve the // latest across all features. Put latestPut = - new Put( - makeBigTableRowKey(row.getEntityKey(), row.getEventTimestamp(), Enum.NONE) - .toByteArray()); - Put timeseriesPut = - new Put( - makeBigTableRowKey(row.getEntityKey(), row.getEventTimestamp(), granularity) - .toByteArray()); - boolean isTimeseries = granularity.getNumber() != Enum.NONE.getNumber(); - + new Put(makeBigTableRowKey(row.getEntityKey()).toByteArray()); for (Feature feature : row.getFeaturesList()) { FeatureSpec featureSpec = specs.getFeatureSpec(feature.getId()); BigTableFeatureOptions options = servingOptionsCache.get(featureSpec); @@ -129,18 +105,9 @@ public List makePut(FeatureRowExtended rowExtended) { byte[] family = options.family.getBytes(Charsets.UTF_8); byte[] qualifier = feature.getId().getBytes(Charsets.UTF_8); byte[] value = feature.getValue().toByteArray(); - // Note version will always the same for row key. long version = DateUtil.toMillis(row.getEventTimestamp()); latestPut.addColumn(family, qualifier, version, value); - if (isTimeseries) { - timeseriesPut.addColumn(family, qualifier, version, value); - } - } - - if (isTimeseries) { - return Lists.newArrayList(timeseriesPut, latestPut); - } else { - return Lists.newArrayList(latestPut); } + return latestPut; } } diff --git a/ingestion/src/main/java/feast/storage/postgres/FeatureRowJdbcIO.java b/ingestion/src/main/java/feast/storage/postgres/FeatureRowJdbcIO.java deleted file mode 100644 index 960b21056d5..00000000000 --- a/ingestion/src/main/java/feast/storage/postgres/FeatureRowJdbcIO.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.postgres; - -import com.google.common.base.Charsets; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import feast.ingestion.util.DateUtil; -import feast.ingestion.transform.FeatureIO; -import feast.ingestion.transform.SplitFeatures.MultiOutputSplit; -import feast.ingestion.model.Specs; -import com.google.inject.Inject; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity; -import feast.types.FeatureProto.Feature; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.ValueProto.Value; -import java.io.ByteArrayOutputStream; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.beam.sdk.io.jdbc.JdbcIO; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.TupleTag; -import org.jtwig.JtwigModel; -import org.jtwig.JtwigTemplate; - -@Slf4j -public class FeatureRowJdbcIO { - - - - public static class Write extends FeatureIO.Write { - - private String driverClassName = "org.postgresql.Driver"; - private PostgresOptions jdbcOptions; - private Specs specs; - private String storeId; - - private static Map driverTemplates = new HashMap<>(); - - static { - driverTemplates.put("org.postgresql.Driver", "upsert_feature_row.postgres.sql.twig"); - } - - @Inject - public Write(PostgresOptions jdbcOptions, Specs specs, String storeId) { - this.jdbcOptions = jdbcOptions; - this.specs = specs; - this.storeId = storeId; - } - - public static String getTableName(FeatureSpec spec) { - String entity = spec.getEntity(); - String granularity = spec.getGranularity().name().toLowerCase(); - return String.format("%s_%s", entity, granularity); - } - - @Override - public PDone expand(PCollection input) { - List storeFeatureSpecs = specs.getFeatureSpecByServingStoreId(storeId); - Set tableNames = new HashSet<>(); - Map> featuresByTable = new HashMap<>(); - - // Group feature specs by table name - for (FeatureSpec featureSpec : storeFeatureSpecs) { - String tableName = getTableName(featureSpec); - tableNames.add(tableName); - Set featureSpecs = featuresByTable.getOrDefault(tableName, new HashSet<>()); - featureSpecs.add(featureSpec); - featuresByTable.put(tableName, featureSpecs); - } - - MultiOutputSplit splitter = - new MultiOutputSplit<>(Write::getTableName, tableNames, specs); - PCollectionTuple splitByTable = input.apply(splitter); - - for (String tableName : tableNames) { - log.info("Initialising write transform for table " + tableName); - TupleTag tag = splitter.getStrategy().getTag(tableName); - writeTable(splitByTable.get(tag), Lists.newArrayList(featuresByTable.get(tableName))); - } - return PDone.in(input.getPipeline()); - } - - private PDone writeTable( - PCollection input, List featureSpecs) { - String tableName = - Strings.nullToEmpty(jdbcOptions.prefix) + getTableName(featureSpecs.get(0)); - - // We create map of positions so that the prepared statement knows what order to set feature - // columns - final Map positions = new HashMap<>(); - for (int i = 0; i < featureSpecs.size(); i++) { - positions.put(featureSpecs.get(i).getId(), i); - } - - JtwigTemplate template = - JtwigTemplate.classpathTemplate("templates/" + driverTemplates.get(driverClassName)); - JtwigModel model = - JtwigModel.newModel() - .with("tableName", tableName) - .with( - "featureNames", - featureSpecs.stream().map(FeatureSpec::getName).collect(Collectors.toList())); - - ByteArrayOutputStream outs = new ByteArrayOutputStream(); - template.render(model, outs); - String statement = new String(outs.toByteArray(), Charsets.UTF_8); - - return input.apply( - JdbcIO.write() - .withDataSourceConfiguration( - JdbcIO.DataSourceConfiguration.create(driverClassName, jdbcOptions.url)) - .withStatement(statement) - .withPreparedStatementSetter( - new JdbcIO.PreparedStatementSetter() { - public void setParameters( - FeatureRowExtended extendedRow, PreparedStatement query) - throws SQLException { - // We assume all features have the same granularity - // (this should be enforced by validation) - - FeatureRow row = extendedRow.getRow(); - Granularity.Enum granularity = row.getGranularity(); - - query.setString(1, row.getEntityKey()); - query.setTimestamp(2, DateUtil.toSqlTimestamp(row.getEventTimestamp())); - - int featuresOffset = 3; - for (Feature feature : row.getFeaturesList()) { - Integer featurePosition = positions.get(feature.getId()); - if (featurePosition == null) { - continue; - } - setPreparedStatementValue( - query, featuresOffset + featurePosition, feature.getValue()); - } - } - })); - } - - private void setPreparedStatementValue(PreparedStatement query, int index, Value value) - throws SQLException { - switch (value.getValCase()) { - case STRINGVAL: - query.setString(index, value.getStringVal()); - break; - case TIMESTAMPVAL: - query.setTimestamp(index, DateUtil.toSqlTimestamp(value.getTimestampVal())); - break; - case DOUBLEVAL: - query.setDouble(index, value.getDoubleVal()); - break; - case BOOLVAL: - query.setBoolean(index, value.getBoolVal()); - break; - case BYTESVAL: - query.setBytes(index, value.getBytesVal().toByteArray()); - break; - case FLOATVAL: - query.setFloat(index, value.getFloatVal()); - break; - case INT64VAL: - query.setLong(index, value.getInt64Val()); - break; - case INT32VAL: - query.setInt(index, value.getInt32Val()); - break; - default: - throw new SQLException("Unhandled type " + value.getValCase().name()); - } - } - } -} diff --git a/ingestion/src/main/java/feast/storage/postgres/PostgresOptions.java b/ingestion/src/main/java/feast/storage/postgres/PostgresOptions.java deleted file mode 100644 index 0d6dda9e31b..00000000000 --- a/ingestion/src/main/java/feast/storage/postgres/PostgresOptions.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.postgres; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.Serializable; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import feast.options.Options; - -public class PostgresOptions implements Options, Serializable { - private static final String OPT_POSTGRES_URL = "postgres.url"; - private static final String OPT_POSTGRES_PREFIX = "postgres.prefix"; - - @JsonProperty(value = OPT_POSTGRES_URL) - @NotNull - @Pattern(regexp = "jdbc:postgresql:.+") - String url; - - @JsonProperty(value = OPT_POSTGRES_PREFIX) - @Pattern(regexp = "[a-zA-Z_][a-zA-Z0-9_]*") - String prefix; -} diff --git a/ingestion/src/main/java/feast/storage/postgres/PostgresServingStore.java b/ingestion/src/main/java/feast/storage/postgres/PostgresServingStore.java deleted file mode 100644 index 9f68833b2fc..00000000000 --- a/ingestion/src/main/java/feast/storage/postgres/PostgresServingStore.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.postgres; - -import com.google.auto.service.AutoService; -import com.google.common.base.Preconditions; -import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; -import feast.options.OptionsParser; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; - -@AutoService(ServingStore.class) -public class PostgresServingStore implements ServingStore { - public static String TYPE_POSTGRES = "postgres"; - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - Preconditions.checkArgument( - storageSpec.getType().equals(TYPE_POSTGRES), "Storage spec type was not " + TYPE_POSTGRES); - - PostgresOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), PostgresOptions.class); - - return new FeatureRowJdbcIO.Write(options, specs, storageSpec.getId()); - } - - @Override - public String getType() { - return TYPE_POSTGRES; - } -} diff --git a/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java index 2ee46a93825..dad225d7ba8 100644 --- a/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java +++ b/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java @@ -19,10 +19,6 @@ import static com.google.common.base.Preconditions.checkArgument; -import java.util.Random; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.commons.codec.digest.DigestUtils; -import org.joda.time.Duration; import feast.SerializableCache; import feast.ingestion.model.Specs; import feast.ingestion.util.DateUtil; @@ -35,7 +31,10 @@ import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; +import java.util.Random; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.commons.codec.digest.DigestUtils; +import org.joda.time.Duration; public class FeatureRowToRedisMutationDoFn extends DoFn { @@ -75,7 +74,9 @@ static String getFeatureIdSha1Prefix(String featureId) { return DigestUtils.sha1Hex(featureId.getBytes()).substring(0, 7); } - /** Output a redis mutation object for every feature in the feature row. */ + /** + * Output a redis mutation object for every feature in the feature row. + */ @ProcessElement public void processElement(ProcessContext context) { FeatureRowExtended rowExtended = context.element(); @@ -110,31 +111,6 @@ public void processElement(ProcessContext context) { .expiryMillis(expiryMillis) .method(Method.SET) .build()); - - if (!featureSpec.getGranularity().equals(Granularity.Enum.NONE)) { - long bucketId = getBucketId(roundedTimestamp, options.getBucketSizeDuration()); - - RedisBucketKey keyForValue = - getRedisBucketKey(entityKey, featureIdHash, roundedTimestamp.getSeconds()); - - context.output( - RedisMutation.builder() - .key(keyForValue.toByteArray()) - .value(value.toByteArray()) - .expiryMillis(expiryMillis) - .method(Method.SET) - .build()); - - RedisBucketKey keyForZset = getRedisBucketKey(entityKey, featureIdHash, bucketId); - context.output( - RedisMutation.builder() - .key(keyForZset.toByteArray()) - .value(keyForValue.toByteArray()) - .score(roundedTimestamp.getSeconds()) - .expiryMillis(expiryMillis) - .method(Method.ZADD) - .build()); - } } } } diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto new file mode 100644 index 00000000000..f8e82a59521 --- /dev/null +++ b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto @@ -0,0 +1,37 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; +import "feast/types/Feature.proto"; +import "feast/types/Granularity.proto"; + +option java_package = "feast_ingestion.types"; +option java_outer_classname = "CoalesceAccumProto"; + +// Accumlator for merging feature rows. +message CoalesceAccum { + string entityKey = 1; + google.protobuf.Timestamp eventTimestamp = 3; + string entityName = 4; + feast.types.Granularity.Enum granularity = 5; + + map features = 6; + // map of features to their counter values when they were last added to accumulator + map featureMarks = 7; + int64 counter = 8; +} \ No newline at end of file diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto new file mode 100644 index 00000000000..9730b49ec3b --- /dev/null +++ b/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto @@ -0,0 +1,25 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_package = "feast_ingestion.types"; +option java_outer_classname = "CoalesceKeyProto"; + +message CoalesceKey { + string entityName = 1; + string entityKey = 2; +} \ No newline at end of file diff --git a/ingestion/src/main/proto/feast_ingestion/types/README.md b/ingestion/src/main/proto/feast_ingestion/types/README.md new file mode 100644 index 00000000000..7736c97ba20 --- /dev/null +++ b/ingestion/src/main/proto/feast_ingestion/types/README.md @@ -0,0 +1 @@ +Internal protobuf messages used in ingestion \ No newline at end of file diff --git a/ingestion/src/test/java/feast/FeastMatchers.java b/ingestion/src/test/java/feast/FeastMatchers.java index c0a6a8a7dbc..eb469b5c934 100644 --- a/ingestion/src/test/java/feast/FeastMatchers.java +++ b/ingestion/src/test/java/feast/FeastMatchers.java @@ -26,7 +26,7 @@ public class FeastMatchers { /** * Can be used with the PAssert.that(..).satisfies(fn) method. */ - public static SerializableFunction, Void> hasCount(int count) { + public static SerializableFunction, Void> hasCount(long count) { return (Iterable iterable) -> { Assert.assertEquals(count, Lists.newArrayList(iterable).size()); return null; diff --git a/ingestion/src/test/java/feast/NormalizeFeatureRows.java b/ingestion/src/test/java/feast/NormalizeFeatureRows.java new file mode 100644 index 00000000000..beb3b1b485b --- /dev/null +++ b/ingestion/src/test/java/feast/NormalizeFeatureRows.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast; + +import com.google.common.collect.Lists; +import com.google.common.primitives.UnsignedBytes; +import feast.types.FeatureProto.Feature; +import feast.types.FeatureRowProto.FeatureRow; +import java.util.List; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; + +public class NormalizeFeatureRows + extends PTransform, PCollection> { + + public static FeatureRow normalize(FeatureRow.Builder row) { + return normalize(row.build()); + } + + public static FeatureRow normalize(FeatureRow row) { + List features = Lists.newArrayList(row.getFeaturesList()); + features.sort( + (f1, f2) -> + UnsignedBytes.lexicographicalComparator().compare(f1.toByteArray(), f2.toByteArray())); + + return row.toBuilder() + .clearFeatures().addAllFeatures(features) + .setEventTimestamp(row.getEventTimestamp()) + .build(); + } + + @Override + public PCollection expand(PCollection input) { + return input + .apply( + "normalize rows", + MapElements.into(TypeDescriptor.of(FeatureRow.class)).via( + NormalizeFeatureRows::normalize)); + } +} diff --git a/ingestion/src/test/java/feast/TestHelper.java b/ingestion/src/test/java/feast/TestHelper.java index ce554d4abee..34984266f6f 100644 --- a/ingestion/src/test/java/feast/TestHelper.java +++ b/ingestion/src/test/java/feast/TestHelper.java @@ -23,9 +23,6 @@ import com.google.gson.Gson; import java.io.IOException; import java.io.OutputStream; -import java.util.Optional; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import feast.ingestion.options.ImportJobOptions; public class TestHelper { public static void writeYaml(Object model, OutputStream outputStream) throws IOException { diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index c6adf7dd22d..bd7460e6372 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -18,7 +18,7 @@ package feast.ingestion; import static feast.FeastMatchers.hasCount; -import static feast.ToOrderedFeatureRows.orderedFeatureRow; +import static feast.NormalizeFeatureRows.normalize; import static feast.storage.MockErrorsStore.MOCK_ERRORS_STORE_TYPE; import static org.junit.Assert.assertEquals; @@ -34,7 +34,7 @@ import feast.ingestion.boot.TestPipelineModule; import feast.ingestion.model.Features; import feast.ingestion.model.Values; -import feast.ingestion.options.ImportJobOptions; +import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.service.SpecRetrievalException; import feast.ingestion.util.ProtoUtil; import feast.specs.ImportSpecProto.ImportSpec; @@ -73,12 +73,13 @@ public class ImportJobCSVTest { public TestPipeline testPipeline = TestPipeline.create(); public ImportSpec initImportSpec(ImportSpec importSpec, String dataFile) { - return importSpec.toBuilder().putOptions("path", dataFile).build(); + return importSpec.toBuilder().putSourceOptions("path", dataFile).build(); } - public ImportJobOptions initOptions() { + public ImportJobPipelineOptions initOptions() { Path path = Paths.get(Resources.getResource("core_specs/").getPath()); - ImportJobOptions options = PipelineOptionsFactory.create().as(ImportJobOptions.class); + ImportJobPipelineOptions options = PipelineOptionsFactory.create() + .as(ImportJobPipelineOptions.class); options.setCoreApiSpecPath(path.toString()); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); return options; @@ -90,7 +91,7 @@ public void testImportCSV() throws IOException { ProtoUtil.decodeProtoYaml( "---\n" + "type: file.csv\n" - + "options:\n" + + "sourceOptions:\n" + " # path: to be overwritten in tests\n" + "entities:\n" + " - testEntity\n" @@ -108,7 +109,7 @@ public void testImportCSV() throws IOException { Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); importSpec = initImportSpec(importSpec, csvFile.toString()); - ImportJobOptions options = initOptions(); + ImportJobPipelineOptions options = initOptions(); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = @@ -134,7 +135,7 @@ public void testImportCSV() throws IOException { List expectedRows = Lists.newArrayList( - orderedFeatureRow( + normalize( FeatureRow.newBuilder() .setGranularity(Granularity.Enum.NONE) .setEventTimestamp(Timestamp.getDefaultInstance()) @@ -143,7 +144,7 @@ public void testImportCSV() throws IOException { .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) .addFeatures(Features.of("testEntity.none.testString", Values.ofString("a"))) .build()), - orderedFeatureRow( + normalize( FeatureRow.newBuilder() .setGranularity(Granularity.Enum.NONE) .setEventTimestamp(Timestamp.getDefaultInstance()) @@ -152,7 +153,7 @@ public void testImportCSV() throws IOException { .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(202))) .addFeatures(Features.of("testEntity.none.testString", Values.ofString("b"))) .build()), - orderedFeatureRow( + normalize( FeatureRow.newBuilder() .setGranularity(Granularity.Enum.NONE) .setEventTimestamp(Timestamp.getDefaultInstance()) @@ -173,14 +174,156 @@ public void testImportCSV() throws IOException { testPipeline.run(); } + @Test + public void testImportCSV_withSample1() throws IOException { + ImportSpec importSpec = + ProtoUtil.decodeProtoYaml( + "---\n" + + "type: file.csv\n" + + "sourceOptions:\n" + + " # path: to be overwritten in tests\n" + + "jobOptions:\n" + + " sample.limit: 1\n" + + "entities:\n" + + " - testEntity\n" + + "schema:\n" + + " entityIdColumn: id\n" + + " timestampValue: 2018-09-25T00:00:00.000Z\n" + + " fields:\n" + + " - name: id\n" + + " - featureId: testEntity.none.testInt32\n" + + " - featureId: testEntity.none.testString\n" + + "\n", + ImportSpec.getDefaultInstance()); + + File csvFile = folder.newFile("data.csv"); + Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); + importSpec = initImportSpec(importSpec, csvFile.toString()); + + ImportJobPipelineOptions options = initOptions(); + options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); + + Injector injector = + Guice.createInjector( + new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + + ImportJob job = injector.getInstance(ImportJob.class); + injector.getInstance(ImportJob.class); + job.expand(); + + PCollection writtenToServing = + PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + .apply("flatten serving input", Flatten.pCollections()); + + PCollection writtenToWarehouse = + PCollectionList.of( + WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + .apply("flatten warehouse input", Flatten.pCollections()); + + PCollection writtenToErrors = + PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + .apply("flatten errors input", Flatten.pCollections()); + + PAssert.that(writtenToServing).satisfies(hasCount(1)); + PAssert.that(writtenToWarehouse).satisfies(hasCount(1)); + PAssert.that(writtenToErrors).satisfies(hasCount(0)); + + testPipeline.run(); + } + + @Test + public void testImportCSV_withCoalesceRows() throws IOException { + ImportSpec importSpec = + ProtoUtil.decodeProtoYaml( + "---\n" + + "type: file.csv\n" + + "sourceOptions:\n" + + " # path: to be overwritten in tests\n" + + "jobOptions:\n" + + " coalesceRows.enabled: true\n" + + "entities:\n" + + " - testEntity\n" + + "schema:\n" + + " entityIdColumn: id\n" + + " timestampColumn: timestamp\n" + + " fields:\n" + + " - name: id\n" + + " - name: timestamp\n" + + " - featureId: testEntity.none.testInt32\n" + + " - featureId: testEntity.none.testString\n" + + "\n", + ImportSpec.getDefaultInstance()); + + File csvFile = folder.newFile("data.csv"); + Files.asCharSink(csvFile, Charsets.UTF_8) + .write("1,2018-09-25T00:00:00.000Z,101,a\n1,2018-09-26T00:00:00.000Z,,b\n"); + importSpec = initImportSpec(importSpec, csvFile.toString()); + + ImportJobPipelineOptions options = initOptions(); + options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); + + Injector injector = + Guice.createInjector( + new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + + ImportJob job = injector.getInstance(ImportJob.class); + injector.getInstance(ImportJob.class); + job.expand(); + + PCollection writtenToServing = + PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + .apply("flatten serving input", Flatten.pCollections()); + + PCollection writtenToWarehouse = + PCollectionList.of( + WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + .apply("flatten warehouse input", Flatten.pCollections()); + + PCollection writtenToErrors = + PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + .apply("flatten errors input", Flatten.pCollections()); + + PAssert.that(writtenToErrors).satisfies(hasCount(0)); + + PAssert.that(writtenToServing.apply("serving toFeatureRows", new ToOrderedFeatureRows())) + .containsInAnyOrder( + normalize( + FeatureRow.newBuilder() + .setGranularity(Granularity.Enum.NONE) + .setEntityKey("1") + .setEntityName("testEntity") + .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) + .addFeatures(Features.of("testEntity.none.testString", Values.ofString("b"))) + .build())); + + PAssert.that(writtenToWarehouse.apply("warehouse toFeatureRows", new ToOrderedFeatureRows())) + .containsInAnyOrder( + normalize( + FeatureRow.newBuilder() + .setGranularity(Granularity.Enum.NONE) + .setEntityKey("1") + .setEntityName("testEntity") + .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) + .addFeatures(Features.of("testEntity.none.testString", Values.ofString("a"))) + .build()), + normalize( + FeatureRow.newBuilder() + .setGranularity(Granularity.Enum.NONE) + .setEntityKey("1") + .setEntityName("testEntity") + .addFeatures(Features.of("testEntity.none.testString", Values.ofString("b"))) + .build())); + + testPipeline.run(); + } + @Test(expected = SpecRetrievalException.class) public void testImportCSVUnknownServingStoreError() throws IOException { ImportSpec importSpec = ProtoUtil.decodeProtoYaml( "---\n" + "type: file.csv\n" - + "options:\n" - + " format: csv\n" + + "sourceOptions:\n" + " # path: to be overwritten in tests\n" + "entities:\n" + " - testEntity\n" @@ -199,7 +342,7 @@ public void testImportCSVUnknownServingStoreError() throws IOException { Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); importSpec = initImportSpec(importSpec, csvFile.toString()); - ImportJobOptions options = initOptions(); + ImportJobPipelineOptions options = initOptions(); Injector injector = Guice.createInjector( @@ -218,7 +361,7 @@ public void testImportWithErrors() throws IOException { ProtoUtil.decodeProtoYaml( "---\n" + "type: file.csv\n" - + "options:\n" + + "sourceOptions:\n" + " # path: to be overwritten in tests\n" + "entities:\n" + " - testEntity\n" @@ -238,7 +381,7 @@ public void testImportWithErrors() throws IOException { Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); importSpec = initImportSpec(importSpec, csvFile.toString()); - ImportJobOptions options = initOptions(); + ImportJobPipelineOptions options = initOptions(); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = @@ -283,7 +426,7 @@ public void testImportWithoutWarehouseStore() throws IOException { ProtoUtil.decodeProtoYaml( "---\n" + "type: file.csv\n" - + "options:\n" + + "sourceOptions:\n" + " # path: to be overwritten in tests\n" + "entities:\n" + " - testEntity\n" @@ -303,7 +446,7 @@ public void testImportWithoutWarehouseStore() throws IOException { Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); importSpec = initImportSpec(importSpec, csvFile.toString()); - ImportJobOptions options = initOptions(); + ImportJobPipelineOptions options = initOptions(); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = diff --git a/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java b/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java index a25a2179ca5..00eacdcaeb4 100644 --- a/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java +++ b/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java @@ -18,11 +18,8 @@ package feast.ingestion.config; import feast.ingestion.util.DateUtil; -import feast.ingestion.options.ImportJobOptions; -import com.google.protobuf.InvalidProtocolBufferException; +import feast.ingestion.options.ImportJobPipelineOptions; import com.google.protobuf.util.JsonFormat; -import feast.ingestion.options.ImportJobOptions; -import feast.ingestion.util.DateUtil; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; @@ -40,9 +37,8 @@ public class ImportSpecSupplierTest { String importSpecYaml = "---\n" - + "type: file\n" - + "options:\n" - + " format: csv\n" + + "type: file.csv\n" + + "sourceOptions:\n" + " path: data.csv\n" + "entities:\n" + " - driver\n" @@ -56,17 +52,10 @@ public class ImportSpecSupplierTest { + " featureId: driver.none.trips_completed\n" + "\n"; - String importSpecJson = - "{\"type\":\"file\",\"options\":{\"format\":\"csv\",\"path\":\"data.csv\"},\"entities\":[\"driver\"]," - + "\"schema\":" - + "{\"fields\":[{\"name\":\"timestamp\"}," - + "{\"name\":\"driver_id\"},{\"name\":\"trips_completed\",\"featureId\":\"driver.none.trips_completed\"}],\"timestampValue\":\"2018-09-25T00:00:00Z\",\"entityIdColumn\":\"driver_id\"}}\n"; - ImportSpec expectedImportSpec = ImportSpec.newBuilder() - .setType("file") - .putOptions("format", "csv") - .putOptions("path", "data.csv") + .setType("file.csv") + .putSourceOptions("path", "data.csv") .addEntities("driver") .setSchema( Schema.newBuilder() @@ -89,7 +78,7 @@ public void testSupplierImportSpecYamlFile() throws IOException { printWriter.print(importSpecYaml); } - ImportJobOptions options = PipelineOptionsFactory.create().as(ImportJobOptions.class); + ImportJobPipelineOptions options = PipelineOptionsFactory.create().as(ImportJobPipelineOptions.class); options.setImportSpecYamlFile(yamlFile.toString()); ImportSpec importSpec = new ImportSpecSupplier(options).get(); diff --git a/ingestion/src/test/java/feast/ingestion/options/JobOptionsTest.java b/ingestion/src/test/java/feast/ingestion/options/JobOptionsTest.java new file mode 100644 index 00000000000..59f4a478103 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/options/JobOptionsTest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.options; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import feast.options.OptionsParser; +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; + +public class JobOptionsTest { + + @Test + public void test_shouldParseAll() { + Map map = new HashMap<>(); + map.put("coalesceRows.enabled", "false"); + map.put("coalesceRows.delaySeconds", "123"); + map.put("coalesceRows.timeoutSeconds", "1800"); + map.put("sample.limit", "1234"); + + + JobOptions options = OptionsParser.parse(map, JobOptions.class); + assertEquals(options.getSampleLimit(), 1234L); + assertEquals(options.getCoalesceRowsDelaySeconds(), 123L); + assertFalse(options.isCoalesceRowsEnabled()); + assertEquals(options.getCoalesceRowsTimeoutSeconds(), 1800L); + } + + @Test + public void test_shouldParseEmptyOptions() { + JobOptions options = OptionsParser.parse(new HashMap<>(), JobOptions.class); + assertEquals(options.getSampleLimit(), 0L); + assertEquals(options.getCoalesceRowsDelaySeconds(), 0L); + assertTrue(options.isCoalesceRowsEnabled()); //defaults to true + assertEquals(options.getCoalesceRowsTimeoutSeconds(), 0L); + } + + @Test + public void test_shouldParseSampleLimit() { + Map map = new HashMap<>(); + map.put("sample.limit", "1234"); + JobOptions options = OptionsParser.parse(map, JobOptions.class); + assertEquals(options.getSampleLimit(), 1234L); + } + + @Test + public void test_shouldParseCoalesceRowsDelaySeconds() { + Map map = new HashMap<>(); + map.put("coalesceRows.delaySeconds", "123"); + JobOptions options = OptionsParser.parse(map, JobOptions.class); + assertEquals(options.getCoalesceRowsDelaySeconds(), 123L); + } + + @Test + public void test_shouldParseCoalesceRowsEnabled() { + Map map = new HashMap<>(); + map.put("coalesceRows.enabled", "true"); + JobOptions options = OptionsParser.parse(map, JobOptions.class); + assertTrue(options.isCoalesceRowsEnabled()); + + } + + + @Test + public void test_shouldParseCoalesceRowsTimeoutSeconds() { + Map map = new HashMap<>(); + map.put("coalesceRows.timeoutSeconds", "1800"); + JobOptions options = OptionsParser.parse(map, JobOptions.class); + assertEquals(options.getCoalesceRowsTimeoutSeconds(), 1800L); + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/ingestion/transform/CoalesceFeatureRowsTest.java b/ingestion/src/test/java/feast/ingestion/transform/CoalesceFeatureRowsTest.java new file mode 100644 index 00000000000..4d46e8925a9 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/transform/CoalesceFeatureRowsTest.java @@ -0,0 +1,568 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.transform; + +import static feast.NormalizeFeatureRows.normalize; +import static feast.ingestion.transform.CoalesceFeatureRows.toFeatureRow; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.Lists; +import com.google.protobuf.Timestamp; +import feast.FeastMatchers; +import feast.NormalizeFeatureRows; +import feast.ingestion.model.Features; +import feast.ingestion.model.Values; +import feast.types.FeatureProto.Feature; +import feast.types.FeatureRowProto.FeatureRow; +import feast_ingestion.types.CoalesceAccumProto.CoalesceAccum; +import java.util.List; +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.Rule; +import org.junit.Test; + +public class CoalesceFeatureRowsTest { + + private static final Timestamp DEFAULT_TIMESTAMP = Timestamp.getDefaultInstance(); + private static final FeatureRow DEFAULT_FEATURE_ROW = FeatureRow.getDefaultInstance().toBuilder() + .setEventTimestamp(DEFAULT_TIMESTAMP).build(); + + @Rule + public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testBatch_withDistictKeys_shouldPassThroughNonIntersectingKeys() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEventTimestamp(Timestamp.getDefaultInstance()).setEntityKey("1") + .build(), + FeatureRow.newBuilder().setEventTimestamp(Timestamp.getDefaultInstance()).setEntityKey("2") + .build()); + + PCollection input = pipeline.apply(Create.of(rows)) + .setCoder(ProtoCoder.of(FeatureRow.class)); + + PCollection output = input.apply(new CoalesceFeatureRows()); + + PAssert.that(output.apply(Count.globally())).containsInAnyOrder(2L); + PAssert.that(output).containsInAnyOrder(rows); + + pipeline.run(); + } + + @Test + public void test_withNoFeaturesSameTimestamp_shouldReturn1() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEventTimestamp(Timestamp.getDefaultInstance()).setEntityKey("1") + .build(), + FeatureRow.newBuilder().setEventTimestamp(Timestamp.getDefaultInstance()).setEntityKey("1") + .build()); + PCollection input = pipeline.apply(Create.of(rows)) + .setCoder(ProtoCoder.of(FeatureRow.class)); + + PCollection output = input.apply(new CoalesceFeatureRows()); + + assertThat(CoalesceFeatureRows.combineFeatureRows(rows), equalTo(rows.get(0))); + PAssert.that(output.apply(Count.globally())).containsInAnyOrder(1L); + PAssert.that(output).containsInAnyOrder(rows.get(0)); + + pipeline.run(); + } + + @Test + public void testBatch_withNoFeaturesDifferentSeconds_shouldReturnLatest() { + List rows1 = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(1)).build(), + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)).build()); + + assertThat(CoalesceFeatureRows.combineFeatureRows(rows1), equalTo(rows1.get(1))); + assertThat(CoalesceFeatureRows.combineFeatureRows(Lists.reverse(rows1)), + equalTo(rows1.get(1))); + } + + + @Test + public void testBatch_withNoFeaturesDifferentNanos_shouldReturnLatest() { + List rows1 = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setNanos(1)).build(), + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setNanos(2)).build()); + + assertThat(CoalesceFeatureRows.combineFeatureRows(rows1), equalTo(rows1.get(1))); + assertThat(CoalesceFeatureRows.combineFeatureRows(Lists.reverse(rows1)), + equalTo(rows1.get(1))); + } + + @Test + public void testBatch_shouldMergeFeatures() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build()); + PCollection input = pipeline.apply(Create.of(rows)) + .setCoder(ProtoCoder.of(FeatureRow.class)); + + PCollection output = input.apply(new CoalesceFeatureRows()); + + PAssert.that(output.apply(Count.globally())).containsInAnyOrder(1L); + PAssert.that(output.apply(new NormalizeFeatureRows())).containsInAnyOrder( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.getDefaultInstance()) + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build() + ); + + pipeline.run(); + } + + @Test + public void testStream_shouldMergeFeatures() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build()); + + Instant start = new Instant(); + Duration delay = Duration.standardSeconds(10); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .advanceWatermarkTo(start) + .addElements(rows.get(0)) + .addElements(rows.get(1)) + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows(delay, Duration.ZERO)); + + PAssert.that(output.apply(Count.globally())).containsInAnyOrder(1L); + PAssert.that(output).containsInAnyOrder( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build() + ); + pipeline.run(); + } + + @Test + public void testStream_shouldIncludeRowAddedOnTimerEdge() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f3").setValue(Values.ofInt32(3))) + .build()); + + Instant start = new Instant(); + Duration delay = Duration.standardSeconds(10); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .advanceWatermarkTo(start) + .addElements(rows.get(0)) + .advanceWatermarkTo(start.plus(delay)) + // This row will be included in the same pane because it's exactly + // on the same water mark as the onTimer event + .addElements(rows.get(1)) + .advanceWatermarkTo(start.plus(delay).plus(delay).plus(delay)) + .addElements(rows.get(2)) + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input + .apply(new CoalesceFeatureRows(delay, Duration.ZERO)); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(2L)); + PAssert.that(output.apply(new NormalizeFeatureRows())).containsInAnyOrder( + normalize(FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build()), + normalize(FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f3").setValue(Values.ofInt32(3))) + .build()) + ); + pipeline.run(); + } + + @Test + public void testStream_shouldMergeFeatures_emittingPanes_overlappingTimers() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build() + ); + + Instant start = new Instant(); + Duration delay = Duration.standardSeconds(10); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .advanceWatermarkTo(start) + .addElements(rows.get(0)) + .advanceWatermarkTo(start.plus(delay.dividedBy(2))) + // second event before time triggers + .addElements(rows.get(1)) + .advanceWatermarkTo(start.plus(delay)) + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows(delay, Duration.ZERO)); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(1L)); + PAssert.that(output.apply(new NormalizeFeatureRows())).containsInAnyOrder( + normalize(FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + ) + ); + pipeline.run(); + } + + @Test + public void testStream_shouldNotSetTimerWhilePending() { + Instant start = new Instant(); + Duration delay = Duration.standardSeconds(10); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .advanceWatermarkTo(start) + .addElements(DEFAULT_FEATURE_ROW) + // this should not reset the timer as the first is still pending. + .advanceWatermarkTo(start.plus(delay.dividedBy(2))) + .addElements(DEFAULT_FEATURE_ROW) + // timer should trigger causing the first output row + .advanceWatermarkTo(start.plus(delay).plus(delay.dividedBy(2))) + .addElements(FeatureRow.getDefaultInstance()) // this should cause a second output row. + .advanceWatermarkTo(start.plus(delay).plus(delay)) + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows(delay, Duration.ZERO)); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(2L)); + PAssert.that(output).containsInAnyOrder(DEFAULT_FEATURE_ROW, DEFAULT_FEATURE_ROW); + pipeline.run(); + } + + @Test + public void testStream_shouldOnlyEmitNewFeaturesInSecondPane() { + Instant start = new Instant(); + Duration delay = Duration.standardSeconds(10); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .advanceWatermarkTo(start) + .addElements( + FeatureRow.newBuilder() + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Features.of("f1", Values.ofString("a"))) + .build()) + // this should should emit a row + .advanceWatermarkTo(start.plus(delay).plus(delay)) + .addElements( + FeatureRow.newBuilder() + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Features.of("f2", Values.ofString("b"))) + .build()) + // this should emit a row with f2 but without f1 because it hasn't had an update + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows(delay, Duration.ZERO)); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(2L)); + PAssert.that(output).containsInAnyOrder( + FeatureRow.newBuilder() + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Features.of("f1", Values.ofString("a"))) + .build(), + FeatureRow.newBuilder() + .setEventTimestamp(DEFAULT_TIMESTAMP) + .addFeatures(Features.of("f2", Values.ofString("b"))) + .build()); + pipeline.run(); + } + + @Test + public void test_combineFeatureRows_shouldCountRows() { + List rows = Lists.newArrayList( + FeatureRow.getDefaultInstance(), + FeatureRow.getDefaultInstance(), + FeatureRow.getDefaultInstance()); + CoalesceAccum accum = CoalesceFeatureRows + .combineFeatureRows(CoalesceAccum.getDefaultInstance(), rows); + assertEquals(3, accum.getCounter()); + } + + @Test + public void test_combineFeatureRows_shouldOverwriteWhenLaterEventTimestampProcessedSecond() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("a"))) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(1)) + .build(), + FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("b"))) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) + .build()); + CoalesceAccum accum = CoalesceFeatureRows + .combineFeatureRows(CoalesceAccum.getDefaultInstance(), rows); + assertEquals(accum.getFeaturesMap().get("f1"), Features.of("f1", Values.ofString("b"))); + } + + @Test + public void test_combineFeatureRows_shouldNotOverwriteWhenEarlierEventTimestampProcessedSecond() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("b"))) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) + .build(), + FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("a"))) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(1)) + .build()); + CoalesceAccum accum = CoalesceFeatureRows + .combineFeatureRows(CoalesceAccum.getDefaultInstance(), rows); + assertEquals(accum.getFeaturesMap().get("f1"), Features.of("f1", Values.ofString("b"))); + } + + @Test + public void test_combineFeatureRows_shouldOverwriteWhenSameEventTimestampProcessedSecond() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("a"))) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) + .build(), + FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("b"))) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) + .build() + ); + CoalesceAccum accum = CoalesceFeatureRows + .combineFeatureRows(CoalesceAccum.getDefaultInstance(), rows); + assertEquals(accum.getFeaturesMap().get("f1"), Features.of("f1", Values.ofString("b"))); + } + + @Test + public void test_shouldPickLatestFeatures() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp( + Timestamp.newBuilder().setSeconds(1)) // old row with non unique feature + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(2))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(1)) // old row with unique feature + .addFeatures(Feature.newBuilder().setId("f3").setValue(Values.ofInt32(3))) + .build()); + + CoalesceAccum accum = CoalesceFeatureRows + .combineFeatureRows(CoalesceAccum.getDefaultInstance(), rows); + assertEquals(3, accum.getCounter()); + assertThat(toFeatureRow(accum, 0), equalTo( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(2))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .addFeatures(Feature.newBuilder().setId("f3").setValue(Values.ofInt32(3))) + .build() + )); + } + + @Test + public void testStream_withNoInput() { + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows()); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(0L)); + pipeline.run(); + } + + @Test + public void testBatch_withNoInput() { + PCollection input = pipeline.apply(Create.empty(ProtoCoder.of(FeatureRow.class))); + PCollection output = input.apply(new CoalesceFeatureRows()); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(0L)); + pipeline.run(); + } + + @Test + public void testStream_withTimeout_shouldRemoveState() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build() + ); + + Instant start = new Instant(); + Duration delay = Duration.standardSeconds(10); + Duration timeout = Duration.standardMinutes(30); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .addElements(rows.get(0)) + .advanceWatermarkTo(start.plus(timeout)) + // first element should get fired, as the delay water mark is reached before the timeout + // watermark, then state should be cleared when it reaches the timeout watermark. + .addElements(rows.get(1)) + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows(delay, timeout)); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(2L)); + PAssert.that(output.apply(new NormalizeFeatureRows())).containsInAnyOrder( + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .setEventTimestamp(Timestamp.getDefaultInstance()) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .setEventTimestamp(Timestamp.getDefaultInstance()) + .build() + ); + pipeline.run(); + } + + @Test + public void testStream_withDelayAfterTimeout_shouldProcessBagBeforeClear() { + List rows = Lists.newArrayList( + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .build(), + FeatureRow.newBuilder().setEntityKey("1") + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build() + ); + + Instant start = new Instant(); + Duration delay = Duration.standardMinutes(40); + Duration timeout = Duration.standardMinutes(30); + TestStream testStream = TestStream.create(ProtoCoder.of(FeatureRow.class)) + .addElements(rows.get(0)) + .addElements(rows.get(1)) + // first element should get fired, as the delay water mark is reached before the timeout + // watermark, then state should be cleared when it reaches the timeout watermark. + // If it didn't process the bag before clearing it, we'd get no output events at all. + .advanceWatermarkToInfinity(); + + PCollection input = pipeline.apply(testStream); + PCollection output = input.apply(new CoalesceFeatureRows(delay, timeout)); + + PAssert.that(output).satisfies(FeastMatchers.hasCount(1L)); + PAssert.that(output.apply(new NormalizeFeatureRows())).containsInAnyOrder( + FeatureRow.newBuilder().setEntityKey("1") + .setEventTimestamp(Timestamp.getDefaultInstance()) + .addFeatures(Feature.newBuilder().setId("f1").setValue(Values.ofInt32(1))) + .addFeatures(Feature.newBuilder().setId("f2").setValue(Values.ofInt32(2))) + .build() + ); + pipeline.run(); + } + + @Test + public void test_toFeatureRow_shouldBeNewMarkedFeaturesOnly() { + CoalesceAccum accum = CoalesceAccum.newBuilder() + .putFeatures("f1", Features.of("f1", Values.ofString("a"))) + .putFeatures("f2", Features.of("f2", Values.ofString("b"))) + .putFeatures("f3", Features.of("f3", Values.ofString("c"))) + .putFeatures("f4", Features.of("f4", Values.ofString("d"))) + .putFeatureMarks("f1", 1) + .putFeatureMarks("f2", 1) + .putFeatureMarks("f3", 2) + .putFeatureMarks("f4", 3) + .setCounter(3) + .build(); + + FeatureRow output = normalize(toFeatureRow(accum, 0)); + assertThat(output, + equalTo(FeatureRow.newBuilder() + .addFeatures(Features.of("f1", Values.ofString("a"))) + .addFeatures(Features.of("f2", Values.ofString("b"))) + .addFeatures(Features.of("f3", Values.ofString("c"))) + .addFeatures(Features.of("f4", Values.ofString("d"))) + .setEventTimestamp(DEFAULT_TIMESTAMP) + .build())); + output = normalize(toFeatureRow(accum, 1)); + assertThat(output, + equalTo(FeatureRow.newBuilder() + .addFeatures(Features.of("f3", Values.ofString("c"))) + .addFeatures(Features.of("f4", Values.ofString("d"))) + .setEventTimestamp(DEFAULT_TIMESTAMP) + .build())); + output = normalize(toFeatureRow(accum, 2)); + assertThat(output, + equalTo(FeatureRow.newBuilder() + .addFeatures(Features.of("f4", Values.ofString("d"))) + .setEventTimestamp(DEFAULT_TIMESTAMP) + .build())); + output = normalize(toFeatureRow(accum, 3)); + assertThat(output, + equalTo(FeatureRow.newBuilder() + .setEventTimestamp(DEFAULT_TIMESTAMP) + .build())); + } + + @Test(expected = IllegalArgumentException.class) + public void test_toFeatureRow_markTooHigh_shouldThrow() { + CoalesceAccum accum = CoalesceAccum.newBuilder() + .putFeatures("f1", Features.of("f1", Values.ofString("a"))) + .putFeatures("f2", Features.of("f2", Values.ofString("b"))) + .putFeatures("f3", Features.of("f3", Values.ofString("c"))) + .putFeatures("f4", Features.of("f4", Values.ofString("d"))) + .putFeatureMarks("f1", 1) + .putFeatureMarks("f2", 1) + .putFeatureMarks("f3", 2) + .putFeatureMarks("f4", 3) + .setCounter(3) + .build(); + normalize(toFeatureRow(accum, 4)); + // we throw an exception because use case should check that we have new features before trying + // to emit them. + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java index 7255c42f225..83c0ef3e025 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java @@ -21,7 +21,7 @@ import static feast.storage.MockErrorsStore.MOCK_ERRORS_STORE_TYPE; import feast.ingestion.model.Specs; -import feast.ingestion.options.ImportJobOptions; +import feast.ingestion.options.ImportJobPipelineOptions; import feast.storage.MockErrorsStore; import feast.storage.service.ErrorsStoreService; import feast.storage.stderr.StderrErrorsStore; @@ -49,14 +49,14 @@ public class ErrorsStoreTransformTest { @Rule public TestPipeline pipeline = TestPipeline.create(); - private ImportJobOptions options; + private ImportJobPipelineOptions options; private Specs specs; private PCollection inputs; private List errors; @Before public void setUp() { - options = PipelineOptionsFactory.create().as(ImportJobOptions.class); + options = PipelineOptionsFactory.create().as(ImportJobPipelineOptions.class); options.setJobName("test"); specs = Specs.builder().jobName("test").build(); diff --git a/ingestion/src/test/java/feast/storage/postgres/PostgresOptionsTest.java b/ingestion/src/test/java/feast/storage/postgres/PostgresOptionsTest.java deleted file mode 100644 index f5f53de26f5..00000000000 --- a/ingestion/src/test/java/feast/storage/postgres/PostgresOptionsTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.postgres; - -import static junit.framework.TestCase.fail; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; -import feast.options.OptionsParser; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@Slf4j -public class PostgresOptionsTest { - @Test - public void testJdbcOptionsUrl() { - PostgresOptions options = - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .build(), - PostgresOptions.class); - assertEquals("jdbc:postgresql://host:port/database", options.url); - } - - @Test(expected = IllegalArgumentException.class) - public void testJdbcOptionsUrlInvalid() { - PostgresOptions options = - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:missingpostgresprefix://fadfas") - .build(), - PostgresOptions.class); - } - - @Test - public void testJdbcOptionsPrefix() { - assertEquals( - "abcd", - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "abcd") - .build(), - PostgresOptions.class) - .prefix); - - assertEquals( - "ABCD", - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "ABCD") - .build(), - PostgresOptions.class) - .prefix); - - assertEquals( - "_abc", - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "_abc") - .build(), - PostgresOptions.class) - .prefix); - - assertEquals( - "abcd_ABCD_1234", - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "abcd_ABCD_1234") - .build(), - PostgresOptions.class) - .prefix); - } - - /** - * Test that invalid prefix is rejected - */ - @Test - public void testJdbcOptionsPrefixInvalid() { - try { - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "9123abcd") - .build(), - PostgresOptions.class); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("property \"prefix\" must match")); - } - - try { - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "-asdf") - .build(), - PostgresOptions.class); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("property \"prefix\" must match")); - } - - try { - OptionsParser.parse( - ImmutableMap.builder() - .put("postgres.url", "jdbc:postgresql://host:port/database") - .put("postgres.prefix", "$asdf") - .build(), - PostgresOptions.class); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("property \"prefix\" must match")); - } - } -} diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java index eb75560adfb..591736c1bed 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java @@ -17,7 +17,6 @@ package feast.storage.redis; -import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getBucketId; import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getFeatureIdSha1Prefix; import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getRedisBucketKey; import static org.junit.Assert.assertEquals; @@ -32,7 +31,6 @@ import feast.ingestion.service.SpecService; import feast.ingestion.transform.FeatureIO; import feast.ingestion.util.DateUtil; -import feast.options.OptionsParser; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; @@ -46,13 +44,10 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Set; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.joda.time.format.ISOPeriodFormat; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; @@ -228,55 +223,8 @@ public void testWriteHourGranularity() throws IOException { RedisBucketKey featureStringLatestKey = getRedisBucketKey("1", getFeatureIdSha1Prefix(featureHourString), 0L); - RedisBucketKey featureInt32ValueKey = - getRedisBucketKey("1", getFeatureIdSha1Prefix(featureHourInt32), - roundedTimestamp.getSeconds()); - RedisBucketKey featureStringValueKey = - getRedisBucketKey( - "1", getFeatureIdSha1Prefix(featureHourString), roundedTimestamp.getSeconds()); - - // TODO have a helper func for loading feature store options - Duration featureInt32BucketSize = - OptionsParser.parse( - specs.getFeatureSpec(featureHourInt32).getDataStores().getServing().getOptionsMap(), - RedisFeatureOptions.class) - .getBucketSizeDuration(); - - RedisBucketKey featureInt32BucketKey = - getRedisBucketKey( - "1", - getFeatureIdSha1Prefix(featureHourInt32), - getBucketId(roundedTimestamp, featureInt32BucketSize)); - RedisBucketKey featureStringBucketKey = - getRedisBucketKey( - "1", - getFeatureIdSha1Prefix(featureHourString), - // No bucketsize specified so uses the default. - getBucketId( - roundedTimestamp, - ISOPeriodFormat.standard() - .parsePeriod(RedisFeatureOptions.DEFAULT_BUCKET_SIZE) - .toStandardDuration())); - checkRedisValue(featureInt32LatestKey, Values.ofInt32(1), roundedTimestamp); checkRedisValue(featureStringLatestKey, Values.ofString("a"), roundedTimestamp); - - checkRedisValue(featureInt32ValueKey, Values.ofInt32(1), roundedTimestamp); - checkRedisValue(featureStringValueKey, Values.ofString("a"), roundedTimestamp); - - checkRedisValue(featureInt32BucketKey, featureInt32ValueKey, roundedTimestamp); - checkRedisValue(featureStringBucketKey, featureStringValueKey, roundedTimestamp); - } - - /** - * Check that a key's value is another key. - */ - void checkRedisValue(RedisBucketKey key, RedisBucketKey expectedValue, Timestamp timestamp) - throws InvalidProtocolBufferException { - Set result = jedis.zrangeByScore(key.toByteArray(), 0, Long.MAX_VALUE); - assertEquals(1, result.size()); - RedisBucketKey value = RedisBucketKey.parseFrom(result.iterator().next()); - assertEquals(expectedValue, value); } void checkRedisValue(RedisBucketKey key, Value expectedValue, Timestamp expectedTimestamp) diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java index f2ca4487dc5..242042fb357 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java +++ b/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java @@ -17,6 +17,8 @@ package feast.storage.redis; +import feast.ingestion.util.DateUtil; +import feast.storage.RedisProto.RedisBucketKey; import org.apache.beam.sdk.testing.NeedsRunner; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.commons.codec.digest.DigestUtils; @@ -25,12 +27,11 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import feast.ingestion.util.DateUtil; -import feast.storage.RedisProto.RedisBucketKey; public class FeatureRowToRedisMutationDoFnTest { - @Rule public final transient TestPipeline pipeline = TestPipeline.create(); + @Rule + public final transient TestPipeline pipeline = TestPipeline.create(); @Test public void testRedisBucketKeySize() { diff --git a/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java b/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java index cc94bfa98af..87b7fc98fad 100644 --- a/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java @@ -19,10 +19,10 @@ import static org.junit.Assert.assertEquals; +import feast.options.OptionsParser; import java.util.HashMap; import java.util.Map; import org.junit.Test; -import feast.options.OptionsParser; public class RedisStoreOptionsTest { diff --git a/ingestion/src/test/resources/logback-test.xml b/ingestion/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..9a301f3bc63 --- /dev/null +++ b/ingestion/src/test/resources/logback-test.xml @@ -0,0 +1,28 @@ + + + + + + + %d{HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n + + + + + + \ No newline at end of file diff --git a/protos/feast/specs/ImportSpec.proto b/protos/feast/specs/ImportSpec.proto index 53838df3338..a6cab4f71d7 100644 --- a/protos/feast/specs/ImportSpec.proto +++ b/protos/feast/specs/ImportSpec.proto @@ -25,14 +25,11 @@ option go_package = "github.com/gojek/feast/protos/generated/go/feast/specs"; import "google/protobuf/timestamp.proto"; message ImportSpec { - // type should define what sort of store it is - // pubsub: topic, subscription (already a featureRow) - // kafka: ? (already a featureRow) - // file: csv, parquet, json - // bigquery: table, query string type = 1; - map options = 2; + map sourceOptions = 2; + + map jobOptions = 5; repeated string entities = 3; @@ -41,7 +38,8 @@ message ImportSpec { message Schema { repeated Field fields = 1; - oneof timestamp { // the event timestamp to set per row. + // the event timestamp to set per row. + oneof timestamp { string timestampColumn = 5; google.protobuf.Timestamp timestampValue = 6; } diff --git a/protos/generated/go/feast/core/DatasetService.pb.go b/protos/generated/go/feast/core/DatasetService.pb.go new file mode 100644 index 00000000000..d2d3b01c932 --- /dev/null +++ b/protos/generated/go/feast/core/DatasetService.pb.go @@ -0,0 +1,388 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: feast/core/DatasetService.proto + +package core // import "github.com/gojek/feast/protos/generated/go/feast/core" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import timestamp "github.com/golang/protobuf/ptypes/timestamp" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DatasetServiceTypes struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasetServiceTypes) Reset() { *m = DatasetServiceTypes{} } +func (m *DatasetServiceTypes) String() string { return proto.CompactTextString(m) } +func (*DatasetServiceTypes) ProtoMessage() {} +func (*DatasetServiceTypes) Descriptor() ([]byte, []int) { + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0} +} +func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasetServiceTypes.Unmarshal(m, b) +} +func (m *DatasetServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasetServiceTypes.Marshal(b, m, deterministic) +} +func (dst *DatasetServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes.Merge(dst, src) +} +func (m *DatasetServiceTypes) XXX_Size() int { + return xxx_messageInfo_DatasetServiceTypes.Size(m) +} +func (m *DatasetServiceTypes) XXX_DiscardUnknown() { + xxx_messageInfo_DatasetServiceTypes.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasetServiceTypes proto.InternalMessageInfo + +type DatasetServiceTypes_CreateDatasetRequest struct { + // set of features for which its training data should be created + FeatureSet *FeatureSet `protobuf:"bytes,1,opt,name=featureSet,proto3" json:"featureSet,omitempty"` + // start date of the training data (inclusive) + StartDate *timestamp.Timestamp `protobuf:"bytes,2,opt,name=startDate,proto3" json:"startDate,omitempty"` + // end date of the training data (inclusive) + EndDate *timestamp.Timestamp `protobuf:"bytes,3,opt,name=endDate,proto3" json:"endDate,omitempty"` + // (optional) number of row that should be generated + // (default: none) + Limit int64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + // (optional) prefix for dataset name + NamePrefix string `protobuf:"bytes,5,opt,name=namePrefix,proto3" json:"namePrefix,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasetServiceTypes_CreateDatasetRequest) Reset() { + *m = DatasetServiceTypes_CreateDatasetRequest{} +} +func (m *DatasetServiceTypes_CreateDatasetRequest) String() string { return proto.CompactTextString(m) } +func (*DatasetServiceTypes_CreateDatasetRequest) ProtoMessage() {} +func (*DatasetServiceTypes_CreateDatasetRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 0} +} +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Unmarshal(m, b) +} +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Marshal(b, m, deterministic) +} +func (dst *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(dst, src) +} +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Size() int { + return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Size(m) +} +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest proto.InternalMessageInfo + +func (m *DatasetServiceTypes_CreateDatasetRequest) GetFeatureSet() *FeatureSet { + if m != nil { + return m.FeatureSet + } + return nil +} + +func (m *DatasetServiceTypes_CreateDatasetRequest) GetStartDate() *timestamp.Timestamp { + if m != nil { + return m.StartDate + } + return nil +} + +func (m *DatasetServiceTypes_CreateDatasetRequest) GetEndDate() *timestamp.Timestamp { + if m != nil { + return m.EndDate + } + return nil +} + +func (m *DatasetServiceTypes_CreateDatasetRequest) GetLimit() int64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *DatasetServiceTypes_CreateDatasetRequest) GetNamePrefix() string { + if m != nil { + return m.NamePrefix + } + return "" +} + +type DatasetServiceTypes_CreateDatasetResponse struct { + // information of the created training dataset + DatasetInfo *DatasetInfo `protobuf:"bytes,1,opt,name=datasetInfo,proto3" json:"datasetInfo,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasetServiceTypes_CreateDatasetResponse) Reset() { + *m = DatasetServiceTypes_CreateDatasetResponse{} +} +func (m *DatasetServiceTypes_CreateDatasetResponse) String() string { return proto.CompactTextString(m) } +func (*DatasetServiceTypes_CreateDatasetResponse) ProtoMessage() {} +func (*DatasetServiceTypes_CreateDatasetResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 1} +} +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Unmarshal(m, b) +} +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Marshal(b, m, deterministic) +} +func (dst *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(dst, src) +} +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Size() int { + return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Size(m) +} +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse proto.InternalMessageInfo + +func (m *DatasetServiceTypes_CreateDatasetResponse) GetDatasetInfo() *DatasetInfo { + if m != nil { + return m.DatasetInfo + } + return nil +} + +// Represent a collection of feature having same entity name +type FeatureSet struct { + // entity related to this feature set + EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` + // list of feature id in this feature set + FeatureIds []string `protobuf:"bytes,2,rep,name=featureIds,proto3" json:"featureIds,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FeatureSet) Reset() { *m = FeatureSet{} } +func (m *FeatureSet) String() string { return proto.CompactTextString(m) } +func (*FeatureSet) ProtoMessage() {} +func (*FeatureSet) Descriptor() ([]byte, []int) { + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{1} +} +func (m *FeatureSet) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FeatureSet.Unmarshal(m, b) +} +func (m *FeatureSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FeatureSet.Marshal(b, m, deterministic) +} +func (dst *FeatureSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSet.Merge(dst, src) +} +func (m *FeatureSet) XXX_Size() int { + return xxx_messageInfo_FeatureSet.Size(m) +} +func (m *FeatureSet) XXX_DiscardUnknown() { + xxx_messageInfo_FeatureSet.DiscardUnknown(m) +} + +var xxx_messageInfo_FeatureSet proto.InternalMessageInfo + +func (m *FeatureSet) GetEntityName() string { + if m != nil { + return m.EntityName + } + return "" +} + +func (m *FeatureSet) GetFeatureIds() []string { + if m != nil { + return m.FeatureIds + } + return nil +} + +// Representation of training dataset information +type DatasetInfo struct { + // name of dataset + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // URL to table location of the training dataset + TableUrl string `protobuf:"bytes,2,opt,name=tableUrl,proto3" json:"tableUrl,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasetInfo) Reset() { *m = DatasetInfo{} } +func (m *DatasetInfo) String() string { return proto.CompactTextString(m) } +func (*DatasetInfo) ProtoMessage() {} +func (*DatasetInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{2} +} +func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasetInfo.Unmarshal(m, b) +} +func (m *DatasetInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasetInfo.Marshal(b, m, deterministic) +} +func (dst *DatasetInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetInfo.Merge(dst, src) +} +func (m *DatasetInfo) XXX_Size() int { + return xxx_messageInfo_DatasetInfo.Size(m) +} +func (m *DatasetInfo) XXX_DiscardUnknown() { + xxx_messageInfo_DatasetInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasetInfo proto.InternalMessageInfo + +func (m *DatasetInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *DatasetInfo) GetTableUrl() string { + if m != nil { + return m.TableUrl + } + return "" +} + +func init() { + proto.RegisterType((*DatasetServiceTypes)(nil), "feast.core.DatasetServiceTypes") + proto.RegisterType((*DatasetServiceTypes_CreateDatasetRequest)(nil), "feast.core.DatasetServiceTypes.CreateDatasetRequest") + proto.RegisterType((*DatasetServiceTypes_CreateDatasetResponse)(nil), "feast.core.DatasetServiceTypes.CreateDatasetResponse") + proto.RegisterType((*FeatureSet)(nil), "feast.core.FeatureSet") + proto.RegisterType((*DatasetInfo)(nil), "feast.core.DatasetInfo") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// DatasetServiceClient is the client API for DatasetService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type DatasetServiceClient interface { + // Create training dataset for a feature set + CreateDataset(ctx context.Context, in *DatasetServiceTypes_CreateDatasetRequest, opts ...grpc.CallOption) (*DatasetServiceTypes_CreateDatasetResponse, error) +} + +type datasetServiceClient struct { + cc *grpc.ClientConn +} + +func NewDatasetServiceClient(cc *grpc.ClientConn) DatasetServiceClient { + return &datasetServiceClient{cc} +} + +func (c *datasetServiceClient) CreateDataset(ctx context.Context, in *DatasetServiceTypes_CreateDatasetRequest, opts ...grpc.CallOption) (*DatasetServiceTypes_CreateDatasetResponse, error) { + out := new(DatasetServiceTypes_CreateDatasetResponse) + err := c.cc.Invoke(ctx, "/feast.core.DatasetService/CreateDataset", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DatasetServiceServer is the server API for DatasetService service. +type DatasetServiceServer interface { + // Create training dataset for a feature set + CreateDataset(context.Context, *DatasetServiceTypes_CreateDatasetRequest) (*DatasetServiceTypes_CreateDatasetResponse, error) +} + +func RegisterDatasetServiceServer(s *grpc.Server, srv DatasetServiceServer) { + s.RegisterService(&_DatasetService_serviceDesc, srv) +} + +func _DatasetService_CreateDataset_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DatasetServiceTypes_CreateDatasetRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DatasetServiceServer).CreateDataset(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.DatasetService/CreateDataset", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DatasetServiceServer).CreateDataset(ctx, req.(*DatasetServiceTypes_CreateDatasetRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _DatasetService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "feast.core.DatasetService", + HandlerType: (*DatasetServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CreateDataset", + Handler: _DatasetService_CreateDataset_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "feast/core/DatasetService.proto", +} + +func init() { + proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_37ae639a8c7b5dd5) +} + +var fileDescriptor_DatasetService_37ae639a8c7b5dd5 = []byte{ + // 414 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, + 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, + 0xa5, 0xd2, 0x22, 0x38, 0x70, 0x29, 0x2b, 0xa4, 0x4a, 0x08, 0x55, 0x6e, 0x91, 0x10, 0x37, 0x67, + 0x33, 0x09, 0x81, 0x24, 0x0e, 0xf6, 0x04, 0xd1, 0x0b, 0xdf, 0xc0, 0x27, 0xf2, 0x11, 0x7c, 0x00, + 0x8a, 0xb3, 0xdb, 0x78, 0xab, 0x4a, 0x88, 0x9b, 0x3d, 0xef, 0xcd, 0x78, 0xde, 0xf8, 0x0d, 0x1c, + 0x16, 0xa8, 0x2c, 0x89, 0xb5, 0x36, 0x28, 0x56, 0x8a, 0x94, 0x45, 0xba, 0x40, 0xf3, 0xbd, 0x5a, + 0x23, 0xef, 0x8c, 0x26, 0xcd, 0xc0, 0x11, 0xf8, 0x40, 0x88, 0x0f, 0x4b, 0xad, 0xcb, 0x1a, 0x85, + 0x43, 0xb2, 0xbe, 0x10, 0x54, 0x35, 0x68, 0x49, 0x35, 0xdd, 0x48, 0x4e, 0x7f, 0xcf, 0xe0, 0xd1, + 0x6e, 0x95, 0xcb, 0xab, 0x0e, 0x6d, 0xfc, 0x27, 0x80, 0x83, 0x37, 0x06, 0x15, 0xe1, 0x06, 0x95, + 0xf8, 0xad, 0x47, 0x4b, 0xec, 0x05, 0x0c, 0xf5, 0xa9, 0x37, 0x78, 0x81, 0x14, 0x05, 0x49, 0xb0, + 0x5c, 0x1c, 0x3d, 0xe1, 0xd3, 0x93, 0xfc, 0xed, 0x35, 0x2a, 0x3d, 0x26, 0x7b, 0x09, 0xa1, 0x25, + 0x65, 0x68, 0xa5, 0x08, 0xa3, 0x99, 0x4b, 0x8b, 0xf9, 0xd8, 0x1d, 0xdf, 0x76, 0xc7, 0x2f, 0xb7, + 0xdd, 0xc9, 0x89, 0xcc, 0x8e, 0xe1, 0x1e, 0xb6, 0xb9, 0xcb, 0x9b, 0xff, 0x33, 0x6f, 0x4b, 0x65, + 0x07, 0xb0, 0x5f, 0x57, 0x4d, 0x45, 0xd1, 0x5e, 0x12, 0x2c, 0xe7, 0x72, 0xbc, 0xb0, 0x67, 0x00, + 0xad, 0x6a, 0xf0, 0xdc, 0x60, 0x51, 0xfd, 0x88, 0xf6, 0x93, 0x60, 0x19, 0x4a, 0x2f, 0x12, 0x4b, + 0x78, 0x7c, 0x43, 0xb5, 0xed, 0x74, 0x6b, 0x91, 0xbd, 0x82, 0x45, 0x3e, 0x86, 0xce, 0xda, 0x42, + 0x6f, 0x74, 0x3f, 0xf5, 0x75, 0xaf, 0x26, 0x58, 0xfa, 0xdc, 0xf4, 0x1d, 0xc0, 0x34, 0x93, 0xa1, + 0x03, 0x6c, 0xa9, 0xa2, 0xab, 0xf7, 0xaa, 0x41, 0x57, 0x27, 0x94, 0x5e, 0x64, 0xc0, 0x37, 0x53, + 0x3b, 0xcb, 0x6d, 0x34, 0x4b, 0xe6, 0x03, 0x3e, 0x45, 0xd2, 0xd7, 0xb0, 0xf0, 0x5e, 0x62, 0x0c, + 0xf6, 0xda, 0xa9, 0x90, 0x3b, 0xb3, 0x18, 0xee, 0x93, 0xca, 0x6a, 0xfc, 0x60, 0x6a, 0x37, 0xe9, + 0x50, 0x5e, 0xdf, 0x8f, 0x7e, 0x05, 0xf0, 0x70, 0xf7, 0xbf, 0xd9, 0x4f, 0x78, 0xb0, 0xa3, 0x99, + 0x1d, 0xdf, 0x22, 0xcb, 0x37, 0x07, 0xbf, 0xcd, 0x18, 0xf1, 0xc9, 0x7f, 0x66, 0x8d, 0x83, 0x4d, + 0xef, 0x9c, 0x7e, 0x04, 0xcf, 0xb1, 0xa7, 0x37, 0xdc, 0x78, 0x3e, 0x7c, 0xf1, 0xa7, 0x93, 0xb2, + 0xa2, 0xcf, 0x7d, 0xc6, 0xd7, 0xba, 0x11, 0xa5, 0xfe, 0x82, 0x5f, 0xc5, 0xb8, 0x04, 0xce, 0x00, + 0x56, 0x94, 0xd8, 0xa2, 0x51, 0x84, 0xb9, 0x28, 0xb5, 0x98, 0xd6, 0x23, 0xbb, 0xeb, 0xf0, 0xe7, + 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x9e, 0x4d, 0xe0, 0x33, 0x03, 0x00, 0x00, +} diff --git a/protos/generated/go/feast/specs/ImportSpec.pb.go b/protos/generated/go/feast/specs/ImportSpec.pb.go index d4cca689ba3..d16f9fa07bd 100644 --- a/protos/generated/go/feast/specs/ImportSpec.pb.go +++ b/protos/generated/go/feast/specs/ImportSpec.pb.go @@ -6,7 +6,6 @@ package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import _ "github.com/gojek/feast/protos/generated/go/feast/types" import timestamp "github.com/golang/protobuf/ptypes/timestamp" // Reference imports to suppress errors if they are not otherwise used. @@ -21,13 +20,9 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type ImportSpec struct { - // type should define what sort of store it is - // pubsub: topic, subscription (already a featureRow) - // kafka: ? (already a featureRow) - // file: csv, parquet, json - // bigquery: table, query Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - Options map[string]string `protobuf:"bytes,2,rep,name=options,proto3" json:"options,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + SourceOptions map[string]string `protobuf:"bytes,2,rep,name=sourceOptions,proto3" json:"sourceOptions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + JobOptions map[string]string `protobuf:"bytes,5,rep,name=jobOptions,proto3" json:"jobOptions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` Entities []string `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty"` Schema *Schema `protobuf:"bytes,4,opt,name=schema,proto3" json:"schema,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -39,7 +34,7 @@ func (m *ImportSpec) Reset() { *m = ImportSpec{} } func (m *ImportSpec) String() string { return proto.CompactTextString(m) } func (*ImportSpec) ProtoMessage() {} func (*ImportSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_d4d069df5227ffcf, []int{0} + return fileDescriptor_ImportSpec_673bc4f248a91137, []int{0} } func (m *ImportSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportSpec.Unmarshal(m, b) @@ -66,9 +61,16 @@ func (m *ImportSpec) GetType() string { return "" } -func (m *ImportSpec) GetOptions() map[string]string { +func (m *ImportSpec) GetSourceOptions() map[string]string { if m != nil { - return m.Options + return m.SourceOptions + } + return nil +} + +func (m *ImportSpec) GetJobOptions() map[string]string { + if m != nil { + return m.JobOptions } return nil } @@ -89,6 +91,8 @@ func (m *ImportSpec) GetSchema() *Schema { type Schema struct { Fields []*Field `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty"` + // the event timestamp to set per row. + // // Types that are valid to be assigned to Timestamp: // *Schema_TimestampColumn // *Schema_TimestampValue @@ -103,7 +107,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_d4d069df5227ffcf, []int{1} + return fileDescriptor_ImportSpec_673bc4f248a91137, []int{1} } func (m *Schema) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Schema.Unmarshal(m, b) @@ -256,7 +260,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_d4d069df5227ffcf, []int{2} + return fileDescriptor_ImportSpec_673bc4f248a91137, []int{2} } func (m *Field) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Field.Unmarshal(m, b) @@ -292,41 +296,44 @@ func (m *Field) GetFeatureId() string { func init() { proto.RegisterType((*ImportSpec)(nil), "feast.specs.ImportSpec") - proto.RegisterMapType((map[string]string)(nil), "feast.specs.ImportSpec.OptionsEntry") + proto.RegisterMapType((map[string]string)(nil), "feast.specs.ImportSpec.JobOptionsEntry") + proto.RegisterMapType((map[string]string)(nil), "feast.specs.ImportSpec.SourceOptionsEntry") proto.RegisterType((*Schema)(nil), "feast.specs.Schema") proto.RegisterType((*Field)(nil), "feast.specs.Field") } func init() { - proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_d4d069df5227ffcf) -} - -var fileDescriptor_ImportSpec_d4d069df5227ffcf = []byte{ - // 413 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x52, 0x5b, 0x6b, 0xd4, 0x40, - 0x14, 0x6e, 0x76, 0xbb, 0xa9, 0x7b, 0x22, 0xad, 0x8c, 0x82, 0x43, 0x28, 0x18, 0x16, 0x91, 0x50, - 0x61, 0x06, 0x2a, 0x88, 0xf6, 0xc1, 0x87, 0xf5, 0x42, 0xf7, 0x49, 0x49, 0xa5, 0x0f, 0xbe, 0x65, - 0x93, 0x93, 0x34, 0x36, 0xc9, 0x84, 0xcc, 0x44, 0xc8, 0x6f, 0xf5, 0xd9, 0xff, 0x21, 0x33, 0x93, - 0xcd, 0x66, 0xf7, 0xed, 0x5c, 0xbe, 0x73, 0xf9, 0xce, 0x77, 0xe0, 0x32, 0xc3, 0x58, 0x2a, 0x2e, - 0x1b, 0x4c, 0x24, 0xdf, 0x54, 0x8d, 0x68, 0xd5, 0x5d, 0x83, 0x09, 0x6b, 0x5a, 0xa1, 0x04, 0xf1, - 0x4c, 0x96, 0x99, 0xac, 0xff, 0xd2, 0x42, 0x55, 0xdf, 0xa0, 0xe4, 0xf7, 0x71, 0xd9, 0xa1, 0x45, - 0xf9, 0xaf, 0x72, 0x21, 0xf2, 0x12, 0xb9, 0xf1, 0xb6, 0x5d, 0xc6, 0x55, 0x51, 0xa1, 0x54, 0x71, - 0xd5, 0x58, 0xc0, 0xea, 0x9f, 0x03, 0xb0, 0xef, 0x4d, 0x08, 0x9c, 0xea, 0x26, 0xd4, 0x09, 0x9c, - 0x70, 0x19, 0x19, 0x9b, 0x7c, 0x82, 0x33, 0xd1, 0xa8, 0x42, 0xd4, 0x92, 0xce, 0x82, 0x79, 0xe8, - 0x5d, 0xbf, 0x66, 0x93, 0xd9, 0x6c, 0xb2, 0xd9, 0x77, 0x0b, 0xfb, 0x5a, 0xab, 0xb6, 0x8f, 0x76, - 0x45, 0xc4, 0x87, 0x27, 0x58, 0xab, 0x42, 0x15, 0x28, 0xe9, 0x3c, 0x98, 0x87, 0xcb, 0x68, 0xf4, - 0xc9, 0x5b, 0x70, 0x65, 0xf2, 0x80, 0x55, 0x4c, 0x4f, 0x03, 0x27, 0xf4, 0xae, 0x9f, 0x1f, 0xb4, - 0xbe, 0x33, 0xa9, 0x68, 0x80, 0xf8, 0x37, 0xf0, 0x74, 0x3a, 0x81, 0x3c, 0x83, 0xf9, 0x23, 0xf6, - 0xc3, 0xae, 0xda, 0x24, 0x2f, 0x60, 0xf1, 0x47, 0xb3, 0xa7, 0x33, 0x13, 0xb3, 0xce, 0xcd, 0xec, - 0x83, 0xb3, 0xfa, 0xeb, 0x80, 0x6b, 0xdb, 0x91, 0x2b, 0x70, 0xb3, 0x02, 0xcb, 0x54, 0x52, 0xc7, - 0xd0, 0x21, 0x07, 0x33, 0xbf, 0xe9, 0x54, 0x34, 0x20, 0xc8, 0x15, 0x5c, 0x8c, 0x17, 0xfb, 0x2c, - 0xca, 0xae, 0xaa, 0xe9, 0x42, 0xb7, 0xbe, 0x3d, 0x89, 0x8e, 0x13, 0xe4, 0x0b, 0x9c, 0x8f, 0x21, - 0xa3, 0x01, 0x75, 0x0d, 0x27, 0x9f, 0x59, 0x11, 0xd8, 0x4e, 0x04, 0xf6, 0x73, 0x07, 0xbb, 0x3d, - 0x89, 0x8e, 0x6a, 0xc8, 0x1b, 0x38, 0x37, 0xd7, 0xe9, 0x37, 0xe9, 0x30, 0xf0, 0xcc, 0x70, 0x39, - 0x8a, 0xae, 0x3d, 0x58, 0x8e, 0x95, 0xab, 0x8f, 0xb0, 0x30, 0x7b, 0x6b, 0xfd, 0xea, 0xb8, 0x1a, - 0xf5, 0xd3, 0x36, 0xb9, 0x84, 0x65, 0x86, 0xb1, 0xea, 0x5a, 0xdc, 0xa4, 0xc3, 0x61, 0xf6, 0x81, - 0xf5, 0x3d, 0x4c, 0x3f, 0x69, 0x7d, 0xb1, 0x97, 0xf3, 0x87, 0x5e, 0xf7, 0xd7, 0xfb, 0xbc, 0x50, - 0x0f, 0xdd, 0x96, 0x25, 0xa2, 0xe2, 0xb9, 0xf8, 0x8d, 0x8f, 0xdc, 0xfe, 0x9a, 0x21, 0x23, 0x79, - 0x8e, 0x35, 0xb6, 0xb1, 0xc2, 0x94, 0xe7, 0x82, 0x4f, 0x1e, 0x76, 0xeb, 0x1a, 0xc0, 0xbb, 0xff, - 0x01, 0x00, 0x00, 0xff, 0xff, 0xd1, 0x43, 0xb0, 0x30, 0xc6, 0x02, 0x00, 0x00, + proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_673bc4f248a91137) +} + +var fileDescriptor_ImportSpec_673bc4f248a91137 = []byte{ + // 440 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8f, 0x93, 0x40, + 0x14, 0x5d, 0xca, 0x16, 0xed, 0x6d, 0xdc, 0x9a, 0xab, 0x0f, 0x84, 0x6c, 0x62, 0xd3, 0x07, 0x6d, + 0x6a, 0x32, 0x93, 0xac, 0x89, 0x51, 0x13, 0x13, 0x53, 0xbf, 0xb6, 0xbe, 0xb8, 0xa1, 0x66, 0x1f, + 0x7c, 0xa3, 0x70, 0x61, 0xd9, 0x05, 0x86, 0x30, 0x83, 0x09, 0x3f, 0xc0, 0x5f, 0xe9, 0x9f, 0x31, + 0x0c, 0x94, 0x52, 0x8c, 0x0f, 0xfb, 0x36, 0xdc, 0x73, 0xee, 0xb9, 0x67, 0x2e, 0x67, 0xe0, 0x3c, + 0x24, 0x4f, 0x2a, 0x2e, 0x73, 0xf2, 0x25, 0xdf, 0xa4, 0xb9, 0x28, 0xd4, 0x36, 0x27, 0x9f, 0xe5, + 0x85, 0x50, 0x02, 0xa7, 0x1a, 0x65, 0x1a, 0x75, 0x9e, 0x45, 0x42, 0x44, 0x09, 0x71, 0x0d, 0xed, + 0xca, 0x90, 0xab, 0x38, 0x25, 0xa9, 0xbc, 0x34, 0x6f, 0xd8, 0x8b, 0xdf, 0x26, 0xc0, 0x41, 0x02, + 0x11, 0x4e, 0x55, 0x95, 0x93, 0x6d, 0xcc, 0x8d, 0xe5, 0xc4, 0xd5, 0x67, 0xbc, 0x82, 0x47, 0x52, + 0x94, 0x85, 0x4f, 0xdf, 0x73, 0x15, 0x8b, 0x4c, 0xda, 0xa3, 0xb9, 0xb9, 0x9c, 0x5e, 0xac, 0x58, + 0x6f, 0x10, 0xeb, 0xd9, 0xd8, 0xf6, 0xc9, 0x9f, 0x33, 0x55, 0x54, 0xee, 0xb1, 0x00, 0x7e, 0x05, + 0xb8, 0x15, 0xbb, 0xbd, 0xdc, 0x58, 0xcb, 0xbd, 0xf8, 0x9f, 0xdc, 0xb7, 0x8e, 0xd9, 0x68, 0xf5, + 0x5a, 0xd1, 0x81, 0x87, 0x94, 0xa9, 0x58, 0xc5, 0x24, 0x6d, 0x73, 0x6e, 0x2e, 0x27, 0x6e, 0xf7, + 0x8d, 0x2f, 0xc1, 0x92, 0xfe, 0x0d, 0xa5, 0x9e, 0x7d, 0x3a, 0x37, 0x96, 0xd3, 0x8b, 0x27, 0x47, + 0x03, 0xb6, 0x1a, 0x72, 0x5b, 0x8a, 0xf3, 0x01, 0xf0, 0x5f, 0xdb, 0xf8, 0x18, 0xcc, 0x3b, 0xaa, + 0xda, 0x65, 0xd4, 0x47, 0x7c, 0x0a, 0xe3, 0x5f, 0x5e, 0x52, 0x92, 0x3d, 0xd2, 0xb5, 0xe6, 0xe3, + 0xdd, 0xe8, 0x8d, 0xe1, 0xbc, 0x87, 0xd9, 0xc0, 0xe9, 0x7d, 0xda, 0x17, 0x7f, 0x0c, 0xb0, 0x1a, + 0x4f, 0xb8, 0x02, 0x2b, 0x8c, 0x29, 0x09, 0xa4, 0x6d, 0xe8, 0xcd, 0xe0, 0x91, 0xf1, 0x2f, 0x35, + 0xe4, 0xb6, 0x0c, 0x5c, 0xc1, 0xac, 0xfb, 0xa3, 0x1f, 0x45, 0x52, 0xa6, 0x99, 0x3d, 0xae, 0xa5, + 0x2f, 0x4f, 0xdc, 0x21, 0x80, 0x9f, 0xe0, 0xac, 0x2b, 0x5d, 0x6b, 0x17, 0x96, 0x5e, 0x8c, 0xc3, + 0x9a, 0x90, 0xb0, 0x7d, 0x48, 0xd8, 0x8f, 0x3d, 0xed, 0xf2, 0xc4, 0x1d, 0xf4, 0xe0, 0x73, 0x38, + 0xd3, 0x2b, 0xae, 0x36, 0x41, 0x3b, 0xf0, 0x81, 0xbe, 0xcb, 0xa0, 0xba, 0x9e, 0xc2, 0xa4, 0xeb, + 0x5c, 0xbc, 0x85, 0xb1, 0xf6, 0x5d, 0xe7, 0x2b, 0xf3, 0xd2, 0x2e, 0x5f, 0xf5, 0x19, 0xcf, 0x61, + 0x12, 0x92, 0xa7, 0xca, 0x82, 0x36, 0x41, 0xbb, 0x98, 0x43, 0x61, 0x7d, 0x0d, 0xfd, 0x40, 0xaf, + 0x67, 0x87, 0x64, 0x5c, 0xd5, 0x76, 0x7f, 0xbe, 0x8e, 0x62, 0x75, 0x53, 0xee, 0x98, 0x2f, 0x52, + 0x1e, 0x89, 0x5b, 0xba, 0xe3, 0xcd, 0xeb, 0xd0, 0x97, 0x91, 0x3c, 0xa2, 0x8c, 0x0a, 0x4f, 0x51, + 0xc0, 0x23, 0xc1, 0x7b, 0xef, 0x66, 0x67, 0x69, 0xc2, 0xab, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, + 0x91, 0xba, 0x56, 0x3c, 0x4d, 0x03, 0x00, 0x00, } diff --git a/protos/generated/go/feast/storage/BigTable.pb.go b/protos/generated/go/feast/storage/BigTable.pb.go index 21151396b28..e65b5e35fce 100644 --- a/protos/generated/go/feast/storage/BigTable.pb.go +++ b/protos/generated/go/feast/storage/BigTable.pb.go @@ -6,7 +6,6 @@ package storage // import "github.com/gojek/feast/protos/generated/go/feast/stor import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import _ "github.com/gojek/feast/protos/generated/go/feast/types" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -34,7 +33,7 @@ func (m *BigTableRowKey) Reset() { *m = BigTableRowKey{} } func (m *BigTableRowKey) String() string { return proto.CompactTextString(m) } func (*BigTableRowKey) ProtoMessage() {} func (*BigTableRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_BigTable_546749cadd35d8e4, []int{0} + return fileDescriptor_BigTable_e25bdeffe0669ae7, []int{0} } func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BigTableRowKey.Unmarshal(m, b) @@ -80,23 +79,22 @@ func init() { } func init() { - proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_546749cadd35d8e4) + proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_e25bdeffe0669ae7) } -var fileDescriptor_BigTable_546749cadd35d8e4 = []byte{ - // 209 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x90, 0xc1, 0x4b, 0x86, 0x30, - 0x18, 0x87, 0xf9, 0x0a, 0x02, 0x07, 0x7a, 0xd8, 0x25, 0x09, 0x89, 0xe8, 0x10, 0x9d, 0x7c, 0x89, - 0x2e, 0x9d, 0xbd, 0x4a, 0x20, 0x12, 0x11, 0xdd, 0x66, 0xbe, 0xce, 0xd5, 0x72, 0xb2, 0xbd, 0x5a, - 0xfb, 0xef, 0xa3, 0xa9, 0x94, 0xdf, 0xf5, 0xf7, 0x3c, 0x83, 0x67, 0x2f, 0xcb, 0x3a, 0x14, 0x8e, - 0xc0, 0x91, 0xb1, 0x42, 0x22, 0x14, 0x4a, 0x3e, 0x89, 0x46, 0x63, 0x3e, 0x5a, 0x43, 0x86, 0xc7, - 0x81, 0xe6, 0x2b, 0xbd, 0x38, 0x5f, 0x64, 0xf2, 0x23, 0x3a, 0x78, 0x16, 0x7a, 0x5a, 0xbd, 0xeb, - 0x99, 0x25, 0xdb, 0xcb, 0xda, 0x7c, 0x95, 0xe8, 0xf9, 0x25, 0x63, 0xae, 0x17, 0x77, 0x95, 0xc5, - 0x4e, 0x7d, 0xa7, 0x87, 0xab, 0xc3, 0x6d, 0x54, 0xff, 0x5b, 0x78, 0xc6, 0x22, 0x1c, 0x48, 0x91, - 0x2f, 0xd1, 0xa7, 0x27, 0x01, 0xff, 0x0d, 0xfc, 0x86, 0x25, 0x16, 0x67, 0xb4, 0x0e, 0xdb, 0x47, - 0xa5, 0xb5, 0x72, 0xe9, 0x69, 0x50, 0x8e, 0xd6, 0xe2, 0x85, 0xed, 0x0b, 0x8b, 0x78, 0xcb, 0xa8, - 0x7e, 0xbb, 0x5e, 0x1f, 0xa4, 0xa2, 0x7e, 0x6a, 0xf2, 0x37, 0xf3, 0x09, 0xd2, 0xbc, 0xe3, 0x07, - 0x2c, 0x3f, 0x08, 0xd5, 0x0e, 0x24, 0x0e, 0x68, 0x05, 0x61, 0x0b, 0xd2, 0xc0, 0xee, 0x10, 0xcd, - 0x59, 0x50, 0xee, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x43, 0x55, 0xcf, 0x14, 0x20, 0x01, 0x00, +var fileDescriptor_BigTable_e25bdeffe0669ae7 = []byte{ + // 193 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x8f, 0xb1, 0x0b, 0x82, 0x40, + 0x18, 0x47, 0xb1, 0x20, 0xf0, 0x40, 0x87, 0x9b, 0x1c, 0x24, 0xa2, 0x21, 0x9a, 0x3c, 0xa2, 0xa5, + 0xd9, 0x55, 0x02, 0x91, 0x86, 0x68, 0x3b, 0xf3, 0xf3, 0xbc, 0x32, 0x2f, 0xee, 0xbe, 0x2c, 0xff, + 0xfb, 0xe8, 0x4a, 0xca, 0xd6, 0xdf, 0xef, 0x0d, 0xef, 0x91, 0xb0, 0x04, 0x6e, 0x90, 0x19, 0x54, + 0x9a, 0x0b, 0x60, 0xb1, 0x14, 0x3b, 0x9e, 0xd7, 0x10, 0x5d, 0xb5, 0x42, 0x45, 0x3d, 0xfb, 0x46, + 0x9f, 0x77, 0xde, 0x12, 0xbf, 0x07, 0x32, 0x75, 0x4f, 0xa0, 0xa3, 0x53, 0x42, 0x4c, 0xc5, 0x57, + 0xa9, 0x86, 0x52, 0x3e, 0x02, 0x67, 0xe6, 0x2c, 0xdd, 0xec, 0x67, 0xa1, 0x21, 0x71, 0xa1, 0x41, + 0x89, 0x5d, 0x02, 0x5d, 0x30, 0xb2, 0xf7, 0x77, 0xa0, 0x0b, 0xe2, 0x6b, 0x68, 0x41, 0x1b, 0x28, + 0xb6, 0xb2, 0xae, 0xa5, 0x09, 0xc6, 0x16, 0xf9, 0x5b, 0xe3, 0x3d, 0x19, 0x8a, 0xc4, 0x5e, 0xaf, + 0x91, 0xbe, 0x34, 0x0f, 0x1b, 0x21, 0xb1, 0xba, 0xe5, 0xd1, 0x51, 0x5d, 0x98, 0x50, 0x27, 0x38, + 0xb3, 0x77, 0x95, 0x8d, 0x30, 0x4c, 0x40, 0x03, 0x9a, 0x23, 0x14, 0x4c, 0x28, 0x36, 0xe8, 0xcd, + 0x27, 0x16, 0x59, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc5, 0xd9, 0xf3, 0xb2, 0x07, 0x01, 0x00, 0x00, } diff --git a/protos/generated/go/feast/types/Feature.pb.go b/protos/generated/go/feast/types/Feature.pb.go index 5421909cfeb..51ed41fb54f 100644 --- a/protos/generated/go/feast/types/Feature.pb.go +++ b/protos/generated/go/feast/types/Feature.pb.go @@ -6,7 +6,6 @@ package types // import "github.com/gojek/feast/protos/generated/go/feast/types" import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import _ "github.com/golang/protobuf/ptypes/timestamp" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -31,7 +30,7 @@ func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_Feature_26989c9d13406a0c, []int{0} + return fileDescriptor_Feature_c2a5d99d9bf3ca9c, []int{0} } func (m *Feature) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Feature.Unmarshal(m, b) @@ -69,21 +68,19 @@ func init() { proto.RegisterType((*Feature)(nil), "feast.types.Feature") } -func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_26989c9d13406a0c) } +func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_c2a5d99d9bf3ca9c) } -var fileDescriptor_Feature_26989c9d13406a0c = []byte{ - // 197 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x8e, 0xb1, 0x6a, 0xc3, 0x30, - 0x14, 0x45, 0xb1, 0xa1, 0x2d, 0x95, 0x4b, 0x07, 0x2d, 0x75, 0xbd, 0xd4, 0x74, 0xf2, 0xa4, 0x07, - 0x09, 0xe4, 0x03, 0x1c, 0xc8, 0x1c, 0x1c, 0xc8, 0x90, 0x4d, 0x8e, 0x9f, 0x15, 0x25, 0x76, 0x64, - 0xac, 0xa7, 0x40, 0xfe, 0x3e, 0x58, 0xf6, 0xa0, 0xf9, 0x1e, 0xce, 0x3d, 0xec, 0xb7, 0x45, 0x69, - 0x09, 0xe8, 0x39, 0xa0, 0x85, 0x1d, 0x4a, 0x72, 0x23, 0x8a, 0x61, 0x34, 0x64, 0x78, 0xe2, 0x27, - 0xe1, 0xa7, 0xec, 0x4f, 0x19, 0xa3, 0x3a, 0x04, 0x3f, 0xd5, 0xae, 0x05, 0xd2, 0x3d, 0x5a, 0x92, - 0xfd, 0x30, 0xd3, 0xd9, 0x4f, 0x28, 0x3a, 0xca, 0xce, 0x2d, 0x9a, 0xff, 0x2d, 0xfb, 0x58, 0xbc, - 0xfc, 0x9b, 0xc5, 0xba, 0x49, 0xa3, 0x3c, 0x2a, 0x3e, 0xab, 0x58, 0x37, 0xbc, 0x60, 0x6f, 0x8f, - 0x89, 0x4c, 0xe3, 0x3c, 0x2a, 0x92, 0x15, 0x17, 0xc1, 0xa3, 0xf0, 0x8e, 0x6a, 0x06, 0xca, 0x03, - 0x0b, 0x6b, 0xca, 0xaf, 0xc5, 0xb8, 0x9f, 0x1e, 0x4e, 0x1b, 0xa5, 0xe9, 0xe2, 0x6a, 0x71, 0x36, - 0x3d, 0x28, 0x73, 0xc5, 0x1b, 0xcc, 0x2d, 0xfe, 0xdf, 0x82, 0xc2, 0x3b, 0x8e, 0x92, 0xb0, 0x01, - 0x65, 0x20, 0xa8, 0xac, 0xdf, 0x3d, 0xb0, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, 0xbf, 0x23, 0xad, - 0x34, 0x04, 0x01, 0x00, 0x00, +var fileDescriptor_Feature_c2a5d99d9bf3ca9c = []byte{ + // 173 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0x4b, 0x4d, 0x2c, + 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0xd5, + 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x06, 0x4b, 0xe9, 0x81, 0xa5, 0xa4, 0xc4, 0x91, 0xd5, + 0x85, 0x25, 0xe6, 0x94, 0x42, 0x55, 0x29, 0x39, 0x73, 0xb1, 0x43, 0xb5, 0x09, 0xf1, 0x71, 0x31, + 0x65, 0xa6, 0x48, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x31, 0x65, 0xa6, 0x08, 0x69, 0x70, 0xb1, + 0x96, 0x81, 0x54, 0x4a, 0x30, 0x29, 0x30, 0x6a, 0x70, 0x1b, 0x09, 0xe9, 0x21, 0x19, 0xa8, 0x07, + 0x36, 0x23, 0x08, 0xa2, 0xc0, 0x29, 0x98, 0x0b, 0xd9, 0x32, 0x27, 0x1e, 0xa8, 0x89, 0x01, 0x20, + 0x1b, 0xa2, 0xcc, 0xd2, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0xd3, 0xf3, + 0xb3, 0x52, 0xb3, 0xf5, 0x21, 0x6e, 0x01, 0xdb, 0x5f, 0xac, 0x9f, 0x9e, 0x9a, 0x97, 0x5a, 0x94, + 0x58, 0x92, 0x9a, 0xa2, 0x9f, 0x9e, 0xaf, 0x8f, 0xe4, 0xca, 0x24, 0x36, 0xb0, 0x02, 0x63, 0x40, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x72, 0x5f, 0x2d, 0xd9, 0xe3, 0x00, 0x00, 0x00, } diff --git a/protos/generated/go/feast/types/FeatureRow.pb.go b/protos/generated/go/feast/types/FeatureRow.pb.go index d4e18a19754..3fb6bc72f9d 100644 --- a/protos/generated/go/feast/types/FeatureRow.pb.go +++ b/protos/generated/go/feast/types/FeatureRow.pb.go @@ -33,7 +33,7 @@ func (m *FeatureRowKey) Reset() { *m = FeatureRowKey{} } func (m *FeatureRowKey) String() string { return proto.CompactTextString(m) } func (*FeatureRowKey) ProtoMessage() {} func (*FeatureRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_185cd01c950d2687, []int{0} + return fileDescriptor_FeatureRow_435f7325616f2983, []int{0} } func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowKey.Unmarshal(m, b) @@ -96,7 +96,7 @@ func (m *FeatureRow) Reset() { *m = FeatureRow{} } func (m *FeatureRow) String() string { return proto.CompactTextString(m) } func (*FeatureRow) ProtoMessage() {} func (*FeatureRow) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_185cd01c950d2687, []int{1} + return fileDescriptor_FeatureRow_435f7325616f2983, []int{1} } func (m *FeatureRow) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRow.Unmarshal(m, b) @@ -157,29 +157,29 @@ func init() { } func init() { - proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_185cd01c950d2687) -} - -var fileDescriptor_FeatureRow_185cd01c950d2687 = []byte{ - // 311 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x92, 0x41, 0x4b, 0xc3, 0x40, - 0x10, 0x85, 0x49, 0xab, 0x62, 0x37, 0x58, 0x21, 0x08, 0xc6, 0xd2, 0x6a, 0xe8, 0x29, 0xa7, 0x5d, - 0x89, 0xe0, 0x55, 0x08, 0xa8, 0x87, 0x82, 0x48, 0x90, 0x1e, 0xbc, 0x6d, 0x74, 0xb2, 0x46, 0x93, - 0x6c, 0x48, 0x66, 0x95, 0x1c, 0xfd, 0x8b, 0xfe, 0x22, 0x71, 0x93, 0x34, 0xab, 0x08, 0x9e, 0xbd, - 0xce, 0xf7, 0x66, 0xe6, 0xbd, 0x61, 0xc8, 0x3c, 0x01, 0x5e, 0x23, 0xc3, 0xa6, 0x84, 0x9a, 0x5d, - 0x01, 0x47, 0x55, 0x41, 0x24, 0xdf, 0x68, 0x59, 0x49, 0x94, 0x8e, 0xad, 0x29, 0xd5, 0x74, 0x76, - 0x22, 0xa4, 0x14, 0x19, 0x30, 0x8d, 0x62, 0x95, 0x30, 0x4c, 0x73, 0xa8, 0x91, 0xe7, 0x65, 0xab, - 0x9e, 0x1d, 0xfd, 0x32, 0xab, 0x43, 0x0b, 0x13, 0x5d, 0x57, 0xbc, 0x50, 0x19, 0xaf, 0x52, 0x6c, - 0x3a, 0x7c, 0x68, 0xe2, 0x35, 0xcf, 0x54, 0xd7, 0xb7, 0xfc, 0xb0, 0xc8, 0xde, 0xe0, 0x6a, 0x05, - 0x8d, 0x33, 0x27, 0x13, 0x28, 0x30, 0xc5, 0x66, 0x05, 0x8d, 0x6b, 0x79, 0x96, 0x3f, 0x89, 0x86, - 0x82, 0x13, 0x92, 0x29, 0xbc, 0x42, 0x81, 0x77, 0xbd, 0x35, 0x77, 0xec, 0x59, 0xbe, 0x1d, 0xcc, - 0x68, 0x6b, 0x9e, 0xf6, 0xe6, 0xe9, 0x46, 0x11, 0xfd, 0xe8, 0x70, 0x8e, 0x09, 0x69, 0x07, 0xde, - 0xf0, 0x1c, 0xdc, 0x2d, 0xbd, 0xc2, 0xa8, 0x38, 0x17, 0xc4, 0x16, 0x43, 0x02, 0x77, 0xdb, 0xb3, - 0xfc, 0x69, 0xb0, 0xa0, 0xc6, 0xa9, 0xa8, 0x99, 0xf0, 0xb2, 0x50, 0x79, 0x64, 0x76, 0x2c, 0xdf, - 0x47, 0x84, 0x0c, 0xa1, 0xfe, 0x48, 0x74, 0x4a, 0x76, 0x93, 0x56, 0x5b, 0xbb, 0x23, 0x6f, 0xec, - 0xdb, 0xc1, 0xc1, 0xb7, 0x55, 0xfd, 0xa0, 0x8d, 0xea, 0x5f, 0xdc, 0x20, 0x5c, 0x13, 0xf3, 0xb7, - 0xc2, 0xfd, 0xe1, 0x1e, 0xb7, 0x5f, 0xee, 0xee, 0xcf, 0x45, 0x8a, 0x4f, 0x2a, 0xa6, 0x0f, 0x32, - 0x67, 0x42, 0x3e, 0xc3, 0x0b, 0x6b, 0x5f, 0x44, 0x7b, 0xaf, 0x99, 0x80, 0x02, 0x2a, 0x8e, 0xf0, - 0xc8, 0x84, 0x64, 0xc6, 0xf3, 0xc4, 0x3b, 0x5a, 0x70, 0xf6, 0x19, 0x00, 0x00, 0xff, 0xff, 0x3d, - 0x8f, 0x82, 0x0d, 0xd8, 0x02, 0x00, 0x00, + proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_435f7325616f2983) +} + +var fileDescriptor_FeatureRow_435f7325616f2983 = []byte{ + // 305 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x92, 0x41, 0x4b, 0xfb, 0x40, + 0x10, 0xc5, 0x49, 0xfb, 0xff, 0x8b, 0x9d, 0x60, 0x85, 0xe0, 0x21, 0x96, 0x56, 0x43, 0x4f, 0x39, + 0xed, 0x4a, 0x04, 0xaf, 0x42, 0x40, 0x3d, 0x14, 0x44, 0x82, 0x78, 0xf0, 0xb6, 0xd1, 0xc9, 0x1a, + 0x6d, 0xb2, 0x21, 0x99, 0x28, 0x39, 0xfa, 0x15, 0xfd, 0x44, 0xe2, 0xae, 0x69, 0x56, 0x11, 0x3c, + 0x7b, 0x9d, 0xf7, 0x66, 0xe6, 0x37, 0xc3, 0x83, 0x79, 0x86, 0xa2, 0x21, 0x4e, 0x5d, 0x85, 0x0d, + 0x3f, 0x47, 0x41, 0x6d, 0x8d, 0x89, 0x7a, 0x61, 0x55, 0xad, 0x48, 0x79, 0xae, 0x56, 0x99, 0x56, + 0x67, 0x87, 0x52, 0x29, 0xb9, 0x46, 0xae, 0xa5, 0xb4, 0xcd, 0x38, 0xe5, 0x05, 0x36, 0x24, 0x8a, + 0xca, 0xb8, 0x67, 0xfb, 0x3f, 0xcc, 0xfa, 0x94, 0x16, 0xb6, 0x74, 0x51, 0x8b, 0xb2, 0x5d, 0x8b, + 0x3a, 0xa7, 0xce, 0xc8, 0xcb, 0x37, 0x07, 0x76, 0x86, 0xe5, 0x2b, 0xec, 0xbc, 0x39, 0x4c, 0xb0, + 0xa4, 0x9c, 0xba, 0x15, 0x76, 0xbe, 0x13, 0x38, 0xe1, 0x24, 0x19, 0x0a, 0x5e, 0x0c, 0x53, 0x7c, + 0xc6, 0x92, 0xae, 0x7b, 0x02, 0x7f, 0x1c, 0x38, 0xa1, 0x1b, 0xcd, 0x98, 0x61, 0x64, 0x3d, 0x23, + 0xdb, 0x38, 0x92, 0x6f, 0x1d, 0xde, 0x01, 0x80, 0x19, 0x78, 0x29, 0x0a, 0xf4, 0xff, 0xe9, 0x15, + 0x56, 0xc5, 0x3b, 0x05, 0x57, 0x0e, 0xa0, 0xfe, 0xff, 0xc0, 0x09, 0xa7, 0xd1, 0x82, 0x59, 0x1f, + 0x61, 0xf6, 0x21, 0x67, 0x65, 0x5b, 0x24, 0x76, 0xc7, 0xf2, 0x75, 0x04, 0x30, 0x1c, 0xf5, 0xcb, + 0x45, 0x47, 0xb0, 0x9d, 0x19, 0x6f, 0xe3, 0x8f, 0x82, 0x71, 0xe8, 0x46, 0x7b, 0x5f, 0x56, 0xf5, + 0x83, 0x36, 0xae, 0x3f, 0xf1, 0x83, 0xf8, 0x06, 0xec, 0x08, 0xc5, 0xbb, 0xc3, 0x3f, 0xae, 0x3e, + 0xe8, 0x6e, 0x4f, 0x64, 0x4e, 0x0f, 0x6d, 0xca, 0xee, 0x54, 0xc1, 0xa5, 0x7a, 0xc4, 0x27, 0x6e, + 0x82, 0xa2, 0xd9, 0x1b, 0x2e, 0xb1, 0xc4, 0x5a, 0x10, 0xde, 0x73, 0xa9, 0xb8, 0x15, 0xa1, 0x74, + 0x4b, 0x1b, 0x8e, 0xdf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xb5, 0xfe, 0x14, 0xc9, 0xbf, 0x02, 0x00, + 0x00, } diff --git a/sdk/python/feast/sdk/importer.py b/sdk/python/feast/sdk/importer.py index d6800c45d94..c7f31491b93 100644 --- a/sdk/python/feast/sdk/importer.py +++ b/sdk/python/feast/sdk/importer.py @@ -75,7 +75,8 @@ def entity(self): @classmethod def from_csv(cls, path, entity, granularity, owner, staging_location=None, id_column=None, feature_columns=None, timestamp_column=None, - timestamp_value=None, serving_store=None, warehouse_store=None): + timestamp_value=None, serving_store=None, warehouse_store=None, + job_options={}): """Creates an importer from a given csv dataset. This file can be either local or remote (in gcs). If it's a local file then staging_location must be determined. @@ -100,12 +101,13 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, Serving store to write the features in this instance to. warehouse_store (feast.sdk.resources.feature.DataStore): Defaults to None. Warehouse store to write the features in this instance to. + job_options (dict): Defaults to empty dict. Additional job options. Returns: Importer: the importer for the dataset provided. """ - import_spec_options = {"format": "csv"} - import_spec_options["path"], require_staging = \ + source_options = {"format": "csv"} + source_options["path"], require_staging = \ _get_remote_location(path, staging_location) if is_gs_path(path): df = gcs_to_df(path) @@ -116,10 +118,10 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) - iport_spec = _create_import("file", import_spec_options, entity, schema) + iport_spec = _create_import("file", source_options, job_options, entity, schema) props = (_properties("csv", len(df.index), require_staging, - import_spec_options["path"])) + source_options["path"])) specs = _specs(iport_spec, Entity(name=entity), features) return cls(specs, df, props) @@ -127,7 +129,8 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, @classmethod def from_bq(cls, bq_path, entity, granularity, owner, limit=10, id_column=None, feature_columns=None, timestamp_column=None, - timestamp_value=None, serving_store=None, warehouse_store=None): + timestamp_value=None, serving_store=None, warehouse_store=None, + job_options={}): """Creates an importer from a given bigquery table. Args: @@ -151,6 +154,7 @@ def from_bq(cls, bq_path, entity, granularity, owner, limit=10, Serving store to write the features in this instance to. warehouse_store (feast.sdk.resources.feature.DataStore): Defaults to None. Warehouse store to write the features in this instance to. + job_options (dict): Defaults to empty dict. Additional job options. Returns: Importer: the importer for the dataset provided. @@ -162,7 +166,7 @@ def from_bq(cls, bq_path, entity, granularity, owner, limit=10, table_ref = dataset_ref.table(table_id) table = cli.get_table(table_ref) - import_spec_options = { + source_options = { "project": project, "dataset": dataset_id, "table": table_id @@ -173,8 +177,8 @@ def from_bq(cls, bq_path, entity, granularity, owner, limit=10, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) - iport_spec = _create_import("bigquery", import_spec_options, - entity, schema) + iport_spec = _create_import("bigquery", source_options, + job_options, entity, schema) props = _properties("bigquery", table.num_rows, False, None) specs = _specs(iport_spec, Entity(name=entity), features) @@ -183,7 +187,8 @@ def from_bq(cls, bq_path, entity, granularity, owner, limit=10, @classmethod def from_df(cls, df, entity, granularity, owner, staging_location, id_column=None, feature_columns=None, timestamp_column=None, - timestamp_value=None, serving_store=None, warehouse_store=None): + timestamp_value=None, serving_store=None, warehouse_store=None, + job_options={}): """Creates an importer from a given pandas dataframe. To import a file from a dataframe, the data will have to be staged. @@ -207,26 +212,28 @@ def from_df(cls, df, entity, granularity, owner, staging_location, Serving store to write the features in this instance to. warehouse_store (feast.sdk.resources.feature.DataStore): Defaults to None. Warehouse store to write the features in this instance to. + job_options (dict): Defaults to empty dict. Additional job options. Returns: Importer: the importer for the dataset provided. """ tmp_file_name = ("tmp_{}_{}.csv" .format(entity, int(round(time.time() * 1000)))) - import_spec_options = { + source_options = { "format": "csv" } - import_spec_options["path"], require_staging = ( + source_options["path"], require_staging = ( _get_remote_location(tmp_file_name, staging_location)) schema, features = \ _detect_schema_and_feature(entity, granularity, owner, id_column, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) - iport_spec = _create_import("file", import_spec_options, entity, schema) + iport_spec = _create_import("file", source_options, + job_options, entity, schema) props = _properties("dataframe", len(df.index), require_staging, - import_spec_options["path"]) + source_options["path"]) specs = _specs(iport_spec, Entity(name=entity), features) return cls(specs, df, props) @@ -434,12 +441,13 @@ def _create_feature(column, entity, granularity, owner, return feature -def _create_import(import_type, options, entity, schema): +def _create_import(import_type, source_options, job_options, entity, schema): """Create an import spec. Args: import_type (str): import type - options (dict): import spec options + source_options (dict): import spec source options + jobOptions (dict): import spec job options entity (str): entity schema (feast.specs.ImportSpec_pb2.Schema): schema of the file @@ -449,7 +457,8 @@ def _create_import(import_type, options, entity, schema): return ImportSpec( type=import_type, - options=options, + sourceOptions=source_options, + jobOptions=job_options, entities=[entity], schema=schema) diff --git a/sdk/python/feast/specs/ImportSpec_pb2.py b/sdk/python/feast/specs/ImportSpec_pb2.py index d87fdafa4ad..60e5b4f400a 100644 --- a/sdk/python/feast/specs/ImportSpec_pb2.py +++ b/sdk/python/feast/specs/ImportSpec_pb2.py @@ -20,29 +20,29 @@ package='feast.specs', syntax='proto3', serialized_options=_b('\n\013feast.specsB\017ImportSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specs'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/specs/ImportSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1fgoogle/protobuf/timestamp.proto\"\xb8\x01\n\nImportSpec\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x35\n\x07options\x18\x02 \x03(\x0b\x32$.feast.specs.ImportSpec.OptionsEntry\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12#\n\x06schema\x18\x04 \x01(\x0b\x32\x13.feast.specs.Schema\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa2\x01\n\x06Schema\x12\"\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x12.feast.specs.Field\x12\x19\n\x0ftimestampColumn\x18\x05 \x01(\tH\x00\x12\x34\n\x0etimestampValue\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x12\x16\n\x0e\x65ntityIdColumn\x18\x07 \x01(\tB\x0b\n\ttimestamp\"(\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tfeatureId\x18\x02 \x01(\tBV\n\x0b\x66\x65\x61st.specsB\x0fImportSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') + serialized_pb=_b('\n\x1c\x66\x65\x61st/specs/ImportSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1fgoogle/protobuf/timestamp.proto\"\xba\x02\n\nImportSpec\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x41\n\rsourceOptions\x18\x02 \x03(\x0b\x32*.feast.specs.ImportSpec.SourceOptionsEntry\x12;\n\njobOptions\x18\x05 \x03(\x0b\x32\'.feast.specs.ImportSpec.JobOptionsEntry\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12#\n\x06schema\x18\x04 \x01(\x0b\x32\x13.feast.specs.Schema\x1a\x34\n\x12SourceOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fJobOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa2\x01\n\x06Schema\x12\"\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x12.feast.specs.Field\x12\x19\n\x0ftimestampColumn\x18\x05 \x01(\tH\x00\x12\x34\n\x0etimestampValue\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x12\x16\n\x0e\x65ntityIdColumn\x18\x07 \x01(\tB\x0b\n\ttimestamp\"(\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tfeatureId\x18\x02 \x01(\tBV\n\x0b\x66\x65\x61st.specsB\x0fImportSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) -_IMPORTSPEC_OPTIONSENTRY = _descriptor.Descriptor( - name='OptionsEntry', - full_name='feast.specs.ImportSpec.OptionsEntry', +_IMPORTSPEC_SOURCEOPTIONSENTRY = _descriptor.Descriptor( + name='SourceOptionsEntry', + full_name='feast.specs.ImportSpec.SourceOptionsEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='feast.specs.ImportSpec.OptionsEntry.key', index=0, + name='key', full_name='feast.specs.ImportSpec.SourceOptionsEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='value', full_name='feast.specs.ImportSpec.OptionsEntry.value', index=1, + name='value', full_name='feast.specs.ImportSpec.SourceOptionsEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -60,8 +60,45 @@ extension_ranges=[], oneofs=[ ], - serialized_start=217, - serialized_end=263, + serialized_start=290, + serialized_end=342, +) + +_IMPORTSPEC_JOBOPTIONSENTRY = _descriptor.Descriptor( + name='JobOptionsEntry', + full_name='feast.specs.ImportSpec.JobOptionsEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='feast.specs.ImportSpec.JobOptionsEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='feast.specs.ImportSpec.JobOptionsEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=_b('8\001'), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=344, + serialized_end=393, ) _IMPORTSPEC = _descriptor.Descriptor( @@ -79,21 +116,28 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='options', full_name='feast.specs.ImportSpec.options', index=1, + name='sourceOptions', full_name='feast.specs.ImportSpec.sourceOptions', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='entities', full_name='feast.specs.ImportSpec.entities', index=2, + name='jobOptions', full_name='feast.specs.ImportSpec.jobOptions', index=2, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='entities', full_name='feast.specs.ImportSpec.entities', index=3, number=3, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='schema', full_name='feast.specs.ImportSpec.schema', index=3, + name='schema', full_name='feast.specs.ImportSpec.schema', index=4, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -102,7 +146,7 @@ ], extensions=[ ], - nested_types=[_IMPORTSPEC_OPTIONSENTRY, ], + nested_types=[_IMPORTSPEC_SOURCEOPTIONSENTRY, _IMPORTSPEC_JOBOPTIONSENTRY, ], enum_types=[ ], serialized_options=None, @@ -112,7 +156,7 @@ oneofs=[ ], serialized_start=79, - serialized_end=263, + serialized_end=393, ) @@ -166,8 +210,8 @@ name='timestamp', full_name='feast.specs.Schema.timestamp', index=0, containing_type=None, fields=[]), ], - serialized_start=266, - serialized_end=428, + serialized_start=396, + serialized_end=558, ) @@ -204,12 +248,14 @@ extension_ranges=[], oneofs=[ ], - serialized_start=430, - serialized_end=470, + serialized_start=560, + serialized_end=600, ) -_IMPORTSPEC_OPTIONSENTRY.containing_type = _IMPORTSPEC -_IMPORTSPEC.fields_by_name['options'].message_type = _IMPORTSPEC_OPTIONSENTRY +_IMPORTSPEC_SOURCEOPTIONSENTRY.containing_type = _IMPORTSPEC +_IMPORTSPEC_JOBOPTIONSENTRY.containing_type = _IMPORTSPEC +_IMPORTSPEC.fields_by_name['sourceOptions'].message_type = _IMPORTSPEC_SOURCEOPTIONSENTRY +_IMPORTSPEC.fields_by_name['jobOptions'].message_type = _IMPORTSPEC_JOBOPTIONSENTRY _IMPORTSPEC.fields_by_name['schema'].message_type = _SCHEMA _SCHEMA.fields_by_name['fields'].message_type = _FIELD _SCHEMA.fields_by_name['timestampValue'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP @@ -226,10 +272,17 @@ ImportSpec = _reflection.GeneratedProtocolMessageType('ImportSpec', (_message.Message,), dict( - OptionsEntry = _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), dict( - DESCRIPTOR = _IMPORTSPEC_OPTIONSENTRY, + SourceOptionsEntry = _reflection.GeneratedProtocolMessageType('SourceOptionsEntry', (_message.Message,), dict( + DESCRIPTOR = _IMPORTSPEC_SOURCEOPTIONSENTRY, + __module__ = 'feast.specs.ImportSpec_pb2' + # @@protoc_insertion_point(class_scope:feast.specs.ImportSpec.SourceOptionsEntry) + )) + , + + JobOptionsEntry = _reflection.GeneratedProtocolMessageType('JobOptionsEntry', (_message.Message,), dict( + DESCRIPTOR = _IMPORTSPEC_JOBOPTIONSENTRY, __module__ = 'feast.specs.ImportSpec_pb2' - # @@protoc_insertion_point(class_scope:feast.specs.ImportSpec.OptionsEntry) + # @@protoc_insertion_point(class_scope:feast.specs.ImportSpec.JobOptionsEntry) )) , DESCRIPTOR = _IMPORTSPEC, @@ -237,7 +290,8 @@ # @@protoc_insertion_point(class_scope:feast.specs.ImportSpec) )) _sym_db.RegisterMessage(ImportSpec) -_sym_db.RegisterMessage(ImportSpec.OptionsEntry) +_sym_db.RegisterMessage(ImportSpec.SourceOptionsEntry) +_sym_db.RegisterMessage(ImportSpec.JobOptionsEntry) Schema = _reflection.GeneratedProtocolMessageType('Schema', (_message.Message,), dict( DESCRIPTOR = _SCHEMA, @@ -255,5 +309,6 @@ DESCRIPTOR._options = None -_IMPORTSPEC_OPTIONSENTRY._options = None +_IMPORTSPEC_SOURCEOPTIONSENTRY._options = None +_IMPORTSPEC_JOBOPTIONSENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/tests/sdk/test_importer.py b/sdk/python/tests/sdk/test_importer.py index ba18deea371..c09cad0db87 100644 --- a/sdk/python/tests/sdk/test_importer.py +++ b/sdk/python/tests/sdk/test_importer.py @@ -151,7 +151,7 @@ def test_from_df(self): import_spec = importer.spec assert import_spec.type == "file" - assert import_spec.options == {"format" : "csv", "path" : importer.remote_path} + assert import_spec.sourceOptions == {"format" : "csv", "path" : importer.remote_path} assert import_spec.entities == ["driver"] schema = import_spec.schema @@ -185,7 +185,7 @@ def _validate_csv_importer(self, import_spec = importer.spec assert import_spec.type == "file" path = importer.remote_path if importer.require_staging else csv_path - assert import_spec.options == {"format" : "csv", "path" : path} + assert import_spec.sourceOptions == {"format" : "csv", "path" : path} assert import_spec.entities == [entity_name] schema = import_spec.schema From 5dbb66ef513e6c18a74b97ff2c373b3b2f3bf2ff Mon Sep 17 00:00:00 2001 From: Willem Pienaar <6728866+woop@users.noreply.github.com> Date: Sun, 17 Feb 2019 13:30:11 +0800 Subject: [PATCH 02/25] Fix Python SDK importer's ability to apply features (#135) This commit fixes a bug where the key is being passed to the _apply_feature method, instead of the object. This causes a failure when trying to register that feature with Feast. --- sdk/python/feast/sdk/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index ea2d2317c6a..d1b09ebf22c 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -166,7 +166,7 @@ def run(self, importer, name_override=None, self._apply_entity(importer.entity) if apply_features: for feature in importer.features: - self._apply_feature(feature) + self._apply_feature(importer.features[feature]) if importer.require_staging: print("Staging file to remote path {}" From 316eac464a62c101bd44b91635f3191d8bf7d49a Mon Sep 17 00:00:00 2001 From: Chen Zhiling Date: Wed, 20 Feb 2019 12:23:13 +0800 Subject: [PATCH 03/25] Add wait flag for jobs, fix go proto path for dataset service (#138) --- cli/feast/cmd/jobs.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cli/feast/cmd/jobs.go b/cli/feast/cmd/jobs.go index 962e37e56e3..5eb3e401ce8 100644 --- a/cli/feast/cmd/jobs.go +++ b/cli/feast/cmd/jobs.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io/ioutil" + "time" "github.com/gojek/feast/cli/feast/pkg/parse" "github.com/gojek/feast/protos/generated/go/feast/core" @@ -26,6 +27,10 @@ import ( "github.com/spf13/cobra" ) +var ( + waitJobComplete = false +) + // jobsCmd represents the jobs command var jobsCmd = &cobra.Command{ Use: "jobs", @@ -63,6 +68,7 @@ var jobsAbortCmd = &cobra.Command{ } func init() { + jobsRunCmd.Flags().BoolVar(&waitJobComplete, "wait", false, "wait for job to run to completion") jobsCmd.AddCommand(jobsRunCmd) jobsCmd.AddCommand(jobsAbortCmd) rootCmd.AddCommand(jobsCmd) @@ -86,9 +92,35 @@ func runJob(ctx context.Context, path string) error { return fmt.Errorf("[jobs] failed to start job: %v", err) } fmt.Printf("[jobs] started job with ID: %s", out.GetJobId()) + if waitJobComplete { + return waitJob(ctx, jobsClient, out.GetJobId()) + } return nil } +func waitJob(ctx context.Context, jobsClient core.JobServiceClient, jobID string) error { + for { + response, err := jobsClient.GetJob(ctx, &core.JobServiceTypes_GetJobRequest{ + Id: jobID, + }) + if err != nil { + return fmt.Errorf("[jobs] error while querying job id %s: %v", jobID, err) + } + + status := response.GetJob().GetStatus() + fmt.Printf("\r[jobs] job id %s is currently: %s\n", jobID, status) + switch status { + case "COMPLETED": + return nil + case "ABORTED": + return fmt.Errorf("[jobs] job id %s failed: Job was aborted", jobID) + case "ERROR": + return fmt.Errorf("[jobs] job id %s failed: Job terminated with error. For more information, refer to job logs", jobID) + } + time.Sleep(5 * time.Second) + } +} + func abortJob(ctx context.Context, id string) error { initConn() jobsClient := core.NewJobServiceClient(coreConn) From 294a6ecd777c1f0cdab22171f7bc06cf7ca989d7 Mon Sep 17 00:00:00 2001 From: Chen Zhiling Date: Wed, 20 Feb 2019 16:42:13 +0800 Subject: [PATCH 04/25] Regenerate go (#142) * Add wait flag for jobs, fix go proto path for dataset service * Regenerate go protos From 6cdd173e41398354a650d33f967bf635a6cccdca Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Sun, 24 Feb 2019 11:26:14 +0800 Subject: [PATCH 05/25] Fix import spec created from Importer.from_csv (#143) --- sdk/python/feast/sdk/importer.py | 8 +- sdk/python/tests/sdk/test_importer.py | 188 ++++++++++++++------------ 2 files changed, 107 insertions(+), 89 deletions(-) diff --git a/sdk/python/feast/sdk/importer.py b/sdk/python/feast/sdk/importer.py index c7f31491b93..f9a97aad31b 100644 --- a/sdk/python/feast/sdk/importer.py +++ b/sdk/python/feast/sdk/importer.py @@ -106,7 +106,8 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, Returns: Importer: the importer for the dataset provided. """ - source_options = {"format": "csv"} + src_type = "file.csv" + source_options = {} source_options["path"], require_staging = \ _get_remote_location(path, staging_location) if is_gs_path(path): @@ -118,9 +119,10 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) - iport_spec = _create_import("file", source_options, job_options, entity, schema) + iport_spec = _create_import(src_type, source_options, job_options, + entity, schema) - props = (_properties("csv", len(df.index), require_staging, + props = (_properties(src_type, len(df.index), require_staging, source_options["path"])) specs = _specs(iport_spec, Entity(name=entity), features) diff --git a/sdk/python/tests/sdk/test_importer.py b/sdk/python/tests/sdk/test_importer.py index c09cad0db87..7bf1c05c3f5 100644 --- a/sdk/python/tests/sdk/test_importer.py +++ b/sdk/python/tests/sdk/test_importer.py @@ -15,7 +15,8 @@ import pandas as pd import pytest import ntpath -from feast.sdk.resources.feature import Feature, Granularity, ValueType, Datastore +from feast.sdk.resources.feature import Feature, Granularity, ValueType, \ + Datastore from feast.sdk.importer import _create_feature, Importer from feast.sdk.utils.gs_utils import is_gs_path from feast.types.Granularity_pb2 import Granularity as Granularity_pb2 @@ -30,56 +31,60 @@ def test_from_csv(self): staging_location = "gs://test-bucket" id_column = "driver_id" feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + "avg_customer_distance_completed"] timestamp_column = "ts" - importer = Importer.from_csv(path = csv_path, - entity = entity_name, - granularity = feature_granularity, - owner = owner, - staging_location=staging_location, - id_column = id_column, - feature_columns=feature_columns, - timestamp_column=timestamp_column) + importer = Importer.from_csv(path=csv_path, + entity=entity_name, + granularity=feature_granularity, + owner=owner, + staging_location=staging_location, + id_column=id_column, + feature_columns=feature_columns, + timestamp_column=timestamp_column) self._validate_csv_importer(importer, csv_path, entity_name, - feature_granularity, owner, staging_location, id_column, - feature_columns, timestamp_column) + feature_granularity, owner, + staging_location, id_column, + feature_columns, timestamp_column) def test_from_csv_id_column_not_specified(self): with pytest.raises(ValueError, - match="Column with name driver is not found") as e_info: + match="Column with name driver is not found"): feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + "avg_customer_distance_completed"] csv_path = "tests/data/driver_features.csv" - importer = Importer.from_csv(path = csv_path, - entity = "driver", - granularity = Granularity.DAY, - owner = "owner@feast.com", - staging_location="gs://test-bucket", - feature_columns=feature_columns, - timestamp_column="ts") + Importer.from_csv(path=csv_path, + entity="driver", + granularity=Granularity.DAY, + owner="owner@feast.com", + staging_location="gs://test-bucket", + feature_columns=feature_columns, + timestamp_column="ts") def test_from_csv_timestamp_column_not_specified(self): feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed", "avg_distance_cancelled"] + "avg_customer_distance_completed", + "avg_distance_cancelled"] csv_path = "tests/data/driver_features.csv" entity_name = "driver" granularity = Granularity.DAY owner = "owner@feast.com" staging_location = "gs://test-bucket" id_column = "driver_id" - importer = Importer.from_csv(path = csv_path, - entity = entity_name, - granularity = granularity, - owner = owner, - staging_location=staging_location, - id_column = id_column, - feature_columns= feature_columns) + importer = Importer.from_csv(path=csv_path, + entity=entity_name, + granularity=granularity, + owner=owner, + staging_location=staging_location, + id_column=id_column, + feature_columns=feature_columns) self._validate_csv_importer(importer, csv_path, entity_name, - granularity, owner, staging_location = staging_location, - id_column=id_column, feature_columns=feature_columns) + granularity, owner, + staging_location=staging_location, + id_column=id_column, + feature_columns=feature_columns) def test_from_csv_feature_columns_not_specified(self): csv_path = "tests/data/driver_features.csv" @@ -89,43 +94,45 @@ def test_from_csv_feature_columns_not_specified(self): staging_location = "gs://test-bucket" id_column = "driver_id" timestamp_column = "ts" - importer = Importer.from_csv(path = csv_path, - entity = entity_name, - granularity = granularity, - owner = owner, - staging_location=staging_location, - id_column = id_column, - timestamp_column=timestamp_column) + importer = Importer.from_csv(path=csv_path, + entity=entity_name, + granularity=granularity, + owner=owner, + staging_location=staging_location, + id_column=id_column, + timestamp_column=timestamp_column) self._validate_csv_importer(importer, csv_path, entity_name, - granularity, owner, staging_location = staging_location, - id_column=id_column, timestamp_column=timestamp_column) + granularity, owner, + staging_location=staging_location, + id_column=id_column, + timestamp_column=timestamp_column) def test_from_csv_staging_location_not_specified(self): with pytest.raises(ValueError, - match="Specify staging_location for importing local file/dataframe") as e_info: + match="Specify staging_location for importing local file/dataframe"): feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + "avg_customer_distance_completed"] csv_path = "tests/data/driver_features.csv" - importer = Importer.from_csv(path = csv_path, - entity = "driver", - granularity = Granularity.DAY, - owner = "owner@feast.com", - feature_columns=feature_columns, - timestamp_column="ts") + Importer.from_csv(path=csv_path, + entity="driver", + granularity=Granularity.DAY, + owner="owner@feast.com", + feature_columns=feature_columns, + timestamp_column="ts") with pytest.raises(ValueError, - match="Staging location must be in GCS") as e_info: + match="Staging location must be in GCS") as e_info: feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + "avg_customer_distance_completed"] csv_path = "tests/data/driver_features.csv" - importer = Importer.from_csv(path = csv_path, - entity = "driver", - granularity = Granularity.DAY, - owner = "owner@feast.com", - staging_location = "/home", - feature_columns=feature_columns, - timestamp_column="ts") + Importer.from_csv(path=csv_path, + entity="driver", + granularity=Granularity.DAY, + owner="owner@feast.com", + staging_location="/home", + feature_columns=feature_columns, + timestamp_column="ts") def test_from_df(self): csv_path = "tests/data/driver_features.csv" @@ -133,59 +140,63 @@ def test_from_df(self): staging_location = "gs://test-bucket" entity = "driver" - importer = Importer.from_df(df = df, - entity = entity, - granularity = Granularity.DAY, - owner = "owner@feast.com", - staging_location=staging_location, - id_column = "driver_id", - timestamp_column="ts") - + importer = Importer.from_df(df=df, + entity=entity, + granularity=Granularity.DAY, + owner="owner@feast.com", + staging_location=staging_location, + id_column="driver_id", + timestamp_column="ts") assert importer.require_staging == True assert ("{}/tmp_{}".format(staging_location, entity) - in importer.remote_path) + in importer.remote_path) for feature in importer.features.values(): assert feature.name in df.columns assert feature.id == "driver.day." + feature.name import_spec = importer.spec assert import_spec.type == "file" - assert import_spec.sourceOptions == {"format" : "csv", "path" : importer.remote_path} + assert import_spec.sourceOptions == {"format": "csv", + "path": importer.remote_path} assert import_spec.entities == ["driver"] schema = import_spec.schema assert schema.entityIdColumn == "driver_id" assert schema.timestampValue is not None feature_columns = ["completed", "avg_distance_completed", - "avg_customer_distance_completed", - "avg_distance_cancelled"] + "avg_customer_distance_completed", + "avg_distance_cancelled"] for col, field in zip(df.columns.values, schema.fields): assert col == field.name if col in feature_columns: assert field.featureId == "driver.day." + col def _validate_csv_importer(self, - importer, csv_path, entity_name, feature_granularity, owner, - staging_location = None, id_column = None, feature_columns = None, - timestamp_column = None, timestamp_value = None): + importer, csv_path, entity_name, + feature_granularity, owner, + staging_location=None, id_column=None, + feature_columns=None, + timestamp_column=None, timestamp_value=None): df = pd.read_csv(csv_path) assert not importer.require_staging == is_gs_path(csv_path) if importer.require_staging: assert importer.remote_path == "{}/{}".format(staging_location, - ntpath.basename(csv_path)) + ntpath.basename( + csv_path)) # check features created for feature in importer.features.values(): assert feature.name in df.columns assert feature.id == "{}.{}.{}".format(entity_name, - Granularity_pb2.Enum.Name(feature_granularity.value).lower(), - feature.name) + Granularity_pb2.Enum.Name( + feature_granularity.value).lower(), + feature.name) import_spec = importer.spec - assert import_spec.type == "file" + assert import_spec.type == "file.csv" path = importer.remote_path if importer.require_staging else csv_path - assert import_spec.sourceOptions == {"format" : "csv", "path" : path} + assert import_spec.sourceOptions == {"path": path} assert import_spec.entities == [entity_name] schema = import_spec.schema @@ -204,19 +215,23 @@ def _validate_csv_importer(self, for col, field in zip(df.columns.values, schema.fields): assert col == field.name if col in feature_columns: - assert field.featureId == "{}.{}.{}".format(entity_name, - Granularity_pb2.Enum.Name(feature_granularity.value).lower(), col) + assert field.featureId == \ + "{}.{}.{}".format(entity_name, + Granularity_pb2.Enum.Name( + feature_granularity.value).lower(), + col) class TestHelpers: def test_create_feature(self): - col = pd.Series([1]*3,dtype='int32',name="test") + col = pd.Series([1] * 3, dtype='int32', name="test") expected = Feature(name="test", - entity="test", - granularity=Granularity.NONE, - owner="person", - value_type=ValueType.INT32) - actual = _create_feature(col, "test", Granularity.NONE, "person", None, None) + entity="test", + granularity=Granularity.NONE, + owner="person", + value_type=ValueType.INT32) + actual = _create_feature(col, "test", Granularity.NONE, "person", None, + None) assert actual.id == expected.id assert actual.value_type == expected.value_type assert actual.owner == expected.owner @@ -231,7 +246,8 @@ def test_create_feature_with_stores(self): serving_store=Datastore(id="SERVING"), warehouse_store=Datastore(id="WAREHOUSE")) actual = _create_feature(col, "test", Granularity.NONE, "person", - Datastore(id="SERVING"), Datastore(id="WAREHOUSE")) + Datastore(id="SERVING"), + Datastore(id="WAREHOUSE")) assert actual.id == expected.id assert actual.value_type == expected.value_type assert actual.owner == expected.owner From dad7e6ea1dd0f4f8733cddd827d4bc4cb61e984e Mon Sep 17 00:00:00 2001 From: Tim Sell <63295+tims@users.noreply.github.com> Date: Mon, 25 Feb 2019 14:53:15 +0700 Subject: [PATCH 06/25] Refactor stores (#110) * rename feature store classes to feature store factories * split stores up into serving and warehouse and rename some things * rename FeatureIO.Write to FeatureStoreWrite --- .../feast/ingestion/boot/ImportJobModule.java | 24 ++--- .../transform/ErrorsStoreTransform.java | 25 +++--- .../feast/ingestion/transform/FeatureIO.java | 44 ---------- .../transform/ServingStoreTransform.java | 12 +-- .../transform/SplitOutputByStore.java | 26 +++--- .../transform/WarehouseStoreTransform.java | 18 ++-- .../transform/fn/ValidateFeatureRowsDoFn.java | 12 +-- .../bigquery/BigQueryToFeatureRowFn.java | 4 +- .../main/java/feast/storage/StoreClient.java | 51 ----------- .../storage/file/json/JsonFileFeatureIO.java | 66 -------------- .../storage/file/json/JsonFileStores.java | 88 ------------------- .../feast/storage/noop/NoOpServingStore.java | 38 -------- .../storage/noop/NoOpWarehouseStore.java | 39 -------- .../FeatureStoreFactory.java} | 7 +- .../java/feast/store/FeatureStoreWrite.java | 25 ++++++ .../file => store}/FileStoreOptions.java | 2 +- .../feast/{storage/noop => store}/NoOpIO.java | 5 +- .../TextFileDynamicIO.java} | 39 ++++---- .../errors/FeatureErrorsFactory.java} | 6 +- .../errors/FeatureErrorsFactoryService.java} | 32 ++++--- .../errors/json/JsonFileErrorsFactory.java | 48 ++++++++++ .../errors/json/JsonFileErrorsWrite.java | 56 ++++++++++++ .../errors/logging}/LogIO.java | 10 +-- .../logging/StderrFeatureErrorsFactory.java} | 18 ++-- .../logging/StdoutFeatureErrorsFactory.java} | 16 ++-- .../serving/FeatureServingFactory.java} | 6 +- .../FeatureServingFactoryService.java} | 30 ++++--- .../serving/FeatureServingStoreClient.java} | 28 +++--- .../bigtable/BigTableFeatureOptions.java | 2 +- .../bigtable/BigTableServingStoreFactory.java | 13 +-- .../bigtable/BigTableStoreOptions.java | 2 +- .../bigtable/FeatureRowBigTableIO.java | 6 +- .../FeatureRowToBigTableMutationDoFn.java | 2 +- .../serving}/redis/FeatureRowRedisIO.java | 9 +- .../redis/FeatureRowToRedisMutationDoFn.java | 6 +- .../serving}/redis/RedisCustomIO.java | 2 +- .../serving}/redis/RedisFeatureOptions.java | 2 +- .../serving/redis/RedisServingFactory.java} | 13 +-- .../serving}/redis/RedisStoreOptions.java | 2 +- .../warehouse/FeatureWarehouseFactory.java} | 8 +- .../FeatureWarehouseFactoryService.java} | 26 +++--- .../bigquery/BigQueryStoreOptions.java | 3 +- .../bigquery/BigQueryWarehouseFactory.java} | 12 +-- .../bigquery/FeatureRowBigQueryIO.java | 6 +- .../FeatureRowToBigQueryTableRowDoFn.java | 4 +- .../bigquery/ValueBigQueryBuilder.java | 2 +- .../json/JsonFileWarehouseFactory.java | 47 ++++++++++ .../json/JsonFileWarehouseWrite.java | 56 ++++++++++++ .../sdk/io/gcp/pubsub/FeastPubsubHelper.java | 58 ------------ .../feast/ingestion/ImportJobCSVTest.java | 63 ++++++++----- .../transform/ErrorsStoreTransformTest.java | 22 ++--- .../transform/SplitOutputByStoreTest.java | 20 ++--- .../file => store}/FileStoreOptionsTest.java | 7 +- .../MockFeatureErrorsFactory.java} | 14 +-- .../{storage => store}/MockFeatureStore.java | 8 +- .../MockServingFactory.java} | 14 +-- .../{storage => store}/MockTransforms.java | 9 +- .../MockWarehouseFactory.java} | 14 +-- .../TextFileDynamicIOTest.java} | 51 ++++++----- .../bigtable/BigTableFeatureOptionsTest.java | 6 +- .../bigtable/BigTableStoreOptionsTest.java | 6 +- .../redis/FeatureRowRedisIOWriteTest.java | 18 ++-- .../FeatureRowToRedisMutationDoFnTest.java | 6 +- .../redis/RedisFeatureOptionsTest.java | 6 +- .../serving}/redis/RedisStoreOptionsTest.java | 6 +- .../bigquery/BigQueryStoreOptionsTest.java | 2 +- .../bigquery/ValueBigQueryBuilderTest.java | 2 +- .../json/JsonFileFeatureIOTest.java | 50 +++++++---- 68 files changed, 659 insertions(+), 721 deletions(-) delete mode 100644 ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java delete mode 100644 ingestion/src/main/java/feast/storage/StoreClient.java delete mode 100644 ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java delete mode 100644 ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java delete mode 100644 ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java delete mode 100644 ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java rename ingestion/src/main/java/feast/{storage/FeatureStore.java => store/FeatureStoreFactory.java} (81%) create mode 100644 ingestion/src/main/java/feast/store/FeatureStoreWrite.java rename ingestion/src/main/java/feast/{storage/file => store}/FileStoreOptions.java (97%) rename ingestion/src/main/java/feast/{storage/noop => store}/NoOpIO.java (89%) rename ingestion/src/main/java/feast/{storage/file/TextFileFeatureIO.java => store/TextFileDynamicIO.java} (65%) rename ingestion/src/main/java/feast/{storage/ServingStore.java => store/errors/FeatureErrorsFactory.java} (81%) rename ingestion/src/main/java/feast/{storage/service/ErrorsStoreService.java => store/errors/FeatureErrorsFactoryService.java} (59%) create mode 100644 ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java create mode 100644 ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java rename ingestion/src/main/java/feast/{storage/stderr => store/errors/logging}/LogIO.java (83%) rename ingestion/src/main/java/feast/{storage/stderr/StderrErrorsStore.java => store/errors/logging/StderrFeatureErrorsFactory.java} (68%) rename ingestion/src/main/java/feast/{storage/stderr/StdoutErrorsStore.java => store/errors/logging/StdoutFeatureErrorsFactory.java} (68%) rename ingestion/src/main/java/feast/{storage/ErrorsStore.java => store/serving/FeatureServingFactory.java} (81%) rename ingestion/src/main/java/feast/{storage/service/ServingStoreService.java => store/serving/FeatureServingFactoryService.java} (59%) rename ingestion/src/main/java/feast/{storage/FeatureStoreMigration.java => store/serving/FeatureServingStoreClient.java} (54%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/BigTableFeatureOptions.java (95%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/BigTableServingStoreFactory.java (79%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/BigTableStoreOptions.java (95%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/FeatureRowBigTableIO.java (95%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/FeatureRowToBigTableMutationDoFn.java (99%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/FeatureRowRedisIO.java (88%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/FeatureRowToRedisMutationDoFn.java (96%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/RedisCustomIO.java (99%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/RedisFeatureOptions.java (97%) rename ingestion/src/main/java/feast/{storage/redis/RedisServingStore.java => store/serving/redis/RedisServingFactory.java} (80%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/RedisStoreOptions.java (97%) rename ingestion/src/main/java/feast/{storage/WarehouseStore.java => store/warehouse/FeatureWarehouseFactory.java} (80%) rename ingestion/src/main/java/feast/{storage/service/WarehouseStoreService.java => store/warehouse/FeatureWarehouseFactoryService.java} (60%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/BigQueryStoreOptions.java (93%) rename ingestion/src/main/java/feast/{storage/bigquery/BigQueryWarehouseStore.java => store/warehouse/bigquery/BigQueryWarehouseFactory.java} (79%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/FeatureRowBigQueryIO.java (97%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/FeatureRowToBigQueryTableRowDoFn.java (96%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/ValueBigQueryBuilder.java (99%) create mode 100644 ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java create mode 100644 ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java delete mode 100644 ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java rename ingestion/src/test/java/feast/{storage/file => store}/FileStoreOptionsTest.java (93%) rename ingestion/src/test/java/feast/{storage/MockErrorsStore.java => store/MockFeatureErrorsFactory.java} (67%) rename ingestion/src/test/java/feast/{storage => store}/MockFeatureStore.java (85%) rename ingestion/src/test/java/feast/{storage/MockServingStore.java => store/MockServingFactory.java} (67%) rename ingestion/src/test/java/feast/{storage => store}/MockTransforms.java (87%) rename ingestion/src/test/java/feast/{storage/MockWarehouseStore.java => store/MockWarehouseFactory.java} (67%) rename ingestion/src/test/java/feast/{storage/file/TextFileFeatureIOTest.java => store/TextFileDynamicIOTest.java} (56%) rename ingestion/src/test/java/feast/{storage => store/serving}/bigtable/BigTableFeatureOptionsTest.java (91%) rename ingestion/src/test/java/feast/{storage => store/serving}/bigtable/BigTableStoreOptionsTest.java (95%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/FeatureRowRedisIOWriteTest.java (93%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/FeatureRowToRedisMutationDoFnTest.java (93%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/RedisFeatureOptionsTest.java (95%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/RedisStoreOptionsTest.java (92%) rename ingestion/src/test/java/feast/{storage => store/warehouse}/bigquery/BigQueryStoreOptionsTest.java (97%) rename ingestion/src/test/java/feast/{storage => store/warehouse}/bigquery/ValueBigQueryBuilderTest.java (98%) rename ingestion/src/test/java/feast/{storage/file => store/warehouse}/json/JsonFileFeatureIOTest.java (59%) diff --git a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java index 7e9e235b5bc..81dadaf38e1 100644 --- a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java +++ b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java @@ -28,12 +28,12 @@ import feast.ingestion.service.SpecService.Builder; import feast.ingestion.service.SpecService.UnsupportedBuilder; import feast.specs.ImportSpecProto.ImportSpec; -import feast.storage.ErrorsStore; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; -import feast.storage.service.ErrorsStoreService; -import feast.storage.service.ServingStoreService; -import feast.storage.service.WarehouseStoreService; +import feast.store.errors.FeatureErrorsFactory; +import feast.store.errors.FeatureErrorsFactoryService; +import feast.store.serving.FeatureServingFactory; +import feast.store.serving.FeatureServingFactoryService; +import feast.store.warehouse.FeatureWarehouseFactory; +import feast.store.warehouse.FeatureWarehouseFactoryService; import java.util.List; import org.apache.beam.sdk.options.PipelineOptions; @@ -78,19 +78,19 @@ Specs provideSpecs(SpecService.Builder specService) { @Provides @Singleton - List provideWarehouseStores() { - return WarehouseStoreService.getAll(); + List provideWarehouseStores() { + return FeatureWarehouseFactoryService.getAll(); } @Provides @Singleton - List provideServingStores() { - return ServingStoreService.getAll(); + List provideServingStores() { + return FeatureServingFactoryService.getAll(); } @Provides @Singleton - List provideErrorsStores() { - return ErrorsStoreService.getAll(); + List provideErrorsStores() { + return FeatureErrorsFactoryService.getAll(); } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java index 7b8e41030e9..92590b91df9 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java @@ -23,10 +23,10 @@ import com.google.inject.Inject; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; -import feast.storage.noop.NoOpIO; +import feast.store.FeatureStoreWrite; +import feast.store.NoOpIO; +import feast.store.errors.FeatureErrorsFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.List; import lombok.extern.slf4j.Slf4j; @@ -35,18 +35,19 @@ import org.apache.hadoop.hbase.util.Strings; @Slf4j -public class ErrorsStoreTransform extends FeatureIO.Write { +public class ErrorsStoreTransform extends FeatureStoreWrite { private String errorsStoreType; private StorageSpec errorsStoreSpec; private Specs specs; - private List errorsStores; + private List errorsStoreFactories; @Inject public ErrorsStoreTransform( - ImportJobPipelineOptions options, Specs specs, List errorsStores) { + ImportJobPipelineOptions options, Specs specs, + List errorsStoreFactories) { this.specs = specs; - this.errorsStores = errorsStores; + this.errorsStoreFactories = errorsStoreFactories; this.errorsStoreType = options.getErrorsStoreType(); if (!Strings.isEmpty(errorsStoreType)) { @@ -60,7 +61,7 @@ public ErrorsStoreTransform( @Override public PDone expand(PCollection input) { - Write write; + FeatureStoreWrite write; if (Strings.isEmpty(errorsStoreType)) { write = new NoOpIO.Write(); } else { @@ -70,11 +71,11 @@ public PDone expand(PCollection input) { return PDone.in(input.getPipeline()); } - ErrorsStore getErrorStore() { + FeatureErrorsFactory getErrorStore() { checkArgument(!errorsStoreType.isEmpty(), "Errors store type not provided"); - for (ErrorsStore errorsStore : errorsStores) { - if (errorsStore.getType().equals(errorsStoreType)) { - return errorsStore; + for (FeatureErrorsFactory errorsStoreFactory : errorsStoreFactories) { + if (errorsStoreFactory.getType().equals(errorsStoreType)) { + return errorsStoreFactory; } } throw new IllegalArgumentException("Errors store type not found"); diff --git a/ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java b/ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java deleted file mode 100644 index c6d9a3b1c09..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.transform; - -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; -import lombok.AllArgsConstructor; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.PInput; - -public class FeatureIO { - - public abstract static class Read extends PTransform> {} - - public abstract static class Write extends PTransform, PDone> {} - - /** Used during setup if read transform can not be determined. */ - @AllArgsConstructor - public static class UnknownRead extends Read { - private String message; - - @Override - public PCollection expand(PInput input) { - throw new IllegalArgumentException(message); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java index feb712da0d1..024687bdfaa 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java @@ -21,21 +21,20 @@ import feast.ingestion.metrics.FeastMetrics; import feast.ingestion.model.Specs; import feast.ingestion.values.PFeatureRows; -import feast.storage.ServingStore; +import feast.store.serving.FeatureServingFactory; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import java.util.List; - @Slf4j public class ServingStoreTransform extends PTransform { - private List stores; + private List stores; private Specs specs; @Inject - public ServingStoreTransform(List stores, Specs specs) { + public ServingStoreTransform(List stores, Specs specs) { this.stores = stores; this.specs = specs; } @@ -50,7 +49,8 @@ public PFeatureRows expand(PFeatureRows input) { output.getMain().apply("metrics.store.lag", ParDo.of(FeastMetrics.lagUpdateDoFn())); output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("serving_stored"))); - output.getErrors().apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("serving_errors"))); + output.getErrors() + .apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("serving_errors"))); return output; } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java index cb11490dd35..c7e58892f02 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java @@ -20,12 +20,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; import feast.ingestion.transform.SplitFeatures.MultiOutputSplit; import feast.ingestion.values.PFeatureRows; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureStore; +import feast.store.FeatureStoreWrite; +import feast.store.FeatureStoreFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collection; import java.util.HashMap; @@ -46,20 +46,20 @@ @Slf4j public class SplitOutputByStore extends PTransform { - private Collection stores; + private Collection stores; private SerializableFunction selector; private Specs specs; @Override public PFeatureRows expand(PFeatureRows input) { - Map transforms = getFeatureStoreTransforms(); + Map transforms = getFeatureStoreTransforms(); Set keys = transforms.keySet(); log.info(String.format("Splitting on keys = [%s]", String.join(",", keys))); MultiOutputSplit splitter = new MultiOutputSplit<>(selector, keys, specs); PCollectionTuple splits = input.getMain().apply(splitter); - Map, Write> taggedTransforms = new HashMap<>(); + Map, FeatureStoreWrite> taggedTransforms = new HashMap<>(); for (String key : transforms.keySet()) { TupleTag tag = splitter.getStrategy().getTag(key); taggedTransforms.put(tag, transforms.get(key)); @@ -72,17 +72,17 @@ public PFeatureRows expand(PFeatureRows input) { input.getErrors()); } - private Map getStoresMap() { - Map storesMap = new HashMap<>(); - for (FeatureStore servingStore : stores) { + private Map getStoresMap() { + Map storesMap = new HashMap<>(); + for (FeatureStoreFactory servingStore : stores) { storesMap.put(servingStore.getType(), servingStore); } return storesMap; } - private Map getFeatureStoreTransforms() { - Map storesMap = getStoresMap(); - Map transforms = new HashMap<>(); + private Map getFeatureStoreTransforms() { + Map storesMap = getStoresMap(); + Map transforms = new HashMap<>(); Map storageSpecs = specs.getStorageSpecs(); for (String storeId : storageSpecs.keySet()) { StorageSpec storageSpec = storageSpecs.get(storeId); @@ -109,14 +109,14 @@ private Map getFeatureStoreTransforms() { public static class WriteTags extends PTransform> { - private Map, Write> transforms; + private Map, FeatureStoreWrite> transforms; private TupleTag mainTag; @Override public PCollection expand(PCollectionTuple tuple) { List> outputList = Lists.newArrayList(); for (TupleTag tag : transforms.keySet()) { - Write write = transforms.get(tag); + FeatureStoreWrite write = transforms.get(tag); Preconditions.checkNotNull(write, String.format("Null transform for tag=%s", tag.getId())); PCollection input = tuple.get(tag); input.apply(String.format("Write to %s", tag.getId()), write); diff --git a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java index dcfb9c44334..de70541961a 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java @@ -18,23 +18,23 @@ package feast.ingestion.transform; import com.google.inject.Inject; +import feast.ingestion.metrics.FeastMetrics; +import feast.ingestion.model.Specs; +import feast.ingestion.values.PFeatureRows; +import feast.store.warehouse.FeatureWarehouseFactory; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import feast.ingestion.metrics.FeastMetrics; -import feast.ingestion.model.Specs; -import feast.ingestion.values.PFeatureRows; -import feast.storage.WarehouseStore; @Slf4j public class WarehouseStoreTransform extends PTransform { - private List stores; + private List stores; private Specs specs; @Inject - public WarehouseStoreTransform(List stores, Specs specs) { + public WarehouseStoreTransform(List stores, Specs specs) { this.stores = stores; this.specs = specs; } @@ -48,8 +48,10 @@ public PFeatureRows expand(PFeatureRows input) { stores, (featureSpec) -> featureSpec.getDataStores().getWarehouse().getId(), specs)); - output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored"))); - output.getErrors().apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("warehouse_errors"))); + output.getMain() + .apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored"))); + output.getErrors() + .apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("warehouse_errors"))); return output; } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java index 40fc6054fd8..2704c5e766b 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java @@ -31,10 +31,10 @@ import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; -import feast.storage.service.ServingStoreService; -import feast.storage.service.WarehouseStoreService; +import feast.store.serving.FeatureServingFactory; +import feast.store.serving.FeatureServingFactoryService; +import feast.store.warehouse.FeatureWarehouseFactory; +import feast.store.warehouse.FeatureWarehouseFactoryService; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity.Enum; @@ -66,10 +66,10 @@ public void setup() { featureIds.add(field.getFeatureId()); } } - for (ServingStore store : ServingStoreService.getAll()) { + for (FeatureServingFactory store : FeatureServingFactoryService.getAll()) { supportedServingTypes.add(store.getType()); } - for (WarehouseStore store : WarehouseStoreService.getAll()) { + for (FeatureWarehouseFactory store : FeatureWarehouseFactoryService.getAll()) { supportedWarehouseTypes.add(store.getType()); } } diff --git a/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java b/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java index 795735410d9..dfe26c45b77 100644 --- a/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java +++ b/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java @@ -23,7 +23,7 @@ import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.common.collect.Maps; import com.google.protobuf.Timestamp; -import feast.storage.bigquery.ValueBigQueryBuilder; +import feast.store.warehouse.bigquery.ValueBigQueryBuilder; import java.util.Map; import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; @@ -34,8 +34,6 @@ import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; import feast.types.ValueProto.Value; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This is a serializable function used with the BigQueryIO for fetching feature rows directly from diff --git a/ingestion/src/main/java/feast/storage/StoreClient.java b/ingestion/src/main/java/feast/storage/StoreClient.java deleted file mode 100644 index 9f1302187b4..00000000000 --- a/ingestion/src/main/java/feast/storage/StoreClient.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import feast.serving.ServingAPIProto.TimestampRange; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.FeatureRowProto.FeatureRow; - -/** Abstraction of Client for Feast Storage. */ -public interface StoreClient { - - /** Get a the latest FeatureRow containing features for a single entity id */ - FeatureRow getLatest(String entityName, String entityId, List featureSpecs); - - /** Get a the latest FeatureRow for containing features for each entity id */ - List getLatest( - String entityName, Collection entityIds, List featureSpecs); - - /** - * Get a the N most recent FeatureRows for single entity id. Each featureRow will contain at least - * one of the features provided - */ - List getLatest( - String entityName, String entityId, List featureSpecs, int limit); - - /** - * Get up to N most recent FeatureRows for each entity id. Each featureRow will contain at least - * one of the features provided - * @return A map of feature ids to lists of FeatureRows - */ - Map> getLatestRange( - String entityName, Collection entityId, List featureSpecs, int limit, TimestampRange tsRange); -} diff --git a/ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java b/ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java deleted file mode 100644 index 0ebbbcb5408..00000000000 --- a/ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.file.json; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.util.JsonFormat; -import feast.ingestion.transform.FeatureIO; -import feast.storage.file.FileStoreOptions; -import feast.storage.file.TextFileFeatureIO; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import lombok.AllArgsConstructor; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; - -public class JsonFileFeatureIO { - - public static Write writeRowExtended(FileStoreOptions options) { - return new Write(options, (rowExtended) -> rowExtended); - } - - public static Write writeRow(FileStoreOptions options) { - return new Write(options, (rowExtended) -> rowExtended.getRow()); - } - - @AllArgsConstructor - public static class Write extends FeatureIO.Write { - - private FileStoreOptions options; - private SerializableFunction messageFunction; - - @Override - public PDone expand(PCollection input) { - return input.apply( - "Write Text Files", - new TextFileFeatureIO.Write( - options, - (rowExtended) -> { - try { - return JsonFormat.printer() - .omittingInsignificantWhitespace() - .print(messageFunction.apply(rowExtended)); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - }, - ".json")); - } - } -} diff --git a/ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java b/ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java deleted file mode 100644 index 11dda1e3b7a..00000000000 --- a/ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.file.json; - -import com.google.auto.service.AutoService; -import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; -import feast.options.OptionsParser; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; -import feast.storage.file.FileStoreOptions; -import lombok.AllArgsConstructor; - -public class JsonFileStores { - - private static final String JSON_FILES_TYPE = "file.json"; - - @AutoService(WarehouseStore.class) - @AllArgsConstructor - public static class JsonFileWarehouseStore implements WarehouseStore { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - FileStoreOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); - options.jobName = specs.getJobName(); - return JsonFileFeatureIO.writeRow(options); - } - - @Override - public String getType() { - return JSON_FILES_TYPE; - } - } - - @AutoService(ServingStore.class) - @AllArgsConstructor - public static class JsonFileServingStore implements ServingStore { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - FileStoreOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); - options.jobName = specs.getJobName(); - return JsonFileFeatureIO.writeRow(options); - } - - @Override - public String getType() { - return JSON_FILES_TYPE; - } - } - - @AutoService(ErrorsStore.class) - @AllArgsConstructor - public static class JsonFileErrorsStore implements ErrorsStore { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - FileStoreOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); - options.jobName = specs.getJobName(); - return JsonFileFeatureIO.writeRowExtended(options); - } - - @Override - public String getType() { - return JSON_FILES_TYPE; - } - } -} diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java b/ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java deleted file mode 100644 index 15de5921443..00000000000 --- a/ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.noop; - -import com.google.auto.service.AutoService; -import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; - -@AutoService(ServingStore.class) -public class NoOpServingStore implements ServingStore { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - return new NoOpIO.Write(); - } - - @Override - public String getType() { - return "noop"; - } -} diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java b/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java deleted file mode 100644 index f69e7c9fbb2..00000000000 --- a/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.noop; - -import com.google.auto.service.AutoService; -import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; - -@AutoService(WarehouseStore.class) -public class NoOpWarehouseStore implements WarehouseStore { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - return new NoOpIO.Write(); - } - - @Override - public String getType() { - return "noop"; - } -} diff --git a/ingestion/src/main/java/feast/storage/FeatureStore.java b/ingestion/src/main/java/feast/store/FeatureStoreFactory.java similarity index 81% rename from ingestion/src/main/java/feast/storage/FeatureStore.java rename to ingestion/src/main/java/feast/store/FeatureStoreFactory.java index a969c1476ad..998df98b924 100644 --- a/ingestion/src/main/java/feast/storage/FeatureStore.java +++ b/ingestion/src/main/java/feast/store/FeatureStoreFactory.java @@ -15,14 +15,13 @@ * */ -package feast.storage; +package feast.store; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; import feast.specs.StorageSpecProto.StorageSpec; -public interface FeatureStore { - FeatureIO.Write create(StorageSpec storageSpec, Specs specs); +public interface FeatureStoreFactory { + FeatureStoreWrite create(StorageSpec storageSpec, Specs specs); String getType(); } diff --git a/ingestion/src/main/java/feast/store/FeatureStoreWrite.java b/ingestion/src/main/java/feast/store/FeatureStoreWrite.java new file mode 100644 index 00000000000..759ff3bfa28 --- /dev/null +++ b/ingestion/src/main/java/feast/store/FeatureStoreWrite.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store; + +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +public abstract class FeatureStoreWrite extends PTransform, PDone> {} diff --git a/ingestion/src/main/java/feast/storage/file/FileStoreOptions.java b/ingestion/src/main/java/feast/store/FileStoreOptions.java similarity index 97% rename from ingestion/src/main/java/feast/storage/file/FileStoreOptions.java rename to ingestion/src/main/java/feast/store/FileStoreOptions.java index 84a5caa1379..5df669f0e98 100644 --- a/ingestion/src/main/java/feast/storage/file/FileStoreOptions.java +++ b/ingestion/src/main/java/feast/store/FileStoreOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.file; +package feast.store; import com.fasterxml.jackson.annotation.JsonIgnore; import javax.validation.constraints.NotEmpty; diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpIO.java b/ingestion/src/main/java/feast/store/NoOpIO.java similarity index 89% rename from ingestion/src/main/java/feast/storage/noop/NoOpIO.java rename to ingestion/src/main/java/feast/store/NoOpIO.java index 45e06e58012..65a68118c9b 100644 --- a/ingestion/src/main/java/feast/storage/noop/NoOpIO.java +++ b/ingestion/src/main/java/feast/store/NoOpIO.java @@ -15,9 +15,8 @@ * */ -package feast.storage.noop; +package feast.store; -import feast.ingestion.transform.FeatureIO; import feast.ingestion.transform.fn.Identity; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import org.apache.beam.sdk.transforms.ParDo; @@ -26,7 +25,7 @@ public class NoOpIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { @Override public PDone expand(PCollection input) { diff --git a/ingestion/src/main/java/feast/storage/file/TextFileFeatureIO.java b/ingestion/src/main/java/feast/store/TextFileDynamicIO.java similarity index 65% rename from ingestion/src/main/java/feast/storage/file/TextFileFeatureIO.java rename to ingestion/src/main/java/feast/store/TextFileDynamicIO.java index 740a2692126..ee7ed107516 100644 --- a/ingestion/src/main/java/feast/storage/file/TextFileFeatureIO.java +++ b/ingestion/src/main/java/feast/store/TextFileDynamicIO.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * */ -package feast.storage.file; +package feast.store; import lombok.AllArgsConstructor; import org.apache.beam.sdk.coders.StringUtf8Coder; @@ -23,43 +23,46 @@ import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk.io.WriteFilesResult; import org.apache.beam.sdk.transforms.Contextful; -import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.windowing.AfterWatermark; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PDone; import org.joda.time.Duration; -import feast.ingestion.transform.FeatureIO; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -public class TextFileFeatureIO { - private TextFileFeatureIO() {} +public class TextFileDynamicIO { + + private TextFileDynamicIO() { + } @AllArgsConstructor - public static class Write extends FeatureIO.Write { + public static class Write extends PTransform>, PDone> { + private final FileStoreOptions options; - private final SerializableFunction toTextFunction; private final String suffix; - /** Writes to different file sinks based on a */ + /** + * Writes to different file sinks based on a + */ @Override - public PDone expand(PCollection input) { + public PDone expand(PCollection> input) { final String folderName = options.jobName != null ? options.jobName : "unknown-jobs"; - FileIO.Write write = - FileIO.writeDynamic() - .by((rowExtended) -> rowExtended.getRow().getEntityName()) + FileIO.Write> write = + FileIO.>writeDynamic() + .by(KV::getKey) .withDestinationCoder(StringUtf8Coder.of()) .withNaming( Contextful.fn( - (entityName) -> FileIO.Write.defaultNaming(folderName + "/" + entityName, suffix))) - .via(Contextful.fn(toTextFunction), Contextful.fn((entityName) -> TextIO.sink())) + (key) -> FileIO.Write.defaultNaming(folderName + "/" + key + "/part-", suffix))) + .via(Contextful.fn(KV::getValue), Contextful.fn((entityName) -> TextIO.sink())) .to(options.path); if (input.isBounded().equals(IsBounded.UNBOUNDED)) { - Window minuteWindow = - Window.into(FixedWindows.of(options.getWindowDuration())) + Window> minuteWindow = + Window.>into(FixedWindows.of(options.getWindowDuration())) .triggering(AfterWatermark.pastEndOfWindow()) .discardingFiredPanes() .withAllowedLateness(Duration.ZERO); diff --git a/ingestion/src/main/java/feast/storage/ServingStore.java b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactory.java similarity index 81% rename from ingestion/src/main/java/feast/storage/ServingStore.java rename to ingestion/src/main/java/feast/store/errors/FeatureErrorsFactory.java index aa08181381e..6047e4c0887 100644 --- a/ingestion/src/main/java/feast/storage/ServingStore.java +++ b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactory.java @@ -15,6 +15,8 @@ * */ -package feast.storage; +package feast.store.errors; -public interface ServingStore extends FeatureStore {} +import feast.store.FeatureStoreFactory; + +public interface FeatureErrorsFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/service/ErrorsStoreService.java b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactoryService.java similarity index 59% rename from ingestion/src/main/java/feast/storage/service/ErrorsStoreService.java rename to ingestion/src/main/java/feast/store/errors/FeatureErrorsFactoryService.java index 0f2e8a30e09..4f5ad2c50a2 100644 --- a/ingestion/src/main/java/feast/storage/service/ErrorsStoreService.java +++ b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactoryService.java @@ -15,37 +15,41 @@ * */ -package feast.storage.service; +package feast.store.errors; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import feast.storage.ErrorsStore; -import lombok.extern.slf4j.Slf4j; - import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; +import lombok.extern.slf4j.Slf4j; +/** + * Service class for fetching all the FeatureErrorsFactory instances available + */ @Slf4j -public class ErrorsStoreService { +public class FeatureErrorsFactoryService { - private static ServiceLoader serviceLoader = ServiceLoader.load(ErrorsStore.class); - private static List manuallyRegistered = new ArrayList<>(); + private static ServiceLoader serviceLoader = ServiceLoader + .load(FeatureErrorsFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (ErrorsStore store : getAll()) { - log.info("ErrorsStore type found: " + store.getType()); + for (FeatureErrorsFactory store : getAll()) { + log.info("FeatureErrorsFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } - /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (ErrorsStore store : getAll()) { + /** + * Get store of the given subclass. + */ + public static T get(Class clazz) { + for (FeatureErrorsFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -54,7 +58,7 @@ public static T get(Class clazz) { return null; } - public static void register(ErrorsStore store) { + public static void register(FeatureErrorsFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java new file mode 100644 index 00000000000..3a56382b1f4 --- /dev/null +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store.errors.json; + +import com.google.auto.service.AutoService; +import feast.ingestion.model.Specs; +import feast.store.FeatureStoreWrite; +import feast.options.OptionsParser; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.store.FileStoreOptions; +import feast.store.errors.FeatureErrorsFactory; +import lombok.AllArgsConstructor; + +@AutoService(FeatureErrorsFactory.class) +@AllArgsConstructor + +public class JsonFileErrorsFactory implements FeatureErrorsFactory { + + private static final String JSON_FILES_TYPE = "file.json"; + + @Override + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { + FileStoreOptions options = + OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); + options.jobName = specs.getJobName(); + return new JsonFileErrorsWrite(options); + } + + @Override + public String getType() { + return JSON_FILES_TYPE; + } +} diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java new file mode 100644 index 00000000000..ef95bf634bf --- /dev/null +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store.errors.json; + +import static org.apache.beam.sdk.values.TypeDescriptors.kvs; +import static org.apache.beam.sdk.values.TypeDescriptors.strings; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.store.FeatureStoreWrite; +import feast.store.FileStoreOptions; +import feast.store.TextFileDynamicIO; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import lombok.AllArgsConstructor; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +@AllArgsConstructor +public class JsonFileErrorsWrite extends FeatureStoreWrite { + + private FileStoreOptions options; + + @Override + public PDone expand(PCollection input) { + return input.apply("Map to strings", MapElements.into(kvs(strings(), strings())).via( + (rowExtended) -> { + try { + return KV.of( + rowExtended.getRow().getEntityName(), + JsonFormat.printer().omittingInsignificantWhitespace() + .print(rowExtended) + ); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + )).apply("Write Error Json Files", new TextFileDynamicIO.Write(options, ".json")); + } +} diff --git a/ingestion/src/main/java/feast/storage/stderr/LogIO.java b/ingestion/src/main/java/feast/store/errors/logging/LogIO.java similarity index 83% rename from ingestion/src/main/java/feast/storage/stderr/LogIO.java rename to ingestion/src/main/java/feast/store/errors/logging/LogIO.java index abd6f7c9ff3..45a6c6826a0 100644 --- a/ingestion/src/main/java/feast/storage/stderr/LogIO.java +++ b/ingestion/src/main/java/feast/store/errors/logging/LogIO.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,9 +15,9 @@ * */ -package feast.storage.stderr; +package feast.store.errors.logging; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.ingestion.transform.fn.LoggerDoFn; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import lombok.AllArgsConstructor; @@ -29,7 +29,7 @@ public class LogIO { @AllArgsConstructor - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private Level level; diff --git a/ingestion/src/main/java/feast/storage/stderr/StderrErrorsStore.java b/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java similarity index 68% rename from ingestion/src/main/java/feast/storage/stderr/StderrErrorsStore.java rename to ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java index 623b7383aa1..0e904928db0 100644 --- a/ingestion/src/main/java/feast/storage/stderr/StderrErrorsStore.java +++ b/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,22 +15,22 @@ * */ -package feast.storage.stderr; +package feast.store.errors.logging; import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; +import feast.store.errors.FeatureErrorsFactory; import org.slf4j.event.Level; -@AutoService(ErrorsStore.class) -public class StderrErrorsStore implements ErrorsStore { +@AutoService(FeatureErrorsFactory.class) +public class StderrFeatureErrorsFactory implements FeatureErrorsFactory { public static final String TYPE_STDERR = "stderr"; - + @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { return new LogIO.Write(Level.ERROR); } diff --git a/ingestion/src/main/java/feast/storage/stderr/StdoutErrorsStore.java b/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java similarity index 68% rename from ingestion/src/main/java/feast/storage/stderr/StdoutErrorsStore.java rename to ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java index 7c5cc464377..afc3a35d1b5 100644 --- a/ingestion/src/main/java/feast/storage/stderr/StdoutErrorsStore.java +++ b/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,22 +15,22 @@ * */ -package feast.storage.stderr; +package feast.store.errors.logging; import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; +import feast.store.errors.FeatureErrorsFactory; import org.slf4j.event.Level; -@AutoService(ErrorsStore.class) -public class StdoutErrorsStore implements ErrorsStore { +@AutoService(FeatureErrorsFactory.class) +public class StdoutFeatureErrorsFactory implements FeatureErrorsFactory { public static final String TYPE_STDOUT = "stdout"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { return new LogIO.Write(Level.INFO); } diff --git a/ingestion/src/main/java/feast/storage/ErrorsStore.java b/ingestion/src/main/java/feast/store/serving/FeatureServingFactory.java similarity index 81% rename from ingestion/src/main/java/feast/storage/ErrorsStore.java rename to ingestion/src/main/java/feast/store/serving/FeatureServingFactory.java index 4fde4ec00fb..ca9491973e9 100644 --- a/ingestion/src/main/java/feast/storage/ErrorsStore.java +++ b/ingestion/src/main/java/feast/store/serving/FeatureServingFactory.java @@ -15,6 +15,8 @@ * */ -package feast.storage; +package feast.store.serving; -public interface ErrorsStore extends FeatureStore {} +import feast.store.FeatureStoreFactory; + +public interface FeatureServingFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/service/ServingStoreService.java b/ingestion/src/main/java/feast/store/serving/FeatureServingFactoryService.java similarity index 59% rename from ingestion/src/main/java/feast/storage/service/ServingStoreService.java rename to ingestion/src/main/java/feast/store/serving/FeatureServingFactoryService.java index a1ab6095948..d720c0dca1f 100644 --- a/ingestion/src/main/java/feast/storage/service/ServingStoreService.java +++ b/ingestion/src/main/java/feast/store/serving/FeatureServingFactoryService.java @@ -15,7 +15,7 @@ * */ -package feast.storage.service; +package feast.store.serving; import avro.shaded.com.google.common.collect.Lists; import com.google.common.collect.Iterators; @@ -23,27 +23,33 @@ import java.util.List; import java.util.ServiceLoader; import lombok.extern.slf4j.Slf4j; -import feast.storage.ServingStore; +/** + * Service class for fetching all the FeatureServingFactory instances available + */ @Slf4j -public class ServingStoreService { - private static ServiceLoader serviceLoader = ServiceLoader.load(ServingStore.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureServingFactoryService { + + private static ServiceLoader serviceLoader = ServiceLoader + .load(FeatureServingFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (ServingStore store : getAll()) { - log.info("ServingStore type found: " + store.getType()); + for (FeatureServingFactory store : getAll()) { + log.info("FeatureServingFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } - /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (ServingStore store : getAll()) { + /** + * Get store of the given subclass. + */ + public static T get(Class clazz) { + for (FeatureServingFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -52,7 +58,7 @@ public static T get(Class clazz) { return null; } - public static void register(ServingStore store) { + public static void register(FeatureServingFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/storage/FeatureStoreMigration.java b/ingestion/src/main/java/feast/store/serving/FeatureServingStoreClient.java similarity index 54% rename from ingestion/src/main/java/feast/storage/FeatureStoreMigration.java rename to ingestion/src/main/java/feast/store/serving/FeatureServingStoreClient.java index a35e7db3fcc..5aa7251b894 100644 --- a/ingestion/src/main/java/feast/storage/FeatureStoreMigration.java +++ b/ingestion/src/main/java/feast/store/serving/FeatureServingStoreClient.java @@ -15,19 +15,27 @@ * */ -package feast.storage; +package feast.store.serving; import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.types.FeatureRowProto.FeatureRow; +import java.util.Collection; import java.util.List; -import lombok.Value; /** - * This class is for notifying storage plugins about new or changed FeatureSpecs so they can update - * appropriate schemas. + * Abstraction of Client for Feast Serving Store */ -@Value -public class FeatureStoreMigration { - List create; - List drop; - List update; -} +public interface FeatureServingStoreClient { + + /** + * Get a the latest FeatureRow containing features for a single entity id + */ + FeatureRow get(String entityName, String entityId, List featureSpecs); + + /** + * Get a the latest FeatureRow for containing features for each entity id + */ + List get( + String entityName, Collection entityIds, List featureSpecs); + +} \ No newline at end of file diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableFeatureOptions.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableFeatureOptions.java similarity index 95% rename from ingestion/src/main/java/feast/storage/bigtable/BigTableFeatureOptions.java rename to ingestion/src/main/java/feast/store/serving/bigtable/BigTableFeatureOptions.java index 6997ae4397d..68bb7e6ec58 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableFeatureOptions.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableFeatureOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import feast.options.Options; diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java similarity index 79% rename from ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java rename to ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java index e6aa7f3232a..9583a5d431b 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java @@ -15,22 +15,23 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; +import feast.store.serving.FeatureServingFactory; + +@AutoService(FeatureServingFactory.class) +public class BigTableServingStoreFactory implements FeatureServingFactory { -@AutoService(ServingStore.class) -public class BigTableServingStoreFactory implements ServingStore { public static String TYPE_BIGTABLE = "bigtable"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { Preconditions.checkArgument(storageSpec.getType().equals(getType())); BigTableStoreOptions options = diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableStoreOptions.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableStoreOptions.java similarity index 95% rename from ingestion/src/main/java/feast/storage/bigtable/BigTableStoreOptions.java rename to ingestion/src/main/java/feast/store/serving/bigtable/BigTableStoreOptions.java index fcfcace69ba..9e1bbfb8b46 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableStoreOptions.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableStoreOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import java.io.Serializable; import javax.validation.constraints.NotEmpty; diff --git a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java similarity index 95% rename from ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java rename to ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java index 7437f59c55a..4dc289aeb2b 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java @@ -15,12 +15,12 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import com.google.cloud.bigtable.beam.CloudBigtableConfiguration; import com.google.cloud.bigtable.beam.CloudBigtableIO; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collections; import lombok.extern.slf4j.Slf4j; @@ -34,7 +34,7 @@ @Slf4j public class FeatureRowBigTableIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private BigTableStoreOptions bigTableOptions; private Specs specs; diff --git a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java similarity index 99% rename from ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java rename to ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java index bfd44ceda87..1d14b1201eb 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import com.google.common.base.Charsets; import feast.SerializableCache; diff --git a/ingestion/src/main/java/feast/storage/redis/FeatureRowRedisIO.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java similarity index 88% rename from ingestion/src/main/java/feast/storage/redis/FeatureRowRedisIO.java rename to ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java index d58abdc0f54..9c5772140b6 100644 --- a/ingestion/src/main/java/feast/storage/redis/FeatureRowRedisIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java @@ -15,11 +15,11 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; -import feast.storage.redis.RedisCustomIO.RedisMutation; +import feast.store.FeatureStoreWrite; +import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; @@ -27,7 +27,8 @@ public class FeatureRowRedisIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { + private final RedisStoreOptions options; private final Specs specs; diff --git a/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java similarity index 96% rename from ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java rename to ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java index dad225d7ba8..dbe57bff9fa 100644 --- a/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import static com.google.common.base.Preconditions.checkArgument; @@ -26,8 +26,8 @@ import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; -import feast.storage.redis.RedisCustomIO.Method; -import feast.storage.redis.RedisCustomIO.RedisMutation; +import feast.store.serving.redis.RedisCustomIO.Method; +import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; diff --git a/ingestion/src/main/java/feast/storage/redis/RedisCustomIO.java b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java similarity index 99% rename from ingestion/src/main/java/feast/storage/redis/RedisCustomIO.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java index d9b8261093b..4dc53467c70 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisCustomIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java @@ -16,7 +16,7 @@ */ // package io.suryawirawan.henry.beam.redis.io; -package feast.storage.redis; +package feast.store.serving.redis; import lombok.Builder; import lombok.Data; diff --git a/ingestion/src/main/java/feast/storage/redis/RedisFeatureOptions.java b/ingestion/src/main/java/feast/store/serving/redis/RedisFeatureOptions.java similarity index 97% rename from ingestion/src/main/java/feast/storage/redis/RedisFeatureOptions.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisFeatureOptions.java index e4654e2a43f..cf868cc47bb 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisFeatureOptions.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisFeatureOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import com.fasterxml.jackson.annotation.JsonIgnore; import org.joda.time.Duration; diff --git a/ingestion/src/main/java/feast/storage/redis/RedisServingStore.java b/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java similarity index 80% rename from ingestion/src/main/java/feast/storage/redis/RedisServingStore.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java index 6b5fc23ce57..ca023c616bd 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisServingStore.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java @@ -15,22 +15,23 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; +import feast.store.serving.FeatureServingFactory; + +@AutoService(FeatureServingFactory.class) +public class RedisServingFactory implements FeatureServingFactory { -@AutoService(ServingStore.class) -public class RedisServingStore implements ServingStore { public static final String TYPE_REDIS = "redis"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { Preconditions.checkArgument( storageSpec.getType().equals(TYPE_REDIS), "Storage spec type was not " + TYPE_REDIS); RedisStoreOptions options = diff --git a/ingestion/src/main/java/feast/storage/redis/RedisStoreOptions.java b/ingestion/src/main/java/feast/store/serving/redis/RedisStoreOptions.java similarity index 97% rename from ingestion/src/main/java/feast/storage/redis/RedisStoreOptions.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisStoreOptions.java index eecffac5b23..9b257d40f00 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisStoreOptions.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisStoreOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import feast.options.Options; import javax.validation.constraints.NotEmpty; diff --git a/ingestion/src/main/java/feast/storage/WarehouseStore.java b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactory.java similarity index 80% rename from ingestion/src/main/java/feast/storage/WarehouseStore.java rename to ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactory.java index b4bda0a33da..72b5617962e 100644 --- a/ingestion/src/main/java/feast/storage/WarehouseStore.java +++ b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactory.java @@ -15,6 +15,10 @@ * */ -package feast.storage; +package feast.store.warehouse; -public interface WarehouseStore extends FeatureStore {} +import feast.store.FeatureStoreFactory; + +public interface FeatureWarehouseFactory extends FeatureStoreFactory { + +} diff --git a/ingestion/src/main/java/feast/storage/service/WarehouseStoreService.java b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactoryService.java similarity index 60% rename from ingestion/src/main/java/feast/storage/service/WarehouseStoreService.java rename to ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactoryService.java index 6236be0ab06..06ae7236134 100644 --- a/ingestion/src/main/java/feast/storage/service/WarehouseStoreService.java +++ b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactoryService.java @@ -15,7 +15,7 @@ * */ -package feast.storage.service; +package feast.store.warehouse; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -23,28 +23,30 @@ import java.util.List; import java.util.ServiceLoader; import lombok.extern.slf4j.Slf4j; -import feast.storage.WarehouseStore; +/** + * Service class for fetching all the FeatureWarehouseFactory instances available + */ @Slf4j -public class WarehouseStoreService { - private static ServiceLoader serviceLoader = - ServiceLoader.load(WarehouseStore.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureWarehouseFactoryService { + private static ServiceLoader serviceLoader = + ServiceLoader.load(FeatureWarehouseFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (WarehouseStore store : getAll()) { - log.info("WarehouseStore type found: " + store.getType()); + for (FeatureWarehouseFactory store : getAll()) { + log.info("FeatureWarehouseFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (WarehouseStore store : getAll()) { + public static T get(Class clazz) { + for (FeatureWarehouseFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -53,7 +55,7 @@ public static T get(Class clazz) { return null; } - public static void register(WarehouseStore store) { + public static void register(FeatureWarehouseFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/storage/bigquery/BigQueryStoreOptions.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryStoreOptions.java similarity index 93% rename from ingestion/src/main/java/feast/storage/bigquery/BigQueryStoreOptions.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryStoreOptions.java index 95c70be24b2..f82efb58e8b 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/BigQueryStoreOptions.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryStoreOptions.java @@ -15,9 +15,8 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; -import java.io.Serializable; import javax.validation.constraints.NotEmpty; import feast.options.Options; diff --git a/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStore.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java similarity index 79% rename from ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStore.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java index a7682908bd1..33ad75a32b3 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStore.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java @@ -15,22 +15,22 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.WarehouseStore; +import feast.store.warehouse.FeatureWarehouseFactory; -@AutoService(WarehouseStore.class) -public class BigQueryWarehouseStore implements WarehouseStore { +@AutoService(FeatureWarehouseFactory.class) +public class BigQueryWarehouseFactory implements FeatureWarehouseFactory { public static String TYPE_BIGQUERY = "bigquery"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { Preconditions.checkArgument( storageSpec.getType().equals(TYPE_BIGQUERY), "Storage spec type was not " + TYPE_BIGQUERY); diff --git a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowBigQueryIO.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java similarity index 97% rename from ingestion/src/main/java/feast/storage/bigquery/FeatureRowBigQueryIO.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java index b5e039cc9e1..c8ec8a7825e 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowBigQueryIO.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java @@ -15,13 +15,13 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.api.services.bigquery.model.TableSchema; import com.google.common.base.Strings; import com.google.inject.Inject; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.ingestion.transform.SplitFeatures.SingleOutputSplit; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureSpecProto.FeatureSpec; @@ -45,7 +45,7 @@ @Slf4j public class FeatureRowBigQueryIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private final BigQueryStoreOptions bigQueryOptions; private final Specs specs; diff --git a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowToBigQueryTableRowDoFn.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java similarity index 96% rename from ingestion/src/main/java/feast/storage/bigquery/FeatureRowToBigQueryTableRowDoFn.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java index a05569f61d3..c666d4d71ab 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowToBigQueryTableRowDoFn.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java @@ -15,13 +15,11 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.api.services.bigquery.model.TableRow; import feast.ingestion.util.DateUtil; import feast.ingestion.model.Specs; -import feast.ingestion.model.Specs; -import feast.ingestion.util.DateUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; diff --git a/ingestion/src/main/java/feast/storage/bigquery/ValueBigQueryBuilder.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/ValueBigQueryBuilder.java similarity index 99% rename from ingestion/src/main/java/feast/storage/bigquery/ValueBigQueryBuilder.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/ValueBigQueryBuilder.java index ab292e3e0d0..34700dc96b7 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/ValueBigQueryBuilder.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/ValueBigQueryBuilder.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.protobuf.ByteString; diff --git a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java new file mode 100644 index 00000000000..5572772fcc2 --- /dev/null +++ b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store.warehouse.json; + +import com.google.auto.service.AutoService; +import feast.ingestion.model.Specs; +import feast.store.FeatureStoreWrite; +import feast.options.OptionsParser; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.store.FileStoreOptions; +import feast.store.warehouse.FeatureWarehouseFactory; +import lombok.AllArgsConstructor; + +@AutoService(FeatureWarehouseFactory.class) +@AllArgsConstructor +public class JsonFileWarehouseFactory implements FeatureWarehouseFactory { + + private static final String JSON_FILES_TYPE = "file.json"; + + @Override + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { + FileStoreOptions options = + OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); + options.jobName = specs.getJobName(); + return new JsonFileWarehouseWrite(options); + } + + @Override + public String getType() { + return JSON_FILES_TYPE; + } +} diff --git a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java new file mode 100644 index 00000000000..fd3050a049f --- /dev/null +++ b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store.warehouse.json; + +import static org.apache.beam.sdk.values.TypeDescriptors.kvs; +import static org.apache.beam.sdk.values.TypeDescriptors.strings; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.store.FeatureStoreWrite; +import feast.store.FileStoreOptions; +import feast.store.TextFileDynamicIO; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import lombok.AllArgsConstructor; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +@AllArgsConstructor +public class JsonFileWarehouseWrite extends FeatureStoreWrite { + + private FileStoreOptions options; + + @Override + public PDone expand(PCollection input) { + return input.apply("Map to strings", MapElements.into(kvs(strings(), strings())).via( + (rowExtended) -> { + try { + return KV.of( + rowExtended.getRow().getEntityName(), + JsonFormat.printer().omittingInsignificantWhitespace() + .print(rowExtended.getRow()) + ); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + )).apply("Write Warehouse Json Files", new TextFileDynamicIO.Write(options, ".json")); + } +} diff --git a/ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java b/ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java deleted file mode 100644 index 9f2853bc47c..00000000000 --- a/ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.beam.sdk.io.gcp.pubsub; - -import com.google.common.collect.Lists; -import com.google.protobuf.Message; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; -import org.apache.beam.sdk.options.PipelineOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This PubSub helper exists because including the canonical client libraries can result in - * dependency problems. We place it in the beam package because we need protected methods. - */ -public class FeastPubsubHelper { - - private final PipelineOptions options; - private transient PubsubClient client; - - public FeastPubsubHelper(PipelineOptions options) { - this.options = options; - } - - private PubsubClient getClient() throws IOException { - if (client == null) { - client = PubsubJsonClient.FACTORY.newClient(null, null, options.as(PubsubOptions.class)); - } - return client; - } - - public int publish(TopicPath topicPath, Message message) throws IOException { - List messages = - Lists.newArrayList( - new OutgoingMessage( - message.toByteArray(), new HashMap<>(), System.currentTimeMillis(), null)); - return getClient().publish(topicPath, messages); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index bd7460e6372..6379e8dfad4 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -19,7 +19,7 @@ import static feast.FeastMatchers.hasCount; import static feast.NormalizeFeatureRows.normalize; -import static feast.storage.MockErrorsStore.MOCK_ERRORS_STORE_TYPE; +import static feast.store.MockFeatureErrorsFactory.MOCK_ERRORS_STORE_TYPE; import static org.junit.Assert.assertEquals; import com.google.common.base.Charsets; @@ -38,12 +38,12 @@ import feast.ingestion.service.SpecRetrievalException; import feast.ingestion.util.ProtoUtil; import feast.specs.ImportSpecProto.ImportSpec; -import feast.storage.MockErrorsStore; -import feast.storage.MockServingStore; -import feast.storage.MockWarehouseStore; -import feast.storage.service.ErrorsStoreService; -import feast.storage.service.ServingStoreService; -import feast.storage.service.WarehouseStoreService; +import feast.store.MockFeatureErrorsFactory; +import feast.store.MockServingFactory; +import feast.store.MockWarehouseFactory; +import feast.store.errors.FeatureErrorsFactoryService; +import feast.store.serving.FeatureServingFactoryService; +import feast.store.warehouse.FeatureWarehouseFactoryService; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity; @@ -121,16 +121,21 @@ public void testImportCSV() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); List expectedRows = @@ -212,16 +217,21 @@ public void testImportCSV_withSample1() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToServing).satisfies(hasCount(1)); @@ -271,16 +281,21 @@ public void testImportCSV_withCoalesceRows() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors).satisfies(hasCount(0)); @@ -394,11 +409,15 @@ public void testImportWithErrors() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors) @@ -459,11 +478,15 @@ public void testImportWithoutWarehouseStore() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors) diff --git a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java index 83c0ef3e025..bddb79515b1 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java @@ -18,14 +18,14 @@ package feast.ingestion.transform; import static feast.ingestion.model.Errors.toError; -import static feast.storage.MockErrorsStore.MOCK_ERRORS_STORE_TYPE; +import static feast.store.MockFeatureErrorsFactory.MOCK_ERRORS_STORE_TYPE; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.storage.MockErrorsStore; -import feast.storage.service.ErrorsStoreService; -import feast.storage.stderr.StderrErrorsStore; -import feast.storage.stderr.StdoutErrorsStore; +import feast.store.MockFeatureErrorsFactory; +import feast.store.errors.FeatureErrorsFactoryService; +import feast.store.errors.logging.StderrFeatureErrorsFactory; +import feast.store.errors.logging.StdoutFeatureErrorsFactory; import feast.types.FeatureRowExtendedProto.Attempt; import feast.types.FeatureRowExtendedProto.Error; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -75,7 +75,7 @@ private FeatureRowExtended errorOf(String transform, Throwable cause) { @Test public void shouldWriteToGivenErrorsStore() { - MockErrorsStore mockStore = new MockErrorsStore(); + MockFeatureErrorsFactory mockStore = new MockFeatureErrorsFactory(); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); ErrorsStoreTransform transform = new ErrorsStoreTransform(options, specs, Lists.newArrayList(mockStore)); @@ -91,9 +91,9 @@ public void shouldWriteToGivenErrorsStore() { @Test public void logErrorsToStdErr() { - options.setErrorsStoreType(StderrErrorsStore.TYPE_STDERR); + options.setErrorsStoreType(StderrFeatureErrorsFactory.TYPE_STDERR); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, ErrorsStoreService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -101,9 +101,9 @@ public void logErrorsToStdErr() { @Test public void logErrorsToStdOut() { - options.setErrorsStoreType(StdoutErrorsStore.TYPE_STDOUT); + options.setErrorsStoreType(StdoutFeatureErrorsFactory.TYPE_STDOUT); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, ErrorsStoreService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -112,7 +112,7 @@ public void logErrorsToStdOut() { public void logToNull() { //options.setErrorsStoreType(...); // no errors store type set ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, ErrorsStoreService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } diff --git a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java index 7758a774899..183f117156b 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java @@ -26,7 +26,6 @@ import feast.ingestion.model.Specs; import feast.ingestion.model.Values; import feast.ingestion.service.MockSpecService; -import feast.ingestion.transform.FeatureIO.Write; import feast.ingestion.transform.SplitOutputByStore.WriteTags; import feast.ingestion.values.PFeatureRows; import feast.specs.EntitySpecProto.EntitySpec; @@ -37,9 +36,10 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureStore; -import feast.storage.MockFeatureStore; -import feast.storage.MockTransforms; +import feast.store.FeatureStoreWrite; +import feast.store.FeatureStoreFactory; +import feast.store.MockFeatureStore; +import feast.store.MockTransforms; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import java.util.Collections; @@ -78,7 +78,7 @@ public void testSplit() { DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store2"))).build()) .addStorage(StorageSpec.newBuilder().setId("store1").setType("type1").build()) .addStorage(StorageSpec.newBuilder().setId("store2").setType("type2").build()); - List stores = + List stores = Lists.newArrayList(new MockFeatureStore("type1"), new MockFeatureStore("type2")); Specs specs = Specs.of( @@ -186,7 +186,7 @@ public void testSplitWhereFeature2HasNoStoreId() { "store1", StorageSpec.newBuilder().setId("store1").setType("type1").build()); specService.storageSpecs.put( "store2", StorageSpec.newBuilder().setId("store2").setType("type2").build()); - List stores = + List stores = Lists.newArrayList(new MockFeatureStore("type1"), new MockFeatureStore("type2")); Specs specs = Specs.of( @@ -284,7 +284,7 @@ public void testSplitWhereNoStorageSpec() { .build(), specService); assertNull(specs.getError()); - List stores = Collections.emptyList(); + List stores = Collections.emptyList(); SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); PCollection input = @@ -346,7 +346,7 @@ public void testSplitWhereNoStorageSpecForAFeature() { .build(), specService); assertNull(specs.getError()); - List stores = Collections.emptyList(); + List stores = Collections.emptyList(); SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); PCollection input = @@ -385,7 +385,7 @@ public void testWriteTags() { TupleTag tag3 = new TupleTag<>("TAG3"); TupleTag mainTag = new TupleTag<>("TAG4"); - Map, Write> transforms = ImmutableMap., Write>builder() + Map, FeatureStoreWrite> transforms = ImmutableMap., FeatureStoreWrite>builder() .put(tag1, new MockTransforms.Write()) .put(tag2, new MockTransforms.Write()) // tag3 and mainTag do not have write transforms. @@ -418,7 +418,7 @@ public void testWriteTags() { // input 4 is is returned in the output because it is the main tag. PAssert.that(output).containsInAnyOrder(rowex1, rowex2, rowex4); - // Each non main tagged input should be written to corresponding Write transform + // Each non main tagged input should be written to corresponding FeatureStoreWrite transform PAssert.that(((MockTransforms.Write) transforms.get(tag1)).getInputs().get(0)) .containsInAnyOrder(rowex1); PAssert.that(((MockTransforms.Write) transforms.get(tag2)).getInputs().get(0)) diff --git a/ingestion/src/test/java/feast/storage/file/FileStoreOptionsTest.java b/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java similarity index 93% rename from ingestion/src/test/java/feast/storage/file/FileStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/FileStoreOptionsTest.java index 4a2be27a709..0d8e06182c6 100644 --- a/ingestion/src/test/java/feast/storage/file/FileStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,10 +15,11 @@ * */ -package feast.storage.file; +package feast.store; import static org.junit.Assert.assertEquals; +import feast.store.FileStoreOptions; import org.joda.time.Duration; import org.junit.Test; import feast.options.OptionsParser; diff --git a/ingestion/src/test/java/feast/storage/MockErrorsStore.java b/ingestion/src/test/java/feast/store/MockFeatureErrorsFactory.java similarity index 67% rename from ingestion/src/test/java/feast/storage/MockErrorsStore.java rename to ingestion/src/test/java/feast/store/MockFeatureErrorsFactory.java index e83ba5641e7..49721147a60 100644 --- a/ingestion/src/test/java/feast/storage/MockErrorsStore.java +++ b/ingestion/src/test/java/feast/store/MockFeatureErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,15 +15,17 @@ * */ -package feast.storage; +package feast.store; import com.google.auto.service.AutoService; +import feast.store.errors.FeatureErrorsFactory; -@AutoService(ErrorsStore.class) -public class MockErrorsStore extends MockFeatureStore implements ErrorsStore { +@AutoService(FeatureErrorsFactory.class) +public class MockFeatureErrorsFactory extends MockFeatureStore implements + FeatureErrorsFactory { public static final String MOCK_ERRORS_STORE_TYPE = "errors.mock"; - public MockErrorsStore() { + public MockFeatureErrorsFactory() { super(MOCK_ERRORS_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/MockFeatureStore.java b/ingestion/src/test/java/feast/store/MockFeatureStore.java similarity index 85% rename from ingestion/src/test/java/feast/storage/MockFeatureStore.java rename to ingestion/src/test/java/feast/store/MockFeatureStore.java index 2e957d4de23..cdde9e97405 100644 --- a/ingestion/src/test/java/feast/storage/MockFeatureStore.java +++ b/ingestion/src/test/java/feast/store/MockFeatureStore.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,13 +15,13 @@ * */ -package feast.storage; +package feast.store; import feast.ingestion.model.Specs; import feast.specs.StorageSpecProto.StorageSpec; import lombok.Getter; -public class MockFeatureStore implements FeatureStore { +public class MockFeatureStore implements FeatureStoreFactory { private final String type; @Getter private MockTransforms.Write write; diff --git a/ingestion/src/test/java/feast/storage/MockServingStore.java b/ingestion/src/test/java/feast/store/MockServingFactory.java similarity index 67% rename from ingestion/src/test/java/feast/storage/MockServingStore.java rename to ingestion/src/test/java/feast/store/MockServingFactory.java index 2c60bcbf784..11195c1385e 100644 --- a/ingestion/src/test/java/feast/storage/MockServingStore.java +++ b/ingestion/src/test/java/feast/store/MockServingFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,15 +15,17 @@ * */ -package feast.storage; +package feast.store; import com.google.auto.service.AutoService; +import feast.store.serving.FeatureServingFactory; -@AutoService(ServingStore.class) -public class MockServingStore extends MockFeatureStore implements ServingStore { +@AutoService(FeatureServingFactory.class) +public class MockServingFactory extends MockFeatureStore implements + FeatureServingFactory { public static final String MOCK_SERVING_STORE_TYPE = "serving.mock"; - public MockServingStore() { + public MockServingFactory() { super(MOCK_SERVING_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/MockTransforms.java b/ingestion/src/test/java/feast/store/MockTransforms.java similarity index 87% rename from ingestion/src/test/java/feast/storage/MockTransforms.java rename to ingestion/src/test/java/feast/store/MockTransforms.java index 1f5a573495f..7d241e77eed 100644 --- a/ingestion/src/test/java/feast/storage/MockTransforms.java +++ b/ingestion/src/test/java/feast/store/MockTransforms.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,10 +15,9 @@ * */ -package feast.storage; +package feast.store; import com.google.common.collect.Lists; -import feast.ingestion.transform.FeatureIO; import feast.ingestion.transform.fn.Identity; import feast.specs.StorageSpecProto.StorageSpec; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -31,7 +30,7 @@ public class MockTransforms { @Getter - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { List> inputs = Lists.newArrayList(); private StorageSpec spec; diff --git a/ingestion/src/test/java/feast/storage/MockWarehouseStore.java b/ingestion/src/test/java/feast/store/MockWarehouseFactory.java similarity index 67% rename from ingestion/src/test/java/feast/storage/MockWarehouseStore.java rename to ingestion/src/test/java/feast/store/MockWarehouseFactory.java index dbe9ab8caf5..ddfd06556ab 100644 --- a/ingestion/src/test/java/feast/storage/MockWarehouseStore.java +++ b/ingestion/src/test/java/feast/store/MockWarehouseFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,15 +15,17 @@ * */ -package feast.storage; +package feast.store; import com.google.auto.service.AutoService; +import feast.store.warehouse.FeatureWarehouseFactory; -@AutoService(WarehouseStore.class) -public class MockWarehouseStore extends MockFeatureStore implements WarehouseStore { +@AutoService(FeatureWarehouseFactory.class) +public class MockWarehouseFactory extends MockFeatureStore implements + FeatureWarehouseFactory { public static final String MOCK_WAREHOUSE_STORE_TYPE = "warehouse.mock"; - public MockWarehouseStore() { + public MockWarehouseFactory() { super(MOCK_WAREHOUSE_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/file/TextFileFeatureIOTest.java b/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java similarity index 56% rename from ingestion/src/test/java/feast/storage/file/TextFileFeatureIOTest.java rename to ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java index 84b87ca671f..0a52b4bc788 100644 --- a/ingestion/src/test/java/feast/storage/file/TextFileFeatureIOTest.java +++ b/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,9 @@ * */ -package feast.storage.file; +package feast.store; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import com.google.common.collect.Lists; @@ -27,43 +27,54 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; -public class TextFileFeatureIOTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); +@Slf4j +public class TextFileDynamicIOTest { - @Rule public TestPipeline testPipeline = TestPipeline.create(); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Rule + public TestPipeline testPipeline = TestPipeline.create(); @Test public void testWrite() throws IOException { File path = folder.newFolder(); FileStoreOptions options = new FileStoreOptions(); - options.jobName = testPipeline.getOptions().getJobName(); + options.jobName = "test-job"; options.path = path.getAbsolutePath(); - TextFileFeatureIO.Write write = - new TextFileFeatureIO.Write( - options, (featureRow) -> "output: " + featureRow.getRow().getEntityName(), ".text"); + TextFileDynamicIO.Write write = + new TextFileDynamicIO.Write( + options, ".text"); - PCollection rowExtended = - testPipeline.apply( - Create.of( - FeatureRowExtended.newBuilder() - .setRow(FeatureRow.newBuilder().setEntityName("testEntity")) - .build())); + PCollection> rowExtended = + testPipeline.apply(Create.of( + KV.of("part1", "line1"), + KV.of("part1", "line2"), + KV.of("part2", "line3"), + KV.of("part2", "line4"))); rowExtended.apply(write); testPipeline.run(); - List files = Files.walk(path.toPath()).collect(Collectors.toList()); + List part1 = getAllLines(path.toPath().resolve("test-job/part1")); + List part2 = getAllLines(path.toPath().resolve("test-job/part2")); + assertThat(part1, containsInAnyOrder("line1", "line2")); + assertThat(part2, containsInAnyOrder("line3", "line4")); + } + + List getAllLines(Path path) throws IOException { + List files = Files.walk(path).collect(Collectors.toList()); List lines = Lists.newArrayList(); for (Path file : files) { System.out.println(file); @@ -71,6 +82,6 @@ public void testWrite() throws IOException { lines.addAll(Files.readAllLines(file)); } } - assertThat(lines, equalTo(Lists.newArrayList("output: testEntity"))); + return lines; } } diff --git a/ingestion/src/test/java/feast/storage/bigtable/BigTableFeatureOptionsTest.java b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableFeatureOptionsTest.java similarity index 91% rename from ingestion/src/test/java/feast/storage/bigtable/BigTableFeatureOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/bigtable/BigTableFeatureOptionsTest.java index 85256265832..dfd3b853399 100644 --- a/ingestion/src/test/java/feast/storage/bigtable/BigTableFeatureOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableFeatureOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/storage/bigtable/BigTableStoreOptionsTest.java b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableStoreOptionsTest.java similarity index 95% rename from ingestion/src/test/java/feast/storage/bigtable/BigTableStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/bigtable/BigTableStoreOptionsTest.java index a777f50ffd3..b9957a2b453 100644 --- a/ingestion/src/test/java/feast/storage/bigtable/BigTableStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableStoreOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java similarity index 93% rename from ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java rename to ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java index 591736c1bed..3dd1b10112f 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,10 +15,10 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; -import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getFeatureIdSha1Prefix; -import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getRedisBucketKey; +import static feast.store.serving.redis.FeatureRowToRedisMutationDoFn.getFeatureIdSha1Prefix; +import static feast.store.serving.redis.FeatureRowToRedisMutationDoFn.getRedisBucketKey; import static org.junit.Assert.assertEquals; import com.google.common.io.Resources; @@ -29,7 +29,7 @@ import feast.ingestion.model.Values; import feast.ingestion.service.FileSpecService; import feast.ingestion.service.SpecService; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.ingestion.util.DateUtil; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; @@ -110,7 +110,7 @@ public void testWriteNoneGranularity() throws IOException { Specs specs = getSpecs(); specs.validate(); - new RedisServingStore().create(specs.getStorageSpec("REDIS1"), specs); + new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowRedisIO.Write write = new FeatureRowRedisIO.Write( RedisStoreOptions.builder().host("localhost").port(REDIS_PORT).build(), specs); @@ -154,7 +154,7 @@ public void testWriteNoneGranularity() throws IOException { @Test public void testWriteNoneGranularityFromOptions() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingStore().create(specs.getStorageSpec("REDIS1"), specs); + FeatureStoreWrite write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() @@ -195,7 +195,7 @@ public void testWriteNoneGranularityFromOptions() throws IOException { @Test public void testWriteHourGranularity() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingStore().create(specs.getStorageSpec("REDIS1"), specs); + FeatureStoreWrite write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java similarity index 93% rename from ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java rename to ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java index 242042fb357..64bc4014268 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import feast.ingestion.util.DateUtil; import feast.storage.RedisProto.RedisBucketKey; diff --git a/ingestion/src/test/java/feast/storage/redis/RedisFeatureOptionsTest.java b/ingestion/src/test/java/feast/store/serving/redis/RedisFeatureOptionsTest.java similarity index 95% rename from ingestion/src/test/java/feast/storage/redis/RedisFeatureOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/redis/RedisFeatureOptionsTest.java index 2cbbffb1db8..bf49aca612c 100644 --- a/ingestion/src/test/java/feast/storage/redis/RedisFeatureOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/RedisFeatureOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import org.joda.time.Duration; import org.junit.Assert; diff --git a/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java b/ingestion/src/test/java/feast/store/serving/redis/RedisStoreOptionsTest.java similarity index 92% rename from ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/redis/RedisStoreOptionsTest.java index 87b7fc98fad..335a3b1fedf 100644 --- a/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/RedisStoreOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import static org.junit.Assert.assertEquals; diff --git a/ingestion/src/test/java/feast/storage/bigquery/BigQueryStoreOptionsTest.java b/ingestion/src/test/java/feast/store/warehouse/bigquery/BigQueryStoreOptionsTest.java similarity index 97% rename from ingestion/src/test/java/feast/storage/bigquery/BigQueryStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/warehouse/bigquery/BigQueryStoreOptionsTest.java index 92a30cf56b0..4563d3e6fb9 100644 --- a/ingestion/src/test/java/feast/storage/bigquery/BigQueryStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/warehouse/bigquery/BigQueryStoreOptionsTest.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/storage/bigquery/ValueBigQueryBuilderTest.java b/ingestion/src/test/java/feast/store/warehouse/bigquery/ValueBigQueryBuilderTest.java similarity index 98% rename from ingestion/src/test/java/feast/storage/bigquery/ValueBigQueryBuilderTest.java rename to ingestion/src/test/java/feast/store/warehouse/bigquery/ValueBigQueryBuilderTest.java index 4ddc2d8c455..648fa5f151e 100644 --- a/ingestion/src/test/java/feast/storage/bigquery/ValueBigQueryBuilderTest.java +++ b/ingestion/src/test/java/feast/store/warehouse/bigquery/ValueBigQueryBuilderTest.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; diff --git a/ingestion/src/test/java/feast/storage/file/json/JsonFileFeatureIOTest.java b/ingestion/src/test/java/feast/store/warehouse/json/JsonFileFeatureIOTest.java similarity index 59% rename from ingestion/src/test/java/feast/storage/file/json/JsonFileFeatureIOTest.java rename to ingestion/src/test/java/feast/store/warehouse/json/JsonFileFeatureIOTest.java index cdc761b4ce6..97e09f9e7f3 100644 --- a/ingestion/src/test/java/feast/storage/file/json/JsonFileFeatureIOTest.java +++ b/ingestion/src/test/java/feast/store/warehouse/json/JsonFileFeatureIOTest.java @@ -15,56 +15,78 @@ * */ -package feast.storage.file.json; +package feast.store.warehouse.json; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import com.google.common.collect.Lists; +import feast.store.FileStoreOptions; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import feast.types.FeatureRowProto.FeatureRow; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import feast.storage.file.FileStoreOptions; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; +@Slf4j public class JsonFileFeatureIOTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Rule public TestPipeline testPipeline = TestPipeline.create(); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Rule + public TestPipeline testPipeline = TestPipeline.create(); @Test public void testWrite() throws IOException { File path = folder.newFolder(); FileStoreOptions options = new FileStoreOptions(); - options.jobName = testPipeline.getOptions().getJobName(); + options.jobName = "test-job"; options.path = path.getAbsolutePath(); - JsonFileFeatureIO.Write write = - new JsonFileFeatureIO.Write(options, FeatureRowExtended::getRow); + JsonFileWarehouseWrite write = new JsonFileWarehouseWrite(options); PCollection rowExtended = testPipeline.apply( Create.of( FeatureRowExtended.newBuilder() .setRow( - FeatureRow.newBuilder().setEntityName("testEntity").setEntityKey("1234")) + FeatureRow.newBuilder().setEntityName("testEntity1").setEntityKey("1234")) + .build(), + FeatureRowExtended.newBuilder() + .setRow( + FeatureRow.newBuilder().setEntityName("testEntity2").setEntityKey("1234")) .build())); rowExtended.apply(write); testPipeline.run(); - List files = Files.walk(path.toPath()).collect(Collectors.toList()); + for (Path p : Files.walk(path.toPath()).collect(Collectors.toList())) { + if (p.toFile().isFile()) { + log.debug(p.toString()); + } + } + List part1 = getAllLines(path.toPath().resolve("test-job/testEntity1")); + List part2 = getAllLines(path.toPath().resolve("test-job/testEntity2")); + assertThat(part1, + containsInAnyOrder("{\"entityKey\":\"1234\",\"entityName\":\"testEntity1\"}")); + assertThat(part2, + containsInAnyOrder("{\"entityKey\":\"1234\",\"entityName\":\"testEntity2\"}")); + } + + List getAllLines(Path path) throws IOException { + List files = Files.walk(path).collect(Collectors.toList()); List lines = Lists.newArrayList(); for (Path file : files) { System.out.println(file); @@ -72,8 +94,6 @@ public void testWrite() throws IOException { lines.addAll(Files.readAllLines(file)); } } - assertThat( - lines, - equalTo(Lists.newArrayList("{\"entityKey\":\"1234\",\"entityName\":\"testEntity\"}"))); + return lines; } } From de6d67c1b2c5125b2f708fde612ffc4f43cd9a7f Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Tue, 26 Feb 2019 15:10:15 +0800 Subject: [PATCH 07/25] Add thirteen37 to OWNERS (#144) --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index eb5ddd5c99b..756bd1cef14 100644 --- a/OWNERS +++ b/OWNERS @@ -3,8 +3,10 @@ approvers: - pradithya - woop - tims + - thirteen37 reviewers: - zhilingc - pradithya - woop - tims + - thirteen37 From 2718eda19bfdbdec74b0f499acbc2bbed9f4fa0c Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Tue, 26 Feb 2019 15:26:16 +0800 Subject: [PATCH 08/25] Remove historical feature in serving store (#87) * Change serving to only get latest value of feature * Update golang generated code * Update python SDK * Refactor python SDK * Remove server-side timestamp filter * Include timestamp filter in Python sdk * Update generated go file * Use datetime type for time range filter --- protos/feast/serving/Serving.proto | 76 +-- .../generated/go/feast/core/CoreService.pb.go | 90 ++-- .../go/feast/core/DatasetService.pb.go | 36 +- .../generated/go/feast/core/JobService.pb.go | 60 ++- .../generated/go/feast/core/UIService.pb.go | 108 +++-- .../generated/go/feast/serving/Serving.pb.go | 446 +++++------------- .../generated/go/feast/specs/EntitySpec.pb.go | 12 +- .../go/feast/specs/FeatureGroupSpec.pb.go | 12 +- .../go/feast/specs/FeatureSpec.pb.go | 24 +- .../generated/go/feast/specs/ImportSpec.pb.go | 24 +- .../go/feast/specs/StorageSpec.pb.go | 12 +- .../generated/go/feast/storage/BigTable.pb.go | 12 +- protos/generated/go/feast/storage/Redis.pb.go | 22 +- protos/generated/go/feast/types/Feature.pb.go | 10 +- .../generated/go/feast/types/FeatureRow.pb.go | 18 +- .../go/feast/types/FeatureRowExtended.pb.go | 22 +- .../go/feast/types/Granularity.pb.go | 16 +- protos/generated/go/feast/types/Value.pb.go | 74 +-- sdk/python/feast/sdk/client.py | 138 +++--- sdk/python/feast/serving/Serving_pb2.py | 306 +++--------- sdk/python/feast/serving/Serving_pb2_grpc.py | 10 +- sdk/python/tests/sdk/test_client.py | 252 ++++++---- .../serving/grpc/ServingGrpcService.java | 29 +- .../serving/http/ServingHttpService.java | 14 +- .../main/java/feast/serving/model/Pair.java | 45 -- .../serving/model/RequestDetailWithSpec.java | 31 -- .../service/BigTableFeatureStorage.java | 188 +------- .../feast/serving/service/FeastServing.java | 41 +- .../service/FeatureRetrievalDispatcher.java | 192 ++------ .../feast/serving/service/FeatureStorage.java | 62 +-- .../serving/service/RedisFeatureStorage.java | 311 +----------- .../feast/serving/util/EntityMapBuilder.java | 45 +- .../serving/util/FeatureValueListBuilder.java | 166 ------- .../feast/serving/util/RequestHelper.java | 82 +--- .../java/feast/serving/util/StatsUtil.java | 12 +- .../java/feast/serving/util/TimeUtil.java | 56 +-- .../feast/serving/grpc/FeastServingTest.java | 55 +-- .../serving/grpc/ServingGrpcServiceTest.java | 149 +----- .../BigTableFeatureStorageTestITCase.java | 224 +-------- .../FeatureRetrievalDispatcherTest.java | 142 +----- .../service/RedisFeatureStorageTest.java | 349 +------------- .../serving/testutil/BigTablePopulator.java | 141 ++---- .../testutil/FeatureStoragePopulator.java | 120 +---- .../serving/testutil/RedisPopulator.java | 76 +-- .../java/feast/serving/testutil/TimeUtil.java | 60 +++ .../serving/util/EntityMapBuilderTest.java | 103 ++-- .../serving/util/FeatureBuilderTest.java | 187 -------- 47 files changed, 1062 insertions(+), 3598 deletions(-) delete mode 100644 serving/src/main/java/feast/serving/model/Pair.java delete mode 100644 serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java delete mode 100644 serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java create mode 100644 serving/src/test/java/feast/serving/testutil/TimeUtil.java delete mode 100644 serving/src/test/java/feast/serving/util/FeatureBuilderTest.java diff --git a/protos/feast/serving/Serving.proto b/protos/feast/serving/Serving.proto index cdaa9d76b9b..cbb1d38657e 100644 --- a/protos/feast/serving/Serving.proto +++ b/protos/feast/serving/Serving.proto @@ -26,73 +26,37 @@ option java_outer_classname = "ServingAPIProto"; option go_package = "github.com/gojek/feast/protos/generated/go/feast/serving"; service ServingAPI { - // Query features from Feast - rpc QueryFeatures (QueryFeatures.Request) returns (QueryFeatures.Response) {}; + // Query features from Feast serving storage + rpc QueryFeatures (QueryFeaturesRequest) returns (QueryFeaturesResponse) {}; } -message QueryFeatures { - message Request { - // e.g. "driver", "customer", "city". - string entityName = 1; - // List of entity ID. - repeated string entityId = 2; - // List of request details, contains: featureId, type of query, and limit. - repeated RequestDetail requestDetails = 3; - // Filter specifying only to retrieve features having timestamp within this range. - TimestampRange timestampRange = 4; - } - - message Response { - // e.g. "driver", "customer", "city". - string entityName = 1; - // map of entity ID and its entity's properties. - map entities = 2; - } -} - -message RequestDetail { - // feature ID to be included in the query. +message QueryFeaturesRequest { + // e.g. "driver", "customer", "city". + string entityName = 1; + // List of entity ID. + repeated string entityId = 2; + // List of feature ID. // feature ID is in the form of [entity_name].[granularity].[feature_name] // e.g: "driver.day.total_accepted_booking" // all requested feature ID shall have same entity name. - string featureId = 1; - // request type either LAST or LIST. - // LAST : return only the latest value based on timestamp. - // LIST : return list of historical data sorted by timestamp. - RequestType type = 2; - // only applicable to LIST. - // length of the returned list <= limit. - // default = 0 - int32 limit = 3; -} - -enum RequestType { - // LAST : return only the latest value based on timestamp. (default) - LAST = 0; - // LIST : return list of historical data sorted by timestamp. - LIST = 1; + repeated string featureId = 3; } -// range of timestamp for querying -// valid timestamp range is having start <= end -message TimestampRange { - // start time of the range query. - google.protobuf.Timestamp start = 1; - // end time of the range query. - google.protobuf.Timestamp end = 2; +message QueryFeaturesResponse { + // Entity name of the response + string entityName = 1; + // map of entity ID and its entity's properties. + map entities = 2; } message Entity { // map of feature ID and its feature value. - map features = 1; + map features = 1; } -message FeatureValueList { - // list of feature value - // if "type" in "requestDetail" is "LAST", then the length will always be 1. - // if "type" in "requestDetail" is "LIST", then the length is <= "limit". - feast.types.ValueList valueList = 1; - // list of timestamp of the value. - // the i-th timestamps correspond to the i-th value. - feast.types.TimestampList timestampList = 2; +message FeatureValue { + // value of feature + feast.types.Value value = 1; + // timestamp of the feature + google.protobuf.Timestamp timestamp = 2; } diff --git a/protos/generated/go/feast/core/CoreService.pb.go b/protos/generated/go/feast/core/CoreService.pb.go index 4037da58af9..d8224c13a9b 100644 --- a/protos/generated/go/feast/core/CoreService.pb.go +++ b/protos/generated/go/feast/core/CoreService.pb.go @@ -35,7 +35,7 @@ func (m *CoreServiceTypes) Reset() { *m = CoreServiceTypes{} } func (m *CoreServiceTypes) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes) ProtoMessage() {} func (*CoreServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0} + return fileDescriptor_d9be266444105411, []int{0} } func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes.Unmarshal(m, b) @@ -43,8 +43,8 @@ func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes.Merge(dst, src) +func (m *CoreServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes.Merge(m, src) } func (m *CoreServiceTypes) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes.Size(m) @@ -66,7 +66,7 @@ func (m *CoreServiceTypes_GetEntitiesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetEntitiesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 0} + return fileDescriptor_d9be266444105411, []int{0, 0} } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Unmarshal(m, b) @@ -74,8 +74,8 @@ func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(m, src) } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Size(m) @@ -104,7 +104,7 @@ func (m *CoreServiceTypes_GetEntitiesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 1} + return fileDescriptor_d9be266444105411, []int{0, 1} } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Unmarshal(m, b) @@ -112,8 +112,8 @@ func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(m, src) } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Size(m) @@ -142,7 +142,7 @@ func (m *CoreServiceTypes_ListEntitiesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 2} + return fileDescriptor_d9be266444105411, []int{0, 2} } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -150,8 +150,8 @@ func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(m, src) } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Size(m) @@ -181,7 +181,7 @@ func (m *CoreServiceTypes_GetFeaturesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 3} + return fileDescriptor_d9be266444105411, []int{0, 3} } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Unmarshal(m, b) @@ -189,8 +189,8 @@ func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(m, src) } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Size(m) @@ -219,7 +219,7 @@ func (m *CoreServiceTypes_GetFeaturesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 4} + return fileDescriptor_d9be266444105411, []int{0, 4} } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Unmarshal(m, b) @@ -227,8 +227,8 @@ func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(m, src) } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Size(m) @@ -257,7 +257,7 @@ func (m *CoreServiceTypes_ListFeaturesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 5} + return fileDescriptor_d9be266444105411, []int{0, 5} } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -265,8 +265,8 @@ func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(m, src) } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Size(m) @@ -296,7 +296,7 @@ func (m *CoreServiceTypes_GetStorageRequest) Reset() { *m = CoreServiceT func (m *CoreServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageRequest) ProtoMessage() {} func (*CoreServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 6} + return fileDescriptor_d9be266444105411, []int{0, 6} } func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -304,8 +304,8 @@ func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(m, src) } func (m *CoreServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Size(m) @@ -334,7 +334,7 @@ func (m *CoreServiceTypes_GetStorageResponse) Reset() { *m = CoreService func (m *CoreServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 7} + return fileDescriptor_d9be266444105411, []int{0, 7} } func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -342,8 +342,8 @@ func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Size(m) @@ -372,7 +372,7 @@ func (m *CoreServiceTypes_ListStorageResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 8} + return fileDescriptor_d9be266444105411, []int{0, 8} } func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -380,8 +380,8 @@ func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Size(m) @@ -411,7 +411,7 @@ func (m *CoreServiceTypes_ApplyEntityResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ApplyEntityResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyEntityResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 9} + return fileDescriptor_d9be266444105411, []int{0, 9} } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Unmarshal(m, b) @@ -419,8 +419,8 @@ func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Size(m) @@ -450,7 +450,7 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyFeatureResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyFeatureResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 10} + return fileDescriptor_d9be266444105411, []int{0, 10} } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Unmarshal(m, b) @@ -458,8 +458,8 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Size(m) @@ -493,7 +493,7 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) String() string { } func (*CoreServiceTypes_ApplyFeatureGroupResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 11} + return fileDescriptor_d9be266444105411, []int{0, 11} } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Unmarshal(m, b) @@ -501,8 +501,8 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) err func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Size(m) @@ -532,7 +532,7 @@ func (m *CoreServiceTypes_ApplyStorageResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 12} + return fileDescriptor_d9be266444105411, []int{0, 12} } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Unmarshal(m, b) @@ -540,8 +540,8 @@ func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Size(m) @@ -1005,11 +1005,9 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/CoreService.proto", } -func init() { - proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_a0e9a1504f969203) -} +func init() { proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) } -var fileDescriptor_CoreService_a0e9a1504f969203 = []byte{ +var fileDescriptor_d9be266444105411 = []byte{ // 602 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5b, 0x6f, 0x12, 0x41, 0x14, 0x86, 0x34, 0x69, 0xe0, 0x40, 0x4c, 0x3b, 0x34, 0x8a, 0x23, 0x35, 0xcd, 0x26, 0x36, 0x7d, diff --git a/protos/generated/go/feast/core/DatasetService.pb.go b/protos/generated/go/feast/core/DatasetService.pb.go index d2d3b01c932..d83ec1a1dc5 100644 --- a/protos/generated/go/feast/core/DatasetService.pb.go +++ b/protos/generated/go/feast/core/DatasetService.pb.go @@ -34,7 +34,7 @@ func (m *DatasetServiceTypes) Reset() { *m = DatasetServiceTypes{} } func (m *DatasetServiceTypes) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes) ProtoMessage() {} func (*DatasetServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0} + return fileDescriptor_3edc37a8b0d37b39, []int{0} } func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes.Unmarshal(m, b) @@ -42,8 +42,8 @@ func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { func (m *DatasetServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes.Merge(dst, src) +func (m *DatasetServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes.Merge(m, src) } func (m *DatasetServiceTypes) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes.Size(m) @@ -77,7 +77,7 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) Reset() { func (m *DatasetServiceTypes_CreateDatasetRequest) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetRequest) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 0} + return fileDescriptor_3edc37a8b0d37b39, []int{0, 0} } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Unmarshal(m, b) @@ -85,8 +85,8 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(dst, src) +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(m, src) } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Size(m) @@ -146,7 +146,7 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) Reset() { func (m *DatasetServiceTypes_CreateDatasetResponse) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetResponse) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 1} + return fileDescriptor_3edc37a8b0d37b39, []int{0, 1} } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Unmarshal(m, b) @@ -154,8 +154,8 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) erro func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(dst, src) +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(m, src) } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Size(m) @@ -188,7 +188,7 @@ func (m *FeatureSet) Reset() { *m = FeatureSet{} } func (m *FeatureSet) String() string { return proto.CompactTextString(m) } func (*FeatureSet) ProtoMessage() {} func (*FeatureSet) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{1} + return fileDescriptor_3edc37a8b0d37b39, []int{1} } func (m *FeatureSet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSet.Unmarshal(m, b) @@ -196,8 +196,8 @@ func (m *FeatureSet) XXX_Unmarshal(b []byte) error { func (m *FeatureSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSet.Marshal(b, m, deterministic) } -func (dst *FeatureSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSet.Merge(dst, src) +func (m *FeatureSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSet.Merge(m, src) } func (m *FeatureSet) XXX_Size() int { return xxx_messageInfo_FeatureSet.Size(m) @@ -237,7 +237,7 @@ func (m *DatasetInfo) Reset() { *m = DatasetInfo{} } func (m *DatasetInfo) String() string { return proto.CompactTextString(m) } func (*DatasetInfo) ProtoMessage() {} func (*DatasetInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{2} + return fileDescriptor_3edc37a8b0d37b39, []int{2} } func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetInfo.Unmarshal(m, b) @@ -245,8 +245,8 @@ func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { func (m *DatasetInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetInfo.Marshal(b, m, deterministic) } -func (dst *DatasetInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetInfo.Merge(dst, src) +func (m *DatasetInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetInfo.Merge(m, src) } func (m *DatasetInfo) XXX_Size() int { return xxx_messageInfo_DatasetInfo.Size(m) @@ -353,11 +353,9 @@ var _DatasetService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/DatasetService.proto", } -func init() { - proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_37ae639a8c7b5dd5) -} +func init() { proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_3edc37a8b0d37b39) } -var fileDescriptor_DatasetService_37ae639a8c7b5dd5 = []byte{ +var fileDescriptor_3edc37a8b0d37b39 = []byte{ // 414 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, diff --git a/protos/generated/go/feast/core/JobService.pb.go b/protos/generated/go/feast/core/JobService.pb.go index 6a3a4e6dc0d..699d2b4d328 100644 --- a/protos/generated/go/feast/core/JobService.pb.go +++ b/protos/generated/go/feast/core/JobService.pb.go @@ -36,7 +36,7 @@ func (m *JobServiceTypes) Reset() { *m = JobServiceTypes{} } func (m *JobServiceTypes) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes) ProtoMessage() {} func (*JobServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0} + return fileDescriptor_7115affcfc7885c4, []int{0} } func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes.Merge(dst, src) +func (m *JobServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes.Merge(m, src) } func (m *JobServiceTypes) XXX_Size() int { return xxx_messageInfo_JobServiceTypes.Size(m) @@ -70,7 +70,7 @@ func (m *JobServiceTypes_SubmitImportJobRequest) Reset() { func (m *JobServiceTypes_SubmitImportJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobRequest) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 0} + return fileDescriptor_7115affcfc7885c4, []int{0, 0} } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Unmarshal(m, b) @@ -78,8 +78,8 @@ func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(dst, src) +func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(m, src) } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Size(m) @@ -117,7 +117,7 @@ func (m *JobServiceTypes_SubmitImportJobResponse) Reset() { func (m *JobServiceTypes_SubmitImportJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobResponse) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 1} + return fileDescriptor_7115affcfc7885c4, []int{0, 1} } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Unmarshal(m, b) @@ -125,8 +125,8 @@ func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(dst, src) +func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(m, src) } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Size(m) @@ -155,7 +155,7 @@ func (m *JobServiceTypes_ListJobsResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_ListJobsResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_ListJobsResponse) ProtoMessage() {} func (*JobServiceTypes_ListJobsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 2} + return fileDescriptor_7115affcfc7885c4, []int{0, 2} } func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Unmarshal(m, b) @@ -163,8 +163,8 @@ func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_ListJobsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(dst, src) +func (m *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(m, src) } func (m *JobServiceTypes_ListJobsResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Size(m) @@ -193,7 +193,7 @@ func (m *JobServiceTypes_GetJobRequest) Reset() { *m = JobServiceTypes_G func (m *JobServiceTypes_GetJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobRequest) ProtoMessage() {} func (*JobServiceTypes_GetJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 3} + return fileDescriptor_7115affcfc7885c4, []int{0, 3} } func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Unmarshal(m, b) @@ -201,8 +201,8 @@ func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_GetJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(dst, src) +func (m *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(m, src) } func (m *JobServiceTypes_GetJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Size(m) @@ -231,7 +231,7 @@ func (m *JobServiceTypes_GetJobResponse) Reset() { *m = JobServiceTypes_ func (m *JobServiceTypes_GetJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobResponse) ProtoMessage() {} func (*JobServiceTypes_GetJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 4} + return fileDescriptor_7115affcfc7885c4, []int{0, 4} } func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Unmarshal(m, b) @@ -239,8 +239,8 @@ func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_GetJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(dst, src) +func (m *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(m, src) } func (m *JobServiceTypes_GetJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Size(m) @@ -269,7 +269,7 @@ func (m *JobServiceTypes_AbortJobRequest) Reset() { *m = JobServiceTypes func (m *JobServiceTypes_AbortJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobRequest) ProtoMessage() {} func (*JobServiceTypes_AbortJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 5} + return fileDescriptor_7115affcfc7885c4, []int{0, 5} } func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Unmarshal(m, b) @@ -277,8 +277,8 @@ func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_AbortJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(dst, src) +func (m *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(m, src) } func (m *JobServiceTypes_AbortJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Size(m) @@ -307,7 +307,7 @@ func (m *JobServiceTypes_AbortJobResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_AbortJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobResponse) ProtoMessage() {} func (*JobServiceTypes_AbortJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 6} + return fileDescriptor_7115affcfc7885c4, []int{0, 6} } func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Unmarshal(m, b) @@ -315,8 +315,8 @@ func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_AbortJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(dst, src) +func (m *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(m, src) } func (m *JobServiceTypes_AbortJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Size(m) @@ -356,7 +356,7 @@ func (m *JobServiceTypes_JobDetail) Reset() { *m = JobServiceTypes_JobDe func (m *JobServiceTypes_JobDetail) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_JobDetail) ProtoMessage() {} func (*JobServiceTypes_JobDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 7} + return fileDescriptor_7115affcfc7885c4, []int{0, 7} } func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_JobDetail.Unmarshal(m, b) @@ -364,8 +364,8 @@ func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_JobDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_JobDetail.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_JobDetail.Merge(dst, src) +func (m *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_JobDetail.Merge(m, src) } func (m *JobServiceTypes_JobDetail) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_JobDetail.Size(m) @@ -638,11 +638,9 @@ var _JobService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/JobService.proto", } -func init() { - proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_JobService_edcd183b773c9f62) -} +func init() { proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_7115affcfc7885c4) } -var fileDescriptor_JobService_edcd183b773c9f62 = []byte{ +var fileDescriptor_7115affcfc7885c4 = []byte{ // 621 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0xdb, 0x4c, 0x10, 0x55, 0x62, 0x08, 0xf1, 0xf0, 0x7d, 0x80, 0x56, 0x15, 0x58, 0x4b, 0x25, 0x52, 0xa4, 0x4a, diff --git a/protos/generated/go/feast/core/UIService.pb.go b/protos/generated/go/feast/core/UIService.pb.go index 772c548c875..87adc7210e9 100644 --- a/protos/generated/go/feast/core/UIService.pb.go +++ b/protos/generated/go/feast/core/UIService.pb.go @@ -36,7 +36,7 @@ func (m *UIServiceTypes) Reset() { *m = UIServiceTypes{} } func (m *UIServiceTypes) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes) ProtoMessage() {} func (*UIServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0} + return fileDescriptor_c13b3edb35d457e8, []int{0} } func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes.Merge(dst, src) +func (m *UIServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes.Merge(m, src) } func (m *UIServiceTypes) XXX_Size() int { return xxx_messageInfo_UIServiceTypes.Size(m) @@ -70,7 +70,7 @@ func (m *UIServiceTypes_EntityDetail) Reset() { *m = UIServiceTypes_Enti func (m *UIServiceTypes_EntityDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_EntityDetail) ProtoMessage() {} func (*UIServiceTypes_EntityDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 0} + return fileDescriptor_c13b3edb35d457e8, []int{0, 0} } func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_EntityDetail.Unmarshal(m, b) @@ -78,8 +78,8 @@ func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_EntityDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_EntityDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(dst, src) +func (m *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(m, src) } func (m *UIServiceTypes_EntityDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_EntityDetail.Size(m) @@ -122,7 +122,7 @@ func (m *UIServiceTypes_GetEntityRequest) Reset() { *m = UIServiceTypes_ func (m *UIServiceTypes_GetEntityRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityRequest) ProtoMessage() {} func (*UIServiceTypes_GetEntityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 1} + return fileDescriptor_c13b3edb35d457e8, []int{0, 1} } func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Unmarshal(m, b) @@ -130,8 +130,8 @@ func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetEntityRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(dst, src) +func (m *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(m, src) } func (m *UIServiceTypes_GetEntityRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Size(m) @@ -160,7 +160,7 @@ func (m *UIServiceTypes_GetEntityResponse) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetEntityResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityResponse) ProtoMessage() {} func (*UIServiceTypes_GetEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 2} + return fileDescriptor_c13b3edb35d457e8, []int{0, 2} } func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Unmarshal(m, b) @@ -168,8 +168,8 @@ func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(dst, src) +func (m *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(m, src) } func (m *UIServiceTypes_GetEntityResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Size(m) @@ -198,7 +198,7 @@ func (m *UIServiceTypes_ListEntitiesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*UIServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 3} + return fileDescriptor_c13b3edb35d457e8, []int{0, 3} } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -206,8 +206,8 @@ func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(dst, src) +func (m *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(m, src) } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Size(m) @@ -242,7 +242,7 @@ func (m *UIServiceTypes_FeatureDetail) Reset() { *m = UIServiceTypes_Fea func (m *UIServiceTypes_FeatureDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 4} + return fileDescriptor_c13b3edb35d457e8, []int{0, 4} } func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Unmarshal(m, b) @@ -250,8 +250,8 @@ func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_FeatureDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(dst, src) +func (m *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(m, src) } func (m *UIServiceTypes_FeatureDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Size(m) @@ -315,7 +315,7 @@ func (m *UIServiceTypes_GetFeatureRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetFeatureRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 5} + return fileDescriptor_c13b3edb35d457e8, []int{0, 5} } func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Unmarshal(m, b) @@ -323,8 +323,8 @@ func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(m, src) } func (m *UIServiceTypes_GetFeatureRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Size(m) @@ -354,7 +354,7 @@ func (m *UIServiceTypes_GetFeatureResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetFeatureResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 6} + return fileDescriptor_c13b3edb35d457e8, []int{0, 6} } func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Unmarshal(m, b) @@ -362,8 +362,8 @@ func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(m, src) } func (m *UIServiceTypes_GetFeatureResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Size(m) @@ -399,7 +399,7 @@ func (m *UIServiceTypes_ListFeaturesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 7} + return fileDescriptor_c13b3edb35d457e8, []int{0, 7} } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -407,8 +407,8 @@ func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(dst, src) +func (m *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(m, src) } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Size(m) @@ -439,7 +439,7 @@ func (m *UIServiceTypes_FeatureGroupDetail) Reset() { *m = UIServiceType func (m *UIServiceTypes_FeatureGroupDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureGroupDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureGroupDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 8} + return fileDescriptor_c13b3edb35d457e8, []int{0, 8} } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Unmarshal(m, b) @@ -447,8 +447,8 @@ func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_FeatureGroupDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(dst, src) +func (m *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(m, src) } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Size(m) @@ -484,7 +484,7 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) Reset() { *m = UIService func (m *UIServiceTypes_GetFeatureGroupRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 9} + return fileDescriptor_c13b3edb35d457e8, []int{0, 9} } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Unmarshal(m, b) @@ -492,8 +492,8 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(m, src) } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Size(m) @@ -524,7 +524,7 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) Reset() { func (m *UIServiceTypes_GetFeatureGroupResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 10} + return fileDescriptor_c13b3edb35d457e8, []int{0, 10} } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Unmarshal(m, b) @@ -532,8 +532,8 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(m, src) } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Size(m) @@ -564,7 +564,7 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) Reset() { func (m *UIServiceTypes_ListFeatureGroupsResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeatureGroupsResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeatureGroupsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 11} + return fileDescriptor_c13b3edb35d457e8, []int{0, 11} } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Unmarshal(m, b) @@ -572,8 +572,8 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(dst, src) +func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(m, src) } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Size(m) @@ -604,7 +604,7 @@ func (m *UIServiceTypes_StorageDetail) Reset() { *m = UIServiceTypes_Sto func (m *UIServiceTypes_StorageDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_StorageDetail) ProtoMessage() {} func (*UIServiceTypes_StorageDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 12} + return fileDescriptor_c13b3edb35d457e8, []int{0, 12} } func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_StorageDetail.Unmarshal(m, b) @@ -612,8 +612,8 @@ func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_StorageDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_StorageDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(dst, src) +func (m *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(m, src) } func (m *UIServiceTypes_StorageDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_StorageDetail.Size(m) @@ -649,7 +649,7 @@ func (m *UIServiceTypes_GetStorageRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageRequest) ProtoMessage() {} func (*UIServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 13} + return fileDescriptor_c13b3edb35d457e8, []int{0, 13} } func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -657,8 +657,8 @@ func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(dst, src) +func (m *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(m, src) } func (m *UIServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Size(m) @@ -687,7 +687,7 @@ func (m *UIServiceTypes_GetStorageResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageResponse) ProtoMessage() {} func (*UIServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 14} + return fileDescriptor_c13b3edb35d457e8, []int{0, 14} } func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -695,8 +695,8 @@ func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(dst, src) +func (m *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(m, src) } func (m *UIServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Size(m) @@ -725,7 +725,7 @@ func (m *UIServiceTypes_ListStorageResponse) Reset() { *m = UIServiceTyp func (m *UIServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListStorageResponse) ProtoMessage() {} func (*UIServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 15} + return fileDescriptor_c13b3edb35d457e8, []int{0, 15} } func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -733,8 +733,8 @@ func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(dst, src) +func (m *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(m, src) } func (m *UIServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Size(m) @@ -1119,11 +1119,9 @@ var _UIService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/UIService.proto", } -func init() { - proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_UIService_04866529701c634c) -} +func init() { proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_c13b3edb35d457e8) } -var fileDescriptor_UIService_04866529701c634c = []byte{ +var fileDescriptor_c13b3edb35d457e8 = []byte{ // 784 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xd3, 0x40, 0x10, 0xcd, 0x47, 0x49, 0x9a, 0x69, 0x5a, 0xe8, 0x82, 0xda, 0xb0, 0x50, 0x51, 0x99, 0x03, 0x91, diff --git a/protos/generated/go/feast/serving/Serving.pb.go b/protos/generated/go/feast/serving/Serving.pb.go index 30048a75bcc..64979c154ad 100644 --- a/protos/generated/go/feast/serving/Serving.pb.go +++ b/protos/generated/go/feast/serving/Serving.pb.go @@ -25,129 +25,68 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type RequestType int32 - -const ( - // LAST : return only the latest value based on timestamp. (default) - RequestType_LAST RequestType = 0 - // LIST : return list of historical data sorted by timestamp. - RequestType_LIST RequestType = 1 -) - -var RequestType_name = map[int32]string{ - 0: "LAST", - 1: "LIST", -} -var RequestType_value = map[string]int32{ - "LAST": 0, - "LIST": 1, -} - -func (x RequestType) String() string { - return proto.EnumName(RequestType_name, int32(x)) -} -func (RequestType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0} -} - -type QueryFeatures struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *QueryFeatures) Reset() { *m = QueryFeatures{} } -func (m *QueryFeatures) String() string { return proto.CompactTextString(m) } -func (*QueryFeatures) ProtoMessage() {} -func (*QueryFeatures) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0} -} -func (m *QueryFeatures) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryFeatures.Unmarshal(m, b) -} -func (m *QueryFeatures) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryFeatures.Marshal(b, m, deterministic) -} -func (dst *QueryFeatures) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeatures.Merge(dst, src) -} -func (m *QueryFeatures) XXX_Size() int { - return xxx_messageInfo_QueryFeatures.Size(m) -} -func (m *QueryFeatures) XXX_DiscardUnknown() { - xxx_messageInfo_QueryFeatures.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryFeatures proto.InternalMessageInfo - -type QueryFeatures_Request struct { +type QueryFeaturesRequest struct { // e.g. "driver", "customer", "city". EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` // List of entity ID. EntityId []string `protobuf:"bytes,2,rep,name=entityId,proto3" json:"entityId,omitempty"` - // List of request details, contains: featureId, type of query, and limit. - RequestDetails []*RequestDetail `protobuf:"bytes,3,rep,name=requestDetails,proto3" json:"requestDetails,omitempty"` - // Filter specifying only to retrieve features having timestamp within this range. - TimestampRange *TimestampRange `protobuf:"bytes,4,opt,name=timestampRange,proto3" json:"timestampRange,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // List of feature ID. + // feature ID is in the form of [entity_name].[granularity].[feature_name] + // e.g: "driver.day.total_accepted_booking" + // all requested feature ID shall have same entity name. + FeatureId []string `protobuf:"bytes,3,rep,name=featureId,proto3" json:"featureId,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *QueryFeatures_Request) Reset() { *m = QueryFeatures_Request{} } -func (m *QueryFeatures_Request) String() string { return proto.CompactTextString(m) } -func (*QueryFeatures_Request) ProtoMessage() {} -func (*QueryFeatures_Request) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0, 0} +func (m *QueryFeaturesRequest) Reset() { *m = QueryFeaturesRequest{} } +func (m *QueryFeaturesRequest) String() string { return proto.CompactTextString(m) } +func (*QueryFeaturesRequest) ProtoMessage() {} +func (*QueryFeaturesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_7609de6de542e6f0, []int{0} } -func (m *QueryFeatures_Request) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryFeatures_Request.Unmarshal(m, b) +func (m *QueryFeaturesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryFeaturesRequest.Unmarshal(m, b) } -func (m *QueryFeatures_Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryFeatures_Request.Marshal(b, m, deterministic) +func (m *QueryFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryFeaturesRequest.Marshal(b, m, deterministic) } -func (dst *QueryFeatures_Request) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeatures_Request.Merge(dst, src) +func (m *QueryFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesRequest.Merge(m, src) } -func (m *QueryFeatures_Request) XXX_Size() int { - return xxx_messageInfo_QueryFeatures_Request.Size(m) +func (m *QueryFeaturesRequest) XXX_Size() int { + return xxx_messageInfo_QueryFeaturesRequest.Size(m) } -func (m *QueryFeatures_Request) XXX_DiscardUnknown() { - xxx_messageInfo_QueryFeatures_Request.DiscardUnknown(m) +func (m *QueryFeaturesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryFeaturesRequest.DiscardUnknown(m) } -var xxx_messageInfo_QueryFeatures_Request proto.InternalMessageInfo +var xxx_messageInfo_QueryFeaturesRequest proto.InternalMessageInfo -func (m *QueryFeatures_Request) GetEntityName() string { +func (m *QueryFeaturesRequest) GetEntityName() string { if m != nil { return m.EntityName } return "" } -func (m *QueryFeatures_Request) GetEntityId() []string { +func (m *QueryFeaturesRequest) GetEntityId() []string { if m != nil { return m.EntityId } return nil } -func (m *QueryFeatures_Request) GetRequestDetails() []*RequestDetail { +func (m *QueryFeaturesRequest) GetFeatureId() []string { if m != nil { - return m.RequestDetails - } - return nil -} - -func (m *QueryFeatures_Request) GetTimestampRange() *TimestampRange { - if m != nil { - return m.TimestampRange + return m.FeatureId } return nil } -type QueryFeatures_Response struct { - // e.g. "driver", "customer", "city". +type QueryFeaturesResponse struct { + // Entity name of the response EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` // map of entity ID and its entity's properties. Entities map[string]*Entity `protobuf:"bytes,2,rep,name=entities,proto3" json:"entities,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -156,171 +95,57 @@ type QueryFeatures_Response struct { XXX_sizecache int32 `json:"-"` } -func (m *QueryFeatures_Response) Reset() { *m = QueryFeatures_Response{} } -func (m *QueryFeatures_Response) String() string { return proto.CompactTextString(m) } -func (*QueryFeatures_Response) ProtoMessage() {} -func (*QueryFeatures_Response) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0, 1} +func (m *QueryFeaturesResponse) Reset() { *m = QueryFeaturesResponse{} } +func (m *QueryFeaturesResponse) String() string { return proto.CompactTextString(m) } +func (*QueryFeaturesResponse) ProtoMessage() {} +func (*QueryFeaturesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_7609de6de542e6f0, []int{1} } -func (m *QueryFeatures_Response) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryFeatures_Response.Unmarshal(m, b) +func (m *QueryFeaturesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryFeaturesResponse.Unmarshal(m, b) } -func (m *QueryFeatures_Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryFeatures_Response.Marshal(b, m, deterministic) +func (m *QueryFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *QueryFeatures_Response) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeatures_Response.Merge(dst, src) +func (m *QueryFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesResponse.Merge(m, src) } -func (m *QueryFeatures_Response) XXX_Size() int { - return xxx_messageInfo_QueryFeatures_Response.Size(m) +func (m *QueryFeaturesResponse) XXX_Size() int { + return xxx_messageInfo_QueryFeaturesResponse.Size(m) } -func (m *QueryFeatures_Response) XXX_DiscardUnknown() { - xxx_messageInfo_QueryFeatures_Response.DiscardUnknown(m) +func (m *QueryFeaturesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_QueryFeaturesResponse.DiscardUnknown(m) } -var xxx_messageInfo_QueryFeatures_Response proto.InternalMessageInfo +var xxx_messageInfo_QueryFeaturesResponse proto.InternalMessageInfo -func (m *QueryFeatures_Response) GetEntityName() string { +func (m *QueryFeaturesResponse) GetEntityName() string { if m != nil { return m.EntityName } return "" } -func (m *QueryFeatures_Response) GetEntities() map[string]*Entity { +func (m *QueryFeaturesResponse) GetEntities() map[string]*Entity { if m != nil { return m.Entities } return nil } -type RequestDetail struct { - // feature ID to be included in the query. - // feature ID is in the form of [entity_name].[granularity].[feature_name] - // e.g: "driver.day.total_accepted_booking" - // all requested feature ID shall have same entity name. - FeatureId string `protobuf:"bytes,1,opt,name=featureId,proto3" json:"featureId,omitempty"` - // request type either LAST or LIST. - // LAST : return only the latest value based on timestamp. - // LIST : return list of historical data sorted by timestamp. - Type RequestType `protobuf:"varint,2,opt,name=type,proto3,enum=feast.serving.RequestType" json:"type,omitempty"` - // only applicable to LIST. - // length of the returned list <= limit. - // default = 0 - Limit int32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RequestDetail) Reset() { *m = RequestDetail{} } -func (m *RequestDetail) String() string { return proto.CompactTextString(m) } -func (*RequestDetail) ProtoMessage() {} -func (*RequestDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{1} -} -func (m *RequestDetail) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RequestDetail.Unmarshal(m, b) -} -func (m *RequestDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RequestDetail.Marshal(b, m, deterministic) -} -func (dst *RequestDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_RequestDetail.Merge(dst, src) -} -func (m *RequestDetail) XXX_Size() int { - return xxx_messageInfo_RequestDetail.Size(m) -} -func (m *RequestDetail) XXX_DiscardUnknown() { - xxx_messageInfo_RequestDetail.DiscardUnknown(m) -} - -var xxx_messageInfo_RequestDetail proto.InternalMessageInfo - -func (m *RequestDetail) GetFeatureId() string { - if m != nil { - return m.FeatureId - } - return "" -} - -func (m *RequestDetail) GetType() RequestType { - if m != nil { - return m.Type - } - return RequestType_LAST -} - -func (m *RequestDetail) GetLimit() int32 { - if m != nil { - return m.Limit - } - return 0 -} - -// range of timestamp for querying -// valid timestamp range is having start <= end -type TimestampRange struct { - // start time of the range query. - Start *timestamp.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` - // end time of the range query. - End *timestamp.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *TimestampRange) Reset() { *m = TimestampRange{} } -func (m *TimestampRange) String() string { return proto.CompactTextString(m) } -func (*TimestampRange) ProtoMessage() {} -func (*TimestampRange) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{2} -} -func (m *TimestampRange) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_TimestampRange.Unmarshal(m, b) -} -func (m *TimestampRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_TimestampRange.Marshal(b, m, deterministic) -} -func (dst *TimestampRange) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampRange.Merge(dst, src) -} -func (m *TimestampRange) XXX_Size() int { - return xxx_messageInfo_TimestampRange.Size(m) -} -func (m *TimestampRange) XXX_DiscardUnknown() { - xxx_messageInfo_TimestampRange.DiscardUnknown(m) -} - -var xxx_messageInfo_TimestampRange proto.InternalMessageInfo - -func (m *TimestampRange) GetStart() *timestamp.Timestamp { - if m != nil { - return m.Start - } - return nil -} - -func (m *TimestampRange) GetEnd() *timestamp.Timestamp { - if m != nil { - return m.End - } - return nil -} - type Entity struct { // map of feature ID and its feature value. - Features map[string]*FeatureValueList `protobuf:"bytes,1,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Features map[string]*FeatureValue `protobuf:"bytes,1,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Entity) Reset() { *m = Entity{} } func (m *Entity) String() string { return proto.CompactTextString(m) } func (*Entity) ProtoMessage() {} func (*Entity) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{3} + return fileDescriptor_7609de6de542e6f0, []int{2} } func (m *Entity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Entity.Unmarshal(m, b) @@ -328,8 +153,8 @@ func (m *Entity) XXX_Unmarshal(b []byte) error { func (m *Entity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Entity.Marshal(b, m, deterministic) } -func (dst *Entity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Entity.Merge(dst, src) +func (m *Entity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Entity.Merge(m, src) } func (m *Entity) XXX_Size() int { return xxx_messageInfo_Entity.Size(m) @@ -340,75 +165,68 @@ func (m *Entity) XXX_DiscardUnknown() { var xxx_messageInfo_Entity proto.InternalMessageInfo -func (m *Entity) GetFeatures() map[string]*FeatureValueList { +func (m *Entity) GetFeatures() map[string]*FeatureValue { if m != nil { return m.Features } return nil } -type FeatureValueList struct { - // list of feature value - // if "type" in "requestDetail" is "LAST", then the length will always be 1. - // if "type" in "requestDetail" is "LIST", then the length is <= "limit". - ValueList *types.ValueList `protobuf:"bytes,1,opt,name=valueList,proto3" json:"valueList,omitempty"` - // list of timestamp of the value. - // the i-th timestamps correspond to the i-th value. - TimestampList *types.TimestampList `protobuf:"bytes,2,opt,name=timestampList,proto3" json:"timestampList,omitempty"` +type FeatureValue struct { + // value of feature + Value *types.Value `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + // timestamp of the feature + Timestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *FeatureValueList) Reset() { *m = FeatureValueList{} } -func (m *FeatureValueList) String() string { return proto.CompactTextString(m) } -func (*FeatureValueList) ProtoMessage() {} -func (*FeatureValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{4} +func (m *FeatureValue) Reset() { *m = FeatureValue{} } +func (m *FeatureValue) String() string { return proto.CompactTextString(m) } +func (*FeatureValue) ProtoMessage() {} +func (*FeatureValue) Descriptor() ([]byte, []int) { + return fileDescriptor_7609de6de542e6f0, []int{3} } -func (m *FeatureValueList) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_FeatureValueList.Unmarshal(m, b) +func (m *FeatureValue) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FeatureValue.Unmarshal(m, b) } -func (m *FeatureValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_FeatureValueList.Marshal(b, m, deterministic) +func (m *FeatureValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FeatureValue.Marshal(b, m, deterministic) } -func (dst *FeatureValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureValueList.Merge(dst, src) +func (m *FeatureValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureValue.Merge(m, src) } -func (m *FeatureValueList) XXX_Size() int { - return xxx_messageInfo_FeatureValueList.Size(m) +func (m *FeatureValue) XXX_Size() int { + return xxx_messageInfo_FeatureValue.Size(m) } -func (m *FeatureValueList) XXX_DiscardUnknown() { - xxx_messageInfo_FeatureValueList.DiscardUnknown(m) +func (m *FeatureValue) XXX_DiscardUnknown() { + xxx_messageInfo_FeatureValue.DiscardUnknown(m) } -var xxx_messageInfo_FeatureValueList proto.InternalMessageInfo +var xxx_messageInfo_FeatureValue proto.InternalMessageInfo -func (m *FeatureValueList) GetValueList() *types.ValueList { +func (m *FeatureValue) GetValue() *types.Value { if m != nil { - return m.ValueList + return m.Value } return nil } -func (m *FeatureValueList) GetTimestampList() *types.TimestampList { +func (m *FeatureValue) GetTimestamp() *timestamp.Timestamp { if m != nil { - return m.TimestampList + return m.Timestamp } return nil } func init() { - proto.RegisterType((*QueryFeatures)(nil), "feast.serving.QueryFeatures") - proto.RegisterType((*QueryFeatures_Request)(nil), "feast.serving.QueryFeatures.Request") - proto.RegisterType((*QueryFeatures_Response)(nil), "feast.serving.QueryFeatures.Response") - proto.RegisterMapType((map[string]*Entity)(nil), "feast.serving.QueryFeatures.Response.EntitiesEntry") - proto.RegisterType((*RequestDetail)(nil), "feast.serving.RequestDetail") - proto.RegisterType((*TimestampRange)(nil), "feast.serving.TimestampRange") + proto.RegisterType((*QueryFeaturesRequest)(nil), "feast.serving.QueryFeaturesRequest") + proto.RegisterType((*QueryFeaturesResponse)(nil), "feast.serving.QueryFeaturesResponse") + proto.RegisterMapType((map[string]*Entity)(nil), "feast.serving.QueryFeaturesResponse.EntitiesEntry") proto.RegisterType((*Entity)(nil), "feast.serving.Entity") - proto.RegisterMapType((map[string]*FeatureValueList)(nil), "feast.serving.Entity.FeaturesEntry") - proto.RegisterType((*FeatureValueList)(nil), "feast.serving.FeatureValueList") - proto.RegisterEnum("feast.serving.RequestType", RequestType_name, RequestType_value) + proto.RegisterMapType((map[string]*FeatureValue)(nil), "feast.serving.Entity.FeaturesEntry") + proto.RegisterType((*FeatureValue)(nil), "feast.serving.FeatureValue") } // Reference imports to suppress errors if they are not otherwise used. @@ -423,8 +241,8 @@ const _ = grpc.SupportPackageIsVersion4 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type ServingAPIClient interface { - // Query features from Feast - QueryFeatures(ctx context.Context, in *QueryFeatures_Request, opts ...grpc.CallOption) (*QueryFeatures_Response, error) + // Query features from Feast serving storage + QueryFeatures(ctx context.Context, in *QueryFeaturesRequest, opts ...grpc.CallOption) (*QueryFeaturesResponse, error) } type servingAPIClient struct { @@ -435,8 +253,8 @@ func NewServingAPIClient(cc *grpc.ClientConn) ServingAPIClient { return &servingAPIClient{cc} } -func (c *servingAPIClient) QueryFeatures(ctx context.Context, in *QueryFeatures_Request, opts ...grpc.CallOption) (*QueryFeatures_Response, error) { - out := new(QueryFeatures_Response) +func (c *servingAPIClient) QueryFeatures(ctx context.Context, in *QueryFeaturesRequest, opts ...grpc.CallOption) (*QueryFeaturesResponse, error) { + out := new(QueryFeaturesResponse) err := c.cc.Invoke(ctx, "/feast.serving.ServingAPI/QueryFeatures", in, out, opts...) if err != nil { return nil, err @@ -446,8 +264,8 @@ func (c *servingAPIClient) QueryFeatures(ctx context.Context, in *QueryFeatures_ // ServingAPIServer is the server API for ServingAPI service. type ServingAPIServer interface { - // Query features from Feast - QueryFeatures(context.Context, *QueryFeatures_Request) (*QueryFeatures_Response, error) + // Query features from Feast serving storage + QueryFeatures(context.Context, *QueryFeaturesRequest) (*QueryFeaturesResponse, error) } func RegisterServingAPIServer(s *grpc.Server, srv ServingAPIServer) { @@ -455,7 +273,7 @@ func RegisterServingAPIServer(s *grpc.Server, srv ServingAPIServer) { } func _ServingAPI_QueryFeatures_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QueryFeatures_Request) + in := new(QueryFeaturesRequest) if err := dec(in); err != nil { return nil, err } @@ -467,7 +285,7 @@ func _ServingAPI_QueryFeatures_Handler(srv interface{}, ctx context.Context, dec FullMethod: "/feast.serving.ServingAPI/QueryFeatures", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServingAPIServer).QueryFeatures(ctx, req.(*QueryFeatures_Request)) + return srv.(ServingAPIServer).QueryFeatures(ctx, req.(*QueryFeaturesRequest)) } return interceptor(ctx, in, info, handler) } @@ -485,47 +303,35 @@ var _ServingAPI_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/serving/Serving.proto", } -func init() { - proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_Serving_3801c4d5794dbe13) -} - -var fileDescriptor_Serving_3801c4d5794dbe13 = []byte{ - // 591 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xdd, 0x6e, 0xd3, 0x4c, - 0x10, 0xad, 0xeb, 0xa6, 0x5f, 0x32, 0x91, 0xf3, 0x45, 0x2b, 0x7e, 0x2c, 0x53, 0x68, 0x08, 0x20, - 0x45, 0x80, 0xd6, 0xc8, 0x05, 0xa9, 0xe2, 0x06, 0x5a, 0xb5, 0x48, 0x91, 0x2a, 0x28, 0xdb, 0x88, - 0x8b, 0x0a, 0x21, 0x39, 0x64, 0x62, 0x4c, 0x13, 0xdb, 0x78, 0xd7, 0x91, 0xfc, 0x0a, 0xbc, 0x08, - 0x6f, 0xc0, 0x6b, 0x20, 0xf1, 0x44, 0x28, 0xbb, 0xeb, 0xc4, 0xb6, 0x22, 0xca, 0x95, 0xbd, 0x9e, - 0x73, 0xce, 0x78, 0xe6, 0xcc, 0x0e, 0xdc, 0x99, 0xa2, 0xcf, 0x85, 0xcb, 0x31, 0x5d, 0x84, 0x51, - 0xe0, 0x5e, 0xa8, 0x27, 0x4d, 0xd2, 0x58, 0xc4, 0xc4, 0x92, 0x41, 0xaa, 0x83, 0xce, 0x7e, 0x10, - 0xc7, 0xc1, 0x0c, 0x5d, 0x19, 0x1c, 0x67, 0x53, 0x57, 0x84, 0x73, 0xe4, 0xc2, 0x9f, 0x27, 0x0a, - 0xef, 0xdc, 0x56, 0x62, 0x22, 0x4f, 0x90, 0xbb, 0x1f, 0xfc, 0x59, 0x86, 0x2a, 0xd0, 0xff, 0x69, - 0x82, 0xf5, 0x3e, 0xc3, 0x34, 0x7f, 0x83, 0xbe, 0xc8, 0x52, 0xe4, 0xce, 0x6f, 0x03, 0xfe, 0x63, - 0xf8, 0x2d, 0x43, 0x2e, 0xc8, 0x3d, 0x00, 0x8c, 0x44, 0x28, 0xf2, 0xb7, 0xfe, 0x1c, 0x6d, 0xa3, - 0x67, 0x0c, 0x5a, 0xac, 0xf4, 0x85, 0x38, 0xd0, 0x54, 0xa7, 0xe1, 0xc4, 0xde, 0xee, 0x99, 0x83, - 0x16, 0x5b, 0x9d, 0xc9, 0x09, 0x74, 0x52, 0x25, 0x73, 0x82, 0xc2, 0x0f, 0x67, 0xdc, 0x36, 0x7b, - 0xe6, 0xa0, 0xed, 0xed, 0xd1, 0xca, 0xbf, 0x53, 0x56, 0x06, 0xb1, 0x1a, 0x87, 0x9c, 0x42, 0x67, - 0x55, 0x0b, 0xf3, 0xa3, 0x00, 0xed, 0x9d, 0x9e, 0x31, 0x68, 0x7b, 0x77, 0x6b, 0x2a, 0xa3, 0x0a, - 0x88, 0xd5, 0x48, 0xce, 0x2f, 0x03, 0x9a, 0x0c, 0x79, 0x12, 0x47, 0x1c, 0xaf, 0xad, 0xea, 0x9d, - 0xae, 0x2a, 0x44, 0x2e, 0xab, 0x6a, 0x7b, 0x07, 0xb5, 0x6c, 0x95, 0x8e, 0xd1, 0x42, 0x98, 0x9e, - 0x6a, 0xd6, 0x69, 0x24, 0xd2, 0x9c, 0xad, 0x44, 0x1c, 0x06, 0x56, 0x25, 0x44, 0xba, 0x60, 0x5e, - 0x61, 0xae, 0x53, 0x2f, 0x5f, 0xc9, 0x13, 0x68, 0x2c, 0x96, 0xb6, 0xd8, 0xdb, 0xb2, 0xbc, 0x9b, - 0xb5, 0x84, 0x92, 0x9e, 0x33, 0x85, 0x79, 0xb9, 0x7d, 0x68, 0xf4, 0x39, 0x58, 0x95, 0xce, 0x91, - 0x3d, 0x68, 0x4d, 0xd5, 0x1f, 0x0d, 0x27, 0x5a, 0x79, 0xfd, 0x81, 0x50, 0xd8, 0x59, 0x9a, 0x2f, - 0xe5, 0x3b, 0x9e, 0xb3, 0xd9, 0x83, 0x51, 0x9e, 0x20, 0x93, 0x38, 0x72, 0x03, 0x1a, 0xb3, 0x70, - 0x1e, 0x0a, 0xdb, 0xec, 0x19, 0x83, 0x06, 0x53, 0x87, 0x7e, 0x02, 0x9d, 0x6a, 0xa3, 0xc9, 0x33, - 0x68, 0x70, 0xe1, 0xa7, 0x42, 0x66, 0x6c, 0x7b, 0x0e, 0x55, 0x93, 0x48, 0x8b, 0x49, 0x2c, 0x19, - 0xa3, 0x80, 0xe4, 0x29, 0x98, 0x18, 0x4d, 0x74, 0x9d, 0x7f, 0xc3, 0x2f, 0x61, 0xfd, 0x1f, 0x06, - 0xec, 0xaa, 0xe2, 0xc9, 0x2b, 0x68, 0xea, 0x7a, 0xb8, 0x6d, 0x48, 0x5b, 0x1e, 0x6c, 0xec, 0x12, - 0x2d, 0x8c, 0xd1, 0x36, 0x14, 0x24, 0xe7, 0x23, 0x58, 0x95, 0xd0, 0x06, 0x1b, 0x5e, 0x54, 0x6d, - 0xd8, 0xaf, 0x25, 0xd0, 0x74, 0x79, 0x81, 0xce, 0x42, 0x2e, 0xca, 0x86, 0x7c, 0x37, 0xa0, 0x5b, - 0x8f, 0x93, 0xe7, 0xd0, 0x5a, 0x14, 0x07, 0xdd, 0xa2, 0x5b, 0x5a, 0x53, 0xde, 0x45, 0xba, 0x96, - 0x5a, 0x03, 0xc9, 0x6b, 0xb0, 0x56, 0xf3, 0x2b, 0x99, 0x45, 0xb3, 0xca, 0xcc, 0x51, 0x19, 0xc1, - 0xaa, 0x84, 0xc7, 0xf7, 0xa1, 0x5d, 0xf2, 0x94, 0x34, 0x61, 0xe7, 0xec, 0xe8, 0x62, 0xd4, 0xdd, - 0x92, 0x6f, 0xc3, 0x8b, 0x51, 0xd7, 0xf0, 0x66, 0x00, 0x7a, 0xa7, 0x1c, 0x9d, 0x0f, 0xc9, 0xa7, - 0xda, 0x1a, 0x20, 0x0f, 0xaf, 0x19, 0x79, 0x29, 0xee, 0x3c, 0xfa, 0xa7, 0x8b, 0xd1, 0xdf, 0x3a, - 0xbe, 0x84, 0xea, 0xca, 0x3a, 0xfe, 0x7f, 0x9d, 0xfc, 0x7c, 0xe9, 0xfd, 0xe5, 0x61, 0x10, 0x8a, - 0x2f, 0xd9, 0x98, 0x7e, 0x8e, 0xe7, 0x6e, 0x10, 0x7f, 0xc5, 0x2b, 0x57, 0xed, 0x2c, 0x39, 0x19, - 0xdc, 0x0d, 0x30, 0xc2, 0xd4, 0x17, 0x38, 0x71, 0x83, 0xd8, 0xad, 0xac, 0xc6, 0xf1, 0xae, 0x84, - 0x1c, 0xfc, 0x09, 0x00, 0x00, 0xff, 0xff, 0x17, 0x94, 0x4b, 0x84, 0x32, 0x05, 0x00, 0x00, +func init() { proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_7609de6de542e6f0) } + +var fileDescriptor_7609de6de542e6f0 = []byte{ + // 429 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xd1, 0x8a, 0xd3, 0x40, + 0x14, 0x75, 0x5a, 0x5c, 0x36, 0x77, 0x0d, 0xca, 0xe0, 0x62, 0xc8, 0x8a, 0x96, 0xac, 0x0f, 0x01, + 0x61, 0x06, 0xe3, 0x4b, 0xf1, 0x45, 0x5c, 0x58, 0xa1, 0x2f, 0x4b, 0x8d, 0x22, 0x52, 0x7c, 0x49, + 0xed, 0x6d, 0x4c, 0xdb, 0x64, 0x62, 0x66, 0x52, 0xc8, 0xf7, 0xf8, 0x59, 0xfe, 0x8c, 0x74, 0x66, + 0xd2, 0x26, 0xa5, 0xe8, 0x3e, 0x25, 0x33, 0xe7, 0xde, 0x73, 0xce, 0xcc, 0x3d, 0x03, 0x57, 0x4b, + 0x4c, 0xa4, 0xe2, 0x12, 0xab, 0x6d, 0x56, 0xa4, 0xfc, 0xb3, 0xf9, 0xb2, 0xb2, 0x12, 0x4a, 0x50, + 0x57, 0x83, 0xcc, 0x82, 0xfe, 0xcb, 0x54, 0x88, 0x74, 0x83, 0x5c, 0x83, 0xf3, 0x7a, 0xc9, 0x55, + 0x96, 0xa3, 0x54, 0x49, 0x5e, 0x9a, 0x7a, 0xff, 0x99, 0x21, 0x53, 0x4d, 0x89, 0x92, 0x7f, 0x4d, + 0x36, 0x35, 0x1a, 0x20, 0x28, 0xe1, 0xe9, 0xa7, 0x1a, 0xab, 0xe6, 0x23, 0x26, 0xaa, 0xae, 0x50, + 0xc6, 0xf8, 0xab, 0x46, 0xa9, 0xe8, 0x0b, 0x00, 0x2c, 0x54, 0xa6, 0x9a, 0xbb, 0x24, 0x47, 0x8f, + 0x8c, 0x48, 0xe8, 0xc4, 0x9d, 0x1d, 0xea, 0xc3, 0xb9, 0x59, 0x4d, 0x16, 0xde, 0x60, 0x34, 0x0c, + 0x9d, 0x78, 0xbf, 0xa6, 0xcf, 0xc1, 0x59, 0x1a, 0xba, 0xc9, 0xc2, 0x1b, 0x6a, 0xf0, 0xb0, 0x11, + 0xfc, 0x21, 0x70, 0x79, 0x24, 0x29, 0x4b, 0x51, 0x48, 0xfc, 0xaf, 0xe6, 0x9d, 0xd5, 0xcc, 0x50, + 0x6a, 0xcd, 0x8b, 0x28, 0x62, 0xbd, 0x7b, 0x60, 0x27, 0x79, 0xd9, 0xad, 0x6d, 0xba, 0x2d, 0x54, + 0xd5, 0xc4, 0x7b, 0x0e, 0x3f, 0x06, 0xb7, 0x07, 0xd1, 0x27, 0x30, 0x5c, 0x63, 0x63, 0x95, 0x77, + 0xbf, 0xf4, 0x35, 0x3c, 0xdc, 0xee, 0x6e, 0xcb, 0x1b, 0x8c, 0x48, 0x78, 0x11, 0x5d, 0x1e, 0xe9, + 0xe9, 0xf6, 0x26, 0x36, 0x35, 0xef, 0x06, 0x63, 0x12, 0xfc, 0x26, 0x70, 0x66, 0x76, 0xe9, 0x7b, + 0x38, 0xb7, 0xa7, 0x96, 0x1e, 0xd1, 0x76, 0xaf, 0x4f, 0xb6, 0xb3, 0xd6, 0xb0, 0xf5, 0xd7, 0x36, + 0xf9, 0xdf, 0xc0, 0xed, 0x41, 0x27, 0xfc, 0xbd, 0xe9, 0xfb, 0xbb, 0x3a, 0x12, 0xb0, 0xed, 0x7a, + 0xe0, 0x5d, 0x97, 0x15, 0x3c, 0xea, 0x42, 0x34, 0x6c, 0x69, 0x88, 0xa6, 0xa1, 0x96, 0x46, 0xc7, + 0x85, 0x75, 0xbb, 0xe9, 0x18, 0x9c, 0x7d, 0xb6, 0xac, 0xa8, 0xcf, 0x4c, 0xfa, 0x58, 0x9b, 0x3e, + 0xf6, 0xa5, 0xad, 0x88, 0x0f, 0xc5, 0xd1, 0x0a, 0xc0, 0x66, 0xf8, 0xc3, 0x74, 0x42, 0xbf, 0x83, + 0xdb, 0x1b, 0x16, 0xbd, 0xfe, 0xf7, 0x28, 0x75, 0x2a, 0xfd, 0x57, 0xf7, 0x99, 0x77, 0xf0, 0xe0, + 0x66, 0x06, 0xfd, 0x07, 0x72, 0xf3, 0xf8, 0x20, 0x3d, 0xdd, 0xb9, 0x9c, 0x8d, 0xd3, 0x4c, 0xfd, + 0xac, 0xe7, 0xec, 0x87, 0xc8, 0x79, 0x2a, 0x56, 0xb8, 0xe6, 0xe6, 0x85, 0xe8, 0x33, 0x48, 0x9e, + 0x62, 0x81, 0x55, 0xa2, 0x70, 0xc1, 0x53, 0xc1, 0x7b, 0x0f, 0x71, 0x7e, 0xa6, 0x4b, 0xde, 0xfe, + 0x0d, 0x00, 0x00, 0xff, 0xff, 0x77, 0xb5, 0x0e, 0xd1, 0xa0, 0x03, 0x00, 0x00, } diff --git a/protos/generated/go/feast/specs/EntitySpec.pb.go b/protos/generated/go/feast/specs/EntitySpec.pb.go index 0f4374c679e..6b2bbca1092 100644 --- a/protos/generated/go/feast/specs/EntitySpec.pb.go +++ b/protos/generated/go/feast/specs/EntitySpec.pb.go @@ -31,7 +31,7 @@ func (m *EntitySpec) Reset() { *m = EntitySpec{} } func (m *EntitySpec) String() string { return proto.CompactTextString(m) } func (*EntitySpec) ProtoMessage() {} func (*EntitySpec) Descriptor() ([]byte, []int) { - return fileDescriptor_EntitySpec_b8950ded39b854cb, []int{0} + return fileDescriptor_a3230229c2278d5e, []int{0} } func (m *EntitySpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_EntitySpec.Unmarshal(m, b) @@ -39,8 +39,8 @@ func (m *EntitySpec) XXX_Unmarshal(b []byte) error { func (m *EntitySpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_EntitySpec.Marshal(b, m, deterministic) } -func (dst *EntitySpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_EntitySpec.Merge(dst, src) +func (m *EntitySpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_EntitySpec.Merge(m, src) } func (m *EntitySpec) XXX_Size() int { return xxx_messageInfo_EntitySpec.Size(m) @@ -76,11 +76,9 @@ func init() { proto.RegisterType((*EntitySpec)(nil), "feast.specs.EntitySpec") } -func init() { - proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_EntitySpec_b8950ded39b854cb) -} +func init() { proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_a3230229c2278d5e) } -var fileDescriptor_EntitySpec_b8950ded39b854cb = []byte{ +var fileDescriptor_a3230229c2278d5e = []byte{ // 177 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0xcd, 0x2b, 0xc9, 0x2c, 0xa9, 0x0c, 0x2e, diff --git a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go index 8349e5a53aa..43bb2bb36d2 100644 --- a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go @@ -31,7 +31,7 @@ func (m *FeatureGroupSpec) Reset() { *m = FeatureGroupSpec{} } func (m *FeatureGroupSpec) String() string { return proto.CompactTextString(m) } func (*FeatureGroupSpec) ProtoMessage() {} func (*FeatureGroupSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureGroupSpec_21c67ee01edd412c, []int{0} + return fileDescriptor_df3b6a9e736b5719, []int{0} } func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureGroupSpec.Unmarshal(m, b) @@ -39,8 +39,8 @@ func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { func (m *FeatureGroupSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureGroupSpec.Marshal(b, m, deterministic) } -func (dst *FeatureGroupSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureGroupSpec.Merge(dst, src) +func (m *FeatureGroupSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureGroupSpec.Merge(m, src) } func (m *FeatureGroupSpec) XXX_Size() int { return xxx_messageInfo_FeatureGroupSpec.Size(m) @@ -76,11 +76,9 @@ func init() { proto.RegisterType((*FeatureGroupSpec)(nil), "feast.specs.FeatureGroupSpec") } -func init() { - proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_FeatureGroupSpec_21c67ee01edd412c) -} +func init() { proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_df3b6a9e736b5719) } -var fileDescriptor_FeatureGroupSpec_21c67ee01edd412c = []byte{ +var fileDescriptor_df3b6a9e736b5719 = []byte{ // 203 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4a, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0x75, diff --git a/protos/generated/go/feast/specs/FeatureSpec.pb.go b/protos/generated/go/feast/specs/FeatureSpec.pb.go index 4b8a3d95055..35e4c5adcba 100644 --- a/protos/generated/go/feast/specs/FeatureSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureSpec.pb.go @@ -41,7 +41,7 @@ func (m *FeatureSpec) Reset() { *m = FeatureSpec{} } func (m *FeatureSpec) String() string { return proto.CompactTextString(m) } func (*FeatureSpec) ProtoMessage() {} func (*FeatureSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_eccd5917f38e2b1f, []int{0} + return fileDescriptor_b8f1468f11147fbe, []int{0} } func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSpec.Unmarshal(m, b) @@ -49,8 +49,8 @@ func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { func (m *FeatureSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSpec.Marshal(b, m, deterministic) } -func (dst *FeatureSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSpec.Merge(dst, src) +func (m *FeatureSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSpec.Merge(m, src) } func (m *FeatureSpec) XXX_Size() int { return xxx_messageInfo_FeatureSpec.Size(m) @@ -157,7 +157,7 @@ func (m *DataStores) Reset() { *m = DataStores{} } func (m *DataStores) String() string { return proto.CompactTextString(m) } func (*DataStores) ProtoMessage() {} func (*DataStores) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_eccd5917f38e2b1f, []int{1} + return fileDescriptor_b8f1468f11147fbe, []int{1} } func (m *DataStores) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStores.Unmarshal(m, b) @@ -165,8 +165,8 @@ func (m *DataStores) XXX_Unmarshal(b []byte) error { func (m *DataStores) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStores.Marshal(b, m, deterministic) } -func (dst *DataStores) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStores.Merge(dst, src) +func (m *DataStores) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStores.Merge(m, src) } func (m *DataStores) XXX_Size() int { return xxx_messageInfo_DataStores.Size(m) @@ -203,7 +203,7 @@ func (m *DataStore) Reset() { *m = DataStore{} } func (m *DataStore) String() string { return proto.CompactTextString(m) } func (*DataStore) ProtoMessage() {} func (*DataStore) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_eccd5917f38e2b1f, []int{2} + return fileDescriptor_b8f1468f11147fbe, []int{2} } func (m *DataStore) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStore.Unmarshal(m, b) @@ -211,8 +211,8 @@ func (m *DataStore) XXX_Unmarshal(b []byte) error { func (m *DataStore) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStore.Marshal(b, m, deterministic) } -func (dst *DataStore) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStore.Merge(dst, src) +func (m *DataStore) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStore.Merge(m, src) } func (m *DataStore) XXX_Size() int { return xxx_messageInfo_DataStore.Size(m) @@ -245,11 +245,9 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.DataStore.OptionsEntry") } -func init() { - proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_FeatureSpec_eccd5917f38e2b1f) -} +func init() { proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_b8f1468f11147fbe) } -var fileDescriptor_FeatureSpec_eccd5917f38e2b1f = []byte{ +var fileDescriptor_b8f1468f11147fbe = []byte{ // 479 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0x4d, 0x6b, 0xdb, 0x40, 0x10, 0x45, 0x56, 0x62, 0x57, 0xa3, 0x10, 0xc2, 0x52, 0x92, 0xc5, 0x6d, 0x40, 0xb8, 0x14, 0x7c, diff --git a/protos/generated/go/feast/specs/ImportSpec.pb.go b/protos/generated/go/feast/specs/ImportSpec.pb.go index d16f9fa07bd..3a4c8d0801c 100644 --- a/protos/generated/go/feast/specs/ImportSpec.pb.go +++ b/protos/generated/go/feast/specs/ImportSpec.pb.go @@ -34,7 +34,7 @@ func (m *ImportSpec) Reset() { *m = ImportSpec{} } func (m *ImportSpec) String() string { return proto.CompactTextString(m) } func (*ImportSpec) ProtoMessage() {} func (*ImportSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{0} + return fileDescriptor_bf21bfd2215b3e0c, []int{0} } func (m *ImportSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportSpec.Unmarshal(m, b) @@ -42,8 +42,8 @@ func (m *ImportSpec) XXX_Unmarshal(b []byte) error { func (m *ImportSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ImportSpec.Marshal(b, m, deterministic) } -func (dst *ImportSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportSpec.Merge(dst, src) +func (m *ImportSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportSpec.Merge(m, src) } func (m *ImportSpec) XXX_Size() int { return xxx_messageInfo_ImportSpec.Size(m) @@ -107,7 +107,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{1} + return fileDescriptor_bf21bfd2215b3e0c, []int{1} } func (m *Schema) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Schema.Unmarshal(m, b) @@ -115,8 +115,8 @@ func (m *Schema) XXX_Unmarshal(b []byte) error { func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Schema.Marshal(b, m, deterministic) } -func (dst *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(dst, src) +func (m *Schema) XXX_Merge(src proto.Message) { + xxx_messageInfo_Schema.Merge(m, src) } func (m *Schema) XXX_Size() int { return xxx_messageInfo_Schema.Size(m) @@ -260,7 +260,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{2} + return fileDescriptor_bf21bfd2215b3e0c, []int{2} } func (m *Field) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Field.Unmarshal(m, b) @@ -268,8 +268,8 @@ func (m *Field) XXX_Unmarshal(b []byte) error { func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Field.Marshal(b, m, deterministic) } -func (dst *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(dst, src) +func (m *Field) XXX_Merge(src proto.Message) { + xxx_messageInfo_Field.Merge(m, src) } func (m *Field) XXX_Size() int { return xxx_messageInfo_Field.Size(m) @@ -302,11 +302,9 @@ func init() { proto.RegisterType((*Field)(nil), "feast.specs.Field") } -func init() { - proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_673bc4f248a91137) -} +func init() { proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_bf21bfd2215b3e0c) } -var fileDescriptor_ImportSpec_673bc4f248a91137 = []byte{ +var fileDescriptor_bf21bfd2215b3e0c = []byte{ // 440 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8f, 0x93, 0x40, 0x14, 0x5d, 0xca, 0x16, 0xed, 0x6d, 0xdc, 0x9a, 0xab, 0x0f, 0x84, 0x6c, 0x62, 0xd3, 0x07, 0x6d, diff --git a/protos/generated/go/feast/specs/StorageSpec.pb.go b/protos/generated/go/feast/specs/StorageSpec.pb.go index fefe56a1be5..924e1f154f6 100644 --- a/protos/generated/go/feast/specs/StorageSpec.pb.go +++ b/protos/generated/go/feast/specs/StorageSpec.pb.go @@ -36,7 +36,7 @@ func (m *StorageSpec) Reset() { *m = StorageSpec{} } func (m *StorageSpec) String() string { return proto.CompactTextString(m) } func (*StorageSpec) ProtoMessage() {} func (*StorageSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_StorageSpec_bfb8a5e5cf34de95, []int{0} + return fileDescriptor_7783b72a0d689614, []int{0} } func (m *StorageSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StorageSpec.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *StorageSpec) XXX_Unmarshal(b []byte) error { func (m *StorageSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StorageSpec.Marshal(b, m, deterministic) } -func (dst *StorageSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_StorageSpec.Merge(dst, src) +func (m *StorageSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_StorageSpec.Merge(m, src) } func (m *StorageSpec) XXX_Size() int { return xxx_messageInfo_StorageSpec.Size(m) @@ -82,11 +82,9 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.StorageSpec.OptionsEntry") } -func init() { - proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_StorageSpec_bfb8a5e5cf34de95) -} +func init() { proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_7783b72a0d689614) } -var fileDescriptor_StorageSpec_bfb8a5e5cf34de95 = []byte{ +var fileDescriptor_7783b72a0d689614 = []byte{ // 227 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x0f, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x0d, diff --git a/protos/generated/go/feast/storage/BigTable.pb.go b/protos/generated/go/feast/storage/BigTable.pb.go index e65b5e35fce..9943d0a884b 100644 --- a/protos/generated/go/feast/storage/BigTable.pb.go +++ b/protos/generated/go/feast/storage/BigTable.pb.go @@ -33,7 +33,7 @@ func (m *BigTableRowKey) Reset() { *m = BigTableRowKey{} } func (m *BigTableRowKey) String() string { return proto.CompactTextString(m) } func (*BigTableRowKey) ProtoMessage() {} func (*BigTableRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_BigTable_e25bdeffe0669ae7, []int{0} + return fileDescriptor_1d33ec3bd45c712c, []int{0} } func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BigTableRowKey.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { func (m *BigTableRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BigTableRowKey.Marshal(b, m, deterministic) } -func (dst *BigTableRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_BigTableRowKey.Merge(dst, src) +func (m *BigTableRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_BigTableRowKey.Merge(m, src) } func (m *BigTableRowKey) XXX_Size() int { return xxx_messageInfo_BigTableRowKey.Size(m) @@ -78,11 +78,9 @@ func init() { proto.RegisterType((*BigTableRowKey)(nil), "feast.storage.BigTableRowKey") } -func init() { - proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_e25bdeffe0669ae7) -} +func init() { proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_1d33ec3bd45c712c) } -var fileDescriptor_BigTable_e25bdeffe0669ae7 = []byte{ +var fileDescriptor_1d33ec3bd45c712c = []byte{ // 193 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x8f, 0xb1, 0x0b, 0x82, 0x40, 0x18, 0x47, 0xb1, 0x20, 0xf0, 0x40, 0x87, 0x9b, 0x1c, 0x24, 0xa2, 0x21, 0x9a, 0x3c, 0xa2, 0xa5, diff --git a/protos/generated/go/feast/storage/Redis.pb.go b/protos/generated/go/feast/storage/Redis.pb.go index 49f80a061e3..f42a1f618d0 100644 --- a/protos/generated/go/feast/storage/Redis.pb.go +++ b/protos/generated/go/feast/storage/Redis.pb.go @@ -43,7 +43,7 @@ func (m *RedisBucketKey) Reset() { *m = RedisBucketKey{} } func (m *RedisBucketKey) String() string { return proto.CompactTextString(m) } func (*RedisBucketKey) ProtoMessage() {} func (*RedisBucketKey) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{0} + return fileDescriptor_64e898a359fc9e5d, []int{0} } func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketKey.Unmarshal(m, b) @@ -51,8 +51,8 @@ func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { func (m *RedisBucketKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketKey.Marshal(b, m, deterministic) } -func (dst *RedisBucketKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketKey.Merge(dst, src) +func (m *RedisBucketKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketKey.Merge(m, src) } func (m *RedisBucketKey) XXX_Size() int { return xxx_messageInfo_RedisBucketKey.Size(m) @@ -99,7 +99,7 @@ func (m *RedisBucketValue) Reset() { *m = RedisBucketValue{} } func (m *RedisBucketValue) String() string { return proto.CompactTextString(m) } func (*RedisBucketValue) ProtoMessage() {} func (*RedisBucketValue) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{1} + return fileDescriptor_64e898a359fc9e5d, []int{1} } func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValue.Unmarshal(m, b) @@ -107,8 +107,8 @@ func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { func (m *RedisBucketValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValue.Marshal(b, m, deterministic) } -func (dst *RedisBucketValue) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValue.Merge(dst, src) +func (m *RedisBucketValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValue.Merge(m, src) } func (m *RedisBucketValue) XXX_Size() int { return xxx_messageInfo_RedisBucketValue.Size(m) @@ -147,7 +147,7 @@ func (m *RedisBucketValueList) Reset() { *m = RedisBucketValueList{} } func (m *RedisBucketValueList) String() string { return proto.CompactTextString(m) } func (*RedisBucketValueList) ProtoMessage() {} func (*RedisBucketValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{2} + return fileDescriptor_64e898a359fc9e5d, []int{2} } func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValueList.Unmarshal(m, b) @@ -155,8 +155,8 @@ func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { func (m *RedisBucketValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValueList.Marshal(b, m, deterministic) } -func (dst *RedisBucketValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValueList.Merge(dst, src) +func (m *RedisBucketValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValueList.Merge(m, src) } func (m *RedisBucketValueList) XXX_Size() int { return xxx_messageInfo_RedisBucketValueList.Size(m) @@ -180,9 +180,9 @@ func init() { proto.RegisterType((*RedisBucketValueList)(nil), "feast.storage.RedisBucketValueList") } -func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_Redis_cef62c817c1622ce) } +func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_64e898a359fc9e5d) } -var fileDescriptor_Redis_cef62c817c1622ce = []byte{ +var fileDescriptor_64e898a359fc9e5d = []byte{ // 325 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcd, 0x4f, 0xf2, 0x40, 0x10, 0xc6, 0xd3, 0x97, 0x57, 0x22, 0x4b, 0x24, 0x66, 0x35, 0xb1, 0x36, 0x26, 0x34, 0x9c, 0x7a, diff --git a/protos/generated/go/feast/types/Feature.pb.go b/protos/generated/go/feast/types/Feature.pb.go index 51ed41fb54f..44534da5c75 100644 --- a/protos/generated/go/feast/types/Feature.pb.go +++ b/protos/generated/go/feast/types/Feature.pb.go @@ -30,7 +30,7 @@ func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_Feature_c2a5d99d9bf3ca9c, []int{0} + return fileDescriptor_4e19474999533fc9, []int{0} } func (m *Feature) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Feature.Unmarshal(m, b) @@ -38,8 +38,8 @@ func (m *Feature) XXX_Unmarshal(b []byte) error { func (m *Feature) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Feature.Marshal(b, m, deterministic) } -func (dst *Feature) XXX_Merge(src proto.Message) { - xxx_messageInfo_Feature.Merge(dst, src) +func (m *Feature) XXX_Merge(src proto.Message) { + xxx_messageInfo_Feature.Merge(m, src) } func (m *Feature) XXX_Size() int { return xxx_messageInfo_Feature.Size(m) @@ -68,9 +68,9 @@ func init() { proto.RegisterType((*Feature)(nil), "feast.types.Feature") } -func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_c2a5d99d9bf3ca9c) } +func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_4e19474999533fc9) } -var fileDescriptor_Feature_c2a5d99d9bf3ca9c = []byte{ +var fileDescriptor_4e19474999533fc9 = []byte{ // 173 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0xd5, diff --git a/protos/generated/go/feast/types/FeatureRow.pb.go b/protos/generated/go/feast/types/FeatureRow.pb.go index 3fb6bc72f9d..ba8712ce346 100644 --- a/protos/generated/go/feast/types/FeatureRow.pb.go +++ b/protos/generated/go/feast/types/FeatureRow.pb.go @@ -33,7 +33,7 @@ func (m *FeatureRowKey) Reset() { *m = FeatureRowKey{} } func (m *FeatureRowKey) String() string { return proto.CompactTextString(m) } func (*FeatureRowKey) ProtoMessage() {} func (*FeatureRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_435f7325616f2983, []int{0} + return fileDescriptor_fbbea9c89787d1c7, []int{0} } func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowKey.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { func (m *FeatureRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowKey.Marshal(b, m, deterministic) } -func (dst *FeatureRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowKey.Merge(dst, src) +func (m *FeatureRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowKey.Merge(m, src) } func (m *FeatureRowKey) XXX_Size() int { return xxx_messageInfo_FeatureRowKey.Size(m) @@ -96,7 +96,7 @@ func (m *FeatureRow) Reset() { *m = FeatureRow{} } func (m *FeatureRow) String() string { return proto.CompactTextString(m) } func (*FeatureRow) ProtoMessage() {} func (*FeatureRow) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_435f7325616f2983, []int{1} + return fileDescriptor_fbbea9c89787d1c7, []int{1} } func (m *FeatureRow) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRow.Unmarshal(m, b) @@ -104,8 +104,8 @@ func (m *FeatureRow) XXX_Unmarshal(b []byte) error { func (m *FeatureRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRow.Marshal(b, m, deterministic) } -func (dst *FeatureRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRow.Merge(dst, src) +func (m *FeatureRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRow.Merge(m, src) } func (m *FeatureRow) XXX_Size() int { return xxx_messageInfo_FeatureRow.Size(m) @@ -156,11 +156,9 @@ func init() { proto.RegisterType((*FeatureRow)(nil), "feast.types.FeatureRow") } -func init() { - proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_435f7325616f2983) -} +func init() { proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_fbbea9c89787d1c7) } -var fileDescriptor_FeatureRow_435f7325616f2983 = []byte{ +var fileDescriptor_fbbea9c89787d1c7 = []byte{ // 305 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x92, 0x41, 0x4b, 0xfb, 0x40, 0x10, 0xc5, 0x49, 0xfb, 0xff, 0x8b, 0x9d, 0x60, 0x85, 0xe0, 0x21, 0x96, 0x56, 0x43, 0x4f, 0x39, diff --git a/protos/generated/go/feast/types/FeatureRowExtended.pb.go b/protos/generated/go/feast/types/FeatureRowExtended.pb.go index ad89f27db23..9aab57b1896 100644 --- a/protos/generated/go/feast/types/FeatureRowExtended.pb.go +++ b/protos/generated/go/feast/types/FeatureRowExtended.pb.go @@ -33,7 +33,7 @@ func (m *Error) Reset() { *m = Error{} } func (m *Error) String() string { return proto.CompactTextString(m) } func (*Error) ProtoMessage() {} func (*Error) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{0} + return fileDescriptor_7823aa2c72575793, []int{0} } func (m *Error) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Error.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *Error) XXX_Unmarshal(b []byte) error { func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Error.Marshal(b, m, deterministic) } -func (dst *Error) XXX_Merge(src proto.Message) { - xxx_messageInfo_Error.Merge(dst, src) +func (m *Error) XXX_Merge(src proto.Message) { + xxx_messageInfo_Error.Merge(m, src) } func (m *Error) XXX_Size() int { return xxx_messageInfo_Error.Size(m) @@ -93,7 +93,7 @@ func (m *Attempt) Reset() { *m = Attempt{} } func (m *Attempt) String() string { return proto.CompactTextString(m) } func (*Attempt) ProtoMessage() {} func (*Attempt) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{1} + return fileDescriptor_7823aa2c72575793, []int{1} } func (m *Attempt) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attempt.Unmarshal(m, b) @@ -101,8 +101,8 @@ func (m *Attempt) XXX_Unmarshal(b []byte) error { func (m *Attempt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Attempt.Marshal(b, m, deterministic) } -func (dst *Attempt) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attempt.Merge(dst, src) +func (m *Attempt) XXX_Merge(src proto.Message) { + xxx_messageInfo_Attempt.Merge(m, src) } func (m *Attempt) XXX_Size() int { return xxx_messageInfo_Attempt.Size(m) @@ -140,7 +140,7 @@ func (m *FeatureRowExtended) Reset() { *m = FeatureRowExtended{} } func (m *FeatureRowExtended) String() string { return proto.CompactTextString(m) } func (*FeatureRowExtended) ProtoMessage() {} func (*FeatureRowExtended) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{2} + return fileDescriptor_7823aa2c72575793, []int{2} } func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowExtended.Unmarshal(m, b) @@ -148,8 +148,8 @@ func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { func (m *FeatureRowExtended) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowExtended.Marshal(b, m, deterministic) } -func (dst *FeatureRowExtended) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowExtended.Merge(dst, src) +func (m *FeatureRowExtended) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowExtended.Merge(m, src) } func (m *FeatureRowExtended) XXX_Size() int { return xxx_messageInfo_FeatureRowExtended.Size(m) @@ -188,10 +188,10 @@ func init() { } func init() { - proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_FeatureRowExtended_bfd3c37956d1a040) + proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_7823aa2c72575793) } -var fileDescriptor_FeatureRowExtended_bfd3c37956d1a040 = []byte{ +var fileDescriptor_7823aa2c72575793 = []byte{ // 338 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x6b, 0xea, 0x40, 0x10, 0xc6, 0xf1, 0xf9, 0xf2, 0x7c, 0x4e, 0x6e, 0x8b, 0x60, 0x08, 0xd2, 0x16, 0xe9, 0xc1, 0x5e, diff --git a/protos/generated/go/feast/types/Granularity.pb.go b/protos/generated/go/feast/types/Granularity.pb.go index 5166f7bbc2e..99aae3d90fc 100644 --- a/protos/generated/go/feast/types/Granularity.pb.go +++ b/protos/generated/go/feast/types/Granularity.pb.go @@ -35,6 +35,7 @@ var Granularity_Enum_name = map[int32]string{ 3: "MINUTE", 4: "SECOND", } + var Granularity_Enum_value = map[string]int32{ "NONE": 0, "DAY": 1, @@ -46,8 +47,9 @@ var Granularity_Enum_value = map[string]int32{ func (x Granularity_Enum) String() string { return proto.EnumName(Granularity_Enum_name, int32(x)) } + func (Granularity_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0, 0} + return fileDescriptor_846024d9f96d5fa2, []int{0, 0} } type Granularity struct { @@ -60,7 +62,7 @@ func (m *Granularity) Reset() { *m = Granularity{} } func (m *Granularity) String() string { return proto.CompactTextString(m) } func (*Granularity) ProtoMessage() {} func (*Granularity) Descriptor() ([]byte, []int) { - return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0} + return fileDescriptor_846024d9f96d5fa2, []int{0} } func (m *Granularity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Granularity.Unmarshal(m, b) @@ -68,8 +70,8 @@ func (m *Granularity) XXX_Unmarshal(b []byte) error { func (m *Granularity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Granularity.Marshal(b, m, deterministic) } -func (dst *Granularity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Granularity.Merge(dst, src) +func (m *Granularity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Granularity.Merge(m, src) } func (m *Granularity) XXX_Size() int { return xxx_messageInfo_Granularity.Size(m) @@ -85,11 +87,9 @@ func init() { proto.RegisterEnum("feast.types.Granularity_Enum", Granularity_Enum_name, Granularity_Enum_value) } -func init() { - proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_Granularity_4b57756f7a751fdb) -} +func init() { proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_846024d9f96d5fa2) } -var fileDescriptor_Granularity_4b57756f7a751fdb = []byte{ +var fileDescriptor_846024d9f96d5fa2 = []byte{ // 183 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x2f, 0x4a, 0xcc, 0x2b, 0xcd, 0x49, 0x2c, diff --git a/protos/generated/go/feast/types/Value.pb.go b/protos/generated/go/feast/types/Value.pb.go index b9c26f8c4b8..4b0121e3260 100644 --- a/protos/generated/go/feast/types/Value.pb.go +++ b/protos/generated/go/feast/types/Value.pb.go @@ -44,6 +44,7 @@ var ValueType_Enum_name = map[int32]string{ 7: "BOOL", 8: "TIMESTAMP", } + var ValueType_Enum_value = map[string]int32{ "UNKNOWN": 0, "BYTES": 1, @@ -59,8 +60,9 @@ var ValueType_Enum_value = map[string]int32{ func (x ValueType_Enum) String() string { return proto.EnumName(ValueType_Enum_name, int32(x)) } + func (ValueType_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{0, 0} + return fileDescriptor_47c504407d284ecc, []int{0, 0} } type ValueType struct { @@ -73,7 +75,7 @@ func (m *ValueType) Reset() { *m = ValueType{} } func (m *ValueType) String() string { return proto.CompactTextString(m) } func (*ValueType) ProtoMessage() {} func (*ValueType) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{0} + return fileDescriptor_47c504407d284ecc, []int{0} } func (m *ValueType) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueType.Unmarshal(m, b) @@ -81,8 +83,8 @@ func (m *ValueType) XXX_Unmarshal(b []byte) error { func (m *ValueType) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueType.Marshal(b, m, deterministic) } -func (dst *ValueType) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueType.Merge(dst, src) +func (m *ValueType) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueType.Merge(m, src) } func (m *ValueType) XXX_Size() int { return xxx_messageInfo_ValueType.Size(m) @@ -113,7 +115,7 @@ func (m *Value) Reset() { *m = Value{} } func (m *Value) String() string { return proto.CompactTextString(m) } func (*Value) ProtoMessage() {} func (*Value) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{1} + return fileDescriptor_47c504407d284ecc, []int{1} } func (m *Value) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Value.Unmarshal(m, b) @@ -121,8 +123,8 @@ func (m *Value) XXX_Unmarshal(b []byte) error { func (m *Value) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Value.Marshal(b, m, deterministic) } -func (dst *Value) XXX_Merge(src proto.Message) { - xxx_messageInfo_Value.Merge(dst, src) +func (m *Value) XXX_Merge(src proto.Message) { + xxx_messageInfo_Value.Merge(m, src) } func (m *Value) XXX_Size() int { return xxx_messageInfo_Value.Size(m) @@ -427,7 +429,7 @@ func (m *ValueList) Reset() { *m = ValueList{} } func (m *ValueList) String() string { return proto.CompactTextString(m) } func (*ValueList) ProtoMessage() {} func (*ValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{2} + return fileDescriptor_47c504407d284ecc, []int{2} } func (m *ValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueList.Unmarshal(m, b) @@ -435,8 +437,8 @@ func (m *ValueList) XXX_Unmarshal(b []byte) error { func (m *ValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueList.Marshal(b, m, deterministic) } -func (dst *ValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueList.Merge(dst, src) +func (m *ValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueList.Merge(m, src) } func (m *ValueList) XXX_Size() int { return xxx_messageInfo_ValueList.Size(m) @@ -761,7 +763,7 @@ func (m *BytesList) Reset() { *m = BytesList{} } func (m *BytesList) String() string { return proto.CompactTextString(m) } func (*BytesList) ProtoMessage() {} func (*BytesList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{3} + return fileDescriptor_47c504407d284ecc, []int{3} } func (m *BytesList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BytesList.Unmarshal(m, b) @@ -769,8 +771,8 @@ func (m *BytesList) XXX_Unmarshal(b []byte) error { func (m *BytesList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BytesList.Marshal(b, m, deterministic) } -func (dst *BytesList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BytesList.Merge(dst, src) +func (m *BytesList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BytesList.Merge(m, src) } func (m *BytesList) XXX_Size() int { return xxx_messageInfo_BytesList.Size(m) @@ -799,7 +801,7 @@ func (m *StringList) Reset() { *m = StringList{} } func (m *StringList) String() string { return proto.CompactTextString(m) } func (*StringList) ProtoMessage() {} func (*StringList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{4} + return fileDescriptor_47c504407d284ecc, []int{4} } func (m *StringList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StringList.Unmarshal(m, b) @@ -807,8 +809,8 @@ func (m *StringList) XXX_Unmarshal(b []byte) error { func (m *StringList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StringList.Marshal(b, m, deterministic) } -func (dst *StringList) XXX_Merge(src proto.Message) { - xxx_messageInfo_StringList.Merge(dst, src) +func (m *StringList) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringList.Merge(m, src) } func (m *StringList) XXX_Size() int { return xxx_messageInfo_StringList.Size(m) @@ -837,7 +839,7 @@ func (m *Int32List) Reset() { *m = Int32List{} } func (m *Int32List) String() string { return proto.CompactTextString(m) } func (*Int32List) ProtoMessage() {} func (*Int32List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{5} + return fileDescriptor_47c504407d284ecc, []int{5} } func (m *Int32List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int32List.Unmarshal(m, b) @@ -845,8 +847,8 @@ func (m *Int32List) XXX_Unmarshal(b []byte) error { func (m *Int32List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int32List.Marshal(b, m, deterministic) } -func (dst *Int32List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int32List.Merge(dst, src) +func (m *Int32List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int32List.Merge(m, src) } func (m *Int32List) XXX_Size() int { return xxx_messageInfo_Int32List.Size(m) @@ -875,7 +877,7 @@ func (m *Int64List) Reset() { *m = Int64List{} } func (m *Int64List) String() string { return proto.CompactTextString(m) } func (*Int64List) ProtoMessage() {} func (*Int64List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{6} + return fileDescriptor_47c504407d284ecc, []int{6} } func (m *Int64List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int64List.Unmarshal(m, b) @@ -883,8 +885,8 @@ func (m *Int64List) XXX_Unmarshal(b []byte) error { func (m *Int64List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int64List.Marshal(b, m, deterministic) } -func (dst *Int64List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int64List.Merge(dst, src) +func (m *Int64List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int64List.Merge(m, src) } func (m *Int64List) XXX_Size() int { return xxx_messageInfo_Int64List.Size(m) @@ -913,7 +915,7 @@ func (m *DoubleList) Reset() { *m = DoubleList{} } func (m *DoubleList) String() string { return proto.CompactTextString(m) } func (*DoubleList) ProtoMessage() {} func (*DoubleList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{7} + return fileDescriptor_47c504407d284ecc, []int{7} } func (m *DoubleList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DoubleList.Unmarshal(m, b) @@ -921,8 +923,8 @@ func (m *DoubleList) XXX_Unmarshal(b []byte) error { func (m *DoubleList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DoubleList.Marshal(b, m, deterministic) } -func (dst *DoubleList) XXX_Merge(src proto.Message) { - xxx_messageInfo_DoubleList.Merge(dst, src) +func (m *DoubleList) XXX_Merge(src proto.Message) { + xxx_messageInfo_DoubleList.Merge(m, src) } func (m *DoubleList) XXX_Size() int { return xxx_messageInfo_DoubleList.Size(m) @@ -951,7 +953,7 @@ func (m *FloatList) Reset() { *m = FloatList{} } func (m *FloatList) String() string { return proto.CompactTextString(m) } func (*FloatList) ProtoMessage() {} func (*FloatList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{8} + return fileDescriptor_47c504407d284ecc, []int{8} } func (m *FloatList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FloatList.Unmarshal(m, b) @@ -959,8 +961,8 @@ func (m *FloatList) XXX_Unmarshal(b []byte) error { func (m *FloatList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FloatList.Marshal(b, m, deterministic) } -func (dst *FloatList) XXX_Merge(src proto.Message) { - xxx_messageInfo_FloatList.Merge(dst, src) +func (m *FloatList) XXX_Merge(src proto.Message) { + xxx_messageInfo_FloatList.Merge(m, src) } func (m *FloatList) XXX_Size() int { return xxx_messageInfo_FloatList.Size(m) @@ -989,7 +991,7 @@ func (m *BoolList) Reset() { *m = BoolList{} } func (m *BoolList) String() string { return proto.CompactTextString(m) } func (*BoolList) ProtoMessage() {} func (*BoolList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{9} + return fileDescriptor_47c504407d284ecc, []int{9} } func (m *BoolList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BoolList.Unmarshal(m, b) @@ -997,8 +999,8 @@ func (m *BoolList) XXX_Unmarshal(b []byte) error { func (m *BoolList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BoolList.Marshal(b, m, deterministic) } -func (dst *BoolList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BoolList.Merge(dst, src) +func (m *BoolList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BoolList.Merge(m, src) } func (m *BoolList) XXX_Size() int { return xxx_messageInfo_BoolList.Size(m) @@ -1027,7 +1029,7 @@ func (m *TimestampList) Reset() { *m = TimestampList{} } func (m *TimestampList) String() string { return proto.CompactTextString(m) } func (*TimestampList) ProtoMessage() {} func (*TimestampList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{10} + return fileDescriptor_47c504407d284ecc, []int{10} } func (m *TimestampList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TimestampList.Unmarshal(m, b) @@ -1035,8 +1037,8 @@ func (m *TimestampList) XXX_Unmarshal(b []byte) error { func (m *TimestampList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TimestampList.Marshal(b, m, deterministic) } -func (dst *TimestampList) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampList.Merge(dst, src) +func (m *TimestampList) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimestampList.Merge(m, src) } func (m *TimestampList) XXX_Size() int { return xxx_messageInfo_TimestampList.Size(m) @@ -1069,9 +1071,9 @@ func init() { proto.RegisterEnum("feast.types.ValueType_Enum", ValueType_Enum_name, ValueType_Enum_value) } -func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_Value_0680a2f024df1112) } +func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_47c504407d284ecc) } -var fileDescriptor_Value_0680a2f024df1112 = []byte{ +var fileDescriptor_47c504407d284ecc = []byte{ // 626 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xd1, 0x6f, 0x9a, 0x50, 0x14, 0xc6, 0xb9, 0x22, 0x0a, 0xc7, 0x36, 0x21, 0x37, 0xd9, 0xda, 0x34, 0x6d, 0x47, 0x7c, 0xe2, diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index d1b09ebf22c..a8d146be89b 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -16,13 +16,11 @@ Main interface for users to interact with the Core API. """ -import enum import os from datetime import datetime import grpc import pandas as pd -import dateutil.parser from google.protobuf.timestamp_pb2 import Timestamp from feast.core.CoreService_pb2_grpc import CoreServiceStub @@ -38,21 +36,10 @@ from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader from feast.sdk.utils.print_utils import spec_to_yaml -from feast.serving.Serving_pb2 import QueryFeatures, RequestDetail, \ - TimestampRange +from feast.serving.Serving_pb2 import QueryFeaturesRequest from feast.serving.Serving_pb2_grpc import ServingAPIStub -class ServingRequestType(enum.Enum): - """ - Request type for serving api - """ - LAST = 0 - """ Get last value of a feature """ - LIST = 1 - """ Get list of value of a feature """ - - class Client: def __init__(self, core_url=None, serving_url=None, verbose=False): """Create an instance of Feast client which is connected to feast @@ -123,7 +110,6 @@ def verbose(self, val): raise TypeError("verbose should be a boolean value") self._verbose = val - def apply(self, obj): """Create or update one or many feast's resource (feature, entity, importer, storage). @@ -180,7 +166,7 @@ def run(self, importer, name_override=None, return response.jobId def create_dataset(self, feature_set, start_date, end_date, - limit=None, name_prefix=None): + limit=None, name_prefix=None): """ Create training dataset for a feature set. The training dataset will be bounded by event timestamp between start_date and end_date. @@ -202,7 +188,7 @@ def create_dataset(self, feature_set, start_date, end_date, the information of training dataset """ self._check_create_dataset_args(feature_set, start_date, - end_date, limit) + end_date, limit) req = DatasetServiceTypes.CreateDatasetRequest( featureSet=feature_set.proto, @@ -222,12 +208,8 @@ def create_dataset(self, feature_set, start_date, end_date, resp.datasetInfo.tableUrl)) return DatasetInfo(resp.datasetInfo.name, resp.datasetInfo.tableUrl) - def get_serving_data(self, feature_set, entity_keys, - request_type=ServingRequestType.LAST, - ts_range=[], limit=10): - """Get data from the feast serving layer. You can either retrieve the - the latest value, or a list of the latest values, up to a provided - limit. + def get_serving_data(self, feature_set, entity_keys, ts_range=None): + """Get feature value from feast serving API. If server_url is not provided, the value stored in the environment variable FEAST_SERVING_URL is used to connect to the serving server instead. @@ -236,26 +218,28 @@ def get_serving_data(self, feature_set, entity_keys, feature_set (feast.sdk.resources.feature_set.FeatureSet): feature set representing the data wanted entity_keys (:obj: `list` of :obj: `str): list of entity keys - request_type (feast.sdk.utils.types.ServingRequestType): - (default: feast.sdk.utils.types.ServingRequestType.LAST) type of - request: one of [LIST, LAST] - ts_range (:obj: `list` of str, optional): size 2 list of start - timestamp and end timestamp, in ISO 8601 format. Only required if - request_type is set to LIST - limit (int, optional): (default: 10) number of values to get. Only - required if request_type is set to LIST + ts_range (:obj: `list` of str, optional): size 2 list of start + and end time, in datetime type. It will + filter out any feature value having event timestamp outside + of the ts_range. Returns: pandas.DataFrame: DataFrame of results """ - - ts_range = [_timestamp_from_datetime(dateutil.parser.parse(dt)) - for dt in ts_range] - request = self._build_serving_request(feature_set, entity_keys, - request_type, ts_range, limit) + start = None + end = None + if ts_range is not None: + if len(ts_range) != 2: + raise ValueError("ts_range must have len 2") + start = ts_range[0] + end = ts_range[1] + if type(start) is not datetime or type(end) is not datetime: + raise TypeError("start and end must be datetime type") + + request = self._build_serving_request(feature_set, entity_keys) self._connect_serving() return self._response_to_df(feature_set, self._serving_service_stub - .QueryFeatures(request)) + .QueryFeatures(request), start, end) def download_dataset(self, dataset_info, dest, staging_location, file_type=FileType.CSV): @@ -309,7 +293,8 @@ def _connect_core(self): self.__core_channel = grpc.insecure_channel(self.core_url) self._core_service_stub = CoreServiceStub(self.__core_channel) self._job_service_stub = JobServiceStub(self.__core_channel) - self._dataset_service_stub = DatasetServiceStub(self.__core_channel) + self._dataset_service_stub = DatasetServiceStub( + self.__core_channel) def _connect_serving(self): """Connect to serving api""" @@ -317,49 +302,28 @@ def _connect_serving(self): self.__serving_channel = grpc.insecure_channel(self.serving_url) self._serving_service_stub = ServingAPIStub(self.__serving_channel) - def _build_serving_request(self, feature_set, entity_keys, request_type, - ts_range, limit): + def _build_serving_request(self, feature_set, entity_keys): """Helper function to build serving service request.""" - request = QueryFeatures.Request(entityName=feature_set.entity, - entityId=entity_keys) - features = [RequestDetail(featureId=feat_id, type=request_type.value) - for feat_id in feature_set.features] - - if request_type == ServingRequestType.LIST: - ts_range = TimestampRange(start=ts_range[0], end=ts_range[1]) - request.timestampRange.CopyFrom(ts_range) - for feature in features: - feature.limit = limit - request.requestDetails.extend(features) - return request - - def _response_to_df(self, feature_set, response): - entity_tables = [] - for entity_key in response.entities: - feature_tables = [] - features = response.entities[entity_key].features - for feature_name in features: - rows = [] - v_list = features[feature_name].valueList - v_list = getattr(v_list, v_list.WhichOneof("valueList")).val - for idx in range(len(v_list)): - row = {response.entityName: entity_key, - feature_name: v_list[idx]} - if features[feature_name].HasField("timestampList"): - ts_seconds = \ - features[feature_name].timestampList.val[idx].seconds - row["timestamp"] = datetime.fromtimestamp(ts_seconds) - rows.append(row) - feature_tables.append(pd.DataFrame(rows)) - entity_table = feature_tables[0] - for idx in range(1, len(feature_tables)): - entity_table = pd.merge(left=entity_table, - right=feature_tables[idx], how='outer') - entity_tables.append(entity_table) - if len(entity_tables) == 0: - return pd.DataFrame(columns=[feature_set.entity, "timestamp"] + - feature_set.features) - df = pd.concat(entity_tables) + return QueryFeaturesRequest(entityName=feature_set.entity, + entityId=entity_keys, + featureId=feature_set.features) + + def _response_to_df(self, feature_set, response, start=None, end=None): + is_filter_time = start is not None and end is not None + df = pd.DataFrame(columns=[feature_set.entity] + feature_set.features) + for entity_id in response.entities: + feature_map = response.entities[entity_id].features + row = {response.entityName: entity_id} + for feature_id in feature_map: + v = feature_map[feature_id].value + if is_filter_time and not _is_granularity_none( + feature_id): + ts = feature_map[feature_id].timestamp.ToDatetime() + if ts < start or ts > end: + continue + v = getattr(v, v.WhichOneof("val")) + row[feature_id] = v + df = df.append(row, ignore_index=True) return df.reset_index(drop=True) def _apply(self, obj): @@ -389,8 +353,9 @@ def _apply_feature(self, feature): """ self._connect_core() response = self._core_service_stub.ApplyFeature(feature.spec) - if self.verbose: print("Successfully applied feature with id: {}\n---\n{}" - .format(response.featureId, feature)) + if self.verbose: print( + "Successfully applied feature with id: {}\n---\n{}" + .format(response.featureId, feature)) return response.featureId def _apply_entity(self, entity): @@ -414,7 +379,8 @@ def _apply_feature_group(self, feature_group): feature group to apply """ self._connect_core() - response = self._core_service_stub.ApplyFeatureGroup(feature_group.spec) + response = self._core_service_stub.ApplyFeatureGroup( + feature_group.spec) if self.verbose: print("Successfully applied feature group with id: " + "{}\n---\n{}".format(response.featureGroupId, feature_group)) @@ -433,7 +399,7 @@ def _apply_storage(self, storage): return response.storageId def _check_create_dataset_args(self, feature_set, start_date, - end_date, limit): + end_date, limit): if len(feature_set.features) < 1: raise ValueError("feature set is empty") @@ -465,3 +431,7 @@ def _timestamp_from_datetime(dt): ts = Timestamp() ts.FromDatetime(dt) return ts + + +def _is_granularity_none(feature_id): + return feature_id.split(".")[1] == "none" diff --git a/sdk/python/feast/serving/Serving_pb2.py b/sdk/python/feast/serving/Serving_pb2.py index 9393a39ca7a..4f7900e2c3e 100644 --- a/sdk/python/feast/serving/Serving_pb2.py +++ b/sdk/python/feast/serving/Serving_pb2.py @@ -3,7 +3,6 @@ import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -22,73 +21,41 @@ package='feast.serving', syntax='proto3', serialized_options=_b('\n\rfeast.servingB\017ServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/serving'), - serialized_pb=_b('\n\x1b\x66\x65\x61st/serving/Serving.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\xde\x02\n\rQueryFeatures\x1a\x9c\x01\n\x07Request\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x10\n\x08\x65ntityId\x18\x02 \x03(\t\x12\x34\n\x0erequestDetails\x18\x03 \x03(\x0b\x32\x1c.feast.serving.RequestDetail\x12\x35\n\x0etimestampRange\x18\x04 \x01(\x0b\x32\x1d.feast.serving.TimestampRange\x1a\xad\x01\n\x08Response\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x45\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x33.feast.serving.QueryFeatures.Response.EntitiesEntry\x1a\x46\n\rEntitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.feast.serving.Entity:\x02\x38\x01\"[\n\rRequestDetail\x12\x11\n\tfeatureId\x18\x01 \x01(\t\x12(\n\x04type\x18\x02 \x01(\x0e\x32\x1a.feast.serving.RequestType\x12\r\n\x05limit\x18\x03 \x01(\x05\"d\n\x0eTimestampRange\x12)\n\x05start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x91\x01\n\x06\x45ntity\x12\x35\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32#.feast.serving.Entity.FeaturesEntry\x1aP\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.feast.serving.FeatureValueList:\x02\x38\x01\"p\n\x10\x46\x65\x61tureValueList\x12)\n\tvalueList\x18\x01 \x01(\x0b\x32\x16.feast.types.ValueList\x12\x31\n\rtimestampList\x18\x02 \x01(\x0b\x32\x1a.feast.types.TimestampList*!\n\x0bRequestType\x12\x08\n\x04LAST\x10\x00\x12\x08\n\x04LIST\x10\x01\x32l\n\nServingAPI\x12^\n\rQueryFeatures\x12$.feast.serving.QueryFeatures.Request\x1a%.feast.serving.QueryFeatures.Response\"\x00\x42Z\n\rfeast.servingB\x0fServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/servingb\x06proto3') + serialized_pb=_b('\n\x1b\x66\x65\x61st/serving/Serving.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"O\n\x14QueryFeaturesRequest\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x10\n\x08\x65ntityId\x18\x02 \x03(\t\x12\x11\n\tfeatureId\x18\x03 \x03(\t\"\xb9\x01\n\x15QueryFeaturesResponse\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x44\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x32.feast.serving.QueryFeaturesResponse.EntitiesEntry\x1a\x46\n\rEntitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.feast.serving.Entity:\x02\x38\x01\"\x8d\x01\n\x06\x45ntity\x12\x35\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32#.feast.serving.Entity.FeaturesEntry\x1aL\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.feast.serving.FeatureValue:\x02\x38\x01\"`\n\x0c\x46\x65\x61tureValue\x12!\n\x05value\x18\x01 \x01(\x0b\x32\x12.feast.types.Value\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp2j\n\nServingAPI\x12\\\n\rQueryFeatures\x12#.feast.serving.QueryFeaturesRequest\x1a$.feast.serving.QueryFeaturesResponse\"\x00\x42Z\n\rfeast.servingB\x0fServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/servingb\x06proto3') , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) -_REQUESTTYPE = _descriptor.EnumDescriptor( - name='RequestType', - full_name='feast.serving.RequestType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='LAST', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LIST', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=914, - serialized_end=947, -) -_sym_db.RegisterEnumDescriptor(_REQUESTTYPE) - -RequestType = enum_type_wrapper.EnumTypeWrapper(_REQUESTTYPE) -LAST = 0 -LIST = 1 -_QUERYFEATURES_REQUEST = _descriptor.Descriptor( - name='Request', - full_name='feast.serving.QueryFeatures.Request', +_QUERYFEATURESREQUEST = _descriptor.Descriptor( + name='QueryFeaturesRequest', + full_name='feast.serving.QueryFeaturesRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='entityName', full_name='feast.serving.QueryFeatures.Request.entityName', index=0, + name='entityName', full_name='feast.serving.QueryFeaturesRequest.entityName', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='entityId', full_name='feast.serving.QueryFeatures.Request.entityId', index=1, + name='entityId', full_name='feast.serving.QueryFeaturesRequest.entityId', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='requestDetails', full_name='feast.serving.QueryFeatures.Request.requestDetails', index=2, - number=3, type=11, cpp_type=10, label=3, + name='featureId', full_name='feast.serving.QueryFeaturesRequest.featureId', index=2, + number=3, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='timestampRange', full_name='feast.serving.QueryFeatures.Request.timestampRange', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -101,26 +68,27 @@ extension_ranges=[], oneofs=[ ], - serialized_start=123, - serialized_end=279, + serialized_start=104, + serialized_end=183, ) -_QUERYFEATURES_RESPONSE_ENTITIESENTRY = _descriptor.Descriptor( + +_QUERYFEATURESRESPONSE_ENTITIESENTRY = _descriptor.Descriptor( name='EntitiesEntry', - full_name='feast.serving.QueryFeatures.Response.EntitiesEntry', + full_name='feast.serving.QueryFeaturesResponse.EntitiesEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='feast.serving.QueryFeatures.Response.EntitiesEntry.key', index=0, + name='key', full_name='feast.serving.QueryFeaturesResponse.EntitiesEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='value', full_name='feast.serving.QueryFeatures.Response.EntitiesEntry.value', index=1, + name='value', full_name='feast.serving.QueryFeaturesResponse.EntitiesEntry.value', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -138,26 +106,26 @@ extension_ranges=[], oneofs=[ ], - serialized_start=385, - serialized_end=455, + serialized_start=301, + serialized_end=371, ) -_QUERYFEATURES_RESPONSE = _descriptor.Descriptor( - name='Response', - full_name='feast.serving.QueryFeatures.Response', +_QUERYFEATURESRESPONSE = _descriptor.Descriptor( + name='QueryFeaturesResponse', + full_name='feast.serving.QueryFeaturesResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='entityName', full_name='feast.serving.QueryFeatures.Response.entityName', index=0, + name='entityName', full_name='feast.serving.QueryFeaturesResponse.entityName', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='entities', full_name='feast.serving.QueryFeatures.Response.entities', index=1, + name='entities', full_name='feast.serving.QueryFeaturesResponse.entities', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -166,75 +134,7 @@ ], extensions=[ ], - nested_types=[_QUERYFEATURES_RESPONSE_ENTITIESENTRY, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=282, - serialized_end=455, -) - -_QUERYFEATURES = _descriptor.Descriptor( - name='QueryFeatures', - full_name='feast.serving.QueryFeatures', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[_QUERYFEATURES_REQUEST, _QUERYFEATURES_RESPONSE, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=105, - serialized_end=455, -) - - -_REQUESTDETAIL = _descriptor.Descriptor( - name='RequestDetail', - full_name='feast.serving.RequestDetail', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='featureId', full_name='feast.serving.RequestDetail.featureId', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='type', full_name='feast.serving.RequestDetail.type', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='limit', full_name='feast.serving.RequestDetail.limit', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], + nested_types=[_QUERYFEATURESRESPONSE_ENTITIESENTRY, ], enum_types=[ ], serialized_options=None, @@ -243,46 +143,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=457, - serialized_end=548, -) - - -_TIMESTAMPRANGE = _descriptor.Descriptor( - name='TimestampRange', - full_name='feast.serving.TimestampRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start', full_name='feast.serving.TimestampRange.start', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='feast.serving.TimestampRange.end', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=550, - serialized_end=650, + serialized_start=186, + serialized_end=371, ) @@ -319,8 +181,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=718, - serialized_end=798, + serialized_start=439, + serialized_end=515, ) _ENTITY = _descriptor.Descriptor( @@ -349,27 +211,27 @@ extension_ranges=[], oneofs=[ ], - serialized_start=653, - serialized_end=798, + serialized_start=374, + serialized_end=515, ) -_FEATUREVALUELIST = _descriptor.Descriptor( - name='FeatureValueList', - full_name='feast.serving.FeatureValueList', +_FEATUREVALUE = _descriptor.Descriptor( + name='FeatureValue', + full_name='feast.serving.FeatureValue', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='valueList', full_name='feast.serving.FeatureValueList.valueList', index=0, + name='value', full_name='feast.serving.FeatureValue.value', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='timestampList', full_name='feast.serving.FeatureValueList.timestampList', index=1, + name='timestamp', full_name='feast.serving.FeatureValue.timestamp', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -387,77 +249,45 @@ extension_ranges=[], oneofs=[ ], - serialized_start=800, - serialized_end=912, + serialized_start=517, + serialized_end=613, ) -_QUERYFEATURES_REQUEST.fields_by_name['requestDetails'].message_type = _REQUESTDETAIL -_QUERYFEATURES_REQUEST.fields_by_name['timestampRange'].message_type = _TIMESTAMPRANGE -_QUERYFEATURES_REQUEST.containing_type = _QUERYFEATURES -_QUERYFEATURES_RESPONSE_ENTITIESENTRY.fields_by_name['value'].message_type = _ENTITY -_QUERYFEATURES_RESPONSE_ENTITIESENTRY.containing_type = _QUERYFEATURES_RESPONSE -_QUERYFEATURES_RESPONSE.fields_by_name['entities'].message_type = _QUERYFEATURES_RESPONSE_ENTITIESENTRY -_QUERYFEATURES_RESPONSE.containing_type = _QUERYFEATURES -_REQUESTDETAIL.fields_by_name['type'].enum_type = _REQUESTTYPE -_TIMESTAMPRANGE.fields_by_name['start'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -_TIMESTAMPRANGE.fields_by_name['end'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -_ENTITY_FEATURESENTRY.fields_by_name['value'].message_type = _FEATUREVALUELIST +_QUERYFEATURESRESPONSE_ENTITIESENTRY.fields_by_name['value'].message_type = _ENTITY +_QUERYFEATURESRESPONSE_ENTITIESENTRY.containing_type = _QUERYFEATURESRESPONSE +_QUERYFEATURESRESPONSE.fields_by_name['entities'].message_type = _QUERYFEATURESRESPONSE_ENTITIESENTRY +_ENTITY_FEATURESENTRY.fields_by_name['value'].message_type = _FEATUREVALUE _ENTITY_FEATURESENTRY.containing_type = _ENTITY _ENTITY.fields_by_name['features'].message_type = _ENTITY_FEATURESENTRY -_FEATUREVALUELIST.fields_by_name['valueList'].message_type = feast_dot_types_dot_Value__pb2._VALUELIST -_FEATUREVALUELIST.fields_by_name['timestampList'].message_type = feast_dot_types_dot_Value__pb2._TIMESTAMPLIST -DESCRIPTOR.message_types_by_name['QueryFeatures'] = _QUERYFEATURES -DESCRIPTOR.message_types_by_name['RequestDetail'] = _REQUESTDETAIL -DESCRIPTOR.message_types_by_name['TimestampRange'] = _TIMESTAMPRANGE +_FEATUREVALUE.fields_by_name['value'].message_type = feast_dot_types_dot_Value__pb2._VALUE +_FEATUREVALUE.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +DESCRIPTOR.message_types_by_name['QueryFeaturesRequest'] = _QUERYFEATURESREQUEST +DESCRIPTOR.message_types_by_name['QueryFeaturesResponse'] = _QUERYFEATURESRESPONSE DESCRIPTOR.message_types_by_name['Entity'] = _ENTITY -DESCRIPTOR.message_types_by_name['FeatureValueList'] = _FEATUREVALUELIST -DESCRIPTOR.enum_types_by_name['RequestType'] = _REQUESTTYPE +DESCRIPTOR.message_types_by_name['FeatureValue'] = _FEATUREVALUE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -QueryFeatures = _reflection.GeneratedProtocolMessageType('QueryFeatures', (_message.Message,), dict( - - Request = _reflection.GeneratedProtocolMessageType('Request', (_message.Message,), dict( - DESCRIPTOR = _QUERYFEATURES_REQUEST, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures.Request) - )) - , +QueryFeaturesRequest = _reflection.GeneratedProtocolMessageType('QueryFeaturesRequest', (_message.Message,), dict( + DESCRIPTOR = _QUERYFEATURESREQUEST, + __module__ = 'feast.serving.Serving_pb2' + # @@protoc_insertion_point(class_scope:feast.serving.QueryFeaturesRequest) + )) +_sym_db.RegisterMessage(QueryFeaturesRequest) - Response = _reflection.GeneratedProtocolMessageType('Response', (_message.Message,), dict( +QueryFeaturesResponse = _reflection.GeneratedProtocolMessageType('QueryFeaturesResponse', (_message.Message,), dict( - EntitiesEntry = _reflection.GeneratedProtocolMessageType('EntitiesEntry', (_message.Message,), dict( - DESCRIPTOR = _QUERYFEATURES_RESPONSE_ENTITIESENTRY, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures.Response.EntitiesEntry) - )) - , - DESCRIPTOR = _QUERYFEATURES_RESPONSE, + EntitiesEntry = _reflection.GeneratedProtocolMessageType('EntitiesEntry', (_message.Message,), dict( + DESCRIPTOR = _QUERYFEATURESRESPONSE_ENTITIESENTRY, __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures.Response) + # @@protoc_insertion_point(class_scope:feast.serving.QueryFeaturesResponse.EntitiesEntry) )) , - DESCRIPTOR = _QUERYFEATURES, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures) - )) -_sym_db.RegisterMessage(QueryFeatures) -_sym_db.RegisterMessage(QueryFeatures.Request) -_sym_db.RegisterMessage(QueryFeatures.Response) -_sym_db.RegisterMessage(QueryFeatures.Response.EntitiesEntry) - -RequestDetail = _reflection.GeneratedProtocolMessageType('RequestDetail', (_message.Message,), dict( - DESCRIPTOR = _REQUESTDETAIL, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.RequestDetail) - )) -_sym_db.RegisterMessage(RequestDetail) - -TimestampRange = _reflection.GeneratedProtocolMessageType('TimestampRange', (_message.Message,), dict( - DESCRIPTOR = _TIMESTAMPRANGE, + DESCRIPTOR = _QUERYFEATURESRESPONSE, __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.TimestampRange) + # @@protoc_insertion_point(class_scope:feast.serving.QueryFeaturesResponse) )) -_sym_db.RegisterMessage(TimestampRange) +_sym_db.RegisterMessage(QueryFeaturesResponse) +_sym_db.RegisterMessage(QueryFeaturesResponse.EntitiesEntry) Entity = _reflection.GeneratedProtocolMessageType('Entity', (_message.Message,), dict( @@ -474,16 +304,16 @@ _sym_db.RegisterMessage(Entity) _sym_db.RegisterMessage(Entity.FeaturesEntry) -FeatureValueList = _reflection.GeneratedProtocolMessageType('FeatureValueList', (_message.Message,), dict( - DESCRIPTOR = _FEATUREVALUELIST, +FeatureValue = _reflection.GeneratedProtocolMessageType('FeatureValue', (_message.Message,), dict( + DESCRIPTOR = _FEATUREVALUE, __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.FeatureValueList) + # @@protoc_insertion_point(class_scope:feast.serving.FeatureValue) )) -_sym_db.RegisterMessage(FeatureValueList) +_sym_db.RegisterMessage(FeatureValue) DESCRIPTOR._options = None -_QUERYFEATURES_RESPONSE_ENTITIESENTRY._options = None +_QUERYFEATURESRESPONSE_ENTITIESENTRY._options = None _ENTITY_FEATURESENTRY._options = None _SERVINGAPI = _descriptor.ServiceDescriptor( @@ -492,16 +322,16 @@ file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=949, - serialized_end=1057, + serialized_start=615, + serialized_end=721, methods=[ _descriptor.MethodDescriptor( name='QueryFeatures', full_name='feast.serving.ServingAPI.QueryFeatures', index=0, containing_service=None, - input_type=_QUERYFEATURES_REQUEST, - output_type=_QUERYFEATURES_RESPONSE, + input_type=_QUERYFEATURESREQUEST, + output_type=_QUERYFEATURESRESPONSE, serialized_options=None, ), ]) diff --git a/sdk/python/feast/serving/Serving_pb2_grpc.py b/sdk/python/feast/serving/Serving_pb2_grpc.py index a17c7b6305d..d3a8da4b4d5 100644 --- a/sdk/python/feast/serving/Serving_pb2_grpc.py +++ b/sdk/python/feast/serving/Serving_pb2_grpc.py @@ -16,8 +16,8 @@ def __init__(self, channel): """ self.QueryFeatures = channel.unary_unary( '/feast.serving.ServingAPI/QueryFeatures', - request_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Request.SerializeToString, - response_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Response.FromString, + request_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesRequest.SerializeToString, + response_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesResponse.FromString, ) @@ -26,7 +26,7 @@ class ServingAPIServicer(object): pass def QueryFeatures(self, request, context): - """Query features from Feast + """Query features from Feast serving storage """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -37,8 +37,8 @@ def add_ServingAPIServicer_to_server(servicer, server): rpc_method_handlers = { 'QueryFeatures': grpc.unary_unary_rpc_method_handler( servicer.QueryFeatures, - request_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Request.FromString, - response_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Response.SerializeToString, + request_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesRequest.FromString, + response_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index 81909349244..c13c350ed43 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -17,6 +17,7 @@ import grpc import pandas as pd import pytest +import numpy as np from google.protobuf.timestamp_pb2 import Timestamp from pandas.util.testing import assert_frame_equal @@ -29,7 +30,6 @@ from feast.core.DatasetService_pb2 import DatasetServiceTypes from feast.core.DatasetService_pb2 import DatasetInfo as DatasetInfo_pb from feast.sdk.client import Client, _parse_date, _timestamp_from_datetime -from feast.sdk.client import ServingRequestType from feast.sdk.importer import Importer from feast.sdk.resources.entity import Entity from feast.sdk.resources.feature import Feature, Granularity @@ -37,12 +37,12 @@ from feast.sdk.resources.feature_set import FeatureSet, DatasetInfo, FileType from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader -from feast.serving.Serving_pb2 import QueryFeatures, RequestDetail, \ - TimestampRange, FeatureValueList +from feast.serving.Serving_pb2 import QueryFeaturesRequest, \ + QueryFeaturesResponse, FeatureValue from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore from feast.specs.ImportSpec_pb2 import ImportSpec from feast.specs.StorageSpec_pb2 import StorageSpec -from feast.types.Value_pb2 import ValueList, TimestampList, Int32List +from feast.types.Value_pb2 import Value class TestClient(object): @@ -146,26 +146,26 @@ def test_create_dataset_invalid_args(self, client): with pytest.raises(ValueError, match="feature set is empty"): inv_feature_set = FeatureSet("entity", []) client.create_dataset(inv_feature_set, "2018-12-01", - "2018-12-02") + "2018-12-02") # invalid start date with pytest.raises(ValueError, match="Incorrect date format, should be YYYY-MM-DD"): client.create_dataset(feature_set, "20181201", - "2018-12-02") + "2018-12-02") # invalid end date with pytest.raises(ValueError, match="Incorrect date format, should be YYYY-MM-DD"): client.create_dataset(feature_set, "2018-12-01", - "20181202") + "20181202") # start date > end date with pytest.raises(ValueError, match="end_date is before start_date"): client.create_dataset(feature_set, "2018-12-02", - "2018-12-01") + "2018-12-01") # invalid limit with pytest.raises(ValueError, match="limit is not a positive integer"): client.create_dataset(feature_set, "2018-12-01", - "2018-12-02", -1) + "2018-12-02", -1) def test_create_dataset(self, client, mocker): entity_name = "myentity" @@ -175,7 +175,7 @@ def test_create_dataset(self, client, mocker): end_date = "2018-12-31" ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + tableUrl="project.dataset.table") mock_trn_stub = training.DatasetServiceStub(grpc.insecure_channel("")) mocker.patch.object(mock_trn_stub, "CreateDataset", @@ -206,7 +206,7 @@ def test_create_dataset_with_limit(self, client, mocker): limit = 100 ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + tableUrl="project.dataset.table") mock_trn_stub = training.DatasetServiceStub(grpc.insecure_channel("")) mocker.patch.object(mock_trn_stub, "CreateDataset", @@ -215,7 +215,7 @@ def test_create_dataset_with_limit(self, client, mocker): client._dataset_service_stub = mock_trn_stub ds = client.create_dataset(fs, start_date, end_date, - limit=limit) + limit=limit) assert "dataset_name" == ds.name assert "project.dataset.table" == ds.table_id @@ -239,17 +239,18 @@ def test_create_dataset_with_name_prefix(self, client, mocker): name_prefix = "feast" ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + tableUrl="project.dataset.table") - mock_dssvc_stub = training.DatasetServiceStub(grpc.insecure_channel("")) + mock_dssvc_stub = training.DatasetServiceStub( + grpc.insecure_channel("")) mocker.patch.object(mock_dssvc_stub, "CreateDataset", return_value=DatasetServiceTypes .CreateDatasetResponse(datasetInfo=ds_pb)) client._dataset_service_stub = mock_dssvc_stub ds = client.create_dataset(fs, start_date, end_date, - limit=limit, - name_prefix=name_prefix) + limit=limit, + name_prefix=name_prefix) assert "dataset_name" == ds.name assert "project.dataset.table" == ds.table_id @@ -263,97 +264,149 @@ def test_create_dataset_with_name_prefix(self, client, mocker): ) ) - def test_build_serving_request_last(self, client): + def test_build_serving_request(self, client): feature_set = FeatureSet("entity", ["entity.none.feat1", "entity.none.feat2"]) - req = client._build_serving_request(feature_set, ["1", "2", "3"], - ServingRequestType.LAST, None, - None) - expected_request_detail = \ - [RequestDetail(featureId=feat_id, type=serving_pb.LAST) - for feat_id in ["entity.none.feat1", "entity.none.feat2"]] - expected = QueryFeatures.Request(entityName="entity", - entityId=["1", "2", "3"], - requestDetails=expected_request_detail) - assert req == expected - - def test_build_serving_request_list(self, client): - feature_set = FeatureSet("entity", - ["entity.none.feat1", "entity.none.feat2"]) - - req = client._build_serving_request( - feature_set, ["1", "2", "3"], ServingRequestType.LIST, - [Timestamp(seconds=10), Timestamp(seconds=11)], 2) - expected_request_detail = \ - [RequestDetail(featureId=feat_id, type=serving_pb.LIST, limit=2) - for feat_id in ["entity.none.feat1", "entity.none.feat2"]] - expected_ts_range = \ - TimestampRange(start=Timestamp(seconds=10), - end=Timestamp(seconds=11)) - expected = QueryFeatures.Request(entityName="entity", - entityId=["1", "2", "3"], - requestDetails=expected_request_detail, - timestampRange=expected_ts_range) + req = client._build_serving_request(feature_set, ["1", "2", "3"]) + expected = QueryFeaturesRequest(entityName="entity", + entityId=["1", "2", "3"], + featureId=feature_set.features) assert req == expected def test_serving_response_to_df(self, client): response = self._create_query_features_response( entity_name="entity", - entities=[{"id": "1", - "feat1": [(3, Timestamp(seconds=10))], - "feat2": [(1, Timestamp(seconds=10))]}, - {"id": "2", - "feat1": [(3, Timestamp(seconds=10))], - "feat2": [(3, Timestamp(seconds=10))]}], - features=["feat1", "feat2"] + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1)), + "entity.feat2": (2, Timestamp(seconds=2))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3)), + "entity.feat2": (4, Timestamp(seconds=4))}} ) expected_df = pd.DataFrame({'entity': ["1", "2"], - 'timestamp': [datetime.fromtimestamp(10), - datetime.fromtimestamp(10)], - 'feat1': [3, 3], - 'feat2': [1, 3]}) \ + 'entity.feat1': [1, 3], + 'entity.feat2': [2, 4]}) \ .reset_index(drop=True) - df = client._response_to_df(FeatureSet("", []), response) \ - .sort_values(['entity', 'timestamp']) \ + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response) \ + .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) - def test_serving_response_to_df_multiple(self, client): + def test_serving_response_to_df_with_missing_value(self, client): response = self._create_query_features_response( entity_name="entity", - entities=[{"id": "1", - "feat1": [(3, Timestamp(seconds=10)), - (4, Timestamp(seconds=11))], - "feat2": [(1, Timestamp(seconds=10)), - (3, Timestamp(seconds=11))]}, - {"id": "2", - "feat1": [(4, Timestamp(seconds=10)), - (6, Timestamp(seconds=11))], - "feat2": [(2, Timestamp(seconds=10)), - (5, Timestamp(seconds=11))]}], - features=["feat1", "feat2"] + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3)), + "entity.feat2": (4, Timestamp(seconds=4))}} ) - expected_df = pd.DataFrame({'entity': ["1", "1", "2", "2"], - 'timestamp': [datetime.fromtimestamp(10), - datetime.fromtimestamp(11), - datetime.fromtimestamp(10), - datetime.fromtimestamp(11)], - 'feat1': [3, 4, 4, 6], - 'feat2': [1, 3, 2, 5]}) \ + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.feat1': [1, 3], + 'entity.feat2': [np.NaN, 4]}) \ + .reset_index(drop=True) + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_response_to_df_with_missing_feature(self, client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.feat1': [1, 3], + 'entity.feat2': [np.NaN, np.NaN]}) \ .reset_index(drop=True) - df = client._response_to_df(FeatureSet("", []), response) \ - .sort_values(['entity', 'timestamp']) \ + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response) \ + .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_response_to_df_no_data(self, client): - response = QueryFeatures.Response(entityName="entity") + response = QueryFeaturesResponse(entityName="entity") expected_df = pd.DataFrame(columns= - ['entity', 'timestamp', 'entity.day.feat1', 'entity.day.feat2']) + ['entity', 'entity.day.feat1', + 'entity.day.feat2']) df = client._response_to_df(FeatureSet("entity", - ["entity.day.feat1", "entity.day.feat2"]), response) - assert_frame_equal(df, expected_df) + ["entity.day.feat1", + "entity.day.feat2"]), response) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_response_to_df_with_time_filter(self, client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.feat1': [np.NaN, 3], + 'entity.feat2': [np.NaN, np.NaN]}) \ + .reset_index(drop=True) + start = datetime.utcfromtimestamp(2) + end = datetime.utcfromtimestamp(5) + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response, start, end) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + print(df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_response_to_df_with_time_filter_granularity_none(self, + client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.none.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.none.feat1": (3, Timestamp(seconds=3))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.none.feat1': [1, 3], + 'entity.none.feat2': [np.NaN, np.NaN]}) \ + .reset_index(drop=True) + start = datetime.utcfromtimestamp(2) + end = datetime.utcfromtimestamp(5) + df = client._response_to_df(FeatureSet("entity", ["entity.none.feat1", + "entity.none.feat2"]), + response, start, end) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + print(df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_invalid_type(self, client): + start = "2018-01-01T01:01:01" + end = "2018-01-01T01:01:01" + ts_range = [start, end] + with pytest.raises(TypeError, match="start and end must be datetime " + "type"): + client.get_serving_data(FeatureSet("entity", ["entity.none.feat1", + "entity.none.feat2"]), + ["1234", "5678"], + ts_range) def test_download_dataset_as_file(self, client, mocker): destination = "/tmp/dest_file" @@ -377,25 +430,16 @@ def test_download_dataset_as_file(self, client, mocker): staging_location, FileType.CSV) - def _create_query_features_response(self, entity_name, entities, features): - # entities should be in the form: - # {"id": "1", - # "feature_1": [(val1, ts1), (val2, ts2)], - # "feature_2": ...} - response = QueryFeatures.Response(entityName=entity_name) - for entity in entities: - features_map = {} - for feature in features: - features_map[feature] = FeatureValueList( - valueList=ValueList( - int32List=Int32List( - val=[v[0] for v in entity[feature]]) - ), - timestampList=TimestampList( - val=[v[1] for v in entity[feature]]) - ) - entity_pb = serving_pb.Entity(features=features_map) - response.entities[entity["id"]].CopyFrom(entity_pb) + def _create_query_features_response(self, entity_name, entities): + response = QueryFeaturesResponse(entityName=entity_name) + for entity_id, feature_map in entities.items(): + feature = {} + for feature_id, feature_value in feature_map.items(): + feature[feature_id] = FeatureValue( + value=Value(int32Val=feature_value[0]), + timestamp=feature_value[1]) + entity_pb = serving_pb.Entity(features=feature) + response.entities[entity_id].CopyFrom(entity_pb) return response diff --git a/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java b/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java index 6877ae33a36..51622d8ea65 100644 --- a/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java +++ b/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java @@ -17,16 +17,14 @@ package feast.serving.grpc; -import static feast.serving.util.RequestHelper.checkTimestampRange; import static feast.serving.util.RequestHelper.validateRequest; import static feast.serving.util.StatsUtil.makeStatsdTags; import static io.grpc.Status.Code.INTERNAL; -import static io.grpc.Status.Code.NOT_FOUND; import com.timgroup.statsd.StatsDClient; import feast.serving.ServingAPIGrpc.ServingAPIImplBase; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.serving.service.FeastServing; import feast.serving.util.TimeUtil; import io.grpc.Status; @@ -39,9 +37,7 @@ import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; -/** - * Grpc service implementation for Serving API. - */ +/** Grpc service implementation for Serving API. */ @Slf4j @GRpcService public class ServingGrpcService extends ServingAPIImplBase { @@ -57,25 +53,19 @@ public ServingGrpcService(FeastServing feast, Tracer tracer, StatsDClient statsD this.statsDClient = statsDClient; } - /** - * Query feature values from Feast storage. - */ + /** Query feature values from Feast storage. */ @Override - public void queryFeatures(Request request, StreamObserver responseObserver) { + public void queryFeatures(QueryFeaturesRequest request, StreamObserver responseObserver) { long currentMicro = TimeUtil.microTime(); - Span span = - tracer - .buildSpan("ServingGrpcService-queryFeatures") - .start(); + Span span = tracer.buildSpan("ServingGrpcService-queryFeatures").start(); String[] tags = makeStatsdTags(request); statsDClient.increment("query_features_count", tags); statsDClient.gauge("query_features_entity_count", request.getEntityIdCount(), tags); - statsDClient.gauge("query_features_feature_count", request.getRequestDetailsCount(), tags); + statsDClient.gauge("query_features_feature_count", request.getFeatureIdCount(), tags); try (Scope scope = tracer.scopeManager().activate(span, false)) { Span innerSpan = scope.span(); validateRequest(request); - Request validRequest = checkTimestampRange(request); - Response response = feast.queryFeatures(validRequest); + QueryFeaturesResponse response = feast.queryFeatures(request); innerSpan.log("calling onNext"); responseObserver.onNext(response); @@ -90,8 +80,7 @@ public void queryFeatures(Request request, StreamObserver responseObse new StatusRuntimeException( Status.fromCode(INTERNAL).withDescription(e.getMessage()).withCause(e))); } finally { - statsDClient - .gauge("query_features_latency_us", TimeUtil.microTime() - currentMicro, tags); + statsDClient.gauge("query_features_latency_us", TimeUtil.microTime() - currentMicro, tags); span.finish(); } } diff --git a/serving/src/main/java/feast/serving/http/ServingHttpService.java b/serving/src/main/java/feast/serving/http/ServingHttpService.java index ef50d45bcd6..48164a9d495 100644 --- a/serving/src/main/java/feast/serving/http/ServingHttpService.java +++ b/serving/src/main/java/feast/serving/http/ServingHttpService.java @@ -17,17 +17,16 @@ package feast.serving.http; -import feast.serving.ServingAPIProto.QueryFeatures; -import feast.serving.ServingAPIProto.QueryFeatures.Request; +import static feast.serving.util.RequestHelper.validateRequest; + +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.serving.service.FeastServing; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import static feast.serving.util.RequestHelper.checkTimestampRange; -import static feast.serving.util.RequestHelper.validateRequest; - /** HTTP endpoint for Serving API. */ @RestController public class ServingHttpService { @@ -42,9 +41,8 @@ public ServingHttpService(FeastServing feastServing) { value = "/api/v1/features/request", produces = "application/json", consumes = "application/json") - public QueryFeatures.Response queryFeature(@RequestBody QueryFeatures.Request request) { + public QueryFeaturesResponse queryFeature(@RequestBody QueryFeaturesRequest request) { validateRequest(request); - Request validRequest = checkTimestampRange(request); - return feastServing.queryFeatures(validRequest); + return feastServing.queryFeatures(request); } } diff --git a/serving/src/main/java/feast/serving/model/Pair.java b/serving/src/main/java/feast/serving/model/Pair.java deleted file mode 100644 index ac4035ee380..00000000000 --- a/serving/src/main/java/feast/serving/model/Pair.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.model; - -import lombok.EqualsAndHashCode; - -/** - * Pair value containing L-typed {@code left} and R-typed {@code right}. - * - * @param type of the left value. - * @param type of the right value. - */ -@EqualsAndHashCode -public class Pair { - private final L left; - private final R right; - - public Pair(L left, R right) { - this.left = left; - this.right = right; - } - - public L getLeft() { - return left; - } - - public R getRight() { - return right; - } -} diff --git a/serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java b/serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java deleted file mode 100644 index d98b1828496..00000000000 --- a/serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.model; - -import feast.serving.ServingAPIProto.RequestDetail; -import feast.specs.FeatureSpecProto.FeatureSpec; -import lombok.AllArgsConstructor; -import lombok.Data; - -/** Composite type to hold information about request detail and the associated feature spec. */ -@Data -@AllArgsConstructor -public class RequestDetailWithSpec { - RequestDetail requestDetail; - FeatureSpec featureSpec; -} diff --git a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java index 50d2b775d61..a9909858392 100644 --- a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java @@ -20,21 +20,15 @@ import com.google.common.base.Strings; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.BigTableProto.BigTableRowKey; -import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.Value; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.hbase.Cell; @@ -42,44 +36,28 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.filter.MultiRowRangeFilter; -import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; /** Connector to BigTable instance. */ @Slf4j public class BigTableFeatureStorage implements FeatureStorage { public static final String TYPE = "bigtable"; - + private static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(); public static String OPT_BIGTABLE_PROJECT = "project"; public static String OPT_BIGTABLE_INSTANCE = "instance"; public static String OPT_BIGTABLE_TABLE_PREFIX = "tablePrefix"; - public static String SERVING_OPT_BIGTABLE_COLUMN_FAMILY = "family"; - private final Connection connection; - private static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(); - public BigTableFeatureStorage(Connection connection) { this.connection = connection; } /** {@inheritDoc} */ @Override - public List getCurrentFeature( - String entityName, List entityIds, FeatureSpec featureSpec) - throws FeatureRetrievalException { - return getCurrentFeatureInternal(entityName, entityIds, featureSpec); - } - - /** {@inheritDoc} */ - @Override - public List getCurrentFeatures( - String entityName, List entityIds, List featureSpecs) { + public List getFeature( + String entityName, Collection entityIds, Collection featureSpecs) { List featureValues = new ArrayList<>(entityIds.size() * featureSpecs.size()); for (FeatureSpec featureSpec : featureSpecs) { featureValues.addAll(getCurrentFeatureInternal(entityName, entityIds, featureSpec)); @@ -87,39 +65,6 @@ public List getCurrentFeatures( return featureValues; } - /** {@inheritDoc} */ - @Override - public List getNLatestFeatureWithinTimestampRange( - String entityName, - List entityIds, - Pair featureSpecLimitPair, - TimestampRange tsRange) - throws FeatureRetrievalException { - FeatureSpec featureSpec = featureSpecLimitPair.getLeft(); - int limit = featureSpecLimitPair.getRight(); - - return getNLatestFeatureWithinTimestampRangeInternal( - entityName, entityIds, featureSpec, limit, tsRange); - } - - /** {@inheritDoc} */ - @Override - public List getNLatestFeaturesWithinTimestampRange( - String entityName, - List entityIds, - List> featureSpecAndLimitPairs, - TimestampRange tsRange) { - List featureValues = new ArrayList<>(entityIds.size() * featureSpecAndLimitPairs.size()); - for(Pair featureSpecLimitPair: featureSpecAndLimitPairs) { - FeatureSpec featureSpec = featureSpecLimitPair.getLeft(); - int limit = featureSpecLimitPair.getRight(); - - featureValues.addAll(getNLatestFeatureWithinTimestampRangeInternal( - entityName, entityIds, featureSpec, limit, tsRange)); - } - return featureValues; - } - /** * Internal implementation of get current value of a feature for a list of entity Ids. * @@ -129,11 +74,11 @@ public List getNLatestFeaturesWithinTimestampRange( * @return list of feature value. */ private List getCurrentFeatureInternal( - String entityName, List entityIds, FeatureSpec featureSpec) { + String entityName, Collection entityIds, FeatureSpec featureSpec) { List features = new ArrayList<>(entityIds.size()); String featureId = featureSpec.getId(); byte[] featureIdBytes = featureSpec.getId().getBytes(); - List gets = createGets(entityIds, featureIdBytes, getColumnFamily(featureSpec)); + List gets = createGets(entityIds, featureSpec); try (Table table = connection.getTable(TableName.valueOf(entityName))) { Result[] results = table.get(gets); for (Result result : results) { @@ -168,76 +113,16 @@ private List getCurrentFeatureInternal( } /** - * Internal implementation of getting N latest feature within certain time range given a list of entity ID. - * - * @param entityName entity name - * @param entityIds list of entity id for which the feature should be retrieved. - * @param featureSpec spec of the feature. - * @param limit maximum number of value to be retrieved per feature. - * @param tsRange timestamp range. - * @return list of feature value. - */ - private List getNLatestFeatureWithinTimestampRangeInternal( - String entityName, - List entityIds, - FeatureSpec featureSpec, - int limit, - TimestampRange tsRange) { - Scan scan = createScanFromFeatureSpec(featureSpec); - Granularity.Enum granularity = featureSpec.getGranularity(); - String featureId = featureSpec.getId(); - byte[] featureIdBytes = featureSpec.getId().getBytes(); - String revMilliStart = roundedReversedMillis(tsRange.getStart(), granularity); - String revMilliEnd = roundedReversedMillis(tsRange.getEnd(), granularity); - scan.setFilter(createMultiRowRangeFilter(entityIds, revMilliStart, revMilliEnd)); - - List features = new ArrayList<>(); - Map featureCount = new HashMap<>(); // map of entity ID to number of feature. - try (Table table = connection.getTable(TableName.valueOf(entityName)); - ResultScanner scanResult = table.getScanner(scan)) { - for (Result row : scanResult) { - Cell currentCell = row.getColumnLatestCell(getColumnFamily(featureSpec), featureIdBytes); - if (currentCell == null) { - continue; - } - - BigTableRowKey rowKey = BigTableRowKey.parseFrom(currentCell.getRowArray()); - String entityId = rowKey.getEntityKey(); - byte[] cellValue = currentCell.getValueArray(); - - if (cellValue == null) { - continue; - } - - Integer count = featureCount.computeIfAbsent(entityId, k -> 0); - if (count < limit) { - Timestamp timestamp = Timestamps.fromMillis(currentCell.getTimestamp()); - Value value = Value.parseFrom(cellValue); - FeatureValue featureValue = new FeatureValue(featureId, entityId, value, timestamp); - features.add(featureValue); - - count++; - featureCount.put(entityId, count); - } - } - - return features; - } catch (IOException e) { - log.error("Error while retrieving feature: {}", e); - throw new FeatureRetrievalException("Unable to retrieve feature from BigTable", e); - } - } - - /** - * Create list of BigTable's Get operation. + * Create list of get operation for retrieving a feature of several entities optionally filtered + * by its timestamp. * * @param entityIds list of entity ID. - * @param featureIdBytes byte array value of a feature ID. - * @param columnFamily byte array value of column family + * @param featureSpec feature spec * @return list of Get operation. */ - private List createGets( - Collection entityIds, byte[] featureIdBytes, byte[] columnFamily) { + private List createGets(Collection entityIds, FeatureSpec featureSpec) { + byte[] featureIdBytes = featureSpec.getId().getBytes(); + byte[] columnFamily = getColumnFamily(featureSpec); List gets = new ArrayList<>(); for (String entityId : entityIds) { String entityIdPrefix = DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7); @@ -245,7 +130,6 @@ private List createGets( Get get = new Get(btKey.toByteArray()); get.addColumn(columnFamily, featureIdBytes); try { - // for some reason Get.readVersions has checked exception. get.readVersions(1); } catch (IOException e) { log.error("should not happen"); @@ -255,40 +139,6 @@ private List createGets( return gets; } - /** - * Create BigTable's scan operation for a certain feature. - * - * @param featureSpec spec of feature. - * @return BigTable's scan operation. - */ - private Scan createScanFromFeatureSpec(FeatureSpec featureSpec) { - Scan scan = new Scan(); - byte[] columnFamily = getColumnFamily(featureSpec); - scan.addColumn(columnFamily, featureSpec.getId().getBytes()); - scan.readVersions(1); - return scan; - } - - /** - * Create BigTables's multi row range filter to scan several entity ID within a timerange - * - * @param entityIds list of entity id to be scanned. - * @param reversedMilliStart starting timerange. - * @param reversedMillisEnd end timerange. - * @return instance of {@link MultiRowRangeFilter}. - */ - private MultiRowRangeFilter createMultiRowRangeFilter( - Collection entityIds, String reversedMilliStart, String reversedMillisEnd) { - List rowRangeList = new ArrayList<>(); - for (String entityId : entityIds) { - String entityIdPrefix = DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7); - BigTableRowKey startRow = createRowKey(entityIdPrefix, entityId, reversedMillisEnd); - BigTableRowKey stopRow = createRowKey(entityIdPrefix, entityId, reversedMilliStart); - rowRangeList.add(new RowRange(startRow.toByteArray(), true, stopRow.toByteArray(), true)); - } - return new MultiRowRangeFilter(rowRangeList); - } - /** * Create BigTableRowKey based on entityId, timestamp, and granularity. * @@ -306,23 +156,9 @@ private BigTableRowKey createRowKey( .build(); } - /** - * Get rounded and reversed millis value of a timestamp. - * - * @param ts timestamp - * @param granularity granularity of feature. - * @return string value of timestamp after being rounded and reversed. - */ - private String roundedReversedMillis(Timestamp ts, Granularity.Enum granularity) { - if (granularity.equals(Granularity.Enum.NONE)) { - return "0"; - } - Timestamp roundedEnd = TimeUtil.roundFloorTimestamp(ts, granularity); - return String.valueOf(Long.MAX_VALUE - roundedEnd.getSeconds() * 1000); - } - /** * Get column family of a feature from its spec. + * * @param fs feature's spec * @return byte array value of the column family. */ diff --git a/serving/src/main/java/feast/serving/service/FeastServing.java b/serving/src/main/java/feast/serving/service/FeastServing.java index 1194c3cf1ee..0e1ee830abc 100644 --- a/serving/src/main/java/feast/serving/service/FeastServing.java +++ b/serving/src/main/java/feast/serving/service/FeastServing.java @@ -17,11 +17,10 @@ package feast.serving.service; +import com.google.common.collect.Sets; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.model.RequestDetailWithSpec; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; import io.opentracing.Scope; @@ -66,15 +65,15 @@ public FeastServing( * @param request feature query request. * @return response of the query containing the feature values. */ - public Response queryFeatures(Request request) { + public QueryFeaturesResponse queryFeatures(QueryFeaturesRequest request) { try (Scope scope = tracer.buildSpan("FeastServing-queryFeatures").startActive(true)) { - List requestDetails = joinRequestDetailsWithFeatureSpec(request); + Collection featureSpecs = getFeatureSpecs(request.getFeatureIdList()); // create connection to feature storage if necessary checkAndConnectFeatureStorage( - requestDetails + featureSpecs .stream() - .map(r -> r.getFeatureSpec().getDataStores().getServing().getId()) + .map(featureSpec -> featureSpec.getDataStores().getServing().getId()) .collect(Collectors.toList())); scope.span().log("start retrieving all feature"); @@ -82,13 +81,12 @@ public Response queryFeatures(Request request) { featureRetrievalDispatcher.dispatchFeatureRetrieval( request.getEntityName(), request.getEntityIdList(), - requestDetails, - request.getTimestampRange()); + featureSpecs); scope.span().log("finished retrieving all feature"); // build response - return Response.newBuilder() + return QueryFeaturesResponse.newBuilder() .setEntityName(request.getEntityName()) .putAllEntities(result) .build(); @@ -121,23 +119,14 @@ private void checkAndConnectFeatureStorage(Collection storageIds) { /** * Attach request details with associated feature spec. * - * @param request - * @return + * @param featureIds collection of feature ID + * @return collection of feature spec */ - private List joinRequestDetailsWithFeatureSpec(Request request) { + private Collection getFeatureSpecs(Collection featureIds) { // dedup feature ID. - Collection featuresId = - request - .getRequestDetailsList() - .stream() - .map(RequestDetail::getFeatureId) - .collect(Collectors.toSet()); + Collection featureIdSet = Sets.newHashSet(featureIds); - Map featureSpecMap = specStorage.getFeatureSpecs(featuresId); - return request - .getRequestDetailsList() - .stream() - .map(rd -> new RequestDetailWithSpec(rd, featureSpecMap.get(rd.getFeatureId()))) - .collect(Collectors.toList()); + Map featureSpecMap = specStorage.getFeatureSpecs(featureIdSet); + return featureSpecMap.values(); } } diff --git a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java index 74b0e98a5dc..97ed05957a4 100644 --- a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java +++ b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java @@ -23,28 +23,21 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.RequestType; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.config.AppConfig; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.model.RequestDetailWithSpec; import feast.serving.util.EntityMapBuilder; import feast.specs.FeatureSpecProto.FeatureSpec; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -78,110 +71,42 @@ public FeatureRetrievalDispatcher( * * @param entityName entity name of the feature. * @param entityIds list of entity ids. - * @param requests list of request. - * @param timestampRange timestamp range of feature to be retrieved. + * @param featureSpecs list of request. * @return map of entityID and Entity instance. */ public Map dispatchFeatureRetrieval( - String entityName, - List entityIds, - List requests, - TimestampRange timestampRange) { + String entityName, Collection entityIds, Collection featureSpecs) { - GroupedRequest groupedRequest = groupRequestByTypeAndStorage(requests); + Map> groupedFeatureSpecs = groupByStorage(featureSpecs); - if (groupedRequest.getNbThreadRequired() <= 1) { - return runInCurrentThread( - entityName, entityIds, timestampRange, groupedRequest.getRequests()); + if (groupedFeatureSpecs.size() <= 1) { + return runInCurrentThread(entityName, entityIds, groupedFeatureSpecs); } else { - return runWithExecutorService( - entityName, entityIds, timestampRange, groupedRequest.getRequests()); + return runWithExecutorService(entityName, entityIds, groupedFeatureSpecs); } } - /** - * Group request by its type and storage ID of the feature. - * - * @param requestDetailWithSpecs list of requests. - * @return requests grouped by request type and storage ID. - */ - private GroupedRequest groupRequestByTypeAndStorage( - List requestDetailWithSpecs) { - Map> groupedByRequestType = - requestDetailWithSpecs.stream() - .collect(groupingBy(r -> r.getRequestDetail().getType())); - - int nbThreadRequired = 0; - Map>> groupedByTypeAndStorageId = - new HashMap<>(); - for (Map.Entry> requestEntry : - groupedByRequestType.entrySet()) { - RequestType requestType = requestEntry.getKey(); - List requestDetails = requestEntry.getValue(); - - Map> groupedByStorageId = - groupRequestByStorage(requestDetails); - groupedByTypeAndStorageId.put(requestType, groupedByStorageId); - - nbThreadRequired += groupedByStorageId.size(); - } - return new GroupedRequest(groupedByTypeAndStorageId, nbThreadRequired); - } - /** * Execute request in current thread. * * @param entityName entity name of of the feature. * @param entityIds list of entity ID of the feature to be retrieved. - * @param tsRange timestamp range of the feature to be retrieved. - * @param groupedRequest request grouped by type and storage ID. + * @param groupedFeatureSpecs feature spec grouped by storage ID. * @return entity map containing the result of feature retrieval. */ private Map runInCurrentThread( String entityName, - List entityIds, - TimestampRange tsRange, - Map>> groupedRequest) { - try (Scope scope = tracer.buildSpan("FeatureRetrievalDispatcher-runInCurrentThread") - .startActive(true)) { - Span span = scope.span(); - if (groupedRequest.size() > 1) { - throw new IllegalArgumentException( - "runInCurrentThread required more than one thread to run"); - } - - RequestType requestType = groupedRequest.keySet().iterator().next(); - Map> request = groupedRequest.get(requestType); + Collection entityIds, + Map> groupedFeatureSpecs) { + try (Scope scope = + tracer.buildSpan("FeatureRetrievalDispatcher-runInCurrentThread").startActive(true)) { - if (request.size() > 1) { - throw new IllegalArgumentException( - "runInCurrentThread required more than one thread to run"); - } - - String storageId = request.keySet().iterator().next(); - List requestDetailWithSpecs = request.get(storageId); + String storageId = groupedFeatureSpecs.keySet().iterator().next(); + List featureSpecs = groupedFeatureSpecs.get(storageId); FeatureStorage featureStorage = featureStorageRegistry.get(storageId); List featureValues; - if (RequestType.LAST == requestType) { - span.setTag("request-type", "last"); - List featureSpecs = - requestDetailWithSpecs - .stream() - .map(RequestDetailWithSpec::getFeatureSpec) - .collect(Collectors.toList()); - featureValues = featureStorage.getCurrentFeatures(entityName, entityIds, featureSpecs); - } else { - span.setTag("request-type", "list"); - List> featureSpecLimitPairs = - requestDetailWithSpecs - .stream() - .map(r -> new Pair<>(r.getFeatureSpec(), r.getRequestDetail().getLimit())) - .collect(Collectors.toList()); - featureValues = - featureStorage.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - } + featureValues = featureStorage.getFeature(entityName, entityIds, featureSpecs); EntityMapBuilder builder = new EntityMapBuilder(); builder.addFeatureValueList(featureValues); @@ -194,61 +119,29 @@ private Map runInCurrentThread( * * @param entityName entity name of the feature. * @param entityIds list of entity ID. - * @param tsRange timestamp range of the feature. - * @param groupedRequest request grouped by type and serving storage ID. + * @param groupedFeatureSpec feature specs grouped by serving storage ID. * @return entity map containing result of feature retrieval. */ private Map runWithExecutorService( String entityName, - List entityIds, - TimestampRange tsRange, - Map>> groupedRequest) { - try (Scope scope = tracer.buildSpan("FeatureRetrievalDispatcher-runWithExecutorService") - .startActive(true)) { + Collection entityIds, + Map> groupedFeatureSpec) { + try (Scope scope = + tracer.buildSpan("FeatureRetrievalDispatcher-runWithExecutorService").startActive(true)) { Span span = scope.span(); List> futures = new ArrayList<>(); EntityMapBuilder entityMapBuilder = new EntityMapBuilder(); - for (Map.Entry>> requestEntryPerType : - groupedRequest.entrySet()) { - RequestType requestType = requestEntryPerType.getKey(); - for (Map.Entry> requestEntryPerStorage : - requestEntryPerType.getValue().entrySet()) { - List requests = requestEntryPerStorage.getValue(); - FeatureStorage featureStorage = featureStorageRegistry - .get(requestEntryPerStorage.getKey()); - if (requestType == RequestType.LAST) { - List featureSpecs = - requests - .stream() - .map(RequestDetailWithSpec::getFeatureSpec) - .collect(Collectors.toList()); - - futures.add( - executorService.submit( - () -> { - List featureValues = - featureStorage.getCurrentFeatures(entityName, entityIds, featureSpecs); - entityMapBuilder.addFeatureValueList(featureValues); - return null; - })); - } else { - List> featureSpecLimitPairs = - requests - .stream() - .map(r -> new Pair<>(r.getFeatureSpec(), r.getRequestDetail().getLimit())) - .collect(Collectors.toList()); - - futures.add( - executorService.submit( - () -> { - List featureValues = - featureStorage.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - entityMapBuilder.addFeatureValueList(featureValues); - return null; - })); - } - } + for (Map.Entry> entry : groupedFeatureSpec.entrySet()) { + FeatureStorage featureStorage = featureStorageRegistry.get(entry.getKey()); + List featureSpecs = entry.getValue(); + futures.add( + executorService.submit( + () -> { + List featureValues = + featureStorage.getFeature(entityName, entityIds, featureSpecs); + entityMapBuilder.addFeatureValueList(featureValues); + return null; + })); } span.log("submitted all task"); ListenableFuture> combined = Futures.allAsList(futures); @@ -273,27 +166,12 @@ private Map runWithExecutorService( /** * Group request by its serving storage ID. * - * @param requestDetailWithSpecs list of request. + * @param featureSpecs list of request. * @return request grouped by serving storage ID. */ - private Map> groupRequestByStorage( - List requestDetailWithSpecs) { - return requestDetailWithSpecs + private Map> groupByStorage(Collection featureSpecs) { + return featureSpecs .stream() - .collect(groupingBy(r -> r.getFeatureSpec().getDataStores().getServing().getId())); - } - - @AllArgsConstructor - @Getter - private static class GroupedRequest { - - /** - * request grouped by type and its serving storage ID - */ - private final Map>> requests; - /** - * number of thread required to execute all request in parallel - */ - private final int nbThreadRequired; + .collect(groupingBy(featureSpec -> featureSpec.getDataStores().getServing().getId())); } } diff --git a/serving/src/main/java/feast/serving/service/FeatureStorage.java b/serving/src/main/java/feast/serving/service/FeatureStorage.java index 992ac5ae6cb..e1ffb0306ec 100644 --- a/serving/src/main/java/feast/serving/service/FeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/FeatureStorage.java @@ -17,77 +17,25 @@ package feast.serving.service; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; import feast.specs.FeatureSpecProto.FeatureSpec; +import java.util.Collection; import java.util.List; /** Abstraction of Feast Storage. */ public interface FeatureStorage { - /** - * Get current value of the given feature for the specified entities. - * - *

It returns the most recent value if the feature is time-series (granularity other than NONE) - * - * @param entityName entity name, e.g. 'driver', 'customer', 'area' - * @param entityIds list of entity id. - * @param featureSpec feature spec for which the feature should be retrieved. - * @return list of feature value. - * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. - */ - List getCurrentFeature( - String entityName, List entityIds, FeatureSpec featureSpec); /** * Get current value of several feature for the specified entities. * - *

It returns the most recent value if the feature is time-series (granularity other than NONE) - * * @param entityName entity name, e.g. 'driver', 'customer', 'area' * @param entityIds list of entity id. - * @param featureSpecs list of feature spec for which the feature should be retrieved. + * @param featureSpecs list of feature spec for which the feature should be retrieved. * @param * @return list of feature value. * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. */ - List getCurrentFeatures( - String entityName, List entityIds, List featureSpecs); - - /** - * Get N latest value of a feature within a timestamp range. - * - *

The number of data returned is less than or equal to {@code n}. - * - * @param entityName entity name - * @param entityIds list of entity id that the feature should be retrieved. - * @param featureSpecAndLimitPair pair between feature spec and the maximum number of value to be - * returned. - * @param tsRange timerange filter. - * @return map of entity id and the feature values. - * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. - */ - List getNLatestFeatureWithinTimestampRange( - String entityName, - List entityIds, - Pair featureSpecAndLimitPair, - TimestampRange tsRange); - - /** - * Get N latest value of several features within a timestamp range. - * - *

The number of data returned is less than or equal to {@code n}. - * - * @param entityName entity name - * @param entityIds list of entity id that the feature should be retrieved. - * @param featureSpecAndLimitPairs list of pair between feature spec and the maximum number of - * value to be returned. - * @param tsRange timerange filter. - * @return map of entity id and the feature values. - * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. - */ - List getNLatestFeaturesWithinTimestampRange( + List getFeature( String entityName, - List entityIds, - List> featureSpecAndLimitPairs, - TimestampRange tsRange); + Collection entityIds, + Collection featureSpecs); } diff --git a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java index 71c9a4b449b..d481c9554e9 100644 --- a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java @@ -18,12 +18,8 @@ package feast.serving.service; import com.google.protobuf.InvalidProtocolBufferException; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.util.SpecUtil; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; @@ -31,41 +27,29 @@ import io.opentracing.Span; import io.opentracing.Tracer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -/** - * Class for retrieving features from a Redis instance. - */ +/** Class for retrieving features from a Redis instance. */ @Slf4j public class RedisFeatureStorage implements FeatureStorage { public static final String TYPE = "redis"; - - public static String OPT_REDIS_HOST = "host"; - public static String OPT_REDIS_PORT = "port"; - - public static String OPT_REDIS_BUCKET_SIZE = "bucketSize"; // ISO8601 Period - // BUCKET_ID_ZERO is used to identify feature with granularity NONE or latest value of a // time-series feature. private static final long BUCKET_ID_ZERO = 0; - private static final byte[][] ZERO_LENGTH_ARRAY = new byte[0][0]; - + public static String OPT_REDIS_HOST = "host"; + public static String OPT_REDIS_PORT = "port"; + public static String OPT_REDIS_BUCKET_SIZE = "bucketSize"; // ISO8601 Period private final JedisPool jedisPool; private final Tracer tracer; @@ -79,144 +63,21 @@ public RedisFeatureStorage(JedisPool jedisPool, Tracer tracer) { this.tracer = tracer; } - /** - * {@inheritDoc} - */ - @Override - public List getCurrentFeature( - String entityName, List entityIds, FeatureSpec featureSpec) - throws FeatureRetrievalException { - try (Scope scope = tracer.buildSpan("Redis-getCurrentFeature").startActive(true)) { - String featureId = featureSpec.getId(); - String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); - List getRequests = new ArrayList<>(entityIds.size()); - for (String entityId : entityIds) { - RedisBucketKey key = makeBucketKey(entityId, featureIdSha1Prefix, BUCKET_ID_ZERO); - getRequests.add(new GetRequest(entityId, featureId, key)); - } - scope.span().log("completed request creation"); - return sendAndProcessMultiGet(getRequests); - } - } - - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override - public List getCurrentFeatures( - String entityName, List entityIds, List featureSpecs) { - try (Scope scope = tracer.buildSpan("Redis-getCurrentFeatures").startActive(true)) { + public List getFeature( + String entityName, Collection entityIds, Collection featureSpecs) { + try (Scope scope = tracer.buildSpan("Redis-getFeature").startActive(true)) { List getRequests = new ArrayList<>(entityIds.size() * featureSpecs.size()); for (FeatureSpec featureSpec : featureSpecs) { String featureId = featureSpec.getId(); String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); for (String entityId : entityIds) { RedisBucketKey key = makeBucketKey(entityId, featureIdSha1Prefix, BUCKET_ID_ZERO); - getRequests.add(new GetRequest(entityId, featureId, key)); - } - } - scope.span().log("completed request creation"); - return sendAndProcessMultiGet(getRequests); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List getNLatestFeatureWithinTimestampRange( - String entityName, - List entityIds, - Pair featureSpecLimitPair, - TimestampRange tsRange) - throws FeatureRetrievalException { - try (Scope scope = tracer.buildSpan("Redis-getNLatestFeatureWithinTimestampRange") - .startActive(true)) { - List buckets = - makePipelinedBucketKeys( - entityIds, featureSpecLimitPair.getLeft(), featureSpecLimitPair.getRight(), tsRange); - sendAndSyncPipelinedBucket(buckets); - - List getRequests = new ArrayList<>(); - Map featureCount = - new HashMap<>(entityIds.size()); // map of entity id to the number of value. - for (PipelinedBucketKeys bucket : buckets) { - for (byte[] rawKey : bucket.getBucket().get()) { - Integer count = featureCount.computeIfAbsent(bucket.getEntityId(), k -> 0); - if (count < bucket.getLimit()) { - RedisBucketKey key; - try { - key = RedisBucketKey.parseFrom(rawKey); - } catch (InvalidProtocolBufferException e) { - log.warn("Unable to parse one of key: {}", rawKey); - continue; - } - getRequests.add(new GetRequest(bucket.entityId, bucket.featureId, key)); - count++; - featureCount.put(bucket.getEntityId(), count); - } else { - break; - } - } - } - - return sendAndProcessMultiGet(getRequests); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List getNLatestFeaturesWithinTimestampRange( - String entityName, - List entityIds, - List> featureSpecAndLimitPairs, - TimestampRange tsRange) { - try (Scope scope = tracer.buildSpan("Redis-getNLatestFeaturesWithinTimestampRange") - .startActive(true)) { - - List buckets = new ArrayList<>(); - for (Pair featureSpecAndLimitPair : featureSpecAndLimitPairs) { - buckets.addAll( - makePipelinedBucketKeys( - entityIds, - featureSpecAndLimitPair.getLeft(), - featureSpecAndLimitPair.getRight(), - tsRange)); - } - sendAndSyncPipelinedBucket(buckets); - - List getRequests = new ArrayList<>(); - Map> entityCountMap = new HashMap<>(entityIds.size()); - for (PipelinedBucketKeys bucket : buckets) { - for (byte[] rawKey : bucket.getBucket().get()) { - Map featureCountMap = - entityCountMap.computeIfAbsent( - bucket.getEntityId(), k -> new HashMap<>(featureSpecAndLimitPairs.size())); - Integer count = featureCountMap.computeIfAbsent(bucket.getFeatureId(), k -> 0); - - if (count < bucket.getLimit()) { - String entityId = bucket.getEntityId(); - String featureId = bucket.getFeatureId(); - RedisBucketKey key; - try { - key = RedisBucketKey.parseFrom(rawKey); - } catch (InvalidProtocolBufferException e) { - log.warn("Unable to parse one of key: {}", rawKey); - continue; - } - getRequests.add(new GetRequest(entityId, featureId, key)); - count++; - featureCountMap.put(bucket.getFeatureId(), count); - entityCountMap.put(bucket.getEntityId(), featureCountMap); - } else { - break; - } } } - + scope.span().log("completed request creation"); return sendAndProcessMultiGet(getRequests); } } @@ -228,8 +89,7 @@ public List getNLatestFeaturesWithinTimestampRange( * @return list of feature value. */ private List sendAndProcessMultiGet(List getRequests) { - try (Scope scope = tracer.buildSpan("Redis-sendAndProcessMultiGet") - .startActive(true)) { + try (Scope scope = tracer.buildSpan("Redis-sendAndProcessMultiGet").startActive(true)) { Span span = scope.span(); if (getRequests.isEmpty()) { @@ -274,8 +134,7 @@ private List sendAndProcessMultiGet(List getRequests) */ private List processMGet(List requests, List results) throws InvalidProtocolBufferException { - try (Scope scope = tracer.buildSpan("Redis-processMGet") - .startActive(true)) { + try (Scope scope = tracer.buildSpan("Redis-processMGet").startActive(true)) { int keySize = requests.size(); List featureValues = new ArrayList<>(keySize); @@ -298,99 +157,6 @@ private List processMGet(List requests, List r } } - /** - * Send and wait for result of a pipelined zrevrangeByScore. - * - * @param buckets list of keys to be scanned. - */ - private void sendAndSyncPipelinedBucket(List buckets) { - try (Scope scope = tracer.buildSpan("Redis-sendAndSyncPipelinedBucket") - .startActive(true); - Jedis jedis = jedisPool.getResource()) { - Pipeline pipeline = jedis.pipelined(); - - // retrieve all redis keys from index bucket. - for (PipelinedBucketKeys bucket : buckets) { - bucket.setBucket( - pipeline.zrevrangeByScore( - bucket.getBucketKey().toByteArray(), - bucket.getMaxTimestamp(), - bucket.getMinTimestamp())); - } - - pipeline.sync(); - } catch (Exception e) { - log.error("Exception while retrieving bucket keys in redis", e); - throw new FeatureRetrievalException("Unable to retrieve feature from Redis", e); - } - } - - /** - * Make a list of bucket keys to scan. - * - * @param entityIds list of entity id. - * @param featureSpec feature spec of the feature to be scanned. - * @param limit maximum number of value returned from request. - * @param tsRange timestamp range of value to be returned. - * @return list of bucket keys to be scanned in pipelined command. - */ - private List makePipelinedBucketKeys( - List entityIds, FeatureSpec featureSpec, int limit, TimestampRange tsRange) { - try (Scope scope = tracer.buildSpan("Redis-makePipelinedBucketKeys") - .startActive(true)) { - List buckets = new ArrayList<>(); - String featureId = featureSpec.getId(); - String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); - - long maxTimestamp = - TimeUtil.roundFloorTimestamp(tsRange.getEnd(), featureSpec.getGranularity()).getSeconds(); - long minTimestamp = - TimeUtil.roundFloorTimestamp(tsRange.getStart(), featureSpec.getGranularity()) - .getSeconds(); - - for (String entityId : entityIds) { - for (RedisBucketKey bucketKey : - makeBucketKeysToScan( - entityId, featureIdSha1Prefix, minTimestamp, maxTimestamp, featureSpec)) { - buckets.add( - new PipelinedBucketKeys( - entityId, featureId, bucketKey, null, minTimestamp, maxTimestamp, limit)); - } - } - return buckets; - } - } - - /** - * Generate list of bucket keys to be scanned for range query. - * - * @param entityId entity ID of the feature - * @param featureIdSha1Prefix first 7 bytes of feature ID SHA1 prefix. - * @param roundedStart start time of range query rounded into granularity. - * @param roundedEnd end time of range query rounded into granualrity. - * @param featureSpec feature spec of the feature. - * @return list of redis bucket key to be scanned. - */ - private List makeBucketKeysToScan( - String entityId, - String featureIdSha1Prefix, - long roundedStart, - long roundedEnd, - FeatureSpec featureSpec) { - List bucketKeys = new ArrayList<>(); - long bucketSizeInSecond = SpecUtil.getBucketSize(featureSpec).getStandardSeconds(); - - for (long i = roundedStart; i <= roundedEnd; i += bucketSizeInSecond) { - bucketKeys.add( - RedisBucketKey.newBuilder() - .setEntityKey(entityId) - .setFeatureIdSha1Prefix(featureIdSha1Prefix) - .setBucketId(i / bucketSizeInSecond) - .build()); - } - return bucketKeys; - } - /** * Create {@link RedisBucketKey}. * @@ -417,64 +183,17 @@ private String makeFeatureIdSha1Prefix(String featureId) { return DigestUtils.sha1Hex(featureId.getBytes()).substring(0, 7); } - @AllArgsConstructor - @Getter - @Setter - private static class PipelinedBucketKeys { - - /** - * Entity Id of the feature - */ - private String entityId; - - /** - * Feature Id of the feature - */ - private String featureId; - - /** - * bucket key - */ - private RedisBucketKey bucketKey; - - /** - * Content of bucket - */ - private Response> bucket; - - /** - * Rounded lower bound of timestamp to be scanned - */ - private long minTimestamp; - - /** - * Rounded upper bound of timestamp to be scanned - */ - private long maxTimestamp; - - /** - * Maximum number of value to be returned - */ - private int limit; - } - @AllArgsConstructor @Getter private static class GetRequest { - /** - * Entity Id of the get request - */ + /** Entity Id of the get request */ private String entityId; - /** - * Feature Id of the get request - */ + /** Feature Id of the get request */ private String featureId; - /** - * Bucket key of the request - */ + /** Bucket key of the request */ private RedisBucketKey key; } } diff --git a/serving/src/main/java/feast/serving/util/EntityMapBuilder.java b/serving/src/main/java/feast/serving/util/EntityMapBuilder.java index 6bec1574b71..5dc6a52e86b 100644 --- a/serving/src/main/java/feast/serving/util/EntityMapBuilder.java +++ b/serving/src/main/java/feast/serving/util/EntityMapBuilder.java @@ -17,40 +17,28 @@ package feast.serving.util; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; +import feast.serving.ServingAPIProto; import feast.serving.ServingAPIProto.Entity; import feast.serving.model.FeatureValue; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.SortedSet; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListSet; /** Utility class for converting response from Feature Storage into {@link Map}. */ public class EntityMapBuilder { - // for sorting based on timestamp, newest first. - private static final Comparator TIMESTAMP_COMPARATOR = - new FeatureValueTimestampComparator().reversed(); - - private final Map>> backingMap = - new ConcurrentHashMap<>(); + private final Map> backingMap = new ConcurrentHashMap<>(); public void addFeatureValueList(List features) { for (FeatureValue feature : features) { String entityId = feature.getEntityId(); String featureId = feature.getFeatureId(); - Map> featureIdToFeatureValueMap = + Map featureIdToFeatureValueMap = backingMap.computeIfAbsent(entityId, k -> new ConcurrentHashMap<>()); - SortedSet featureValues = - featureIdToFeatureValueMap.computeIfAbsent( - featureId, k -> new ConcurrentSkipListSet<>(TIMESTAMP_COMPARATOR)); - featureValues.add(feature); + featureIdToFeatureValueMap.put(featureId, feature); } } @@ -61,28 +49,21 @@ public void addFeatureValueList(List features) { */ public Map toEntityMap() { Map resultMap = new HashMap<>(); - for (Map.Entry>> entity : backingMap.entrySet()) { + for (Map.Entry> entity : backingMap.entrySet()) { String entityId = entity.getKey(); Entity.Builder entityBuilder = Entity.newBuilder(); - for (Map.Entry> feature : entity.getValue().entrySet()) { + for (Map.Entry feature : entity.getValue().entrySet()) { String featureId = feature.getKey(); - FeatureValueListBuilder featureBuilder = new FeatureValueListBuilder(); - for (FeatureValue featureValue : feature.getValue()) { - featureBuilder.addValue(featureValue.getValue(), featureValue.getTimestamp()); - } - entityBuilder.putFeatures(featureId, featureBuilder.build()); + FeatureValue featureValue = feature.getValue(); + ServingAPIProto.FeatureValue featureValueProto = ServingAPIProto.FeatureValue.newBuilder() + .setTimestamp(featureValue.getTimestamp()) + .setValue(featureValue.getValue()) + .build(); + + entityBuilder.putFeatures(featureId, featureValueProto); } resultMap.put(entityId, entityBuilder.build()); } return resultMap; } - - private static class FeatureValueTimestampComparator implements Comparator { - @Override - public int compare(FeatureValue feature1, FeatureValue feature2) { - Timestamp t1 = feature1.getTimestamp(); - Timestamp t2 = feature2.getTimestamp(); - return Timestamps.compare(t1, t2); - } - } } diff --git a/serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java b/serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java deleted file mode 100644 index 7db0ced77a2..00000000000 --- a/serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.util; - -import com.google.protobuf.GeneratedMessageV3; -import com.google.protobuf.Timestamp; -import feast.serving.ServingAPIProto.FeatureValueList; -import feast.types.ValueProto.*; - -/** Utilitu class for building {@link FeatureValueList}. */ -public class FeatureValueListBuilder { - - private boolean isInitialized = false; - private Value.ValCase type; - - private GeneratedMessageV3.Builder valueBuilder; - private TimestampList.Builder tsBuilder; - - public boolean addValue(Value newVal, Timestamp ts) { - initializeIfNecessary(newVal, ts); - switch (newVal.getValCase()) { - case BYTESVAL: - BytesList.Builder bytesBuilder = (BytesList.Builder) valueBuilder; - bytesBuilder.addVal(newVal.getBytesVal()); - break; - case STRINGVAL: - StringList.Builder stringBuilder = (StringList.Builder) valueBuilder; - stringBuilder.addVal(newVal.getStringVal()); - break; - case INT32VAL: - Int32List.Builder int32Builder = (Int32List.Builder) valueBuilder; - int32Builder.addVal(newVal.getInt32Val()); - break; - case INT64VAL: - Int64List.Builder int64Builder = (Int64List.Builder) valueBuilder; - int64Builder.addVal(newVal.getInt64Val()); - break; - case DOUBLEVAL: - DoubleList.Builder doubleBuilder = (DoubleList.Builder) valueBuilder; - doubleBuilder.addVal(newVal.getDoubleVal()); - break; - case FLOATVAL: - FloatList.Builder floatBuilder = (FloatList.Builder) valueBuilder; - floatBuilder.addVal(newVal.getFloatVal()); - break; - case BOOLVAL: - BoolList.Builder boolBuilder = (BoolList.Builder) valueBuilder; - boolBuilder.addVal(newVal.getBoolVal()); - break; - case TIMESTAMPVAL: - TimestampList.Builder tsListBuilder = (TimestampList.Builder) valueBuilder; - tsListBuilder.addVal(newVal.getTimestampVal()); - break; - default: - throw new IllegalArgumentException("unsupported type"); - } - if (ts != null) { - tsBuilder.addVal(ts); - } - return true; - } - - public FeatureValueList build() { - ValueList valueList = null; - switch (type) { - case BYTESVAL: - valueList = - ValueList.newBuilder().setBytesList(((BytesList.Builder) valueBuilder).build()).build(); - break; - case STRINGVAL: - valueList = - ValueList.newBuilder() - .setStringList(((StringList.Builder) valueBuilder).build()) - .build(); - break; - case INT32VAL: - valueList = - ValueList.newBuilder().setInt32List(((Int32List.Builder) valueBuilder).build()).build(); - break; - case INT64VAL: - valueList = - ValueList.newBuilder().setInt64List(((Int64List.Builder) valueBuilder).build()).build(); - break; - case DOUBLEVAL: - valueList = - ValueList.newBuilder() - .setDoubleList(((DoubleList.Builder) valueBuilder).build()) - .build(); - break; - case FLOATVAL: - valueList = - ValueList.newBuilder().setFloatList(((FloatList.Builder) valueBuilder).build()).build(); - break; - case BOOLVAL: - valueList = - ValueList.newBuilder().setBoolList(((BoolList.Builder) valueBuilder).build()).build(); - break; - case TIMESTAMPVAL: - valueList = - ValueList.newBuilder() - .setTimestampList(((TimestampList.Builder) valueBuilder).build()) - .build(); - break; - } - - FeatureValueList.Builder featureBuilder = FeatureValueList.newBuilder().setValueList(valueList); - if (tsBuilder != null) { - featureBuilder.setTimestampList(tsBuilder.build()); - } - return featureBuilder.build(); - } - - private void initializeIfNecessary(Value val, Timestamp ts) { - if (isInitialized) { - return; - } - - this.type = val.getValCase(); - this.tsBuilder = (ts != null) ? TimestampList.newBuilder() : null; - switch (type) { - case BYTESVAL: - valueBuilder = BytesList.newBuilder(); - break; - case STRINGVAL: - valueBuilder = StringList.newBuilder(); - break; - case INT32VAL: - valueBuilder = Int32List.newBuilder(); - break; - case INT64VAL: - valueBuilder = Int64List.newBuilder(); - break; - case DOUBLEVAL: - valueBuilder = DoubleList.newBuilder(); - break; - case FLOATVAL: - valueBuilder = FloatList.newBuilder(); - break; - case BOOLVAL: - valueBuilder = BoolList.newBuilder(); - break; - case TIMESTAMPVAL: - valueBuilder = TimestampList.newBuilder(); - break; - default: - throw new IllegalArgumentException("unsupported type"); - } - - isInitialized = true; - } -} diff --git a/serving/src/main/java/feast/serving/util/RequestHelper.java b/serving/src/main/java/feast/serving/util/RequestHelper.java index 7b978ed61fc..c5e915dc27e 100644 --- a/serving/src/main/java/feast/serving/util/RequestHelper.java +++ b/serving/src/main/java/feast/serving/util/RequestHelper.java @@ -18,72 +18,34 @@ package feast.serving.util; import com.google.common.base.Strings; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.TimestampRange; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; public class RequestHelper { - private RequestHelper() {} + private RequestHelper() {} - public static void validateRequest(Request request) { - // entity name shall present - if (Strings.isNullOrEmpty(request.getEntityName())) { - throw new IllegalArgumentException("entity name must be set"); - } - - // entity id list shall not be empty - if (request.getEntityIdList().size() <= 0) { - throw new IllegalArgumentException("entity ID must be provided"); - } - - // request detail shall not be empty - if (request.getRequestDetailsList().size() <= 0) { - throw new IllegalArgumentException("request details must be provided"); - } - - // feature id in each request detail shall have same entity name - String entityName = request.getEntityName(); - for (RequestDetail requestDetail : request.getRequestDetailsList()) { - String featureId = requestDetail.getFeatureId(); - if (!featureId.substring(0, featureId.indexOf(".")).equals(entityName)) { - throw new IllegalArgumentException( - "entity name of all feature ID in request details must be: " + entityName); - } - } + public static void validateRequest(QueryFeaturesRequest request) { + // entity name shall present + if (Strings.isNullOrEmpty(request.getEntityName())) { + throw new IllegalArgumentException("entity name must be set"); } - public static Request checkTimestampRange(Request request) { - Request.Builder requestBuilder = Request.newBuilder(request); - - Timestamp defaultTs = - Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - - // default timestamp range - if (!request.hasTimestampRange()) { - requestBuilder.setTimestampRange( - TimestampRange.newBuilder().setStart(defaultTs).setEnd(defaultTs).build()); - } else if (request.getTimestampRange().getStart().getSeconds() == 0 - && request.getTimestampRange().getStart().getNanos() == 0) { - Timestamp end = request.getTimestampRange().getEnd(); - requestBuilder.setTimestampRange( - TimestampRange.newBuilder(request.getTimestampRange()).setStart(end)); - } else if (request.getTimestampRange().getEnd().getSeconds() == 0 - && request.getTimestampRange().getEnd().getNanos() == 0) { - requestBuilder.setTimestampRange( - TimestampRange.newBuilder(request.getTimestampRange()).setEnd(defaultTs)); - } - - Request newRequest = requestBuilder.build(); - Timestamp start = newRequest.getTimestampRange().getStart(); - Timestamp end = newRequest.getTimestampRange().getEnd(); + // entity id list shall not be empty + if (request.getEntityIdList().size() <= 0) { + throw new IllegalArgumentException("entity ID must be provided"); + } - if (Timestamps.compare(start, end) > 0) { - throw new IllegalArgumentException( - "'end' of timestampRange must be before or same time as 'start'"); - } + // feature IDs shall not be empty + if (request.getFeatureIdCount() <= 0) { + throw new IllegalArgumentException("feature id must be provided"); + } - return newRequest; + // feature id in each request detail shall have same entity name + String entityName = request.getEntityName(); + for (String featureId : request.getFeatureIdList()) { + if (!featureId.substring(0, featureId.indexOf(".")).equals(entityName)) { + throw new IllegalArgumentException( + "entity name of all feature ID in request details must be: " + entityName); + } } + } } diff --git a/serving/src/main/java/feast/serving/util/StatsUtil.java b/serving/src/main/java/feast/serving/util/StatsUtil.java index 7782bcb6e0e..86437218e8d 100644 --- a/serving/src/main/java/feast/serving/util/StatsUtil.java +++ b/serving/src/main/java/feast/serving/util/StatsUtil.java @@ -17,8 +17,7 @@ package feast.serving.util; import com.google.common.base.Strings; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.RequestDetail; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; import io.grpc.Context; import io.grpc.Context.Key; import java.net.SocketAddress; @@ -40,7 +39,7 @@ private StatsUtil() { *

The tags contain information about feature's ids of the request and the client requesting * it. */ - public static String[] makeStatsdTags(Request request) { + public static String[] makeStatsdTags(QueryFeaturesRequest request) { List featureTags = makeFeatureTags(request); String remoteAddrTag = makeRemoteAddressTag(); String[] tags = featureTags.toArray(new String[featureTags.size() + 1]); @@ -48,10 +47,9 @@ public static String[] makeStatsdTags(Request request) { return tags; } - private static List makeFeatureTags(Request request) { - List tags = new ArrayList<>(request.getRequestDetailsCount()); - for (RequestDetail requestDetail : request.getRequestDetailsList()) { - String featureId = requestDetail.getFeatureId(); + private static List makeFeatureTags(QueryFeaturesRequest request) { + List tags = new ArrayList<>(request.getFeatureIdCount()); + for (String featureId : request.getFeatureIdList()) { if (Strings.isNullOrEmpty(featureId)) { continue; } diff --git a/serving/src/main/java/feast/serving/util/TimeUtil.java b/serving/src/main/java/feast/serving/util/TimeUtil.java index bb3ba178731..24c4aa8950d 100644 --- a/serving/src/main/java/feast/serving/util/TimeUtil.java +++ b/serving/src/main/java/feast/serving/util/TimeUtil.java @@ -17,71 +17,21 @@ package feast.serving.util; -import com.google.protobuf.Timestamp; -import feast.types.GranularityProto.Granularity.Enum; -import org.joda.time.*; - /** Utility class for time-related function. */ public class TimeUtil { - public static final int NANO_IN_MILLI = 1000000; public static final int NANO_IN_MICRO = 1000; - public static final int MILLI_IN_SECOND = 1000; private TimeUtil() {} - /** - * Round down timestamp to the nearest granularity. - * - * @param timestamp original timestamp. - * @param granularity granularity of the rounded timestamp. - * @return - */ - public static Timestamp roundFloorTimestamp(Timestamp timestamp, Enum granularity) { - MutableDateTime dt = new MutableDateTime(DateTimeZone.UTC); - DateTimeField roundingField; - switch (granularity) { - case DAY: - roundingField = dt.getChronology().dayOfMonth(); - break; - case HOUR: - roundingField = dt.getChronology().hourOfDay(); - break; - case MINUTE: - roundingField = dt.getChronology().minuteOfHour(); - break; - case SECOND: - roundingField = dt.getChronology().secondOfMinute(); - break; - case NONE: - return Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); - default: - throw new RuntimeException("Unrecognised time series granularity"); - } - dt.setMillis(timestamp.getSeconds() * 1000 + timestamp.getNanos() / 1000000); - dt.setRounding(roundingField, MutableDateTime.ROUND_FLOOR); - return dateTimeToTimestamp(dt.toDateTime()); - } /** - * Returns the current value of the running Java Virtual Machine's - * high-resolution time source, in microseconds. + * Returns the current value of the running Java Virtual Machine's high-resolution time source, in + * microseconds. + * * @return current micro time. * @see System#nanoTime() */ public static long microTime() { return System.nanoTime() / NANO_IN_MICRO; } - - /** - * Convert {@link DateTime} into {@link Timestamp} - * - * @param dateTime - * @return - */ - private static Timestamp dateTimeToTimestamp(DateTime dateTime) { - return Timestamp.newBuilder() - .setSeconds(dateTime.getMillis() / MILLI_IN_SECOND) - .setNanos((int) (dateTime.getMillis() % MILLI_IN_SECOND) * NANO_IN_MILLI) - .build(); - } } diff --git a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java index 2a7cac282df..3ba9c2b7354 100644 --- a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java +++ b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java @@ -17,24 +17,20 @@ package feast.serving.grpc; -import static junit.framework.TestCase.assertNotNull; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.verify; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.TimestampRange; -import feast.serving.model.RequestDetailWithSpec; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.serving.service.FeastServing; import feast.serving.service.FeatureRetrievalDispatcher; import feast.serving.service.FeatureStorageRegistry; import feast.serving.service.SpecStorage; import feast.serving.testutil.FakeSpecStorage; +import feast.specs.FeatureSpecProto.FeatureSpec; import io.opentracing.util.GlobalTracer; import java.util.Arrays; import java.util.Collection; @@ -67,21 +63,13 @@ public void setUp() throws Exception { @Test public void shouldReturnSameEntityNameAsRequest() { String entityName = "driver"; - RequestDetail requestDetail = - RequestDetail.newBuilder().setFeatureId("driver.day.total_completed_booking").build(); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.EPOCH) - .setEnd(Timestamp.newBuilder().setSeconds(1234).build()) - .build(); - Request request = - Request.newBuilder() + QueryFeaturesRequest request = + QueryFeaturesRequest.newBuilder() .setEntityName(entityName) - .addRequestDetails(requestDetail) - .setTimestampRange(tsRange) + .addFeatureId("driver.day.total_completed_booking") .build(); - Response response = feast.queryFeatures(request); + QueryFeaturesResponse response = feast.queryFeatures(request); assertNotNull(response); assertThat(response.getEntityName(), equalTo(entityName)); @@ -91,38 +79,27 @@ public void shouldReturnSameEntityNameAsRequest() { public void shouldPassValidRequestToFeatureRetrievalDispatcher() { String entityName = "driver"; Collection entityIds = Arrays.asList("entity1", "entity2", "entity3"); - RequestDetail req1 = - RequestDetail.newBuilder().setFeatureId("driver.day.total_completed_booking").build(); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.EPOCH) - .setEnd(Timestamp.newBuilder().setSeconds(1234).build()) - .build(); - Request request = - Request.newBuilder() + Collection featureIds = Arrays.asList("driver.day.total_completed_booking"); + QueryFeaturesRequest request = + QueryFeaturesRequest.newBuilder() .setEntityName(entityName) .addAllEntityId(entityIds) - .addRequestDetails(req1) - .setTimestampRange(tsRange) + .addAllFeatureId(featureIds) .build(); ArgumentCaptor entityNameArg = ArgumentCaptor.forClass(String.class); ArgumentCaptor> entityIdsArg = ArgumentCaptor.forClass(List.class); - ArgumentCaptor> requestsArg = ArgumentCaptor.forClass(List.class); - ArgumentCaptor tsRangeArg = ArgumentCaptor.forClass(TimestampRange.class); + ArgumentCaptor> featureSpecArg = + ArgumentCaptor.forClass(Collection.class); - Response response = feast.queryFeatures(request); + QueryFeaturesResponse response = feast.queryFeatures(request); verify(featureRetrievalDispatcher) .dispatchFeatureRetrieval( - entityNameArg.capture(), - entityIdsArg.capture(), - requestsArg.capture(), - tsRangeArg.capture()); + entityNameArg.capture(), entityIdsArg.capture(), featureSpecArg.capture()); assertNotNull(response); assertThat(response.getEntityName(), equalTo(entityName)); assertThat(entityNameArg.getValue(), equalTo(entityName)); assertThat(entityIdsArg.getValue(), containsInAnyOrder(entityIds.toArray())); - assertThat(tsRangeArg.getValue(), equalTo(tsRange)); } } diff --git a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java index e52848bc0b1..75c80f8aa28 100644 --- a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java +++ b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java @@ -17,45 +17,32 @@ package feast.serving.grpc; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; + import com.timgroup.statsd.StatsDClient; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; +import feast.serving.service.FeastServing; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import io.jaegertracing.Configuration; import io.opentracing.Tracer; +import java.util.Arrays; import org.junit.Before; import org.junit.Test; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.RequestType; -import feast.serving.ServingAPIProto.TimestampRange; -import feast.serving.service.FeastServing; -import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.util.Arrays; - -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class ServingGrpcServiceTest { @Mock private FeastServing mockFeast; - @Mock private StreamObserver mockStreamObserver; - - @Mock private Response mockResponse; + @Mock private StreamObserver mockStreamObserver; @Mock private StatsDClient statsDClient; - private Request validRequest; + private QueryFeaturesRequest validRequest; private ServingGrpcService service; @@ -63,28 +50,12 @@ public class ServingGrpcServiceTest { public void setUp() throws Exception { MockitoAnnotations.initMocks(this); - RequestDetail req1 = - RequestDetail.newBuilder().setFeatureId("driver.day.completed_booking").build(); - - RequestDetail req2 = - RequestDetail.newBuilder() - .setFeatureId("driver.none.last_opportunity") - .setType(RequestType.LIST) - .setLimit(5) - .build(); - - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamp.newBuilder().setSeconds(10).build()) - .setEnd(Timestamp.newBuilder().setSeconds(11).build()) - .build(); - validRequest = - Request.newBuilder() + QueryFeaturesRequest.newBuilder() .setEntityName("driver") .addAllEntityId(Arrays.asList("driver1", "driver2", "driver3")) - .addAllRequestDetails(Arrays.asList(req1, req2)) - .setTimestampRange(tsRange) + .addAllFeatureId( + Arrays.asList("driver.day.completed_booking", "driver.none.last_opportunity")) .build(); Tracer tracer = Configuration.fromEnv("dummy").getTracer(); @@ -97,70 +68,10 @@ public void shouldPassValidRequestAsIs() { verify(mockFeast).queryFeatures(validRequest); } - @Test - public void shouldUseNowAsStartAndEndInTimestampRangeIfMissing() { - Request missingTsRangeReq = Request.newBuilder(validRequest).clearTimestampRange().build(); - - ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Request.class); - - when(mockFeast.queryFeatures(argCaptor.capture())).thenReturn(mockResponse); - service.queryFeatures(missingTsRangeReq, mockStreamObserver); - - Timestamp now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - TimestampRange actualTsRange = argCaptor.getValue().getTimestampRange(); - assertThat( - (double) actualTsRange.getStart().getSeconds(), - closeTo((double) now.getSeconds(), (double) 1)); - assertThat( - (double) actualTsRange.getEnd().getSeconds(), - closeTo((double) now.getSeconds(), (double) 1)); - System.out.println(argCaptor.getValue()); - } - - @Test - public void shouldUseEndIfStartTimestampRangeIsMissing() { - Timestamp end = Timestamp.newBuilder().setSeconds(1234).build(); - Request missingStartTsRangeReq = - Request.newBuilder(validRequest) - .clearTimestampRange() - .setTimestampRange(TimestampRange.newBuilder().setEnd(end)) - .build(); - - ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Request.class); - - when(mockFeast.queryFeatures(argCaptor.capture())).thenReturn(mockResponse); - service.queryFeatures(missingStartTsRangeReq, mockStreamObserver); - - TimestampRange actualTsRange = argCaptor.getValue().getTimestampRange(); - assertThat(actualTsRange.getStart(), equalTo(end)); - System.out.println(argCaptor.getValue()); - } - - @Test - public void shouldUseNowIfEndTimestampRangeIsMissing() { - Timestamp start = Timestamp.newBuilder().setSeconds(1234).build(); - Request missingEndTsRangeReq = - Request.newBuilder(validRequest) - .clearTimestampRange() - .setTimestampRange(TimestampRange.newBuilder().setStart(start)) - .build(); - - ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Request.class); - - when(mockFeast.queryFeatures(argCaptor.capture())).thenReturn(mockResponse); - service.queryFeatures(missingEndTsRangeReq, mockStreamObserver); - - Timestamp now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - TimestampRange actualTsRange = argCaptor.getValue().getTimestampRange(); - assertThat( - (double) actualTsRange.getEnd().getSeconds(), - closeTo((double) now.getSeconds(), (double) 1)); - System.out.println(argCaptor.getValue()); - } - @Test public void shouldCallOnErrorIfEntityNameIsNotSet() { - Request missingEntityName = Request.newBuilder(validRequest).clearEntityName().build(); + QueryFeaturesRequest missingEntityName = + QueryFeaturesRequest.newBuilder(validRequest).clearEntityName().build(); service.queryFeatures(missingEntityName, mockStreamObserver); @@ -169,7 +80,8 @@ public void shouldCallOnErrorIfEntityNameIsNotSet() { @Test public void shouldCallOnErrorIfEntityIdsIsNotSet() { - Request missingEntityIds = Request.newBuilder(validRequest).clearEntityId().build(); + QueryFeaturesRequest missingEntityIds = + QueryFeaturesRequest.newBuilder(validRequest).clearEntityId().build(); service.queryFeatures(missingEntityIds, mockStreamObserver); @@ -177,8 +89,9 @@ public void shouldCallOnErrorIfEntityIdsIsNotSet() { } @Test - public void shouldCallOnErrorIfRequestDetailsIsNotSet() { - Request missingRequestDetails = Request.newBuilder(validRequest).clearRequestDetails().build(); + public void shouldCallOnErrorIfFeatureIdsIsNotSet() { + QueryFeaturesRequest missingRequestDetails = + QueryFeaturesRequest.newBuilder(validRequest).clearFeatureId().build(); service.queryFeatures(missingRequestDetails, mockStreamObserver); @@ -186,27 +99,13 @@ public void shouldCallOnErrorIfRequestDetailsIsNotSet() { } @Test - public void shouldCallOnErrorIfRequestDetailsContainsDifferentEntity() { - RequestDetail req3 = RequestDetail.newBuilder().setFeatureId("customer.day.order_made").build(); - Request differentEntityReq = Request.newBuilder(validRequest).addRequestDetails(req3).build(); - - service.queryFeatures(differentEntityReq, mockStreamObserver); - - verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); - } - - @Test - public void shouldCallOnErrorIfTimestampRangeStartIsAfterEnd() { - TimestampRange invalidTsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(1001)) - .setEnd(Timestamps.fromSeconds(1000)) + public void shouldCallOnErrorIfFeatureIdsContainsDifferentEntity() { + QueryFeaturesRequest differentEntityReq = + QueryFeaturesRequest.newBuilder(validRequest) + .addFeatureId("customer.day.order_made") .build(); - Request invalidTsRangeReq = - Request.newBuilder(validRequest).setTimestampRange(invalidTsRange).build(); - - service.queryFeatures(invalidTsRangeReq, mockStreamObserver); + service.queryFeatures(differentEntityReq, mockStreamObserver); verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } diff --git a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java index bac3110456d..f8905e6342a 100644 --- a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java +++ b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java @@ -20,27 +20,23 @@ import com.google.cloud.bigtable.hbase.BigtableConfiguration; import com.google.cloud.bigtable.hbase.BigtableOptionsFactory; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; -import com.google.protobuf.util.Timestamps; -import feast.serving.model.Pair; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Connection; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; import feast.serving.testutil.BigTablePopulator; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.GranularityProto.Granularity; import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; - -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Connection; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class BigTableFeatureStorageTestITCase { private static final String ENTITY_NAME = "test_entity"; - private static final int NUM_OF_DAYS_DATA = 2; // 2 days data. // The object under test BigTableFeatureStorage featureStorage; @@ -48,8 +44,7 @@ public class BigTableFeatureStorageTestITCase { private BigTablePopulator bigTablePopulator; private List entityIds; - private Timestamp start; - private Timestamp end; + private Timestamp now; private Connection connection; @Before @@ -62,8 +57,7 @@ public void setUp() throws Exception { featureStorage = new BigTableFeatureStorage(connection); entityIds = createEntityIds(10); - end = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - start = Timestamps.subtract(end, Durations.fromSeconds(NUM_OF_DAYS_DATA * 24 * 60 * 60)); + now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); } @After @@ -72,67 +66,7 @@ public void tearDown() throws Exception { } @Test - public void getCurrentFeature_shouldWorkForGranularityNone() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_granularity_none") - .setName("feature_granularity_none") - .setGranularity(Granularity.Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Arrays.asList(featureSpec); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); - List results = - featureStorage.getCurrentFeature(ENTITY_NAME, entityIds, featureSpec); - - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldGracefullyHandleMissingEntity() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_granularity_none") - .setName("feature_granularity_none") - .setGranularity(Granularity.Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Arrays.asList(featureSpec); - List entityIdsWithMissingEntity = new ArrayList<>(entityIds); - entityIdsWithMissingEntity.add("100"); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); - List results = - featureStorage.getCurrentFeature(ENTITY_NAME, entityIdsWithMissingEntity, featureSpec); - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldWorkForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - - List featureSpecs = Collections.singletonList(spec); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - List result = - featureStorage.getCurrentFeature(ENTITY_NAME, entityIds, spec); - - bigTablePopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } - } - - - @Test - public void getCurrentFeatures_shouldWorkForGranularityNone() { + public void getFeatures_shouldWorkForGranularityNone() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) @@ -152,16 +86,14 @@ public void getCurrentFeatures_shouldWorkForGranularityNone() { .build(); List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); - List results = - featureStorage.getCurrentFeatures(ENTITY_NAME, entityIds, featureSpecs); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); + List results = featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs); - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); + bigTablePopulator.validate(results, entityIds, featureSpecs); } @Test - public void getCurrentFeatures_shouldGracefullyHandleMissingEntity() { + public void getFeatures_shouldGracefullyHandleMissingEntity() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) @@ -183,82 +115,14 @@ public void getCurrentFeatures_shouldGracefullyHandleMissingEntity() { List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); List entityIdsWithMissingEntity = new ArrayList<>(entityIds); entityIdsWithMissingEntity.add("100"); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); List results = - featureStorage.getCurrentFeatures(ENTITY_NAME, entityIdsWithMissingEntity, featureSpecs); - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeatures_shouldWorkForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); - - List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - List result = - featureStorage.getCurrentFeatures(ENTITY_NAME, entityIds, featureSpecs); - - bigTablePopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldWorkForTimeseriesData() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - - List featureSpecs = Collections.singletonList(spec); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - featureStorage.getNLatestFeatureWithinTimestampRange( - ENTITY_NAME, entityIds, featureSpecLimitPair, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } + featureStorage.getFeature(ENTITY_NAME, entityIdsWithMissingEntity, featureSpecs); + bigTablePopulator.validate(results, entityIds, featureSpecs); } @Test - public void getNLatestFeatureWithinTimerange_shouldGracefullyHandleMissingEntity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - - List entityIdsWithMissingEntity = new ArrayList<>(entityIds); - entityIdsWithMissingEntity.add("100"); - - Collection featureSpecs = Collections.singletonList(spec); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - featureStorage.getNLatestFeatureWithinTimestampRange( - ENTITY_NAME, entityIdsWithMissingEntity, featureSpecLimitPair, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - - @Test - public void getNLatestFeaturesWithinTimerange_shouldWorkForTimeseriesData() { + public void getFeatures_shouldWorkForOtherGranularity() { for (Granularity.Enum granularity : Granularity.Enum.values()) { if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { continue; @@ -267,56 +131,16 @@ public void getNLatestFeaturesWithinTimerange_shouldWorkForTimeseriesData() { FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); - int limit1 = 5; - int limit2 = 3; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); + List result = featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - featureStorage.getNLatestFeaturesWithinTimestampRange( - ENTITY_NAME, entityIds, featureSpecLimitPairs, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, featureSpecLimitPairs, tsRange); + bigTablePopulator.validate(result, entityIds, featureSpecs); } } - @Test - public void getNLatestFeaturesWithinTimerange_shouldGracefullyHandleMissingEntity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); - - - List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - int limit1 = 5; - int limit2 = 3; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - - List entityIdsWithMissingEntity = new ArrayList<>(entityIds); - entityIdsWithMissingEntity.add("100"); - List result = - featureStorage.getNLatestFeaturesWithinTimestampRange( - ENTITY_NAME, entityIdsWithMissingEntity, featureSpecLimitPairs, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, featureSpecLimitPairs, tsRange); - } - } - - - private FeatureSpec createFeatureSpec(String featureName, Enum granularity, ValueType.Enum valType) { + private FeatureSpec createFeatureSpec( + String featureName, Enum granularity, ValueType.Enum valType) { String entityName = ENTITY_NAME; String featureId = String.format("%s.%s.%s", entityName, granularity.toString().toLowerCase(), featureName); diff --git a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java index 14509804eae..11b3f8fa87c 100644 --- a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java +++ b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java @@ -27,13 +27,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.RequestType; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.config.AppConfig; -import feast.serving.model.Pair; -import feast.serving.model.RequestDetailWithSpec; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; @@ -71,22 +65,18 @@ public void setUp() throws Exception { public void shouldUseCurrentThreadIfRequestIsSmallEnough() { String entityName = "entity"; FeatureStorage featureStorage = mock(FeatureStorage.class); - when(featureStorage.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(featureStorage.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); when(featureStorageRegistry.get(any(String.class))).thenReturn(featureStorage); String featureId = "entity.none.feature_1"; FeatureSpec featureSpec = FeatureSpec.newBuilder().setId(featureId).build(); - RequestDetail requestDetail = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId).build(); - RequestDetailWithSpec requestDetailWithSpec = - new RequestDetailWithSpec(requestDetail, featureSpec); dispatcher.dispatchFeatureRetrieval( - entityName, entityIds, Collections.singletonList(requestDetailWithSpec), null); + entityName, entityIds, Collections.singletonList(featureSpec)); verifyZeroInteractions(executorService); verify(featureStorage) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec)); + .getFeature(entityName, entityIds, Collections.singletonList(featureSpec)); } @Test @@ -96,62 +86,32 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { FeatureStorage redis1 = mock(FeatureStorage.class); when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); - when(redis1.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(redis1.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); String entityName = "entity"; String featureId1 = "entity.none.feature_1"; - int limit1 = 5; FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setId(featureId1) .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - RequestDetail requestDetail1 = - RequestDetail.newBuilder() - .setType(RequestType.LIST) - .setLimit(limit1) - .setFeatureId(featureId1) - .build(); String featureId2 = "entity.none.feature_2"; - int limit2 = 1; FeatureSpec featureSpec2 = FeatureSpec.newBuilder() - .setId(featureId1) + .setId(featureId2) .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - RequestDetail requestDetail2 = - RequestDetail.newBuilder() - .setType(RequestType.LIST) - .setLimit(limit2) - .setFeatureId(featureId2) - .build(); - RequestDetailWithSpec requestDetailWithSpec1 = - new RequestDetailWithSpec(requestDetail1, featureSpec1); - RequestDetailWithSpec requestDetailWithSpec2 = - new RequestDetailWithSpec(requestDetail2, featureSpec2); - - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(0)) - .setEnd(Timestamps.fromSeconds(10)) - .build(); dispatcher.dispatchFeatureRetrieval( entityName, entityIds, - Arrays.asList(requestDetailWithSpec1, requestDetailWithSpec2), - tsRange); + Arrays.asList(featureSpec1, featureSpec2)); - verify(redis1) - .getNLatestFeaturesWithinTimestampRange( - entityName, - entityIds, - Arrays.asList(new Pair<>(featureSpec1, limit1), new Pair<>(featureSpec2, limit2)), - tsRange); + verify(redis1).getFeature(entityName, entityIds, Arrays.asList(featureSpec1, featureSpec2)); verifyZeroInteractions(executorService); } @@ -165,60 +125,9 @@ public void shouldUseExecutorServiceIfRequestFromMoreThanOneStorage() { when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); when(featureStorageRegistry.get(storageId2)).thenReturn(redis2); - when(redis1.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) - .thenReturn(Collections.emptyList()); - when(redis2.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) - .thenReturn(Collections.emptyList()); - - String entityName = "entity"; - String featureId1 = "entity.none.feature_1"; - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() - .setId(featureId1) - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) - .build(); - RequestDetail requestDetail1 = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId1).build(); - - String featureId2 = "entity.none.feature_2"; - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setId(featureId1) - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId2))) - .build(); - RequestDetail requestDetail2 = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId2).build(); - - RequestDetailWithSpec requestDetailWithSpec1 = - new RequestDetailWithSpec(requestDetail1, featureSpec1); - RequestDetailWithSpec requestDetailWithSpec2 = - new RequestDetailWithSpec(requestDetail2, featureSpec2); - - dispatcher.dispatchFeatureRetrieval( - entityName, entityIds, Arrays.asList(requestDetailWithSpec1, requestDetailWithSpec2), null); - - verify(redis1) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec1)); - verify(redis2) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec2)); - verify(executorService, atLeast(2)).submit(any(Callable.class)); - } - - @Test - public void shouldUseExecutorServiceIfMultipleRequestType() { - String storageId1 = "REDIS1"; - String storageId2 = "REDIS2"; - - FeatureStorage redis1 = mock(FeatureStorage.class); - FeatureStorage redis2 = mock(FeatureStorage.class); - when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); - when(featureStorageRegistry.get(storageId2)).thenReturn(redis2); - - when(redis1.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(redis1.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); - when(redis2.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(redis2.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); String entityName = "entity"; @@ -229,45 +138,20 @@ public void shouldUseExecutorServiceIfMultipleRequestType() { .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - RequestDetail requestDetail1 = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId1).build(); String featureId2 = "entity.none.feature_2"; - int limit = 5; FeatureSpec featureSpec2 = FeatureSpec.newBuilder() - .setId(featureId1) + .setId(featureId2) .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId2))) .build(); - RequestDetail requestDetail2 = - RequestDetail.newBuilder() - .setType(RequestType.LIST) - .setLimit(limit) - .setFeatureId(featureId2) - .build(); - - RequestDetailWithSpec requestDetailWithSpec1 = - new RequestDetailWithSpec(requestDetail1, featureSpec1); - RequestDetailWithSpec requestDetailWithSpec2 = - new RequestDetailWithSpec(requestDetail2, featureSpec2); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(0)) - .setEnd(Timestamps.fromSeconds(10)) - .build(); dispatcher.dispatchFeatureRetrieval( - entityName, - entityIds, - Arrays.asList(requestDetailWithSpec1, requestDetailWithSpec2), - tsRange); + entityName, entityIds, Arrays.asList(featureSpec1, featureSpec2)); - verify(redis1) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec1)); - verify(redis2) - .getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, Collections.singletonList(new Pair<>(featureSpec2, limit)), tsRange); + verify(redis1).getFeature(entityName, entityIds, Collections.singletonList(featureSpec1)); + verify(redis2).getFeature(entityName, entityIds, Collections.singletonList(featureSpec2)); verify(executorService, atLeast(2)).submit(any(Callable.class)); } diff --git a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java index c6cd539069e..be70fee48db 100644 --- a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java +++ b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java @@ -18,15 +18,9 @@ package feast.serving.service; import static junit.framework.TestCase.fail; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; import feast.serving.testutil.RedisPopulator; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; @@ -37,9 +31,7 @@ import io.opentracing.util.GlobalTracer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; -import org.joda.time.Duration; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -50,7 +42,6 @@ public class RedisFeatureStorageTest { public static final String REDIS_HOST = "localhost"; public static final int REDIS_PORT = 6377; - private static final int NUM_OF_DAYS_DATA = 2; // embedded redis RedisServer redisServer; @@ -61,8 +52,7 @@ public class RedisFeatureStorageTest { private List entityIds; private String entityName; - private Timestamp end; - private Timestamp start; + private Timestamp now; @Before public void setUp() throws Exception { @@ -81,68 +71,11 @@ public void setUp() throws Exception { entityIds = createEntityIds(10); entityName = "entity"; - end = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - start = Timestamps.subtract(end, Durations.fromSeconds(NUM_OF_DAYS_DATA * 24 * 60 * 60)); + now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); } @Test - public void getCurrentFeature_shouldWorkForGranularityNone() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setId("entity.none.feature_1") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Collections.singletonList(featureSpec); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); - - List result = redisFs.getCurrentFeature(entityName, entityIds, featureSpec); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldNotReturnMissingValue() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setId("entity.none.feature_1") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Collections.singletonList(featureSpec); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); - - // add entity without feature - List requestEntityIds = new ArrayList<>(entityIds); - requestEntityIds.add("100"); - List result = - redisFs.getCurrentFeature(entityName, requestEntityIds, featureSpec); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldReturnLastValueForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1",granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - List result = redisFs.getCurrentFeature(entityName, entityIds, spec); - redisPopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } - } - - - @Test - public void getCurrentFeatures_shouldWorkForGranularityNone() { + public void getFeatures_shouldNotReturnMissingValue() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setId("entity.none.feature_1") @@ -160,292 +93,40 @@ public void getCurrentFeatures_shouldWorkForGranularityNone() { .build(); List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); - - List result = redisFs.getCurrentFeatures(entityName, entityIds, featureSpecs); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeatures_shouldNotReturnMissingValue() { - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() - .setId("entity.none.feature_1") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setId("entity.none.feature_2") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); + redisPopulator.populate(entityName, entityIds, featureSpecs, now); // add entity without feature List requestEntityIds = new ArrayList<>(entityIds); requestEntityIds.add("100"); - List result = - redisFs.getCurrentFeatures(entityName, requestEntityIds, featureSpecs); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); + List result = redisFs.getFeature(entityName, requestEntityIds, featureSpecs); + redisPopulator.validate(result, entityIds, featureSpecs); } @Test - public void getCurrentFeatures_shouldReturnLastValueForOtherGranularity() { + public void getFeatures_shouldReturnLastValue() { for (Granularity.Enum granularity : Granularity.Enum.values()) { if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { continue; } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); + FeatureSpec spec1 = createFeatureSpec("feature_1", Enum.NONE); + FeatureSpec spec2 = createFeatureSpec("feature_2", granularity); List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - List result = redisFs.getCurrentFeatures(entityName, entityIds, featureSpecs); - redisPopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldWorkForTimeseriesData() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); + redisPopulator.populate(entityName, entityIds, featureSpecs, now); - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); + List result = redisFs.getFeature(entityName, entityIds, featureSpecs); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, entityIds, featureSpecLimitPair, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldNotReturnMissingEntity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - - List entityIdsWithMissing = new ArrayList<>(entityIds); - entityIdsWithMissing.add("100"); // missing entity - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, entityIdsWithMissing, featureSpecLimitPair, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldNotReturnMissingValueInCertainTimestamp() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, entityIds, featureSpecLimitPair, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldReturnEmptyListIfNoFeatureIsRetrieved() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - - String missingEntity = "missingEntity"; - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, Arrays.asList(missingEntity), featureSpecLimitPair, tsRange); - assertThat(result.size(), equalTo(0)); - } - } - - - @Test - public void getNLatestFeaturesWithinTimerange_shouldWorkForTimeseriesData() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit1 = 2; - int limit2 = 3; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, featureSpecLimitPairs, tsRange); - } - } - - @Test - public void getNLatestFeaturesWithinTimerange_shouldNotReturnMissingEntity() { - - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit1 = 5; - int limit2 = 1; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - - List entityIdsWithMissing = new ArrayList<>(entityIds); - entityIdsWithMissing.add("100"); // missing entity - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, entityIdsWithMissing, featureSpecLimitPairs, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, featureSpecLimitPairs, tsRange); - } - } - - @Test - public void getNLatestFeaturesWithinTimerange_shouldNotReturnMissingValueInCertainTimestamp() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit1 = 5; - int limit2 = 10; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, featureSpecLimitPairs, tsRange); - } - } - - @Test - public void getNLatestFeaturesWithinTimerange_shouldReturnEmptyListIfNoFeatureIsRetrieved() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit1 = 5; - int limit2 = 10; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - - String missingEntity = "missingEntity"; - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, Arrays.asList(missingEntity), featureSpecLimitPairs, tsRange); - assertThat(result.size(), equalTo(0)); + redisPopulator.validate(result, entityIds, featureSpecs); } } private FeatureSpec createFeatureSpec(String featureName, Enum granularity) { - DataStore servingDatastoreSpec = - DataStore.newBuilder() - .putOptions( - RedisFeatureStorage.OPT_REDIS_BUCKET_SIZE, - Duration.standardSeconds(86400).toString()) - .build(); + DataStore servingDatastoreSpec = DataStore.newBuilder().setId("REDIS").build(); return createFeatureSpec(featureName, granularity, ValueType.Enum.STRING, servingDatastoreSpec); } - private FeatureSpec createFeatureSpec(String featureName, - Enum granularity, ValueType.Enum valType, DataStore dataStoreSpec) { + private FeatureSpec createFeatureSpec( + String featureName, Enum granularity, ValueType.Enum valType, DataStore dataStoreSpec) { String entityName = "entity"; String featureId = String.format("%s.%s.%s", entityName, granularity.toString().toLowerCase(), featureName); diff --git a/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java b/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java index c2317f040b9..9f7b2292b38 100644 --- a/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java +++ b/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java @@ -17,58 +17,56 @@ package feast.serving.testutil; -import com.google.protobuf.Duration; +import static junit.framework.TestCase.fail; + import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableExistsException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.BigTableProto.BigTableRowKey; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueType; - import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; - -import static junit.framework.TestCase.fail; - -/** Helper class to populate a BigTable instance with fake data. */ +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; + +/** + * Helper class to populate a BigTable instance with fake data. It mimic ingesting data to BigTable. + */ public class BigTablePopulator extends FeatureStoragePopulator { - private final Connection connection; private static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(); + private static final String LATEST_KEY = "0"; + private final Connection connection; public BigTablePopulator(Connection connection) { this.connection = connection; } /** - * Populate big table instance with fake data ranging between 'start' and 'end'. The data will be - * dense, meaning for each entity the data will be available for each granularity units. (except - * for granularity second, for which the data is available every 30 seconds) + * Populate big table instance with fake data. * - * @param entityIds - * @param featureSpecs - * @param start - * @param end + * @param entityName entity name of the feature + * @param entityIds collection of entity ID for which the feature should be populated + * @param featureSpecs collection of feature specs for which the feature should be populated + * @param timestamp timestamp of the features */ @Override public void populate( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { createTableIfNecessary(entityName); - populateTableWithFakeData(entityName, entityIds, featureSpecs, start, end); + populateTableWithFakeData(entityName, entityIds, featureSpecs, timestamp); } private void createTableIfNecessary(String entityName) { @@ -97,17 +95,15 @@ private void populateTableWithFakeData( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { TableName tableName = TableName.valueOf(entityName); try (Table table = connection.getTable(tableName)) { for (FeatureSpec featureSpec : featureSpecs) { for (String entityId : entityIds) { - Timestamp roundedStart = - TimeUtil.roundFloorTimestamp(start, featureSpec.getGranularity()); - Timestamp roundedEnd = TimeUtil.roundFloorTimestamp(end, featureSpec.getGranularity()); - List puts = makeMultiplePut(entityId, featureSpec, roundedStart, roundedEnd); - table.put(puts); + Timestamp roundedTimestamp = + TimeUtil.roundFloorTimestamp(timestamp, featureSpec.getGranularity()); + Put put = makePut(entityId, featureSpec, roundedTimestamp); + table.put(put); } } @@ -118,45 +114,11 @@ private void populateTableWithFakeData( } } - private List makeMultiplePut( - String entityId, FeatureSpec fs, Timestamp start, Timestamp end) { - String featureId = fs.getId(); - List puts = new ArrayList<>(); - if (fs.getGranularity().equals(Enum.NONE)) { - Value val = createValue(entityId, featureId, fs.getValueType()); - puts.add(makeSinglePutForGranularityNone(entityId, fs.getId(), val)); - return puts; - } - - Duration timesteps = getTimestep(fs.getGranularity()); - for (Timestamp iter = end; - Timestamps.compare(iter, start) > 0; - iter = Timestamps.subtract(iter, timesteps)) { - Timestamp roundedIter = TimeUtil.roundFloorTimestamp(iter, fs.getGranularity()); - if (Timestamps.compare(iter, end) == 0) { - Value value = createValue(entityId, featureId, iter, fs.getValueType()); - BigTableRowKey rowKey = - BigTableRowKey.newBuilder() - .setEntityKey(entityId) - .setReversedMillis("0") - .setSha1Prefix(DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7)) - .build(); - Put put = new Put(rowKey.toByteArray()); - long ts = Timestamps.toMillis(roundedIter); - put.addColumn(DEFAULT_COLUMN_FAMILY, fs.getId().getBytes(), ts, value.toByteArray()); - puts.add(put); - } - puts.add(makeSinglePut(entityId, fs, roundedIter)); - } - - return puts; - } - - private Put makeSinglePut(String entityId, FeatureSpec fs, Timestamp roundedTimestamp) { + private Put makePut(String entityId, FeatureSpec fs, Timestamp roundedTimestamp) { BigTableRowKey rowKey = BigTableRowKey.newBuilder() .setEntityKey(entityId) - .setReversedMillis(roundedReversedMillis(roundedTimestamp, fs.getGranularity())) + .setReversedMillis(LATEST_KEY) .setSha1Prefix(DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7)) .build(); Put put = new Put(rowKey.toByteArray()); @@ -165,41 +127,4 @@ private Put makeSinglePut(String entityId, FeatureSpec fs, Timestamp roundedTime put.addColumn(DEFAULT_COLUMN_FAMILY, fs.getId().getBytes(), timestamp, val.toByteArray()); return put; } - - private Put makeSinglePutForGranularityNone(String entityId, String featureId, Value value) { - BigTableRowKey rowKey = - BigTableRowKey.newBuilder() - .setEntityKey(entityId) - .setReversedMillis("0") - .setSha1Prefix(DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7)) - .build(); - Put put = new Put(rowKey.toByteArray()); - put.addColumn(DEFAULT_COLUMN_FAMILY, featureId.getBytes(), 0, value.toByteArray()); - return put; - } - - String roundedReversedMillis(Timestamp ts, Enum granularity) { - if (granularity.equals(Enum.NONE)) { - return "0"; - } - Timestamp roundedEnd = TimeUtil.roundFloorTimestamp(ts, granularity); - return String.valueOf(Long.MAX_VALUE - roundedEnd.getSeconds() * 1000); - } - - // use at least hourly time step since big table is slow - @Override - protected Duration getTimestep(Enum granularity) { - switch (granularity) { - case SECOND: - return Durations.fromSeconds(60 * 60); - case MINUTE: - return Durations.fromSeconds(60 * 60); - case HOUR: - return Durations.fromSeconds(60 * 60); - case DAY: - return Durations.fromSeconds(24 * 60 * 60); - default: - throw new IllegalArgumentException("unsupported granularity: " + granularity); - } - } } diff --git a/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java b/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java index 8cbea948869..823be09d212 100644 --- a/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java +++ b/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java @@ -20,20 +20,11 @@ import static java.util.stream.Collectors.groupingBy; import static org.apache.hadoop.hbase.shaded.org.junit.Assert.assertNotNull; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.assertThat; -import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; import feast.types.ValueProto.ValueType; import java.util.Collection; @@ -49,15 +40,13 @@ public abstract class FeatureStoragePopulator { * @param entityName entity name. * @param entityIds collection of entity ID to be added. * @param featureSpecs collection of feature's spec for which the data should be added. - * @param start oldest timestamp of data that should be added. (exclusive) - * @param end newest timestamp of data that should be added. (inclusive) + * @param timestamp event timestamp of data */ public abstract void populate( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end); + Timestamp timestamp); /** * Create a historical feature value based on parameters. @@ -81,75 +70,10 @@ protected Value createValue( } } - /** - * Create single feature value based on parameters. - * - * @param entityId - * @param featureId - * @param valType - * @return - */ - protected Value createValue(String entityId, String featureId, ValueType.Enum valType) { - switch (valType) { - case INT64: - return Value.newBuilder().setInt64Val(Long.parseLong(entityId)).build(); - case STRING: - String value = String.format("%s_%s", entityId, featureId); - return Value.newBuilder().setStringVal(value).build(); - default: - throw new IllegalArgumentException("not yet supported"); - } - } - - /** - * Get the duration between 2 fake data based on granularity of a feature. - * - * @param granularity feature's granualarity. - * @return duration between two fake time-series data. - */ - protected Duration getTimestep(Enum granularity) { - switch (granularity) { - case SECOND: - return Durations.fromSeconds(30); - case MINUTE: - return Durations.fromSeconds(60); - case HOUR: - return Durations.fromSeconds(60 * 60); - case DAY: - return Durations.fromSeconds(24 * 60 * 60); - default: - throw new IllegalArgumentException("unsupported granularity: " + granularity); - } - } - - public void validateCurrentValueGranularityNone( + public void validate( List result, List entityIds, List featureSpecs) { Map>> entityMap = toEntityMap(result); - assertNotNull(entityMap); - assertThat(entityMap.size(), equalTo(entityIds.size())); - for (String entityId : entityIds) { - Map> featureMap = entityMap.get(entityId); - assertNotNull(featureMap); - assertThat(featureMap.size(), equalTo(featureSpecs.size())); - for (FeatureSpec featureSpec : featureSpecs) { - List featureValueList = featureMap.get(featureSpec.getId()); - assertNotNull(featureValueList); - assertThat(featureValueList.size(), equalTo(1)); - - FeatureValue featureValue = featureValueList.get(0); - validateValue(featureValue, entityId, featureSpec); - } - } - } - - public void validateCurrentValueOtherGranularity( - List result, - List entityIds, - List featureSpecs, - Timestamp lastTimestamp) { - Map>> entityMap = toEntityMap(result); - assertNotNull(entityMap); assertThat(entityMap.size(), equalTo(entityIds.size())); @@ -165,48 +89,10 @@ public void validateCurrentValueOtherGranularity( FeatureValue featureValue = featureValueList.get(0); Timestamp timestamp = featureValue.getTimestamp(); validateValue(featureValue, entityId, featureSpec, timestamp); - lastTimestamp = TimeUtil.roundFloorTimestamp(lastTimestamp, featureSpec.getGranularity()); - assertThat(timestamp, equalTo(lastTimestamp)); - } - } - } - - public void validateValueWithinTimerange( - List result, - List entityIds, - List> featureSpecLimitPairs, - TimestampRange tsRange) { - Map>> entityMap = toEntityMap(result); - - assertNotNull(entityMap); - assertThat(entityMap.size(), equalTo(entityIds.size())); - for (String entityId : entityIds) { - Map> featureMap = entityMap.get(entityId); - assertThat(featureMap.size(), equalTo(featureSpecLimitPairs.size())); - for (Pair featureSpecLimitPair : featureSpecLimitPairs) { - FeatureSpec featureSpec = featureSpecLimitPair.getLeft(); - int limit = featureSpecLimitPair.getRight(); - List featureValues = featureMap.get(featureSpec.getId()); - assertThat(featureValues.size(), lessThanOrEqualTo(limit)); - - for (FeatureValue featureValue : featureValues) { - Timestamp timestamp = featureValue.getTimestamp(); - validateValue(featureValue, entityId, featureSpec, timestamp); - - // check timestamp is within specified range - assertThat(Timestamps.compare(timestamp, tsRange.getEnd()), lessThanOrEqualTo(0)); - assertThat(Timestamps.compare(timestamp, tsRange.getStart()), greaterThanOrEqualTo(0)); - } } } } - private void validateValue(FeatureValue featureValue, String entityId, FeatureSpec featureSpec) { - Value actualValue = featureValue.getValue(); - Value expectedValue = createValue(entityId, featureSpec.getId(), featureSpec.getValueType()); - assertThat(actualValue, equalTo(expectedValue)); - } - private void validateValue( FeatureValue featureValue, String entityId, FeatureSpec featureSpec, Timestamp timestamp) { Value actualValue = featureValue.getValue(); diff --git a/serving/src/test/java/feast/serving/testutil/RedisPopulator.java b/serving/src/test/java/feast/serving/testutil/RedisPopulator.java index b8260cb59df..68340405e6b 100644 --- a/serving/src/test/java/feast/serving/testutil/RedisPopulator.java +++ b/serving/src/test/java/feast/serving/testutil/RedisPopulator.java @@ -17,20 +17,13 @@ package feast.serving.testutil; -import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.util.SpecUtil; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; -import feast.types.GranularityProto.Granularity.Enum; +import java.util.Collection; import org.apache.commons.codec.digest.DigestUtils; import redis.clients.jedis.Jedis; -import redis.clients.jedis.Pipeline; - -import java.util.Collection; public class RedisPopulator extends FeatureStoragePopulator { private final Jedis jedis; @@ -44,76 +37,29 @@ public void populate( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { for (FeatureSpec fs : featureSpecs) { for (String entityId : entityIds) { - if (fs.getGranularity().equals(Enum.NONE)) { - addGranularityNoneData(entityId, fs); - } else { - start = TimeUtil.roundFloorTimestamp(start, fs.getGranularity()); - end = TimeUtil.roundFloorTimestamp(end, fs.getGranularity()); - addOtherGranularityData(entityId, fs, start, end); - } - } - } - } - - /** - * Add time series data for entityId within specified time range. - * - * @param entityId entity id of the data to be added. - * @param fs feature spec of the feature to be added. - * @param start start time of the available data (exclusive). - * @param end end time of the available data (inclusive). - */ - private void addOtherGranularityData( - String entityId, FeatureSpec fs, Timestamp start, Timestamp end) { - Duration dur = getTimestep(fs.getGranularity()); - long bucketSizeInSecond = SpecUtil.getBucketSize(fs).getStandardSeconds(); - String featureId = fs.getId(); - String featureIdSha1Prefix = getFeatureIdSha1Prefix(featureId); - Pipeline pipeline = jedis.pipelined(); - - for (Timestamp iter = end; - Timestamps.compare(iter, start) > 0; - iter = Timestamps.subtract(iter, dur)) { - long bucketId = - TimeUtil.roundFloorTimestamp(iter, fs.getGranularity()).getSeconds() / bucketSizeInSecond; - RedisBucketKey bucketKeyProto = createBucketKey(entityId, featureIdSha1Prefix, bucketId); - RedisBucketKey keyProto = createBucketKey(entityId, featureIdSha1Prefix, iter.getSeconds()); - - RedisBucketValue valueProto = - RedisBucketValue.newBuilder() - .setValue(createValue(entityId, featureId, iter, fs.getValueType())) - .setEventTimestamp(iter) - .build(); - - byte[] bucketKey = bucketKeyProto.toByteArray(); - byte[] key = keyProto.toByteArray(); - byte[] value = valueProto.toByteArray(); - pipeline.set(key, value); - pipeline.zadd(bucketKey, iter.getSeconds(), key); - - // add last value in bucket 0 - if (Timestamps.compare(iter, end) == 0) { - RedisBucketKey lastValueKey = createBucketKey(entityId, featureIdSha1Prefix, 0); - pipeline.set(lastValueKey.toByteArray(), value); + Timestamp roundedTimestamp = TimeUtil.roundFloorTimestamp(timestamp, fs.getGranularity()); + addData(entityId, fs, roundedTimestamp); } } - pipeline.sync(); } /** - * Add singular data as key value. + * Add feature value. * * @param entityId entityId of data to be added. * @param fs feature spec of the feature to be added. + * @param timestamp timestamp of data */ - private void addGranularityNoneData(String entityId, FeatureSpec fs) { + private void addData(String entityId, FeatureSpec fs, Timestamp timestamp) { RedisBucketKey bucketKey = createBucketKey(entityId, getFeatureIdSha1Prefix(fs.getId()), 0); RedisBucketValue bucketValue = - RedisBucketValue.newBuilder().setValue(createValue(entityId, fs.getId(), fs.getValueType())).build(); + RedisBucketValue.newBuilder() + .setValue(createValue(entityId, fs.getId(), timestamp, fs.getValueType())) + .setEventTimestamp(timestamp) + .build(); byte[] key = bucketKey.toByteArray(); byte[] value = bucketValue.toByteArray(); jedis.set(key, value); diff --git a/serving/src/test/java/feast/serving/testutil/TimeUtil.java b/serving/src/test/java/feast/serving/testutil/TimeUtil.java new file mode 100644 index 00000000000..77e4edea978 --- /dev/null +++ b/serving/src/test/java/feast/serving/testutil/TimeUtil.java @@ -0,0 +1,60 @@ +package feast.serving.testutil; + +import com.google.protobuf.Timestamp; +import feast.types.GranularityProto.Granularity.Enum; +import org.joda.time.DateTime; +import org.joda.time.DateTimeField; +import org.joda.time.DateTimeZone; +import org.joda.time.MutableDateTime; + +public class TimeUtil { + public static final int NANO_IN_MILLI = 1000000; + public static final int NANO_IN_MICRO = 1000; + public static final int MILLI_IN_SECOND = 1000; + + /** + * Round down timestamp to the nearest granularity. + * + * @param timestamp original timestamp. + * @param granularity granularity of the rounded timestamp. + * @return + */ + public static Timestamp roundFloorTimestamp(Timestamp timestamp, Enum granularity) { + MutableDateTime dt = new MutableDateTime(DateTimeZone.UTC); + DateTimeField roundingField; + switch (granularity) { + case DAY: + roundingField = dt.getChronology().dayOfMonth(); + break; + case HOUR: + roundingField = dt.getChronology().hourOfDay(); + break; + case MINUTE: + roundingField = dt.getChronology().minuteOfHour(); + break; + case SECOND: + roundingField = dt.getChronology().secondOfMinute(); + break; + case NONE: + return Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); + default: + throw new RuntimeException("Unrecognised time series granularity"); + } + dt.setMillis(timestamp.getSeconds() * 1000 + timestamp.getNanos() / 1000000); + dt.setRounding(roundingField, MutableDateTime.ROUND_FLOOR); + return dateTimeToTimestamp(dt.toDateTime()); + } + + /** + * Convert {@link DateTime} into {@link Timestamp} + * + * @param dateTime + * @return + */ + private static Timestamp dateTimeToTimestamp(DateTime dateTime) { + return Timestamp.newBuilder() + .setSeconds(dateTime.getMillis() / MILLI_IN_SECOND) + .setNanos((int) (dateTime.getMillis() % MILLI_IN_SECOND) * NANO_IN_MILLI) + .build(); + } +} diff --git a/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java b/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java index 526cab17d3d..e2cf1938765 100644 --- a/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java +++ b/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java @@ -19,7 +19,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -27,16 +26,12 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; +import feast.serving.ServingAPIProto; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.FeatureValueList; import feast.serving.model.FeatureValue; -import feast.types.ValueProto.TimestampList; import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueList; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -58,15 +53,14 @@ public void addFeatureValueList_shouldWorkForOneEntityIdAndOneFeatureId() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(1); List featureIds = createFeatureIds(1); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); - List featureValueList = createFeatureValues(entityIds, featureIds, start, end); + Timestamp timestamp = Timestamps.fromSeconds(0); + List featureValueList = createFeatureValues(entityIds, featureIds, timestamp); builder.addFeatureValueList(featureValueList); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -74,14 +68,13 @@ public void addFeatureValueList_shouldWorkForSeveralEntityIdAndOneFeatureId() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(10); List featureIds = createFeatureIds(1); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); - List featureValueList = createFeatureValues(entityIds, featureIds, start, end); + Timestamp timestamp = Timestamps.fromSeconds(0); + List featureValueList = createFeatureValues(entityIds, featureIds, timestamp); builder.addFeatureValueList(featureValueList); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -89,15 +82,14 @@ public void addFeatureValueList_shouldWorkForSeveralEntityIdAndSeveralFeatureId( EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(10); List featureIds = createFeatureIds(10); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); - List featureValueList = createFeatureValues(entityIds, featureIds, start, end); + Timestamp timestamp = Timestamps.fromSeconds(0); + List featureValueList = createFeatureValues(entityIds, featureIds, timestamp); builder.addFeatureValueList(featureValueList); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -105,19 +97,18 @@ public void addFeatureValueList_shouldWorkForMultipleCallOfDifferentEntityId() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(20); List featureIds = createFeatureIds(20); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); + Timestamp timestamp = Timestamps.fromSeconds(0); List featureValueList1 = - createFeatureValues(entityIds.subList(0, 10), featureIds, start, end); + createFeatureValues(entityIds.subList(0, 10), featureIds, timestamp); List featureValueList2 = - createFeatureValues(entityIds.subList(10, 20), featureIds, start, end); + createFeatureValues(entityIds.subList(10, 20), featureIds, timestamp); builder.addFeatureValueList(featureValueList1); builder.addFeatureValueList(featureValueList2); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -125,19 +116,18 @@ public void addFeatureValueList_shouldWorkForMultipleCallOfDifferentFeature() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(20); List featureIds = createFeatureIds(20); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); + Timestamp timestamp = Timestamps.fromSeconds(0); List featureValueList1 = - createFeatureValues(entityIds, featureIds.subList(0, 10), start, end); + createFeatureValues(entityIds, featureIds.subList(0, 10), timestamp); List featureValueList2 = - createFeatureValues(entityIds, featureIds.subList(10, 20), start, end); + createFeatureValues(entityIds, featureIds.subList(10, 20), timestamp); builder.addFeatureValueList(featureValueList1); builder.addFeatureValueList(featureValueList2); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -153,8 +143,7 @@ public void shouldBeThreadSafe() throws Exception { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(20); List featureIds = createFeatureIds(featurePerThread * nbThread); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); + Timestamp timestamp = Timestamps.fromSeconds(0); List> featureValueList = new ArrayList<>(); for (int i = 0; i < nbThread; i++) { @@ -162,8 +151,7 @@ public void shouldBeThreadSafe() throws Exception { createFeatureValues( entityIds, featureIds.subList(i * featurePerThread, (i + 1) * featurePerThread), - start, - end)); + timestamp)); } List> futures = new ArrayList<>(); @@ -192,7 +180,7 @@ public void shouldBeThreadSafe() throws Exception { all.get(); - validate(builder.toEntityMap(), entityIds, featureIds, start, end); + validate(builder.toEntityMap(), entityIds, featureIds, timestamp); assertThat(overlaps.get(), greaterThan(0)); } @@ -200,34 +188,19 @@ private void validate( Map result, List entityIds, List featureIds, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { assertThat(result.size(), equalTo(entityIds.size())); for (String entityId : entityIds) { Entity entity = result.get(entityId); assertNotNull(entity); - Map featureValueListMap = entity.getFeaturesMap(); - assertNotNull(featureValueListMap); - assertThat(featureValueListMap.size(), equalTo(featureIds.size())); + Map featureValueMap = entity.getFeaturesMap(); + assertNotNull(featureValueMap); + assertThat(featureValueMap.size(), equalTo(featureIds.size())); for (String featureId : featureIds) { - FeatureValueList featureValueList = featureValueListMap.get(featureId); - Duration duration = Timestamps.between(start, end); - int count = (int) duration.getSeconds(); - - ValueList valueList = featureValueList.getValueList(); - TimestampList tsList = featureValueList.getTimestampList(); - assertThat(valueList.getInt64List().getValCount(), equalTo(count)); - assertThat(tsList.getValCount(), equalTo(count)); - - // check ordering - for (int i = 0; i < count; i++) { - if (i == 0) { - continue; - } - - // compare that timestamp is ordered in descending order. - assertThat(Timestamps.compare(tsList.getVal(i), tsList.getVal(i - 1)), lessThan(0)); - } + ServingAPIProto.FeatureValue featureValue = featureValueMap.get(featureId); + + assertThat(featureValue.getTimestamp(), equalTo(timestamp)); + assertThat(timestamp.getSeconds(), equalTo(featureValue.getValue().getInt64Val())); } } } @@ -249,20 +222,16 @@ private List createEntityIds(int count) { } private List createFeatureValues( - List entityIds, List featureIds, Timestamp start, Timestamp end) { + List entityIds, List featureIds, Timestamp timestamp) { List featureValues = new ArrayList<>(); for (String entityId : entityIds) { for (String featureId : featureIds) { - for (Timestamp iter = start; - Timestamps.compare(iter, end) < 0; - iter = Timestamps.add(iter, Durations.fromSeconds(1))) { - featureValues.add( - new FeatureValue( - featureId, - entityId, - Value.newBuilder().setInt64Val(iter.getSeconds()).build(), - iter)); - } + featureValues.add( + new FeatureValue( + featureId, + entityId, + Value.newBuilder().setInt64Val(timestamp.getSeconds()).build(), + timestamp)); } } return featureValues; diff --git a/serving/src/test/java/feast/serving/util/FeatureBuilderTest.java b/serving/src/test/java/feast/serving/util/FeatureBuilderTest.java deleted file mode 100644 index c7442b43cc3..00000000000 --- a/serving/src/test/java/feast/serving/util/FeatureBuilderTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.util; - -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.FeatureValueList; -import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueList; -import feast.types.ValueProto.ValueType; -import feast.types.ValueProto.ValueType.Enum; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -public class FeatureBuilderTest { - - private static final Random random = new Random(); - - @Test - public void shouldBeAbleToStoreValueWithoutTimestamp() { - for (ValueType.Enum type : ValueType.Enum.values()) { - if (type.equals(Enum.UNKNOWN) || type.equals(Enum.UNRECOGNIZED)) { - continue; - } - - Value val = createRandomValue(type); - - FeatureValueListBuilder featureBuilder = new FeatureValueListBuilder(); - featureBuilder.addValue(val, null); - - FeatureValueList feature = featureBuilder.build(); - assertThat(getValueList(feature.getValueList()).get(0), equalTo(getValue(val))); - } - } - - @Test - public void shouldBeAbleToStoreValueWithTimestamp() { - for (ValueType.Enum type : ValueType.Enum.values()) { - if (type.equals(Enum.UNKNOWN) || type.equals(Enum.UNRECOGNIZED)) { - continue; - } - - Value val = createRandomValue(type); - - FeatureValueListBuilder featureBuilder = new FeatureValueListBuilder(); - Timestamp ts = Timestamp.newBuilder().setSeconds(1234).build(); - - featureBuilder.addValue(val, ts); - - FeatureValueList feature = featureBuilder.build(); - assertThat(getValueList(feature.getValueList()).get(0), equalTo(getValue(val))); - assertThat(feature.getTimestampList().getVal(0), equalTo(ts)); - } - } - - @Test - public void shouldBeAbleToStoreMultipleValueWithTimestamp() { - - for (ValueType.Enum type : ValueType.Enum.values()) { - if (type.equals(Enum.UNKNOWN) || type.equals(Enum.UNRECOGNIZED)) { - continue; - } - - FeatureValueListBuilder fb = new FeatureValueListBuilder(); - List values = new ArrayList<>(); - List timestamps = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - Value val = createRandomValue(type); - Timestamp ts = Timestamp.newBuilder().setSeconds(i).build(); - fb.addValue(val, ts); - - values.add(val); - timestamps.add(ts); - } - - FeatureValueList feature = fb.build(); - for (int i = 0; i < 10; i++) { - assertThat(getValueList(feature.getValueList()).get(i), equalTo(getValue(values.get(i)))); - assertThat(feature.getTimestampList().getVal(i), equalTo(timestamps.get(i))); - } - } - } - - private Value createRandomValue(ValueType.Enum type) { - Value.Builder valBuilder = Value.newBuilder(); - switch (type) { - case INT32: - valBuilder.setInt32Val(random.nextInt()); - break; - case INT64: - valBuilder.setInt64Val(random.nextLong()); - break; - case BYTES: - byte[] randomBytes = new byte[5]; - random.nextBytes(randomBytes); - valBuilder.setBytesVal(ByteString.copyFrom(randomBytes)); - break; - case STRING: - valBuilder.setStringVal("value_" + random.nextInt()); - break; - case BOOL: - valBuilder.setBoolVal(true); - break; - case FLOAT: - valBuilder.setFloatVal(random.nextFloat()); - break; - case DOUBLE: - valBuilder.setDoubleVal(random.nextDouble()); - break; - case TIMESTAMP: - valBuilder.setTimestampVal(Timestamps.fromSeconds(random.nextInt())); - break; - default: - throw new IllegalArgumentException("unknown type: " + type); - } - - return valBuilder.build(); - } - - private Object getValue(Value value) { - switch (value.getValCase()) { - case FLOATVAL: - return value.getFloatVal(); - case BOOLVAL: - return value.getBoolVal(); - case BYTESVAL: - return value.getBytesVal(); - case INT32VAL: - return value.getInt32Val(); - case INT64VAL: - return value.getInt64Val(); - case STRINGVAL: - return value.getStringVal(); - case DOUBLEVAL: - return value.getDoubleVal(); - case TIMESTAMPVAL: - return value.getTimestampVal(); - default: - throw new IllegalArgumentException("Unknown type"); - } - } - - private List getValueList(ValueList valueList) { - switch (valueList.getValueListCase()) { - case BOOLLIST: - return valueList.getBoolList().getValList(); - case BYTESLIST: - return valueList.getBytesList().getValList(); - case FLOATLIST: - return valueList.getFloatList().getValList(); - case INT32LIST: - return valueList.getInt32List().getValList(); - case INT64LIST: - return valueList.getInt64List().getValList(); - case DOUBLELIST: - return valueList.getDoubleList().getValList(); - case STRINGLIST: - return valueList.getStringList().getValList(); - case TIMESTAMPLIST: - return valueList.getTimestampList().getValList(); - default: - throw new IllegalArgumentException("Unknown type"); - } - } -} From 2e1bc6c171f0249669be4c3c8007cff67d3a34ee Mon Sep 17 00:00:00 2001 From: Tim Sell <63295+tims@users.noreply.github.com> Date: Tue, 26 Feb 2019 16:55:16 +0800 Subject: [PATCH 09/25] Flat JSON for pubsub and text files (#141) * pubsub json support + common transform from value map to featurerow * Text file json now must be flat json + lots of tests for json parsing and value map transforms * fix tests * fix pubsub options test * fix pubsub source json type (and move class packages names a bit again, to align with separate PR refactoring stores) * csv test for duplicate headers * add test for convert types dofn --- .../java/feast/options/OptionsParser.java | 1 - .../common/ValueMapToFeatureRowTransform.java | 122 +++++++++ .../source/csv/CsvFileFeatureSource.java | 58 +---- .../{CsvIO.java => ParseCsvTransform.java} | 77 +++--- .../source/csv/StringToValueMapTransform.java | 61 +++++ .../source/json/JsonFileFeatureSource.java | 32 +-- .../feast/source/json/ParseJsonTransform.java | 99 ++++++++ .../source/kafka/KafkaFeatureSource.java | 9 +- .../source/pubsub/PubSubFeatureSource.java | 77 ++++-- .../feast/ingestion/ImportJobCSVTest.java | 78 ++++++ .../transform/fn/ConvertTypesDoFnTest.java | 97 ++++++++ .../ValueMapToFeatureRowTransformTest.java | 234 ++++++++++++++++++ .../test/java/feast/source/csv/CsvIOTest.java | 56 ----- .../source/csv/ParseCSVTransformTest.java | 155 ++++++++++++ .../csv/StringToValueMapTransformTest.java | 77 ++++++ .../source/json/ParseJsonTransformTest.java | 157 ++++++++++++ .../source/pubsub/PubSubReadOptionsTest.java | 74 ++++++ 17 files changed, 1266 insertions(+), 198 deletions(-) create mode 100644 ingestion/src/main/java/feast/source/common/ValueMapToFeatureRowTransform.java rename ingestion/src/main/java/feast/source/csv/{CsvIO.java => ParseCsvTransform.java} (61%) create mode 100644 ingestion/src/main/java/feast/source/csv/StringToValueMapTransform.java create mode 100644 ingestion/src/main/java/feast/source/json/ParseJsonTransform.java create mode 100644 ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java create mode 100644 ingestion/src/test/java/feast/source/common/ValueMapToFeatureRowTransformTest.java delete mode 100644 ingestion/src/test/java/feast/source/csv/CsvIOTest.java create mode 100644 ingestion/src/test/java/feast/source/csv/ParseCSVTransformTest.java create mode 100644 ingestion/src/test/java/feast/source/csv/StringToValueMapTransformTest.java create mode 100644 ingestion/src/test/java/feast/source/json/ParseJsonTransformTest.java create mode 100644 ingestion/src/test/java/feast/source/pubsub/PubSubReadOptionsTest.java diff --git a/ingestion/src/main/java/feast/options/OptionsParser.java b/ingestion/src/main/java/feast/options/OptionsParser.java index 7eab269637d..aca61d7f144 100644 --- a/ingestion/src/main/java/feast/options/OptionsParser.java +++ b/ingestion/src/main/java/feast/options/OptionsParser.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.module.jsonSchema.JsonSchema; import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator; import com.google.common.collect.Lists; -import feast.source.csv.CsvFileFeatureSource.CsvFileFeatureSourceOptions; import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/ingestion/src/main/java/feast/source/common/ValueMapToFeatureRowTransform.java b/ingestion/src/main/java/feast/source/common/ValueMapToFeatureRowTransform.java new file mode 100644 index 00000000000..490eb62bd61 --- /dev/null +++ b/ingestion/src/main/java/feast/source/common/ValueMapToFeatureRowTransform.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.common; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import feast.ingestion.metrics.FeastMetrics; +import feast.ingestion.model.Values; +import feast.specs.ImportSpecProto.Field; +import feast.specs.ImportSpecProto.Schema; +import feast.types.FeatureProto.Feature; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.Value.ValCase; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; + +public class ValueMapToFeatureRowTransform extends + PTransform>, PCollection> { + + private String entity; + private Schema schema; + + public ValueMapToFeatureRowTransform(String entity, Schema schema) { + this.entity = entity; + this.schema = schema; + } + + @Override + public PCollection expand(PCollection> input) { + return input.apply(ParDo.of(ValueMapToFeatureRowDoFn.of(entity, schema))) + .setCoder(ProtoCoder.of(FeatureRow.class)); + } + + public static class ValueMapToFeatureRowDoFn extends + DoFn, FeatureRow> { + + private String entityIdColumn; + private String timestampColumn; + private com.google.protobuf.Timestamp timestampValue; + private String entity; + private Map fields; + + private ValueMapToFeatureRowDoFn(String entity, Map fields, + String entityIdColumn, + String timestampColumn, + com.google.protobuf.Timestamp timestampValue) { + this.entity = entity; + this.fields = fields; + this.entityIdColumn = entityIdColumn; + this.timestampColumn = timestampColumn; + this.timestampValue = timestampValue; + } + + public static ValueMapToFeatureRowDoFn of(String entity, Schema schema) { + final Map fields = Maps.newHashMap(); + for (Field field : schema.getFieldsList()) { + String displayName = !field.getName().isEmpty() ? field.getName() : field.getFeatureId(); + fields.put(displayName, field); + } + return new ValueMapToFeatureRowDoFn(entity, fields, schema.getEntityIdColumn(), + schema.getTimestampColumn(), schema.getTimestampValue()); + } + + @ProcessElement + public void processElement(ProcessContext context) { + FeatureRow.Builder builder = FeatureRow.newBuilder(); + try { + Map valueMap = context.element(); + builder.setEntityName(entity); + for (Entry entry : valueMap.entrySet()) { + String name = entry.getKey(); + Value value = entry.getValue(); + Field field = fields.get(name); + + if (value == null || value.getValCase().equals(ValCase.VAL_NOT_SET)) { + continue; + } + + // A feature can only be one of these things + if (entityIdColumn.equals(name)) { + builder.setEntityKey(Values.asString(value).getStringVal()); + } else if (timestampColumn.equals(name)) { + builder.setEventTimestamp(Values.asTimestamp(value).getTimestampVal()); + } else if (!Strings.isNullOrEmpty(field.getFeatureId())) { + String featureId = field.getFeatureId(); + builder.addFeatures( + Feature.newBuilder().setId(featureId).setValue(value)); + } + // else silently ignore this column + } + if (!timestampValue.equals(com.google.protobuf.Timestamp.getDefaultInstance())) { + // This overrides any column event timestamp. + builder.setEventTimestamp(timestampValue); + } + context.output(builder.build()); + } catch (Exception e) { + FeastMetrics.inc(builder.build(), "input_errors"); + } + } + } +} \ No newline at end of file diff --git a/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java b/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java index f1a99dd017f..0d713134f39 100644 --- a/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java +++ b/ingestion/src/main/java/feast/source/csv/CsvFileFeatureSource.java @@ -24,25 +24,20 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import feast.ingestion.metrics.FeastMetrics; -import feast.ingestion.model.Values; -import feast.ingestion.util.DateUtil; import feast.options.Options; import feast.options.OptionsParser; import feast.source.FeatureSource; import feast.source.FeatureSourceFactory; -import feast.source.csv.CsvIO.StringMap; +import feast.source.common.ValueMapToFeatureRowTransform.ValueMapToFeatureRowDoFn; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; -import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import javax.validation.constraints.NotEmpty; import lombok.AllArgsConstructor; -import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; @@ -98,51 +93,10 @@ public PCollection expand(PInput input) { Preconditions.checkArgument(fieldNames.contains(timestampColumn), String.format("timestampColumn %s, does not match any field", timestampColumn)); } - - PCollection stringMaps = input.getPipeline().apply(CsvIO.read(path, fieldNames)); - - return stringMaps.apply( - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext context) { - FeatureRow.Builder builder = FeatureRow.newBuilder(); - try { - StringMap stringMap = context.element(); - builder.setEntityName(entity); - - for (Entry entry : stringMap.entrySet()) { - String name = entry.getKey(); - String value = entry.getValue(); - if (value.isEmpty()) { - continue; - } - Field field = fields.get(name); - - // A feature can only be one of these things - if (entityIdColumn.equals(name)) { - builder.setEntityKey(value); - } else if (timestampColumn.equals(name)) { - builder.setEventTimestamp(DateUtil.toTimestamp(value)); - } else if (!Strings.isNullOrEmpty(field.getFeatureId())) { - String featureId = field.getFeatureId(); - builder.addFeatures( - Feature.newBuilder().setId(featureId).setValue(Values.ofString(value))); - } - // else silently ignore this column - } - if (!schema.getTimestampValue() - .equals(com.google.protobuf.Timestamp.getDefaultInstance())) { - // This overrides any column event timestamp. - builder.setEventTimestamp(schema.getTimestampValue()); - } - context.output(builder.build()); - } catch (Exception e) { - FeastMetrics.inc(builder.build(), "input_errors"); - } - } - })); - + PCollection text = input.getPipeline().apply(TextIO.read().from(path)); + return text.apply(ParseCsvTransform.builder().header(fieldNames).build()) + .apply(new StringToValueMapTransform()) + .apply(ParDo.of(ValueMapToFeatureRowDoFn.of(entity, schema))); } public static class CsvFileFeatureSourceOptions implements Options { diff --git a/ingestion/src/main/java/feast/source/csv/CsvIO.java b/ingestion/src/main/java/feast/source/csv/ParseCsvTransform.java similarity index 61% rename from ingestion/src/main/java/feast/source/csv/CsvIO.java rename to ingestion/src/main/java/feast/source/csv/ParseCsvTransform.java index e6cf0f9dc98..95bcdfc5081 100644 --- a/ingestion/src/main/java/feast/source/csv/CsvIO.java +++ b/ingestion/src/main/java/feast/source/csv/ParseCsvTransform.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,21 +26,41 @@ import java.util.stream.Collectors; import lombok.Builder; import lombok.NonNull; -import org.apache.beam.sdk.coders.SerializableCoder; -import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.coders.MapCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PInput; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class CsvIO { - - private static final Logger LOGGER = LoggerFactory.getLogger(CsvIO.class); +/** + * Transform for reading text CSV jsonfiles into a map of strings to strings. CSV format is assumed + * to be RFC4180. + */ +@Builder +public class ParseCsvTransform extends + PTransform, PCollection>> { + + @NonNull + private List header; + + @Override + public PCollection> expand(PCollection input) { + CSVLineParser csvLineParser = new CSVLineParser(header); + return input.apply(ParDo.of(new DoFn>() { + @ProcessElement + public void processElement(ProcessContext context) { + String line = context.element(); + if (line != null && !line.isEmpty()) { + for (StringMap map : csvLineParser.records(line)) { + context.output(map); + } + } + } + })).setCoder(MapCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())); + } public static class StringMap extends HashMap { @@ -59,44 +79,6 @@ public StringMap thisput(String key, String value) { } } - - public static Read read(String inputPath, List header) { - return Read.builder() - .inputPath(inputPath) - .csvLineParser(new CSVLineParser(header)) - .build(); - } - - /** - * Transform for reading text CSV jsonfiles into a map of strings to strings. CSV format is assumed to - * be RFC4180. - */ - @Builder - public static class Read extends PTransform> { - - @NonNull - private final CSVLineParser csvLineParser; - @NonNull - private String inputPath; - - @Override - public PCollection expand(PInput input) { - PCollection text = input.getPipeline().apply(TextIO.read().from(inputPath)); - return text.apply(ParDo.of(new DoFn() { - @ProcessElement - public void processElement(ProcessContext context) { - String line = context.element(); - if (line != null && !line.isEmpty()) { - for (StringMap map : csvLineParser.records(line)) { - context.output(map); - } - } - } - })).setCoder(SerializableCoder.of(StringMap.class)); - } - } - - /** * This is a helper class to make it easy to use the commons csv parser object for one line * without recreating it. @@ -123,5 +105,4 @@ public List records(String input) { } } } - } diff --git a/ingestion/src/main/java/feast/source/csv/StringToValueMapTransform.java b/ingestion/src/main/java/feast/source/csv/StringToValueMapTransform.java new file mode 100644 index 00000000000..c583a077710 --- /dev/null +++ b/ingestion/src/main/java/feast/source/csv/StringToValueMapTransform.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.csv; + +import com.google.common.base.Strings; +import feast.ingestion.model.Values; +import feast.types.ValueProto.Value; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.beam.sdk.coders.MapCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; + +public class StringToValueMapTransform extends + PTransform>, PCollection>> { + + public static MapCoder VALUE_MAP_CODER = MapCoder.of(StringUtf8Coder.of(), + ProtoCoder.of(Value.class)); + + @Override + public PCollection> expand(PCollection> input) { + return input.apply(ParDo.of(new StringToValueMapDoFn())) + .setCoder(VALUE_MAP_CODER); + } + + public static class StringToValueMapDoFn extends + DoFn, Map> { + + @ProcessElement + public void processElement(ProcessContext context) { + Map row = new HashMap<>(); + for (Entry entry : context.element().entrySet()) { + String stringVal = entry.getValue(); + if (!Strings.isNullOrEmpty(stringVal)) { + row.put(entry.getKey(), Values.ofString(entry.getValue())); + } + } + context.output(row); + } + } +} \ No newline at end of file diff --git a/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java b/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java index 08131c16f8b..3420d1f5639 100644 --- a/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java +++ b/ingestion/src/main/java/feast/source/json/JsonFileFeatureSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,19 +20,18 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; +import com.google.common.base.Preconditions; import feast.options.Options; import feast.options.OptionsParser; import feast.source.FeatureSource; import feast.source.FeatureSourceFactory; +import feast.source.common.ValueMapToFeatureRowTransform; import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportSpecProto.Schema; import feast.types.FeatureRowProto.FeatureRow; import javax.validation.constraints.NotEmpty; import lombok.AllArgsConstructor; import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; @@ -58,21 +57,14 @@ public PCollection expand(PInput input) { JsonFileFeatureSourceOptions options = OptionsParser .parse(importSpec.getSourceOptionsMap(), JsonFileFeatureSourceOptions.class); PCollection jsonLines = input.getPipeline().apply(TextIO.read().from(options.path)); - return jsonLines.apply( - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext context) { - String line = context.element(); - FeatureRow.Builder builder = FeatureRow.newBuilder(); - try { - JsonFormat.parser().merge(line, builder); - context.output(builder.build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - })); + + Preconditions + .checkArgument(importSpec.getEntitiesCount() == 1, "Import spec must have only 1 entity"); + String entity = importSpec.getEntities(0); + Schema schema = importSpec.getSchema(); + + return jsonLines.apply(new ParseJsonTransform()) + .apply(new ValueMapToFeatureRowTransform(entity, schema)); } public static class JsonFileFeatureSourceOptions implements Options { diff --git a/ingestion/src/main/java/feast/source/json/ParseJsonTransform.java b/ingestion/src/main/java/feast/source/json/ParseJsonTransform.java new file mode 100644 index 00000000000..ee2b0035853 --- /dev/null +++ b/ingestion/src/main/java/feast/source/json/ParseJsonTransform.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.json; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; +import feast.ingestion.model.Values; +import feast.types.ValueProto.Value; +import java.lang.reflect.Type; +import java.util.Map; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; + +public class ParseJsonTransform extends + PTransform, PCollection>> { + + @Override + public PCollection> expand(PCollection input) { + return input.apply(ParDo.of(new ParseJsonDoFn())); + } + + + public static class ParseJsonDoFn extends DoFn> { + + private transient Gson gson; + private transient Type valueMapType; + + @ProcessElement + public void processElement(ProcessContext context) { + context.output(parseJson(context.element())); + } + + Map parseJson(String json) { + if (gson == null) { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(Value.class, new ValueDeserializer()); + gson = gsonBuilder.create(); + valueMapType = new TypeToken>() { + }.getType(); + } + return gson.fromJson(json, valueMapType); + } + } + + private static class ValueDeserializer implements JsonDeserializer { + + @Override + public Value deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + if (json.isJsonNull()) { + return Value.newBuilder().build(); // return UNKNOWN value. + } else if (json.isJsonPrimitive()) { + JsonPrimitive primitive = json.getAsJsonPrimitive(); + if (primitive.isBoolean()) { + return Values.ofBool(primitive.getAsBoolean()); + } else if (primitive.isString()) { + return Values.ofString(primitive.getAsString()); + } else { + // Find out if it is an int type + Double doubleVal = primitive.getAsDouble(); + Long int64Val = primitive.getAsLong(); + + if (Double.compare(int64Val.doubleValue(), doubleVal) != 0.0) { + return Values.ofDouble(doubleVal); + } else { + return Values.ofInt64(int64Val); + } + } + } else if (json.isJsonArray() || json.isJsonObject()) { + return Values.ofString(json.toString()); + } else { + throw new JsonParseException("Unknown json element type "); + } + } + } +} \ No newline at end of file diff --git a/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java b/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java index 9246c5a9fdc..a91add6924a 100644 --- a/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java +++ b/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java @@ -63,15 +63,12 @@ public PCollection expand(PInput input) { List topicsList = new ArrayList<>(Arrays.asList(options.topics.split(","))); - KafkaIO.Read kafkaIOReader = - KafkaIO.read() + PCollection> featureRowRecord = + input.getPipeline().apply( KafkaIO.read() .withBootstrapServers(options.server) .withTopics(topicsList) .withKeyDeserializer(FeatureRowKeyDeserializer.class) - .withValueDeserializer(FeatureRowDeserializer.class); - - PCollection> featureRowRecord = - input.getPipeline().apply(kafkaIOReader); + .withValueDeserializer(FeatureRowDeserializer.class)); PCollection featureRow = featureRowRecord.apply( ParDo.of( diff --git a/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java b/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java index c8bafbafc70..36f00074aff 100644 --- a/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java +++ b/ingestion/src/main/java/feast/source/pubsub/PubSubFeatureSource.java @@ -19,13 +19,20 @@ import static com.google.common.base.Preconditions.checkArgument; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.service.AutoService; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.Lists; import feast.ingestion.transform.fn.FilterFeatureRowDoFn; import feast.options.Options; import feast.options.OptionsParser; import feast.source.FeatureSource; import feast.source.FeatureSourceFactory; +import feast.source.csv.ParseCsvTransform; +import feast.source.json.ParseJsonTransform; +import feast.source.csv.StringToValueMapTransform; +import feast.source.common.ValueMapToFeatureRowTransform; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.types.FeatureRowProto.FeatureRow; @@ -54,7 +61,8 @@ public class PubSubFeatureSource extends FeatureSource { public static final String PUBSUB_FEATURE_SOURCE_TYPE = "pubsub"; - @NonNull private ImportSpec importSpec; + @NonNull + private ImportSpec importSpec; @Override public PCollection expand(PInput input) { @@ -62,45 +70,84 @@ public PCollection expand(PInput input) { PubSubReadOptions options = OptionsParser.parse(importSpec.getSourceOptionsMap(), PubSubReadOptions.class); - PubsubIO.Read read = readProtos(); - - if (!Strings.isNullOrEmpty(options.subscription)) { - read = read.fromSubscription(options.subscription); - } else if (!Strings.isNullOrEmpty(options.topic)) { - read = read.fromTopic(options.topic); + PCollection featureRows; + switch (options.messageFormat) { + case FEATURE_ROW: + PubsubIO.Read read = fromSubscriptionOrTopic( + PubsubIO.readProtos(FeatureRow.class), options); + featureRows = input.getPipeline().apply(read); + break; + case CSV: + Preconditions.checkArgument(importSpec.getEntitiesCount() == 1, + "pubsub source with format csv, import spec must have one entity"); + featureRows = input.getPipeline() + .apply(fromSubscriptionOrTopic(PubsubIO.readStrings(), options)) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList()).build()) + .apply(new StringToValueMapTransform()) + .apply(new ValueMapToFeatureRowTransform(importSpec.getEntities(0), + importSpec.getSchema())); + break; + case JSON: + Preconditions.checkArgument(importSpec.getEntitiesCount() == 1, + "pubsub source with format json, import spec must have one entity"); + featureRows = input.getPipeline() + .apply(fromSubscriptionOrTopic(PubsubIO.readStrings(), options)) + .apply(new ParseJsonTransform()) + .apply(new ValueMapToFeatureRowTransform(importSpec.getEntities(0), + importSpec.getSchema())); + break; + default: + throw new IllegalArgumentException( + String.format("Unhandled message format %s", options.messageFormat)); } - PCollection featureRow = input.getPipeline().apply(read); - if (options.discardUnknownFeatures) { List featureIds = new ArrayList<>(); - for(Field field: importSpec.getSchema().getFieldsList()) { + for (Field field : importSpec.getSchema().getFieldsList()) { String featureId = field.getFeatureId(); if (!Strings.isNullOrEmpty(featureId)) { featureIds.add(featureId); } } - return featureRow.apply(ParDo.of(new FilterFeatureRowDoFn(featureIds))); + return featureRows.apply(ParDo.of(new FilterFeatureRowDoFn(featureIds))); } - return featureRow; + return featureRows; } - PubsubIO.Read readProtos() { - return PubsubIO.readProtos(FeatureRow.class); + private PubsubIO.Read fromSubscriptionOrTopic(PubsubIO.Read read, + PubSubReadOptions options) { + if (!Strings.isNullOrEmpty(options.subscription)) { + read = read.fromSubscription(options.subscription); + } else if (!Strings.isNullOrEmpty(options.topic)) { + read = read.fromTopic(options.topic); + } + return read; + } + + public enum MessageFormat { + @JsonProperty(value = "featureRow") + FEATURE_ROW, + @JsonProperty(value = "json") + JSON, + @JsonProperty(value = "csv") + CSV } public static class PubSubReadOptions implements Options { + public String subscription; public String topic; + public MessageFormat messageFormat = MessageFormat.FEATURE_ROW; + public boolean discardUnknownFeatures = false; @AssertTrue(message = "subscription or topic must be set") boolean isValid() { return !Strings.isNullOrEmpty(subscription) || !Strings.isNullOrEmpty(topic); } - public boolean discardUnknownFeatures = false; } @AutoService(FeatureSourceFactory.class) public static class Factory implements FeatureSourceFactory { + @Override public String getType() { return PUBSUB_FEATURE_SOURCE_TYPE; diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index 6379e8dfad4..1543df42532 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -179,6 +179,84 @@ public void testImportCSV() throws IOException { testPipeline.run(); } + @Test + public void testImportFileJson() throws IOException { + ImportSpec importSpec = + ProtoUtil.decodeProtoYaml( + "---\n" + + "type: file.json\n" + + "sourceOptions:\n" + + " # path: to be overwritten in tests\n" + + "entities:\n" + + " - testEntity\n" + + "schema:\n" + + " entityIdColumn: id\n" + + " timestampValue: 2018-09-25T00:00:00.000Z\n" + + " fields:\n" + + " - name: id\n" + + " - name: x\n" + + " featureId: testEntity.none.testInt32\n" + + "\n", + ImportSpec.getDefaultInstance()); + + File jsonFile = folder.newFile("data.json"); + Files.asCharSink(jsonFile, Charsets.UTF_8).write("{\"id\":1,\"x\":101}\n{\"id\":2,\"x\":202}\n"); + importSpec = initImportSpec(importSpec, jsonFile.toString()); + + ImportJobPipelineOptions options = initOptions(); + options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); + + Injector injector = + Guice.createInjector( + new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + + ImportJob job = injector.getInstance(ImportJob.class); + injector.getInstance(ImportJob.class); + job.expand(); + + PCollection writtenToServing = + PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + .apply("flatten serving input", Flatten.pCollections()); + + PCollection writtenToWarehouse = + PCollectionList.of( + WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + .apply("flatten warehouse input", Flatten.pCollections()); + + PCollection writtenToErrors = + PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + .apply("flatten errors input", Flatten.pCollections()); + + List expectedRows = + Lists.newArrayList( + normalize( + FeatureRow.newBuilder() + .setGranularity(Granularity.Enum.NONE) + .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEntityKey("1") + .setEntityName("testEntity") + .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) + .build()), + normalize( + FeatureRow.newBuilder() + .setGranularity(Granularity.Enum.NONE) + .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEntityKey("2") + .setEntityName("testEntity") + .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(202))) + .build())); + + PAssert.that(writtenToErrors).satisfies(hasCount(0)); + + PAssert.that(writtenToServing.apply("serving toFeatureRows", new ToOrderedFeatureRows())) + .containsInAnyOrder(expectedRows); + + PAssert.that(writtenToWarehouse.apply("warehouse toFeatureRows", new ToOrderedFeatureRows())) + .containsInAnyOrder(expectedRows); + + testPipeline.run(); + } + @Test public void testImportCSV_withSample1() throws IOException { ImportSpec importSpec = diff --git a/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java b/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java new file mode 100644 index 00000000000..f69753aad52 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.transform.fn; + + +import static feast.FeastMatchers.hasCount; + +import com.google.common.collect.Lists; +import feast.ingestion.model.Features; +import feast.ingestion.model.Specs; +import feast.ingestion.model.Values; +import feast.ingestion.util.DateUtil; +import feast.ingestion.values.PFeatureRows; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.types.FeatureRowExtendedProto.Attempt; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.ValueType.Enum; +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +@Slf4j +public class ConvertTypesDoFnTest { + + @Rule + public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testStringTo() { + FeatureRowExtended row = FeatureRowExtended.newBuilder().setRow( + FeatureRow.newBuilder().addAllFeatures(Lists.newArrayList( + Features.of("STRING_TO_INT32", Values.ofString("123")), + Features.of("STRING_TO_INT64", Values.ofString("123")), + Features.of("STRING_TO_FLOAT", Values.ofString("123")), + Features.of("STRING_TO_DOUBLE", Values.ofString("123")), + Features.of("STRING_TO_STRING", Values.ofString("123")), + Features.of("STRING_TO_BOOL", Values.ofString("true")), + Features.of("STRING_TO_TIMESTAMP", Values.ofString("2019-01-31T19:19:19.123Z")) + ))).build(); + + Map featureSpecs = new HashMap<>(); + featureSpecs.put("STRING_TO_INT32", FeatureSpec.newBuilder().setValueType(Enum.INT32).build()); + featureSpecs.put("STRING_TO_INT64", FeatureSpec.newBuilder().setValueType(Enum.INT64).build()); + featureSpecs.put("STRING_TO_FLOAT", FeatureSpec.newBuilder().setValueType(Enum.FLOAT).build()); + featureSpecs.put("STRING_TO_DOUBLE", FeatureSpec.newBuilder().setValueType(Enum.DOUBLE).build()); + featureSpecs.put("STRING_TO_BOOL", FeatureSpec.newBuilder().setValueType(Enum.BOOL).build()); + featureSpecs.put("STRING_TO_STRING", FeatureSpec.newBuilder().setValueType(Enum.STRING).build()); + featureSpecs.put("STRING_TO_TIMESTAMP", FeatureSpec.newBuilder().setValueType(Enum.TIMESTAMP).build()); + + PFeatureRows output = PFeatureRows.of(pipeline.apply(Create.of(row))) + .applyDoFn("name", + new ConvertTypesDoFn(Specs.builder().featureSpecs(featureSpecs).build())); + + + PAssert.that(output.getErrors()).satisfies(rows -> { + if (rows.iterator().hasNext()) { + log.error(rows.iterator().next().getLastAttempt().getError().toString()); + } + return null; + }); + + PAssert.that(output.getMain()).containsInAnyOrder(FeatureRowExtended.newBuilder().setRow( + FeatureRow.newBuilder().addAllFeatures(Lists.newArrayList( + Features.of("STRING_TO_INT32", Values.ofInt32(123)), + Features.of("STRING_TO_INT64", Values.ofInt64(123L)), + Features.of("STRING_TO_FLOAT", Values.ofFloat(123F)), + Features.of("STRING_TO_DOUBLE", Values.ofDouble(123.0)), + Features.of("STRING_TO_STRING", Values.ofString("123")), + Features.of("STRING_TO_BOOL", Values.ofBool(true)), + Features.of("STRING_TO_TIMESTAMP", + Values.ofTimestamp(DateUtil.toTimestamp("2019-01-31T19:19:19.123Z"))) + ))).setLastAttempt(Attempt.getDefaultInstance()).build()); + pipeline.run(); + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/source/common/ValueMapToFeatureRowTransformTest.java b/ingestion/src/test/java/feast/source/common/ValueMapToFeatureRowTransformTest.java new file mode 100644 index 00000000000..246dc6317de --- /dev/null +++ b/ingestion/src/test/java/feast/source/common/ValueMapToFeatureRowTransformTest.java @@ -0,0 +1,234 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.common; + +import static feast.source.csv.StringToValueMapTransform.VALUE_MAP_CODER; +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.Lists; +import feast.ingestion.model.Features; +import feast.ingestion.model.Values; +import feast.ingestion.util.DateUtil; +import feast.specs.ImportSpecProto.Field; +import feast.specs.ImportSpecProto.Schema; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.joda.time.DateTime; +import org.junit.Rule; +import org.junit.Test; + +public class ValueMapToFeatureRowTransformTest { + + @Rule + public TestPipeline pipeline = TestPipeline.create(); + + + @Test + public void testTimestampValue() { + Schema schema = Schema.newBuilder() + .setTimestampValue(DateUtil.toTimestamp(DateTime.now())) + .build(); + Map map = new HashMap<>(); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals("entity", row.getEntityName()); + assertEquals(schema.getTimestampValue(), row.getEventTimestamp()); + return null; + }); + pipeline.run(); + } + + @Test + public void testTimestampColumn() { + Schema schema = Schema.newBuilder() + .setTimestampColumn("timestamp") + .addFields(Field.newBuilder().setName("c1").build()).build(); + + Map map = new HashMap<>(); + map.put("timestamp", Values.asTimestamp(Values.ofString("2019-01-30T19:30:12Z"))); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals("entity", row.getEntityName()); + assertEquals(map.get("timestamp").getTimestampVal(), row.getEventTimestamp()); + return null; + }); + pipeline.run(); + } + + @Test + public void testTimestampColumnOptional() { + // schema field for timestamp column is optional, as long as it can be provided by the source + Schema schema = Schema.newBuilder() + .setTimestampColumn("timestamp").build(); + + Map map = new HashMap<>(); + map.put("timestamp", Values.asTimestamp(Values.ofString("2019-01-30T19:30:12Z"))); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals("entity", row.getEntityName()); + assertEquals(map.get("timestamp").getTimestampVal(), row.getEventTimestamp()); + return null; + }); + pipeline.run(); + } + + @Test + public void testEntityKeyColumn() { + Schema schema = Schema.newBuilder() + .setEntityIdColumn("driver_id").build(); + + Map map = new HashMap<>(); + map.put("driver_id", Values.ofString("1234")); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals("entity", row.getEntityName()); + assertEquals("1234", row.getEntityKey()); + return null; + }); + pipeline.run(); + } + + @Test + public void testEntityKeyColumnTypeConverted() { + Schema schema = Schema.newBuilder() + .setEntityIdColumn("driver_id").build(); + + Map map = new HashMap<>(); + map.put("driver_id", Values.ofInt64(12345)); //integer Value gets converted to string. + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals("entity", row.getEntityName()); + assertEquals("12345", row.getEntityKey()); + return null; + }); + pipeline.run(); + } + + @Test + public void testFieldPickedWithFeatureId() { + Schema schema = Schema.newBuilder() + .addFields(Field.newBuilder().setName("x").setFeatureId("f1")).build(); + + Map map = new HashMap<>(); + map.put("x", Values.ofInt64(1)); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals(1, row.getFeaturesList().size()); + assertEquals(Features.of("f1", Values.ofInt64(1)), row.getFeatures(0)); + + return null; + }); + pipeline.run(); + } + + @Test + public void testUnsetFeatureValue() { + Schema schema = Schema.newBuilder() + .addFields(Field.newBuilder().setName("x").setFeatureId("f1")).build(); + + Map map = new HashMap<>(); + map.put("x", Value.getDefaultInstance()); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals(0, row.getFeaturesList().size()); + return null; + }); + pipeline.run(); + } + + @Test + public void testFieldNonFeatureIdIgnored() { + Schema schema = Schema.newBuilder() + .addFields(Field.newBuilder().setName("x")).build(); + + Map map = new HashMap<>(); + map.put("x", Values.ofInt64(1)); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals(0, row.getFeaturesList().size()); + return null; + }); + pipeline.run(); + } + + @Test + public void testFeatureWithValueNotSetIsIgnored() { + Schema schema = Schema.newBuilder() + .addFields(Field.newBuilder().setName("x").setFeatureId("f1")).build(); + + Map map = new HashMap<>(); + map.put("x", Value.newBuilder().build()); + + PCollection output = pipeline + .apply(Create.of(Lists.newArrayList(map)).withCoder(VALUE_MAP_CODER)) + .apply(new ValueMapToFeatureRowTransform("entity", schema)); + + PAssert.that(output).satisfies(maps -> { + FeatureRow row = maps.iterator().next(); + assertEquals(0, row.getFeaturesList().size()); + return null; + }); + pipeline.run(); + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/source/csv/CsvIOTest.java b/ingestion/src/test/java/feast/source/csv/CsvIOTest.java deleted file mode 100644 index 5e5217af966..00000000000 --- a/ingestion/src/test/java/feast/source/csv/CsvIOTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.source.csv; - -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.*; - -import com.google.common.collect.Lists; -import feast.source.csv.CsvIO.CSVLineParser; -import feast.source.csv.CsvIO.StringMap; -import java.util.List; -import junit.framework.TestCase; - -public class CsvIOTest extends TestCase { - - public void testCSVLineParser_multiLines() { - CSVLineParser parser = new CSVLineParser(Lists.newArrayList("c1", "c2")); - List actual = parser.records("a,b\nc,d"); - - List expected = Lists.newArrayList( - new StringMap().thisput("c1", "a").thisput("c2", "b"), - new StringMap().thisput("c1", "c").thisput("c2", "d") - ); - assertThat(actual, is(expected)); - } - - public void testCSVLineParser_repeatedLines() { - CSVLineParser parser = new CSVLineParser(Lists.newArrayList("c1", "c2")); - List actual = parser.records("a,b"); - List expected = Lists.newArrayList( - new StringMap().thisput("c1", "a").thisput("c2", "b") - ); - assertThat(actual, is(expected)); - - actual = parser.records("c,d"); - expected = Lists.newArrayList( - new StringMap().thisput("c1", "c").thisput("c2", "d") - ); - assertThat(actual, is(expected)); - } -} diff --git a/ingestion/src/test/java/feast/source/csv/ParseCSVTransformTest.java b/ingestion/src/test/java/feast/source/csv/ParseCSVTransformTest.java new file mode 100644 index 00000000000..5fada5073b5 --- /dev/null +++ b/ingestion/src/test/java/feast/source/csv/ParseCSVTransformTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.csv; + +import static feast.FeastMatchers.hasCount; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import com.google.common.collect.Lists; +import feast.source.csv.ParseCsvTransform.CSVLineParser; +import feast.source.csv.ParseCsvTransform.StringMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.beam.sdk.Pipeline.PipelineExecutionException; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +@Slf4j +public class ParseCSVTransformTest { + + @Rule + public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testEmptyLine() { + PCollection> output = pipeline.apply(Create.of("")) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList()).build()); + PAssert.that(output).satisfies(hasCount(0)); + pipeline.run(); + } + + @Test + public void testAllEmptyFields() { + PCollection> output = pipeline.apply(Create.of(",,")) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList("a", "b", "c")).build()); + PAssert.that(output).satisfies(hasCount(1)); + PAssert.that(output).satisfies(rows -> { + Map row = rows.iterator().next(); + assertEquals(row.size(), 3); + assertEquals(row.get("a"), ""); + assertEquals(row.get("b"), ""); + assertEquals(row.get("c"), ""); + return null; + }); + pipeline.run(); + } + + @Test(expected = PipelineExecutionException.class) + public void testDuplicateHeaders() { + pipeline.apply(Create.of("1,2,3")) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList("a", "a", "a")).build()); + pipeline.run(); + } + + @Test + public void testSomeEmptyFields() { + PCollection> output = pipeline.apply(Create.of("1,,3")) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList("a", "b", "c")).build()); + PAssert.that(output).satisfies(hasCount(1)); + PAssert.that(output).satisfies(rows -> { + Map row = rows.iterator().next(); + assertEquals(row.size(), 3); + assertEquals(row.get("a"), "1"); + assertEquals(row.get("b"), ""); + assertEquals(row.get("c"), "3"); + return null; + }); + pipeline.run(); + } + + @Test + public void testTooManyHeaders() { + PCollection> output = pipeline.apply(Create.of("1,2")) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList("a", "b", "c")).build()); + PAssert.that(output).satisfies(hasCount(1)); + PAssert.that(output).satisfies(rows -> { + Map row = rows.iterator().next(); + assertEquals(row.size(), 2); + assertEquals(row.get("a"), "1"); + assertEquals(row.get("b"), "2"); + return null; + }); + pipeline.run(); + } + + @Test + public void testNotEnoughHeaders() { + PCollection> output = pipeline.apply(Create.of("1,2,3")) + .apply(ParseCsvTransform.builder().header(Lists.newArrayList("a", "b")).build()); + PAssert.that(output).satisfies(hasCount(1)); + PAssert.that(output).satisfies(rows -> { + Map row = rows.iterator().next(); + assertEquals(row.size(), 2); + assertEquals(row.get("a"), "1"); + assertEquals(row.get("b"), "2"); + return null; + }); + pipeline.run(); + } + + @Test + public void testCSVLineParser_multiLines() { + CSVLineParser parser = new CSVLineParser(Lists.newArrayList("c1", "c2")); + List actual = parser.records("a,b\nc,d"); + + List expected = Lists.newArrayList( + new StringMap().thisput("c1", "a").thisput("c2", "b"), + new StringMap().thisput("c1", "c").thisput("c2", "d") + ); + assertThat(actual, is(expected)); + } + + @Test(expected = IllegalArgumentException.class) + public void testCSVLineParser_duplicateHeaders() { + CSVLineParser parser = new CSVLineParser(Lists.newArrayList("c1", "c1")); + List actual = parser.records("a,b"); + } + + @Test + public void testCSVLineParser_repeatedLines() { + CSVLineParser parser = new CSVLineParser(Lists.newArrayList("c1", "c2")); + List actual = parser.records("a,b"); + List expected = Lists.newArrayList( + new StringMap().thisput("c1", "a").thisput("c2", "b") + ); + assertThat(actual, is(expected)); + + actual = parser.records("c,d"); + expected = Lists.newArrayList( + new StringMap().thisput("c1", "c").thisput("c2", "d") + ); + assertThat(actual, is(expected)); + } +} diff --git a/ingestion/src/test/java/feast/source/csv/StringToValueMapTransformTest.java b/ingestion/src/test/java/feast/source/csv/StringToValueMapTransformTest.java new file mode 100644 index 00000000000..3d523a6609f --- /dev/null +++ b/ingestion/src/test/java/feast/source/csv/StringToValueMapTransformTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.csv; + +import static feast.source.csv.StringToValueMapTransform.VALUE_MAP_CODER; +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.Lists; +import feast.ingestion.model.Values; +import feast.source.csv.StringToValueMapTransform; +import feast.types.ValueProto.Value; +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.sdk.coders.MapCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +public class StringToValueMapTransformTest { + + @Rule + public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testEmptyValues() { + Map map = new HashMap<>(); + map.put("a", ""); + map.put("b", ""); + PCollection> input = pipeline.apply(Create.of(Lists.newArrayList(map)) + .withCoder(MapCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))); + + PCollection> output = input.apply(new StringToValueMapTransform()) + .setCoder(VALUE_MAP_CODER); + + PAssert.that(output).containsInAnyOrder(new HashMap<>()); + pipeline.run(); + } + + @Test + public void testStringValues() { + Map map = new HashMap<>(); + map.put("a", "abcd"); + map.put("b", "1234"); + PCollection> input = pipeline.apply(Create.of(Lists.newArrayList(map)) + .withCoder(MapCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))); + + PCollection> output = input.apply(new StringToValueMapTransform()) + .setCoder(VALUE_MAP_CODER); + + PAssert.that(output).satisfies(maps -> { + Map outMap = maps.iterator().next(); + assertEquals(outMap.get("a"), Values.ofString("abcd")); + assertEquals(outMap.get("b"), Values.ofString("1234")); + return null; + }); + pipeline.run(); + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/source/json/ParseJsonTransformTest.java b/ingestion/src/test/java/feast/source/json/ParseJsonTransformTest.java new file mode 100644 index 00000000000..5b19954ca91 --- /dev/null +++ b/ingestion/src/test/java/feast/source/json/ParseJsonTransformTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.json; + +import static org.junit.Assert.assertEquals; + +import feast.ingestion.model.Values; +import feast.source.json.ParseJsonTransform; +import feast.types.ValueProto.Value; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +@Slf4j +public class ParseJsonTransformTest { + + @Rule + public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testJsonBool() { + PCollection> output = pipeline.apply(Create.of("{\"x\": true}")) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals(true, map.get("x").getBoolVal()); + return null; + }); + pipeline.run(); + } + + @Test + public void testJsonString() { + PCollection> output = pipeline.apply(Create.of("{\"x\": \"abc\"}")) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals("abc", map.get("x").getStringVal()); + return null; + }); + pipeline.run(); + } + + + @Test + public void testJsonObject() { + PCollection> output = pipeline.apply(Create.of("{\"x\": {}}")) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals("{}", map.get("x").getStringVal()); + return null; + }); + pipeline.run(); + } + + + @Test + public void testJsonLong() { + PCollection> output = pipeline + .apply(Create + .of(String.format("{\"max\": %s, \"min\": %s}", Long.MAX_VALUE, Long.MIN_VALUE))) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals(Values.ofInt64(Long.MAX_VALUE), map.get("max")); + assertEquals(Values.ofInt64(Long.MIN_VALUE), map.get("min")); + return null; + }); + pipeline.run(); + } + + @Test + public void testJsonInteger() { + PCollection> output = pipeline + .apply(Create + .of(String.format("{\"max\": %s, \"min\": %s}", Integer.MAX_VALUE, Integer.MIN_VALUE))) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals(Values.ofInt64(Integer.MAX_VALUE), map.get("max")); + assertEquals(Values.ofInt64(Integer.MIN_VALUE), map.get("min")); + return null; + }); + pipeline.run(); + } + + + @Test + public void testJsonDouble() { + PCollection> output = pipeline + .apply(Create + .of(String.format("{\"max\": %s, \"min\": %s}", Double.MAX_VALUE, Double.MIN_VALUE))) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals(Values.ofDouble(Double.MAX_VALUE), map.get("max")); + assertEquals(Values.ofDouble(Double.MIN_VALUE), map.get("min")); + return null; + }); + pipeline.run(); + } + + @Test + public void testJsonFloat() { + PCollection> output = pipeline + .apply(Create + .of(String + .format("{\"max\": %s, \"min\": %s, \"x\": %s}", Float.MAX_VALUE, Float.MIN_VALUE, + 0.12345678f))) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals(Values.ofDouble(Float.MAX_VALUE).getDoubleVal(), + map.get("max").getDoubleVal(), Float.MAX_VALUE / 2E7); + assertEquals(Values.ofDouble(0.12345678F).getDoubleVal(), map.get("x").getDoubleVal(), + 0.00000001); + assertEquals(Values.ofDouble(Float.MIN_VALUE).getDoubleVal(), + map.get("min").getDoubleVal(), Float.MIN_VALUE * 2E8); + + return null; + }); + pipeline.run(); + } + + @Test + public void testJsonArray() { + PCollection> output = pipeline.apply(Create.of("{\"x\": []}")) + .apply(new ParseJsonTransform()); + PAssert.that(output).satisfies((maps) -> { + Map map = maps.iterator().next(); + assertEquals("[]", map.get("x").getStringVal()); + return null; + }); + pipeline.run(); + } +} diff --git a/ingestion/src/test/java/feast/source/pubsub/PubSubReadOptionsTest.java b/ingestion/src/test/java/feast/source/pubsub/PubSubReadOptionsTest.java new file mode 100644 index 00000000000..ece96942d3d --- /dev/null +++ b/ingestion/src/test/java/feast/source/pubsub/PubSubReadOptionsTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.source.pubsub; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import feast.options.OptionsParser; +import feast.source.pubsub.PubSubFeatureSource.MessageFormat; +import feast.source.pubsub.PubSubFeatureSource.PubSubReadOptions; +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; + +public class PubSubReadOptionsTest { + + @Test + public void testParseSubscription() { + Map opts = new HashMap<>(); + opts.put("subscription", "foo"); + + PubSubReadOptions options = OptionsParser.parse(opts, PubSubReadOptions.class); + assertFalse(options.discardUnknownFeatures); + assertEquals(options.messageFormat, MessageFormat.FEATURE_ROW); // default format + } + + @Test + public void testParseSubscriptionWithCsvFormat() { + Map opts = new HashMap<>(); + opts.put("subscription", "foo"); + opts.put("messageFormat", "csv"); + + PubSubReadOptions options = OptionsParser.parse(opts, PubSubReadOptions.class); + assertFalse(options.discardUnknownFeatures); + assertEquals(options.messageFormat, MessageFormat.CSV); + } + + @Test + public void testParseSubscriptionWithJsonFormat() { + Map opts = new HashMap<>(); + opts.put("subscription", "foo"); + opts.put("messageFormat", "json"); + + PubSubReadOptions options = OptionsParser.parse(opts, PubSubReadOptions.class); + assertFalse(options.discardUnknownFeatures); + assertEquals(options.messageFormat, MessageFormat.JSON); + } + + @Test + public void testParseSubscriptionWithFeatureRowFormat() { + Map opts = new HashMap<>(); + opts.put("subscription", "foo"); + opts.put("messageFormat", "featureRow"); + + PubSubReadOptions options = OptionsParser.parse(opts, PubSubReadOptions.class); + assertFalse(options.discardUnknownFeatures); + assertEquals(options.messageFormat, MessageFormat.FEATURE_ROW); + } +} From 81f3ede76f52274ec2e28011e60e38cd6936a7cf Mon Sep 17 00:00:00 2001 From: Mansi Date: Fri, 1 Mar 2019 16:23:17 +0800 Subject: [PATCH 10/25] Fix tests (#146) --- .../src/test/java/feast/ingestion/ImportJobCSVTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index 1543df42532..7f67bb15af2 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -215,16 +215,16 @@ public void testImportFileJson() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite().getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite().getInputs()) .apply("flatten errors input", Flatten.pCollections()); List expectedRows = From 0a89bd6659e7560e65a92914e798458642d3c56c Mon Sep 17 00:00:00 2001 From: Mansi Date: Sun, 3 Mar 2019 21:15:18 +0800 Subject: [PATCH 11/25] Add job identifier to FeatureRow (#147) * Add column for job_id in BQ * Add job id for feature row in warehouse store --- .../java/feast/core/storage/BigQueryStorageManager.java | 6 +++++- .../java/feast/core/storage/BigQueryStorageManagerTest.java | 6 ++++-- .../bigquery/FeatureRowToBigQueryTableRowDoFn.java | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java index 5ad463ec4f4..197e0016aeb 100644 --- a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java +++ b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java @@ -44,6 +44,7 @@ public class BigQueryStorageManager implements StorageManager { private static final String FIELD_ID = "id"; private static final String FIELD_EVENT_TIMESTAMP = "event_timestamp"; private static final String FIELD_CREATED_TIMESTAMP = "created_timestamp"; + private static final String FIELD_JOB_ID = "job_id"; private String id; private final BigQuery bigQuery; @@ -103,6 +104,7 @@ public void registerNewFeature(FeatureSpec featureSpec) { createField(FIELD_ID, Enum.STRING, ""), createField(FIELD_EVENT_TIMESTAMP, Enum.TIMESTAMP, FIELD_EVENT_TIMESTAMP), createField(FIELD_CREATED_TIMESTAMP, Enum.TIMESTAMP, FIELD_CREATED_TIMESTAMP), + createField(FIELD_JOB_ID, Enum.STRING, FIELD_JOB_ID), createFeatureField(featureSpec)); TableDefinition tableDefinition = StandardTableDefinition.newBuilder() @@ -137,7 +139,9 @@ public void registerNewFeature(FeatureSpec featureSpec) { f -> !f.equals(FIELD_ID) && !f.equals(FIELD_CREATED_TIMESTAMP) - && !f.equals(FIELD_EVENT_TIMESTAMP)) + && !f.equals(FIELD_EVENT_TIMESTAMP) + && !f.equals(FIELD_JOB_ID)) + .collect(Collectors.toList())); AuditLogger.log( Resource.STORAGE, diff --git a/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java b/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java index 09a2a2d481b..3cbc7069a07 100644 --- a/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java +++ b/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java @@ -115,15 +115,17 @@ public void shouldCreateNewDatasetAndTableAndViewIfNotExist() throws Interrupted actualTable.getTableId().getTable(), equalTo(String.format("%s_%s", entityName, granularity.toString().toLowerCase()))); List fields = actualTable.getDefinition().getSchema().getFields(); - assertThat(fields.size(), equalTo(4)); + assertThat(fields.size(), equalTo(5)); Field idField = fields.get(0); assertThat(idField.getName(), equalTo("id")); Field etsField = fields.get(1); assertThat(etsField.getName(), equalTo("event_timestamp")); Field ctsField = fields.get(2); assertThat(ctsField.getName(), equalTo("created_timestamp")); - Field field = fields.get(3); + Field field = fields.get(4); assertThat(field.getDescription(), equalTo(description)); + Field jobIdField = fields.get(3); + assertThat(jobIdField.getName(), equalTo("job_id")); assertThat(field.getType(), equalTo(LegacySQLTypeName.INTEGER)); assertThat(field.getName(), equalTo(featureName)); diff --git a/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java index c666d4d71ab..828a8cc899f 100644 --- a/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java @@ -36,6 +36,8 @@ public class FeatureRowToBigQueryTableRowDoFn extends DoFn Date: Sun, 10 Mar 2019 12:29:20 +0800 Subject: [PATCH 12/25] Preload spec in serving cache (#152) * Preload spec in serving cache * Disable async refresh and only periodically refresh cache --- serving/pom.xml | 7 ++ .../config/ServingApiConfiguration.java | 30 +++-- .../serving/service/CachedSpecStorage.java | 82 ++++---------- .../src/main/resources/application.properties | 1 + .../service/CachedSpecStorageTest.java | 106 ++++++++++++++++++ 5 files changed, 160 insertions(+), 66 deletions(-) create mode 100644 serving/src/test/java/feast/serving/service/CachedSpecStorageTest.java diff --git a/serving/pom.xml b/serving/pom.xml index 34cfcd777ed..c6db97ec91b 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -232,6 +232,13 @@ 2.23.0 test + + + com.google.guava + guava-testlib + 26.0-jre + test + diff --git a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java index c8e5225a3c5..dcee2f73c65 100644 --- a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java +++ b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java @@ -30,6 +30,8 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -39,15 +41,12 @@ import org.springframework.http.converter.protobuf.ProtobufJsonFormatHttpMessageConverter; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -/** - * Global bean configuration. - */ +/** Global bean configuration. */ @Slf4j @Configuration public class ServingApiConfiguration implements WebMvcConfigurer { - @Autowired - private ProtobufJsonFormatHttpMessageConverter protobufConverter; + @Autowired private ProtobufJsonFormatHttpMessageConverter protobufConverter; @Bean public AppConfig getAppConfig( @@ -66,9 +65,24 @@ public AppConfig getAppConfig( @Bean public SpecStorage getCoreServiceSpecStorage( @Value("${feast.core.host}") String coreServiceHost, - @Value("${feast.core.grpc.port}") String coreServicePort) { - return new CachedSpecStorage( - new CoreService(coreServiceHost, Integer.parseInt(coreServicePort))); + @Value("${feast.core.grpc.port}") String coreServicePort, + @Value("${feast.cacheDurationMinute}") int cacheDurationMinute) { + ScheduledExecutorService scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(); + final CachedSpecStorage cachedSpecStorage = + new CachedSpecStorage(new CoreService(coreServiceHost, Integer.parseInt(coreServicePort))); + + // reload all specs including new ones periodically + scheduledExecutorService.schedule( + () -> cachedSpecStorage.populateCache(), cacheDurationMinute, TimeUnit.MINUTES); + + // load all specs during start up + try { + cachedSpecStorage.populateCache(); + } catch (Exception e) { + log.error("Unable to preload feast's spec"); + } + return cachedSpecStorage; } @Bean diff --git a/serving/src/main/java/feast/serving/service/CachedSpecStorage.java b/serving/src/main/java/feast/serving/service/CachedSpecStorage.java index dcbb74f6c88..bddcdbc23f8 100644 --- a/serving/src/main/java/feast/serving/service/CachedSpecStorage.java +++ b/serving/src/main/java/feast/serving/service/CachedSpecStorage.java @@ -20,19 +20,19 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import lombok.extern.slf4j.Slf4j; import feast.serving.exception.SpecRetrievalException; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; - -import java.time.Duration; import java.util.Collections; import java.util.Map; +import lombok.extern.slf4j.Slf4j; /** SpecStorage implementation with built-in in-memory cache. */ @Slf4j public class CachedSpecStorage implements SpecStorage { + private static final int MAX_SPEC_COUNT = 10000; + private final CoreService coreService; private final LoadingCache entitySpecCache; private final CacheLoader entitySpecLoader; @@ -41,70 +41,24 @@ public class CachedSpecStorage implements SpecStorage { private final LoadingCache storageSpecCache; private final CacheLoader storageSpecLoader; - private static final Duration CACHE_DURATION; - private static final int MAX_SPEC_COUNT = 1000; - - static { - CACHE_DURATION = Duration.ofMinutes(30); - } - public CachedSpecStorage(CoreService coreService) { this.coreService = coreService; entitySpecLoader = - new CacheLoader() { - @Override - public EntitySpec load(String key) throws Exception { - return coreService.getEntitySpecs(Collections.singletonList(key)).get(key); - } - - @Override - public Map loadAll(Iterable keys) throws Exception { - return coreService.getEntitySpecs((Iterable) keys); - } - }; - entitySpecCache = - CacheBuilder.newBuilder() - .maximumSize(MAX_SPEC_COUNT) - .expireAfterAccess(CACHE_DURATION) - .build(entitySpecLoader); + CacheLoader.from( + (String key) -> coreService.getEntitySpecs(Collections.singletonList(key)).get(key)); + entitySpecCache = CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(entitySpecLoader); featureSpecLoader = - new CacheLoader() { - @Override - public FeatureSpec load(String key) throws Exception { - return coreService.getFeatureSpecs(Collections.singletonList(key)).get(key); - } - - @Override - public Map loadAll(Iterable keys) - throws Exception { - return coreService.getFeatureSpecs((Iterable) keys); - } - }; + CacheLoader.from( + (String key) -> coreService.getFeatureSpecs(Collections.singletonList(key)).get(key)); featureSpecCache = - CacheBuilder.newBuilder() - .maximumSize(MAX_SPEC_COUNT) - .expireAfterAccess(CACHE_DURATION) - .build(featureSpecLoader); + CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(featureSpecLoader); storageSpecLoader = - new CacheLoader() { - @Override - public Map loadAll(Iterable keys) - throws Exception { - return coreService.getStorageSpecs((Iterable) keys); - } - - @Override - public StorageSpec load(String key) throws Exception { - return coreService.getStorageSpecs(Collections.singleton(key)).get(key); - } - }; + CacheLoader.from( + (String key) -> coreService.getStorageSpecs(Collections.singletonList(key)).get(key)); storageSpecCache = - CacheBuilder.newBuilder() - .maximumSize(MAX_SPEC_COUNT) - .expireAfterAccess(CACHE_DURATION) - .build(storageSpecLoader); + CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(storageSpecLoader); } @Override @@ -177,4 +131,16 @@ public Map getAllStorageSpecs() { public boolean isConnected() { return coreService.isConnected(); } + + /** Preload all spec into cache. */ + public void populateCache() { + Map featureSpecMap = coreService.getAllFeatureSpecs(); + featureSpecCache.putAll(featureSpecMap); + + Map entitySpecMap = coreService.getAllEntitySpecs(); + entitySpecCache.putAll(entitySpecMap); + + Map storageSpecMap = coreService.getAllStorageSpecs(); + storageSpecCache.putAll(storageSpecMap); + } } diff --git a/serving/src/main/resources/application.properties b/serving/src/main/resources/application.properties index ff14ded7422..0327f187696 100644 --- a/serving/src/main/resources/application.properties +++ b/serving/src/main/resources/application.properties @@ -24,6 +24,7 @@ feast.maxentity=${FEAST_MAX_ENTITY_PER_BATCH:2000} feast.timeout=${FEAST_RETRIEVAL_TIMEOUT:5} feast.redispool.maxsize=${FEAST_REDIS_POOL_MAX_SIZE:128} feast.redispool.maxidle=${FEAST_REDIS_POOL_MAX_IDLE:16} +feast.cacheDurationMinute=${FEAST_SPEC_CACHE_DURATION_MINUTE:5} statsd.host= ${STATSD_HOST:localhost} statsd.port= ${STATSD_PORT:8125} diff --git a/serving/src/test/java/feast/serving/service/CachedSpecStorageTest.java b/serving/src/test/java/feast/serving/service/CachedSpecStorageTest.java new file mode 100644 index 00000000000..76a7b780618 --- /dev/null +++ b/serving/src/test/java/feast/serving/service/CachedSpecStorageTest.java @@ -0,0 +1,106 @@ +package feast.serving.service; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.testing.FakeTicker; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.StorageSpecProto.StorageSpec; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.Before; +import org.junit.Test; + +public class CachedSpecStorageTest { + + private CoreService coreService; + private CachedSpecStorage cachedSpecStorage; + + @Before + public void setUp() throws Exception { + coreService = mock(CoreService.class); + cachedSpecStorage = new CachedSpecStorage(coreService); + } + + @Test + public void testPopulateCache() { + Map featureSpecMap = new HashMap<>(); + featureSpecMap.put("feature_1", mock(FeatureSpec.class)); + + Map storageSpecMap = new HashMap<>(); + storageSpecMap.put("storage_1", mock(StorageSpec.class)); + + Map entitySpecMap = new HashMap<>(); + entitySpecMap.put("entity_1", mock(EntitySpec.class)); + + when(coreService.getAllFeatureSpecs()).thenReturn(featureSpecMap); + when(coreService.getAllEntitySpecs()).thenReturn(entitySpecMap); + when(coreService.getAllStorageSpecs()).thenReturn(storageSpecMap); + + cachedSpecStorage.populateCache(); + Map result = + cachedSpecStorage.getFeatureSpecs(Collections.singletonList("feature_1")); + Map result1 = + cachedSpecStorage.getStorageSpecs(Collections.singletonList("storage_1")); + Map result2 = + cachedSpecStorage.getEntitySpecs(Collections.singletonList("entity_1")); + + assertThat(result.size(), equalTo(1)); + assertThat(result1.size(), equalTo(1)); + assertThat(result2.size(), equalTo(1)); + + verify(coreService, times(0)).getFeatureSpecs(any(Iterable.class)); + verify(coreService, times(0)).getStorageSpecs(any(Iterable.class)); + verify(coreService, times(0)).getEntitySpecs(any(Iterable.class)); + } + + @Test + public void reloadFailureShouldReturnOldValue() { + Map featureSpecMap = new HashMap<>(); + featureSpecMap.put("feature_1", mock(FeatureSpec.class)); + + Map storageSpecMap = new HashMap<>(); + storageSpecMap.put("storage_1", mock(StorageSpec.class)); + + Map entitySpecMap = new HashMap<>(); + entitySpecMap.put("entity_1", mock(EntitySpec.class)); + + when(coreService.getAllFeatureSpecs()).thenReturn(featureSpecMap); + when(coreService.getFeatureSpecs(any(Iterable.class))).thenThrow(new RuntimeException("error")); + when(coreService.getAllEntitySpecs()).thenReturn(entitySpecMap); + when(coreService.getEntitySpecs(any(Iterable.class))).thenThrow(new RuntimeException("error")); + when(coreService.getAllStorageSpecs()).thenReturn(storageSpecMap); + when(coreService.getStorageSpecs(any(Iterable.class))).thenThrow(new RuntimeException("error")); + + cachedSpecStorage.populateCache(); + Map result = + cachedSpecStorage.getFeatureSpecs(Collections.singletonList("feature_1")); + Map result1 = + cachedSpecStorage.getStorageSpecs(Collections.singletonList("storage_1")); + Map result2 = + cachedSpecStorage.getEntitySpecs(Collections.singletonList("entity_1")); + + assertThat(result.size(), equalTo(1)); + assertThat(result1.size(), equalTo(1)); + assertThat(result2.size(), equalTo(1)); + verify(coreService, times(0)).getFeatureSpecs(any(Iterable.class)); + verify(coreService, times(0)).getStorageSpecs(any(Iterable.class)); + verify(coreService, times(0)).getEntitySpecs(any(Iterable.class)); + + result = cachedSpecStorage.getFeatureSpecs(Collections.singletonList("feature_1")); + result1 = cachedSpecStorage.getStorageSpecs(Collections.singletonList("storage_1")); + result2 = cachedSpecStorage.getEntitySpecs(Collections.singletonList("entity_1")); + assertThat(result.size(), equalTo(1)); + assertThat(result1.size(), equalTo(1)); + assertThat(result2.size(), equalTo(1)); + } +} \ No newline at end of file From 8715fb96de2e3e8eb43b92d7ab475040e5c4830b Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Sun, 17 Mar 2019 19:18:23 +0700 Subject: [PATCH 13/25] Add Feast admin quickstart guide (#155) * Installation quickstart Guide for Feast administrators * Add link to installation docs from main README --- README.md | 4 +++ docs/install.md | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 docs/install.md diff --git a/README.md b/README.md index 6773da01df8..59f8a4402f8 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ __Discovery__: Feast allows users to easily explore and discover features and th * [Components](docs/components.md) * [Concepts](docs/concepts.md) +For Feast administrators: +* [Installation quickstart](docs/install.md} +* [Helm charts](charts/README.md} details + ## Notice Feast is still under active development. Your feedback and contributions are important to us. diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 00000000000..6cab5f80168 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,87 @@ +# Feast Installation Quickstart Guide + +This is a quickstart guide for Feast administrators setting up a Feast +deployment for the first time. + +Feast is meant to run on Kubernetes, and currently requires managed +services from GCP for certain operations +* Dataflow (for loading feature data into Feast) +* BigQuery (to act as feature warehouse) +* Pub/Sub (for event data) +* Cloud Storage (for staging data and logs) + +This guide will assume that users are using GKE (Google Container +Engine), and that these managed services are available. In addition, +Redis will be used for the feature serving database. + +## Prerequisites + +* Kubernetes cluster + * The user should have a GKE cluster provisioned, with `kubectl` set + up to access this cluster + * This cluster should have the right scopes to start jobs on + Dataflow and to modify BigQuery datasets and tables. The simplest + way to set this up is by setting the scope to `cloud-platform` when + provisioning the Kubernetes cluster. +* [Helm](https://helm.sh/) + * Helm should be installed locally and Tiller should be installed + within this cluster. As noted + [here](https://medium.com/google-cloud/helm-on-gke-cluster-quick-hands-on-guide-ecffad94b0), + make sure you have the cluster-admin role attached to the Helm + service account. +* Feast repository + * You have cloned the [Feast + repository](https://github.com/gojek/feast/) and your command line + is active in the root of the repository + +## Set up the environment + +Set the following environmental variables before beginning the installation + +```sh +GCP_PROJECT=my-feast-project +FEAST_CLUSTER=feast +FEAST_REPO=$(pwd) +FEAST_VERSION=0.1.0 +FEAST_STORAGE_BUCKET=gs://${PROJECT}-feast +``` + +Ensure that your `kubectl` context is set to the correct cluster + +```sh +gcloud container clusters get-credentials "${FEAST_CLUSTER}" --project "${GCP_PROJECT}" +``` + +Create a storage bucket for Feast to stage data + +```sh +gsutil mb "${FEAST_STORAGE_BUCKET}" +``` + +## Install Feast using Helm + +Create a copy of the Helm `values.yaml` file. This file will need to +be configured with the specifics of your environment. + +```sh +cp charts/feast/values.yaml . +``` + +Keys that likely need to be changed based on the user’s environment are +* `core.projectId` - your GCP project +* `core.image.tag` - the current Feast version +* `serving.image.tag` - the current Feast version +* `core.jobs.*` + * `DataflowRunner` or `DirectRunner` or `FlinkRunner` settings for Beam + * Note: `DirectRunner` should not be used in production it is for testing only. + +See the [documentation page](#) for the complete list of options. + +You can update `values.yaml` directly, or specify overrides on the Helm command line: + +```sh +helm install --name feast charts/dist/feast-${FEAST_VERSION}.tgz \ +--set core.projectId=${GCP_PROJECT} \ +--set core.image.tag=${FEAST_VERSION} \ +--set serving.image.tag=${FEAST_VERSION} +``` From 1e3e495dffc99bf6cfb39c7456be5a7f30617c31 Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Mon, 18 Mar 2019 12:08:37 +0800 Subject: [PATCH 14/25] Fix typo link typo in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 59f8a4402f8..7bef49fefda 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ __Discovery__: Feast allows users to easily explore and discover features and th * [Concepts](docs/concepts.md) For Feast administrators: -* [Installation quickstart](docs/install.md} -* [Helm charts](charts/README.md} details +* [Installation quickstart](docs/install.md) +* [Helm charts](charts/README.md) details ## Notice From f027a890906cc5587174bdcb40c4a0be75a110ca Mon Sep 17 00:00:00 2001 From: Yu-Xi Lim Date: Tue, 19 Mar 2019 20:22:24 +0800 Subject: [PATCH 15/25] Simplify and document CLI building steps (#158) * Use vendored protoc-gen-go * Simplified CLI building README Building the CLI is now two steps to install dev dependencies and one step to build. * Cleanup Gopkg.toml --- Gopkg.lock | 9 +- Gopkg.toml | 31 +---- cli/README.md | 50 ++++---- .../generated/go/feast/core/CoreService.pb.go | 90 ++++++++------- .../go/feast/core/DatasetService.pb.go | 36 +++--- .../generated/go/feast/core/JobService.pb.go | 60 +++++----- .../generated/go/feast/core/UIService.pb.go | 108 +++++++++--------- .../generated/go/feast/serving/Serving.pb.go | 30 ++--- .../generated/go/feast/specs/EntitySpec.pb.go | 12 +- .../go/feast/specs/FeatureGroupSpec.pb.go | 12 +- .../go/feast/specs/FeatureSpec.pb.go | 24 ++-- .../generated/go/feast/specs/ImportSpec.pb.go | 24 ++-- .../go/feast/specs/StorageSpec.pb.go | 12 +- .../generated/go/feast/storage/BigTable.pb.go | 12 +- protos/generated/go/feast/storage/Redis.pb.go | 22 ++-- protos/generated/go/feast/types/Feature.pb.go | 10 +- .../generated/go/feast/types/FeatureRow.pb.go | 18 +-- .../go/feast/types/FeatureRowExtended.pb.go | 22 ++-- .../go/feast/types/Granularity.pb.go | 16 +-- protos/generated/go/feast/types/Value.pb.go | 74 ++++++------ 20 files changed, 340 insertions(+), 332 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index b3324dc3e9d..18004a22c77 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -18,10 +18,16 @@ version = "v1.0.0" [[projects]] - digest = "1:5ac24c0ad5dc117fb2b6d90d205c876ec599c09cccb702a36586680e09cb20b5" + digest = "1:54a9254d3845e3a80d4957975abfc5e771a49d08379509ec4f9c99c1ccbe13a9" name = "github.com/golang/protobuf" packages = [ "proto", + "protoc-gen-go", + "protoc-gen-go/descriptor", + "protoc-gen-go/generator", + "protoc-gen-go/generator/internal/remap", + "protoc-gen-go/grpc", + "protoc-gen-go/plugin", "ptypes", "ptypes/any", "ptypes/duration", @@ -259,6 +265,7 @@ input-imports = [ "github.com/ghodss/yaml", "github.com/golang/protobuf/proto", + "github.com/golang/protobuf/protoc-gen-go", "github.com/golang/protobuf/ptypes/empty", "github.com/golang/protobuf/ptypes/timestamp", "github.com/google/go-cmp/cmp", diff --git a/Gopkg.toml b/Gopkg.toml index 9d9e7e9ec12..c06fd9fc914 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,29 +1,4 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - +required = ["github.com/golang/protobuf/protoc-gen-go"] [[constraint]] name = "github.com/ghodss/yaml" @@ -49,10 +24,6 @@ name = "github.com/spf13/viper" version = "1.2.1" -[[constraint]] - branch = "master" - name = "golang.org/x/net" - [[constraint]] name = "google.golang.org/grpc" version = "1.15.0" diff --git a/cli/README.md b/cli/README.md index aeb665b74de..d35ca2d3a5c 100644 --- a/cli/README.md +++ b/cli/README.md @@ -1,6 +1,7 @@ # Feast CLI -The feast command-line tool, `feast`, is used to register resources to feast, as well as manage and run ingestion jobs. +The feast command-line tool, `feast`, is used to register resources to +feast, as well as manage and run ingestion jobs. ## Installation @@ -10,32 +11,39 @@ The quickest way to get the CLI is to download the compiled binary: #TODO ### Building from source -The following dependencies are required to install the CLI from source: -#### protoc -To install protoc, find the compiled binary for your distribution [here](https://github.com/protocolbuffers/protobuf/releases), then install it: -``` -PROTOC_VERSION=3.3.0 +The following dependencies are required to build the CLI from source: +* [`go`](https://golang.org/) +* [`protoc`](https://developers.google.com/protocol-buffers/) +* [`dep`](https://github.com/golang/dep) + +See below for specific instructions on how to install the dependencies. -curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP -unzip -o $PROTOC_ZIP -d /usr/local bin/protoc -rm -f $PROTOC_ZIP +After the dependencies are installed, you can build the CLI using: +```sh +# at feast top-level directory +$ make build-cli ``` -#### dep -On MacOS you can install or upgrade to the latest released version with Homebrew: +### Dependencies -``` -brew install dep -brew upgrade dep -``` +#### `protoc-gen-go` -On other platforms you can use the install.sh script: -``` -curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +To ensure you have a matching version of `protoc-gen-go`, install the vendored version: +```sh +$ go install ./vendor/github.com/golang/protobuf/protoc-gen-go +$ which protoc-gen-go +~/go/bin/protoc-gen-go ``` -Then just install the cli using: +#### `dep` + +On MacOS you can install or upgrade to the latest released version with Homebrew: +```sh +$ brew install dep +$ brew upgrade dep ``` -# at feast top-level directory -make install-cli + +On other platforms you can use the `install.sh` script: +```sh +$ curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh ``` diff --git a/protos/generated/go/feast/core/CoreService.pb.go b/protos/generated/go/feast/core/CoreService.pb.go index d8224c13a9b..76f38974337 100644 --- a/protos/generated/go/feast/core/CoreService.pb.go +++ b/protos/generated/go/feast/core/CoreService.pb.go @@ -35,7 +35,7 @@ func (m *CoreServiceTypes) Reset() { *m = CoreServiceTypes{} } func (m *CoreServiceTypes) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes) ProtoMessage() {} func (*CoreServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0} } func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes.Unmarshal(m, b) @@ -43,8 +43,8 @@ func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes.Merge(m, src) +func (dst *CoreServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes.Merge(dst, src) } func (m *CoreServiceTypes) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes.Size(m) @@ -66,7 +66,7 @@ func (m *CoreServiceTypes_GetEntitiesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetEntitiesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 0} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 0} } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Unmarshal(m, b) @@ -74,8 +74,8 @@ func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(m, src) +func (dst *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(dst, src) } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Size(m) @@ -104,7 +104,7 @@ func (m *CoreServiceTypes_GetEntitiesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 1} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 1} } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Unmarshal(m, b) @@ -112,8 +112,8 @@ func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(m, src) +func (dst *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(dst, src) } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Size(m) @@ -142,7 +142,7 @@ func (m *CoreServiceTypes_ListEntitiesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 2} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 2} } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -150,8 +150,8 @@ func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(m, src) +func (dst *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(dst, src) } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Size(m) @@ -181,7 +181,7 @@ func (m *CoreServiceTypes_GetFeaturesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 3} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 3} } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Unmarshal(m, b) @@ -189,8 +189,8 @@ func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(m, src) +func (dst *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(dst, src) } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Size(m) @@ -219,7 +219,7 @@ func (m *CoreServiceTypes_GetFeaturesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 4} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 4} } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Unmarshal(m, b) @@ -227,8 +227,8 @@ func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(m, src) +func (dst *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(dst, src) } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Size(m) @@ -257,7 +257,7 @@ func (m *CoreServiceTypes_ListFeaturesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 5} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 5} } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -265,8 +265,8 @@ func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(m, src) +func (dst *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(dst, src) } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Size(m) @@ -296,7 +296,7 @@ func (m *CoreServiceTypes_GetStorageRequest) Reset() { *m = CoreServiceT func (m *CoreServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageRequest) ProtoMessage() {} func (*CoreServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 6} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 6} } func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -304,8 +304,8 @@ func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(m, src) +func (dst *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(dst, src) } func (m *CoreServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Size(m) @@ -334,7 +334,7 @@ func (m *CoreServiceTypes_GetStorageResponse) Reset() { *m = CoreService func (m *CoreServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 7} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 7} } func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -342,8 +342,8 @@ func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(m, src) +func (dst *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(dst, src) } func (m *CoreServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Size(m) @@ -372,7 +372,7 @@ func (m *CoreServiceTypes_ListStorageResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 8} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 8} } func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -380,8 +380,8 @@ func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(m, src) +func (dst *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(dst, src) } func (m *CoreServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Size(m) @@ -411,7 +411,7 @@ func (m *CoreServiceTypes_ApplyEntityResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ApplyEntityResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyEntityResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 9} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 9} } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Unmarshal(m, b) @@ -419,8 +419,8 @@ func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(m, src) +func (dst *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(dst, src) } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Size(m) @@ -450,7 +450,7 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyFeatureResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyFeatureResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 10} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 10} } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Unmarshal(m, b) @@ -458,8 +458,8 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(m, src) +func (dst *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(dst, src) } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Size(m) @@ -493,7 +493,7 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) String() string { } func (*CoreServiceTypes_ApplyFeatureGroupResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 11} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 11} } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Unmarshal(m, b) @@ -501,8 +501,8 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) err func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(m, src) +func (dst *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(dst, src) } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Size(m) @@ -532,7 +532,7 @@ func (m *CoreServiceTypes_ApplyStorageResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 12} + return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 12} } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Unmarshal(m, b) @@ -540,8 +540,8 @@ func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(m, src) +func (dst *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(dst, src) } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Size(m) @@ -1005,9 +1005,11 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/CoreService.proto", } -func init() { proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) } +func init() { + proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_f716411f9ac886b5) +} -var fileDescriptor_d9be266444105411 = []byte{ +var fileDescriptor_CoreService_f716411f9ac886b5 = []byte{ // 602 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5b, 0x6f, 0x12, 0x41, 0x14, 0x86, 0x34, 0x69, 0xe0, 0x40, 0x4c, 0x3b, 0x34, 0x8a, 0x23, 0x35, 0xcd, 0x26, 0x36, 0x7d, diff --git a/protos/generated/go/feast/core/DatasetService.pb.go b/protos/generated/go/feast/core/DatasetService.pb.go index d83ec1a1dc5..61b0a09975f 100644 --- a/protos/generated/go/feast/core/DatasetService.pb.go +++ b/protos/generated/go/feast/core/DatasetService.pb.go @@ -34,7 +34,7 @@ func (m *DatasetServiceTypes) Reset() { *m = DatasetServiceTypes{} } func (m *DatasetServiceTypes) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes) ProtoMessage() {} func (*DatasetServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_3edc37a8b0d37b39, []int{0} + return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{0} } func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes.Unmarshal(m, b) @@ -42,8 +42,8 @@ func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { func (m *DatasetServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes.Marshal(b, m, deterministic) } -func (m *DatasetServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes.Merge(m, src) +func (dst *DatasetServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes.Merge(dst, src) } func (m *DatasetServiceTypes) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes.Size(m) @@ -77,7 +77,7 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) Reset() { func (m *DatasetServiceTypes_CreateDatasetRequest) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetRequest) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_3edc37a8b0d37b39, []int{0, 0} + return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{0, 0} } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Unmarshal(m, b) @@ -85,8 +85,8 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Marshal(b, m, deterministic) } -func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(m, src) +func (dst *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(dst, src) } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Size(m) @@ -146,7 +146,7 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) Reset() { func (m *DatasetServiceTypes_CreateDatasetResponse) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetResponse) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_3edc37a8b0d37b39, []int{0, 1} + return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{0, 1} } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Unmarshal(m, b) @@ -154,8 +154,8 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) erro func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Marshal(b, m, deterministic) } -func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(m, src) +func (dst *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(dst, src) } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Size(m) @@ -188,7 +188,7 @@ func (m *FeatureSet) Reset() { *m = FeatureSet{} } func (m *FeatureSet) String() string { return proto.CompactTextString(m) } func (*FeatureSet) ProtoMessage() {} func (*FeatureSet) Descriptor() ([]byte, []int) { - return fileDescriptor_3edc37a8b0d37b39, []int{1} + return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{1} } func (m *FeatureSet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSet.Unmarshal(m, b) @@ -196,8 +196,8 @@ func (m *FeatureSet) XXX_Unmarshal(b []byte) error { func (m *FeatureSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSet.Marshal(b, m, deterministic) } -func (m *FeatureSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSet.Merge(m, src) +func (dst *FeatureSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSet.Merge(dst, src) } func (m *FeatureSet) XXX_Size() int { return xxx_messageInfo_FeatureSet.Size(m) @@ -237,7 +237,7 @@ func (m *DatasetInfo) Reset() { *m = DatasetInfo{} } func (m *DatasetInfo) String() string { return proto.CompactTextString(m) } func (*DatasetInfo) ProtoMessage() {} func (*DatasetInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_3edc37a8b0d37b39, []int{2} + return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{2} } func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetInfo.Unmarshal(m, b) @@ -245,8 +245,8 @@ func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { func (m *DatasetInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetInfo.Marshal(b, m, deterministic) } -func (m *DatasetInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetInfo.Merge(m, src) +func (dst *DatasetInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetInfo.Merge(dst, src) } func (m *DatasetInfo) XXX_Size() int { return xxx_messageInfo_DatasetInfo.Size(m) @@ -353,9 +353,11 @@ var _DatasetService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/DatasetService.proto", } -func init() { proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_3edc37a8b0d37b39) } +func init() { + proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_9ba96186e7cec93b) +} -var fileDescriptor_3edc37a8b0d37b39 = []byte{ +var fileDescriptor_DatasetService_9ba96186e7cec93b = []byte{ // 414 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, diff --git a/protos/generated/go/feast/core/JobService.pb.go b/protos/generated/go/feast/core/JobService.pb.go index 699d2b4d328..e830edfe5fa 100644 --- a/protos/generated/go/feast/core/JobService.pb.go +++ b/protos/generated/go/feast/core/JobService.pb.go @@ -36,7 +36,7 @@ func (m *JobServiceTypes) Reset() { *m = JobServiceTypes{} } func (m *JobServiceTypes) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes) ProtoMessage() {} func (*JobServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0} + return fileDescriptor_JobService_c970f49698845549, []int{0} } func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes.Marshal(b, m, deterministic) } -func (m *JobServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes.Merge(m, src) +func (dst *JobServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes.Merge(dst, src) } func (m *JobServiceTypes) XXX_Size() int { return xxx_messageInfo_JobServiceTypes.Size(m) @@ -70,7 +70,7 @@ func (m *JobServiceTypes_SubmitImportJobRequest) Reset() { func (m *JobServiceTypes_SubmitImportJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobRequest) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 0} + return fileDescriptor_JobService_c970f49698845549, []int{0, 0} } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Unmarshal(m, b) @@ -78,8 +78,8 @@ func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(m, src) +func (dst *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(dst, src) } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Size(m) @@ -117,7 +117,7 @@ func (m *JobServiceTypes_SubmitImportJobResponse) Reset() { func (m *JobServiceTypes_SubmitImportJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobResponse) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 1} + return fileDescriptor_JobService_c970f49698845549, []int{0, 1} } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Unmarshal(m, b) @@ -125,8 +125,8 @@ func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(m, src) +func (dst *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(dst, src) } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Size(m) @@ -155,7 +155,7 @@ func (m *JobServiceTypes_ListJobsResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_ListJobsResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_ListJobsResponse) ProtoMessage() {} func (*JobServiceTypes_ListJobsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 2} + return fileDescriptor_JobService_c970f49698845549, []int{0, 2} } func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Unmarshal(m, b) @@ -163,8 +163,8 @@ func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_ListJobsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(m, src) +func (dst *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(dst, src) } func (m *JobServiceTypes_ListJobsResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Size(m) @@ -193,7 +193,7 @@ func (m *JobServiceTypes_GetJobRequest) Reset() { *m = JobServiceTypes_G func (m *JobServiceTypes_GetJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobRequest) ProtoMessage() {} func (*JobServiceTypes_GetJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 3} + return fileDescriptor_JobService_c970f49698845549, []int{0, 3} } func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Unmarshal(m, b) @@ -201,8 +201,8 @@ func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_GetJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(m, src) +func (dst *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(dst, src) } func (m *JobServiceTypes_GetJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Size(m) @@ -231,7 +231,7 @@ func (m *JobServiceTypes_GetJobResponse) Reset() { *m = JobServiceTypes_ func (m *JobServiceTypes_GetJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobResponse) ProtoMessage() {} func (*JobServiceTypes_GetJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 4} + return fileDescriptor_JobService_c970f49698845549, []int{0, 4} } func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Unmarshal(m, b) @@ -239,8 +239,8 @@ func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_GetJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(m, src) +func (dst *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(dst, src) } func (m *JobServiceTypes_GetJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Size(m) @@ -269,7 +269,7 @@ func (m *JobServiceTypes_AbortJobRequest) Reset() { *m = JobServiceTypes func (m *JobServiceTypes_AbortJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobRequest) ProtoMessage() {} func (*JobServiceTypes_AbortJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 5} + return fileDescriptor_JobService_c970f49698845549, []int{0, 5} } func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Unmarshal(m, b) @@ -277,8 +277,8 @@ func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_AbortJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(m, src) +func (dst *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(dst, src) } func (m *JobServiceTypes_AbortJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Size(m) @@ -307,7 +307,7 @@ func (m *JobServiceTypes_AbortJobResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_AbortJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobResponse) ProtoMessage() {} func (*JobServiceTypes_AbortJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 6} + return fileDescriptor_JobService_c970f49698845549, []int{0, 6} } func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Unmarshal(m, b) @@ -315,8 +315,8 @@ func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_AbortJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(m, src) +func (dst *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(dst, src) } func (m *JobServiceTypes_AbortJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Size(m) @@ -356,7 +356,7 @@ func (m *JobServiceTypes_JobDetail) Reset() { *m = JobServiceTypes_JobDe func (m *JobServiceTypes_JobDetail) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_JobDetail) ProtoMessage() {} func (*JobServiceTypes_JobDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_7115affcfc7885c4, []int{0, 7} + return fileDescriptor_JobService_c970f49698845549, []int{0, 7} } func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_JobDetail.Unmarshal(m, b) @@ -364,8 +364,8 @@ func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_JobDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_JobDetail.Marshal(b, m, deterministic) } -func (m *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_JobDetail.Merge(m, src) +func (dst *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_JobDetail.Merge(dst, src) } func (m *JobServiceTypes_JobDetail) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_JobDetail.Size(m) @@ -638,9 +638,11 @@ var _JobService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/JobService.proto", } -func init() { proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_7115affcfc7885c4) } +func init() { + proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_JobService_c970f49698845549) +} -var fileDescriptor_7115affcfc7885c4 = []byte{ +var fileDescriptor_JobService_c970f49698845549 = []byte{ // 621 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0xdb, 0x4c, 0x10, 0x55, 0x62, 0x08, 0xf1, 0xf0, 0x7d, 0x80, 0x56, 0x15, 0x58, 0x4b, 0x25, 0x52, 0xa4, 0x4a, diff --git a/protos/generated/go/feast/core/UIService.pb.go b/protos/generated/go/feast/core/UIService.pb.go index 87adc7210e9..d3cacf7afd1 100644 --- a/protos/generated/go/feast/core/UIService.pb.go +++ b/protos/generated/go/feast/core/UIService.pb.go @@ -36,7 +36,7 @@ func (m *UIServiceTypes) Reset() { *m = UIServiceTypes{} } func (m *UIServiceTypes) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes) ProtoMessage() {} func (*UIServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0} } func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes.Marshal(b, m, deterministic) } -func (m *UIServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes.Merge(m, src) +func (dst *UIServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes.Merge(dst, src) } func (m *UIServiceTypes) XXX_Size() int { return xxx_messageInfo_UIServiceTypes.Size(m) @@ -70,7 +70,7 @@ func (m *UIServiceTypes_EntityDetail) Reset() { *m = UIServiceTypes_Enti func (m *UIServiceTypes_EntityDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_EntityDetail) ProtoMessage() {} func (*UIServiceTypes_EntityDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 0} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 0} } func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_EntityDetail.Unmarshal(m, b) @@ -78,8 +78,8 @@ func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_EntityDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_EntityDetail.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(m, src) +func (dst *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(dst, src) } func (m *UIServiceTypes_EntityDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_EntityDetail.Size(m) @@ -122,7 +122,7 @@ func (m *UIServiceTypes_GetEntityRequest) Reset() { *m = UIServiceTypes_ func (m *UIServiceTypes_GetEntityRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityRequest) ProtoMessage() {} func (*UIServiceTypes_GetEntityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 1} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 1} } func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Unmarshal(m, b) @@ -130,8 +130,8 @@ func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetEntityRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(m, src) +func (dst *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(dst, src) } func (m *UIServiceTypes_GetEntityRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Size(m) @@ -160,7 +160,7 @@ func (m *UIServiceTypes_GetEntityResponse) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetEntityResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityResponse) ProtoMessage() {} func (*UIServiceTypes_GetEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 2} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 2} } func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Unmarshal(m, b) @@ -168,8 +168,8 @@ func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(m, src) +func (dst *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(dst, src) } func (m *UIServiceTypes_GetEntityResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Size(m) @@ -198,7 +198,7 @@ func (m *UIServiceTypes_ListEntitiesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*UIServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 3} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 3} } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -206,8 +206,8 @@ func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(m, src) +func (dst *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(dst, src) } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Size(m) @@ -242,7 +242,7 @@ func (m *UIServiceTypes_FeatureDetail) Reset() { *m = UIServiceTypes_Fea func (m *UIServiceTypes_FeatureDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 4} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 4} } func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Unmarshal(m, b) @@ -250,8 +250,8 @@ func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_FeatureDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(m, src) +func (dst *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(dst, src) } func (m *UIServiceTypes_FeatureDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Size(m) @@ -315,7 +315,7 @@ func (m *UIServiceTypes_GetFeatureRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetFeatureRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 5} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 5} } func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Unmarshal(m, b) @@ -323,8 +323,8 @@ func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(m, src) +func (dst *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(dst, src) } func (m *UIServiceTypes_GetFeatureRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Size(m) @@ -354,7 +354,7 @@ func (m *UIServiceTypes_GetFeatureResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetFeatureResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 6} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 6} } func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Unmarshal(m, b) @@ -362,8 +362,8 @@ func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(m, src) +func (dst *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(dst, src) } func (m *UIServiceTypes_GetFeatureResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Size(m) @@ -399,7 +399,7 @@ func (m *UIServiceTypes_ListFeaturesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 7} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 7} } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -407,8 +407,8 @@ func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(m, src) +func (dst *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(dst, src) } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Size(m) @@ -439,7 +439,7 @@ func (m *UIServiceTypes_FeatureGroupDetail) Reset() { *m = UIServiceType func (m *UIServiceTypes_FeatureGroupDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureGroupDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureGroupDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 8} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 8} } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Unmarshal(m, b) @@ -447,8 +447,8 @@ func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_FeatureGroupDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(m, src) +func (dst *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(dst, src) } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Size(m) @@ -484,7 +484,7 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) Reset() { *m = UIService func (m *UIServiceTypes_GetFeatureGroupRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 9} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 9} } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Unmarshal(m, b) @@ -492,8 +492,8 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(m, src) +func (dst *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(dst, src) } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Size(m) @@ -524,7 +524,7 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) Reset() { func (m *UIServiceTypes_GetFeatureGroupResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 10} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 10} } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Unmarshal(m, b) @@ -532,8 +532,8 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(m, src) +func (dst *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(dst, src) } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Size(m) @@ -564,7 +564,7 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) Reset() { func (m *UIServiceTypes_ListFeatureGroupsResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeatureGroupsResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeatureGroupsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 11} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 11} } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Unmarshal(m, b) @@ -572,8 +572,8 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(m, src) +func (dst *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(dst, src) } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Size(m) @@ -604,7 +604,7 @@ func (m *UIServiceTypes_StorageDetail) Reset() { *m = UIServiceTypes_Sto func (m *UIServiceTypes_StorageDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_StorageDetail) ProtoMessage() {} func (*UIServiceTypes_StorageDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 12} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 12} } func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_StorageDetail.Unmarshal(m, b) @@ -612,8 +612,8 @@ func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_StorageDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_StorageDetail.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(m, src) +func (dst *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(dst, src) } func (m *UIServiceTypes_StorageDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_StorageDetail.Size(m) @@ -649,7 +649,7 @@ func (m *UIServiceTypes_GetStorageRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageRequest) ProtoMessage() {} func (*UIServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 13} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 13} } func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -657,8 +657,8 @@ func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(m, src) +func (dst *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(dst, src) } func (m *UIServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Size(m) @@ -687,7 +687,7 @@ func (m *UIServiceTypes_GetStorageResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageResponse) ProtoMessage() {} func (*UIServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 14} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 14} } func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -695,8 +695,8 @@ func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(m, src) +func (dst *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(dst, src) } func (m *UIServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Size(m) @@ -725,7 +725,7 @@ func (m *UIServiceTypes_ListStorageResponse) Reset() { *m = UIServiceTyp func (m *UIServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListStorageResponse) ProtoMessage() {} func (*UIServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_c13b3edb35d457e8, []int{0, 15} + return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 15} } func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -733,8 +733,8 @@ func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (m *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(m, src) +func (dst *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(dst, src) } func (m *UIServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Size(m) @@ -1119,9 +1119,11 @@ var _UIService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/UIService.proto", } -func init() { proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_c13b3edb35d457e8) } +func init() { + proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_UIService_07f94aecf154b17a) +} -var fileDescriptor_c13b3edb35d457e8 = []byte{ +var fileDescriptor_UIService_07f94aecf154b17a = []byte{ // 784 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xd3, 0x40, 0x10, 0xcd, 0x47, 0x49, 0x9a, 0x69, 0x5a, 0xe8, 0x82, 0xda, 0xb0, 0x50, 0x51, 0x99, 0x03, 0x91, diff --git a/protos/generated/go/feast/serving/Serving.pb.go b/protos/generated/go/feast/serving/Serving.pb.go index 64979c154ad..7b00a6cd413 100644 --- a/protos/generated/go/feast/serving/Serving.pb.go +++ b/protos/generated/go/feast/serving/Serving.pb.go @@ -44,7 +44,7 @@ func (m *QueryFeaturesRequest) Reset() { *m = QueryFeaturesRequest{} } func (m *QueryFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*QueryFeaturesRequest) ProtoMessage() {} func (*QueryFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{0} + return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{0} } func (m *QueryFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_QueryFeaturesRequest.Unmarshal(m, b) @@ -52,8 +52,8 @@ func (m *QueryFeaturesRequest) XXX_Unmarshal(b []byte) error { func (m *QueryFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_QueryFeaturesRequest.Marshal(b, m, deterministic) } -func (m *QueryFeaturesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeaturesRequest.Merge(m, src) +func (dst *QueryFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesRequest.Merge(dst, src) } func (m *QueryFeaturesRequest) XXX_Size() int { return xxx_messageInfo_QueryFeaturesRequest.Size(m) @@ -99,7 +99,7 @@ func (m *QueryFeaturesResponse) Reset() { *m = QueryFeaturesResponse{} } func (m *QueryFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*QueryFeaturesResponse) ProtoMessage() {} func (*QueryFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{1} + return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{1} } func (m *QueryFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_QueryFeaturesResponse.Unmarshal(m, b) @@ -107,8 +107,8 @@ func (m *QueryFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *QueryFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_QueryFeaturesResponse.Marshal(b, m, deterministic) } -func (m *QueryFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeaturesResponse.Merge(m, src) +func (dst *QueryFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesResponse.Merge(dst, src) } func (m *QueryFeaturesResponse) XXX_Size() int { return xxx_messageInfo_QueryFeaturesResponse.Size(m) @@ -145,7 +145,7 @@ func (m *Entity) Reset() { *m = Entity{} } func (m *Entity) String() string { return proto.CompactTextString(m) } func (*Entity) ProtoMessage() {} func (*Entity) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{2} + return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{2} } func (m *Entity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Entity.Unmarshal(m, b) @@ -153,8 +153,8 @@ func (m *Entity) XXX_Unmarshal(b []byte) error { func (m *Entity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Entity.Marshal(b, m, deterministic) } -func (m *Entity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Entity.Merge(m, src) +func (dst *Entity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Entity.Merge(dst, src) } func (m *Entity) XXX_Size() int { return xxx_messageInfo_Entity.Size(m) @@ -186,7 +186,7 @@ func (m *FeatureValue) Reset() { *m = FeatureValue{} } func (m *FeatureValue) String() string { return proto.CompactTextString(m) } func (*FeatureValue) ProtoMessage() {} func (*FeatureValue) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{3} + return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{3} } func (m *FeatureValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureValue.Unmarshal(m, b) @@ -194,8 +194,8 @@ func (m *FeatureValue) XXX_Unmarshal(b []byte) error { func (m *FeatureValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureValue.Marshal(b, m, deterministic) } -func (m *FeatureValue) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureValue.Merge(m, src) +func (dst *FeatureValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureValue.Merge(dst, src) } func (m *FeatureValue) XXX_Size() int { return xxx_messageInfo_FeatureValue.Size(m) @@ -303,9 +303,11 @@ var _ServingAPI_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/serving/Serving.proto", } -func init() { proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_7609de6de542e6f0) } +func init() { + proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_Serving_fa8a820cedf8e8f2) +} -var fileDescriptor_7609de6de542e6f0 = []byte{ +var fileDescriptor_Serving_fa8a820cedf8e8f2 = []byte{ // 429 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xd1, 0x8a, 0xd3, 0x40, 0x14, 0x75, 0x5a, 0x5c, 0x36, 0x77, 0x0d, 0xca, 0xe0, 0x62, 0xc8, 0x8a, 0x96, 0xac, 0x0f, 0x01, diff --git a/protos/generated/go/feast/specs/EntitySpec.pb.go b/protos/generated/go/feast/specs/EntitySpec.pb.go index 6b2bbca1092..8543c6c2684 100644 --- a/protos/generated/go/feast/specs/EntitySpec.pb.go +++ b/protos/generated/go/feast/specs/EntitySpec.pb.go @@ -31,7 +31,7 @@ func (m *EntitySpec) Reset() { *m = EntitySpec{} } func (m *EntitySpec) String() string { return proto.CompactTextString(m) } func (*EntitySpec) ProtoMessage() {} func (*EntitySpec) Descriptor() ([]byte, []int) { - return fileDescriptor_a3230229c2278d5e, []int{0} + return fileDescriptor_EntitySpec_df39fb3786bb4912, []int{0} } func (m *EntitySpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_EntitySpec.Unmarshal(m, b) @@ -39,8 +39,8 @@ func (m *EntitySpec) XXX_Unmarshal(b []byte) error { func (m *EntitySpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_EntitySpec.Marshal(b, m, deterministic) } -func (m *EntitySpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_EntitySpec.Merge(m, src) +func (dst *EntitySpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_EntitySpec.Merge(dst, src) } func (m *EntitySpec) XXX_Size() int { return xxx_messageInfo_EntitySpec.Size(m) @@ -76,9 +76,11 @@ func init() { proto.RegisterType((*EntitySpec)(nil), "feast.specs.EntitySpec") } -func init() { proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_a3230229c2278d5e) } +func init() { + proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_EntitySpec_df39fb3786bb4912) +} -var fileDescriptor_a3230229c2278d5e = []byte{ +var fileDescriptor_EntitySpec_df39fb3786bb4912 = []byte{ // 177 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0xcd, 0x2b, 0xc9, 0x2c, 0xa9, 0x0c, 0x2e, diff --git a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go index 43bb2bb36d2..a8f4c6c1336 100644 --- a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go @@ -31,7 +31,7 @@ func (m *FeatureGroupSpec) Reset() { *m = FeatureGroupSpec{} } func (m *FeatureGroupSpec) String() string { return proto.CompactTextString(m) } func (*FeatureGroupSpec) ProtoMessage() {} func (*FeatureGroupSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_df3b6a9e736b5719, []int{0} + return fileDescriptor_FeatureGroupSpec_1fd10deeef20baa7, []int{0} } func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureGroupSpec.Unmarshal(m, b) @@ -39,8 +39,8 @@ func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { func (m *FeatureGroupSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureGroupSpec.Marshal(b, m, deterministic) } -func (m *FeatureGroupSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureGroupSpec.Merge(m, src) +func (dst *FeatureGroupSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureGroupSpec.Merge(dst, src) } func (m *FeatureGroupSpec) XXX_Size() int { return xxx_messageInfo_FeatureGroupSpec.Size(m) @@ -76,9 +76,11 @@ func init() { proto.RegisterType((*FeatureGroupSpec)(nil), "feast.specs.FeatureGroupSpec") } -func init() { proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_df3b6a9e736b5719) } +func init() { + proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_FeatureGroupSpec_1fd10deeef20baa7) +} -var fileDescriptor_df3b6a9e736b5719 = []byte{ +var fileDescriptor_FeatureGroupSpec_1fd10deeef20baa7 = []byte{ // 203 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4a, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0x75, diff --git a/protos/generated/go/feast/specs/FeatureSpec.pb.go b/protos/generated/go/feast/specs/FeatureSpec.pb.go index 35e4c5adcba..293a5303b0d 100644 --- a/protos/generated/go/feast/specs/FeatureSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureSpec.pb.go @@ -41,7 +41,7 @@ func (m *FeatureSpec) Reset() { *m = FeatureSpec{} } func (m *FeatureSpec) String() string { return proto.CompactTextString(m) } func (*FeatureSpec) ProtoMessage() {} func (*FeatureSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_b8f1468f11147fbe, []int{0} + return fileDescriptor_FeatureSpec_97d610673462505c, []int{0} } func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSpec.Unmarshal(m, b) @@ -49,8 +49,8 @@ func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { func (m *FeatureSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSpec.Marshal(b, m, deterministic) } -func (m *FeatureSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSpec.Merge(m, src) +func (dst *FeatureSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSpec.Merge(dst, src) } func (m *FeatureSpec) XXX_Size() int { return xxx_messageInfo_FeatureSpec.Size(m) @@ -157,7 +157,7 @@ func (m *DataStores) Reset() { *m = DataStores{} } func (m *DataStores) String() string { return proto.CompactTextString(m) } func (*DataStores) ProtoMessage() {} func (*DataStores) Descriptor() ([]byte, []int) { - return fileDescriptor_b8f1468f11147fbe, []int{1} + return fileDescriptor_FeatureSpec_97d610673462505c, []int{1} } func (m *DataStores) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStores.Unmarshal(m, b) @@ -165,8 +165,8 @@ func (m *DataStores) XXX_Unmarshal(b []byte) error { func (m *DataStores) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStores.Marshal(b, m, deterministic) } -func (m *DataStores) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStores.Merge(m, src) +func (dst *DataStores) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStores.Merge(dst, src) } func (m *DataStores) XXX_Size() int { return xxx_messageInfo_DataStores.Size(m) @@ -203,7 +203,7 @@ func (m *DataStore) Reset() { *m = DataStore{} } func (m *DataStore) String() string { return proto.CompactTextString(m) } func (*DataStore) ProtoMessage() {} func (*DataStore) Descriptor() ([]byte, []int) { - return fileDescriptor_b8f1468f11147fbe, []int{2} + return fileDescriptor_FeatureSpec_97d610673462505c, []int{2} } func (m *DataStore) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStore.Unmarshal(m, b) @@ -211,8 +211,8 @@ func (m *DataStore) XXX_Unmarshal(b []byte) error { func (m *DataStore) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStore.Marshal(b, m, deterministic) } -func (m *DataStore) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStore.Merge(m, src) +func (dst *DataStore) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStore.Merge(dst, src) } func (m *DataStore) XXX_Size() int { return xxx_messageInfo_DataStore.Size(m) @@ -245,9 +245,11 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.DataStore.OptionsEntry") } -func init() { proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_b8f1468f11147fbe) } +func init() { + proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_FeatureSpec_97d610673462505c) +} -var fileDescriptor_b8f1468f11147fbe = []byte{ +var fileDescriptor_FeatureSpec_97d610673462505c = []byte{ // 479 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0x4d, 0x6b, 0xdb, 0x40, 0x10, 0x45, 0x56, 0x62, 0x57, 0xa3, 0x10, 0xc2, 0x52, 0x92, 0xc5, 0x6d, 0x40, 0xb8, 0x14, 0x7c, diff --git a/protos/generated/go/feast/specs/ImportSpec.pb.go b/protos/generated/go/feast/specs/ImportSpec.pb.go index 3a4c8d0801c..9bcb9b3ae77 100644 --- a/protos/generated/go/feast/specs/ImportSpec.pb.go +++ b/protos/generated/go/feast/specs/ImportSpec.pb.go @@ -34,7 +34,7 @@ func (m *ImportSpec) Reset() { *m = ImportSpec{} } func (m *ImportSpec) String() string { return proto.CompactTextString(m) } func (*ImportSpec) ProtoMessage() {} func (*ImportSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_bf21bfd2215b3e0c, []int{0} + return fileDescriptor_ImportSpec_a027fec3e71fec8d, []int{0} } func (m *ImportSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportSpec.Unmarshal(m, b) @@ -42,8 +42,8 @@ func (m *ImportSpec) XXX_Unmarshal(b []byte) error { func (m *ImportSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ImportSpec.Marshal(b, m, deterministic) } -func (m *ImportSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportSpec.Merge(m, src) +func (dst *ImportSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportSpec.Merge(dst, src) } func (m *ImportSpec) XXX_Size() int { return xxx_messageInfo_ImportSpec.Size(m) @@ -107,7 +107,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_bf21bfd2215b3e0c, []int{1} + return fileDescriptor_ImportSpec_a027fec3e71fec8d, []int{1} } func (m *Schema) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Schema.Unmarshal(m, b) @@ -115,8 +115,8 @@ func (m *Schema) XXX_Unmarshal(b []byte) error { func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Schema.Marshal(b, m, deterministic) } -func (m *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(m, src) +func (dst *Schema) XXX_Merge(src proto.Message) { + xxx_messageInfo_Schema.Merge(dst, src) } func (m *Schema) XXX_Size() int { return xxx_messageInfo_Schema.Size(m) @@ -260,7 +260,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_bf21bfd2215b3e0c, []int{2} + return fileDescriptor_ImportSpec_a027fec3e71fec8d, []int{2} } func (m *Field) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Field.Unmarshal(m, b) @@ -268,8 +268,8 @@ func (m *Field) XXX_Unmarshal(b []byte) error { func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Field.Marshal(b, m, deterministic) } -func (m *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(m, src) +func (dst *Field) XXX_Merge(src proto.Message) { + xxx_messageInfo_Field.Merge(dst, src) } func (m *Field) XXX_Size() int { return xxx_messageInfo_Field.Size(m) @@ -302,9 +302,11 @@ func init() { proto.RegisterType((*Field)(nil), "feast.specs.Field") } -func init() { proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_bf21bfd2215b3e0c) } +func init() { + proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_a027fec3e71fec8d) +} -var fileDescriptor_bf21bfd2215b3e0c = []byte{ +var fileDescriptor_ImportSpec_a027fec3e71fec8d = []byte{ // 440 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8f, 0x93, 0x40, 0x14, 0x5d, 0xca, 0x16, 0xed, 0x6d, 0xdc, 0x9a, 0xab, 0x0f, 0x84, 0x6c, 0x62, 0xd3, 0x07, 0x6d, diff --git a/protos/generated/go/feast/specs/StorageSpec.pb.go b/protos/generated/go/feast/specs/StorageSpec.pb.go index 924e1f154f6..8c76b758847 100644 --- a/protos/generated/go/feast/specs/StorageSpec.pb.go +++ b/protos/generated/go/feast/specs/StorageSpec.pb.go @@ -36,7 +36,7 @@ func (m *StorageSpec) Reset() { *m = StorageSpec{} } func (m *StorageSpec) String() string { return proto.CompactTextString(m) } func (*StorageSpec) ProtoMessage() {} func (*StorageSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_7783b72a0d689614, []int{0} + return fileDescriptor_StorageSpec_ceee6ec6c0eb3849, []int{0} } func (m *StorageSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StorageSpec.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *StorageSpec) XXX_Unmarshal(b []byte) error { func (m *StorageSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StorageSpec.Marshal(b, m, deterministic) } -func (m *StorageSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_StorageSpec.Merge(m, src) +func (dst *StorageSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_StorageSpec.Merge(dst, src) } func (m *StorageSpec) XXX_Size() int { return xxx_messageInfo_StorageSpec.Size(m) @@ -82,9 +82,11 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.StorageSpec.OptionsEntry") } -func init() { proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_7783b72a0d689614) } +func init() { + proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_StorageSpec_ceee6ec6c0eb3849) +} -var fileDescriptor_7783b72a0d689614 = []byte{ +var fileDescriptor_StorageSpec_ceee6ec6c0eb3849 = []byte{ // 227 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x0f, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x0d, diff --git a/protos/generated/go/feast/storage/BigTable.pb.go b/protos/generated/go/feast/storage/BigTable.pb.go index 9943d0a884b..6dc2c35c800 100644 --- a/protos/generated/go/feast/storage/BigTable.pb.go +++ b/protos/generated/go/feast/storage/BigTable.pb.go @@ -33,7 +33,7 @@ func (m *BigTableRowKey) Reset() { *m = BigTableRowKey{} } func (m *BigTableRowKey) String() string { return proto.CompactTextString(m) } func (*BigTableRowKey) ProtoMessage() {} func (*BigTableRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_1d33ec3bd45c712c, []int{0} + return fileDescriptor_BigTable_967f8b41f64b081b, []int{0} } func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BigTableRowKey.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { func (m *BigTableRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BigTableRowKey.Marshal(b, m, deterministic) } -func (m *BigTableRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_BigTableRowKey.Merge(m, src) +func (dst *BigTableRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_BigTableRowKey.Merge(dst, src) } func (m *BigTableRowKey) XXX_Size() int { return xxx_messageInfo_BigTableRowKey.Size(m) @@ -78,9 +78,11 @@ func init() { proto.RegisterType((*BigTableRowKey)(nil), "feast.storage.BigTableRowKey") } -func init() { proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_1d33ec3bd45c712c) } +func init() { + proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_967f8b41f64b081b) +} -var fileDescriptor_1d33ec3bd45c712c = []byte{ +var fileDescriptor_BigTable_967f8b41f64b081b = []byte{ // 193 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x8f, 0xb1, 0x0b, 0x82, 0x40, 0x18, 0x47, 0xb1, 0x20, 0xf0, 0x40, 0x87, 0x9b, 0x1c, 0x24, 0xa2, 0x21, 0x9a, 0x3c, 0xa2, 0xa5, diff --git a/protos/generated/go/feast/storage/Redis.pb.go b/protos/generated/go/feast/storage/Redis.pb.go index f42a1f618d0..c277a8f03e9 100644 --- a/protos/generated/go/feast/storage/Redis.pb.go +++ b/protos/generated/go/feast/storage/Redis.pb.go @@ -43,7 +43,7 @@ func (m *RedisBucketKey) Reset() { *m = RedisBucketKey{} } func (m *RedisBucketKey) String() string { return proto.CompactTextString(m) } func (*RedisBucketKey) ProtoMessage() {} func (*RedisBucketKey) Descriptor() ([]byte, []int) { - return fileDescriptor_64e898a359fc9e5d, []int{0} + return fileDescriptor_Redis_749687aada0bf97b, []int{0} } func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketKey.Unmarshal(m, b) @@ -51,8 +51,8 @@ func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { func (m *RedisBucketKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketKey.Marshal(b, m, deterministic) } -func (m *RedisBucketKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketKey.Merge(m, src) +func (dst *RedisBucketKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketKey.Merge(dst, src) } func (m *RedisBucketKey) XXX_Size() int { return xxx_messageInfo_RedisBucketKey.Size(m) @@ -99,7 +99,7 @@ func (m *RedisBucketValue) Reset() { *m = RedisBucketValue{} } func (m *RedisBucketValue) String() string { return proto.CompactTextString(m) } func (*RedisBucketValue) ProtoMessage() {} func (*RedisBucketValue) Descriptor() ([]byte, []int) { - return fileDescriptor_64e898a359fc9e5d, []int{1} + return fileDescriptor_Redis_749687aada0bf97b, []int{1} } func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValue.Unmarshal(m, b) @@ -107,8 +107,8 @@ func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { func (m *RedisBucketValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValue.Marshal(b, m, deterministic) } -func (m *RedisBucketValue) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValue.Merge(m, src) +func (dst *RedisBucketValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValue.Merge(dst, src) } func (m *RedisBucketValue) XXX_Size() int { return xxx_messageInfo_RedisBucketValue.Size(m) @@ -147,7 +147,7 @@ func (m *RedisBucketValueList) Reset() { *m = RedisBucketValueList{} } func (m *RedisBucketValueList) String() string { return proto.CompactTextString(m) } func (*RedisBucketValueList) ProtoMessage() {} func (*RedisBucketValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_64e898a359fc9e5d, []int{2} + return fileDescriptor_Redis_749687aada0bf97b, []int{2} } func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValueList.Unmarshal(m, b) @@ -155,8 +155,8 @@ func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { func (m *RedisBucketValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValueList.Marshal(b, m, deterministic) } -func (m *RedisBucketValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValueList.Merge(m, src) +func (dst *RedisBucketValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValueList.Merge(dst, src) } func (m *RedisBucketValueList) XXX_Size() int { return xxx_messageInfo_RedisBucketValueList.Size(m) @@ -180,9 +180,9 @@ func init() { proto.RegisterType((*RedisBucketValueList)(nil), "feast.storage.RedisBucketValueList") } -func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_64e898a359fc9e5d) } +func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_Redis_749687aada0bf97b) } -var fileDescriptor_64e898a359fc9e5d = []byte{ +var fileDescriptor_Redis_749687aada0bf97b = []byte{ // 325 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcd, 0x4f, 0xf2, 0x40, 0x10, 0xc6, 0xd3, 0x97, 0x57, 0x22, 0x4b, 0x24, 0x66, 0x35, 0xb1, 0x36, 0x26, 0x34, 0x9c, 0x7a, diff --git a/protos/generated/go/feast/types/Feature.pb.go b/protos/generated/go/feast/types/Feature.pb.go index 44534da5c75..55c70f46b2e 100644 --- a/protos/generated/go/feast/types/Feature.pb.go +++ b/protos/generated/go/feast/types/Feature.pb.go @@ -30,7 +30,7 @@ func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_4e19474999533fc9, []int{0} + return fileDescriptor_Feature_9650e908dedbbf49, []int{0} } func (m *Feature) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Feature.Unmarshal(m, b) @@ -38,8 +38,8 @@ func (m *Feature) XXX_Unmarshal(b []byte) error { func (m *Feature) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Feature.Marshal(b, m, deterministic) } -func (m *Feature) XXX_Merge(src proto.Message) { - xxx_messageInfo_Feature.Merge(m, src) +func (dst *Feature) XXX_Merge(src proto.Message) { + xxx_messageInfo_Feature.Merge(dst, src) } func (m *Feature) XXX_Size() int { return xxx_messageInfo_Feature.Size(m) @@ -68,9 +68,9 @@ func init() { proto.RegisterType((*Feature)(nil), "feast.types.Feature") } -func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_4e19474999533fc9) } +func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_9650e908dedbbf49) } -var fileDescriptor_4e19474999533fc9 = []byte{ +var fileDescriptor_Feature_9650e908dedbbf49 = []byte{ // 173 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0xd5, diff --git a/protos/generated/go/feast/types/FeatureRow.pb.go b/protos/generated/go/feast/types/FeatureRow.pb.go index ba8712ce346..5e9d6b0e74a 100644 --- a/protos/generated/go/feast/types/FeatureRow.pb.go +++ b/protos/generated/go/feast/types/FeatureRow.pb.go @@ -33,7 +33,7 @@ func (m *FeatureRowKey) Reset() { *m = FeatureRowKey{} } func (m *FeatureRowKey) String() string { return proto.CompactTextString(m) } func (*FeatureRowKey) ProtoMessage() {} func (*FeatureRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_fbbea9c89787d1c7, []int{0} + return fileDescriptor_FeatureRow_44b5ccd93feeaf74, []int{0} } func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowKey.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { func (m *FeatureRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowKey.Marshal(b, m, deterministic) } -func (m *FeatureRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowKey.Merge(m, src) +func (dst *FeatureRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowKey.Merge(dst, src) } func (m *FeatureRowKey) XXX_Size() int { return xxx_messageInfo_FeatureRowKey.Size(m) @@ -96,7 +96,7 @@ func (m *FeatureRow) Reset() { *m = FeatureRow{} } func (m *FeatureRow) String() string { return proto.CompactTextString(m) } func (*FeatureRow) ProtoMessage() {} func (*FeatureRow) Descriptor() ([]byte, []int) { - return fileDescriptor_fbbea9c89787d1c7, []int{1} + return fileDescriptor_FeatureRow_44b5ccd93feeaf74, []int{1} } func (m *FeatureRow) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRow.Unmarshal(m, b) @@ -104,8 +104,8 @@ func (m *FeatureRow) XXX_Unmarshal(b []byte) error { func (m *FeatureRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRow.Marshal(b, m, deterministic) } -func (m *FeatureRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRow.Merge(m, src) +func (dst *FeatureRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRow.Merge(dst, src) } func (m *FeatureRow) XXX_Size() int { return xxx_messageInfo_FeatureRow.Size(m) @@ -156,9 +156,11 @@ func init() { proto.RegisterType((*FeatureRow)(nil), "feast.types.FeatureRow") } -func init() { proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_fbbea9c89787d1c7) } +func init() { + proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_44b5ccd93feeaf74) +} -var fileDescriptor_fbbea9c89787d1c7 = []byte{ +var fileDescriptor_FeatureRow_44b5ccd93feeaf74 = []byte{ // 305 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x92, 0x41, 0x4b, 0xfb, 0x40, 0x10, 0xc5, 0x49, 0xfb, 0xff, 0x8b, 0x9d, 0x60, 0x85, 0xe0, 0x21, 0x96, 0x56, 0x43, 0x4f, 0x39, diff --git a/protos/generated/go/feast/types/FeatureRowExtended.pb.go b/protos/generated/go/feast/types/FeatureRowExtended.pb.go index 9aab57b1896..a01715d8db2 100644 --- a/protos/generated/go/feast/types/FeatureRowExtended.pb.go +++ b/protos/generated/go/feast/types/FeatureRowExtended.pb.go @@ -33,7 +33,7 @@ func (m *Error) Reset() { *m = Error{} } func (m *Error) String() string { return proto.CompactTextString(m) } func (*Error) ProtoMessage() {} func (*Error) Descriptor() ([]byte, []int) { - return fileDescriptor_7823aa2c72575793, []int{0} + return fileDescriptor_FeatureRowExtended_f06d01003da6c18e, []int{0} } func (m *Error) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Error.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *Error) XXX_Unmarshal(b []byte) error { func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Error.Marshal(b, m, deterministic) } -func (m *Error) XXX_Merge(src proto.Message) { - xxx_messageInfo_Error.Merge(m, src) +func (dst *Error) XXX_Merge(src proto.Message) { + xxx_messageInfo_Error.Merge(dst, src) } func (m *Error) XXX_Size() int { return xxx_messageInfo_Error.Size(m) @@ -93,7 +93,7 @@ func (m *Attempt) Reset() { *m = Attempt{} } func (m *Attempt) String() string { return proto.CompactTextString(m) } func (*Attempt) ProtoMessage() {} func (*Attempt) Descriptor() ([]byte, []int) { - return fileDescriptor_7823aa2c72575793, []int{1} + return fileDescriptor_FeatureRowExtended_f06d01003da6c18e, []int{1} } func (m *Attempt) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attempt.Unmarshal(m, b) @@ -101,8 +101,8 @@ func (m *Attempt) XXX_Unmarshal(b []byte) error { func (m *Attempt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Attempt.Marshal(b, m, deterministic) } -func (m *Attempt) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attempt.Merge(m, src) +func (dst *Attempt) XXX_Merge(src proto.Message) { + xxx_messageInfo_Attempt.Merge(dst, src) } func (m *Attempt) XXX_Size() int { return xxx_messageInfo_Attempt.Size(m) @@ -140,7 +140,7 @@ func (m *FeatureRowExtended) Reset() { *m = FeatureRowExtended{} } func (m *FeatureRowExtended) String() string { return proto.CompactTextString(m) } func (*FeatureRowExtended) ProtoMessage() {} func (*FeatureRowExtended) Descriptor() ([]byte, []int) { - return fileDescriptor_7823aa2c72575793, []int{2} + return fileDescriptor_FeatureRowExtended_f06d01003da6c18e, []int{2} } func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowExtended.Unmarshal(m, b) @@ -148,8 +148,8 @@ func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { func (m *FeatureRowExtended) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowExtended.Marshal(b, m, deterministic) } -func (m *FeatureRowExtended) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowExtended.Merge(m, src) +func (dst *FeatureRowExtended) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowExtended.Merge(dst, src) } func (m *FeatureRowExtended) XXX_Size() int { return xxx_messageInfo_FeatureRowExtended.Size(m) @@ -188,10 +188,10 @@ func init() { } func init() { - proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_7823aa2c72575793) + proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_FeatureRowExtended_f06d01003da6c18e) } -var fileDescriptor_7823aa2c72575793 = []byte{ +var fileDescriptor_FeatureRowExtended_f06d01003da6c18e = []byte{ // 338 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x6b, 0xea, 0x40, 0x10, 0xc6, 0xf1, 0xf9, 0xf2, 0x7c, 0x4e, 0x6e, 0x8b, 0x60, 0x08, 0xd2, 0x16, 0xe9, 0xc1, 0x5e, diff --git a/protos/generated/go/feast/types/Granularity.pb.go b/protos/generated/go/feast/types/Granularity.pb.go index 99aae3d90fc..a47e55ec373 100644 --- a/protos/generated/go/feast/types/Granularity.pb.go +++ b/protos/generated/go/feast/types/Granularity.pb.go @@ -35,7 +35,6 @@ var Granularity_Enum_name = map[int32]string{ 3: "MINUTE", 4: "SECOND", } - var Granularity_Enum_value = map[string]int32{ "NONE": 0, "DAY": 1, @@ -47,9 +46,8 @@ var Granularity_Enum_value = map[string]int32{ func (x Granularity_Enum) String() string { return proto.EnumName(Granularity_Enum_name, int32(x)) } - func (Granularity_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_846024d9f96d5fa2, []int{0, 0} + return fileDescriptor_Granularity_1271850b4f4bfac4, []int{0, 0} } type Granularity struct { @@ -62,7 +60,7 @@ func (m *Granularity) Reset() { *m = Granularity{} } func (m *Granularity) String() string { return proto.CompactTextString(m) } func (*Granularity) ProtoMessage() {} func (*Granularity) Descriptor() ([]byte, []int) { - return fileDescriptor_846024d9f96d5fa2, []int{0} + return fileDescriptor_Granularity_1271850b4f4bfac4, []int{0} } func (m *Granularity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Granularity.Unmarshal(m, b) @@ -70,8 +68,8 @@ func (m *Granularity) XXX_Unmarshal(b []byte) error { func (m *Granularity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Granularity.Marshal(b, m, deterministic) } -func (m *Granularity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Granularity.Merge(m, src) +func (dst *Granularity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Granularity.Merge(dst, src) } func (m *Granularity) XXX_Size() int { return xxx_messageInfo_Granularity.Size(m) @@ -87,9 +85,11 @@ func init() { proto.RegisterEnum("feast.types.Granularity_Enum", Granularity_Enum_name, Granularity_Enum_value) } -func init() { proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_846024d9f96d5fa2) } +func init() { + proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_Granularity_1271850b4f4bfac4) +} -var fileDescriptor_846024d9f96d5fa2 = []byte{ +var fileDescriptor_Granularity_1271850b4f4bfac4 = []byte{ // 183 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x2f, 0x4a, 0xcc, 0x2b, 0xcd, 0x49, 0x2c, diff --git a/protos/generated/go/feast/types/Value.pb.go b/protos/generated/go/feast/types/Value.pb.go index 4b0121e3260..2b0af9e51c7 100644 --- a/protos/generated/go/feast/types/Value.pb.go +++ b/protos/generated/go/feast/types/Value.pb.go @@ -44,7 +44,6 @@ var ValueType_Enum_name = map[int32]string{ 7: "BOOL", 8: "TIMESTAMP", } - var ValueType_Enum_value = map[string]int32{ "UNKNOWN": 0, "BYTES": 1, @@ -60,9 +59,8 @@ var ValueType_Enum_value = map[string]int32{ func (x ValueType_Enum) String() string { return proto.EnumName(ValueType_Enum_name, int32(x)) } - func (ValueType_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{0, 0} + return fileDescriptor_Value_8f69b75784c97601, []int{0, 0} } type ValueType struct { @@ -75,7 +73,7 @@ func (m *ValueType) Reset() { *m = ValueType{} } func (m *ValueType) String() string { return proto.CompactTextString(m) } func (*ValueType) ProtoMessage() {} func (*ValueType) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{0} + return fileDescriptor_Value_8f69b75784c97601, []int{0} } func (m *ValueType) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueType.Unmarshal(m, b) @@ -83,8 +81,8 @@ func (m *ValueType) XXX_Unmarshal(b []byte) error { func (m *ValueType) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueType.Marshal(b, m, deterministic) } -func (m *ValueType) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueType.Merge(m, src) +func (dst *ValueType) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueType.Merge(dst, src) } func (m *ValueType) XXX_Size() int { return xxx_messageInfo_ValueType.Size(m) @@ -115,7 +113,7 @@ func (m *Value) Reset() { *m = Value{} } func (m *Value) String() string { return proto.CompactTextString(m) } func (*Value) ProtoMessage() {} func (*Value) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{1} + return fileDescriptor_Value_8f69b75784c97601, []int{1} } func (m *Value) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Value.Unmarshal(m, b) @@ -123,8 +121,8 @@ func (m *Value) XXX_Unmarshal(b []byte) error { func (m *Value) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Value.Marshal(b, m, deterministic) } -func (m *Value) XXX_Merge(src proto.Message) { - xxx_messageInfo_Value.Merge(m, src) +func (dst *Value) XXX_Merge(src proto.Message) { + xxx_messageInfo_Value.Merge(dst, src) } func (m *Value) XXX_Size() int { return xxx_messageInfo_Value.Size(m) @@ -429,7 +427,7 @@ func (m *ValueList) Reset() { *m = ValueList{} } func (m *ValueList) String() string { return proto.CompactTextString(m) } func (*ValueList) ProtoMessage() {} func (*ValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{2} + return fileDescriptor_Value_8f69b75784c97601, []int{2} } func (m *ValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueList.Unmarshal(m, b) @@ -437,8 +435,8 @@ func (m *ValueList) XXX_Unmarshal(b []byte) error { func (m *ValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueList.Marshal(b, m, deterministic) } -func (m *ValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueList.Merge(m, src) +func (dst *ValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueList.Merge(dst, src) } func (m *ValueList) XXX_Size() int { return xxx_messageInfo_ValueList.Size(m) @@ -763,7 +761,7 @@ func (m *BytesList) Reset() { *m = BytesList{} } func (m *BytesList) String() string { return proto.CompactTextString(m) } func (*BytesList) ProtoMessage() {} func (*BytesList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{3} + return fileDescriptor_Value_8f69b75784c97601, []int{3} } func (m *BytesList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BytesList.Unmarshal(m, b) @@ -771,8 +769,8 @@ func (m *BytesList) XXX_Unmarshal(b []byte) error { func (m *BytesList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BytesList.Marshal(b, m, deterministic) } -func (m *BytesList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BytesList.Merge(m, src) +func (dst *BytesList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BytesList.Merge(dst, src) } func (m *BytesList) XXX_Size() int { return xxx_messageInfo_BytesList.Size(m) @@ -801,7 +799,7 @@ func (m *StringList) Reset() { *m = StringList{} } func (m *StringList) String() string { return proto.CompactTextString(m) } func (*StringList) ProtoMessage() {} func (*StringList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{4} + return fileDescriptor_Value_8f69b75784c97601, []int{4} } func (m *StringList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StringList.Unmarshal(m, b) @@ -809,8 +807,8 @@ func (m *StringList) XXX_Unmarshal(b []byte) error { func (m *StringList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StringList.Marshal(b, m, deterministic) } -func (m *StringList) XXX_Merge(src proto.Message) { - xxx_messageInfo_StringList.Merge(m, src) +func (dst *StringList) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringList.Merge(dst, src) } func (m *StringList) XXX_Size() int { return xxx_messageInfo_StringList.Size(m) @@ -839,7 +837,7 @@ func (m *Int32List) Reset() { *m = Int32List{} } func (m *Int32List) String() string { return proto.CompactTextString(m) } func (*Int32List) ProtoMessage() {} func (*Int32List) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{5} + return fileDescriptor_Value_8f69b75784c97601, []int{5} } func (m *Int32List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int32List.Unmarshal(m, b) @@ -847,8 +845,8 @@ func (m *Int32List) XXX_Unmarshal(b []byte) error { func (m *Int32List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int32List.Marshal(b, m, deterministic) } -func (m *Int32List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int32List.Merge(m, src) +func (dst *Int32List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int32List.Merge(dst, src) } func (m *Int32List) XXX_Size() int { return xxx_messageInfo_Int32List.Size(m) @@ -877,7 +875,7 @@ func (m *Int64List) Reset() { *m = Int64List{} } func (m *Int64List) String() string { return proto.CompactTextString(m) } func (*Int64List) ProtoMessage() {} func (*Int64List) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{6} + return fileDescriptor_Value_8f69b75784c97601, []int{6} } func (m *Int64List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int64List.Unmarshal(m, b) @@ -885,8 +883,8 @@ func (m *Int64List) XXX_Unmarshal(b []byte) error { func (m *Int64List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int64List.Marshal(b, m, deterministic) } -func (m *Int64List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int64List.Merge(m, src) +func (dst *Int64List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int64List.Merge(dst, src) } func (m *Int64List) XXX_Size() int { return xxx_messageInfo_Int64List.Size(m) @@ -915,7 +913,7 @@ func (m *DoubleList) Reset() { *m = DoubleList{} } func (m *DoubleList) String() string { return proto.CompactTextString(m) } func (*DoubleList) ProtoMessage() {} func (*DoubleList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{7} + return fileDescriptor_Value_8f69b75784c97601, []int{7} } func (m *DoubleList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DoubleList.Unmarshal(m, b) @@ -923,8 +921,8 @@ func (m *DoubleList) XXX_Unmarshal(b []byte) error { func (m *DoubleList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DoubleList.Marshal(b, m, deterministic) } -func (m *DoubleList) XXX_Merge(src proto.Message) { - xxx_messageInfo_DoubleList.Merge(m, src) +func (dst *DoubleList) XXX_Merge(src proto.Message) { + xxx_messageInfo_DoubleList.Merge(dst, src) } func (m *DoubleList) XXX_Size() int { return xxx_messageInfo_DoubleList.Size(m) @@ -953,7 +951,7 @@ func (m *FloatList) Reset() { *m = FloatList{} } func (m *FloatList) String() string { return proto.CompactTextString(m) } func (*FloatList) ProtoMessage() {} func (*FloatList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{8} + return fileDescriptor_Value_8f69b75784c97601, []int{8} } func (m *FloatList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FloatList.Unmarshal(m, b) @@ -961,8 +959,8 @@ func (m *FloatList) XXX_Unmarshal(b []byte) error { func (m *FloatList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FloatList.Marshal(b, m, deterministic) } -func (m *FloatList) XXX_Merge(src proto.Message) { - xxx_messageInfo_FloatList.Merge(m, src) +func (dst *FloatList) XXX_Merge(src proto.Message) { + xxx_messageInfo_FloatList.Merge(dst, src) } func (m *FloatList) XXX_Size() int { return xxx_messageInfo_FloatList.Size(m) @@ -991,7 +989,7 @@ func (m *BoolList) Reset() { *m = BoolList{} } func (m *BoolList) String() string { return proto.CompactTextString(m) } func (*BoolList) ProtoMessage() {} func (*BoolList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{9} + return fileDescriptor_Value_8f69b75784c97601, []int{9} } func (m *BoolList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BoolList.Unmarshal(m, b) @@ -999,8 +997,8 @@ func (m *BoolList) XXX_Unmarshal(b []byte) error { func (m *BoolList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BoolList.Marshal(b, m, deterministic) } -func (m *BoolList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BoolList.Merge(m, src) +func (dst *BoolList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BoolList.Merge(dst, src) } func (m *BoolList) XXX_Size() int { return xxx_messageInfo_BoolList.Size(m) @@ -1029,7 +1027,7 @@ func (m *TimestampList) Reset() { *m = TimestampList{} } func (m *TimestampList) String() string { return proto.CompactTextString(m) } func (*TimestampList) ProtoMessage() {} func (*TimestampList) Descriptor() ([]byte, []int) { - return fileDescriptor_47c504407d284ecc, []int{10} + return fileDescriptor_Value_8f69b75784c97601, []int{10} } func (m *TimestampList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TimestampList.Unmarshal(m, b) @@ -1037,8 +1035,8 @@ func (m *TimestampList) XXX_Unmarshal(b []byte) error { func (m *TimestampList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TimestampList.Marshal(b, m, deterministic) } -func (m *TimestampList) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampList.Merge(m, src) +func (dst *TimestampList) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimestampList.Merge(dst, src) } func (m *TimestampList) XXX_Size() int { return xxx_messageInfo_TimestampList.Size(m) @@ -1071,9 +1069,9 @@ func init() { proto.RegisterEnum("feast.types.ValueType_Enum", ValueType_Enum_name, ValueType_Enum_value) } -func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_47c504407d284ecc) } +func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_Value_8f69b75784c97601) } -var fileDescriptor_47c504407d284ecc = []byte{ +var fileDescriptor_Value_8f69b75784c97601 = []byte{ // 626 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xd1, 0x6f, 0x9a, 0x50, 0x14, 0xc6, 0xb9, 0x22, 0x0a, 0xc7, 0x36, 0x21, 0x37, 0xd9, 0xda, 0x34, 0x6d, 0x47, 0x7c, 0xe2, From 66464029c2ba0ca0cf8a8f215d6148b775620a43 Mon Sep 17 00:00:00 2001 From: Tim Sell <63295+tims@users.noreply.github.com> Date: Thu, 21 Mar 2019 11:50:25 +0800 Subject: [PATCH 16/25] Pass all specs to ingestion by file (#154) * Ingestion to read all specs into a consolidated ImportJobSpecs proto from a yaml file * allow setting only one warehouse and serving store and write to those whenever they are set, ignore what the feature specs say * all the tests! * clean up redundant test file * Tweak ImportJobSpecs proto * revert to continue support for multiple stores for now as it is too much change for one PR * update ingestion to take a workspace option and get error store from the importJobSpecs instead of options * update core to write importJobSpecs.yaml to a workspace dir and pass that to ingestion * add on context refresh to initialize default serving store and warehouse stores with a test * write a test to register features and run a mock job * update application.properties * update core helm chart * validate errors,serving,warehouse storage specs separately for defaults + tests * more tests for validation * add gcs File support to core * update readme with new chart options * fix readme: bigtable, not postgres * fix test * update packaged help chart * fix type in helm charts * fix helm chart once more * add test to ensure gcs filesystem is supported * add test and ensure support for no warehouse * test to ensure no default serving store is required, but adding features will fail validation if none is registered * add dirtiesContext to spring boot application tests * pass workspace as uri * proto util tests and fix bug where it couldn't load protos from yaml in gcs * compile test protos as needed --- charts/README.md | 23 +- charts/dist/feast-0.1.0.tgz | Bin 30453 -> 30628 bytes charts/dist/index.yaml | 6 +- charts/feast/templates/core-deploy.yaml | 26 +- charts/feast/values.yaml | 13 +- core/pom.xml | 37 +- .../java/feast/core/OnContextRefresh.java | 29 ++ .../java/feast/core/config/AppConfig.java | 12 +- .../feast/core/config/ImportJobDefaults.java | 11 +- .../java/feast/core/config/StorageConfig.java | 83 ++++ .../java/feast/core/grpc/CoreServiceImpl.java | 72 ++- .../main/java/feast/core/job/JobManager.java | 10 +- .../core/job/dataflow/DataflowJobManager.java | 9 +- .../job/direct/DirectRunnerJobManager.java | 23 +- .../feast/core/job/flink/FlinkJobManager.java | 24 +- .../core/service/JobManagementService.java | 134 +++++- .../main/java/feast/core/util/PathUtil.java | 32 +- .../feast/core/validators/SpecValidator.java | 70 ++- .../src/main/resources/application.properties | 12 +- .../java/feast/core/CoreApplicationTest.java | 159 +++++++ .../CoreApplicationWithNoServingTest.java | 121 +++++ .../CoreApplicationWithNoWarehouseTest.java | 152 +++++++ .../feast/core/config/StorageConfigTest.java | 71 +++ .../job/dataflow/DataflowJobManagerTest.java | 61 ++- .../core/job/flink/FlinkJobManagerTest.java | 69 +-- .../service/JobManagementServiceTest.java | 42 +- .../java/feast/core/util/PathUtilTest.java | 19 + .../core/validators/SpecValidatorTest.java | 84 ++++ .../src/test/resources/application.properties | 57 +++ ingestion/pom.xml | 2 +- .../main/java/feast/ingestion/ImportJob.java | 22 +- .../feast/ingestion/boot/ImportJobModule.java | 32 +- .../config/ImportJobSpecsSupplier.java | 61 +++ .../ingestion/config/ImportSpecSupplier.java | 63 --- .../java/feast/ingestion/model/Specs.java | 181 +++----- .../options/ImportJobPipelineOptions.java | 43 +- .../ingestion/service/CoreSpecService.java | 177 -------- .../ingestion/service/FileSpecService.java | 165 ------- .../feast/ingestion/service/SpecService.java | 72 --- .../transform/ErrorsStoreTransform.java | 43 +- .../transform/ReadFeaturesTransform.java | 5 +- .../transform/ServingStoreTransform.java | 3 +- .../transform/SplitOutputByStore.java | 4 +- .../transform/WarehouseStoreTransform.java | 4 +- .../transform/fn/ValidateFeatureRowsDoFn.java | 21 +- .../java/feast/ingestion/util/ProtoUtil.java | 12 +- .../java/feast/store/TextFileDynamicIO.java | 2 +- .../errors/json/JsonFileErrorsFactory.java | 2 +- .../feast/ingestion/ImportJobCSVTest.java | 155 +++++-- .../config/ImportJobSpecsSupplierTest.java | 140 ++++++ .../config/ImportSpecSupplierTest.java | 88 ---- .../java/feast/ingestion/model/SpecsTest.java | 131 ++---- .../service/FileSpecServiceTest.java | 61 --- .../ingestion/service/MockSpecService.java | 84 ---- .../transform/ErrorsStoreTransformTest.java | 64 ++- .../transform/SplitOutputByStoreTest.java | 428 ------------------ .../transform/fn/ConvertTypesDoFnTest.java | 30 +- .../feast/ingestion/util/ProtoUtilTest.java | 66 +++ .../java/feast/store/MockFeatureStore.java | 3 +- .../java/feast/store/MockServingFactory.java | 1 + .../feast/store/MockWarehouseFactory.java | 1 + .../redis/FeatureRowRedisIOWriteTest.java | 34 +- .../core_specs/entity/testEntity.json | 5 - .../feature/testEntity.day.testInt64.json | 20 - .../feature/testEntity.hour.redisInt32.json | 21 - .../feature/testEntity.hour.redisString.json | 17 - .../feature/testEntity.none.redisInt32.json | 20 - .../feature/testEntity.none.redisString.json | 17 - .../feature/testEntity.none.testInt32.json | 20 - .../feature/testEntity.none.testInt64.json | 20 - .../testEntity.none.testInt64NoWarehouse.json | 17 - .../feature/testEntity.none.testString.json | 18 - ...testEntity.none.testStringNoWarehouse.json | 15 - .../feature/testEntity.none.unknownInt32.json | 18 - .../resources/core_specs/storage/REDIS1.json | 9 - .../core_specs/storage/TEST_SERVING.json | 5 - .../core_specs/storage/TEST_WAREHOUSE.json | 5 - .../test/resources/specs/importJobSpecs.yaml | 172 +++++++ pom.xml | 1 + protos/feast/specs/ImportJobSpecs.proto | 38 ++ 80 files changed, 2159 insertions(+), 1940 deletions(-) create mode 100644 core/src/main/java/feast/core/OnContextRefresh.java create mode 100644 core/src/main/java/feast/core/config/StorageConfig.java rename ingestion/src/main/java/feast/ingestion/service/SpecRetrievalException.java => core/src/main/java/feast/core/util/PathUtil.java (52%) create mode 100644 core/src/test/java/feast/core/CoreApplicationTest.java create mode 100644 core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java create mode 100644 core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java create mode 100644 core/src/test/java/feast/core/config/StorageConfigTest.java create mode 100644 core/src/test/java/feast/core/util/PathUtilTest.java create mode 100644 core/src/test/resources/application.properties create mode 100644 ingestion/src/main/java/feast/ingestion/config/ImportJobSpecsSupplier.java delete mode 100644 ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java delete mode 100644 ingestion/src/main/java/feast/ingestion/service/CoreSpecService.java delete mode 100644 ingestion/src/main/java/feast/ingestion/service/FileSpecService.java delete mode 100644 ingestion/src/main/java/feast/ingestion/service/SpecService.java create mode 100644 ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java delete mode 100644 ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java delete mode 100644 ingestion/src/test/java/feast/ingestion/service/FileSpecServiceTest.java delete mode 100644 ingestion/src/test/java/feast/ingestion/service/MockSpecService.java delete mode 100644 ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java create mode 100644 ingestion/src/test/java/feast/ingestion/util/ProtoUtilTest.java delete mode 100644 ingestion/src/test/resources/core_specs/entity/testEntity.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.day.testInt64.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisInt32.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisString.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.redisInt32.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.redisString.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt32.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64NoWarehouse.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.testString.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.testStringNoWarehouse.json delete mode 100644 ingestion/src/test/resources/core_specs/feature/testEntity.none.unknownInt32.json delete mode 100644 ingestion/src/test/resources/core_specs/storage/REDIS1.json delete mode 100644 ingestion/src/test/resources/core_specs/storage/TEST_SERVING.json delete mode 100644 ingestion/src/test/resources/core_specs/storage/TEST_WAREHOUSE.json create mode 100644 ingestion/src/test/resources/specs/importJobSpecs.yaml create mode 100644 protos/feast/specs/ImportJobSpecs.proto diff --git a/charts/README.md b/charts/README.md index 7eaaccebed3..ce5ab45715c 100644 --- a/charts/README.md +++ b/charts/README.md @@ -34,8 +34,8 @@ The components included in this chart are: Components that Feast supports, but this installation will not include are: -- Storage - - `Redis` or `Postgres` dbs for feature storage +- Serving Storage + - `Redis` or `Bigtable` dbs for feature serving storage - [TICK stack](https://www.influxdata.com/time-series-platform/) for metrics monitoring - Set `statsd.host` and `statsd.port` to direct job metrics to your metrics store. - Note that if you do not provision a metrics store, feast will only retain the latest metrics from your jobs. @@ -62,13 +62,12 @@ The following table lists the configurable parameters of the Feast chart and the | `core.image.registry` | core docker image registry | feast | | `core.image.repository` | core docker image repository | feast-core | | `core.image.tag` | core docker image version | 0.1.0 | -| `core.jobs.errorStoreType` | type of errors store to write errors to. One of `stdout`, `stderr`, `file.json` | stdout | -| `core.jobs.errorStoreOptions` | additional options for the error store in json string format | `{}` | -| `core.jobs.monitoring.initialDelay` | delay before a job starts to be monitored in ms | 60000 | +| `core.jobs.monitoring.initialDelay` | delay before a job starts to be monitored in ms | 60000 | | `core.jobs.monitoring.period` | polling interval for jobs monitoring in ms | 5000 | | `core.jobs.options` | additional options to be provided to the beam job. Should be a char escaped json k-v object | {} | | `core.jobs.runner` | beam job runner - one of `DirectRunner`, `FlinkRunner` or `DataflowRunner` | DirectRunner | -| `core.replicaCount` | core deployment replica count | 3 | +| `core.jobs.workspace` | workspace path for ingestion jobs, used for separate job workspaces to share importJobSpecs.yaml with ingestion and for writing errors to if no default errors store is configured | nil | +| `core.replicaCount` | core deployment replica count | 3 | | `core.resources.limits.cpu` | core cpu limits | 1 | | `core.resources.limits.memory` | core memory limits | 2G | | `core.resources.requests.cpu` | core cpu requested | 1 | @@ -103,10 +102,16 @@ The following table lists the configurable parameters of the Feast chart and the | `serving.service.extIPAdr` | Internal load balancer IP Address for serving, required so jobs on external runners can connect to the service | nil | | `serving.service.loadBalancerSourceRanges` | IP source ranges that will have access to serving. If not set, will default to 0.0.0.0/0 | nil | | `serviceAccount.name` | service account secret name to mount to deployments | nil | -| `serviceAccount.key` | service account secret key to mount to deployments | nil | +| `serviceAccount.key` | service account secret key to mount to deployments | nil | | `statsd.host` | host of statsd daemon for job metrics to be sent to | nil | -| `statsd.port` | port of statsd daemon for job metrics to be sent to | nil | -| `postgresql.provision` | Provision PostgreSQL | true | +| `statsd.port` | port of statsd daemon for job metrics to be sent to | nil | +| `store.errors.type` | type of default errors store to write errors to. One of `stdout`, `stderr`, `file.json` | nil | +| `store.errors.options` | additional options for the default error store in json string format | `{}` | +| `store.serving.type` | type of default serving store to write errors to. One of `redis`, `bigtable` | nil | +| `store.serving.options` | additional options for the default serving store in json string format | `{}` | +| `store.warehouse.type` | type of default warehouse store to write errors to. One of `bigquery`, `file.json` | nil | +| `store.warehouse.options` | additional options for the default warehouse store in json string format | `{}` | +| `postgresql.provision` | Provision PostgreSQL | true | | `postgresql.postgresPassword` | specify password if you want the postgres password secret to be generated | nil | | `postgresql.resources.requests.cpu` | postgres requested cpu | 100m | | `postgresql.resources.requests.memory` | postgres requested memory | 256Mi | diff --git a/charts/dist/feast-0.1.0.tgz b/charts/dist/feast-0.1.0.tgz index c376dccded5f7b2f7132a11ec902b7496ab19370..2ba968c8951f30021f93052446db3adee8c0176b 100644 GIT binary patch literal 30628 zcmV)YK&-zXiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYaciT9YC_aDlQ`{$|Cw7fVNnTUxNxyMySISd~?Pohx)4y~* z3nC#2ZHiY{l%X($?>H7g z9Km%q#A$>XrcQ*WxDNuqoEy8%UwhEcPyWdLe}lpd)2~GU7VQ76?d@)P|L<&VKJWjh zcwE;7$B$FocL0(!{wMa?anOg>*iT&b+a*EE0U%Q}#(i)A0Fo>W&*G5yvpyV;PGWYJ zVu~XsKBjm~C`)I382hP5;?DKRm2^OUO=3z|%)g7bF7IA^VrbllR&R4_oA;OEBqTmM zh_i_GVN1NBahCd+DkwrSB~1SS@RO_$uhfqzo{BDBy;Z&bl3`jdyjd!|%?qb-7?NoG zAqf!Ekh3VlX&(+rihXu2eu)pU zpysp>tHDT#wV>~y;tb`lXr zPCvirxU1CVazhpR_wdC7;5?+38DLijg(S8G3frO8L6zJOe@{+e8R+|so z>?6eip*0;$Vk#Q&JdZcY*Ovd>-P-$>H|HcVW#bgnUqZ3|NgCe}d0?_M!+P&s^6l}h zx9jzK6?Bv6va`J<@zW8I@HH`xVOPAYY2VR<;cfYGG$P{y?~fuz*z63aZxHh*ec03s zJ4K&Pvgr_~r=!bBic!#q-sUT{+X6z*;yBFFd5S(A2YjdWc8fO`N_k>Dn$;w774px93_gLvdQZCz_$R+1ae#A@~3 zRjrEe*Lc?O*}cXy$Ds@{8t}33t=WfG82cy`+tkVDqPMy8hi6qK|Htq&2@%7zb2Y(X zf>Y|TPw{;I-1!D>$g~d>GZ>K&^P}wbluwI4!F~9q z^W}@fOF2iM5)=hufAZ}I@ksnFE`y$<%DE8W5s9!mEInSb1#a;TWg!Dk3LH*&Gv1H# z^1(}<-rzJP0fsw%FhxG>Zizq1^dcLL$R}vIMPoP$#NQISgA^l%0p)#+vM`*%FBuBS zh~NNFk_fmx=O-*%7v&l6gO7p25c?>j7-&4j@I!u+m7^UI90n9nisdnY15UEogiu%; z&csX{o?P&1TCheU zKg@y}?DN*^5Kwg0h=oP-tpzn6uqH zrVH8WJj^tfoU><2)XtY%EW5%XSt>+B95$a1yiBWcmd*2B{;gC6a3}80$j8w7j~2A9 zS}RTeugwDgsh{%wj}vk?z)2X-9!(uE|NhtO?QCtA?teSmJJ0vOCwWZ$AO6H}#Q&t7 zo8AyJ)N`&$6!hUx;KLM0%$Z__0>n@uTKxPOH2TyqnjWWJvZ#_!fBtL}Z1Gx_668tA zv{91Owrw>F;K$P>j&Q{KkgE{{9g`gyH0#S3w|LV`xm3IAc&&n~3ruiIn2=GLB`%bN z>&q91AJzRrtVm@+lV36gjcSW@!r!y zX!!D_(^NtWqT$Pzetj8+#_&ZcR`Nd24B>ZuwL&~@0-VGtGbWu6I-dwB)-AWD1f|}D zvBWFnzh#?kJH3o+&nK;G#`>19AQe&(s+o?L2)SFV%E2sND!O!p_6Da zzd0rM9yawSm_zpc<>i_2!PISCU_Xum+K1j|*C--2V#P=E8_j$lQ+hc`F`dL=P^bu? z(1_$!cZ>~y0Nzv*Wqj;`}u>cp^m?}M1^*vq5^OLMRX05wfqP8q* zH=uM#t3V^G+z%F2w+cz=0&go>%z`&@m`(9}{<=g@gudoXpjNAcnco@vVVnh>lE#I$ z>NdCmvfa0!z7`z>h480QIMW&@a~S+0VE~My8yim9!{4Rx)W&8|SLGk@>>Q8E-)h;$`eB!`9kgOk7k3jx>1~__H9)^ReS3BJ z_p_shHL(_3nHtnj>Dl?|e;*xOULDsAz6Nn}8=CsLygc7OJ~=*ldv&;fxqoqVd3APv z^xg6QEY)Q8u`2Fe0@#X3H)!EVc@#*2!``RulQ!M%@j9NT- zOwE~N-OcgI@#XRUyQ{;acl&?8Z*S(gS?aS)u31i`qH+D~UR<7@A6;D>o&U(uZHcX5 zwdt|O4PiID+ydHd&t1eCirz5%+|XwpoImZKAANuN;o@ldDahMhI0;4Lt4u@D{v%C< zG>Do9=+XK4>G{R-Qy^PhIDXaaDuY)ouRijk=l#oz!+Gaz(IH|q@Fp>3bDLV`#8hq7 zoRXy#;Wyt=7~eLW&-<7A-@QBi>1zI|VsxzA%MX^r1uefjJ=o{Vv244X2jOc`#j}OT z*T^#(T-WL8+jmD-`)6nGjwM`I2j@qJM<<+G)?&;&PO@{2XWl<)9I1+uH#OK&_g?I! z7h~a_^erqW7nG3zS-i15FwnSXpCvKKl5u} zwVLu@_i5M_pcmlfCr|T_T)wGE{)DkaEXEQNt;CRJnU=0AiNSQo<23P?=vYV+R_(Y} zSW0n-eHNz=R+P-$3%@pHdVZebDWCu9iuR}~;05RZ-gd97|J~i}KKp+@$@5k8zqa|= z6RLns=Z@b7voKc+tmojqa+@#bvP>g_M3q0ae1_<9QD z6#-jCUAp@_Rv~}IY3X%;Yg**G{oSZZww0P%eY<26xf0iGrBR^RFisjZK z{p)u}`xlqF+JLY1a)Q@|xKyW_u{&E@^}CpmHFtJ!dVbU>m2R!32Chn08dSgwzcJO}xf38m6TwQ*Dezd<(HL538Iv0Bt*Wrad9i3brU;h2-?CAXJ&Hm-V_p5hk z`!Ub$@ciiV{P^g{{dZTF$M26$KP-(msy9Vk7W8p=d~tPldiqX`@8bCXtJTcR=_Plv z_LM92D*YWFzI#xA$3fWC-)~$&qems(nLXvn+@PV}KYDv~esy%R|K{D%(nwxuxmI~B zo8EtWbaFYDY&5j4CT(HUtB^g_^qj4)n!eb7fA;R^yiuPxr*WnD^C+4oAx@Ru#S(p; z?Vsl;wjVH%u#gxU?@<&uZHVw# zTAQsU!h9JAa7)+({@1z(r%^bAI1)8ENRZ$ZLK0!m@eVJpLBzs?d7`-VE-c=R@LzVU|W+WE#~ z)5Q7a|99H(BT7k}Q8+$4qK-Djb377Y)RE=V_@BQ=qA2%&ihs#Sil;bY)C*((`ceB> zy#Mz~`+sY1``Q2ZNuB^FI0|s&6VAG6?xpa@aO@a*oavzbPN#_T>k5usTKA$cRva)& z2<2R`17ODf{nqQ|edu6>kJbFH*Y&ziK*pGIMrMLGcXs>!aQjuax4pTuy|p>q+1^Gw z+uOUF!S2ppfJQr`(eTyK@4dpqozdpbaO+z<+S%FdqTOADx4+%n!9!=PB0&Uw*zESc zb-SBxcki;-?eA>&clNx!-p=;7JFj+n|IvNb@BR)W8`tdrM~?!$X#a0*mhb;tdpq6d z{r?n?z5k2*i$wv9UG{&~E{0NY&DMWWhK3<-sT)NU5E(LDH{|6(M~z-I4KO>9`*`X0 zMH5BizxI6o-X6LCC6VnEi~gt?0T%85-Mw=B$DO^s=l%aAj~Vd)TW{Cfv;+R14n-tn z$sq1J9HeI6SREsODc-V!r!mC!;uZ#~BH5N3>2m?Igu+k_ppCSH9z*w=0Hr7VP;)8OM!QW~rucKaPBy zF!6`7L(GAovfUg84v=(Iqi!6z$W7v)1eWT#57U{pHxv=D+ifU98)}-)$IZVS%7AqR zZF7os|3Gotw+f{w3gRjBxeW2n(PX zOf$+<-12Ia0l3C9ZKcSou?dC%QSIS2p8K*fcbTd@SDXr(@dB*Aqp6LZL)s zSXd4QS%_0L>=tEldfoDt*=G#VYuZ7EbA$YGDJ2Fb;&ImEm+s#2CgY^05dix)j(sdANo7(E zKepxnI&Ix5i6b{2acZEbl=0y$@9)euoZRYKNR?(0BQmV$h>RgYDVk!2Q!0aq2E=C@ zkWsz|d^E-30isy00B_}Ui%w9AgE!e|gj3q;Lu;qoegCHQ<-QoA;Bl29QVB~AWWow- z1PWDQ6sM4kuO>qzteC9L7Q_(6Oi}@Cmx5&c{Sfhn`*7!qi~_<&4^SxoV`)ZZrU7Ax zqhIG)1j+@bvx0A)QNFXG_wA+gO9piQwXHg&$$7uR1VCXBz^XV7!S_S$-$`?7{lvV*!;z#jS`h@?d z`RZ?Z4aIS^+*VzsR%-)pC&ZsXKq#L93c)NwQ{opnS)}?^ww6T-0$7uWjX>RXzAmbx zC=0_4h~i3T!leO-8Vm698{kjkm=;Huj7ppLj8hL_ZJca!RM`LNVk5tufsopWcHaJQ zw81~Zzr$&QqYH-o>vi!_Ouh+5pHG7WfP^9T`#8XTIL9dXDJ2Y_Mm}G7DpJ6_dQ14< z*-%B3DrkZm40#hqsHx5k63nD|Tj7h^NyhvivEG(>27LbFI8)40;!|t8=txqE&AmP^?%_I8~bAB$3Qnvx}X*@iq%0<*LGo@n0_ktL?vor>R* zT9F6?3Z%RXh2UqD#ZyoN(&d^C_c?=V^G2hw8{u15wdURs0&0FrkOmEmA$2^8>-k&j?R{pn_L}8cJKJiDP_;XpQMe} z@fq#I$DbwPd#_I)PIph@01KNNAtfmtbJ{*f_M(dlsuZaoeaDF~;|!d>G>Rrv3MK6z z$EWm@-4pC)8W^Ce|q7>VSVBv9^ zVxP&@{E+Z-QrPOr(^lM%)BqH5moe+Hm`FI2o8jXByyI8$P#m=RsTM~H3V!l$m}N!V zyt#yN{w>133=uR+W6r0{Kn`9uvsMO{<1AAwt^i{_pClB9@$HdXRh{yz-4<)3Y!C&K zvN*(&b*GYXi~~TFlCijPu6lPD`$4qUk&SS6+~p|HY4iVUQ?18rm)}eobx{03#Ih7SyyS&86XR#2QBvSL8jbjMY$gxRHspyEG{c~s>EPcBCo;}O3^8@&KOCWT zDY-YxIRK@lTdfn;2$3*LvEjJ2>lp58b?xocHn$}W@a(J34xjJ+f;|u`E!iRMJ`RxH9WzQODEGV-P#aSmL?47r>sWhiB@Pw)*+`R^&XE@Lxr3Aj=?U!d7P zI7qI`L2j8VR{Y(9e<;ir!_4!ine3kynykYbN9_7C7O%6eEWBM;ayhSCD-H8C=adv= zMJzzq)vMvUkkDU3a6f>N$)z`0AhJS-Gr4!-j8V?w@tyU< zD_W57!hbSP?9|eCnpQa|q8%^v?Ctlb7ndcvB-pAYNu@z=UVion!&CaP`?K)$%&3Qg zAms;@c&C1RdAY9pK07@x_f2p7uSO7Ge)edBr5}laNZ<81NO~3pWw)j>zl1I~m%YtB zuj_Tao;6ByfAF)q+Lm@Nk=5F8^a=YJ!}U~FT;|ePe=n|LN^m9RM{ss}cy+S>{-`v` z0`Hew-w;|TNnSV2b}`c%$`$2xjVDU0d@kj9qP^EdLtk17Gc>Gx4DKjSZ&4a72k#3# zVpw{@`>QR@ui%TinE!uxM&Jef|5kUiWdF6<-P(DM|M?`(*USG~*~!mIfS)Q^<8vn9 zUoR8zIc8ZZ?=L7x(l|+pK(XA4>qq|s?g!M5}T^cj{E9JW^r5Kgq{)V!P{@O><^YetyV*XF69HuC_w=wV{`~SV| z?K1zj`z-%`nx{7Yht%f2N6DfHAoKDz%@6KTd(SIH_^2(dh_?55!94W%N}Q8Y-g}nm zgKSj3(nb?vUr2!CzoTimpoKg5C5stu2xwsdklUcC2p5HcniUtKDAv^Sy!Bta{|jx? zeF?w<`~U5&o!!#@@AZ1m{y$IhJaYW!KLkPGTSrK4^ISP~O5mC}Kn4kbJ0MYjBL>?h zf0ajsuJoa`x8?ZHOSM@J{34<*yt6_nuQKBcC$Tm>HI_QFN5}^}53&WYRp!-J(--Y5 zUbt74Nw(uem&mm?n=rc2h7?ZqY{*O7-4!ic$(Kr9nI;@XAw`bqFPqu0#=a**bS2dK9m%P6Opk)OU0MN? z^R%O<7ng6(k1no0Tv$0gizuqgDqL&U1Ryt94785~q0&%-Na2+X^sUUgyefEazXn{D z!5_Yv-^Cm?@G{LGpB!HvzPZ{ze|xcNKZS47+@9VZ?qBX#+g@bE-hhiGU$?nq)20Pi z-yK__el-LfFKY}?EIbYAa($GWuO4};P?!_Kx4v$hBzAXDP=lPcE|E*%v69s6|9GDM(AtpmoCsr>4in8 z_zT{Yx#e9~I4QlT%t&nre)H}rse^{OoWRy-scf4$rxClT-AvUsbWTI%!Zs7xl>Oj0 zJkL4M51Qzrm3cFrH1ai-W|Dfw#&UqEa)$ZGf-mfK&D!;e=k><2$fV^UWP)nAoaH6# zapq1(4b{|lv(oBBS(By0rKSRPr^F`|3GbDQT+w%lo-M<#e3Ukk$D?{ro& zkNdCO%BRmyhW|}zIE@4RTDc51t@|8#r49Sp+pDAhIXgYSJUYKR`k%|Alf$e3^Wo_H z?^kF0m)|!r3N<3nc{tf{8kSeSauj~sd$o7(0oD_YqIY`?7OkGM5m(uY_ewKV2aj|} zDu20O%AQr)pF^+p^f%J&%6ycvGuKe=GU=*%2wO;%d+kc6UFG0wAQOLaqPfZ1R!!ev zxLZMYYTwsOj4_@jY<5VbKTQL681}tQlyXT-Ux3fP*4~&?b1wk`fH0;jGd_alo<|T& z^lL7+NFdXAuFgMItI530RNnWvWi-!0^R>gAfu>T6#$R)g&pXDSX4&h78pdEN-xm$D z9I2)#Sczd%IZw(`tlDP&Q&-DK(o|;=p-7U=Wl54zB ztqc26c~qR$UDlkEx!Iq=nxrOIRBM96sYfTBQb?{=qteNkaXY$Kjr`3ljBL|~>aD)1 z@B;G{GpLgql-qRYE=t|2FoRMVMc42z!a#dbm33|OQVr+j z*wTQSB2CcSAhn#5Jhk-aSF_7rO#fAU(FdFXi|Bv1yIJx7+I_bFdy=PysjGSiG$aY+ zpZ^pjTpGBYCg_(88|K0mxhY4oRaxmuqMtrkY4Y=D5rTI= zOeW|g6M=Y}8>*0)UmF9TpI`G?g8qMX{(rCR|I7LRooD+06wgEQzy2X;`!dSw3Ikur z5Ihohq!_`HJ=g`LD53Rj@6}$bj+d6_OF7r46$2s{PAmg&ow_x#^{Zx*(W00oGHEbj zs!wQYL^3EhZ`IeVw+PLGf2kjF5n)8dXIT>pwn||gV`F_i*U#l47V99?AEE3aQ zrpwilAY~OZG-ST@5opYO`oFoiy;aWtx3l#u z|9_gNZ3Gc=oR7tipVxGNLOVeFIuI(7Bi!P2l!Y4*Cpc1}IU#ON7bJ58d2q~Nf+$Sm z6a$-}2zom(!ziT?j{x~1b7+8Xa2O|fEJh?^RO1Lx>Q4y6KFefn*%n4=gaaT95KxxJ z(HK&kWGbxrC{Fo)A<{d2wjm>clZfGQDt;p&3l~8|g*lrD<9PY*uZQP%2Jf@>8qXS#W{f1%_x;| z#2YF?RLF+P5hd4kEGH;pr+xe$htr~8C5>9wnO;?F_f*crS&Bt^KuQ>V@eQV+m)o3Ooz9#BqBQo2i2GQ?3l@kW*D4eh4E*q{Xx2z?VlzsX zzyV{z^A+K*joDY>am*NPdCzQuNt&;_%C+SfX0r*mB_Rk~c+ zxdXTUk!?48Y^<_k4ZAvrW*|&F2K6Hy=uJ7Oq_Jw48>7a3=4PPHPpj2ypkZh{w?P<) z%vY;}VnI_FvJT9)k7YEnP6+65^N33lm~+V=pp?&zh$(-6EG|27nR6qqH!^n8 zZNXMHFV*9%RN?A1Fy=<&-P+fAw-z?|U@W%$S_gL9IvqrPa;&cbjjbdYSZ&v(UE>S4 zFQP5ysPu@?*iNE0H0LUrm2x0GD^BQ!1|JTMHr%Wt<%X@}wwB;$YNwvx>a3TR-h zFRa|5||b<3YpR2x!(aD)`c5g0wzZwNZ!>*B3%lD9s1? zN~w%jh>9#q;*u1nMMkAEG*+smm1e05O(6gsh;ynY=~rPpP&+rDzua@7YF3K;%Q_i! zej0(eYLezQ2!mH|$)n+obP&t;)I~Owv6VUNsV)HuC>E=nmWWxx5B0u(BwuYb+q{GL+8E78C1<$)R7sBN;TfA z3_}BvVra0zo9M%!*X>RhK_s9lN$V|c>w<~fAy5)v*GH*osHjtRl~(G3+?M48SK=s+ zbdU1>vbaGQ%tQuJoODN$WhgAeD6t4fzaF*nK>`S?skHJ7=lpizEu2ZN74Kj@q?m0{(( z=Hk`JPO{_x;m(~RVFq&#lyS?A54qqSkm)j^DvHHGO@8q zitUta{AOjQOM471-3%2~#v3zhS~W{r-Q0ZsaxX}7XmqTsI!W8Uj-y2};}?cSFg+X^ zBcj!y49#W!ZMhk`gO5Ky5Sn>GrUt5$ho`1qyx%16HzyuQPL$v2^l_rJj(q`vh-G2U z&0x3tusg%tz#@ZMXjGYeb_{5C?h8ykH09_dgBoa58DMz`lVWZj1)BPZA{uYb?d*O{ zOZ8}T(-fIxPzz1NQr*E|>ye?UE>#tphNXIhxv>Hm+On;>Jch-sO>Aqs!3R^n+EAE- zrJylX*N*~?(94OWU#8k^iV~fS)JzdZbuMpC`aWu?a+UcKs&@~*i&JU0K%!BcPF1Q= zsTxl4NGcA5e*He}tJ_2n`$D3u^|CsnJx2bd7^k7kAGj{RRA1xSYr#8x+{pj@c>qWW zs5xi@Q@uTq3R>`gH~_LwQ?c@TI&(9M(~Yuz4o7->FzXqSVY%)m!Y20&kU`K9Mre0B$E*%VH?lWv-R` zE}T2ks^%G*HrlWlh5&z$GXYmdv1O?6(P-x9HRFf0&p0b2KyuMa`(asI7r7xQi&Hgw z#v%(zmbBb4boCUF)K@JQVrh|>QsK)WG@ddt5Q=&sN|VMqWpQV{sZAw=brG<{Rk2Vy z>k>jjA&h;)&_fC32+b2!${ileA~Yqw{+`>@kZA0|u`)5Jm66j)oP~kV8Vt2bkDVi9 zLnoM+BbE0YmGn;JY?d$?r2$7Yx5b*p8DPRDIMrwjs7~J*i+q+99nuEGyzVXG-vlaW zNq`t0$kaGvoiGCl4WLqd=3Dx9Li`DzKQfNuRE|`4&8HZt3_CKr562ogm?55^8xm(J zV6jyt0m!eP{M<-z6lg0RNjNx~w+kIt?E+B8>3N_sm((rOg3r=nvgc`Yr()fjcf26V zO>6o3?8gI{zpRe1Dg;wooLaJK1hiSGZ770`w`m-UGY8$oBrra_q%a{tfFscDandMq z5R!=DbcBU*k|5UuB9yEH4?tygMJa|X8K<1}8p~}Y_o`@Mh)4W%)@fk|U2og#4mO0$ zxn5+?55-n%Agz9sAj_ZtagrYioD5PWq16(tTq?`S`Nu+BYnV=&X5i~`r{}e!Ac;xD zU`@z`zU{r*6SkFn!wOLh4g`TCOeB~5lwe=xuT>;Yx5MYekHb)YRx_C=bG$B+IjY@D z1xVsAG9@8O0Z~QWsh^UeWRV0Lqs|yOE4$1?$tV-eYCT|grk&j2$-LS>?A z2jg%qV`X}or{-5OXKV+e(@<1Mq zf)0tp1@nu6kIk9CLSc9%vtp`kl{ckwIO~j*s$ru*;R+sM<_$4Jo;}I-^u7N4kCuw~ z@sC!Y|Ddt7@BBxr|Bse2oPV^wd}%q(Pn^oyQ;;XIWhoksh@YpWGxFX2lTTxw8C=b~ zz|SPbK0m%CJR5ZjXNfjnOQ|U*jA{xUkvtd4NF8|!T|RC4{10sLr=1qJz;jR9Uduvy z5CNe{h-L+ei!+vF!Wx60aYE-l%0hjjdVhT9NX!2t#Q~umnd|IfE`Qdu(Eq!f|EagV zyZapf<4GQrB)R|Jw%7G~n@&)e8Ta9|@b1_E6x^W5#{pd9nJX1F;&vvyItYb$#Bd}< zA*m!4NZ+7k(hz0f&v|Q{0u_0*uklRYe$_3Emm|@bZomZ538ov;)>;q~Do$d`&(1(N zP%-K`I2w}(_hB5fBy=Y66!$@0&jdnqLg^5nk47I=L&EsAN*vFK@@6`eA^%!$0WuxS zzg+R!-Q4Vc+T7aql4$H);~Bqj(7xk>DC3CGcw#ynHy0N;Q$&h94ldv+B4OW1?(km* z5V?Ql8~M&CkAsx>{}o`$o13DPk;ogRc?Ii(Bi~7=^57aS)G#HgbN#QYF5$B3jmC&w zuj_UH_(bqe{M68YgAsn5J{I}^@xM#{f4$AkXZrsXPrD6o!+3~7I205h_nsIN%F>w- z=LzZI2EUkMef`u7Z>&@La%Re|D(_HAZXrI=4Ul1WgGTiwtCZd+B!Vwx$3Piu0)D_!Z}A|1&<D&yvKst=#Bv;BpXM6a{W7L|2U5U{#b~ z8{LpZ*AfZ`c%S(JFZ$Mc1yl~1ESolD7=);ATO0O2}vGt=yA5iNR8OHZ4qT_ zI#W3R1D;*vQH!PDniBMATL?#8w|QlCvCCta(-7U@gE)&=A9{{sUDEm{iV#fUc7lai z7nN>{O2$0IdbTv|d2ix^=g9Uhg$Q3%oW_B?yP+frXP%TUEBp~vHb7hi?JJ=2(v_`Q zw?ILRCg`RRenK=A8KU!buUJ;=3al2yQS#<}-xq7ryaE?i!R959KlKXa3t*=1FEu48ufp$R z85D}INug=ulP4z$sl4^nxR)0?NgWO(4`!hGRmVw($Zs6o`J4TNA}-vrW8+jf4plt- zb{o#K5L44L=$I-0P37wuhgj?7qz*;}2u*oK9V``GKnZ!9##vIllUZGKn#|&DDVAMZ z>ZnDuQ;Pj8C2S^SA_Nq_kFEyg1ERcS^?tDxA>V+zCxOKt$OTr^HuR zIvr?Cqz;Kf zlHwZ@XT_YXJ8iJlR+90MX7<{j5mBK4rX?pilE%^kEH_GUO73?#Mi*LCsp%BcT&NS3 zGQ}~J8Y=ZbyAAur8K$*0hpzJ6%gddDPHT7Yc_pEg> zvNtV%6M=^X4YV{%Dj5)JzrT!E-@${Zb#W_N4vTl1qf5zqz@Is3rR zNvUhksAM-ax1g(pB*I0#4Z+ANO+Y`VKq@B`v^r@Pd34e;J2#XLR->a3jaLOk)Tsxe z&F7J&=sR9SO%1rva@{HVG!<6f+Q;HXNPdnrI@%Z52|iASzJHe3y^ z9-iu6L)aGql)iiS;o^Id!%+Rb|L&b*B8ozF17DIj(6*xjeG8ka<@ioNjyTUkJN#;P zL-1`!X?R@Dqqs6c3+>1*Ix;A^in}c-h;45sUe-WdaW(*nuv5ayfG%YD`66b516GHq zV1ey3wOxu3E6>5H@?;Q>4Eb9>$@h0XMj>c?NS#CO|dX}5Pns{wL3Ni|YWrVmSip;;76z_FcZr+V6yXmxz z%Hqws9$rlbr1CM+knV0rMi;Zw&dHbMoObf9($^Td0DIznHAmo%Q=dEO@geK23|YQ9vVlU5&EXd3Kpw*>oR<)h9IEt{{1gAQ*z&(GaK5hJetr6I_(cQ;T# z9YZil;{>Nna`x&LmtRHdtFT!j(M<;a%iU^>_P-_N8&Pp{73W}2-3_CCVX3R$$u5z5)KOBmmfNbU)@p62U8xvcnwzAWwx~vX zS)?uP*9@ECbr4*==qk#>a08;a(w(>f0m(-sG`t(&PvSTiEoqY&BeI+aur^LMA&y}G zrwcv`6a^^d6JNCS_J^Yl{u%yV=(H~w@~_uzl4XL^=My5gZ#B_@6hMM zZL1H$a+TjU5X0lyhNfO>2F_z4*jA(ZEm6cph;FPS zzTH+vUuE}H1pMBI1jWcD7mlJQDVCSylFdKj?7lRZ)~2QqryD?wU9=&I9f?FFV_=h* z@=YMj=Y(>?94&`k2}Q zdnHJBRs=Nvuo|Q$jtMFX=eW@FJ%&80+QX{X{KvW$|Bggs=5+B1DWsEP&ZO{C5hh!d z*63H%h{*gHIbM>B}vtZ8~79K zU%S_MM*A?G{eWj1)7dGX@gMN4V1Y!HqWj8R7|NcfCjS*!7LwO8F|ATv@k&XRgqaOOU z+bjD2AsbrWglF zJ-d~8&&^51O?i1s<6`3NX|J`B6uzn9?f-YTcDJ7G|DWRd<{M?-FM^LCCTwbD?2#f_2B9#-%4k1~XE=awzHxNYEJe>~ zk&;XhY9`~gJr{g{Eo11?C$=ScgbBKl+|4wm3_@~^!&x32fUl3bOe(pe z4uKiYgw1rEF)7rvO+Tfe!k?gs(5WY({;tO6Y6EVu%BRFHY}t6s?;Clf4|+?OG_Gpp zHQ5C z9)F4FCuinJRmvcz1RrKvK6TVdX|Yr)VVO@s7^fTQ>J-kXe0n`BvClbS}umG>CESqIeC<#;(sIYL9;KuMz8b=9Ylh>`!UVs?#q`#~N?@J5r zV2CK*-3HeUupb9_-5T+w<+v}#R!}oB+dYr7vVd~^Lr_p39hOQs7)w_WIU9P=X%;=w zXkF!L;=1V+T$iyKxP<@0A|TA`mnZlJr!QsfvU%$Wu8X6a$~xJ<;U7hoX2|npic@gW z3D|r}R!%|Jm12k2t(xIihAzo5hs?-c39b~4)lH!IpmD_o5v<7B^x!}(c;P7`m-gV@ zaeo;aDy|(wGekBx6Ch}ebj z__uw?2rYt+lzhGfjxKSwm{{s)+aM`{vWSeVTuidtK!@^>f8r>JGnR)_(xH^_C+ue` za944J__^_Sg*h%5qPcwXmoFAcKRG=-x;j29(Q;v+xg2pT5o*Hllz!~~>`{hTMm-b+ zDL*mAJN4sBYrO)(+3ERZ35ff2&lzKJ0E_;#CjLvS2)4HDhF({q2xEJ^P?H62qWzeM-pm6leA5vdtn-a?^eIufYb_^o8I#yi=otHAX6*+lA%y269FZOfYft-!g2ycdB&%K!$M(* zW&BziPpzECQgA_V7)Yy9BU+zO!mg=NDE^&IXd9-lI9c36;H#Hgf6SL$muW1p6i#Tip8t3ufbYLv9_6+wOyn zoouX4HnweBn`~^`wr$(?6Wg}Ev2Eu$dEf6lb?W>zJv}pBGgaMp)jfS(znf7m>HdyE zQp~-2+p;3wv~l{Jfu#oHfd1NV2$Sh7pc1Baw4vqn(Sr-U2R>B4s078Hp%}C!j3W;m zJ#7q4VOJG=F&3?J7oHii;?_3!L~r5TnJA1gS_eRU&A$PHGeC=hf!+V5K9ataLR_OZ z#z26!?f4{&1BV|J>&7g!4wWRbuV6Qu@iXK^m$?_uEO}ilL_sVba88>tk;g{`NJ;`E zwJE&LHS9^B7PPz7J9GQyd)Ip&FVWH0sBx^E@QLrfX0J*#wW^>^O%V_*zE$9st#Z|s z*miJ@eJ> zbyaX1N$FU-ezmfG;`^m#UR1IPP*_<;Z>&53S6HZ4F*rrM)F`L~XbQ^ey2KZAQ>&nL zsd%GXyisFnw2w%M&Y3R36_m_4)H&wrJ3?4VyImQ+=-ZY_spbFgTKH>S6I_cs3ux zF0;>Em#u1)##FUlFLIe8eLm@wiq>4LX}mzp*0h|4Y&xgy;m2IeY~a1jn7E#Pef0V5 z$;>WvmrOXlxuxzf%lKq^D(xJa0>>h!*_PKPJ+Ym^pU&MhTutFhxCf_IQ^7}vV4x#d zRlXanyyEud{(2Ni75$_T-CqlMP{^zyqJ}iyuh0agmd`Mfc43qss-d~!dlgtQ;@RWU z9`*QR3HJ8JtrA}={5hvl5kH)L`DCJNSHAd-oGBJ=bTD@COIaf$v-^K;Mc%`x^O(+E5Z<(bu zqA11tL~>N7N(>VscidDo70UnKB@nkm;Ouu7_bmHUC=(Tzov1o(0{kX@>J^4APe3Up z_RL6lB*u?SabBIT`-zyH%~ME-`idtLy+;%rFjcPoL_lM(tWpsr6zrWNuiP;~CC`%% z#bo}Xz?4g^tBfff{Logwk*A@fK2DoSLbRpK^QYI%Te9Fd4c3*C@KVMPhp#Lu=I@-$ zkC;n$wV*OR3N&@JMF3AuhFFwB7gQcRcMey4?yT0EasP9z8v!$O73t zx2jEn-{!clpoi^bvur}}A%x|^#P-CzpGBI&g>84#N!}j3Hzyi-+IivLZrzcBp?!0Ctfpk~*Ic zAM-D9W-aFCs^VfdC>r!EgS0QYwfWzs!MkP%eqi#pKCuegd*UihnXWIf43s3PhpdBo ze<98I`)A}t;r)>1AFAghY2~wa`{IkeZ(Fj~GF`gO)x*c#*}=d&yZEturyr|9ewJVy z=TQwz@DM%Vec4#DC4C95(h zpo2x|(_{M1G_!fHTUwNCllup6i=Or%;NI8u`xn1uLrYiJyVn>0(4TH`ilt_vY8K9* z3Y3^VEJR26dbJQ^d84Jv6%!_huY8pOfkr7WiOL)lb553cAqVc5!Jmx2chi18V~C>l z9E08zWP*Is2)e$?0$7eSxi>yHH2j4~K?D`U>^R`}WlZDfSwp_!3wCn#9~GbDHXB@g zmSA}09DFR9E=3>8e$|KRmh)K`!LH@QyIq1GSf*%Z<#z#O&C7j(+<9n?Z4J^5r%e0I z0jD&0Md~i$@fLX5;C_us3}CA0(aGTUkF=FV2`u{djvzLIFr5HW^q7F7M+?8Lr~u`k z%?7`x(&Z|jl+6AtagWj}i(PSj2ANt_lW9>vMN@jzgp)5Yk%~BKm{oI>n>>_vR$+5k zOebYHr)JC%!^ViwMgKMyA{L%sEh;j?H`!ID%+@EgD5(_qdl=*f)nmjsV8SgC{6_Wr zE7Fo@!w+*_3NTDXgY;(1My6bPbv#qR=Jk4)0i?_9Hr`+*A0m7Q8*o32TIQdu$nDj^tQifp=^auEWYhkG)u@YS}Y1y)ONN! zwlBR|aZl5+&bL`mBkB*VW`I&`EnXaYxOVy~!fk-rI}f0;Z<3{Ukg4`HTn(u(JnEOy z!L;T2T3*2}x+>aycCh3;MW^OHWbyB;aT2WQ9^E9BJS{Zb`th&0l_(h=_s(c`W~k1u zD-?N9D~2qQb%H{p=Sh-*jh1>62=s?^ND++xT+C}GzGL;;*to#v(I;4LPVdI##CAOU zcaD))Hi?LUHzHgjQg6h%Y7ZbL_~`rb9Q#1y3}grM@(F(TwRN>^Z~*!~0X;L09c8`Z z6Oi?~-iBhv)PK^1!$m-6=D^=l5_23`guqoMze6KtiJ3J2&^O``sytHpIg4{;=c65o z$RFKWptDt9G;icS-Na&}(XNNSS=mUEc$uU|Wa#pM|5Mx_TDIJQc>JaJ2)k($^$e@B zy_ERqY>$*?rD@qNAJ5cWaTxWD(q95|i6HuCk4+&QP_+(9BjRfT7f6I50C(x@(LwsQ zeR*ZWzwph|`I`t#k$fi9UFFVYb05XnP5u!z_2H!bDW*vzD`A}WVq%dB1v{rddP9F7 znPRZk%JeakwY$!^pL}RZ8TY{hBK^o(mQl1O7diY{a%GaZs*3$zs!j30P|2nA-fF+& zN{Sn>#un{6j$Ok0T%+m5f6CW(XqN!FHD{i3Yni)NpEF>taKi5zA6cLd{q+5Ag{bWbOLzZ2Jkgo`wL<8RW<_VReMeF>t?6*Ly@2P!drBb0wp?QEW zcZk4ITq=%J7ySe=c%B30#)Y2f1fe-!qXo#x)qp0x`d14CuxEwlfN;xMZ z{_8`1{T=?5pl)kK3m_dS*7khE$W-=R5JT1g0k!KMOF30$)RKlF-XY+wI8dz+)f6?$ zu{eT?p$>2w*&O=#+0(K*y$tu<3s_pLuc7-RT6!{GZCbZV&1wordOk-zzOR)T zT@1O*PVwG79(1er|{U?j)YrF1Fjn_jmh)$*0(Y(BDP)+i0+e0@kO#gEN`=*CKxVo0bW$G@T zqAyioyi($9X}gUPn8WD^2)j)@g-QdEm3u1QYU1}c3gK6+#0IpuE%xS<8hEMp?u#14 z7kgB#P0BJuDsh(Oe&Ib37a6uBRZz?;Vy$C78h`#CChPi{YY3(SD~MEe~&p{zV~G#emAf{SL6EbqPs(>H0G1BY@(+Ru^4p8|d_1Z-kpvX0rT zMCxMIkF-Fq!FJYJUsTe7Q2x;nh#Y^)RYgx3rCz(KwBKkigWWUAJLN zga3Gz4R)IbRKr!$;z^STGSZ0NQBf5sq_B=!I;FnmYEMt}B=fl6$i;??Tz7XitlY2a zbd|!lhG_ehNP~A(=H+3kW!EY*kXKU5Mj~v4;{;rqnTN0*&- z&>a8Ok1momq_ACzEg3s-CH*3 zTTPwzjPe#M4J|(Uj~`CKOn=;fFF`O^tlvnCM382?M&rhEg7z4t0TS%7jO{a!9H-Z0 zzEqgcAo0`G<=N;P)ssCeUohk0EXLl>m(z{C6vmVP{zfRL?$+xh+icpm`FFM)O9O*E zfJq|?A8YDxRoQ$KLC#W@bm1W@=D6fxju%3Lb>>OwztcoOaam z`5@*@z3RKQ0b`aAG;O&6E}TE*T=wJ~xO-jaY{w6&CSLg(U+28PTrD2IR6A~!Q?{)- zj?^>)_NO<`aHfktU0|yv=kv(sc+<~N7m7>G^2eG>5-QY_mV1sL)oYkXxSdpx3QQgv zqYiz#K6`$A9>?)ffP8?xZEw+z0M9QkulPoQ-U+2ga*2rXwT>og*FT7h)WgbGkB{bv zx(lOZ_mV!Okw=NoOSqJ$_iuE_$FKJ(P!^QPg{fvX6OEkJzSJFnNRmI}%)I!S>db)G zmzT+}JyX@DU)Pd3^X3*;pDItS*lXUyV9WFPS5H@0wkdF%+Pt^|`0G;hizZ_{lfS=0@1SZ)O|VwRL5}2^={(I=cVzO)$0JoGlFl z(kfg$3xX60l*+J){#f90XPzovW<5J>*N9Fi3Q0zd#tYjFFAz>%9c2rxr0>-DS%qGh zeom^gp0%w)bMM(}Q(-i2Zw-euG`!PXG0LTO<%nc)in5o_?&D2(9C`XulA@Ha=2 zmGV!=bCMn(ab^n&*bakl?_&z-7q5DM~tL7*8Ba zp4%_@m|SD5^ZBM~*t4u8+a}~0`OjsL7uzAjrZUnS3Ic4_C>eX*irVNO_Oh~4QKHzH zi^{-2uGtKC>z)XxOCI^v?I5ITZzO;vE`>^UYe2e$_6D3cyXq*G(zk#gKcpVw$VJPv zdR9;VrQ?pRCO65n)?`7#`wH*#srS=4tB&x3t?OZ#F1peKp^?WmQabOm{T6oB3#JjL zuEwtx1Kilt3qv9K-!?c+N3cn&g*K;jD*8flgFeC?@%hYE&f}D3u8(Z$r14hQ883Qn zcCQb(oZbwdZqJ|GTF%9SKIik5TN@9LnZV=YmK&OCpsxU8TJRSk_u+@`)|TmePpSLg z1t5P!=;*b?nl8O|kLgwF(R!t4?5g#YJwZ8T>#XZwO+&@`=;k`o6W-XuPnttKg8@L2 z?dS)U{0RuCVq8&^A<>lZz_W5t5-fCCI*d zJIOJ#Rc;#rFaIx1Wq)V2;B$M6^3eC>(#h2|_xT&A@-K~{6rq$q8!ub8H%xa&+#Z(S zNH|L@P5*3Eex{|L|L9M{|1&+*zom5mJ-2jhI}U)`8~?ocfVoHJ18sUa*|FdG;tE+5 z>1T{mZenW#Vh~sa2n2gpIMNQzj8^*Fiul(LxMhC;#}EM?vknIVBc5P{IESn~2o0nN zC@4R>1f@Jjp^@FHCfni1Upe#wwu@GR*RbxfcT?3jKxY7DiOa5%*+R;F&}+tWdLr z%b4bDkAW?4&zdqd!5HN?00sr{Mm0BE+RH6gQ=3)jM`9XbjbS};ul*XZ-@Oo*n12be zLrf_&Oo2Wm4j)u#py)N{*#AvFMV4RJ<5JW46@f>CmkYY)qQcX!C~ zeinQo&~?;Pz!=gLr)AiB>Yn)zqt!PsQ}$adx8g4XdK(`E#)v2tLE7aiF(@8@g&l$w zLMgTqU@`tg90lR`=ch5n50V9G*qe>IfsK^=VteT_P||*sjB9_&4Euw;mY`QOJb^66 z`dvkqKdg)Pd8%%DID!axe356#gbASVl0ixVzvLJ)3K9mO^CF1rsr_+DOgDk8cc^_ILC;<0+h1T00<^AgIgM9hs z$4nMFD?DD}Lf=ibAn-SlHiE_}n#0~t1@n^<-NL|`X0hJm9do1RGh|tV=vVf= zRCGWLq-)6vnl2hhtRZhBq6QIG>Z9?}I^P-Xg3hWbu@krc;Y9q0lPh4TTMlaDXKyn4 zdr|A1h+wEZ2LUzuefYG`9Wyt$9rX7F?jy&v-}zhE8%1xeI7QJB%6>-v@}PSIA%q`z zKRicKaa>qzbwV?&eh;FSb&fd>1%gt4{U(KArT4_LcIp)B$6GE-Os}j?&&Nv}Iyo&0 zbkcRzGGyb+zXMw?FBmQyIIdx_go(AcyZZ?hkXb_yD`Xa1Z$10-JujRgeXQ+>ebGJD z>CuBML|X;9Fd9;9CQ_mZyLBF9KjkVWW0NBBzo)qMoM|8WU}bH!k4QJjYXI4%EM`q#ZienAN!$I_H_*;LMxh=sX?`H5W$Kiq)+buW5kBDd zk^J`(5t3%=reYruq9P(>W^si0Uxe(Q&4BcRIEXi$7sE+)=U)r8W(yZSf6Vo1H{~#l zpHD-?%Rj4bsoiRlM6fe=!ccU>y{=fRJb&lsN9fG@iS*-L)B<^az(|udz=e{*=ic|h zz%mjOHD9fBv=f=*g?DTsNWpA3>EaraI)U4#x2=efOwiWY&mU5ElCduC@mU!1_jvt- zQl!Q&pa?77g&*eVS2C+VsB=i%FGjMHuA^G2qnQpqjD6uTVi!x>8a{2}={hEWceNww z?hjW#RD_%b+5W7nGf@lMhK}W9%n~c(k$1Jj5~s(eXwkY6?r+INWf-E=9hMj$Bs4we z4H{hxaj;aYh9liB`X-oI1^f{xUbPV*x5t1KN~72Cfd0#8)Fd^qNApLR*;ObMrF3Pl`x#pHjCus^SEE0y;D$^cu9X zIJ8MkD{rRS3q)7*;})4w&|Ph6FCawUo7cS$AAT=xbEk1*D>)#5dh1z>tedOK{r_FG zY5yj6ZfL3~H)8tC+hU;=LWP<5IgfhlU3>1BP_nnqQ`;MlsJegMg0vnLLu^!sDPVpGMrYyQr-IPaO|G_MP*z0a7@bEzV zFoo#w%(&T5%sR_1#R+89AIh^?Y7FU00U)nEwa@JisBg{J4bEl!+BUmKltUARy086( zAR3jA0&9*TI^Sr`>PB_L3x=8!LOSVUL*qV-??t%XPmw%Ku^TnR`1mJj$IG$W_~(FK z$`t|E~eo8qM_rhs$4bx2F(A ztFfhj@E|zUw*B-?q&O)|{Ji+(g7Y(3S3B{WG;*V=vP28BddihilI z$Y1XSd`LrAKqM=yKBMPyV|_E22t;5q9WN0hbc{+ic#r~Gv(g}wWBIB$xRRbyV;QU7 zoVvjES4x$hO^QK%l8+gutAh)ozU@|R?#5m;oRB;ZIO>=e-GV)=;wiVR!%q|gQkY|y z*W-1ivE7;D_=qkoPuFKfbCQQuZ!^ozt#mtN%(i-HCfyNKbRENE`+wbmS3PcZAYpD3cAH!K zeK-QQI+m57(mA;n@{8)3Eg#`s(Yg$X!t3sM-LvMrKVJEkzE_Xr4;<&Q$ATPp#I5=U&UTzi9LlL1`v(B+07n8wA*_T?0w1 zXB#zX5{QL0AJklp=8%iiTe&~4F?m9ZDAJn`>v<$?4w1&xQbY*4@p7KYi`IUEZJ!cz z`EF{OGO?*+r1c(QwdhXDYGV6n4mL4p;wbA?Ez#g zk24-O!uM~)z}z)~f^E}AgQxaEy)-quUVxlfJr!6x5-dkuP5>{?UaL@^bMC-V_R824BrEAy|GTb2Q_uuGQmwcvGb4fWVynr+B@`?P|*WtG|K5t=ECWVcVGp1l5ObN@*DT0E*$hLj){G-Fa?(5;+L zR`p0}sc)+?mXM$;lX$Hrq7HDpdAE^OjDdYe2Z~M{Gq7trHk8;w zw#_ky3V~JZn{FyI=^l`IkVJ=$ z_}&I0TkOTB7jF6F2LhK^-lewc&0!FW4eF*>V!laP)AM3=CfF%X^L(quuf*C+Q$LJ^ zYc769BS=URlh8T*)>_EVmyfn<4(wirA9(vyfAr_RxPb0geK9n}fap zRQ>#vaB9qglJ+33DAxHQcq%6iU(C);#dCNsFlxsp>PJgMgMU-2>^x#Uzs-*myFuFa z>Y?OcZ!XaD2>(8LHX%n9**`mV8Zdwtzp7{r?co}he#!Mq&`S*7?vsq_=;JLLlBy%@ zkyse1+oQ&zh3KQkt7M(!*F|UxQS#KQCjD5WjvA2s8+a2Ia*INH7u8k3o@K-u7Fjm6 ze7jHlJpis800%1)S2ObQAGezL)NHZ>C(5RQlJi_Q6=rG0gTy+ zD#N!YW&dQt+tX{hpaBr?GV%S<*iLo80aqvN+&JV`^Iece-h4gjng83nZF`h)-DS1* z{dPtu7ZSf2@GXo#moW4budIJrs$vKxvIMx8Z%~i*eScftd@Xf_@&=QOSlA^Zie)-V zLKoO7PW)jQvgJ?WWh!G&h&oo{2^6Ab5|OH{*L7;YsqcrN>oewkKcE=_svwz(Rgk-n zhySG92Zf-ER&Jjh-qveVJX>EI{1X=@m$?UqV$TnFqN;3~IH16~Z%2{C{x~5{S^=-O6=zFyW|uoTMfHo4J0|^#Qu+Ht z4Eag#etXc3Vp`Dpf~d_AqX?Kv?+?)R<7q?mNfyZ22s$mh0*rsVzHXq}R3fZi zCtZ&r+l1=6ayPex?lEdp30>OgYScVvcgge-G=W?pF^opA$ z%JY=Q)W!}B%tdD%!hgxa9fj_*Xj7m+$aE1gkkwvsHXj1gDrmv)gs{af!Ut8so(k6K zV)l5$hMfIxEKQO?!lOC~z27fq)%^i769|vYI9)P)fuqH6Ib*@B`)fkr+HXEhWL!Hc zOr)~=NEt0S4MMsL2VJFaWRm}wDCvlZqmM{Zq<>CWEQ3&1lCB<-l^0Z}j18gWMlh7_ zNd>VaD1#P+3Qa=)W|K7=23zt6@jNZm=1{vOPOF>X;63%U2(_ibqHkdzsL`Oc8^jZ!>l{oFKhJ_lT2ohQ)WQb0Y&zyVBUNT;g@?k9u>HcV zMSr&6N8Q&*hNo|(F(?npI0hTa?MIv1bwVj8ojS(C$QR-^)Q2tF@58Ak(M`{om7jvi zzZjgr&X3s>R?Che!!DeIjZ6#g=8F285Z{Jk>%C3BZCmYjv8rmVi1Y~7C4Sj zcvTCg17}9~6@h6l#O%(s%ScDtA>{{n&a7coR)ARZ9$bAd_q1a0!)6157)}Swi zk!w{aDdF6Lf2Pj(&U)F695y;nHhmitG>(ItY;P0JD7BZ=l0G=6w_xc#2L$R)P!Yp~ zRfcpkW~oy9#J)LEDY@KX3f+)VALc{ifi2Y$PJJrTEY#JQ8CA;9!D4rNN{At8GRQmS z7!DLXc_Ot=8uFn81;FIqIfs5K!=wFIKZRJx`8w5DaE3Q2@{@jbz#a|e6|;Hh&j3<#T$0 z1G{x!K)VN{7d@w^m)o@)FSjSR7d;LSE-udxKX}a|$Nkbcp|QZ-KI^6McPTzT_bHHi z#XEw!Vu@nh3c&{DpXq?=PB%#aF}jh`pd<-&S5ZQ$@&&pIbOT2d?Js<&7bX78C!n*m zC$KO33HV(I!Mz1|y&t$9ob;Qv328{`cUXY`* zM7C(_QNo;eA*X?&cYZWi0x1w0OreQZdVoKQs7268PvewqV3a% zR<2+2k|ndd1zDXa%F*ABNcqxO#`(m)@iTDfDbjc;()3wiKmZ~^Wea~98iZ$5w5z~6 z2%?s%uENERvW*TzI2*I8fc6|I)H3=>B2`p#&7Z5(d zNdFKr8Qr*aS_@^Ba>a45FAn-*GVB9@@XPtFuuvACd&pdr)D`DSB^EaK;iR$fQHQIX zSc0?SydZ2Lpow9y(9T+wUko^l_x3Ud;&xG*I73MjB%4GkCVVrN#Y={#FZ7PA%dny>SO#|Sf^RYp@t zfc48|RdkL1Spr>K>9)rwgXuI%SFQ8zbCX&?@5AEu8YuL@lpGWxC@n)C8{voG6Bq#)@GPN6}4zk&zA$!?v1elHSU)2%C>h#!~AnQqT{VpfyFejW$VRkp3H&ZUkUZz1^%eV zU)V21!GLm*BM5bifkvN$!F$WNd&7;naA*PsGD}6YoND8E;8(ckj=TFm8Z#++^@AJc z@jjtkf*M=vR_7oVtfv@F4ub{`V?{zc-3ORbB(?>B)QwV&{N+RT4RJ8OI8}|nuU0ME zcDgtiKzvV1g>hSj8yWU@X*ZeanH(QWU|C=wyg-?L3ZHwvmOL1Qh&Mphp>)cTl`vAj zZQWeRgUw7}=4xQM^Mdyse8%b|5>GBWsah=TM-k=^xHof9qCF}BfoZ6zVvf{f$(e3u z!wHhK*;AE%dT8|}n;T<1yE=WU7f1)qI|eivxujet53reR&6Z5nba#ayc-1!B zI-Zfrt>Tb6Tgi5LDQN<9OdU|c!O2AC>g*A2Z|i_mJpQY^mk`eYQ>G-T)uB78lz375 zCOVaVr$M0MO~6$(QC0dawE+bgB30q*^W)KSMu)<7`350{XMRksE@Hwf+oWJGsZ?ni zIe)Rd#?S?=f+JU8933?GgFf?!c#uqJRg`K)k}<&|0a?rG!HxFbZzn)|(7mgW+t`^i z90I!fYzMr&$pi7tE=d2+8#fW0$|%O!YDVRPF(X;wu0lNrh@#^W$s-J|7NUgw0Z+%- zC21r#ai-L*mm-OKr;M7O25QPLov{)ePY>hDQ2T}QI4EvP8HCPqBEPiCEAW12EJTCx0o-J!C-^l7_2cG2l}jwMjP3!91x+;L zjM#mT9yrARbf;7{D{bYUPS(w?YdD)Re+8OpNo85Wg${sX0LUH7+|-4<5URi>hHejg z+8t9Tf~qRW_Uf{ego)#>SGsyz<-6uLJ_@Pb*U_62+w zcxS&+j)LGLakr!s4nIee?~GZXpMSqvgrCcPF_M*pV9+%x)y>(?xnR^0ob;<=g%|wR zy`++=Gw@k_UV}*8q1dvOVg+$hfQ&w~%%|V}mdpe(u{wFY8NC|{i^43l0cLo(;V=eJ zF%+l-LeWL2b__J)2_K{dmueSu%Ndg$!DgEPv^ivzDWTsbh$J|o1H#V&crhAX^2RNf zU^NU^-aICL-^ob}QRRr#PHCi#k-3xrsCRkrD7)lH$to;YYsN1DjUdGVwqeSf?{Tgo ze{rNY#Zu#mgt75bl+(p5&LDoU*F)b3!}Xbn3fUadEbXS1@~Y-kh?MxxhB5zj;%SznA<6t4c!_fg0<^nR8xJWD z>5Q)@d3m0>2s7u+VdmHC*?ad#`bYFfqtR(cmRzk|h1TTNv>GDx)d}ifW$_u6Szddv zeg4%nulpG_YXDA|@%CWc9`uJ*p~!eVBUX~}W(ulOi=?aQbVC{$@gj5*s5Tv%aTN@? z^58=62ZF`lZYRoVxcjI_5fS_pM|)j^yc<%}DcP~%em ztnQghBlcu8fb*$B3^F-IlVGj8w&3*=EK|hdR;0q=XLCorq+0~GQ5Ae*10bL*X~&R3 z;H=EJzkx5jns!#a8@<#qW$5l=PpC$3G!g{U-y3?gJFr>?mn;80ngl8HpO}L5z}DSZq668MbdyWd4(s+Cq+}G;X_7Na&Idc+HZQtJzv|vNn>-9+d?M@|y7O0nc z-Z-`SXH>3aNRw#?S*jRM#~v3Q{{@J9UT@5umX*4UXL{^y-d4Z*L~o~TJb7}|MXw;F z8Fd9BK4TI*%rwm3ik3LCb-~uGN=1zsJVsQ%MJ~xZ4C>aaQ1h^LPgsYetXWF}@q-YP^3LM}zIhNL z?^<#OSry)mbD;*{IHBtY`>;64ZBa5!oc1eWsr>Z$?Y61Bxc0}hw-@}$U=YgX$TvIp z?=$e!%xV45cWc6QTRU?$ye=hlZLqM`G1?scRh8jjSAv0LmpgU6*i1GW1l9$KNBo7 zhLHVDrT?_EnVJr)*m#hDG#^$-|W!NMgZuNxNISG0wLd}HOyU@e+|8xN6x=XBBOeZ|Y z$+Y=2Q)iYB8=POQbS?NYJBw&!kOi>7sg{Jn&kt{&h@CUC04?HKP@U{w2$tJdBG#C*vvOqD8R87i2?X#w;fO646}{VX`w!sskssIb!jp zP=zZ|e|JwZ-pE=w zU#3ILk!a0&+=>x3{-leZ2rIN_&y~MY-5b(G(KOvY404E*gxPHE$Gr;M-Fh+AgfL(1 z_|KNBl)x2@8w|(pes#}br*gmsiEH@0hBVqmf@iDr-HGu1-WT+{ATRL9^)wXZ3Jm1` E04;_+NdN!< literal 30453 zcmV)nK%KuIiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0POvHd)u~_C=Q>$`6+PZ?1|kmDar9A)lBvo$97Xs6Whmj+SzY* zj|GvCgl&ps3DAyqocFW;9xMPPz>6-n>^9xz^R%%@To!2%8fcDlRyH~GH1)!ps>FZ39&RGy4+oc^y)=b`PC2ltIUIH5n2 zl+ifqp_{IQlO+Gy@w#5e2?+C3ns9ivkA5WK1o;A85RVa!7{_6Veh^rM;s{-51CmCB z6XryCLV5`B%ek@J{JjVL{N#_^|2H_y2zx97FlYb2>g?>6_y6|R^ZtL5$8}xg_;E^l z4nj#9|AYAaIOw6)$WL7L+oeIvL5NQ9i1d(y5JE{7hG%g|{b>&!4^Lu#mJ&uH4j)r8 zqKv209vbJL(0;>4@+%`}LUf zq*Uc~r>4T|4!r(HJkY~QvnV2I4;|8!`1~Ax!G{rM`G1zH0o&(NlN3Ohjuz04JM5!r#L($A)fZoZbyFMDW(yPMu(VV zMg*=5>E|A54Vh?q4TbSY0`uP&rzeh+#Eg$p!hQ>3cu5-HP2)ya;7H=--KS@!E(+P<)F^b3B zdTOY1ci;AcyRUXbHSNA`sqoe}wfis`kyLI45(zkh9vb415yx3WNAL#4Q8-0(h^BFd z#`uOH93k?VGESqBh8SVr7n2NyoHcZdBjBwj@RE-SV#H4gNAb{e)^x3XS*ho&AsegJ zcUQFv->=EE;j?>9rjEln<}46n8OMzGP%Dgm9E!gMCaAgSZtXlXZhyK*@_(F6k`Qyk z+E-%|CM0DZPgceX&f))dwmPp${NMJ@&ek*k_Y}{UFYWKp4W0B5BODEBNcc1%Zzf_| z{4wdF@7iC#I--7ExzF_DPXd5P0l&~k44)eLxAM^RtWf}`CG~ppDT6zJ%_ueeNrUsn(JjhnOb2&+}at^EY zc4+{Ultz4rT7O~gUs$Wun(TKOENl9^gnebK6L*^@N4o};5wMgtB+SFRFdpDAU#H$1 zbM8c?7Fwr~A7()f_IVq12nby@VqwvIYfg;^ta__6~|d$Yj*(ogyRCkVM4kR*(!D^mx|zW;SQJFhyW``^y1S3A%5zo&Ri z{U7hM=MBJGWjswhbAx!=91!?rDVKhBKyJS(8U|*KE z2sXTyrGPv!DO1BqQrouGEJA)fN#ck^yoYi%0?;wpkwLS*eszmC&6G>EtB%(yxVpfY zq?ChH)+}+MWO`q}I^wAA6@pYI3!40rDPon)GA>S?(Ns`t9u$U+Ua$;jumsr`R)m>MF&C zl)$&jlLPNPEp|s=zqXr7Xwf_R`n6YIhT{?XsYyce4rDtSJaLC91{PRwVR!|wmbJddtIOw0M^jOvRbS2ME zvhtXu|!Um4129QOD zXkAz3K$DrHJKMjw_~rEcu-fnsX*{v98L6xCPh@&dhUIUyY-9bf%h-;zDpMDC6U^9c zoCY;OzdwCLwN9X6K z=NDHOmjWLa-4D>0Z79^dDpn86E>XqcX!c@VT<%|99L}P(`I!PH4c<6r4fv^=T7+7v zwrVJ3=@1pSo?#f@Hqhq%%l#kTpZ;<+o1z#U>-KV@ayX~u_ooN@VgVLySDZg$+U;g* z&eSFN1>?j$J$?6HP@%K;#}clqgY%=qqZ5JswKy`1i)deysrL^SN2;RaOAMC8y%%dC z&no+rDnz(5Te~LZomb4?zS-ghTp`bKG$O3ppZT??T1~~T`!wtV z=mmKB>C^lpmv3pZKju7v#aKY1kr-^NY1_G!7({n0*x>~_2CjM8j%!(MMndBAIDN37 zeCA&Gy(!c4^Au0{{8wffD^&r`Isf;zyW6Gnf48&u?Emv5&!g!7Z1bZhQ~{dq41Ww} zVWt*P&(q^80`s2kuM2E!Vo7zNpAul5F6o3YjweYEMOhfGR9R@^!9weJZ+&4cs>`QF zWvENf)EbrxH7E}4V$0|b?V2l4AL<5|Qt{<%)AH>vq@m3DLG*YE#3cb+MqRf14XY3z zaay{aKbjV?Zhtpw5?i(Ey9Jxbm3Xa1^og?@wRDR2fvT)nY|YcZ{cyB@ad~xdbpG@4 z$-67D){6;W=i*YG=-cjWVb$M!Le|_FXk3~k)2-#yfK|y#g9>mCI1kWy&d>+S4nFMv z&(+D>tIHqHkM`%P#`MHWcUrIFEIYTSqm#?y%fDTn9i3mj-M>8earq8y&)u0Fo*!MF zA0Pd^|NiRo_`}iZ$A$4m^`?l+oIVbZFRso`Pv67%E{^|ywW^sJz2t7io^l>uroZFE z_YdmtI0&2i`-3ZK^r)l*ucv%>8}wWINAHf#uZ~Xk-@ZRu7|AOw*DB{^)BEp^PA+GX zjfU3Mq|I%58M3FEp0V{+(--?6&fXuLH!2ioG_I7g9>bF)B&pIZF3{K6{`vlghjn&_ zQ#@%@+5AgNyqe@ct5gBa)&K03?Z0=owsxQ8KTq*IT>taL!k#~bROg9RKxMXcVJ*wpXDJYSzlsGWiQAW)_li%-2TMUTE&oeeI?Cp{gNMaQ3Vn`WJZQ`R|QM zIH98`PRS$mG3Wg6>~t&fA7AZ0`+q&jvxd$v7iSWKc#ePq=ypsZG{|Te&}f7b>|f&% zVV<*wF2|H1mL))L5gU^*M58bspozFt(`d9Q$Oh*0hJeJ-c#osNSwj&SNsF`f1k6`S zfNm)tqyN3(q0=awqBw$@0wk1>6ooV*p5q-}T!EROvxW}h$s~@@&j%MMpeb{_5#??8 zzXad$2LDXk@PGYcJZg*o>7VQ-Y8MR+uz#H;5c`BV-+AmdalZ2g_}cl-!z1Qsb3DhR0l{rqE{*^3EfPh!|5Nf?MpH5&5ocZ)``0V&WB&f%EA9VR zyRV+(|2)YPkc315iF_(pH_g2i_V|t+HBK-ctk-TAab8{E$eG7@oZ=Bt957BO6I`%^ zkQw{+d#{uCp~LU3s`*{7<8_>XjtCQs%ouO&?DqV@_Ule}duwO=)z)BVdmHa;Z|`me zyE}UU9_|c>gVzJU`SKpK2&dzQJ@9tu<{r%n!88{;q2_Wd9txorQx3lGT z_Aa}f-p+P!XV2T~?reX*^LnTI@157Z&Nmp@wr2mYJPPc*{r_sKeE)y7`)d1n|3AfJ z@BiZdVo?BNm;FDri%}_@Wb3~e$Agfx)Qutvs0(R2+9UVBB(m*d(N`J~e%}7y-7Clc+u7ZD-v3YXm;v9v_jbK4JK+2200KZuo<5Fn zhyu*<05gInIKm@1G7y_GP9|vm^!)Jchl>r#(c=h_NixO^c6c0V^RfXADW5{Dnm9s? z<0!ysz+69`Bq7C-PdtZ2BN~w&8pS*bopG!sDuOq-y}dR47vpr)cBn6aol#;!0*c#o zGHT;Nobh&&62{sjB*_>@+)JX7b4{kVaT)*@WKh52hvVjhcU&aCRY+xSIpF5Nj3^&x z1CR(7T#edr^~HkNzb2_0(m{&TX}&3)G9$H;^>5n$tBeRgd;fR4dzJVvyWP&S{QpUw zwY407rOlXe>N?tjO_yjn)#3^(#g~)}i%lr7Szc- zW3l^KJ7XkuM;Q%B8(L@WwIn5Ol7Z=?E8hvw*D8i~7VP;)8OMuNW~rucKaPBoaQH*n zA?83Z*=`Pl07yEkF*lA}>?Uzg0!#JWLzAhtH-wnaYip$r#bqf1n{Lf^e^)kBp$BBHT8FQZt&9mZ;GP#SsCZIGSXP zt4Q5dFLQKFrrJtTRO4fU0>WrY0+f5AuOT~jTMxaIJ-@7VtAIOB;swxF{geTCy6VY@y2`-8bdyYxBKac_44(^5yBxT(KrFi z!5|Ass)pTSJWj7$-Xi;qV|LBjIGE6gigCKi8mleMU9yz8lPu&k35lEcBMH`8(J?rQ z0&voSS#U^MDn*<$5=a+E0aFVbpoowFB(0pJ6AEJ(rPwDBRB|D(bMkN-r`KVO195E2 z|Fze2t2B<>cqpiWqEg0(ccQ;D+i>z#$3m(!i#U~GC5Lo`5}e`*;Ur};h-5&0zKJp> z_JEHkBs{>3$Q2N+d}*;UPD$`K8xBdzT0PX->2yB4ZGF8jhA4PkWr$S5k^`Bd!5V== zRWyuKl#H&%0}NJ7)@BQ06vbRp0b-Xz$>_%c77h2%oeMGa!A1|^Q2xi$jLF#gV27h$ z=W+AP1t!yiZ=N!-vr+f^Ew8)#THJQqTia+7?7#@l#Cgg%&{jWo6CH(G;(P``E)Y3&WEoC-a*;Kt8>KJd^1j?jRhh=wRmQR|wJq~$mW zt*xPb!S!c245y0y4>Y_2x}PQ47M%S-i0MSuIhV2}h772^GRT7Z4plhHU{k;$N%4TF z^$Cie2V@whL?045#_1r=q5!gqX!;8<^c3Xz^uxSXfxeLPMFBWtRe+TE5xz>l@c%Sl z{WY(lIF6Rvs;ks$ZKB&T^~WfnOw0g7!8F1X>K9o^r2191mPHBzv@Q=DK-~?oE~=v_ z3&Twm#g)#$r2$bj79`4VB7YpmtT?)4RMNa>l6nZOkCH8c3j4oYZ05H!1X3Gl=iN_7 zo8lw-H*}hi=z?SadILVfnbP1x;{+foQ@AHPpVrz)YIA6|t!8WF-E9^|s72(3h`{Ga)>sKC`xqjwGbm+}kwH z2#ZLVw|qrDCvnQbB%@$bZ#v$#x9imSSd`MSlq|{1HspEQlbziURQsMlUX)H}0>34- zq7gwjkn%1ZB0pn1o**?KU9RbHpE0O4(P%VsBXaAi*4!J4U99M=JEWoXGg?!fekdkU zyhMU%202@zUn2BdMnGnFarFM^;1UI1fH{O-*oQ#IWD_0iUmT%dempu6P57AOFdlhh z%<|^0l9b-ikc`NBUVRh2JUBf$IXbv}xq&WAl?_m?1X2~%Hqg=gizC!QM<<5@0`V37 zaDMusp*jlC?U;3-QqxY@KXG<*8Ix!M3Xt-OYZp1jR22)O0b7$}NKfXJf*BRIrmpT&@0&UC7+JWYo za-`4E9nJ&kEwJ3p%a12X#)+G6BjCNPO=WMX>V_(|lcVzk&Kmt%rdjY%Ak5ZH($38x9&nWIQ>ic)L`!J^|d zB|ev}`5_hOB-rZ7(-!VWY5=&2aI5--|1G2nTI`s>P9lLY(}YW?5)k zG?#F}zeU8C5rKwjEclcul7pActe1fmILj4_3t(*IlZ3-CzCBW_s*`-Jt-;zT8$>`- z9*0D-?o2X{aey#mbOblfW$*5=AE33iY(%i*u0VnIn)tsq)mmk{;%3U3i=)6b2ICG0 z$L^3|o~6VQB`>s@m^d4al7R1cI26~hsXRE>P##EMGYs094*gr8lGls88Q4_X2333X-t&$pE;MNz!WFcDsl8S-TlfqMP!^T{HOYSq zi|~`v%cBdAf9Cg=G0xHdZ0}a&zk9ECUOmfypW-p-uP^U?lAseQivxNt&P8!05W8d; z3t}p+5GpGLk@Q)RV;Z4poTW$!QhLr}F&@1p3-PLtoKwy_Qq7ZaY-HOLj2jLG<(THz zNK3>9(w{s8m>?HoH~Ku(QUN721S=5go94Kco{{V3R_~2Q@5pta#C;_fzb-GKOUK>h;}e;jW(hQYQNoI2wL>i}cSS<5slV7#^QB0l!dqJN-pP3YpG$r5uB2O ztcVBbx_UKS7bWbs5V;?bk>sS%7FEK{eq<2dQ_aVCQN&xUaV;N5GC|6t%;ibUNe^An z$R{P%oHEUeS2${z`vqyCHS>;^Z2^vRYsWx>Ad`42GP&?33m{c+IF)-R&Nve+p4eGG zy=DaoFZ?I-#6B&3r)iY~Bf_00_oO^~_v7isWr;2UTeTpmH0aICFCOJ&!aj9=1y9e6 zc{m7CaZtfK_2cWy4c+(I>3O+tcH=)9L45heV+oOdBmj}V>j{wbECOY>rZT^TE;g6l ztv#>fb-b=MN^^gRv%1=rb}y0D*3i*s;%A)HQ(3sorLq2AT*Z{&O3IJW+3Dfc$^M6< z(j)`kFSfoRYT+b#(=^+~Om8Yzls7e=D6R6jl;er^UW10dwiIS)Sj8CJVVvILG*}GY z7kb2K;R)|8w=}vP=HnN7D22gwK5bPpTXyIJvhm@I3qf zz3uHX|F`oj|9zULHvWgy=6=A*ya*t(^7hOQ?ooTsD@FLIEv<;Qx4K{+dVD3$Nh$9= z%k)7uDqm@%3D_4B;P`KN63%Ji4*izJoHPWqFaXGH&{Twr!a&W63sDqnYI)ZB&)@%r zw&}hEV2=I&_N$%U^8Vl3d-nf%l4r&7pZ^pDL2Mn6+~)am>Xg7WaexdGfbI~D0uphw zZSq%nMCeK%T6Y1gN_m+XpF4@Q;i-w#nXMon>@3LUz*d=8TTNfI zGk@V;S0>qx6I~+L+HAt;LK{*z)w3ZlZFg6+Y$aeSb!D1x6oo9qj+!p5TKDPJw2#Tl znARgTotky-)2C%oqv-Wcy}OL_y41skluwP5`5`8gA70+36^ zq#Z~+_n(>$fGR);2u~W_D=> zOwQ9zo?cwOJ3qR(`gmcbP%NUTE~;>?SrdeEgT+AmNKhsXC8!i$$w1%Atjnu{_wF&^ zsto?{?d&dQsDT%0{`lnh^6>4|{`tF$W&0_7lV_^cLm z^hJvL1+e;W>3yJ-&q*8{pP{d)F=0YO91Z!DG94(pWBD{@Dj=T=)4)=Vdjd`+d9q@J;{9AK)PVg9k;3%eb&c75V;y|FAZX*md)pc*b` zc?o-*xzkZYHTB)Bv^-JPWT|kesX*N>@d-u3bLApe^j)H7i|{KsAR~7?kPbc#U`VcI zDL@(oXO+n68VJ*M3Dxtdx{W%1)%tc%zSU}DHvd}7orzQZ>f-d{`N0vGgjCPhg7Ej+ z?WN4){wufg>4}r!|1cI#;()wSE`v?$K0{tga#YPR}oo&aaOC=kn;}@aq45 zJUai|)!F{#k4=n1jmUErPBxr|<(01-h2M8y@7;TV^#r5n-5!HQt7mM)WwzqI(hSwX zBOQ{;U+$N(XPNeA&}%*YjdZ&*AEgYEC1EulNL@9QPTm`oBrJ*3i~rU5$)``#u>xge&`!DnAHJ6(wkZC+uXP>Iogx+R4?|a-bn&+VT+F{N>Q>l65uQ|wP9pg{4?DawoW3ZL) z^M+ZDR8thJ#IUKHCuJ#CZ8QJk89rt|^YwqWsnUH^fOGW!-R<3y|JPP`r~4fL`$?X% zW}(hWLhKx=2()}298-7LsK05nUIUU-RVOa!qSE@76pC7Fl!#mMQ|jqxBr4|q|NCJW z-y#}8tQ6D7+L1)n;@Vx?D?X>&3;N3xFb)l2pj({-`P9sWrrt~V&Gb-9dd}1wJbL&^ z8czrxlZ+wer#K-qM-La0Kk*4NpZ+H|{sY56&ZGakJLUMVd)uAPGyQ*xr{-2tcMG$; zwkBYOp%bHU%H~UwxCaK94)f$s<;<0hN6q2jesB|)B+>Y*5BY!gsW83th zdaG|LJjZ;+4C19^LsuE(Ems0|M$xNzk>hoKGXlFcpj4f^-n?D7g1hU z82Cbl;L*4(#R!({!7d<039awDulHJYytF)D%DFzR7!bK|Vi9=j)UAoFUp14A7R4-) zNrMSfeL_PgjpUXom)}l_lvT{okTsX7x_S%S>@TxRR+pN!iThmJWK zV}>SiN)R982z7VRl;D)1c!;nMnL`6|L&7-8V=-cgQB5L*Q-4f3@p&d|%eDwkBN8CW z5k`!saWq0HNir4Id>E%dq;RLcr;4nlo8JRIPqM*ub!P7X%K(6jMkS|qwcn{*3GoIo^OkeFp&(8t0 zVMeKpBhgS1qCz%QjwrdIV>zKBcG{;ONjNF`Rnn*po#|D@c2DI@oTUWP12W19a~#(U ziqZA9|6RJjHm&+4L`jk{ z(d7DAp}AD#L}S|Yil~{;0zWIbs|R6xKUwd7U0<_)Zy+WWI1pr4Wtk|FHNXZ@Q62N7 z76A&fR8Be!Q+DP!e?{MYcS)1)zU!g)RHZaf<7NtV`Y=7BD~akKBighQnf}xcNJvy9 zDRbHlM6RxYs#6rN&T~p8@eN@}FSj|nI-NN~n6cQW5cjc&7Yv9Y*D8bx27Y)}G;6Fk zu^A-`aKM=Gd_}}-WA;^e95Y5+-ZLK)n&#`Sa&0-z9XiwXQRTTqhecHG>gSs0PUpH< zsC2oqbBEmeN4DMYv9ZdMHSFpFnm(9#^y^33*PF6mNn_P8H%5*7%uQdLpH{2aK*RB9 zW`n2?nXi@y#e$|VWbK=6AIoTzSr{Uf`eROms4vrN^#xOG43sj|mkGA|D#2FYOnK0+ zLZggv`t{H#(ehkqA{yRnW)w80eQm$gws@zbSw-Yl23qB@ivQ}>LGwVoDGU9~xyc2A zIw-XUsS1dyQImUX&V1U?l#J|bypc(|{I7tz>h9 z01aAiOjflq15yc1Rh{40Z9_se$hO2|&1=j1Q6n@w&F;6sYoIZ02l{vDOk5TMNb#mW z)teDmbMv5;YS75`(OsSq$_Eu~JVXQRUjs-#A2iGjK(n4P;7gM+*7{)8Mj=Mrmnc-TO>V6&mYaRe@&i#pY5sSIA2^E%|McsF&M8wF zR=#Uq9-PU9DvLIV`a9dN?k5zmq2benl%ctPo#!e81tbs|kn@4AQDxNaYgJtT4)ybB z^-F_fL8H}h*6q$psU`!>8X7^$3`o#Q@TS()LA9+ivDqmeW;OU=T0Fy&UL2c;arBbI z#-=H;Q?iMhm6(5Nzq=OIjrxmgJ`^$|rh-kjUn z{hF3)WpmRMnWSF}O~X>%q5i8CLsMOF{q7^>@)Km&R?Ncv@}-6lBE$wjMIrq zH7ZrZDH%$|0qEE7)4sY*1hEeiWv!Rh8SM%7$Hh1eWq#jv#ijb1Oy2znHBz8nwvj3-G*HkZ(X|J^}I_Gv0s-b|)$#z?wZ)^EI#5@0R%-7L%aMa5j4 z51GMc>DlFaYb~%8hcbjsH7lra1keF!oHNk+fciHVJ@;*x`krxb38#p1qvag7qDlR# z?>WxClDE?v5>7V}Mt|G?@LsA&K>yBQ{isqiOI2^NI|{s2diq4lKm&9;)>;-r!7g*H z+;`#Jp;k3d@ubm)#V`c;Tbu!086%dV!Y8AdU)PNv(mvy?kO0X=EA59xZC&JsU_4IM z>=}y;k}PStW9aG`!ct!~Ux=keVoHTCgV1=&$N&`eAWDSRr=iivL&wU*pjJlC#&H$~pfwn1lO8)q z#->g%F+(cvIV$NLC{ZsrUw{ zoFxI~q%TwBjC8^b3>rYC_{_KT?U?#wF@JOv#i<;r?pjPSRvC6=b{~N?axeoj#y2$1 zQbb^@N&=8yKgGF`kSNerJd$t-G+!%pT(t|p9H;9cmARyDndW?!4wGF^n>!Wj*1Y3^ zC^xO;o3oz}Wd5=`!m1EVZE@RFf14FVD&-JT$g zkb{s$oTNhn#z{b~`xKO{eGeg()fJ}%WyvTNtk+0xBe_?hfdLtc(^;p5>36(suhZWI znRC6!UL1<8R$p5EC_$D%0pcV-5(F7!NN4xYjV8G|j*_dpA+KC{I!b2>2}1N_;DD@&uS*~WRBNGGDo$0 z8GrVK#zw#~e{HXld49+txX@L`)FOV#qaT zA;;{RwIR#0*n6p~P_Z2lKg20<6#z;Tv?h13JB>4UKt?p$*mT6y!v=1U*NH6*(k{<& z1Ycwr0^W4J9rSPFpWRI$ACn9W8(EAXM^_w`Bz%KhjO0KnH%SW*Ni;=8e#k;mFB+tb%Qb(RbS4^8e{{vh6X{W_4@Z6KO*Rqfv zL_k>*;%PzR;*2L5SYwDY4s`CrEYv5e_vd$xwERC(5>VEbxy~Nu@@GAB{lClkpSs&S zJJ0byp5(zvlKcN{dmXR4`pH(deUUK)JY9!SRflXr|3L_OJC8pp%jO z%Z1nO)>h~9)~jtViAK&fnTiVs>p3ohG7f~sgXwVGTwLHxFfH;pxQI+J4SPm%hyOBw z$o(td%6CS25~S4suYfSo+ytkbM&2;ZD_9>K`A$NW2iI^>4O5~z*Z;ce5-zLWXpGqP zI$r0`PXzykpBnmaFv6?pW1jz?nF~<)|8+XM&-DK(p0zdfE{q2_M2A2Da_@-|WjvjN zI1i+Uo8n?h^z~CSys=K{E0`&}s=PxfxdnV;n@EP)1;G}GRY>Gh@!oJx;&Co-97)I| zXk62rD zDx;G_ydu>VQq^b-Z?&IE4_PfZ2sJ>&@koBO;8~D3uPHY=61W0H7e|4cg6N9V8={Je zYoi;|=vqSIAkk-jz(e0!uYk)T)8*$wLk{#(dV{<~bZEG|=-ZX=^iUEby1Rr-_HkNBYnUEfgm9W-5LuOdQam*e$_EAR zhEh-a}o-u`X#n6GbRt=yps% ztcy#xMI~b%5ItKO_PjT^;5o9rOAz5h#Yr5tzrHU&fVP26VpR70YT}fz^UIO5VKh`>-a>D{x^IY+eHSQ?Ee20A}j`Qd5%hD*Qf{ zL7~K&6q+_Md2*7l%3IHjdwHRg)Zsw#Ui7T&{d4Mc8j@%(l8F$l6umMgUIZICxDLE^xOC1tJ zNlI>LoE3Aj;jAHBZ6z5GX=bne88HI|Fe^F9(KMD8V7XC(Q*yt{F}l#AN=+wBbD>UD z$`r>?YN*r$Yinq~IK#Boug+g1KvKgbw~VwbWT?hlC_geJRZw;mo8SkdPfdMT<~?g& zjOqbcm1x-0-~yt?DR*>0n%!4>-o1 zV=*C|gj3xbD1K8SC`2@pRx@JrL^4N*qC|B(TrMl(aYB8FdkG+{$%X}SiHEK}He3y^ z9-iu6L)e1=NFWaTU^a< zD7kGb4Ua2$6jw%QVQtw(TLvXpaknJ}vF**^Wevm?X9FY=c0zd>(77zXSi}rCV0DNB z3v8dMYo!RW@*JEfPX=&g$lv-&)f*v)l~U#B2LeB%@$hMu-cd$n2|0@m_c3=H0lmoA#Pf z8QyH@;nie7Djy>a>F#!9bTK>aoP1f%X(!(*eT|U|unX_2IRbZ_`rJvYhpf9aWHoyW zEbMh~FuBlX8s(}<`I3~uQr@>^Gk^q99FX>!_`mLbwJi$_<5FZ{49Fc2j@==_JWGqy z!cJ$6zO-u8pE&i$tsZLONwB-!0`|wsN1Y#9HeV43ZP9q1pSz7CPF?*=Ly%|hZlHiV zhG3G$2}!x+?A0wUzlzjX;nPH-n+*JyyVV%+^?;qxt!P;d$RTV;8Er}UKE}aY9OB4V zyn60XWBT~0M83-`skVc13tIvGuDvGOgtb+yn3_7hd=Q8)A4TEbJv9$J({orFaZRw` zAGNnjptQrj#XRUnL1s`Mzvb4%V`*6=`ii4)y5cF&ZM$KN19zZ6Fy{i7PE+wNG{8MR z>($}cSsb7X%?3f(fX^H-OukZYnh4pD!Oa!U!LGU+M*Co?tKP{jm8X+w(kHL+)hmOt zYFqMfud#v51$oBprC}IKXaUBWhIN@{|G+|x1PJp@ff&Oqlwre^?IaxzNh;~1F~3rp zDA7Qr$jejGss3h|yXAIqJkI;15emqSdBtP8H4LVM1fBgXt#}k?kee5J>cCJ*NL#G} zI|Zz8`KvC4j!gi;3Um$gZdMIM|DWcozZ!Lv)UD;V>Z-L`n`&1o2AAa~sirNe(Owp5 zOZ#=hW_SYuR}Wo9Sr~4jD6Vt|7a&CQ5rKwx6Zzvf&P7YwB*us==OMH{O14lOq5WSj z#3*nS;8aX}(bBu0jyA<-^lzZkzTnuu-mpoQ2~JN;h}^!_LNN^uCvYc4R+#Yqw> z5l?%78TH-#6C=F61{SsAGN4~G0=R1Yl!KI}w-*Zjo(*wM(Q(RwjH}}Lcy)nAsH>o; zf*istp2i`t%LYL0wM6%$g=eV=O%6pZ85NvTlF}GxrbaX=T`#|tC%q+#xCrr$b;Pf& zDWk8ldn!ia-iLu=Enx-l7vfaR@h-%tJo6?0Vtk33%e4l9o4mK?60Ta|(fdV&MHNvB z$g?X*dVe;4{#{%4@CUzYuT?MGYc)*NTGO@mRY!d--E6!1eciQtt?_!h)_C<^tGRqH z1?kR`faV{TgVe+^prYuU6k5Jj$fK$~EPKsg)wTE=5{;SDB_^bhPQsi?;iV!>wkUIL zo-ng$wC1L@R(IEG!rStFYpwCdRqlBy=0qM>+8HAc{nE~zSJu`X`L)??zo_W{EvF*rsHkB_B!%5WmsmIMj-lRyN89w{cSWJ-%7*aDw1-R z;@P$uCHAXdm6QV3q&_SG% z(+l*;AnV%UQ~aCCOak?u(_<1&wCyfW3DKT)+OxYlpKCk}QZ4qdI@L%e7kpLS)x2}N zMgKo#1IxScae&C3^<>AalzzCV(gR;Cw|FsYpDcN9%ftng8y*0LC;&`Rq=fN|q!Y@R z++qh~9F3$gY7Awdd3>>CK`@{=I7VG(a?UD6lnS4$_-@=Urt-{Le9Wi}ky-m#wD z%Dm_1Bydw++|sz1czfDwZDavUevBEv5GR)Gy2k#$Nd0zldU65zN7+R>mGFl4Vea21FU{hw+pI=)3P6oit0)GghP| z1A<)2=w&jnEhFWYnRwK+2b9T*Rc7k-=A83tCvBkh1#;3umN4i3X((zR2mP{tesX;B zP6`y?=64%0VXd0Ucw^6n7+}j7y7Y-{0go`oHr^XR#=^w4)F!ffy z#Iut#3#2M#kW+#WGcBJw>ZG(-DwT-LrvS$3M!GtMGb*2+jyzIipgBCbkbJB(Mi)y> zBQdQh$_@SJDlHteZz0}zemb%RdIN#S zeL8c&5y0rn%DKJzS$)u?E-^r-gg++wlqlh4j7Ee@U~=qFvqo?vAW&h-yLL49;sDsV8Ct{`$Y^q|u$TG41- z)kh=TqcT@ryt}m^Uv^$PGzf%GhPI))BcbiEb+EWdBBgFS0bFJYOc9g7Z$m z=2NnA3c9WoJG^Pt48JmTNsa|%M)peNO3_%|1i}Z6D=vayMb4Io4%C7do+5H-58fa5 z7NMbV{Sav8hZMQv1=ddP`q`_3QE3gV6o`2rkW;A}Cm)X}yb$<02$2?;m4uVvi znD9>h_}bd2fN*wtepv$IKHUq(7!F|QPix}8wu)eDi*D!*C5kY%#|t%C$W7GW0_t+U zXWk_LQ^= zpMuF&oLIe%&4}O37HdNq5)e^M#I*Qh z(nH_1VS}~5bB;bIDtE2rL@r|zJF1+kW6D~03AgwLYBW#A7z3N%oP)#@a;n3L+##N3 zk&kv?!Jl+;kqw9RGitd-V`9gUkGh2Jt9f-(`|Bmqc0=NBR;0LpVQ4FVR121LfM zW%0zyc`OANl!Sq_Dm9|@ff9CIjamsgz}Gn;{)Xoq5A8|H4<r0l@d(ERJ>tocTP+R;(knfYo}tG=OD< zvRaerDj zo+JVnTYr7i^|sy4Ci>5=x94`m&cTUH$4?3K1RASWymyqB_Yh1HY86oz+Be++;kY|% zQgc9QH1e)rg97oUJA2A1(Z@BQ*`g!ITEv5OF{UDln5L;KHv4 zF+WborZdPmQjLEPqJu53+d)B0SoD$?dH?w)VllcUV37g#&pOVmwiIz;CBt|`eb3ns zSqwS|AB$!{+J_j8Mj>#}niPikpHoBx&?>c!455r8W!GCQmqC40WGN|SDRZInqi4mJ z(nWRgVpW}Ua3(>!_G3F6+qUg&>||rxb~YQ^wr$(Ct<4iV*_hAC`<^;q)%RETJu@}c zRns$#`?`K}(Fz>&uJ^H6VPs;Y&$e;RE4sZ@ur1EqtBg1{M@F{x+dx#aC1|d7=;D=R z^R43fQ}f}&#;NYqz)VNq(lyC(a?RQ$dS$P(aLLG4w9Lw=oJwM^g6+#b>o&q}dZ*=3 z-nsn2)yCpMky$CYktxNto9H0C1WKZA*vin&F>ws-lJ17Fpf|9(?s1{QbD+!UvG6jI z-0{oq*~adn&$ohIMb%!p+~xx8iuM&`+0`aRvt6tU(^cJ_uKa9@vC+|@#Cl$54UT&x`B4ORT0&Z`o!ot^4c%|<}rvc-)#j;_D-;p zGmO|kItD_#wTVBN>$L?brgl&EExNYRGuAzI*Xr*s2f-U7(>d1uR&R#aOOb3!2b?Wg zn$~Hvb(_u7SDEnF3tB0-ZRL7a3k1APtI>$oYdXIEvlUz>KARj^dpS)PpCV7@_Q8jA zLfIKzO^5j=mvf70=SU1#mf5Y&A}(pEeH{L*9@a5B%GZMAnC-gie!AGh-C)%f`ypy; zo?l*f$5FH~Pf9U^b&U^7S+z!VP-X{}I-qolnWnPtEQ%ww^w)x~!fR&y0A9UuNmUyV zzt5b?@wI|4ONM2sgIUkA>~NPXBCC-lYDbffDCMi>^#FLAcEDv}y>ScuP%WQdd zZ9tdU zq26?sen<+Tzk;b&nbJ)z-qugH*z~Z*ixNUx`~#yWGFAsOeuL?_vYkMH7*ZM^aUiJJ z_IGK5!7~z4M#WSkIqY}O&AsFaP3$W74zv6bA6|5|2pXZLyHn`1AR)ZOYu169gh5K8 zF5N||6`7q6FLMy_YIcMlm9eUnNm_FNFiENoL0a^Y@LN?~MiJXBvI)`@D%E(&ygh?r*atV9l)l^9Jxl zI@e$1?M9|~HZA}Fb}Yg1B16uU>{cQwbl$c5K1 zK^yF^2nbK$Vp>hnE@y6ddq*Ia*Hd2!mQm+;3vf1(lERNQ8CA!$_slt5QJiCnL5^_G zN_BU04Hqm}g-3LP|GeQUcXRjtR~z#C6WJ*&LQNhJXVR4a`!mCtfIQ`#W5NuUPAeJt z9V_kU$^BZX+BHpD`Lf%A%*Kd6e@}b09_!Bb+3V5D#2B(e+-#mFsLik_PY|)ov>qh8 z-;#F&;rx%<;To)*RmrIZ27PIjSzybG-=p=#>&Lg)$u~}^(oRg<${9wT41a_L z{}|i20U;`PymD*IoYncOKz&HKS;kkYDi_s?hdx2nnJ;!2!o~k%_s{PEqykO+L_%6h z5kYuF1AjGPY!{~7+uyhJf`=$bX32EiallZj{b=J?_4k&YRyW^O2yPW8 zUrkOc*;krBHAfj2i`kbU9u=eeJtF|@6O(0?qX5cQ&iJrv6ug6)9KBR|Oq@JaIg4xJ znMU|%wS!Xxkm8bH)b>itYW(mj3oVM2V%YmEv2h0KfM2b}q{{Q%`iS*e%JGqd8n(qa zOI9oTFm{#R&>bO@<9cOzx6w5#!SF}zMl9Q{4=(A}{*e(k)&uHV-c}R8(_8QaCn+U2!{ZdeYBODp_ zg;m$ZuwxP?f6BQ&{CO-;z$UuVi_(wxHKYaju?8&cw^lul@ZFb+_7^YVUl5 zbgUr|t)pLWvh}Z6lsTF{{;=AX1ROT*^H9FE;n#mzTUh!k85V=IV|b+=)gM}w+q4aA z$2LMFc~$Kwh;6ZT*g#6MGJAAy;+~nTjk3sU?KJ_RTZhJLhqRunJ?(I%f|tt_B+vH_ zGr9q9jFUCM_|c@?J2V=Blm9xo_A_$2hfj=?6S^$5|UWTu6GnbBPTEC z`E{8^9Z@Tqce?2^rBpG3Qu( zlorbMxI-(T(lqp^;ATFEnP(`z5zWJi8Efk`yCAR)roRwRji_S>Ft!cBI+Cs}KZEm9kg&2^nZiw6=Ja#`pYnT@ z=8OJYI3K%I{rvBF_7+{Xz@{C)R-kWUnfRWySP$%Us_9bVGyM_dtvU}E-JIJAT z3`Zmeo?M-rVBJ~uGP)H%Cd|vaH^u4a29`zLg!n6^0z3&KBR!$vu#U z@E&1Iy70=o^+7<9;SI8*%qUJy_Dw!#$K^-_o&fS3X8ypybNjqk^-Nx&yc^P2R6qaRSj}jtzJdyGPrX%XKZ2rYP8cAXHaQQj z)_8OnA82#oy zw{hAOdByVtNw{czAOa?k-&UUPmGHhj&{UuFfD5%`D1X$?%e=lW&!%1H9SGy2wtv^( z3Sl>t8*OrH=CDMDFFy7Pv^IF==~N8gmDQ>zQwbeuTo-@~T6+>VF~f2s!1}Ws^@kgE zn^F}4Rd>x&#{@j~{_n0^`PNLmLdc;$&(-duk>au{NC&qnjFI^$Ca`ox;N!(w#}egD z0&87Vx~E@bi7i zpf*-8TkosJNl9Qi6KyqfMZ(JcyTMg4?Bd@x&sM}%H)i9aokTM|lj3(t9y2}LQrrSF ziq5p(nT5VPOpd<<-(*1Q%*b5M$|yzG78BZ>`D>&4-QwD{-zsA2TGW>5yLw1{H3A4r z$nY}un7as^02jk9YFXcATV#wb|_t2Gco&7!LsDEyC*~W{x%m z>5)^!Yoeg&K$vxoEh$Z8^U@f{q|fHk>Pfmjkfr8OMn7Njtczn+?*x~h?w~A+)>#Jh z^-&QCK@zl;#%z`%6p1mgEt6&4r@<)KTyv1Y-Vrdc>vKo_-Q~|v(Q;X2&*x$e1h&dG zgB(S(vRT?PRevH4-BFBrH-z!v;Y>Jb+m^ZQ@KyD&>n1nkXLOg$&|gba?>D_OD)+N3 z_N4Gkc=s;cH-A`oHN4ZlEeHBk3dK4Utx{TdoB)7R6NT1`$Q)6<=UH_d_B94BmUtl! zsUS5?#2oKMC_!G0X&L2J&}2&MDHKzhzdW4kNgrmMjad1(l9HI5EQb+@ZFo5+^K8aF zeurxybD|6S-P>_*nHwr5F5=-PqQZKIm*Hf@Xx_ViVOV5^-Y>1O_{u4zM#rnFsj=^i z;l~r6UtV12orT=^avWCpUUz}3arL) z3qqQ%rpki0(eWqH)@n2TFBVMc6QL|y)VN^qrR${AEBCM!_$3k9RF6Gx1))<5EVJ?|Qnr zc3*ehp=>^r9MKwWC#iywMbA4DGt<_Fwj#5j`AeW`&N&BXw_wT}g`=p2dyuyZ#Rj zi(K)Ho#Xk7)n_{zl9uyLCxXcz@cC^n)LQx*Onb3CisOvJ8vYlcI?y;E6wc&HgQZp7 z!CH~CqLy@5BmWz79au?{QgB8{Zurd&*rj_1x_cN1yaNR%R8z}1&1W;+l$Y1r=j|Pa zryPv;fC;nL>ikicK$N}suwjtA9$;5jPL1;yG7Ri(H!$yS1`*c-4Zt5f(hmG+inVh5g2fH&!2YP{6%KA8GSQS&Z?vHAUqKpd)_04Pt+SFG8 zLoRjJqxRO2Fk=%(9d%Q1t!o!#Yb(gR`Me%dtmnw;+LWvF^=?DSj*bieI!@sB@6>M? zScA44&sQTuEOvB`uBLyuacyH1)M=M{?l?vU3@hJWJ<01H2m40c0Z~Q$-`(qkna7im zNG1~FiIaLz#2J;)-HwSmCrCf~17FSa*(W30(-~@0gz^FDe**;DV<{>HXN$SXkB_+X zg@v3+EkxHW@99WcO`jZBi-;TG*0%bHI$A@6pqKeA1BNb-EaSAkx@y?6^MwT@YU zs-Jxd^f&5QhE}bnG`VWA=<(pK5;Ha4U0!SLbw8dxyH4mX?^Nxo-fh5MBL9VWy1ajk zdGnpg$3(2v>5KDNTJj;}7xE9bF!}_7JNHjF0MD+klkffA+;gvhxyM=xz&^oRJyLyhqcpB^4{OB;TrK*hV|7ytA{nW+dpzFJwxuTlCYCFQ?!HXjTV zcUFP^Q{hEFHIW((VSTUo;t4QCu)6=(QPC>Ak-D+coCKg=|F55d5+KN1gp;-6&$h<7xFD)@hUr~8D(oC1hOY2Ik?;@L^F znScKF?#1}x$;uwV4rNu^rU-$UG5BGNI)1(tkXo znZ>XpT+mo@%?0OF0+T7T7RIRx?d@Na(kN~-P_v8NnZ=C44FPaeNI!lS76i_o(D~=|{j=;2 zNEQb%lbgmRLVq~>iKY-aJCZNJxho9&RCgDi4;y*ou`o}pzl6jdPP6}6HWZa5f&MCIny1}sIu^Z$^tP9QwHSzA`QP6&5 zUL+{wZ|p}Z)n7fQ&v5v3B#9TI`-QfM-ujWb$QvV1gy{~Opy1I zAe>2XTKxJsZw$kR`ToNt<{FZB+zhTDFaQJ1Mu@WL2eh#ox9Hn!$g%*$I8Ed@L*`%q zVVcf_G^VkQUreUZ=PfT7F)-T5XB;Bm?$jxtknO(0EQW8}E$6pg^V|d$@)#^RS<3rn zUMd>Gg%<`Ew-~|zd?ooxA{gWkI=&gVxMpb@#O>OAtfkIu+9-n(xJ+;n{T?_vKgRfg zJSUxyL=}AH9(m~Rc{BoOJ^V=-B3BIsgxMEl`gmbvfX^jBMwA1}0H$j77wWGnt$RrkxvL(9b zDH&PHtWx$l?kiouC=4DI|Y8E(*FD0Cw*{74?DkL4zZ$=SW`LzWU4Dm~YvssQPCe+_P5;ACkMX9G4>BQr zWYdtIo4Vr_TBf?!L7?hSKPp3&gb-eBtn?;H%1@`LPt=~vVs0rc6>J$CgqP@ycISBh zmza2f@?S%-Ay}Ojr{Q6nT(&;WrOWz)n4B209!j8ftgA>edzb>w6pYQymEDzhtP4F1 zMbtw4Z%W&U%2P~EUe37H-z#l($cH+%cuv-`mf+%HB{b7qSg+Z^%Vqr8aI4cU*+iDX zLynyh)CNeSYdOCxnNzA@T_qVplMy4Xh(%sHa5)#2RHzg&ugNfF&m{x+QVvjRB4F8{ zQgHuLvd$uS1!vj_5f6R4a7Bug=ePwRZUFa6T!z7ZCb|7Mhn<4vf=J_8BuE)v!|mqf z9NZ31MIZ@(oWYT-Xdvzy)E6~So-X=Vjx$OF7Ktgqree^lH`{D?++M$tonHe0n2w38 zw7lMCKrv*1@W|{p5+rp!3m~sUFQQ{2vb8Sd%%fxA@7I9<;w@JNC33$CjoF5Dvc!xu z^nw|S6=A?bmgK?^40ZwiBat{C&XR@!S^ z)N!;Nt>-43sWC;4I#_@s&Ra49PR;#1!5S;0GPr5e4tZa$UP%N;GgBt@t08>s5c~YR z#Z<)7qtu%rOm()kvpqo4>lZMNbgqcr)K^)6^PHgXeH$3+CDz zfF2^?4XisIu5L^OW$*V8qOZom#N-+*n#(RmWt>|g;EgUy!>4J}Jq%N`<)Sc*RBeb( zNRFs8zYqu>Uk-J)(X4@|*faf>m%gMI!=xE^k#NQ~F@qQhHb0ghhMHds zhoM$1DQs`biUAsMr@%vvJzl=45Fwp%c-$DR(r!bnU=FhaNR?G?U+%zs$4l)qZ2{XB znFR+6=Y$g!B9E0Q)}o=WGJ~{}HbYGrwH6c?-vJS+U~I(ji-R8&R{W?_sVo?NhN%1V zfLj5>mWTS|DENnt6y(1RBSgrk=*eL#^o*9+s|OSC*GfW2XNb~#IeIkI?TdbMrhuxU zOp5<^68J5WaqF%rnXfn3Y#N5_3rqj&>4|pVl+3*7>r$;&MWm6`{x=H^{V@5KP4Q09 zk{8d12hHr8%#^VV0uw8p9iQKp^93zqJSx0XZEGeNBQ0ZHqnr6OjeE+@is0`sJ zaf}w+QDG&k$4iwi!ov z3be0QVV4@|n5an>;bPZ$Q1YTHQ=gcT)e)Uo2oD`29m1#=mQyV{ADdjoamK40pRTQo2!E3t*i_WPmFJ4`h2}p z%c;=C(-}c$J`h5LLkTv-FYe|0D}3ScAd+0qvu59SDq-F^b5GoVG|XPE`&th;RoXPD zHP(DQ^L{s6D^qzkF98*tY>ITTsiiJbPIa=^$N5DuCXC!8xeD3kFz+xEZ^l2muEARE zn(KR4i{R7wZ%6IAzVF@o<~__2r^8~XTL1ge^de4X2zOFiDP~t@LBGmi-e3=;y*p$Cz%Id-fQ6tr_BHsvoXU<<* zTOtng-f#BWFnPzP7c!72Vsf;qEZVb9G2pGllW!)hC7;jM-qqMb%_vbVmnn3Hcq71# z(WFGUzKK8NR}LDt+XYNDp!@4h+bmn$j{~;^ih(;<8Y$!HL#&Cg>|5=6$06ouAL@wq zkX0o~aMNm?*B81F(T+&=(sBIN(CEH&&!S1U+F958F#9NSY7i#{HiEVYEd+w;7wJri z2EvqmyxQfwzJKI;aY1}Dj0Q+PJ%_wc?{txT)P0mIa5M-hFHl2UMagY7^7FgkOf1i& zn{$oa=4hoxbs;C|5hDfaF=&)DnzS{3?0BmOH8L%m4XCF3GLq!y-_eN;hQX~*4&kzn zwjY#b;Hu2W&D#9b^_omt*Y*|h9VP8T|wLN2Ycw5g7esF_eoH+geOgfWVM`yq`ns*Qrn?;s( zj1zv~Y!;4nl0}75&n)jebQxs3NJS=E=@9z-Yfk}6jW&}1yLIy1?aQ0uA9^F!nm+Wk zr#lXU237DJ(x+ChWwI7;v+3lvZ?5 z-R;(>L8@qK3(H#bp}KpX$aiR+K9!BOb7R-4Aghr;A3`6+t9P@(1fRfT+OG#kC%%mw z=0C^Cbe-vYKp~rAnf&FI0N)Z%W7qh6X_iPaPr94A<9k%-*CPHg(%L zD6e2i)R{Nq2Bxr1JT(S~mt8Ntbf&d`X7L>eK~dooO~B%rzH07|{S6*#jE2$@)Og`> z9{j)ZbAd`@*-tBD_L}J-n4p3n>EL5r@or#rV z$yS*2cyhL03MXaR=|Wb|zh=Fp2&tgUxVG;U!vRb-^jTnw3*~q*c@+?D9E{h)#_vXggtPlUdDz=WCrnULjIDI!%#g^Mt+6;exUIO zNpZVyPg@thPzX_^bf7CQtP6xEvosG2ZxxWoADlC(=^MCVjY?%yf<+4WIv`&cF~XlU z!O&s=kmH*2IH9GZOB$gysAiuBI7jLk)9||~B}4F0M-RzrMBRp`-l0+*;y_C{curWO zVlAZCyvIcuLWkIWheq`Fb_|?8HgU;) z;n0iRXy!9vogh(971Qs2N`6t-0JG)l0I&7l2(wv?nw{Djl!)JL+5$uS*IUR}~XcjKzHMzJP{OCUtHQC(lkQ=LRF!MQ!8mVWcahktJ>7xN@wQEkWe7X0;m?G>4!74Gbc|YE zQ@3jq)-RpcMZLQ$3ah+YU6Z#rU=|5}>bTdGa0vo!K9(iQE)Vi} zA<{BVlTk+|@;>O^UkjT}WApxi)t8cqVv<^9e(|>KAcBuweEVtBrAKHDqHD~7r zUKmcp+#)lGI}X|9_AmHr>>D-8t|O?inyF*#&9mwZvThaen!!xt^;*h!5Q6- zE?p;nDbb_HKjH%ub&>dY?UI@V4abM=>o_Z+4a_Vhm5+T6`+?OnG+s81H7tZ9?R61E zFVv`qk{6&3#~{Ci9NMPcB)T{MqkI3DlGq_7^%r)+D12%^T1jIjqVS|Qtou5-Tk{HS z2X`tP2Yq{|h8Q8ZdiZGpDqCnndI>*P!nLq?55fc^evKK9{D^vD&LeowP#b1+*j0?v zY!LBFFBIpj{b&&U6V}U(a$b$u1WEwv)DXsIr+%4*wW=?8z77`fO~cftOk2mS8i#qE z6W!TskJ4%qLJry7d{e}KZ0UiZS{(1z!hi!dehf#A?bG-Ks>{zOgBmIq;ri`X;Ok~f zKT+>Uw4$vC)>k=ESb6V>(!mFb$HB+{V*x(Uz>sAvOgPRa?}@~HYcNJ*9U?!5uN?q|pp0R|)x@n_m0TxbD=CpWQb`ZSD(t*AS-DmvxnJ|7i}JBUL-lLF z0MHOKK8iQu_t8GwtP!v;@yiTMrreKl@x#i~oI9I_v;{B{I7sShX;Ek!;L59cz;T zuzAncW5UnOJhI|=!#&RFS*+80bywvtUS zYQSl;F;MHHx-wmbt>$JIKhBoP|L%&Z+C3$PaQ15MD=28vIoVRFQ331Qi1G07syKpV z0tOX=Saeedu1{x61e8gYo^;{PIZfAS;?K~y>RxaT)S*_wQnT%+_K8i3RCu1VqAe3^ zJn0Yo8G*FNvTWDk@pBeRi)o2Xx8A(^Z#pHkXpF5E=A`h1b?5uc{lP?)MLp_o9i5MB zQt6~Ujz&K9-N2vIuJfhPINR75`Zh(PhW{wmFh#&a*PwJS0=i4fTnRXnm(qM&rC7;9 z6{TXE=*=PPd&t4hYNgC6oX7I3}rAuiMk_s|Wqt_rCl zYr4WOju=w9u+$GddGh1NNWMw3=t!l)qO1Z(t1H=yaqt+^#4Nb?1{?=1kvJq$l~MXR zjLL!Z8)SobekoB|;;EICExa)2q*&8`PXYZuIrvf~OC`1TU?FSO@7Yc^8Gg~H;*hG$ zwWoG5;e{HK=}~eGR?9{G1!WQ*-jYlG1x{Tw{K7lNf9XCZ0b~@c?g_4ar9&|Dm&%k6|NeFS#6xQ`H7+`5lDQ|dCDlA zz#-q~Aa7MHY8|{SYf-Qi(ZF#4Xu!PhJr&G5wP0+gMfhONJsHwsUAAFxZk{FE1cCau zG6MM}gd39yj2p#T+4J=KSUL55faD{|*b<_Ep)iP+9P2HU-nRG$y8#~%9@FnLp9&_! zr4fXgLF;0t>j^j$@$`k|vH737 zpX_`AQHT~9g=1Zp1<~kgh$EpIiRa-06(wut)u?RoA&`1YyxvT{(-9v;z%PR%Rz}pZ z+aI=u&tF1)*fG7fxj?KqLbrFInD3w?Rjk0~&3xGcw7Qrqc*>*NZa6;|X0YTn)2Zm_ zwfNCrkzaWw6r{An#uSa3rcDsFc>4j>aHANamLKF&5JuR+Z=f1wpo<1P#4BM~!RQdkjwb%bkByg89}noCvauAv`uN-%tCX5hylt_~JYcu7=C8sMESJY9CD?G6j!#eN0?`ZKOR93cBjQ)1+-+L(B45^ftniOd+WIbq2V;I;0nS`Wr#_O9s^t$lW*$O*M2 zH%Ki+wriJMfQHmwNCc|{t(CAOc`=%Mw{rF1!q1yWQ_ZAPO*n;{8~31M$SSW(v-V(_ zmGXi3UFobDWddY1BoMc-V#@e+*diN>xcF_|6GlWaq$S}{*?)7Eo416r*-Dlp*>{cM^1GaL$)~|3-1#zHTW#g#+ zp%IQ?r?;ZMWpSd=jk#s}(|u=631y~%Q+)S}^iR)g zN^}h){cnPqKE|>GxZnVrztxIn4UYQ&>U&PhY}D){n6%l^BA8-n!J+r1l|YLlmBR_S zG>0&;VLep^qUpg+?n)h^nBluZV-i=1m+^Vj3HnT`4%)(Q2rXh}A-QQ{trwD*#tP75 z3>}=IX0Jnf7`*s<^tAXPZ#(t~`$1h?u-5N|)tZ}G~{<;)v zts2@4@Pkkw<>f+}f#1QGWnd7!aJgA3iBF}TVkCsPOXHez*_8_-QIuK7sMVl1qowfN z)zEDBCHQ;Lob4GrLYegBD%mjUYE&Tw&@O@mz$*-L9E3$Yiy^V;ax*(XB68br7%o11nhroWtjHm=q82idHyizlW&z^Y){NcsFRcF z(fh>)3$7CGs^{W-2`0;WtRN zN~l&r-AGL)3twxjpbXI3cr)5`^t7(gU>yS_%lht+V=Ub_kWwi0>F6 zq0Wp9{OK|f#af4ia7Xn+-=P+yqJ|=4PT+r{En*y2sCoLnrnXJ;$*?E+n;fjkLk+7JuEvGEl2_;3j6dP$Xq zi?2iy8#K3>6L!curlV;C9flSOGYsp#4*b*n-BTV* z@?%3qwcL{6)S;wHIcCGK3WQDHY14+Nj14*K0(h%i0y4IbXA7?dtBK-oFGRIl`ePmh z-^Wsaeqb%KP*Yz$(JxXTz1xjPt(>Tios9lJ^Rcyt67mnah&xG?%&&TK>uW%%2TGei znoo8JG%@{(j56fW(IffQfY#>tPT5X6_zWt3b+I5}oVyj;mC?-oW9b7Q)LBXS z_NMbz9V-b0p}d)U4D}PjQ5N}=PTSY3-@AO~@lmvLvlDOt;sGklKkUj=a>stD+&cM+ zzH}3qq>6tp6X7(Nk0tV>`UN2JZ!GE9LKQB5(@hMOw54n>OyKyl@`%NmwT~oAR+e*U zdtHezZLa%sR+Cw3>}nQ!U`;(&%`Fd7)e3}Vy}6twgbn8Ky0%NGBetC`>}$AjY*)&L z5!$`w=j!9HE58pJ3An9ctIxeitIf8mrl)i2aI4t~rDVS2Xmw?p#nJ|m#o}y`ahO!< zP!0J#O`Mkq&PjH(!X7y>9nI}I&*G9oL2y8)(2U$)TuQcB>-3*->GaCI*v|My z%Em9&Q&vM?RGjpL#6InE9o#!9-Xy2Z40pV@9fu+Q*0I~Vx$SS&=Q2&4LGs=;U?ZmXigeL zr{9?>1-w>B%iNiVS0l-U=9)3-trXEK2 z_qmU|F@IHG@yzc931lM^@ZbZW%=Y#E0Vs?DexVRe87`S--uud5`uW#4_v$fykBPB$5 zpc`R5Bd}@;c>b@i^P}$nstTzi66CcAo+m_wd6 zO$TC2%AYvV$QIcg4#_Y}%uzR*w}l1$z-)%#{e$5g_1arP6~Lgwi{9gjktE%FH2@|NLv;PD0G tI=-(1tp8pNT_f?1T-cMxxJ&Wxl)bwWJ>UC-e*fVXsEH312XYAp@;?dUqptt} diff --git a/charts/dist/index.yaml b/charts/dist/index.yaml index aee00449a73..dc07348c221 100755 --- a/charts/dist/index.yaml +++ b/charts/dist/index.yaml @@ -3,11 +3,11 @@ entries: feast: - apiVersion: v1 appVersion: 0.1.0 - created: 2019-02-08T15:00:47.594234304+08:00 + created: 2019-03-15T19:35:11.848995-07:00 description: A Helm chart to install Feast on kubernetes - digest: 099023ae667dd8129fbf72adf14732cf1f32af60784d694634ef4e1983957a5a + digest: cbc97b4be6a84a33055900b620d2f6b6176ee81d86eb2e8bfcc9096ea040ff47 name: feast urls: - feast-0.1.0.tgz version: 0.1.0 -generated: 2019-02-08T15:00:47.592388768+08:00 +generated: 2019-03-15T19:35:11.844526-07:00 diff --git a/charts/feast/templates/core-deploy.yaml b/charts/feast/templates/core-deploy.yaml index 1f5c8c315fd..dca2c5ea502 100644 --- a/charts/feast/templates/core-deploy.yaml +++ b/charts/feast/templates/core-deploy.yaml @@ -81,20 +81,34 @@ spec: value: "{{ .Values.core.projectId}}" - name: TRAINING_DATASET_PREFIX value: "{{ .Values.core.trainingDatasetPrefix }}" - - name: CORE_API_URI - value: {{ .Values.core.service.extIPAdr }}:{{ .Values.core.service.grpc.targetPort }} - name: JOB_RUNNER value: {{ .Values.core.jobs.runner }} + - name: JOB_WORKSPACE + value: {{ .Values.core.jobs.workspace }} - name: JOB_OPTIONS value: {{ .Values.core.jobs.options | toJson}} - name: JOB_MONITOR_PERIOD_MS value: "{{ .Values.core.jobs.monitoring.period }}" - name: JOB_MONITOR_INITIAL_DELAY_MS value: "{{ .Values.core.jobs.monitoring.initialDelay }}" - - name: JOB_ERRORS_STORE_TYPE - value: {{ .Values.core.jobs.errorStoreType }} - - name: JOB_ERRORS_STORE_OPTIONS - value: {{ .Values.core.jobs.errorStoreOptions | toJson}} + {{- if .Values.store.serving }} + - name: STORE_SERVING_TYPE + value: {{ .Values.store.serving.type }} + - name: STORE_SERVING_OPTIONS + value: {{ .Values.store.serving.options | toJson}} + {{- end }} + {{- if .Values.store.warehouse }} + - name: STORE_WAREHOUSE_TYPE + value: {{ .Values.store.warehouse.type }} + - name: STORE_WAREHOUSE_OPTIONS + value: {{ .Values.store.warehouse.options | toJson}} + {{- end }} + {{- if .Values.store.errors }} + - name: STORE_ERRORS_TYPE + value: {{ .Values.store.errors.type }} + - name: STORE_ERRORS_OPTIONS + value: {{ .Values.store.errors.options | toJson}} + {{- end }} - name: STATSD_HOST value: {{ .Values.statsd.host }} - name: STATSD_PORT diff --git a/charts/feast/values.yaml b/charts/feast/values.yaml index ac49033c939..fde0f7e7d90 100644 --- a/charts/feast/values.yaml +++ b/charts/feast/values.yaml @@ -24,7 +24,8 @@ core: http: port: 80 targetPort: 8080 - jobs: + jobs: + workspace: "/tmp" runner: DirectRunner options: "{}" errorStoreType: "stdout" @@ -35,6 +36,16 @@ core: trainingDatasetPrefix: "fs" # logType: JSON +#store: +# errors: +# type: "stdout" +# warehouse: +# type: "bigquery" +# options: '{"project": "gcp-project-id", "dataset": "feast"}' +# serving: +# type: "redis" +# options: '{"host": "...", "port": "6379"}' + postgresql: provision: true imageTag: 9.6.11 diff --git a/core/pom.xml b/core/pom.xml index 2f72f41f1f2..65565286b34 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -221,6 +221,11 @@ google-cloud-bigquery 1.48.0 + + com.google.cloud + google-cloud-nio + 0.83.0-alpha + org.apache.flink flink-clients_2.11 @@ -243,7 +248,11 @@ jackson-annotations 2.9.8 - + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.9.8 + @@ -330,5 +339,31 @@ 3.11.0 test + + + org.springframework + spring-test + 5.0.8.RELEASE + test + + + org.springframework.boot + spring-boot-test + ${springBootVersion} + test + + + org.springframework.boot + spring-boot-test-autoconfigure + ${springBootVersion} + test + + + com.h2database + h2 + 1.4.198 + test + + diff --git a/core/src/main/java/feast/core/OnContextRefresh.java b/core/src/main/java/feast/core/OnContextRefresh.java new file mode 100644 index 00000000000..66e9139c739 --- /dev/null +++ b/core/src/main/java/feast/core/OnContextRefresh.java @@ -0,0 +1,29 @@ +package feast.core; + +import feast.core.config.StorageConfig.StorageSpecs; +import feast.core.service.SpecService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class OnContextRefresh { + + @Autowired + private SpecService specService; + @Autowired + private StorageSpecs storageSpecs; + + @EventListener + public void onApplicationEvent(ContextRefreshedEvent event) { + if (storageSpecs.getServingStorageSpec() != null) { + specService.registerStorage(storageSpecs.getServingStorageSpec()); + } + if (storageSpecs.getWarehouseStorageSpec() != null) { + specService.registerStorage(storageSpecs.getWarehouseStorageSpec()); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/feast/core/config/AppConfig.java b/core/src/main/java/feast/core/config/AppConfig.java index 76dd6dcc227..f4533d64828 100644 --- a/core/src/main/java/feast/core/config/AppConfig.java +++ b/core/src/main/java/feast/core/config/AppConfig.java @@ -28,13 +28,15 @@ public class AppConfig { @Bean public ImportJobDefaults getImportJobDefaults( - @Value("${feast.jobs.coreUri}") String coreApiUri, @Value("${feast.jobs.runner}") String runner, @Value("${feast.jobs.options}") String options, @Value("${feast.jobs.executable}") String executable, - @Value("${feast.jobs.errorsStoreType}") String errorsStoreType, - @Value("${feast.jobs.errorsStoreOptions}") String errorsStoreOptions) { - return new ImportJobDefaults( - coreApiUri, runner, options, executable, errorsStoreType, errorsStoreOptions); + @Value("${feast.jobs.workspace}") String workspace) { + return ImportJobDefaults.builder() + .importJobOptions(options) + .runner(runner) + .executable(executable) + .workspace(workspace) + .build(); } } diff --git a/core/src/main/java/feast/core/config/ImportJobDefaults.java b/core/src/main/java/feast/core/config/ImportJobDefaults.java index 989777da448..63dbe9e5896 100644 --- a/core/src/main/java/feast/core/config/ImportJobDefaults.java +++ b/core/src/main/java/feast/core/config/ImportJobDefaults.java @@ -17,7 +17,7 @@ package feast.core.config; -import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; import lombok.Setter; @@ -26,13 +26,12 @@ */ @Getter @Setter -@AllArgsConstructor +@Builder public class ImportJobDefaults { - private String coreApiUri; - private String runner; + private String importJobOptions; + private String runner; + private String workspace; private String executable; - private String errorsStoreType; - private String errorsStoreOptions; } diff --git a/core/src/main/java/feast/core/config/StorageConfig.java b/core/src/main/java/feast/core/config/StorageConfig.java new file mode 100644 index 00000000000..8c9509c4582 --- /dev/null +++ b/core/src/main/java/feast/core/config/StorageConfig.java @@ -0,0 +1,83 @@ +package feast.core.config; + +import static feast.core.util.TypeConversion.convertJsonStringToMap; + +import com.google.common.base.Strings; +import feast.core.validators.SpecValidator; +import feast.specs.StorageSpecProto.StorageSpec; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@AllArgsConstructor +@NoArgsConstructor +public class StorageConfig { + + public static final String DEFAULT_SERVING_ID = "SERVING"; + public static final String DEFAULT_WAREHOUSE_ID = "WAREHOUSE"; + public static final String DEFAULT_ERRORS_ID = "ERRORS"; + + @Autowired + private SpecValidator validator; + + private StorageSpec buildStorageSpec( + String id, + String type, + String options) { + options = Strings.isNullOrEmpty(options) ? "{}" : options; + Map optionsMap = convertJsonStringToMap(options); + if (Strings.isNullOrEmpty(type)) { + return null; + } + StorageSpec storageSpec = StorageSpec.newBuilder() + .setId(id) + .setType(type) + .putAllOptions(optionsMap) + .build(); + switch (id) { + case DEFAULT_SERVING_ID: + validator.validateServingStorageSpec(storageSpec); + break; + case DEFAULT_WAREHOUSE_ID: + validator.validateWarehouseStorageSpec(storageSpec); + break; + case DEFAULT_ERRORS_ID: + validator.validateErrorsStorageSpec(storageSpec); + break; + } + return storageSpec; + } + + @Bean + public StorageSpecs getStorageSpecs( + @Value("${feast.store.serving.type}") String servingType, + @Value("${feast.store.serving.options}") String servingOptions, + @Value("${feast.store.warehouse.type}") String warehouseType, + @Value("${feast.store.warehouse.options}") String warehouseOptions, + @Value("${feast.store.errors.type}") String errorsType, + @Value("${feast.store.errors.options}") String errorsOptions) { + StorageSpecs storageSpecs = StorageSpecs.builder() + .servingStorageSpec(buildStorageSpec(DEFAULT_SERVING_ID, servingType, servingOptions)) + .warehouseStorageSpec( + buildStorageSpec(DEFAULT_WAREHOUSE_ID, warehouseType, warehouseOptions)) + .errorsStorageSpec(buildStorageSpec(DEFAULT_ERRORS_ID, errorsType, errorsOptions)) + .build(); + return storageSpecs; + } + + @Builder + @Getter + public static class StorageSpecs { + + private StorageSpec servingStorageSpec; + private StorageSpec warehouseStorageSpec; + private StorageSpec errorsStorageSpec; + } +} diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index a48222a536a..2787f3e262c 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -17,10 +17,24 @@ package feast.core.grpc; +import com.google.common.base.Strings; import com.google.protobuf.Empty; import com.timgroup.statsd.StatsDClient; import feast.core.CoreServiceGrpc.CoreServiceImplBase; -import feast.core.CoreServiceProto.CoreServiceTypes.*; +import feast.core.CoreServiceProto.CoreServiceTypes.ApplyEntityResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ApplyFeatureGroupResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ApplyFeatureResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ApplyStorageResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.GetEntitiesRequest; +import feast.core.CoreServiceProto.CoreServiceTypes.GetEntitiesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.GetFeaturesRequest; +import feast.core.CoreServiceProto.CoreServiceTypes.GetFeaturesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.GetStorageRequest; +import feast.core.CoreServiceProto.CoreServiceTypes.GetStorageResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ListEntitiesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ListFeaturesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ListStorageResponse; +import feast.core.config.StorageConfig.StorageSpecs; import feast.core.exception.RegistrationException; import feast.core.exception.RetrievalException; import feast.core.model.EntityInfo; @@ -31,27 +45,37 @@ import feast.core.validators.SpecValidator; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureGroupSpecProto; +import feast.specs.FeatureSpecProto.DataStore; +import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; +import java.util.List; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; -import java.util.List; -import java.util.stream.Collectors; - -/** Implementation of the feast core GRPC service. */ +/** + * Implementation of the feast core GRPC service. + */ @Slf4j @GRpcService public class CoreServiceImpl extends CoreServiceImplBase { - @Autowired private SpecService specService; - @Autowired private SpecValidator validator; + @Autowired + private SpecService specService; + + @Autowired + private SpecValidator validator; - @Autowired private StatsDClient statsDClient; + @Autowired + private StatsDClient statsDClient; + + @Autowired + private StorageSpecs storageSpecs; /** * Gets specs for all entities requested in the request. If the retrieval of any one of them @@ -84,7 +108,9 @@ public void getEntities( } } - /** Gets specs for all entities registered in the registry. */ + /** + * Gets specs for all entities registered in the registry. + */ @Override public void listEntities(Empty request, StreamObserver responseObserver) { long now = System.currentTimeMillis(); @@ -142,7 +168,9 @@ public void getFeatures( } } - /** Gets specs for all features registered in the registry. TODO: some kind of pagination */ + /** + * Gets specs for all features registered in the registry. TODO: some kind of pagination + */ @Override public void listFeatures(Empty request, StreamObserver responseObserver) { long now = System.currentTimeMillis(); @@ -200,7 +228,9 @@ public void getStorage( } } - /** Gets specs for all storage registered in the registry. */ + /** + * Gets specs for all storage registered in the registry. + */ @Override public void listStorage(Empty request, StreamObserver responseObserver) { long now = System.currentTimeMillis(); @@ -236,6 +266,7 @@ public void listStorage(Empty request, StreamObserver respo public void applyFeature( FeatureSpec request, StreamObserver responseObserver) { try { + request = applyDefaultStores(request); validator.validateFeatureSpec(request); FeatureInfo feature = specService.applyFeature(request); ApplyFeatureResponse response = @@ -251,6 +282,25 @@ public void applyFeature( } } + public FeatureSpec applyDefaultStores(FeatureSpec featureSpec) { + DataStores.Builder dataStoreBuilder = DataStores.newBuilder(); + if (Strings.isNullOrEmpty(featureSpec.getDataStores().getServing().getId())) { + log.info("Feature has no serving store specified using default"); + if (storageSpecs.getServingStorageSpec() != null) { + dataStoreBuilder.setServing(DataStore.newBuilder() + .setId(storageSpecs.getServingStorageSpec().getId())); + } + } + if (Strings.isNullOrEmpty(featureSpec.getDataStores().getServing().getId())) { + if (storageSpecs.getWarehouseStorageSpec() != null) { + log.info("Feature has no warehouse store specified using default"); + dataStoreBuilder.setWarehouse(DataStore.newBuilder() + .setId(storageSpecs.getWarehouseStorageSpec().getId())); + } + } + return featureSpec.toBuilder().setDataStores(dataStoreBuilder).build(); + } + /** * Registers a single feature group spec to the registry. If validation fails, will returns a bad * request error. If registration fails (e.g. connection to the db is interrupted), an internal diff --git a/core/src/main/java/feast/core/job/JobManager.java b/core/src/main/java/feast/core/job/JobManager.java index 7dd098debec..c250a5ee189 100644 --- a/core/src/main/java/feast/core/job/JobManager.java +++ b/core/src/main/java/feast/core/job/JobManager.java @@ -17,17 +17,19 @@ package feast.core.job; -import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import java.nio.file.Path; public interface JobManager { /** * Submit an ingestion job into runner - * @param importSpec import spec of the ingestion job to run - * @param jobName name of the job + * + * @param importJobSpecs wrapper of all the specs needed for the ingestion job to run + * @param workspace path for working directory of the job, for errors and specs * @return extId runner specific job ID. */ - String submitJob(ImportSpec importSpec, String jobName); + String submitJob(ImportJobSpecs importJobSpecs, Path workspace); /** * abort a job given runner-specific job ID. diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java index 61fb5d87b53..65d3046eb79 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java +++ b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java @@ -24,7 +24,8 @@ import com.google.common.base.Strings; import feast.core.config.ImportJobDefaults; import feast.core.job.direct.DirectRunnerJobManager; -import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import java.nio.file.Path; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -33,7 +34,6 @@ public class DataflowJobManager extends DirectRunnerJobManager { private final String projectId; private final String location; private final Dataflow dataflow; - private ImportJobDefaults defaults; public DataflowJobManager( Dataflow dataflow, String projectId, String location, ImportJobDefaults importJobDefaults) { @@ -43,12 +43,11 @@ public DataflowJobManager( this.projectId = projectId; this.location = location; this.dataflow = dataflow; - this.defaults = importJobDefaults; } @Override - public String submitJob(ImportSpec importSpec, String jobId) { - return super.submitJob(importSpec, jobId); + public String submitJob(ImportJobSpecs importJobSpecs, Path workspace) { + return super.submitJob(importJobSpecs, workspace); } @Override diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java index 38acbecb5ad..c4055b929bc 100644 --- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java +++ b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java @@ -22,11 +22,11 @@ import feast.core.exception.JobExecutionException; import feast.core.job.JobManager; import feast.core.util.TypeConversion; -import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import java.io.BufferedReader; import java.io.InputStreamReader; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Optional; @@ -45,8 +45,9 @@ public DirectRunnerJobManager(ImportJobDefaults importJobDefaults) { } @Override - public String submitJob(ImportSpec importSpec, String jobId) { - ProcessBuilder pb = getProcessBuilder(importSpec, jobId); + public String submitJob(ImportJobSpecs importJobSpecs, Path workspace) { + ProcessBuilder pb = getProcessBuilder(importJobSpecs, workspace); + log.info(String.format("Executing command: %s", String.join(" ", pb.command()))); try { @@ -66,25 +67,19 @@ public void abortJob(String extId) { /** * Builds the command to execute the ingestion job * - * @param importSpec - * @param jobId * @return configured ProcessBuilder */ @VisibleForTesting - public ProcessBuilder getProcessBuilder(ImportSpec importSpec, String jobId) { + public ProcessBuilder getProcessBuilder(ImportJobSpecs importJobSpecs, Path workspace) { Map options = TypeConversion.convertJsonStringToMap(defaults.getImportJobOptions()); List commands = new ArrayList<>(); commands.add("java"); commands.add("-jar"); commands.add(defaults.getExecutable()); - commands.add(option("jobName", jobId)); + commands.add(option("jobName", importJobSpecs.getJobId())); + commands.add(option("workspace", workspace.toUri().toString())); commands.add(option("runner", defaults.getRunner())); - commands.add( - option("importSpecBase64", Base64.getEncoder().encodeToString(importSpec.toByteArray()))); - commands.add(option("coreApiUri", defaults.getCoreApiUri())); - commands.add(option("errorsStoreType", defaults.getErrorsStoreType())); - commands.add(option("errorsStoreOptions", defaults.getErrorsStoreOptions())); options.forEach((k, v) -> commands.add(option(k, v))); return new ProcessBuilder(commands); @@ -99,7 +94,7 @@ public ProcessBuilder getProcessBuilder(ImportSpec importSpec, String jobId) { @VisibleForTesting public String runProcess(Process p) { try (BufferedReader outputStream = - new BufferedReader(new InputStreamReader(p.getInputStream())); + new BufferedReader(new InputStreamReader(p.getInputStream())); BufferedReader errorsStream = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { String extId = ""; diff --git a/core/src/main/java/feast/core/job/flink/FlinkJobManager.java b/core/src/main/java/feast/core/job/flink/FlinkJobManager.java index 1ec16ec17da..72cfdb58a75 100644 --- a/core/src/main/java/feast/core/job/flink/FlinkJobManager.java +++ b/core/src/main/java/feast/core/job/flink/FlinkJobManager.java @@ -20,9 +20,9 @@ import feast.core.config.ImportJobDefaults; import feast.core.job.JobManager; import feast.core.util.TypeConversion; -import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Base64; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -48,10 +48,10 @@ public FlinkJobManager( } @Override - public String submitJob(ImportSpec importSpec, String jobId) { - flinkCli.parseParameters(createRunArgs(importSpec, jobId)); + public String submitJob(ImportJobSpecs importJobSpecs, Path workspace) { + flinkCli.parseParameters(createRunArgs(importJobSpecs, workspace)); - return getFlinkJobId(jobId); + return getFlinkJobId(importJobSpecs.getJobId()); } @Override @@ -70,7 +70,7 @@ private String getFlinkJobId(String jobId) { return ""; } - private String[] createRunArgs(ImportSpec importSpec, String jobId) { + private String[] createRunArgs(ImportJobSpecs importJobSpecs, Path workspace) { Map options = TypeConversion.convertJsonStringToMap(defaults.getImportJobOptions()); List commands = new ArrayList<>(); @@ -79,16 +79,12 @@ private String[] createRunArgs(ImportSpec importSpec, String jobId) { commands.add("-m"); commands.add(masterUrl); commands.add(defaults.getExecutable()); - commands.add(option("jobName", jobId)); + commands.add(option("jobName", importJobSpecs.getJobId())); commands.add(option("runner", defaults.getRunner())); - commands.add( - option("importSpecBase64", Base64.getEncoder().encodeToString(importSpec.toByteArray()))); - commands.add(option("coreApiUri", defaults.getCoreApiUri())); - commands.add(option("errorsStoreType", defaults.getErrorsStoreType())); - commands.add(option("errorsStoreOptions", defaults.getErrorsStoreOptions())); + commands.add(option("workspace", workspace.toString())); options.forEach((k, v) -> commands.add(option(k, v))); - return commands.toArray(new String[] {}); + return commands.toArray(new String[]{}); } private String[] createStopArgs(String extId) { @@ -97,7 +93,7 @@ private String[] createStopArgs(String extId) { commands.add("-m"); commands.add(masterUrl); commands.add(extId); - return commands.toArray(new String[] {}); + return commands.toArray(new String[]{}); } private String option(String key, String value) { diff --git a/core/src/main/java/feast/core/service/JobManagementService.java b/core/src/main/java/feast/core/service/JobManagementService.java index 02fa40b280c..960ee7e91a5 100644 --- a/core/src/main/java/feast/core/service/JobManagementService.java +++ b/core/src/main/java/feast/core/service/JobManagementService.java @@ -17,9 +17,21 @@ package feast.core.service; +import static com.google.common.base.Predicates.not; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; import feast.core.JobServiceProto.JobServiceTypes.JobDetail; import feast.core.config.ImportJobDefaults; +import feast.core.config.StorageConfig; +import feast.core.config.StorageConfig.StorageSpecs; import feast.core.dao.JobInfoRepository; import feast.core.dao.MetricsRepository; import feast.core.exception.JobExecutionException; @@ -29,13 +41,28 @@ import feast.core.log.Action; import feast.core.log.AuditLogger; import feast.core.log.Resource; +import feast.core.model.EntityInfo; +import feast.core.model.FeatureInfo; import feast.core.model.JobInfo; import feast.core.model.JobStatus; import feast.core.model.Metrics; +import feast.core.model.StorageInfo; +import feast.core.util.PathUtil; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.StorageSpecProto.StorageSpec; +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -45,24 +72,106 @@ @Slf4j @Service public class JobManagementService { + private static final String JOB_PREFIX_DEFAULT = "feastimport"; private static final String UNKNOWN_EXT_JOB_ID = ""; + private static final String IMPORT_JOB_SPECS_FILENAME = "importJobSpecs.yaml"; private JobInfoRepository jobInfoRepository; private MetricsRepository metricsRepository; private JobManager jobManager; private ImportJobDefaults defaults; + private SpecService specService; + private StorageSpecs storageSpecs; @Autowired public JobManagementService( JobInfoRepository jobInfoRepository, MetricsRepository metricsRepository, JobManager jobManager, - ImportJobDefaults defaults) { + ImportJobDefaults defaults, + SpecService specService, + StorageSpecs storageSpecs) { this.jobInfoRepository = jobInfoRepository; this.metricsRepository = metricsRepository; this.jobManager = jobManager; this.defaults = defaults; + this.specService = specService; + this.storageSpecs = storageSpecs; + } + + public void writeImportJobSpecs(ImportJobSpecs importJobSpecs, Path workspace) { + Path destination = workspace.resolve(IMPORT_JOB_SPECS_FILENAME); + log.info("Writing ImportJobSpecs to {}", destination); + try { + String json = JsonFormat.printer().omittingInsignificantWhitespace().print(importJobSpecs); + TypeToken> typeToken = new TypeToken>() { + }; + Map objectMap = new Gson().fromJson(json, typeToken.getType()); + ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + String yaml = yamlMapper.writer().writeValueAsString(objectMap); + Files.write(destination, Lists.newArrayList(yaml)); + } catch (JsonProcessingException | InvalidProtocolBufferException e) { + throw new JobExecutionException("Cannot serialise to ImportJobSpecs to YAML", e); + } catch (IOException e) { + throw new JobExecutionException( + String.format("Cannot write ImportJobSpecs to workspace %s", destination), e); + } + } + + private ImportJobSpecs buildImportJobSpecs(ImportSpec importSpec, String jobId) { + List entitySpecs = specService.getEntities(importSpec.getEntitiesList()) + .stream() + .map(EntityInfo::getEntitySpec) + .collect(Collectors.toList()); + Set featureIds = importSpec.getSchema().getFieldsList().stream() + .map(Field::getFeatureId).filter(not(Strings::isNullOrEmpty)).collect(Collectors.toSet()); + List featureSpecs = specService.getFeatures(Lists.newArrayList(featureIds)) + .stream() + .map(FeatureInfo::getFeatureSpec) + .collect(Collectors.toList()); + + Set servingStoreIds = featureSpecs.stream() + .map(featureSpec -> featureSpec.getDataStores().getServing().getId()) + .filter(not(Strings::isNullOrEmpty)) + .collect(Collectors.toSet()); + if (!servingStoreIds.contains(StorageConfig.DEFAULT_SERVING_ID) + && storageSpecs.getServingStorageSpec() != null) { + servingStoreIds.add(StorageConfig.DEFAULT_SERVING_ID); + } + List servingStorageSpecs = specService + .getStorage(Lists.newArrayList(servingStoreIds)).stream() + .map(StorageInfo::getStorageSpec) + .collect(Collectors.toList()); + + Set warehouseStoreIds = featureSpecs.stream() + .map(featureSpec -> featureSpec.getDataStores().getWarehouse().getId()) + .filter(not(Strings::isNullOrEmpty)) + .collect(Collectors.toSet()); + if (!warehouseStoreIds.contains(StorageConfig.DEFAULT_WAREHOUSE_ID) + && storageSpecs.getWarehouseStorageSpec() != null) { + warehouseStoreIds.add(StorageConfig.DEFAULT_WAREHOUSE_ID); + } + List warehouseStorageSpecs = Lists.newArrayList(); + if (warehouseStoreIds.size() > 0) { + warehouseStorageSpecs = specService + .getStorage(Lists.newArrayList(warehouseStoreIds)).stream() + .map(StorageInfo::getStorageSpec) + .collect(Collectors.toList()); + } + + ImportJobSpecs.Builder importJobSpecsBuilder = ImportJobSpecs.newBuilder() + .setJobId(jobId) + .setImportSpec(importSpec) + .addAllEntitySpecs(entitySpecs) + .addAllFeatureSpecs(featureSpecs) + .addAllServingStorageSpecs(servingStorageSpecs) + .addAllWarehouseStorageSpecs(warehouseStorageSpecs); + if (storageSpecs.getErrorsStorageSpec() != null) { + importJobSpecsBuilder.setErrorsStorageSpec(storageSpecs.getErrorsStorageSpec()); + } + + return importJobSpecsBuilder.build(); } /** @@ -96,6 +205,7 @@ public JobDetail getJob(String id) { return jobDetailBuilder.build(); } + /** * Submit ingestion job to runner. * @@ -105,11 +215,23 @@ public JobDetail getJob(String id) { */ public String submitJob(ImportSpec importSpec, String namePrefix) { String jobId = createJobId(namePrefix); + Path workspace = PathUtil.getPath(defaults.getWorkspace()).resolve(jobId); + try { + Files.createDirectory(workspace); + } catch (FileAlreadyExistsException e) { + } catch (IOException e) { + throw new RuntimeException( + String.format("Could not initialise job workspace job: %s", workspace.toString()), e); + } + ImportJobSpecs importJobSpecs = buildImportJobSpecs(importSpec, jobId); + writeImportJobSpecs(importJobSpecs, workspace); + boolean isDirectRunner = Runner.DIRECT.getName().equals(defaults.getRunner()); try { if (!isDirectRunner) { JobInfo jobInfo = - new JobInfo(jobId, UNKNOWN_EXT_JOB_ID, defaults.getRunner(), importSpec, JobStatus.PENDING); + new JobInfo(jobId, UNKNOWN_EXT_JOB_ID, defaults.getRunner(), importSpec, + JobStatus.PENDING); jobInfoRepository.save(jobInfo); } @@ -120,7 +242,7 @@ public String submitJob(ImportSpec importSpec, String namePrefix) { "Building graph and submitting to %s", defaults.getRunner()); - String extId = jobManager.submitJob(importSpec, jobId); + String extId = jobManager.submitJob(importJobSpecs, workspace); if (extId.isEmpty()) { throw new RuntimeException( String.format("Could not submit job: \n%s", "unable to retrieve job external id")); @@ -180,9 +302,6 @@ public void abortJob(String id) { /** * Update a given job's status - * - * @param jobId - * @param status */ void updateJobStatus(String jobId, JobStatus status) { Optional jobRecordOptional = jobInfoRepository.findById(jobId); @@ -195,9 +314,6 @@ void updateJobStatus(String jobId, JobStatus status) { /** * Update a given job's external id - * - * @param jobId - * @param jobExtId */ void updateJobExtId(String jobId, String jobExtId) { Optional jobRecordOptional = jobInfoRepository.findById(jobId); diff --git a/ingestion/src/main/java/feast/ingestion/service/SpecRetrievalException.java b/core/src/main/java/feast/core/util/PathUtil.java similarity index 52% rename from ingestion/src/main/java/feast/ingestion/service/SpecRetrievalException.java rename to core/src/main/java/feast/core/util/PathUtil.java index 33140325d11..53e74526f60 100644 --- a/ingestion/src/main/java/feast/ingestion/service/SpecRetrievalException.java +++ b/core/src/main/java/feast/core/util/PathUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,27 @@ * */ -package feast.ingestion.service; +package feast.core.util; -/** Application-specific exception for any failure of retrieving feature/entity/storage spec. */ -public class SpecRetrievalException extends RuntimeException { - public SpecRetrievalException() { - super(); - } +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; - public SpecRetrievalException(String message) { - super(message); - } +public class PathUtil { - public SpecRetrievalException(String message, Throwable cause) { - super(message, cause); + /** + * Gets a path with a schema if present + */ + public static Path getPath(String value) { + if (value.contains("://")) { + try { + return Paths.get(new URI(value)); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } else { + return Paths.get(value); + } } } diff --git a/core/src/main/java/feast/core/validators/SpecValidator.java b/core/src/main/java/feast/core/validators/SpecValidator.java index 18095b25914..7ec2f33d46b 100644 --- a/core/src/main/java/feast/core/validators/SpecValidator.java +++ b/core/src/main/java/feast/core/validators/SpecValidator.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.google.common.collect.Streams; import feast.core.dao.EntityInfoRepository; import feast.core.dao.FeatureGroupInfoRepository; import feast.core.dao.FeatureInfoRepository; @@ -45,21 +46,22 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Autowired; public class SpecValidator { - private static final String FILE_ERROR_STORE_TYPE = "file.json"; private static final String NO_STORE = ""; - private static String[] SUPPORTED_WAREHOUSE_STORES = + private static final String[] SUPPORTED_WAREHOUSE_STORES = new String[]{ - BigQueryStorageManager.TYPE, FILE_ERROR_STORE_TYPE, + BigQueryStorageManager.TYPE, "file.json", }; - private static String[] SUPPORTED_SERVING_STORES = + private static final String[] SUPPORTED_SERVING_STORES = new String[]{ BigTableStorageManager.TYPE, PostgresStorageManager.TYPE, RedisStorageManager.TYPE, }; + private static final String[] SUPPORTED_ERRORS_STORES = new String[]{"file.json", "stderr", + "stdout"}; + private StorageInfoRepository storageInfoRepository; private EntityInfoRepository entityInfoRepository; private FeatureGroupInfoRepository featureGroupInfoRepository; @@ -200,21 +202,63 @@ public void validateEntitySpec(EntitySpec spec) throws IllegalArgumentException } } - // TODO: add validation for storage types and options public void validateStorageSpec(StorageSpec spec) throws IllegalArgumentException { try { checkArgument(!spec.getId().equals(""), "Id field cannot be empty"); Matchers.checkUpperSnakeCase(spec.getId(), "Id"); - checkArgument( - Stream.concat( - Arrays.stream(SUPPORTED_SERVING_STORES), - Arrays.stream(SUPPORTED_WAREHOUSE_STORES)) - .collect(Collectors.toList()) - .contains(spec.getType())); + checkArgument(Streams.concat( + Arrays.stream(SUPPORTED_SERVING_STORES), + Arrays.stream(SUPPORTED_WAREHOUSE_STORES)).collect(Collectors.toList()) + .contains(spec.getType()), + "Store type not supported " + spec.getType()); + } catch (NullPointerException | IllegalArgumentException e) { + throw new IllegalArgumentException( + Strings.lenientFormat( + "Validation for storage spec with id %s failed: %s", spec.getId(), e.getMessage()), + e); + } + } + + // TODO: add validation for storage types and options + public void validateServingStorageSpec(StorageSpec spec) throws IllegalArgumentException { + try { + checkArgument(!spec.getId().equals(""), "Id field cannot be empty"); + Matchers.checkUpperSnakeCase(spec.getId(), "Id"); + checkArgument(Arrays.asList(SUPPORTED_SERVING_STORES).contains(spec.getType()), + "Serving store type not supported " + spec.getType()); + } catch (NullPointerException | IllegalArgumentException e) { + throw new IllegalArgumentException( + Strings.lenientFormat( + "Validation for storage spec with id %s failed: %s", spec.getId(), e.getMessage()), + e); + } + } + + public void validateWarehouseStorageSpec(StorageSpec spec) throws IllegalArgumentException { + try { + checkArgument(!spec.getId().equals(""), "Id field cannot be empty"); + Matchers.checkUpperSnakeCase(spec.getId(), "Id"); + checkArgument(Arrays.asList(SUPPORTED_WAREHOUSE_STORES).contains(spec.getType()), + "Warehouse store type not supported " + spec.getType()); + } catch (NullPointerException | IllegalArgumentException e) { + throw new IllegalArgumentException( + Strings.lenientFormat( + "Validation for storage spec with id %s failed: %s", spec.getId(), e.getMessage()), + e); + } + } + + public void validateErrorsStorageSpec(StorageSpec spec) throws IllegalArgumentException { + try { + checkArgument(!spec.getId().equals(""), "Id field cannot be empty"); + Matchers.checkUpperSnakeCase(spec.getId(), "Id"); + checkArgument(Arrays.asList(SUPPORTED_ERRORS_STORES).contains(spec.getType()), + "Errors store type not supported " + spec.getType()); } catch (NullPointerException | IllegalArgumentException e) { throw new IllegalArgumentException( Strings.lenientFormat( - "Validation for storage spec with id %s failed: %s", spec.getId(), e.getMessage())); + "Validation for storage spec with id %s failed: %s", spec.getId(), e.getMessage()), + e); } } diff --git a/core/src/main/resources/application.properties b/core/src/main/resources/application.properties index c69b9e35941..58d8af33b3f 100644 --- a/core/src/main/resources/application.properties +++ b/core/src/main/resources/application.properties @@ -19,12 +19,10 @@ grpc.port=${GRPC_PORT:6565} feast.core.projectId = ${PROJECT_ID:} feast.core.datasetPrefix = ${DATASET_PREFIX:fs} -feast.jobs.coreUri=${CORE_API_URI:localhost:6565} feast.jobs.runner=${JOB_RUNNER:DirectRunner} +feast.jobs.workspace=${JOB_WORKSPACE} feast.jobs.options=${JOB_OPTIONS:{}} feast.jobs.executable=${JOB_EXECUTABLE:feast-ingestion.jar} -feast.jobs.errorsStoreType=${JOB_ERRORS_STORE_TYPE:stdout} -feast.jobs.errorsStoreOptions=${JOB_ERRORS_STORE_OPTIONS:{}} feast.jobs.dataflow.projectId = ${DATAFLOW_PROJECT_ID:} feast.jobs.dataflow.location = ${DATAFLOW_LOCATION:} @@ -35,6 +33,14 @@ feast.jobs.flink.masterUrl = ${FLINK_MASTER_URL:localhost:8081} feast.jobs.monitor.period = ${JOB_MONITOR_PERIOD_MS:5000} feast.jobs.monitor.initialDelay = ${JOB_MONITOR_INITIAL_DELAY_MS:60000} +feast.store.serving.type = ${STORE_SERVING_TYPE:} +feast.store.serving.options = ${STORE_SERVING_OPTIONS:{}} +feast.store.warehouse.type = ${STORE_WAREHOUSE_TYPE:} +feast.store.warehouse.options = ${STORE_WAREHOUSE_OPTIONS:{}} +feast.store.errors.type = ${STORE_ERRORS_TYPE:} +feast.store.errors.options = ${STORE_ERRORS_OPTIONS:{}} + + spring.jpa.properties.hibernate.format_sql=true spring.datasource.url=jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/postgres spring.datasource.username=${DB_USERNAME:postgres} diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java new file mode 100644 index 00000000000..42e22b25660 --- /dev/null +++ b/core/src/test/java/feast/core/CoreApplicationTest.java @@ -0,0 +1,159 @@ +package feast.core; + +import static feast.core.config.StorageConfig.DEFAULT_ERRORS_ID; +import static feast.core.config.StorageConfig.DEFAULT_SERVING_ID; +import static feast.core.config.StorageConfig.DEFAULT_WAREHOUSE_ID; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; + +import com.google.protobuf.Timestamp; +import feast.core.JobServiceProto.JobServiceTypes.SubmitImportJobRequest; +import feast.core.JobServiceProto.JobServiceTypes.SubmitImportJobResponse; +import feast.core.config.ImportJobDefaults; +import feast.core.job.JobManager; +import feast.core.model.StorageInfo; +import feast.core.service.SpecService; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.DataStore; +import feast.specs.FeatureSpecProto.DataStores; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import feast.specs.ImportSpecProto.Field; +import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportSpecProto.Schema; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.types.GranularityProto.Granularity.Enum; +import feast.types.ValueProto.ValueType; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * Starts the application context with some properties + */ +@RunWith(SpringRunner.class) +@SpringBootTest(properties = { + "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "spring.datasource.url=jdbc:h2:mem:testdb", + "feast.store.warehouse.type=file.json", + "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", + "feast.store.serving.type=redis", + "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", + "feast.store.errors.type=stderr" +}) +@DirtiesContext +public class CoreApplicationTest { + + @Autowired + SpecService specService; + @Autowired + ImportJobDefaults jobDefaults; + @Autowired + JobManager jobManager; + + @Test + public void test_withProperties_systemServingAndWarehouseStoresRegistered() throws IOException { + Files.createDirectory(Paths.get(jobDefaults.getWorkspace())); + + List warehouseStorageInfo = specService + .getStorage(Collections.singletonList(DEFAULT_WAREHOUSE_ID)); + assertEquals(warehouseStorageInfo.size(), 1); + assertEquals(warehouseStorageInfo.get(0).getStorageSpec(), StorageSpec.newBuilder() + .setId(DEFAULT_WAREHOUSE_ID).setType("file.json").putOptions("path", "/tmp/foobar") + .build()); + + List servingStorageInfo = specService + .getStorage(Collections.singletonList(DEFAULT_SERVING_ID)); + assertEquals(servingStorageInfo.size(), 1); + assertEquals(servingStorageInfo.get(0).getStorageSpec(), StorageSpec.newBuilder() + .setId(DEFAULT_SERVING_ID).setType("redis") + .putOptions("host", "localhost") + .putOptions("port", "1234") + .build()); + + ManagedChannelBuilder channelBuilder = ManagedChannelBuilder.forAddress("localhost", 6565); + ManagedChannel channel = channelBuilder.usePlaintext(true).build(); + CoreServiceGrpc.CoreServiceBlockingStub coreService = CoreServiceGrpc.newBlockingStub(channel); + JobServiceGrpc.JobServiceBlockingStub jobService = JobServiceGrpc.newBlockingStub(channel); + + EntitySpec entitySpec = EntitySpec.newBuilder().setName("test").build(); + FeatureSpec featureSpec = FeatureSpec.newBuilder() + .setId("test.none.int64") + .setName("int64") + .setEntity("test") + .setGranularity(Enum.NONE) + .setValueType(ValueType.Enum.INT64) + .setOwner("hermione@example.com") + .setDescription("Test is a test") + .setUri("http://example.com/test.none.int64").build(); + ImportSpec importSpec = ImportSpec.newBuilder() + .setSchema(Schema.newBuilder() + .setEntityIdColumn("id") + .setTimestampValue(Timestamp.getDefaultInstance()) + .addFields(Field.newBuilder().setName("id")) + .addFields(Field.newBuilder().setName("a").setFeatureId("test.none.int64"))) + .addEntities("test") + .setType("file.csv") + .putSourceOptions("path", "/tmp/foobar").build(); + + coreService.applyEntity(entitySpec); + coreService.applyFeature(featureSpec); + SubmitImportJobRequest jobSubmitReq = SubmitImportJobRequest.newBuilder() + .setImportSpec(importSpec).build(); + + Map args = new HashMap<>(); + Mockito.when(jobManager.submitJob(any(), any())).thenAnswer((Answer) invocation -> { + args.put(0, invocation.getArgument(0)); + args.put(1, invocation.getArgument(1)); + return "externalJobId1234"; + }); + SubmitImportJobResponse jobSubmitRes = jobService.submitJob(jobSubmitReq); + String jobId = jobSubmitRes.getJobId(); + assertEquals(args.get(1), Paths.get(jobDefaults.getWorkspace()).resolve(jobId)); + assertEquals(ImportJobSpecs.newBuilder() + .setJobId(jobId) + .setImportSpec(importSpec) + .setErrorsStorageSpec(StorageSpec.newBuilder() + .setId(DEFAULT_ERRORS_ID) + .setType("stderr")) + .addEntitySpecs(entitySpec) + .addFeatureSpecs(featureSpec.toBuilder() + .setDataStores(DataStores.newBuilder() + .setWarehouse(DataStore.newBuilder().setId(DEFAULT_WAREHOUSE_ID)) + .setServing(DataStore.newBuilder().setId(DEFAULT_SERVING_ID)))) + .addServingStorageSpecs(StorageSpec.newBuilder() + .setId(DEFAULT_SERVING_ID) + .setType("redis") + .putOptions("host", "localhost").putOptions("port", "1234")) + .addWarehouseStorageSpecs(StorageSpec.newBuilder() + .setId(DEFAULT_WAREHOUSE_ID) + .setType("file.json") + .putOptions("path", "/tmp/foobar")) + .build(), args.get(0)); + } + + @TestConfiguration + public static class MockProvider { + + @Bean + public JobManager jobManager() { + return Mockito.mock(JobManager.class); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java new file mode 100644 index 00000000000..93298c76045 --- /dev/null +++ b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java @@ -0,0 +1,121 @@ +package feast.core; + +import static feast.core.config.StorageConfig.DEFAULT_ERRORS_ID; +import static feast.core.config.StorageConfig.DEFAULT_SERVING_ID; +import static feast.core.config.StorageConfig.DEFAULT_WAREHOUSE_ID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; + +import com.google.protobuf.Timestamp; +import feast.core.JobServiceProto.JobServiceTypes.SubmitImportJobRequest; +import feast.core.JobServiceProto.JobServiceTypes.SubmitImportJobResponse; +import feast.core.config.ImportJobDefaults; +import feast.core.job.JobManager; +import feast.core.model.StorageInfo; +import feast.core.service.SpecService; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.DataStore; +import feast.specs.FeatureSpecProto.DataStores; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import feast.specs.ImportSpecProto.Field; +import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportSpecProto.Schema; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.types.GranularityProto.Granularity.Enum; +import feast.types.ValueProto.ValueType; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * Starts the application context with some properties + */ +@RunWith(SpringRunner.class) +@SpringBootTest(properties = { + "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "spring.datasource.url=jdbc:h2:mem:testdb", + "feast.store.warehouse.type=file.json", + "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", + "feast.store.errors.type=stderr" +}) +@DirtiesContext +public class CoreApplicationWithNoServingTest { + + @Autowired + SpecService specService; + @Autowired + ImportJobDefaults jobDefaults; + @Autowired + JobManager jobManager; + + @Test + public void test_withProperties_systemServingAndWarehouseStoresRegistered() throws IOException { + Files.createDirectory(Paths.get(jobDefaults.getWorkspace())); + + List warehouseStorageInfo = specService + .getStorage(Collections.singletonList(DEFAULT_WAREHOUSE_ID)); + assertEquals(warehouseStorageInfo.size(), 1); + assertEquals(warehouseStorageInfo.get(0).getStorageSpec(), StorageSpec.newBuilder() + .setId(DEFAULT_WAREHOUSE_ID).setType("file.json").putOptions("path", "/tmp/foobar") + .build()); + + ManagedChannelBuilder channelBuilder = ManagedChannelBuilder.forAddress("localhost", 6565); + ManagedChannel channel = channelBuilder.usePlaintext(true).build(); + CoreServiceGrpc.CoreServiceBlockingStub coreService = CoreServiceGrpc.newBlockingStub(channel); + JobServiceGrpc.JobServiceBlockingStub jobService = JobServiceGrpc.newBlockingStub(channel); + + EntitySpec entitySpec = EntitySpec.newBuilder().setName("test").build(); + FeatureSpec featureSpec = FeatureSpec.newBuilder() + .setId("test.none.int64") + .setName("int64") + .setEntity("test") + .setGranularity(Enum.NONE) + .setValueType(ValueType.Enum.INT64) + .setOwner("hermione@example.com") + .setDescription("Test is a test") + .setUri("http://example.com/test.none.int64").build(); + ImportSpec importSpec = ImportSpec.newBuilder() + .setSchema(Schema.newBuilder() + .setEntityIdColumn("id") + .setTimestampValue(Timestamp.getDefaultInstance()) + .addFields(Field.newBuilder().setName("id")) + .addFields(Field.newBuilder().setName("a").setFeatureId("test.none.int64"))) + .addEntities("test") + .setType("file.csv") + .putSourceOptions("path", "/tmp/foobar").build(); + + coreService.applyEntity(entitySpec); + try { + coreService.applyFeature(featureSpec); + fail("should fail validation as there is not serving store"); + } catch (Exception e) { + } + } + + @TestConfiguration + public static class MockProvider { + + @Bean + public JobManager jobManager() { + return Mockito.mock(JobManager.class); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java new file mode 100644 index 00000000000..5b4c0801e91 --- /dev/null +++ b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java @@ -0,0 +1,152 @@ +package feast.core; + +import static feast.core.config.StorageConfig.DEFAULT_ERRORS_ID; +import static feast.core.config.StorageConfig.DEFAULT_SERVING_ID; +import static feast.core.config.StorageConfig.DEFAULT_WAREHOUSE_ID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; + +import com.google.protobuf.Timestamp; +import feast.core.JobServiceProto.JobServiceTypes.SubmitImportJobRequest; +import feast.core.JobServiceProto.JobServiceTypes.SubmitImportJobResponse; +import feast.core.config.ImportJobDefaults; +import feast.core.job.JobManager; +import feast.core.model.StorageInfo; +import feast.core.service.SpecService; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.DataStore; +import feast.specs.FeatureSpecProto.DataStores; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import feast.specs.ImportSpecProto.Field; +import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportSpecProto.Schema; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.types.GranularityProto.Granularity.Enum; +import feast.types.ValueProto.ValueType; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * Starts the application context with some properties + */ +@RunWith(SpringRunner.class) +@SpringBootTest(properties = { + "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "spring.datasource.url=jdbc:h2:mem:testdb", + "feast.store.serving.type=redis", + "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", + "feast.store.errors.type=stderr" +}) +@DirtiesContext +public class CoreApplicationWithNoWarehouseTest { + + @Autowired + SpecService specService; + @Autowired + ImportJobDefaults jobDefaults; + @Autowired + JobManager jobManager; + + @Test + public void test_withProperties_systemServingAndWarehouseStoresRegistered() throws IOException { + Files.createDirectory(Paths.get(jobDefaults.getWorkspace())); + + try { + specService.getStorage(Collections.singletonList(DEFAULT_WAREHOUSE_ID)); + fail("should have thrown exception"); + } catch (feast.core.exception.RetrievalException e) { + } + + List servingStorageInfo = specService + .getStorage(Collections.singletonList(DEFAULT_SERVING_ID)); + assertEquals(servingStorageInfo.size(), 1); + assertEquals(servingStorageInfo.get(0).getStorageSpec(), StorageSpec.newBuilder() + .setId(DEFAULT_SERVING_ID).setType("redis") + .putOptions("host", "localhost") + .putOptions("port", "1234") + .build()); + + ManagedChannelBuilder channelBuilder = ManagedChannelBuilder.forAddress("localhost", 6565); + ManagedChannel channel = channelBuilder.usePlaintext(true).build(); + CoreServiceGrpc.CoreServiceBlockingStub coreService = CoreServiceGrpc.newBlockingStub(channel); + JobServiceGrpc.JobServiceBlockingStub jobService = JobServiceGrpc.newBlockingStub(channel); + + EntitySpec entitySpec = EntitySpec.newBuilder().setName("test").build(); + FeatureSpec featureSpec = FeatureSpec.newBuilder() + .setId("test.none.int64") + .setName("int64") + .setEntity("test") + .setGranularity(Enum.NONE) + .setValueType(ValueType.Enum.INT64) + .setOwner("hermione@example.com") + .setDescription("Test is a test") + .setUri("http://example.com/test.none.int64").build(); + ImportSpec importSpec = ImportSpec.newBuilder() + .setSchema(Schema.newBuilder() + .setEntityIdColumn("id") + .setTimestampValue(Timestamp.getDefaultInstance()) + .addFields(Field.newBuilder().setName("id")) + .addFields(Field.newBuilder().setName("a").setFeatureId("test.none.int64"))) + .addEntities("test") + .setType("file.csv") + .putSourceOptions("path", "/tmp/foobar").build(); + + coreService.applyEntity(entitySpec); + coreService.applyFeature(featureSpec); + SubmitImportJobRequest jobSubmitReq = SubmitImportJobRequest.newBuilder() + .setImportSpec(importSpec).build(); + + Map args = new HashMap<>(); + Mockito.when(jobManager.submitJob(any(), any())).thenAnswer((Answer) invocation -> { + args.put(0, invocation.getArgument(0)); + args.put(1, invocation.getArgument(1)); + return "externalJobId1234"; + }); + SubmitImportJobResponse jobSubmitRes = jobService.submitJob(jobSubmitReq); + String jobId = jobSubmitRes.getJobId(); + assertEquals(args.get(1), Paths.get(jobDefaults.getWorkspace()).resolve(jobId)); + assertEquals(ImportJobSpecs.newBuilder() + .setJobId(jobId) + .setImportSpec(importSpec) + .setErrorsStorageSpec(StorageSpec.newBuilder() + .setId(DEFAULT_ERRORS_ID) + .setType("stderr")) + .addEntitySpecs(entitySpec) + .addFeatureSpecs(featureSpec.toBuilder() + .setDataStores(DataStores.newBuilder() + .setServing(DataStore.newBuilder().setId(DEFAULT_SERVING_ID)))) + .addServingStorageSpecs(StorageSpec.newBuilder() + .setId(DEFAULT_SERVING_ID) + .setType("redis") + .putOptions("host", "localhost").putOptions("port", "1234")) + .build(), args.get(0)); + } + + @TestConfiguration + public static class MockProvider { + + @Bean + public JobManager jobManager() { + return Mockito.mock(JobManager.class); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/feast/core/config/StorageConfigTest.java b/core/src/test/java/feast/core/config/StorageConfigTest.java new file mode 100644 index 00000000000..a8be939117c --- /dev/null +++ b/core/src/test/java/feast/core/config/StorageConfigTest.java @@ -0,0 +1,71 @@ +package feast.core.config; + +import static feast.core.config.StorageConfig.DEFAULT_ERRORS_ID; +import static feast.core.config.StorageConfig.DEFAULT_SERVING_ID; +import static feast.core.config.StorageConfig.DEFAULT_WAREHOUSE_ID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.verify; + +import feast.core.config.StorageConfig.StorageSpecs; +import feast.core.validators.SpecValidator; +import feast.specs.StorageSpecProto.StorageSpec; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class StorageConfigTest { + + public SpecValidator validator = Mockito.mock(SpecValidator.class); + + StorageConfig config; + + @Before + public void before() { + config = new StorageConfig(validator); + } + + @Test + public void testBuildErrorsStorageSpec() { + StorageSpecs storageSpecs = config.getStorageSpecs("", "", "", + "", "file.json", "{\"path\": \"/tmp/errors\"}"); + StorageSpec storageSpec = StorageSpec.newBuilder() + .setId(DEFAULT_ERRORS_ID).setType("file.json") + .putOptions("path", "/tmp/errors").build(); + assertEquals(storageSpec, storageSpecs.getErrorsStorageSpec()); + assertNull(storageSpecs.getServingStorageSpec()); + assertNull(storageSpecs.getWarehouseStorageSpec()); + verify(validator).validateErrorsStorageSpec(storageSpec); + } + + @Test + public void testBuildServingStorageSpec() { + StorageSpecs storageSpecs = config.getStorageSpecs( + "redis", "{\"host\": \"localhost\", \"port\": \"1234\"}", "", + "", "", ""); + StorageSpec storageSpec = StorageSpec.newBuilder() + .setId(DEFAULT_SERVING_ID).setType("redis") + .putOptions("host", "localhost") + .putOptions("port", "1234").build(); + assertEquals(storageSpec, storageSpecs.getServingStorageSpec()); + assertNull(storageSpecs.getErrorsStorageSpec()); + assertNull(storageSpecs.getWarehouseStorageSpec()); + verify(validator).validateServingStorageSpec(storageSpec); + } + + @Test + public void testBuildWarehouseStorageSpec() { + StorageSpecs storageSpecs = config.getStorageSpecs( + "", "", + "bigquery", "{\"project\": \"project1\", \"dataset\": \"feast\"}", + "", ""); + StorageSpec storageSpec = StorageSpec.newBuilder() + .setId(DEFAULT_WAREHOUSE_ID).setType("bigquery") + .putOptions("project", "project1") + .putOptions("dataset", "feast").build(); + assertEquals(storageSpec, storageSpecs.getWarehouseStorageSpec()); + assertNull(storageSpecs.getErrorsStorageSpec()); + assertNull(storageSpecs.getServingStorageSpec()); + verify(validator).validateWarehouseStorageSpec(storageSpec); + } +} diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java index 678b76c1048..508492ef58a 100644 --- a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java +++ b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java @@ -25,57 +25,88 @@ import com.google.api.services.dataflow.Dataflow; import com.google.common.collect.Lists; import feast.core.config.ImportJobDefaults; +import feast.core.util.PathUtil; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.specs.ImportSpecProto.ImportSpec; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import org.mockito.Mock; import org.mockito.Mockito; public class DataflowJobManagerTest { - @Rule public final ExpectedException expectedException = ExpectedException.none(); + @Rule + public final ExpectedException expectedException = ExpectedException.none(); - @Mock Dataflow dataflow; + @Mock + Dataflow dataflow; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); private ImportJobDefaults defaults; private DataflowJobManager dfJobManager; + private Path workspace; @Before - public void setUp() { + public void setUp() throws IOException { initMocks(this); + workspace = Paths.get(tempFolder.newFolder().toString()); defaults = - new ImportJobDefaults( - "localhost:8080", - "DataflowRunner", - "{\"key\":\"value\"}", - "ingestion.jar", - "STDOUT", - "{}"); + ImportJobDefaults.builder() + .runner("DataflowRunner") + .importJobOptions("{\"key\":\"value\"}") + .executable("ingestion.jar") + .workspace(workspace.toString()).build(); dfJobManager = new DataflowJobManager(dataflow, "project", "location", defaults); } @Test public void shouldBuildProcessBuilderWithCorrectOptions() { ImportSpec importSpec = ImportSpec.newBuilder().setType("file").build(); - ProcessBuilder pb = dfJobManager.getProcessBuilder(importSpec, "test"); + String jobName = "test"; + ImportJobSpecs importJobSpecs = ImportJobSpecs.newBuilder().setJobId(jobName) + .setImportSpec(importSpec).build(); + + ProcessBuilder pb = dfJobManager.getProcessBuilder(importJobSpecs, Paths.get("/tmp/foobar")); + List expected = + Lists.newArrayList( + "java", + "-jar", + "ingestion.jar", + "--jobName=test", + "--workspace=file:///tmp/foobar", + "--runner=DataflowRunner", + "--key=value"); + assertThat(pb.command(), equalTo(expected)); + } + + @Test + public void shouldBuildProcessBuilderWithGCSWorkspace() { + ImportSpec importSpec = ImportSpec.newBuilder().setType("file").build(); + String jobName = "test"; + ImportJobSpecs importJobSpecs = ImportJobSpecs.newBuilder().setJobId(jobName) + .setImportSpec(importSpec).build(); + + ProcessBuilder pb = dfJobManager.getProcessBuilder(importJobSpecs, PathUtil.getPath("gs://bucket/tmp/foobar")); List expected = Lists.newArrayList( "java", "-jar", "ingestion.jar", "--jobName=test", + "--workspace=gs://bucket/tmp/foobar", "--runner=DataflowRunner", - "--importSpecBase64=CgRmaWxl", - "--coreApiUri=localhost:8080", - "--errorsStoreType=STDOUT", - "--errorsStoreOptions={}", "--key=value"); assertThat(pb.command(), equalTo(expected)); } diff --git a/core/src/test/java/feast/core/job/flink/FlinkJobManagerTest.java b/core/src/test/java/feast/core/job/flink/FlinkJobManagerTest.java index cd892696fb9..35faaf2782d 100644 --- a/core/src/test/java/feast/core/job/flink/FlinkJobManagerTest.java +++ b/core/src/test/java/feast/core/job/flink/FlinkJobManagerTest.java @@ -6,62 +6,71 @@ import static org.mockito.Mockito.when; import feast.core.config.ImportJobDefaults; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.specs.ImportSpecProto.ImportSpec; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import org.apache.flink.client.cli.CliFrontend; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; public class FlinkJobManagerTest { - @Mock private CliFrontend flinkCli; - @Mock private FlinkRestApi flinkRestApi; + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + @Mock + private CliFrontend flinkCli; + @Mock + private FlinkRestApi flinkRestApi; private FlinkJobConfig config; private ImportJobDefaults defaults; private FlinkJobManager flinkJobManager; + private Path workspace; @Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); config = new FlinkJobConfig("localhost:8081", "/etc/flink/conf"); + workspace = Paths.get(tempFolder.newFolder().toString()); defaults = - new ImportJobDefaults( - "localhost:8080", - "FlinkRunner", - "{\"key\":\"value\"}", - "ingestion.jar", - "stderr", - "{}"); + ImportJobDefaults.builder(). + runner("FlinkRunner").importJobOptions("{\"key\":\"value\"}") + .executable("ingestion.jar") + .workspace(workspace.toString()).build(); flinkJobManager = new FlinkJobManager(flinkCli, config, flinkRestApi, defaults); } @Test - public void shouldPassCorrectArgumentForSubmittingJob() { + public void shouldPassCorrectArgumentForSubmittingJob() throws IOException { FlinkJobList response = new FlinkJobList(); response.setJobs(Collections.singletonList(new FlinkJob("1234", "job1", "RUNNING"))); when(flinkRestApi.getJobsOverview()).thenReturn(response); ImportSpec importSpec = ImportSpec.newBuilder().setType("file").build(); String jobName = "importjob"; - flinkJobManager.submitJob(importSpec, jobName); + ImportJobSpecs importJobSpecs = ImportJobSpecs.newBuilder().setJobId(jobName) + .setImportSpec(importSpec).build(); + + flinkJobManager.submitJob(importJobSpecs, Paths.get("/tmp/foobar")); String[] expected = - new String[] { - "run", - "-d", - "-m", - config.getMasterUrl(), - defaults.getExecutable(), - "--jobName=" + jobName, - "--runner=FlinkRunner", - "--importSpecBase64=CgRmaWxl", - "--coreApiUri=" + defaults.getCoreApiUri(), - "--errorsStoreType=" + defaults.getErrorsStoreType(), - "--errorsStoreOptions=" + defaults.getErrorsStoreOptions(), - "--key=value" + new String[]{ + "run", + "-d", + "-m", + config.getMasterUrl(), + defaults.getExecutable(), + "--jobName=" + jobName, + "--runner=FlinkRunner", + "--workspace=/tmp/foobar", + "--key=value" }; ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(String[].class); @@ -80,7 +89,9 @@ public void shouldReturnFlinkJobId() { when(flinkRestApi.getJobsOverview()).thenReturn(response); ImportSpec importSpec = ImportSpec.newBuilder().setType("file").build(); - String jobId = flinkJobManager.submitJob(importSpec, jobName); + ImportJobSpecs importJobSpecs = ImportJobSpecs.newBuilder().setJobId(jobName) + .setImportSpec(importSpec).build(); + String jobId = flinkJobManager.submitJob(importJobSpecs, workspace); assertThat(jobId, equalTo(flinkJobId)); } @@ -92,10 +103,10 @@ public void shouldPassCorrectArgumentForStoppingJob() { flinkJobManager.abortJob(jobId); String[] expected = new String[]{ - "cancel", - "-m", - config.getMasterUrl(), - jobId + "cancel", + "-m", + config.getMasterUrl(), + jobId }; ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(String[].class); diff --git a/core/src/test/java/feast/core/service/JobManagementServiceTest.java b/core/src/test/java/feast/core/service/JobManagementServiceTest.java index 1902ccc578d..71967f8da88 100644 --- a/core/src/test/java/feast/core/service/JobManagementServiceTest.java +++ b/core/src/test/java/feast/core/service/JobManagementServiceTest.java @@ -28,12 +28,14 @@ import com.google.protobuf.Timestamp; import feast.core.JobServiceProto.JobServiceTypes.JobDetail; import feast.core.config.ImportJobDefaults; +import feast.core.config.StorageConfig.StorageSpecs; import feast.core.dao.JobInfoRepository; import feast.core.dao.MetricsRepository; import feast.core.exception.RetrievalException; import feast.core.job.JobManager; import feast.core.model.JobInfo; import feast.core.model.JobStatus; +import feast.specs.StorageSpecProto.StorageSpec; import java.time.Instant; import java.util.Collections; import java.util.Date; @@ -47,18 +49,28 @@ import org.mockito.Mock; public class JobManagementServiceTest { - @Rule public final ExpectedException exception = ExpectedException.none(); - @Mock private JobInfoRepository jobInfoRepository; - @Mock private MetricsRepository metricsRepository; - @Mock private JobManager jobManager; + + @Rule + public final ExpectedException exception = ExpectedException.none(); + @Mock + private JobInfoRepository jobInfoRepository; + @Mock + private MetricsRepository metricsRepository; + @Mock + private JobManager jobManager; private ImportJobDefaults defaults; + @Mock + private SpecService specService; + private StorageSpecs storageSpecs; @Before public void setUp() { initMocks(this); defaults = - new ImportJobDefaults( - "localhost:8433", "DirectRunner", "", "/feast-import.jar", "stderr", ""); + ImportJobDefaults.builder() + .runner("DirectRunner").executable("/feast-import.jar").build(); + storageSpecs = StorageSpecs.builder() + .errorsStorageSpec(StorageSpec.newBuilder().setType("stderr").build()).build(); } @Test @@ -95,7 +107,8 @@ public void shouldListAllJobDetails() { jobInfo2.setLastUpdated(Date.from(Instant.ofEpochSecond(1))); when(jobInfoRepository.findAll()).thenReturn(Lists.newArrayList(jobInfo1, jobInfo2)); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, + specService, storageSpecs); List actual = jobManagementService.listJobs(); List expected = Lists.newArrayList( @@ -133,7 +146,8 @@ public void shouldReturnDetailOfRequestedJobId() { jobInfo1.setLastUpdated(Date.from(Instant.ofEpochSecond(1))); when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(jobInfo1)); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, + specService, storageSpecs); JobDetail actual = jobManagementService.getJob("job1"); JobDetail expected = JobDetail.newBuilder() @@ -149,7 +163,7 @@ public void shouldReturnDetailOfRequestedJobId() { public void shouldThrowErrorIfJobIdNotFoundWhenGettingJob() { when(jobInfoRepository.findById("job1")).thenReturn(Optional.empty()); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); exception.expect(RetrievalException.class); exception.expectMessage("Unable to retrieve job with id job1"); jobManagementService.getJob("job1"); @@ -159,7 +173,7 @@ public void shouldThrowErrorIfJobIdNotFoundWhenGettingJob() { public void shouldThrowErrorIfJobIdNotFoundWhenAbortingJob() { when(jobInfoRepository.findById("job1")).thenReturn(Optional.empty()); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); exception.expect(RetrievalException.class); exception.expectMessage("Unable to retrieve job with id job1"); jobManagementService.abortJob("job1"); @@ -171,7 +185,7 @@ public void shouldThrowErrorIfJobInTerminalStateWhenAbortingJob() { job.setStatus(JobStatus.COMPLETED); when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(job)); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); exception.expect(IllegalStateException.class); exception.expectMessage("Unable to stop job already in terminal state"); jobManagementService.abortJob("job1"); @@ -184,7 +198,7 @@ public void shouldUpdateJobAfterAborting() { job.setExtId("extId1"); when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(job)); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); jobManagementService.abortJob("job1"); ArgumentCaptor jobCapture = ArgumentCaptor.forClass(JobInfo.class); verify(jobInfoRepository).saveAndFlush(jobCapture.capture()); @@ -198,7 +212,7 @@ public void shouldUpdateJobStatusIfExists() { ArgumentCaptor jobInfoArgumentCaptor = ArgumentCaptor.forClass(JobInfo.class); JobManagementService jobExecutionService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); jobExecutionService.updateJobStatus("jobid", JobStatus.PENDING); verify(jobInfoRepository, times(1)).save(jobInfoArgumentCaptor.capture()); @@ -215,7 +229,7 @@ public void shouldUpdateJobExtIdIfExists() { ArgumentCaptor jobInfoArgumentCaptor = ArgumentCaptor.forClass(JobInfo.class); JobManagementService jobExecutionService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); jobExecutionService.updateJobExtId("jobid", "extid"); verify(jobInfoRepository, times(1)).save(jobInfoArgumentCaptor.capture()); diff --git a/core/src/test/java/feast/core/util/PathUtilTest.java b/core/src/test/java/feast/core/util/PathUtilTest.java new file mode 100644 index 00000000000..1a8f6196903 --- /dev/null +++ b/core/src/test/java/feast/core/util/PathUtilTest.java @@ -0,0 +1,19 @@ +package feast.core.util; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class PathUtilTest { + + @Test + public void testGetPath() { + assertEquals("file:///tmp/foo/bar", PathUtil.getPath("/tmp/foo/bar").toUri().toString()); + } + + @Test + public void testGetPath_withGcs() { + + assertEquals("gs://tmp/foo/bar", PathUtil.getPath("gs://tmp/foo/bar").toUri().toString()); + } +} \ No newline at end of file diff --git a/core/src/test/java/feast/core/validators/SpecValidatorTest.java b/core/src/test/java/feast/core/validators/SpecValidatorTest.java index 052fa609508..9bb2cf461e5 100644 --- a/core/src/test/java/feast/core/validators/SpecValidatorTest.java +++ b/core/src/test/java/feast/core/validators/SpecValidatorTest.java @@ -17,6 +17,9 @@ package feast.core.validators; +import static feast.core.config.StorageConfig.DEFAULT_ERRORS_ID; +import static feast.core.config.StorageConfig.DEFAULT_SERVING_ID; +import static feast.core.config.StorageConfig.DEFAULT_WAREHOUSE_ID; import static org.mockito.Mockito.when; import com.google.common.base.Strings; @@ -624,6 +627,87 @@ public void entitySpecWithInvalidNameShouldThrowIllegalArgumentException() { validator.validateEntitySpec(input); } + @Test + public void testServingStorageSpec_withValidTypes() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + validator.validateServingStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_SERVING_ID) + .setType("redis").build()); + validator.validateServingStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_SERVING_ID) + .setType("bigtable").build()); + } + + @Test(expected = IllegalArgumentException.class) + public void testServingStorageSpec_withInvalidType() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + validator.validateServingStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_SERVING_ID) + .setType("invalid").build()); + } + + @Test + public void testWarehouseStorageSpec_withValidTypes() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + validator.validateWarehouseStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_WAREHOUSE_ID) + .setType("file.json").build()); + validator.validateWarehouseStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_WAREHOUSE_ID) + .setType("bigquery").build()); + } + + @Test(expected = IllegalArgumentException.class) + public void testWarehouseStorageSpec_withInvalidType() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + validator.validateWarehouseStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_WAREHOUSE_ID) + .setType("invalid").build()); + } + + @Test + public void testErrorsStorageSpec_withValidTypes() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + validator.validateErrorsStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_ERRORS_ID) + .setType("file.json").build()); + validator.validateErrorsStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_ERRORS_ID) + .setType("stderr").build()); + validator.validateErrorsStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_ERRORS_ID) + .setType("stderr").build()); + } + + @Test(expected = IllegalArgumentException.class) + public void testErrorsStorageSpec_withInvalidType() { + SpecValidator validator = + new SpecValidator( + storageInfoRepository, + entityInfoRepository, + featureGroupInfoRepository, + featureInfoRepository); + validator.validateErrorsStorageSpec(StorageSpec.newBuilder().setId(DEFAULT_ERRORS_ID) + .setType("invalid").build()); + } + + @Test public void storageSpecWithoutIdShouldThrowIllegalArgumentException() { SpecValidator validator = diff --git a/core/src/test/resources/application.properties b/core/src/test/resources/application.properties new file mode 100644 index 00000000000..33c45dd0c5f --- /dev/null +++ b/core/src/test/resources/application.properties @@ -0,0 +1,57 @@ +# +# Copyright 2018 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +grpc.port=${GRPC_PORT:6565} + +feast.core.projectId = ${PROJECT_ID:} +feast.core.datasetPrefix = ${DATASET_PREFIX:fs} + +feast.jobs.workspace=${JOB_WORKSPACE} +feast.jobs.runner=${JOB_RUNNER:DirectRunner} +feast.jobs.options=${JOB_OPTIONS:{}} +feast.jobs.executable=${JOB_EXECUTABLE:feast-ingestion.jar} + +feast.jobs.dataflow.projectId = ${DATAFLOW_PROJECT_ID:} +feast.jobs.dataflow.location = ${DATAFLOW_LOCATION:} + +feast.jobs.flink.configDir = ${FLINK_CONF_DIR:/etc/flink/flink-1.5.5/conf} +feast.jobs.flink.masterUrl = ${FLINK_MASTER_URL:localhost:8081} + +feast.jobs.monitor.period = ${JOB_MONITOR_PERIOD_MS:5000} +feast.jobs.monitor.initialDelay = ${JOB_MONITOR_INITIAL_DELAY_MS:60000} + +feast.store.serving.type = ${STORE_SERVING_TYPE:} +feast.store.serving.options = ${STORE_SERVING_OPTIONS:{}} +feast.store.warehouse.type = ${STORE_WAREHOUSE_TYPE:} +feast.store.warehouse.options = ${STORE_WAREHOUSE_OPTIONS:{}} +feast.store.errors.type = ${STORE_ERRORS_TYPE:} +feast.store.errors.options = ${STORE_ERRORS_OPTIONS:{}} + + +spring.jpa.properties.hibernate.format_sql=true +spring.datasource.url=jdbc:h2:mem:testdb +spring.datasource.username=${DB_USERNAME:postgres} +spring.datasource.password=${DB_PASSWORD:password} +spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl +spring.jpa.hibernate.ddl-auto=update + +statsd.host= ${STATSD_HOST:localhost} +statsd.port= ${STATSD_PORT:8125} + +management.metrics.export.simple.enabled=false +management.metrics.export.statsd.enabled=true +management.metrics.export.statsd.host=${STATSD_HOST:localhost} +management.metrics.export.statsd.port=${STATSD_PORT:8125} diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 3abce057dd1..faf60b5a955 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -96,7 +96,7 @@ com.google.cloud google-cloud-nio - 0.64.0-alpha + 0.83.0-alpha diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index 32420698253..e05362c15c7 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -25,7 +25,7 @@ import com.google.protobuf.util.JsonFormat; import feast.ingestion.boot.ImportJobModule; import feast.ingestion.boot.PipelineModule; -import feast.ingestion.config.ImportSpecSupplier; +import feast.ingestion.config.ImportJobSpecsSupplier; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.options.JobOptions; @@ -41,7 +41,7 @@ import feast.ingestion.transform.fn.RoundEventTimestampsDoFn; import feast.ingestion.values.PFeatureRows; import feast.options.OptionsParser; -import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import java.util.Arrays; @@ -75,7 +75,7 @@ public class ImportJob { private static Random random = new Random(System.currentTimeMillis()); private final Pipeline pipeline; - private final ImportSpec importSpec; + private final ImportJobSpecs importJobSpecs; private final ReadFeaturesTransform readFeaturesTransform; private final ServingStoreTransform servingStoreTransform; private final WarehouseStoreTransform warehouseStoreTransform; @@ -87,7 +87,7 @@ public class ImportJob { @Inject public ImportJob( Pipeline pipeline, - ImportSpec importSpec, + ImportJobSpecs importJobSpecs, ReadFeaturesTransform readFeaturesTransform, ServingStoreTransform servingStoreTransform, WarehouseStoreTransform warehouseStoreTransform, @@ -95,7 +95,7 @@ public ImportJob( ImportJobPipelineOptions options, Specs specs) { this.pipeline = pipeline; - this.importSpec = importSpec; + this.importJobSpecs = importJobSpecs; this.readFeaturesTransform = readFeaturesTransform; this.servingStoreTransform = servingStoreTransform; this.warehouseStoreTransform = warehouseStoreTransform; @@ -117,9 +117,10 @@ public static PipelineResult mainWithResult(String[] args) { options.setJobName(generateName()); } log.info("options: " + options.toString()); - ImportSpec importSpec = new ImportSpecSupplier(options).get(); + ImportJobSpecs importJobSpecs = new ImportJobSpecsSupplier(options.getWorkspace()) + .get(); Injector injector = - Guice.createInjector(new ImportJobModule(options, importSpec), new PipelineModule()); + Guice.createInjector(new ImportJobModule(options, importJobSpecs), new PipelineModule()); ImportJob job = injector.getInstance(ImportJob.class); job.expand(); @@ -141,10 +142,11 @@ public void expand() { TypeDescriptor.of(FeatureRowExtended.class), ProtoCoder.of(FeatureRowExtended.class)); coderRegistry.registerCoderForType(TypeDescriptor.of(TableRow.class), TableRowJsonCoder.of()); - JobOptions jobOptions = OptionsParser.parse(importSpec.getJobOptionsMap(), JobOptions.class); + JobOptions jobOptions = OptionsParser + .parse(importJobSpecs.getImportSpec().getJobOptionsMap(), JobOptions.class); try { - log.info(JsonFormat.printer().print(importSpec)); + log.info(JsonFormat.printer().print(importJobSpecs)); } catch (InvalidProtocolBufferException e) { // pass } @@ -179,7 +181,7 @@ public void expand() { if (!dryRun) { servingRows.apply("Write to Serving Stores", servingStoreTransform); warehouseRows.apply("Write to Warehouse Stores", warehouseStoreTransform); - pFeatureRows.getErrors().apply("Write errors", errorsStoreTransform); + pFeatureRows.getErrors().apply(errorsStoreTransform); } } diff --git a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java index 81dadaf38e1..0d4b96519aa 100644 --- a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java +++ b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java @@ -22,12 +22,7 @@ import com.google.inject.Singleton; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.ingestion.service.CoreSpecService; -import feast.ingestion.service.FileSpecService; -import feast.ingestion.service.SpecService; -import feast.ingestion.service.SpecService.Builder; -import feast.ingestion.service.SpecService.UnsupportedBuilder; -import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.store.errors.FeatureErrorsFactory; import feast.store.errors.FeatureErrorsFactoryService; import feast.store.serving.FeatureServingFactory; @@ -43,37 +38,24 @@ public class ImportJobModule extends AbstractModule { private final ImportJobPipelineOptions options; - private ImportSpec importSpec; + private ImportJobSpecs importJobSpecs; - public ImportJobModule(ImportJobPipelineOptions options, ImportSpec importSpec) { + public ImportJobModule(ImportJobPipelineOptions options, ImportJobSpecs importJobSpecs) { this.options = options; - this.importSpec = importSpec; + this.importJobSpecs = importJobSpecs; } @Override protected void configure() { bind(ImportJobPipelineOptions.class).toInstance(options); bind(PipelineOptions.class).toInstance(options); - bind(ImportSpec.class).toInstance(importSpec); + bind(ImportJobSpecs.class).toInstance(importJobSpecs); } @Provides @Singleton - Builder provideSpecService(ImportJobPipelineOptions options) { - if (options.getCoreApiUri() != null) { - return new CoreSpecService.Builder(options.getCoreApiUri()); - } else if (options.getCoreApiSpecPath() != null) { - return new FileSpecService.Builder(options.getCoreApiSpecPath()); - } else { - return new UnsupportedBuilder( - "Cannot initialise spec service as coreApiHost or specPath was not set."); - } - } - - @Provides - @Singleton - Specs provideSpecs(SpecService.Builder specService) { - return Specs.of(options.getJobName(), importSpec, specService.build()); + Specs provideSpecs() { + return Specs.of(options.getJobName(), importJobSpecs); } @Provides diff --git a/ingestion/src/main/java/feast/ingestion/config/ImportJobSpecsSupplier.java b/ingestion/src/main/java/feast/ingestion/config/ImportJobSpecsSupplier.java new file mode 100644 index 00000000000..030ca816ec0 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/config/ImportJobSpecsSupplier.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.config; + +import com.google.common.base.Strings; +import feast.ingestion.util.PathUtil; +import feast.ingestion.util.ProtoUtil; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import java.io.IOException; +import java.nio.file.Path; +import java.util.function.Supplier; + +public class ImportJobSpecsSupplier implements Supplier { + + public static final String IMPORT_JOB_SPECS_FILENAME = "importJobSpecs.yaml"; + + private Path importJobSpecsPath; + private ImportJobSpecs importJobSpecs; + + public ImportJobSpecsSupplier(String workspace) { + if (!Strings.isNullOrEmpty(workspace)) { + this.importJobSpecsPath = PathUtil.getPath(workspace).resolve(IMPORT_JOB_SPECS_FILENAME); + } + } + + private ImportJobSpecs create() { + try { + if (importJobSpecsPath != null) { + return ProtoUtil + .decodeProtoYamlFile(importJobSpecsPath, ImportJobSpecs.getDefaultInstance()); + } else { + return ImportJobSpecs.getDefaultInstance(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ImportJobSpecs get() { + if (importJobSpecs == null) { + importJobSpecs = create(); + } + return importJobSpecs; + } +} diff --git a/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java b/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java deleted file mode 100644 index ba1b431bfae..00000000000 --- a/ingestion/src/main/java/feast/ingestion/config/ImportSpecSupplier.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.config; - -import com.google.inject.Inject; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Base64; -import java.util.function.Supplier; -import feast.ingestion.options.ImportJobPipelineOptions; -import feast.ingestion.util.ProtoUtil; -import feast.specs.ImportSpecProto.ImportSpec; - -public class ImportSpecSupplier implements Supplier { - - private ImportJobPipelineOptions options; - private ImportSpec importSpec; - - @Inject - public ImportSpecSupplier(ImportJobPipelineOptions options) { - this.options = options; - } - - private ImportSpec create() { - try { - if (options.getImportSpecYamlFile() != null) { - return ProtoUtil.decodeProtoYamlFile( - Paths.get(options.getImportSpecYamlFile()), ImportSpec.getDefaultInstance()); - } else if (options.getImportSpecBase64() != null) { - byte[] bytes = Base64.getDecoder().decode(options.getImportSpecBase64()); - return ImportSpec.parseFrom(bytes); - } else { - return ImportSpec.newBuilder().build(); - } - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public ImportSpec get() { - if (importSpec == null) { - importSpec = create(); - } - return importSpec; - } -} diff --git a/ingestion/src/main/java/feast/ingestion/model/Specs.java b/ingestion/src/main/java/feast/ingestion/model/Specs.java index 5189f63130b..be87b85227e 100644 --- a/ingestion/src/main/java/feast/ingestion/model/Specs.java +++ b/ingestion/src/main/java/feast/ingestion/model/Specs.java @@ -17,122 +17,104 @@ package feast.ingestion.model; +import static com.google.common.base.Predicates.not; + import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import feast.ingestion.service.SpecService; +import com.google.common.base.Strings; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.StorageSpecProto.StorageSpec; import java.io.Serializable; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import lombok.Builder; +import java.util.stream.Collectors; import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@Builder -@Getter @Slf4j @ToString public class Specs implements Serializable { + @Getter + ImportSpec importSpec; + @Getter + Map entitySpecs; + @Getter + Map featureSpecs; + @Getter + Map servingStorageSpecs; + @Getter + Map warehouseStorageSpecs; + @Getter + StorageSpec errorsStoreSpec; + + @Getter private String jobName; - private ImportSpec importSpec; - private Map entitySpecs; - private Map featureSpecs; - private Map storageSpecs; - private transient SpecService specService; - private RuntimeException error; - - public static Specs of(String jobName, ImportSpec importSpec, SpecService specService) { - try { - Specs.SpecsBuilder specsBuilder = Specs.builder().jobName(jobName).importSpec(importSpec); - - List fields = importSpec.getSchema().getFieldsList(); - List featureIds = new ArrayList<>(); - for (Field field : fields) { - if (!field.getFeatureId().isEmpty()) { - featureIds.add(field.getFeatureId()); - } - } - specsBuilder.featureSpecs(specService.getFeatureSpecs(featureIds)); - List entityNames = importSpec.getEntitiesList(); - Set storageIds = Sets.newHashSet(); - for (FeatureSpec featureSpec : specsBuilder.featureSpecs.values()) { - Preconditions.checkArgument( - entityNames.contains(featureSpec.getEntity()), - "Feature has entity not listed in import spec featureSpec=" + featureSpec.toString()); - String servingId = featureSpec.getDataStores().getServing().getId(); - if (!servingId.isEmpty()) { - storageIds.add(servingId); - } - String warehouseId = featureSpec.getDataStores().getWarehouse().getId(); - if (!warehouseId.isEmpty()) { - storageIds.add(warehouseId); - } - } - specsBuilder.entitySpecs(specService.getEntitySpecs(entityNames)); + public Specs(String jobName, ImportJobSpecs importJobSpecs) { + this.jobName = jobName; + if (importJobSpecs != null) { + this.importSpec = importJobSpecs.getImportSpec(); + + this.entitySpecs = importJobSpecs.getEntitySpecsList().stream().collect(Collectors.toMap( + EntitySpec::getName, + entitySpec -> entitySpec + )); + this.featureSpecs = importJobSpecs.getFeatureSpecsList().stream().collect(Collectors.toMap( + FeatureSpec::getId, + featureSpec -> featureSpec + )); + + this.servingStorageSpecs = importJobSpecs.getServingStorageSpecsList().stream() + .collect(Collectors.toMap( + StorageSpec::getId, + storageSpec -> storageSpec + )); + this.warehouseStorageSpecs = importJobSpecs.getWarehouseStorageSpecsList().stream() + .collect(Collectors.toMap( + StorageSpec::getId, + storageSpec -> storageSpec + )); + this.errorsStoreSpec = importJobSpecs.getErrorsStorageSpec(); + } + } - specsBuilder.storageSpecs(specService.getStorageSpecs(storageIds)); + public static Specs of(String jobName, ImportJobSpecs importJobSpecs) { + return new Specs(jobName, filterToUtilized(importJobSpecs)); + } - return specsBuilder.build(); - } catch (RuntimeException e) { - return Specs.builder().error(e).build(); - } + static ImportJobSpecs filterToUtilized(ImportJobSpecs importJobSpecs) { + ImportSpec importSpec = importJobSpecs.getImportSpec(); + List fields = importSpec.getSchema().getFieldsList(); + List featureIds = fields.stream().map(Field::getFeatureId) + .filter(not(Strings::isNullOrEmpty)) + .collect(Collectors.toList()); + List entityNames = importSpec.getEntitiesList(); + + List featureSpecs = + importJobSpecs.getFeatureSpecsList().stream() + .filter(featureSpec -> featureIds.contains(featureSpec.getId())) + .collect(Collectors.toList()); + List entitySpecs = + importJobSpecs.getEntitySpecsList().stream() + .filter(entitySpec -> entityNames.contains(entitySpec.getName())) + .collect(Collectors.toList()); + return importJobSpecs.toBuilder() + .clearFeatureSpecs().addAllFeatureSpecs(featureSpecs) + .clearEntitySpecs().addAllEntitySpecs(entitySpecs).build(); } public void validate() { - if (error != null) { - throw error; - } - - // Sanity checks that our maps are built correctly - for (Entry entry : featureSpecs.entrySet()) { - Preconditions.checkArgument(entry.getKey().equals(entry.getValue().getId()), - String.format("Feature id does not match spec %s!=%s", entry.getKey(), - entry.getValue().getId())); + for (String entityName : importSpec.getEntitiesList()) { + getEntitySpec(entityName); } - for (Entry entry : entitySpecs.entrySet()) { - Preconditions.checkArgument(entry.getKey().equals(entry.getValue().getName()), - String.format("Entity name does not match spec %s!=%s", entry.getKey(), - entry.getValue().getName())); - } - for (Entry entry : storageSpecs.entrySet()) { - Preconditions.checkArgument(entry.getKey().equals(entry.getValue().getId()), - String.format("Storage id does not match spec %s!=%s", entry.getKey(), - entry.getValue().getId())); - } - - for (FeatureSpec featureSpec : featureSpecs.values()) { - // Check that feature has a matching entity - Preconditions.checkArgument( - entitySpecs.containsKey(featureSpec.getEntity()), - String.format( - "Feature %s references unknown entity %s", - featureSpec.getId(), featureSpec.getEntity())); - // Check that feature has a matching serving store - if (!featureSpec.getDataStores().getServing().getId().isEmpty()) { - Preconditions.checkArgument( - storageSpecs.containsKey(featureSpec.getDataStores().getServing().getId()), - String.format( - "Feature %s references unknown serving store %s", - featureSpec.getId(), featureSpec.getDataStores().getServing().getId())); - } - // Check that feature has a matching warehouse store - if (!featureSpec.getDataStores().getWarehouse().getId().isEmpty()) { - Preconditions.checkArgument( - storageSpecs.containsKey(featureSpec.getDataStores().getWarehouse().getId()), - String.format( - "Feature %s references unknown warehouse store %s", - featureSpec.getId(), featureSpec.getDataStores().getWarehouse().getId())); + for (Field field : importSpec.getSchema().getFieldsList()) { + if (!Strings.isNullOrEmpty(field.getFeatureId())) { + getFeatureSpec(field.getFeatureId()); } } } @@ -150,21 +132,4 @@ public FeatureSpec getFeatureSpec(String featureId) { String.format("Unknown feature %s, spec was not initialized", featureId)); return featureSpecs.get(featureId); } - - public List getFeatureSpecByServingStoreId(String storeId) { - List out = new ArrayList<>(); - for (FeatureSpec featureSpec : featureSpecs.values()) { - if (featureSpec.getDataStores().getServing().getId().equals(storeId)) { - out.add(featureSpec); - } - } - return out; - } - - public StorageSpec getStorageSpec(String storeId) { - Preconditions.checkArgument( - storageSpecs.containsKey(storeId), - String.format("Unknown store %s, spec was not initialized", storeId)); - return storageSpecs.get(storeId); - } } diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java index 72e8b9a53a1..25dd62d4abc 100644 --- a/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java +++ b/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java @@ -19,7 +19,6 @@ import com.google.auto.service.AutoService; import java.util.Collections; -import org.apache.beam.sdk.metrics.MetricsSink; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptions; @@ -31,17 +30,11 @@ */ public interface ImportJobPipelineOptions extends PipelineOptions { - @Description("Import spec yaml file path") - @Required(groups = {"importSpec"}) - String getImportSpecYamlFile(); + @Description("Path to a workspace directory containing importJobSpecs.yaml") + @Required + String getWorkspace(); - void setImportSpecYamlFile(String value); - - @Description("Import spec as native proto binary encoding converted to Base64 string") - @Required(groups = {"importSpec"}) - String getImportSpecBase64(); - - void setImportSpecBase64(String value); + void setWorkspace(String value); @Description("If dry run is set, execute up to feature row validation") @Default.Boolean(false) @@ -49,34 +42,6 @@ public interface ImportJobPipelineOptions extends PipelineOptions { void setDryRun(Boolean value); - @Description("Uri of the core api service. eg: localhost:8001") - @Required(groups = {"coreApi"}) - String getCoreApiUri(); - - void setCoreApiUri(String value); - - @Description( - "Path to root directory of json spec jsonfiles, substitutes for the core api service") - @Required(groups = {"coreApi"}) - String getCoreApiSpecPath(); - - void setCoreApiSpecPath(String value); - - @Description( - "Set an errors store type. One of: [stderr, stdout, file.json]. Note that you should not use " - + "stderr/stdout in production unless your data volume is extremely small.") - String getErrorsStoreType(); - - void setErrorsStoreType(String value); - - @Description( - "Provide errors store options as a json string containing key-values. Options required" - + "depend on the type of store set.") - @Default.String("{}") - String getErrorsStoreOptions(); - - void setErrorsStoreOptions(String value); - @AutoService(PipelineOptionsRegistrar.class) class ImportJobPipelineOptionsRegistrar implements PipelineOptionsRegistrar { diff --git a/ingestion/src/main/java/feast/ingestion/service/CoreSpecService.java b/ingestion/src/main/java/feast/ingestion/service/CoreSpecService.java deleted file mode 100644 index 9e57c289984..00000000000 --- a/ingestion/src/main/java/feast/ingestion/service/CoreSpecService.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.service; - -import com.google.protobuf.Empty; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import feast.core.CoreServiceGrpc; -import feast.core.CoreServiceProto.CoreServiceTypes.GetEntitiesRequest; -import feast.core.CoreServiceProto.CoreServiceTypes.GetEntitiesResponse; -import feast.core.CoreServiceProto.CoreServiceTypes.GetFeaturesRequest; -import feast.core.CoreServiceProto.CoreServiceTypes.GetFeaturesResponse; -import feast.core.CoreServiceProto.CoreServiceTypes.GetStorageRequest; -import feast.core.CoreServiceProto.CoreServiceTypes.GetStorageResponse; -import feast.core.CoreServiceProto.CoreServiceTypes.ListEntitiesResponse; -import feast.core.CoreServiceProto.CoreServiceTypes.ListFeaturesResponse; -import feast.core.CoreServiceProto.CoreServiceTypes.ListStorageResponse; -import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.specs.StorageSpecProto.StorageSpec; - -/** Class responsible for retrieving Feature, Entity, and Storage Spec from Core API. */ -@Slf4j -public class CoreSpecService implements SpecService { - private final ManagedChannel channel; - private final CoreServiceGrpc.CoreServiceBlockingStub blockingStub; - - public CoreSpecService(String uri) { - this(ManagedChannelBuilder.forTarget(uri)); - } - - public CoreSpecService(ManagedChannelBuilder channelBuilder) { - channel = channelBuilder.usePlaintext(true).build(); - blockingStub = CoreServiceGrpc.newBlockingStub(channel); - } - - /** - * Get map of entity ID and {@link EntitySpec} from Core API, given a collection of entityId. - */ - public Map getEntitySpecs(Iterable entityIds) { - GetEntitiesRequest request = GetEntitiesRequest.newBuilder().addAllIds(entityIds).build(); - - try { - GetEntitiesResponse response = blockingStub.getEntities(request); - return response - .getEntitiesList() - .stream() - .collect(Collectors.toMap(EntitySpec::getName, Function.identity())); - } catch (StatusRuntimeException e) { - log.error("GRPC error in getEntitySpecs: {}", e.getStatus()); - throw new SpecRetrievalException("Unable to retrieve entity spec", e); - } - } - - /** - * Get all {@link EntitySpec} from Core API. - */ - public Map getAllEntitySpecs() { - try { - ListEntitiesResponse response = blockingStub.listEntities(Empty.getDefaultInstance()); - return response - .getEntitiesList() - .stream() - .collect(Collectors.toMap(EntitySpec::getName, Function.identity())); - } catch (StatusRuntimeException e) { - log.error("GRPC error in getAllEntitySpecs: {}", e.getStatus()); - throw new SpecRetrievalException("Unable to retrieve entity spec", e); - } - } - - /** - * Get map of {@link FeatureSpec} from Core API, given a collection of featureId. - */ - public Map getFeatureSpecs(Iterable featureIds) { - try { - GetFeaturesRequest request = GetFeaturesRequest.newBuilder().addAllIds(featureIds).build(); - GetFeaturesResponse response = blockingStub.getFeatures(request); - return response - .getFeaturesList() - .stream() - .collect(Collectors.toMap(FeatureSpec::getId, Function.identity())); - } catch (StatusRuntimeException e) { - log.error("GRPC error in getFeatureSpecs: {}", e.getStatus()); - throw new SpecRetrievalException("Unable to retrieve feature specs", e); - } - } - - /** - * Get all {@link FeatureSpec} available in Core API. - */ - public Map getAllFeatureSpecs() { - try { - ListFeaturesResponse response = blockingStub.listFeatures(Empty.getDefaultInstance()); - return response - .getFeaturesList() - .stream() - .collect(Collectors.toMap(FeatureSpec::getId, Function.identity())); - } catch (StatusRuntimeException e) { - log.error("GRPC error in getAllFeatureSpecs, {}", e.getStatus()); - throw new SpecRetrievalException("Unable to retrieve feature specs", e); - } - } - - /** - * Get map of {@link StorageSpec} from Core API, given a collection of storageId. - */ - public Map getStorageSpecs(Iterable storageIds) { - try { - GetStorageRequest request = GetStorageRequest.newBuilder().addAllIds(storageIds).build(); - - GetStorageResponse response = blockingStub.getStorage(request); - return response - .getStorageSpecsList() - .stream() - .collect(Collectors.toMap(StorageSpec::getId, Function.identity())); - } catch (StatusRuntimeException e) { - log.error("GRPC error in getStorageSpecs: {}", e.getStatus()); - throw new SpecRetrievalException("Unable to retrieve storage specs", e); - } - } - - /** - * Get all {@link StorageSpec} from Core API. - */ - public Map getAllStorageSpecs() { - try { - ListStorageResponse response = blockingStub.listStorage(Empty.getDefaultInstance()); - return response - .getStorageSpecsList() - .stream() - .collect(Collectors.toMap(StorageSpec::getId, Function.identity())); - } catch (StatusRuntimeException e) { - log.error("GRPC error in getAllStorageSpecs, {}", e.getStatus()); - throw new SpecRetrievalException("Unable to retrieve storage specs", e); - } - } - - /** - * Shutdown GRPC channel. - */ - public void shutdown() throws InterruptedException { - log.info("Shutting down CoreSpecService"); - channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); - } - - @AllArgsConstructor - public static class Builder implements SpecService.Builder { - private String uri; - - @Override - public SpecService build() { - return new CoreSpecService(uri); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/service/FileSpecService.java b/ingestion/src/main/java/feast/ingestion/service/FileSpecService.java deleted file mode 100644 index a6bc9c24543..00000000000 --- a/ingestion/src/main/java/feast/ingestion/service/FileSpecService.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.service; - -import com.google.common.base.Charsets; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.util.JsonFormat; -import feast.ingestion.exceptions.SpecNotFound; -import feast.ingestion.util.PathUtil; -import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.specs.StorageSpecProto.StorageSpec; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; - -@AllArgsConstructor -public class FileSpecService implements SpecService { - - private static final String ENTITY_SPEC = "entity"; - private static final String FEATURE_SPEC = "feature"; - private static final String STORAGE_SPEC = "storage"; - - private String basePath; - - private void mergeJsonProto(Path path, Message.Builder builder) { - try (InputStream inputStream = Files.newInputStream(path)) { - Reader reader = new InputStreamReader(inputStream); - JsonFormat.parser().merge(reader, builder); - } catch (FileNotFoundException e) { - throw new SpecNotFound("Spec not found at filepath " + path.toString(), e); - } catch (IOException e) { - throw new SpecRetrievalException("Error parsing spec at filepath " + path.toString(), e); - } - } - - private Path getBasePath() { - return PathUtil.getPath(basePath); - } - - private Path getSpecDir(String type) { - return getBasePath().resolve(type); - } - - private Path getSpecFile(String type, String id) { - return getBasePath().resolve(Paths.get(type, String.format("%s.json", id)).toString()); - } - - private Map getSpecs( - T prototype, String type, Iterable ids) { - Map specs = new HashMap<>(); - for (String id : ids) { - Message.Builder builder = prototype.toBuilder(); - mergeJsonProto(getSpecFile(type, id), builder); - //noinspection unchecked - specs.put(id, (T) builder.build()); - } - return specs; - } - - private Map getAllSpecs( - T prototype, String type, Function keyFunc) { - prototype.toBuilder(); - Map specs = new HashMap<>(); - Path dir = getSpecDir(type); - try { - List paths = Files.list(dir).collect(Collectors.toList()); - for (Path path : paths) { - if (path.toString().endsWith(".json")) { - Message.Builder builder = prototype.toBuilder(); - mergeJsonProto(path, builder); - @SuppressWarnings("unchecked") - T spec = (T) builder.build(); - specs.put(keyFunc.apply(spec), spec); - } - } - } catch (IOException e) { - throw new SpecRetrievalException("Error listing spec directory " + dir.toString(), e); - } - return specs; - } - - @Override - public Map getEntitySpecs(Iterable entityIds) { - return getSpecs(EntitySpec.getDefaultInstance(), ENTITY_SPEC, entityIds); - } - - @Override - public Map getFeatureSpecs(Iterable featureIds) { - return getSpecs(FeatureSpec.getDefaultInstance(), FEATURE_SPEC, featureIds); - } - - @Override - public Map getStorageSpecs(Iterable storageIds) { - return getSpecs(StorageSpec.getDefaultInstance(), STORAGE_SPEC, storageIds); - } - - private void putSpecs( - String type, Function keyFunc, Iterable specs) { - for (T spec : specs) { - Path path = getSpecFile(type, keyFunc.apply(spec)); - try { - String specJson = JsonFormat.printer().print(spec); - Files.write(path, specJson.getBytes(Charsets.UTF_8)); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException( - "Invalid proto while writing spec to file " + path.toString(), e); - } catch (IOException e) { - throw new RuntimeException("Unable to write spec to file " + path.toString(), e); - } - } - } - - // These below functions are useful in tests - - public void putEntitySpecs(Iterable entitySpecs) { - putSpecs(ENTITY_SPEC, EntitySpec::getName, entitySpecs); - } - - public void putFeatureSpecs(Iterable featureSpecs) { - putSpecs(FEATURE_SPEC, FeatureSpec::getId, featureSpecs); - } - - public void putStorageSpecs(Iterable storageSpecs) { - putSpecs(STORAGE_SPEC, StorageSpec::getId, storageSpecs); - } - - @AllArgsConstructor - public static class Builder implements SpecService.Builder { - - private String basePath; - - @Override - public SpecService build() { - return new FileSpecService(basePath); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/service/SpecService.java b/ingestion/src/main/java/feast/ingestion/service/SpecService.java deleted file mode 100644 index 6dd9c28d9ba..00000000000 --- a/ingestion/src/main/java/feast/ingestion/service/SpecService.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.service; - -import java.io.Serializable; -import java.util.Map; -import lombok.AllArgsConstructor; -import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.specs.StorageSpecProto.StorageSpec; - -/** - * Abstraction of service which provides {@link EntitySpec}, {@link FeatureSpec}, and {@link - * StorageSpec}. - */ -public interface SpecService { - /** - * Get a map of {@link EntitySpec} from Core API, given a collection of entityId. - * - * @param entityIds collection of entityId to retrieve. - * @return map of {@link EntitySpec}, where the key is entity name. - * @throws SpecRetrievalException if any error happens during retrieval - */ - Map getEntitySpecs(Iterable entityIds); - - /** - * Get a map of {@link FeatureSpec} from Core API, given a collection of featureId. - * - * @param featureIds collection of entityId to retrieve. - * @return map of {@link FeatureSpec}, where the key is feature id. - * @throws SpecRetrievalException if any error happens during retrieval - */ - Map getFeatureSpecs(Iterable featureIds); - - /** - * Get map of {@link StorageSpec} from Core API, given a collection of storageId. - * - * @param storageIds collection of storageId to retrieve. - * @return map of {@link StorageSpec}, where the key is storage id. - * @throws SpecRetrievalException if any error happens during retrieval - */ - Map getStorageSpecs(Iterable storageIds); - - interface Builder extends Serializable { - SpecService build(); - } - - @AllArgsConstructor - class UnsupportedBuilder implements Builder { - private String message; - - @Override - public SpecService build() { - throw new UnsupportedOperationException(message); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java index 92590b91df9..70f7f2d3c75 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java @@ -18,27 +18,28 @@ package feast.ingestion.transform; import static com.google.common.base.Preconditions.checkArgument; -import static feast.ingestion.util.JsonUtil.convertJsonStringToMap; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.inject.Inject; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; +import feast.ingestion.util.PathUtil; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.FeatureStoreWrite; -import feast.store.NoOpIO; import feast.store.errors.FeatureErrorsFactory; +import feast.store.errors.json.JsonFileErrorsFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import java.nio.file.Path; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.hadoop.hbase.util.Strings; @Slf4j public class ErrorsStoreTransform extends FeatureStoreWrite { - private String errorsStoreType; - private StorageSpec errorsStoreSpec; + private String workspace; private Specs specs; private List errorsStoreFactories; @@ -46,35 +47,31 @@ public class ErrorsStoreTransform extends FeatureStoreWrite { public ErrorsStoreTransform( ImportJobPipelineOptions options, Specs specs, List errorsStoreFactories) { + this.workspace = options.getWorkspace(); this.specs = specs; this.errorsStoreFactories = errorsStoreFactories; - this.errorsStoreType = options.getErrorsStoreType(); - - if (!Strings.isEmpty(errorsStoreType)) { - this.errorsStoreSpec = - StorageSpec.newBuilder() - .setType(errorsStoreType) - .putAllOptions(convertJsonStringToMap(options.getErrorsStoreOptions())) - .build(); - } } @Override public PDone expand(PCollection input) { - FeatureStoreWrite write; - if (Strings.isEmpty(errorsStoreType)) { - write = new NoOpIO.Write(); - } else { - write = getErrorStore().create(this.errorsStoreSpec, specs); + StorageSpec errorsStoreSpec = specs.getErrorsStoreSpec(); + if (Strings.isNullOrEmpty(errorsStoreSpec.getType())) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(workspace), "workspace must be provided"); + Path workspaceErrorsPath = PathUtil.getPath(workspace).resolve("errors"); + errorsStoreSpec = StorageSpec.newBuilder() + .setId("workspace/errors") + .setType(JsonFileErrorsFactory.JSON_FILES_TYPE) + .putOptions("path", workspaceErrorsPath.toString()).build(); } - input.apply("errors to " + String.valueOf(errorsStoreType), write); + input.apply("Write errors" + errorsStoreSpec.getType(), + getErrorStore(errorsStoreSpec.getType()).create(errorsStoreSpec, specs)); return PDone.in(input.getPipeline()); } - FeatureErrorsFactory getErrorStore() { - checkArgument(!errorsStoreType.isEmpty(), "Errors store type not provided"); + FeatureErrorsFactory getErrorStore(String type) { + checkArgument(!type.isEmpty(), "Errors store type not provided"); for (FeatureErrorsFactory errorsStoreFactory : errorsStoreFactories) { - if (errorsStoreFactory.getType().equals(errorsStoreType)) { + if (errorsStoreFactory.getType().equals(type)) { return errorsStoreFactory; } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/ReadFeaturesTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ReadFeaturesTransform.java index 5da8f6e7532..15dc223cc2f 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ReadFeaturesTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ReadFeaturesTransform.java @@ -19,6 +19,7 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; +import feast.ingestion.model.Specs; import feast.source.FeatureSourceFactory; import feast.source.FeatureSourceFactoryService; import feast.specs.ImportSpecProto.ImportSpec; @@ -32,8 +33,8 @@ public class ReadFeaturesTransform extends PTransform featureSpec.getDataStores().getServing().getId(), specs)); + stores, (featureSpec) -> featureSpec.getDataStores().getServing().getId(), specs, + specs.getServingStorageSpecs())); output.getMain().apply("metrics.store.lag", ParDo.of(FeastMetrics.lagUpdateDoFn())); output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("serving_stored"))); diff --git a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java index c7e58892f02..ffb1393698e 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java @@ -24,8 +24,8 @@ import feast.ingestion.values.PFeatureRows; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.store.FeatureStoreWrite; import feast.store.FeatureStoreFactory; +import feast.store.FeatureStoreWrite; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collection; import java.util.HashMap; @@ -49,6 +49,7 @@ public class SplitOutputByStore extends PTransform { private Collection stores; private SerializableFunction selector; private Specs specs; + private Map storageSpecs; @Override public PFeatureRows expand(PFeatureRows input) { @@ -83,7 +84,6 @@ private Map getStoresMap() { private Map getFeatureStoreTransforms() { Map storesMap = getStoresMap(); Map transforms = new HashMap<>(); - Map storageSpecs = specs.getStorageSpecs(); for (String storeId : storageSpecs.keySet()) { StorageSpec storageSpec = storageSpecs.get(storeId); String type = storageSpec.getType(); diff --git a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java index de70541961a..90e8211f2a2 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java @@ -15,6 +15,7 @@ * */ + package feast.ingestion.transform; import com.google.inject.Inject; @@ -47,7 +48,8 @@ public PFeatureRows expand(PFeatureRows input) { new SplitOutputByStore( stores, (featureSpec) -> featureSpec.getDataStores().getWarehouse().getId(), - specs)); + specs, + specs.getWarehouseStorageSpecs())); output.getMain() .apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored"))); output.getErrors() diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java index 2704c5e766b..8336c8dd323 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java @@ -102,19 +102,20 @@ public void processElementImpl(ProcessContext context) { checkNotNull( featureSpec, String.format("Feature spec not found featureId=%s", feature.getId())); - String storageStoreId = featureSpec.getDataStores().getServing().getId(); - StorageSpec servingStorageSpec = specs.getStorageSpec(storageStoreId); - checkArgument( - supportedServingTypes.contains(servingStorageSpec.getType()), - String.format("Serving storage type=%s not supported", servingStorageSpec.getType())); + String servingStoreId = featureSpec.getDataStores().getServing().getId(); + if (!servingStoreId.equals(EMPTY_STORE)) { + StorageSpec servingStorageSpec = specs.getServingStorageSpecs() + .getOrDefault(servingStoreId, null); + checkNotNull(servingStorageSpec, + "Serving storage specs not found for store id " + servingStoreId); + } String warehouseStoreId = featureSpec.getDataStores().getWarehouse().getId(); if (!warehouseStoreId.equals(EMPTY_STORE)) { - StorageSpec warehouseStorageSpec = specs.getStorageSpec(warehouseStoreId); - checkArgument( - supportedWarehouseTypes.contains(warehouseStorageSpec.getType()), - String.format( - "Warehouse storage type=%s not supported", servingStorageSpec.getType())); + StorageSpec warehouseStorageSpec = specs.getWarehouseStorageSpecs() + .getOrDefault(warehouseStoreId, null); + checkNotNull(warehouseStorageSpec, + "Warehouse storage specs not found for store id " + servingStoreId); } checkArgument( diff --git a/ingestion/src/main/java/feast/ingestion/util/ProtoUtil.java b/ingestion/src/main/java/feast/ingestion/util/ProtoUtil.java index de59dac3818..9b71faff276 100644 --- a/ingestion/src/main/java/feast/ingestion/util/ProtoUtil.java +++ b/ingestion/src/main/java/feast/ingestion/util/ProtoUtil.java @@ -26,18 +26,18 @@ import com.google.protobuf.Message; import com.google.protobuf.util.JsonFormat; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; public class ProtoUtil { - private ProtoUtil() {} + + private ProtoUtil() { + } public static T decodeProtoYamlFile(Path path, T prototype) throws IOException { - ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); - ObjectMap map = yamlMapper.readerFor(ObjectMap.class).readValue(path.toFile()); - ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory()); - String json = jsonMapper.writerFor(ObjectMap.class).writeValueAsString(map); - return decodeProtoJson(json, prototype); + String yaml = String.join("\n", Files.readAllLines(path)); + return decodeProtoYaml(yaml, prototype); } public static T decodeProtoYaml(String yamlString, T prototype) diff --git a/ingestion/src/main/java/feast/store/TextFileDynamicIO.java b/ingestion/src/main/java/feast/store/TextFileDynamicIO.java index ee7ed107516..250b46ea6b7 100644 --- a/ingestion/src/main/java/feast/store/TextFileDynamicIO.java +++ b/ingestion/src/main/java/feast/store/TextFileDynamicIO.java @@ -56,7 +56,7 @@ public PDone expand(PCollection> input) { .withDestinationCoder(StringUtf8Coder.of()) .withNaming( Contextful.fn( - (key) -> FileIO.Write.defaultNaming(folderName + "/" + key + "/part-", suffix))) + (key) -> FileIO.Write.defaultNaming(folderName + "/" + key + "/part", suffix))) .via(Contextful.fn(KV::getValue), Contextful.fn((entityName) -> TextIO.sink())) .to(options.path); diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java index 3a56382b1f4..42df872ff7d 100644 --- a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java @@ -31,7 +31,7 @@ public class JsonFileErrorsFactory implements FeatureErrorsFactory { - private static final String JSON_FILES_TYPE = "file.json"; + public static final String JSON_FILES_TYPE = "file.json"; @Override public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index 7f67bb15af2..67e8ed8e862 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -19,7 +19,6 @@ import static feast.FeastMatchers.hasCount; import static feast.NormalizeFeatureRows.normalize; -import static feast.store.MockFeatureErrorsFactory.MOCK_ERRORS_STORE_TYPE; import static org.junit.Assert.assertEquals; import com.google.common.base.Charsets; @@ -32,11 +31,12 @@ import feast.ToOrderedFeatureRows; import feast.ingestion.boot.ImportJobModule; import feast.ingestion.boot.TestPipelineModule; +import feast.ingestion.config.ImportJobSpecsSupplier; import feast.ingestion.model.Features; import feast.ingestion.model.Values; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.ingestion.service.SpecRetrievalException; import feast.ingestion.util.ProtoUtil; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.specs.ImportSpecProto.ImportSpec; import feast.store.MockFeatureErrorsFactory; import feast.store.MockServingFactory; @@ -72,16 +72,17 @@ public class ImportJobCSVTest { @Rule public TestPipeline testPipeline = TestPipeline.create(); - public ImportSpec initImportSpec(ImportSpec importSpec, String dataFile) { - return importSpec.toBuilder().putSourceOptions("path", dataFile).build(); + public ImportJobSpecs getImportJobSpecs(ImportSpec importSpec, String dataFile) { + Path workspacePath = Paths.get(Resources.getResource("specs").getPath()); + ImportJobSpecs importJobSpecs = new ImportJobSpecsSupplier(workspacePath.toUri().toString()).get(); + return importJobSpecs.toBuilder().setImportSpec( + importSpec.toBuilder().putSourceOptions("path", dataFile) + ).build(); } public ImportJobPipelineOptions initOptions() { - Path path = Paths.get(Resources.getResource("core_specs/").getPath()); ImportJobPipelineOptions options = PipelineOptionsFactory.create() .as(ImportJobPipelineOptions.class); - options.setCoreApiSpecPath(path.toString()); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); return options; } @@ -107,14 +108,13 @@ public void testImportCSV() throws IOException { File csvFile = folder.newFile("data.csv"); Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); - importSpec = initImportSpec(importSpec, csvFile.toString()); ImportJobPipelineOptions options = initOptions(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, getImportJobSpecs(importSpec, csvFile.toString())), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); injector.getInstance(ImportJob.class); @@ -200,22 +200,23 @@ public void testImportFileJson() throws IOException { ImportSpec.getDefaultInstance()); File jsonFile = folder.newFile("data.json"); - Files.asCharSink(jsonFile, Charsets.UTF_8).write("{\"id\":1,\"x\":101}\n{\"id\":2,\"x\":202}\n"); - importSpec = initImportSpec(importSpec, jsonFile.toString()); + Files.asCharSink(jsonFile, Charsets.UTF_8) + .write("{\"id\":1,\"x\":101}\n{\"id\":2,\"x\":202}\n"); ImportJobPipelineOptions options = initOptions(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, getImportJobSpecs(importSpec, jsonFile.toString())), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); injector.getInstance(ImportJob.class); job.expand(); PCollection writtenToServing = - PCollectionList.of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = @@ -224,7 +225,9 @@ public void testImportFileJson() throws IOException { .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); List expectedRows = @@ -281,14 +284,13 @@ public void testImportCSV_withSample1() throws IOException { File csvFile = folder.newFile("data.csv"); Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); - importSpec = initImportSpec(importSpec, csvFile.toString()); ImportJobPipelineOptions options = initOptions(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, getImportJobSpecs(importSpec, csvFile.toString())), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); injector.getInstance(ImportJob.class); @@ -345,14 +347,13 @@ public void testImportCSV_withCoalesceRows() throws IOException { File csvFile = folder.newFile("data.csv"); Files.asCharSink(csvFile, Charsets.UTF_8) .write("1,2018-09-25T00:00:00.000Z,101,a\n1,2018-09-26T00:00:00.000Z,,b\n"); - importSpec = initImportSpec(importSpec, csvFile.toString()); ImportJobPipelineOptions options = initOptions(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, getImportJobSpecs(importSpec, csvFile.toString())), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); injector.getInstance(ImportJob.class); @@ -410,7 +411,10 @@ public void testImportCSV_withCoalesceRows() throws IOException { testPipeline.run(); } - @Test(expected = SpecRetrievalException.class) + /* + * ingestion no longer cares what the feature themselves say about which store they should write + * it instead always writes to the specs.getServingStoreSpec() and specs.getWarehouseStoreSpec(). + */ public void testImportCSVUnknownServingStoreError() throws IOException { ImportSpec importSpec = ProtoUtil.decodeProtoYaml( @@ -433,19 +437,36 @@ public void testImportCSVUnknownServingStoreError() throws IOException { File csvFile = folder.newFile("data.csv"); Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); - importSpec = initImportSpec(importSpec, csvFile.toString()); ImportJobPipelineOptions options = initOptions(); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, getImportJobSpecs(importSpec, csvFile.toString())), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); injector.getInstance(ImportJob.class); // Job should fail during expand(), so we don't even need to start the pipeline. job.expand(); + + PCollection writtenToServing = + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) + .apply("flatten serving input", Flatten.pCollections()); + + PCollection writtenToWarehouse = + PCollectionList.of( + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) + .apply("flatten warehouse input", Flatten.pCollections()); + + PAssert.that(writtenToServing).satisfies(hasCount(1)); + PAssert.that(writtenToWarehouse).satisfies(hasCount(1)); + + testPipeline.run(); } @Test @@ -472,14 +493,13 @@ public void testImportWithErrors() throws IOException { // Note the string and integer features are in the wrong positions for the import spec. Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); - importSpec = initImportSpec(importSpec, csvFile.toString()); ImportJobPipelineOptions options = initOptions(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, getImportJobSpecs(importSpec, csvFile.toString())), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); @@ -541,14 +561,16 @@ public void testImportWithoutWarehouseStore() throws IOException { // Note the string and integer features are in the wrong positions for the import spec. Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); - importSpec = initImportSpec(importSpec, csvFile.toString()); ImportJobPipelineOptions options = initOptions(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); + + ImportJobSpecs importJobSpecs = getImportJobSpecs(importSpec, csvFile.toString()).toBuilder() + .clearWarehouseStorageSpecs().build(); Injector injector = Guice.createInjector( - new ImportJobModule(options, importSpec), new TestPipelineModule(testPipeline)); + new ImportJobModule(options, importJobSpecs), + new TestPipelineModule(testPipeline)); ImportJob job = injector.getInstance(ImportJob.class); @@ -561,16 +583,83 @@ public void testImportWithoutWarehouseStore() throws IOException { .getInputs()) .apply("flatten serving input", Flatten.pCollections()); + PCollection writtenToWarehouse = + PCollectionList + .of(FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) + .apply("flatten warehouse input", Flatten.pCollections()); + PCollection writtenToErrors = PCollectionList .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() .getInputs()) .apply("flatten errors input", Flatten.pCollections()); - PAssert.that(writtenToErrors) - .satisfies(hasCount(0)); + PAssert.that(writtenToErrors).satisfies(hasCount(0)); + PAssert.that(writtenToWarehouse).satisfies(hasCount(0)); + PAssert.that(writtenToServing).satisfies(hasCount(3)); + testPipeline.run(); + } + + @Test + public void testImportWithoutWarehouseStoreSetByFeature() throws IOException { + ImportSpec importSpec = + ProtoUtil.decodeProtoYaml( + "---\n" + + "type: file.csv\n" + + "sourceOptions:\n" + + " # path: to be overwritten in tests\n" + + "entities:\n" + + " - testEntity\n" + + "schema:\n" + + " entityIdColumn: id\n" + + " timestampValue: 2018-09-25T00:00:00.000Z\n" + + " fields:\n" + + " - name: id\n" + + " - featureId: testEntity.none.testInt64NoWarehouse\n" + + " - featureId: testEntity.none.testStringNoWarehouse\n" + + "\n", + ImportSpec.getDefaultInstance()); + + File csvFile = folder.newFile("data.csv"); + + // Note the string and integer features are in the wrong positions for the import spec. + Files.asCharSink(csvFile, Charsets.UTF_8).write("1,101,a\n2,202,b\n3,303,c\n"); + + ImportJobPipelineOptions options = initOptions(); + + Injector injector = + Guice.createInjector( + new ImportJobModule(options, getImportJobSpecs(importSpec, csvFile.toString())), + new TestPipelineModule(testPipeline)); + + ImportJob job = injector.getInstance(ImportJob.class); + + injector.getInstance(ImportJob.class); + job.expand(); + + PCollection writtenToServing = + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) + .apply("flatten serving input", Flatten.pCollections()); + + PCollection writtenToWarehouse = + PCollectionList + .of(FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) + .apply("flatten warehouse input", Flatten.pCollections()); + + PCollection writtenToErrors = + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) + .apply("flatten errors input", Flatten.pCollections()); + + PAssert.that(writtenToErrors).satisfies(hasCount(0)); PAssert.that(writtenToServing).satisfies(hasCount(3)); + PAssert.that(writtenToWarehouse).satisfies(hasCount(0)); testPipeline.run(); } } diff --git a/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java b/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java new file mode 100644 index 00000000000..3494e4c41bb --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.ingestion.config; + +import static org.junit.Assert.assertEquals; + +import com.google.protobuf.util.JsonFormat; +import feast.ingestion.model.Specs; +import feast.ingestion.util.DateUtil; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import feast.specs.ImportSpecProto.Field; +import feast.specs.ImportSpecProto.ImportSpec; +import feast.specs.ImportSpecProto.Schema; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.types.GranularityProto.Granularity; +import feast.types.ValueProto.ValueType.Enum; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class ImportJobSpecsSupplierTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + String importSpecYaml = + "---\n" + + "servingStorageSpecs:\n" + + " - id: TEST_SERVING\n" + + " type: serving.mock\n" + + " options: {}\n" + + "warehouseStorageSpecs:\n" + + " - id: TEST_WAREHOUSE\n" + + " type: warehouse.mock\n" + + " options: {}\n" + + "entitySpecs:\n" + + " - name: testEntity\n" + + " description: This is a test entity\n" + + " tags: []\n" + + "featureSpecs:\n" + + " - id: testEntity.day.testInt64\n" + + " entity: testEntity\n" + + " granularity: DAY\n" + + " name: testInt64\n" + + " owner: feast@example.com\n" + + " description: This is test feature of type integer\n" + + " uri: https://example.com/\n" + + " valueType: INT64\n" + + " tags: []\n" + + " options: {}\n" + + "importSpec:\n" + + " type: file.csv\n" + + " sourceOptions:\n" + + " path: data.csv\n" + + " entities:\n" + + " - driver\n" + + " schema:\n" + + " entityIdColumn: driver_id\n" + + " timestampValue: 2018-09-25T00:00:00.000Z\n" + + " fields:\n" + + " - name: timestamp\n" + + " - name: driver_id\n" + + " - name: trips_completed\n" + + " featureId: driver.none.trips_completed\n" + + "\n"; + + @Test + public void testSupplierImportSpecYamlFile() throws IOException { + File yamlFile = temporaryFolder.newFile("importJobSpecs.yaml"); + try (PrintWriter printWriter = new PrintWriter(Files.newOutputStream(yamlFile.toPath()))) { + printWriter.print(importSpecYaml); + } + + ImportJobSpecs importJobSpecs = new ImportJobSpecsSupplier(yamlFile.getParent()).get(); + Specs specs = new Specs("", importJobSpecs); + System.out + .println(JsonFormat.printer().omittingInsignificantWhitespace().print(importJobSpecs)); + assertEquals(ImportSpec.newBuilder() + .setType("file.csv") + .putSourceOptions("path", "data.csv") + .addEntities("driver") + .setSchema( + Schema.newBuilder() + .addFields(Field.newBuilder().setName("timestamp")) + .addFields(Field.newBuilder().setName("driver_id")) + .addFields( + Field.newBuilder() + .setName("trips_completed") + .setFeatureId("driver.none.trips_completed")) + .setEntityIdColumn("driver_id") + .setTimestampValue(DateUtil.toTimestamp("2018-09-25T00:00:00.000Z"))) + .build(), importJobSpecs.getImportSpec()); + + assertEquals(StorageSpec.newBuilder() + .setId("TEST_SERVING") + .setType("serving.mock") + .build(), importJobSpecs.getServingStorageSpecs(0)); + + assertEquals(StorageSpec.newBuilder() + .setId("TEST_WAREHOUSE") + .setType("warehouse.mock") + .build(), importJobSpecs.getWarehouseStorageSpecs(0)); + + assertEquals(EntitySpec.newBuilder() + .setName("testEntity") + .setDescription("This is a test entity") + .build(), specs.getEntitySpec("testEntity")); + + assertEquals(FeatureSpec.newBuilder() + .setId("testEntity.day.testInt64") + .setEntity("testEntity") + .setName("testInt64") + .setOwner("feast@example.com") + .setUri("https://example.com/") + .setValueType(Enum.INT64) + .setGranularity(Granularity.Enum.DAY) + .setDescription("This is test feature of type integer") + .build(), specs.getFeatureSpec("testEntity.day.testInt64")); + } +} diff --git a/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java b/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java deleted file mode 100644 index 00eacdcaeb4..00000000000 --- a/ingestion/src/test/java/feast/ingestion/config/ImportSpecSupplierTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.config; - -import feast.ingestion.util.DateUtil; -import feast.ingestion.options.ImportJobPipelineOptions; -import com.google.protobuf.util.JsonFormat; -import feast.specs.ImportSpecProto.Field; -import feast.specs.ImportSpecProto.ImportSpec; -import feast.specs.ImportSpecProto.Schema; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.Rule; -import static org.junit.Assert.*; - -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class ImportSpecSupplierTest { - String importSpecYaml = - "---\n" - + "type: file.csv\n" - + "sourceOptions:\n" - + " path: data.csv\n" - + "entities:\n" - + " - driver\n" - + "schema:\n" - + " entityIdColumn: driver_id\n" - + " timestampValue: 2018-09-25T00:00:00.000Z\n" - + " fields:\n" - + " - name: timestamp\n" - + " - name: driver_id\n" - + " - name: trips_completed\n" - + " featureId: driver.none.trips_completed\n" - + "\n"; - - ImportSpec expectedImportSpec = - ImportSpec.newBuilder() - .setType("file.csv") - .putSourceOptions("path", "data.csv") - .addEntities("driver") - .setSchema( - Schema.newBuilder() - .addFields(Field.newBuilder().setName("timestamp")) - .addFields(Field.newBuilder().setName("driver_id")) - .addFields( - Field.newBuilder() - .setName("trips_completed") - .setFeatureId("driver.none.trips_completed")) - .setEntityIdColumn("driver_id") - .setTimestampValue(DateUtil.toTimestamp("2018-09-25T00:00:00.000Z"))) - .build(); - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Test - public void testSupplierImportSpecYamlFile() throws IOException { - File yamlFile = temporaryFolder.newFile("importSpec.yaml"); - try (PrintWriter printWriter = new PrintWriter(Files.newOutputStream(yamlFile.toPath()))) { - printWriter.print(importSpecYaml); - } - - ImportJobPipelineOptions options = PipelineOptionsFactory.create().as(ImportJobPipelineOptions.class); - options.setImportSpecYamlFile(yamlFile.toString()); - - ImportSpec importSpec = new ImportSpecSupplier(options).get(); - System.out.println(JsonFormat.printer().omittingInsignificantWhitespace().print(importSpec)); - assertEquals(expectedImportSpec, importSpec); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java b/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java index d0707310548..b6f2d5ed984 100644 --- a/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java +++ b/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java @@ -18,26 +18,24 @@ package feast.ingestion.model; import static junit.framework.TestCase.assertTrue; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; -import com.google.common.collect.Lists; import com.google.common.io.Resources; -import feast.ingestion.service.FileSpecService; -import feast.specs.FeatureSpecProto.DataStore; -import feast.specs.FeatureSpecProto.DataStores; -import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.ingestion.config.ImportJobSpecsSupplier; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import org.junit.Before; import org.junit.Test; public class SpecsTest { - FileSpecService specService; + ImportJobSpecs importJobSpecs; private Field.Builder newField(String featureId) { return Field.newBuilder().setFeatureId(featureId); @@ -45,23 +43,23 @@ private Field.Builder newField(String featureId) { @Before public void before() { - Path path = Paths.get(Resources.getResource("core_specs/").getPath()); - specService = new FileSpecService(path.toString()); + Path path = Paths.get(Resources.getResource("specs/").getPath()); + importJobSpecs = new ImportJobSpecsSupplier(path.toString()).get(); } @Test public void testSingleFeatureAndEntity() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); + ).build(); - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); assertEquals("testjob", specs.getJobName()); - assertEquals(importSpec, specs.getImportSpec()); + assertEquals(importJobSpecs.getImportSpec(), specs.getImportSpec()); assertEquals(1, specs.getEntitySpecs().size()); assertTrue(specs.getEntitySpecs().containsKey("testEntity")); @@ -69,18 +67,18 @@ public void testSingleFeatureAndEntity() { assertEquals(1, specs.getFeatureSpecs().size()); assertTrue(specs.getFeatureSpecs().containsKey("testEntity.none.testInt32")); - assertTrue(specs.getStorageSpecs().containsKey("TEST_SERVING")); + assertTrue(specs.getServingStorageSpecs().containsKey("TEST_SERVING")); } @Test(expected = IllegalArgumentException.class) public void testErrorOnUnknownEntity() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); + ).build(); - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); specs.getEntitySpec("unknown"); @@ -88,13 +86,13 @@ public void testErrorOnUnknownEntity() { @Test(expected = IllegalArgumentException.class) public void testErrorOnUnknownFeature() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); + ).build(); - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); specs.getFeatureSpec("unknown"); @@ -102,13 +100,13 @@ public void testErrorOnUnknownFeature() { @Test public void testGetFeatureSpec() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); + ).build(); - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); assertEquals( @@ -117,13 +115,13 @@ public void testGetFeatureSpec() { @Test public void testGetEntitySpec() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); + ).build(); - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); assertEquals("testEntity", specs.getEntitySpec("testEntity").getName()); @@ -131,79 +129,28 @@ public void testGetEntitySpec() { @Test public void testGetStorageSpec() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); + ).build(); - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); - assertEquals("TEST_SERVING", specs.getStorageSpec("TEST_SERVING").getId()); - } - - @Test(expected = IllegalArgumentException.class) - public void testErrorOnUnknownStore() { - ImportSpec importSpec = - ImportSpec.newBuilder() - .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); - - Specs specs = Specs.of("testjob", importSpec, specService); - specs.validate(); - - specs.getStorageSpec("Unknown feature unknown, spec was not initialized"); + assertThat(specs.getWarehouseStorageSpecs().keySet(), containsInAnyOrder("TEST_WAREHOUSE")); + assertThat(specs.getWarehouseStorageSpecs().keySet(), containsInAnyOrder("TEST_WAREHOUSE")); } @Test(expected = IllegalArgumentException.class) public void testFeatureSpecReferencesUnknownEntity() { - ImportSpec importSpec = - ImportSpec.newBuilder() + ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() + .setImportSpec(ImportSpec.newBuilder() .addEntities("totally_different_entity") .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) - .build(); - - Specs specs = Specs.of("testjob", importSpec, specService); - specs.validate(); - } + ).build(); - @Test - public void testGetFeatureSpecByStoreId() { - ImportSpec importSpec = - ImportSpec.newBuilder() - .addEntities("testEntity") - .setSchema( - Schema.newBuilder() - .addAllFields( - Lists.newArrayList( - newField("testEntity.none.testInt32").build(), - newField("testEntity.none.testString").build()))) - .build(); - - Specs specs = Specs.of("testjob", importSpec, specService); + Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); - - FeatureSpec testStringSpec = specs.getFeatureSpec("testEntity.none.testString"); - DataStores dataStores = - testStringSpec - .getDataStores() - .toBuilder() - .setServing(DataStore.newBuilder().setId("differentStoreId")) - .build(); - testStringSpec = testStringSpec.toBuilder().setDataStores(dataStores).build(); - - // we change one of the specs to point at a different store id. - specs.getFeatureSpecs().put("testEntity.none.testString", testStringSpec); - - List featureSpecs1 = specs.getFeatureSpecByServingStoreId("TEST_SERVING"); - assertEquals(1, featureSpecs1.size()); - assertEquals("testEntity.none.testInt32", featureSpecs1.get(0).getId()); - - - List featureSpecs2 = specs.getFeatureSpecByServingStoreId("differentStoreId"); - assertEquals(1, featureSpecs2.size()); - assertEquals(testStringSpec, featureSpecs2.get(0)); } } diff --git a/ingestion/src/test/java/feast/ingestion/service/FileSpecServiceTest.java b/ingestion/src/test/java/feast/ingestion/service/FileSpecServiceTest.java deleted file mode 100644 index 023ecd55c2e..00000000000 --- a/ingestion/src/test/java/feast/ingestion/service/FileSpecServiceTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.service; - -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.assertThat; - -import com.google.common.collect.Lists; -import com.google.common.io.Resources; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import org.junit.Test; - -public class FileSpecServiceTest { - - static Path getLocalCoreSpecsPath() { - return Paths.get(Resources.getResource("core_specs/").getPath()); - } - - public FileSpecService getFileSpecService() { - return new FileSpecService(getLocalCoreSpecsPath().toString()); - } - - @Test - public void testGetSpecs() { - FileSpecService service = getFileSpecService(); - - List storageIds = Lists.newArrayList("TEST_WAREHOUSE", "TEST_SERVING"); - List entities = Lists.newArrayList("testEntity"); - List featureIds = - Lists.newArrayList( - "testEntity.hour.redisString", - "testEntity.none.testString", - "testEntity.none.redisString", - "testEntity.day.testInt64", - "testEntity.none.redisInt32", - "testEntity.none.testInt32", - "testEntity.none.testInt64"); - assertThat(service.getEntitySpecs(entities).keySet(), containsInAnyOrder(entities.toArray())); - assertThat( - service.getFeatureSpecs(featureIds).keySet(), containsInAnyOrder(featureIds.toArray())); - assertThat( - service.getStorageSpecs(storageIds).keySet(), containsInAnyOrder(storageIds.toArray())); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/service/MockSpecService.java b/ingestion/src/test/java/feast/ingestion/service/MockSpecService.java deleted file mode 100644 index 54e70b39a6d..00000000000 --- a/ingestion/src/test/java/feast/ingestion/service/MockSpecService.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.service; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.specs.StorageSpecProto.StorageSpec; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import org.testcontainers.shaded.com.google.common.base.Preconditions; - -public class MockSpecService implements SpecService { - - public Map entitySpecs = new HashMap<>(); - public Map featureSpecs = new HashMap<>(); - public Map storageSpecs = new HashMap<>(); - - public void checkValuesNotNull(Map map) { - for (Entry entry : map.entrySet()) { - Preconditions.checkNotNull(entry.getValue(), "not found for " + entry.getKey()); - } - } - - @Override - public Map getEntitySpecs(Iterable entityIds) { - Set entityIdsSet = Sets.newHashSet(entityIds); - Map map = Maps.newHashMap( - Maps.filterEntries(entitySpecs, (entry) -> entityIdsSet.contains(entry.getKey()))); - checkValuesNotNull(map); - return map; - } - - @Override - public Map getFeatureSpecs(Iterable featureIds) { - Set featureIdsSet = Sets.newHashSet(featureIds); - Map map = Maps.newHashMap( - Maps.filterEntries(featureSpecs, (entry) -> featureIdsSet.contains(entry.getKey()))); - checkValuesNotNull(map); - return map; - } - - @Override - public Map getStorageSpecs(Iterable storageIds) { - Set storageIdsSet = Sets.newHashSet(storageIds); - Map map = Maps.newHashMap( - Maps.filterEntries(storageSpecs, (entry) -> storageIdsSet.contains(entry.getKey()))); - checkValuesNotNull(map); - return map; - } - - public MockSpecService addStorage(StorageSpec storageSpec) { - storageSpecs.put(storageSpec.getId(), storageSpec); - return this; - } - - public MockSpecService addFeature(FeatureSpec featureSpec) { - featureSpecs.put(featureSpec.getId(), featureSpec); - return this; - } - - public MockSpecService addEntity(EntitySpec entitySpec) { - entitySpecs.put(entitySpec.getName(), entitySpec); - return this; - } -} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java index bddb79515b1..192002bfe9a 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java @@ -19,18 +19,26 @@ import static feast.ingestion.model.Errors.toError; import static feast.store.MockFeatureErrorsFactory.MOCK_ERRORS_STORE_TYPE; +import static feast.store.errors.logging.StderrFeatureErrorsFactory.TYPE_STDERR; +import static feast.store.errors.logging.StdoutFeatureErrorsFactory.TYPE_STDOUT; +import static org.junit.Assert.assertEquals; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; +import feast.specs.StorageSpecProto.StorageSpec; import feast.store.MockFeatureErrorsFactory; import feast.store.errors.FeatureErrorsFactoryService; -import feast.store.errors.logging.StderrFeatureErrorsFactory; -import feast.store.errors.logging.StdoutFeatureErrorsFactory; import feast.types.FeatureRowExtendedProto.Attempt; import feast.types.FeatureRowExtendedProto.Error; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.PAssert; @@ -43,22 +51,37 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +@Slf4j public class ErrorsStoreTransformTest { + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + @Rule public TestPipeline pipeline = TestPipeline.create(); private ImportJobPipelineOptions options; - private Specs specs; private PCollection inputs; private List errors; + public Specs getSpecs(StorageSpec errorsStorageSpec) { + return new Specs("test", ImportJobSpecs.newBuilder() + .setErrorsStorageSpec(errorsStorageSpec).build() + ); + } + + public Specs getSpecs(String errorsStorageType) { + return new Specs("test", ImportJobSpecs.newBuilder() + .setErrorsStorageSpec(StorageSpec.newBuilder().setType(errorsStorageType)).build() + ); + } + @Before public void setUp() { options = PipelineOptionsFactory.create().as(ImportJobPipelineOptions.class); options.setJobName("test"); - specs = Specs.builder().jobName("test").build(); errors = Arrays.asList( @@ -76,9 +99,9 @@ private FeatureRowExtended errorOf(String transform, Throwable cause) { @Test public void shouldWriteToGivenErrorsStore() { MockFeatureErrorsFactory mockStore = new MockFeatureErrorsFactory(); - options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, Lists.newArrayList(mockStore)); + new ErrorsStoreTransform(options, getSpecs(MOCK_ERRORS_STORE_TYPE), + Lists.newArrayList(mockStore)); transform.expand(inputs); PCollection writtenToErrors = @@ -91,9 +114,9 @@ public void shouldWriteToGivenErrorsStore() { @Test public void logErrorsToStdErr() { - options.setErrorsStoreType(StderrFeatureErrorsFactory.TYPE_STDERR); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); + new ErrorsStoreTransform(options, getSpecs(TYPE_STDERR), + FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -101,20 +124,33 @@ public void logErrorsToStdErr() { @Test public void logErrorsToStdOut() { - options.setErrorsStoreType(StdoutFeatureErrorsFactory.TYPE_STDOUT); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); + new ErrorsStoreTransform(options, getSpecs(TYPE_STDOUT), + FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @Test - public void logToNull() { - //options.setErrorsStoreType(...); // no errors store type set + public void logErrorsToWorkspace() throws IOException, InterruptedException { + String tempWorkspace = tempFolder.newFolder().toString(); + options.setWorkspace(tempWorkspace); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); + new ErrorsStoreTransform(options, getSpecs(""), FeatureErrorsFactoryService.getAll()); inputs.apply(transform); - pipeline.run(); + pipeline.run().waitUntilFinish(); + + int lineCount = Files.list(Paths.get(tempWorkspace) + .resolve("errors") // errors workspace dir + .resolve("test") // test entity dir + ).flatMap(path -> { + try { + return Files.readAllLines(path).stream(); + } catch (IOException e) { + throw new RuntimeException(); + } + }).collect(Collectors.toList()).size(); + assertEquals(2, lineCount); } } diff --git a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java deleted file mode 100644 index 183f117156b..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright 2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.transform; - -import static junit.framework.TestCase.assertNull; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.protobuf.Timestamp; -import feast.ingestion.model.Features; -import feast.ingestion.model.Specs; -import feast.ingestion.model.Values; -import feast.ingestion.service.MockSpecService; -import feast.ingestion.transform.SplitOutputByStore.WriteTags; -import feast.ingestion.values.PFeatureRows; -import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.DataStore; -import feast.specs.FeatureSpecProto.DataStores; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.specs.ImportSpecProto.Field; -import feast.specs.ImportSpecProto.ImportSpec; -import feast.specs.ImportSpecProto.Schema; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.store.FeatureStoreWrite; -import feast.store.FeatureStoreFactory; -import feast.store.MockFeatureStore; -import feast.store.MockTransforms; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.junit.Rule; -import org.junit.Test; - -public class SplitOutputByStoreTest { - - @Rule - public TestPipeline pipeline = TestPipeline.create(); - - @Test - public void testSplit() { - // Note we are stores on the group, instead of warehouse or serving store id. - SerializableFunction selector = (featureSpec) -> featureSpec - .getDataStores().getServing().getId(); - MockSpecService specService = new MockSpecService(); - specService - .addEntity(EntitySpec.newBuilder().setName("e1").build()) - .addFeature(FeatureSpec.newBuilder().setId("f1").setEntity("e1").setGroup("store1") - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store1"))).build()) - .addFeature(FeatureSpec.newBuilder().setId("f2").setEntity("e1").setGroup("store2") - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store2"))).build()) - .addStorage(StorageSpec.newBuilder().setId("store1").setType("type1").build()) - .addStorage(StorageSpec.newBuilder().setId("store2").setType("type2").build()); - List stores = - Lists.newArrayList(new MockFeatureStore("type1"), new MockFeatureStore("type2")); - Specs specs = - Specs.of( - "jobname", - ImportSpec.newBuilder() - .addEntities("e1") - .setSchema( - Schema.newBuilder() - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setFeatureId("f1").build(), - Field.newBuilder().setFeatureId("f2").build()))) - .build(), - specService); - assertNull(specs.getError()); - - SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); - - PCollection input = - pipeline - .apply( - Create.of( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .addFeatures(Features.of("f2", Values.ofInt32(2))) - .build())) - .apply(new ToFeatureRowExtended()); - PFeatureRows pfrows = PFeatureRows.of(input); - pfrows = pfrows.apply("do split", split); - - PAssert.that( - pfrows - .getErrors()).empty(); - PAssert.that( - pfrows - .getMain() - .apply( - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow))) - .containsInAnyOrder( - Lists.newArrayList( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build(), - FeatureRow.newBuilder() - .addFeatures(Features.of("f2", Values.ofInt32(2))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build())); - - MockTransforms.Write mockSpecService1 = ((MockFeatureStore) stores.get(0)).getWrite(); - MockTransforms.Write mockSpecService2 = ((MockFeatureStore) stores.get(1)).getWrite(); - - PCollection store1Output = - mockSpecService1 - .getInputs() - .get(0) - .apply("map store1 outputs", - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow)); - - PCollection store2Output = - mockSpecService2 - .getInputs() - .get(0) - .apply("map store2 outputs", - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow)); - - PAssert.that(store1Output) - .containsInAnyOrder( - Lists.newArrayList( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build())); - - PAssert.that(store2Output) - .containsInAnyOrder( - Lists.newArrayList( - FeatureRow.newBuilder() - .addFeatures(Features.of("f2", Values.ofInt32(2))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build())); - pipeline.run(); - } - - @Test - public void testSplitWhereFeature2HasNoStoreId() { - // Feature2 should get thrown away harmlessly - - SerializableFunction selector = (fs) -> fs.getDataStores().getServing() - .getId(); - MockSpecService specService = new MockSpecService(); - specService.entitySpecs.put("e1", EntitySpec.getDefaultInstance()); - specService.featureSpecs.put( - "f1", FeatureSpec.newBuilder().setEntity("e1").setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store1"))) - .build()); - - // f2 doesn't set a serving store, this is considered okay, this feature is ignored without error. - specService.featureSpecs.put( - "f2", FeatureSpec.newBuilder().setEntity("e1").build()); - specService.storageSpecs.put( - "store1", StorageSpec.newBuilder().setId("store1").setType("type1").build()); - specService.storageSpecs.put( - "store2", StorageSpec.newBuilder().setId("store2").setType("type2").build()); - List stores = - Lists.newArrayList(new MockFeatureStore("type1"), new MockFeatureStore("type2")); - Specs specs = - Specs.of( - "jobname", - ImportSpec.newBuilder() - .addEntities("e1") - .setSchema( - Schema.newBuilder() - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setFeatureId("f1").build(), - Field.newBuilder().setFeatureId("f2").build()))) - .build(), - specService); - assertNull(specs.getError()); - - SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); - - PCollection input = - pipeline - .apply( - Create.of( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .addFeatures(Features.of("f2", Values.ofInt32(2))) - .build())) - .apply(new ToFeatureRowExtended()); - PFeatureRows pfrows = PFeatureRows.of(input); - pfrows = pfrows.apply("do split", split); - - PAssert.that( - pfrows - .getErrors()).empty(); - PAssert.that( - pfrows - .getMain() - .apply( - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow))) - .containsInAnyOrder( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build(), - FeatureRow.newBuilder() - .addFeatures(Features.of("f2", Values.ofInt32(2))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build()); - - MockTransforms.Write mockSpecService1 = ((MockFeatureStore) stores.get(0)).getWrite(); - - PCollection store1Output = - mockSpecService1 - .getInputs() - .get(0) - .apply("map store1 outputs", - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow)); - - PAssert.that(store1Output) - .containsInAnyOrder( - Lists.newArrayList( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build())); - - pipeline.run(); - } - - - @Test - public void testSplitWhereNoStorageSpec() { - // There is no storage spec registered and feature 1 has no storage set - // feature1 should get thrown harmlessly - - SerializableFunction selector = (fs) -> fs.getDataStores().getServing() - .getId(); - MockSpecService specService = new MockSpecService(); - specService.entitySpecs.put("e1", EntitySpec.getDefaultInstance()); - specService.featureSpecs.put( - "f1", FeatureSpec.newBuilder().setEntity("e1") - .build()); - - Specs specs = - Specs.of( - "jobname", - ImportSpec.newBuilder() - .addEntities("e1") - .setSchema( - Schema.newBuilder() - .addAllFields( - Collections.singletonList( - Field.newBuilder().setFeatureId("f1").build()))) - .build(), - specService); - assertNull(specs.getError()); - List stores = Collections.emptyList(); - SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); - - PCollection input = - pipeline - .apply( - Create.of( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .build())) - .apply(new ToFeatureRowExtended()); - PFeatureRows pfrows = PFeatureRows.of(input); - pfrows = pfrows.apply("do split", split); - - PAssert.that( - pfrows - .getErrors()).empty(); - PAssert.that( - pfrows - .getMain() - .apply( - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow))) - .containsInAnyOrder( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build()); - - pipeline.run(); - } - - - @Test - public void testSplitWhereNoStorageSpecForAFeature() { - // There is no storage spec registered but feature 1 has storage set - // feature1 should get thrown harmlessly - - SerializableFunction selector = (fs) -> fs.getDataStores().getServing() - .getId(); - MockSpecService specService = new MockSpecService(); - specService.entitySpecs.put("e1", EntitySpec.getDefaultInstance()); - specService.featureSpecs.put( - "f1", FeatureSpec.newBuilder().setEntity("e1") - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store1") - .build())) - .build()); - - Specs specs = - Specs.of( - "jobname", - ImportSpec.newBuilder() - .addEntities("e1") - .setSchema( - Schema.newBuilder() - .addAllFields( - Collections.singletonList( - Field.newBuilder().setFeatureId("f1").build()))) - .build(), - specService); - assertNull(specs.getError()); - List stores = Collections.emptyList(); - SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); - - PCollection input = - pipeline - .apply( - Create.of( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .build())) - .apply(new ToFeatureRowExtended()); - PFeatureRows pfrows = PFeatureRows.of(input); - pfrows = pfrows.apply("do split", split); - - PAssert.that( - pfrows - .getErrors()).empty(); - PAssert.that( - pfrows - .getMain() - .apply( - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(FeatureRowExtended::getRow))) - .containsInAnyOrder( - FeatureRow.newBuilder() - .addFeatures(Features.of("f1", Values.ofInt32(1))) - .setEventTimestamp(Timestamp.getDefaultInstance()) - .build()); - - pipeline.run(); - } - - @Test - public void testWriteTags() { - TupleTag tag1 = new TupleTag<>("TAG1"); - TupleTag tag2 = new TupleTag<>("TAG2"); - TupleTag tag3 = new TupleTag<>("TAG3"); - TupleTag mainTag = new TupleTag<>("TAG4"); - - Map, FeatureStoreWrite> transforms = ImmutableMap., FeatureStoreWrite>builder() - .put(tag1, new MockTransforms.Write()) - .put(tag2, new MockTransforms.Write()) - // tag3 and mainTag do not have write transforms. - .build(); - - FeatureRowExtended rowex1 = FeatureRowExtended.newBuilder() - .setRow(FeatureRow.newBuilder().setEntityKey("1")).build(); - FeatureRowExtended rowex2 = FeatureRowExtended.newBuilder() - .setRow(FeatureRow.newBuilder().setEntityKey("2")).build(); - FeatureRowExtended rowex3 = FeatureRowExtended.newBuilder() - .setRow(FeatureRow.newBuilder().setEntityKey("3")).build(); - FeatureRowExtended rowex4 = FeatureRowExtended.newBuilder() - .setRow(FeatureRow.newBuilder().setEntityKey("4")).build(); - - PCollection pcollection1 = pipeline.apply("input1", Create.of(rowex1)); - PCollection pcollection2 = pipeline.apply("input2", Create.of(rowex2)); - PCollection pcollection3 = pipeline.apply("input3", Create.of(rowex3)); - PCollection pcollection4 = pipeline.apply("input4", Create.of(rowex4)); - - PCollectionTuple tuple = PCollectionTuple.of(tag1, pcollection1).and(tag2, pcollection2) - .and(tag3, pcollection3) - .and(mainTag, pcollection4); // input 3 is included in the output - - PCollection output = tuple.apply(new WriteTags(transforms, mainTag)); - - // All input should be in the main output. - // input 1 is returned in the output because we wrote it to a write transform. - // input 2 is returned in the output because we wrote it to a write transform. - // input 3 is NOT returned in the output because it has no write transform. - // input 4 is is returned in the output because it is the main tag. - PAssert.that(output).containsInAnyOrder(rowex1, rowex2, rowex4); - - // Each non main tagged input should be written to corresponding FeatureStoreWrite transform - PAssert.that(((MockTransforms.Write) transforms.get(tag1)).getInputs().get(0)) - .containsInAnyOrder(rowex1); - PAssert.that(((MockTransforms.Write) transforms.get(tag2)).getInputs().get(0)) - .containsInAnyOrder(rowex2); - pipeline.run(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java b/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java index f69753aad52..7723329d78a 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/fn/ConvertTypesDoFnTest.java @@ -18,8 +18,6 @@ package feast.ingestion.transform.fn; -import static feast.FeastMatchers.hasCount; - import com.google.common.collect.Lists; import feast.ingestion.model.Features; import feast.ingestion.model.Specs; @@ -27,17 +25,16 @@ import feast.ingestion.util.DateUtil; import feast.ingestion.values.PFeatureRows; import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.ImportJobSpecsProto; import feast.types.FeatureRowExtendedProto.Attempt; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import feast.types.ValueProto.ValueType.Enum; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.values.PCollection; import org.junit.Rule; import org.junit.Test; @@ -60,19 +57,20 @@ public void testStringTo() { Features.of("STRING_TO_TIMESTAMP", Values.ofString("2019-01-31T19:19:19.123Z")) ))).build(); - Map featureSpecs = new HashMap<>(); - featureSpecs.put("STRING_TO_INT32", FeatureSpec.newBuilder().setValueType(Enum.INT32).build()); - featureSpecs.put("STRING_TO_INT64", FeatureSpec.newBuilder().setValueType(Enum.INT64).build()); - featureSpecs.put("STRING_TO_FLOAT", FeatureSpec.newBuilder().setValueType(Enum.FLOAT).build()); - featureSpecs.put("STRING_TO_DOUBLE", FeatureSpec.newBuilder().setValueType(Enum.DOUBLE).build()); - featureSpecs.put("STRING_TO_BOOL", FeatureSpec.newBuilder().setValueType(Enum.BOOL).build()); - featureSpecs.put("STRING_TO_STRING", FeatureSpec.newBuilder().setValueType(Enum.STRING).build()); - featureSpecs.put("STRING_TO_TIMESTAMP", FeatureSpec.newBuilder().setValueType(Enum.TIMESTAMP).build()); - + List featureSpecs = Lists.newArrayList( + FeatureSpec.newBuilder().setId("STRING_TO_INT32").setValueType(Enum.INT32).build(), + FeatureSpec.newBuilder().setId("STRING_TO_INT64").setValueType(Enum.INT64).build(), + FeatureSpec.newBuilder().setId("STRING_TO_FLOAT").setValueType(Enum.FLOAT).build(), + FeatureSpec.newBuilder().setId("STRING_TO_DOUBLE").setValueType(Enum.DOUBLE).build(), + FeatureSpec.newBuilder().setId("STRING_TO_BOOL").setValueType(Enum.BOOL).build(), + FeatureSpec.newBuilder().setId("STRING_TO_STRING").setValueType(Enum.STRING).build(), + FeatureSpec.newBuilder().setId("STRING_TO_TIMESTAMP").setValueType(Enum.TIMESTAMP).build() + ); PFeatureRows output = PFeatureRows.of(pipeline.apply(Create.of(row))) .applyDoFn("name", - new ConvertTypesDoFn(Specs.builder().featureSpecs(featureSpecs).build())); - + new ConvertTypesDoFn( + new Specs("", ImportJobSpecsProto.ImportJobSpecs.newBuilder() + .addAllFeatureSpecs(featureSpecs).build()))); PAssert.that(output.getErrors()).satisfies(rows -> { if (rows.iterator().hasNext()) { diff --git a/ingestion/src/test/java/feast/ingestion/util/ProtoUtilTest.java b/ingestion/src/test/java/feast/ingestion/util/ProtoUtilTest.java new file mode 100644 index 00000000000..f99a432b822 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/util/ProtoUtilTest.java @@ -0,0 +1,66 @@ +package feast.ingestion.util; + +import static org.junit.Assert.assertEquals; + +import feast.DriverAreaProto.DriverArea; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import lombok.extern.slf4j.Slf4j; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +@Slf4j +public class ProtoUtilTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testDecodeProtoYaml() throws IOException { + + String yaml = "" + + "driverId: 1\n" + + "areaId: 2"; + + DriverArea da = ProtoUtil.decodeProtoYaml(yaml, DriverArea.getDefaultInstance()); + assertEquals(1, da.getDriverId()); + assertEquals(2, da.getAreaId()); + } + + @Test + public void testDecodeProtoYamlFile() throws IOException { + String yaml = "" + + "driverId: 1\n" + + "areaId: 2"; + Path path = tempFolder.newFile("file.yaml").toPath(); + try (BufferedWriter w = Files.newBufferedWriter(path)) { + w.write(yaml); + } + + DriverArea da = ProtoUtil.decodeProtoYamlFile(path, DriverArea.getDefaultInstance()); + assertEquals(1, da.getDriverId()); + assertEquals(2, da.getAreaId()); + } + + @Test + @Ignore + // Uncomment this during manual testing, as we don't want running unit tests to require GCS access + public void testDecodeProtoYamlFileGCS() throws IOException { + String yaml = "" + + "driverId: 1\n" + + "areaId: 2"; + Path path = PathUtil.getPath("gs://feast-temp/file.yaml"); + log.info(path.toUri().toString()); + try (BufferedWriter w = Files.newBufferedWriter(path)) { + w.write(yaml); + } + + DriverArea da = ProtoUtil.decodeProtoYamlFile(path, DriverArea.getDefaultInstance()); + assertEquals(1, da.getDriverId()); + assertEquals(2, da.getAreaId()); + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/store/MockFeatureStore.java b/ingestion/src/test/java/feast/store/MockFeatureStore.java index cdde9e97405..7a5ebe8ee59 100644 --- a/ingestion/src/test/java/feast/store/MockFeatureStore.java +++ b/ingestion/src/test/java/feast/store/MockFeatureStore.java @@ -24,7 +24,8 @@ public class MockFeatureStore implements FeatureStoreFactory { private final String type; - @Getter private MockTransforms.Write write; + @Getter + private MockTransforms.Write write; public MockFeatureStore(String type) { this.type = type; diff --git a/ingestion/src/test/java/feast/store/MockServingFactory.java b/ingestion/src/test/java/feast/store/MockServingFactory.java index 11195c1385e..125fdfdb79d 100644 --- a/ingestion/src/test/java/feast/store/MockServingFactory.java +++ b/ingestion/src/test/java/feast/store/MockServingFactory.java @@ -23,6 +23,7 @@ @AutoService(FeatureServingFactory.class) public class MockServingFactory extends MockFeatureStore implements FeatureServingFactory { + public static final String MOCK_SERVING_STORE_TYPE = "serving.mock"; public MockServingFactory() { diff --git a/ingestion/src/test/java/feast/store/MockWarehouseFactory.java b/ingestion/src/test/java/feast/store/MockWarehouseFactory.java index ddfd06556ab..05bf86ead98 100644 --- a/ingestion/src/test/java/feast/store/MockWarehouseFactory.java +++ b/ingestion/src/test/java/feast/store/MockWarehouseFactory.java @@ -23,6 +23,7 @@ @AutoService(FeatureWarehouseFactory.class) public class MockWarehouseFactory extends MockFeatureStore implements FeatureWarehouseFactory { + public static final String MOCK_WAREHOUSE_STORE_TYPE = "warehouse.mock"; public MockWarehouseFactory() { diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java index 3dd1b10112f..940778ffdbe 100644 --- a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java @@ -24,19 +24,19 @@ import com.google.common.io.Resources; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; +import feast.ingestion.config.ImportJobSpecsSupplier; import feast.ingestion.model.Features; import feast.ingestion.model.Specs; import feast.ingestion.model.Values; -import feast.ingestion.service.FileSpecService; -import feast.ingestion.service.SpecService; -import feast.store.FeatureStoreWrite; import feast.ingestion.util.DateUtil; +import feast.specs.ImportJobSpecsProto.ImportJobSpecs; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; +import feast.store.FeatureStoreWrite; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity; @@ -66,7 +66,7 @@ public class FeatureRowRedisIOWriteTest { private static int REDIS_PORT = 51234; private static Redis redis; private static Jedis jedis; - private static SpecService fileSpecService; + private static ImportJobSpecs importJobSpecs; @Rule public TestPipeline testPipeline = TestPipeline.create(); @@ -75,8 +75,8 @@ public class FeatureRowRedisIOWriteTest { public static void classSetup() throws IOException { redis = new RedisServer(REDIS_PORT); redis.start(); - Path path = Paths.get(Resources.getResource("core_specs/").getPath()); - fileSpecService = new FileSpecService.Builder(path.toString()).build(); + Path path = Paths.get(Resources.getResource("specs/").getPath()); + importJobSpecs = new ImportJobSpecsSupplier(path.toString()).get(); jedis = new Jedis("localhost", REDIS_PORT); } @@ -88,7 +88,7 @@ public static void teardown() { Specs getSpecs() { Specs specs = Specs.of( "test job", - ImportSpec.newBuilder() + importJobSpecs.toBuilder().setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") .setSchema( Schema.newBuilder() @@ -96,11 +96,13 @@ Specs getSpecs() { .addFields(Field.newBuilder().setFeatureId(featureHourString)) .addFields(Field.newBuilder().setFeatureId(featureNoneInt32)) .addFields(Field.newBuilder().setFeatureId(featureNoneString))) - .build(), - fileSpecService); - StorageSpec redisSpec = specs.getStorageSpec("REDIS1"); - specs.getStorageSpecs().put("REDIS1", - redisSpec.toBuilder().putOptions("port", String.valueOf(REDIS_PORT)).build()); + ).addServingStorageSpecs(StorageSpec.newBuilder() + .setId("REDIS1").setType("redis") + .putOptions("port", String.valueOf(REDIS_PORT)) + .putOptions("host", "localhost") + .putOptions("batchSize", "1") + .putOptions("timeout", "2000") + .build()).build()); specs.validate(); return specs; } @@ -110,7 +112,7 @@ public void testWriteNoneGranularity() throws IOException { Specs specs = getSpecs(); specs.validate(); - new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); + new RedisServingFactory().create(specs.getServingStorageSpecs().get("REDIS1"), specs); FeatureRowRedisIO.Write write = new FeatureRowRedisIO.Write( RedisStoreOptions.builder().host("localhost").port(REDIS_PORT).build(), specs); @@ -154,7 +156,8 @@ public void testWriteNoneGranularity() throws IOException { @Test public void testWriteNoneGranularityFromOptions() throws IOException { Specs specs = getSpecs(); - FeatureStoreWrite write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); + FeatureStoreWrite write = new RedisServingFactory() + .create(specs.getServingStorageSpecs().get("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() @@ -195,7 +198,8 @@ public void testWriteNoneGranularityFromOptions() throws IOException { @Test public void testWriteHourGranularity() throws IOException { Specs specs = getSpecs(); - FeatureStoreWrite write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); + FeatureStoreWrite write = new RedisServingFactory() + .create(specs.getServingStorageSpecs().get("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() diff --git a/ingestion/src/test/resources/core_specs/entity/testEntity.json b/ingestion/src/test/resources/core_specs/entity/testEntity.json deleted file mode 100644 index cf58a5326c0..00000000000 --- a/ingestion/src/test/resources/core_specs/entity/testEntity.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "testEntity", - "description": "This is a test entity", - "tags": [] -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.day.testInt64.json b/ingestion/src/test/resources/core_specs/feature/testEntity.day.testInt64.json deleted file mode 100644 index 18cc99e656c..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.day.testInt64.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "testEntity.day.testInt64", - "entity": "testEntity", - "granularity": "DAY", - "name": "testInt64", - "owner": "feast@example.com", - "description": "This is test feature of type integer", - "uri": "https://example.com/", - "valueType": "INT64", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "TEST_SERVING" - }, - "warehouse": { - "id": "TEST_WAREHOUSE" - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisInt32.json b/ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisInt32.json deleted file mode 100644 index 4c2c046c17b..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisInt32.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "id": "testEntity.hour.redisInt32", - "entity": "testEntity", - "granularity": "HOUR", - "name": "redisInt32", - "owner": "feast@example.com", - "description": "This is test feature of type integer that goes to redis", - "uri": "https://example.com/", - "valueType": "INT32", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "REDIS1", - "options": { - "expiry": "PT1H", - "bucketSize": "PT10H" - } - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisString.json b/ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisString.json deleted file mode 100644 index 98726d26bcd..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.hour.redisString.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "id": "testEntity.hour.redisString", - "entity": "testEntity", - "granularity": "HOUR", - "name": "redisString", - "owner": "feast@example.com", - "description": "This is test feature of type integer that goes to redis", - "uri": "https://example.com/", - "valueType": "STRING", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "REDIS1" - } - } -} diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.redisInt32.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.redisInt32.json deleted file mode 100644 index 0d62bbdaf7b..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.redisInt32.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "testEntity.none.redisInt32", - "entity": "testEntity", - "granularity": "NONE", - "name": "redisInt32", - "owner": "feast@example.com", - "description": "This is test feature of type integer the goes to redis", - "uri": "https://example.com/", - "valueType": "INT32", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "REDIS1", - "options": { - "expiry": "PT1H" - } - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.redisString.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.redisString.json deleted file mode 100644 index ed7467001df..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.redisString.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "id": "testEntity.none.redisString", - "entity": "testEntity", - "granularity": "NONE", - "name": "redisString", - "owner": "feast@example.com", - "description": "This is test feature of type integer that goes to redis", - "uri": "https://example.com/", - "valueType": "STRING", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "REDIS1" - } - } -} diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt32.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt32.json deleted file mode 100644 index 7a37876570b..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt32.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "testEntity.none.testInt32", - "entity": "testEntity", - "granularity": "NONE", - "name": "testInt32", - "owner": "feast@example.com", - "description": "This is test feature of type integer", - "uri": "https://example.com/", - "valueType": "INT32", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "TEST_SERVING" - }, - "warehouse": { - "id": "TEST_WAREHOUSE" - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64.json deleted file mode 100644 index 025dbad01b1..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "testEntity.none.testInt64", - "entity": "testEntity", - "granularity": "NONE", - "name": "testInt64", - "owner": "feast@example.com", - "description": "This is test feature of type integer", - "uri": "https://example.com/", - "valueType": "INT64", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "TEST_SERVING" - }, - "warehouse": { - "id": "TEST_WAREHOUSE" - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64NoWarehouse.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64NoWarehouse.json deleted file mode 100644 index e12d5fdcb49..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testInt64NoWarehouse.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "id": "testEntity.none.testInt64NoWarehouse", - "entity": "testEntity", - "granularity": "NONE", - "name": "testInt64NoWarehouse", - "owner": "feast@example.com", - "description": "This is test feature of type integer", - "uri": "https://example.com/", - "valueType": "INT64", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "TEST_SERVING" - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testString.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.testString.json deleted file mode 100644 index 8c820d3a2ea..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testString.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id": "testEntity.none.testString", - "entity": "testEntity", - "granularity": "NONE", - "name": "testString", - "owner": "feast@example.com", - "description": "This is test feature of type integer", - "uri": "https://example.com/", - "valueType": "STRING", - "dataStores": { - "serving": { - "id": "TEST_SERVING" - }, - "warehouse": { - "id": "TEST_WAREHOUSE" - } - } -} diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testStringNoWarehouse.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.testStringNoWarehouse.json deleted file mode 100644 index 2b7f7f87e49..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.testStringNoWarehouse.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "testEntity.none.testStringNoWarehouse", - "entity": "testEntity", - "granularity": "NONE", - "name": "testStringNoWarehouse", - "owner": "feast@example.com", - "description": "This is test feature of type integer", - "uri": "https://example.com/", - "valueType": "STRING", - "dataStores": { - "serving": { - "id": "TEST_SERVING" - } - } -} diff --git a/ingestion/src/test/resources/core_specs/feature/testEntity.none.unknownInt32.json b/ingestion/src/test/resources/core_specs/feature/testEntity.none.unknownInt32.json deleted file mode 100644 index 991efc2042f..00000000000 --- a/ingestion/src/test/resources/core_specs/feature/testEntity.none.unknownInt32.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id": "testEntity.none.unknownInt32", - "entity": "testEntity", - "granularity": "NONE", - "name": "unknownInt32", - "owner": "feast@example.com", - "description": "This is test feature of type integer that goes to an unknown serving store", - "uri": "https://example.com/", - "valueType": "INT32", - "tags": [], - "options": {}, - "dataStores": { - "serving": { - "id": "UNKNOWN1", - "options": {} - } - } -} \ No newline at end of file diff --git a/ingestion/src/test/resources/core_specs/storage/REDIS1.json b/ingestion/src/test/resources/core_specs/storage/REDIS1.json deleted file mode 100644 index da5d6aebf97..00000000000 --- a/ingestion/src/test/resources/core_specs/storage/REDIS1.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "id": "REDIS1", - "type": "redis", - "options": { - "host": "localhost", - "batchSize": "1", - "timeout": "2000" - } -} diff --git a/ingestion/src/test/resources/core_specs/storage/TEST_SERVING.json b/ingestion/src/test/resources/core_specs/storage/TEST_SERVING.json deleted file mode 100644 index a8520839d61..00000000000 --- a/ingestion/src/test/resources/core_specs/storage/TEST_SERVING.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "TEST_SERVING", - "type": "serving.mock", - "options": {} -} diff --git a/ingestion/src/test/resources/core_specs/storage/TEST_WAREHOUSE.json b/ingestion/src/test/resources/core_specs/storage/TEST_WAREHOUSE.json deleted file mode 100644 index 13e57513983..00000000000 --- a/ingestion/src/test/resources/core_specs/storage/TEST_WAREHOUSE.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "TEST_WAREHOUSE", - "type": "warehouse.mock", - "options": {} -} \ No newline at end of file diff --git a/ingestion/src/test/resources/specs/importJobSpecs.yaml b/ingestion/src/test/resources/specs/importJobSpecs.yaml new file mode 100644 index 00000000000..cbc48b95582 --- /dev/null +++ b/ingestion/src/test/resources/specs/importJobSpecs.yaml @@ -0,0 +1,172 @@ +importSpec: {} +servingStorageSpecs: + - id: TEST_SERVING + type: serving.mock + options: {} +warehouseStorageSpecs: + - id: TEST_WAREHOUSE + type: warehouse.mock + options: {} +errorsStorageSpec: + id: errors + type: errors.mock + options: {} +entitySpecs: + - name: testEntity + description: This is a test entity + tags: [] +featureSpecs: + - id: testEntity.day.testInt64 + entity: testEntity + granularity: DAY + name: testInt64 + owner: feast@example.com + description: This is test feature of type integer + uri: https://example.com/ + valueType: INT64 + tags: [] + options: {} + dataStores: + serving: + id: TEST_SERVING + warehouse: + id: TEST_WAREHOUSE + - id: testEntity.hour.redisInt32 + entity: testEntity + granularity: HOUR + name: redisInt32 + owner: feast@example.com + description: This is test feature of type integer that goes to redis + uri: https://example.com/ + valueType: INT32 + tags: [] + options: {} + dataStores: + serving: + id: REDIS1 + options: + expiry: PT1H + bucketSize: PT10H + - id: testEntity.hour.redisString + entity: testEntity + granularity: HOUR + name: redisString + owner: feast@example.com + description: This is test feature of type integer that goes to redis + uri: https://example.com/ + valueType: STRING + tags: [] + options: {} + dataStores: + serving: + id: REDIS1 + - id: testEntity.none.redisInt32 + entity: testEntity + granularity: NONE + name: redisInt32 + owner: feast@example.com + description: This is test feature of type integer the goes to redis + uri: https://example.com/ + valueType: INT32 + tags: [] + options: {} + dataStores: + serving: + id: REDIS1 + options: + expiry: PT1H + - id: testEntity.none.redisString + entity: testEntity + granularity: NONE + name: redisString + owner: feast@example.com + description: This is test feature of type integer that goes to redis + uri: https://example.com/ + valueType: STRING + tags: [] + options: {} + dataStores: + serving: + id: REDIS1 + - id: testEntity.none.testInt32 + entity: testEntity + granularity: NONE + name: testInt32 + owner: feast@example.com + description: This is test feature of type integer + uri: https://example.com/ + valueType: INT32 + tags: [] + options: {} + dataStores: + serving: + id: TEST_SERVING + warehouse: + id: TEST_WAREHOUSE + - id: testEntity.none.testInt64 + entity: testEntity + granularity: NONE + name: testInt64 + owner: feast@example.com + description: This is test feature of type integer + uri: https://example.com/ + valueType: INT64 + tags: [] + options: {} + dataStores: + serving: + id: TEST_SERVING + warehouse: + id: TEST_WAREHOUSE + - id: testEntity.none.testInt64NoWarehouse + entity: testEntity + granularity: NONE + name: testInt64NoWarehouse + owner: feast@example.com + description: This is test feature of type integer + uri: https://example.com/ + valueType: INT64 + tags: [] + options: {} + dataStores: + serving: + id: TEST_SERVING + - id: testEntity.none.testString + entity: testEntity + granularity: NONE + name: testString + owner: feast@example.com + description: This is test feature of type integer + uri: https://example.com/ + valueType: STRING + dataStores: + serving: + id: TEST_SERVING + warehouse: + id: TEST_WAREHOUSE + - id: testEntity.none.testStringNoWarehouse + entity: testEntity + granularity: NONE + name: testStringNoWarehouse + owner: feast@example.com + description: This is test feature of type integer + uri: https://example.com/ + valueType: STRING + dataStores: + serving: + id: TEST_SERVING + - id: testEntity.none.unknownInt32 + entity: testEntity + granularity: NONE + name: unknownInt32 + owner: feast@example.com + description: This is test feature of type integer that goes to an unknown serving + store + uri: https://example.com/ + valueType: INT32 + tags: [] + options: {} + dataStores: + serving: + id: UNKNOWN1 + options: {} diff --git a/pom.xml b/pom.xml index 728c78fa852..37c3df33114 100644 --- a/pom.xml +++ b/pom.xml @@ -120,6 +120,7 @@ compile compile-custom + test-compile diff --git a/protos/feast/specs/ImportJobSpecs.proto b/protos/feast/specs/ImportJobSpecs.proto new file mode 100644 index 00000000000..fea7c7decec --- /dev/null +++ b/protos/feast/specs/ImportJobSpecs.proto @@ -0,0 +1,38 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package feast.specs; + +import "feast/specs/ImportSpec.proto"; +import "feast/specs/EntitySpec.proto"; +import "feast/specs/FeatureSpec.proto"; +import "feast/specs/StorageSpec.proto"; + +option java_package = "feast.specs"; +option java_outer_classname = "ImportJobSpecsProto"; +option go_package = "github.com/gojek/feast/protos/generated/go/feast/specs"; + +message ImportJobSpecs { + string jobId = 1; + feast.specs.ImportSpec importSpec = 2; + repeated feast.specs.EntitySpec entitySpecs = 3; + repeated feast.specs.FeatureSpec featureSpecs = 4; + repeated StorageSpec servingStorageSpecs = 5; + repeated StorageSpec warehouseStorageSpecs = 6; + StorageSpec errorsStorageSpec = 7; +} From cf45d0f17fd7b63f20020d4b8f305d96486b3b15 Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Fri, 22 Mar 2019 20:51:25 +0800 Subject: [PATCH 17/25] Remove granularity (#162) * Remove granularity from proto files * Remove granularity from serving * Remove granularity from proto files * Remove granularity from serving * Remove granularity from core * Remove granularity from Python SDK * Update generated protos, remove granularity * Remove granularity from ingestion * Scrub additional mentions of granularity * Update core-deploy template * Update chart distribution * Fix core service not overriding storage correctly * Shade core jar to include google nio lib * Remove granularity from UI * Update feature details page * Update Importer in Feast Python SDK to handle source options properly When the import source type is DataFrame --- charts/dist/feast-0.1.0.tgz | Bin 30628 -> 30604 bytes charts/dist/index.yaml | 6 +- charts/feast/templates/core-deploy.yaml | 2 + cli/feast/pkg/parse/yaml.go | 5 - cli/feast/pkg/parse/yaml_test.go | 10 +- cli/feast/pkg/printer/printer_test.go | 10 +- core/pom.xml | 45 ++- .../java/feast/core/grpc/CoreServiceImpl.java | 2 +- .../feast/core/job/StatsdMetricPusher.java | 2 +- .../java/feast/core/model/FeatureInfo.java | 12 +- .../core/storage/BigQueryStorageManager.java | 3 +- .../core/storage/PostgresStorageManager.java | 3 +- .../training/BigQueryDatasetTemplater.java | 102 +---- .../feast/core/validators/SpecValidator.java | 11 +- .../main/resources/templates/bq_training.tmpl | 41 +- .../java/feast/core/CoreApplicationTest.java | 8 +- .../CoreApplicationWithNoServingTest.java | 8 +- .../CoreApplicationWithNoWarehouseTest.java | 8 +- .../core/grpc/DatasetServiceImplTest.java | 8 +- .../feast/core/grpc/UIServiceImplTest.java | 6 +- .../feast/core/model/FeatureInfoTest.java | 22 +- .../feast/core/service/SpecServiceTest.java | 16 +- .../storage/BigQueryStorageManagerTest.java | 29 +- .../training/BigQueryDatasetCreatorTest.java | 6 +- .../BigQueryDatasetTemplaterTest.java | 90 ++--- .../core/validators/SpecValidatorTest.java | 62 +-- core/src/test/resources/sql/expQuery1.sql | 66 +--- core/src/test/resources/sql/expQuery2.sql | 62 +-- core/src/test/resources/sql/expQuery3.sql | 150 ------- .../feature/product.day.completed_orders.json | 3 +- .../core_specs/feature/user.none.age.json | 3 +- .../feature/user.none.completed_orders.json | 3 +- ingestion/example/import_products.yaml | 2 +- ingestion/example/import_users.yaml | 4 +- .../main/java/feast/ingestion/ImportJob.java | 14 +- .../feast/ingestion/metrics/FeastMetrics.java | 22 +- .../transform/CoalesceFeatureRows.java | 91 +++-- .../transform/fn/ConvertTypesDoFn.java | 2 - .../fn/RoundEventTimestampsDoFn.java | 37 -- .../transform/fn/SplitFeaturesDoFn.java | 1 - .../transform/fn/ValidateFeatureRowsDoFn.java | 11 - .../java/feast/ingestion/util/DateUtil.java | 45 +-- .../FeatureRowToBigTableMutationDoFn.java | 10 +- .../redis/FeatureRowToRedisMutationDoFn.java | 12 +- .../bigquery/FeatureRowBigQueryIO.java | 33 +- .../feast_ingestion/types/CoalesceAccum.proto | 2 - .../feast/ingestion/ImportJobCSVTest.java | 89 ++--- .../config/ImportJobSpecsSupplierTest.java | 90 ++--- .../java/feast/ingestion/model/SpecsTest.java | 18 +- .../fn/FilterFeatureRowDoFnTest.java | 8 +- .../feast/ingestion/util/DateUtilTest.java | 60 --- .../source/bigquery/BQToFeatureRowFnTest.java | 9 +- .../redis/FeatureRowRedisIOWriteTest.java | 79 +--- .../resources/import-specs/csv_to_store1.yaml | 4 +- .../test/resources/specs/importJobSpecs.yaml | 56 +-- protos/feast/specs/FeatureSpec.proto | 2 - protos/feast/storage/BigTable.proto | 1 - protos/feast/types/FeatureRow.proto | 3 - protos/feast/types/Granularity.proto | 33 -- .../generated/go/feast/core/CoreService.pb.go | 32 +- .../go/feast/core/DatasetService.pb.go | 14 +- .../generated/go/feast/core/JobService.pb.go | 22 +- .../generated/go/feast/core/UIService.pb.go | 38 +- .../generated/go/feast/serving/Serving.pb.go | 12 +- .../generated/go/feast/specs/EntitySpec.pb.go | 6 +- .../go/feast/specs/FeatureGroupSpec.pb.go | 6 +- .../go/feast/specs/FeatureSpec.pb.go | 111 +++--- .../go/feast/specs/ImportJobSpecs.pb.go | 137 +++++++ .../generated/go/feast/specs/ImportSpec.pb.go | 10 +- .../go/feast/specs/StorageSpec.pb.go | 6 +- .../generated/go/feast/storage/BigTable.pb.go | 6 +- protos/generated/go/feast/storage/Redis.pb.go | 10 +- protos/generated/go/feast/types/Feature.pb.go | 6 +- .../generated/go/feast/types/FeatureRow.pb.go | 67 ++-- .../go/feast/types/FeatureRowExtended.pb.go | 10 +- .../go/feast/types/Granularity.pb.go | 8 +- protos/generated/go/feast/types/Value.pb.go | 28 +- rfcs/0001-python-sdk.md | 5 +- .../examples/quickstart/Quickstart.ipynb | 2 +- sdk/python/feast/sdk/client.py | 113 +++--- sdk/python/feast/sdk/env.py | 7 +- sdk/python/feast/sdk/importer.py | 117 +++--- sdk/python/feast/sdk/resources/entity.py | 22 +- sdk/python/feast/sdk/resources/feature.py | 89 ++--- .../feast/sdk/resources/feature_group.py | 21 +- sdk/python/feast/sdk/resources/feature_set.py | 6 +- sdk/python/feast/sdk/resources/storage.py | 11 +- sdk/python/feast/sdk/utils/bq_util.py | 24 +- sdk/python/feast/sdk/utils/gs_utils.py | 6 +- sdk/python/feast/sdk/utils/print_utils.py | 8 +- sdk/python/feast/sdk/utils/types.py | 6 +- sdk/python/feast/specs/FeatureSpec_pb2.py | 45 +-- sdk/python/feast/specs/ImportJobSpecs_pb2.py | 123 ++++++ sdk/python/feast/types/FeatureRow_pb2.py | 29 +- sdk/python/feast/types/Granularity_pb2.py | 99 ----- sdk/python/tests/sample/valid_feature.yaml | 3 +- sdk/python/tests/sdk/resources/test_entity.py | 13 +- .../tests/sdk/resources/test_feature.py | 42 +- .../tests/sdk/resources/test_feature_group.py | 11 +- .../tests/sdk/resources/test_feature_set.py | 16 +- .../tests/sdk/resources/test_storage.py | 22 +- sdk/python/tests/sdk/test_client.py | 371 +++++++++--------- sdk/python/tests/sdk/test_importer.py | 261 ++++++------ sdk/python/tests/sdk/utils/test_bq_utils.py | 84 ++-- sdk/python/tests/sdk/utils/test_gs_utils.py | 9 +- sdk/python/tests/sdk/utils/test_types.py | 29 +- .../service/BigTableFeatureStorage.java | 2 +- .../feast/serving/grpc/FeastServingTest.java | 4 +- .../serving/grpc/ServingGrpcServiceTest.java | 4 +- .../BigTableFeatureStorageTestITCase.java | 51 +-- .../serving/service/CoreServiceTest.java | 58 +-- .../FeatureRetrievalDispatcherTest.java | 10 +- .../service/RedisFeatureStorageTest.java | 35 +- .../serving/testutil/BigTablePopulator.java | 4 +- .../serving/testutil/FakeSpecStorage.java | 7 +- .../serving/testutil/RedisPopulator.java | 3 +- .../java/feast/serving/testutil/TimeUtil.java | 60 --- .../serving/util/EntityMapBuilderTest.java | 2 +- ui/src/components/Feature/SearchTable.vue | 18 +- ui/src/components/Job/SearchTable.vue | 18 +- ui/src/filters.js | 6 - ui/src/router.js | 5 - ui/src/views/Feature/Details.vue | 28 +- 123 files changed, 1606 insertions(+), 2374 deletions(-) delete mode 100644 core/src/test/resources/sql/expQuery3.sql delete mode 100644 ingestion/src/main/java/feast/ingestion/transform/fn/RoundEventTimestampsDoFn.java delete mode 100644 protos/feast/types/Granularity.proto create mode 100644 protos/generated/go/feast/specs/ImportJobSpecs.pb.go create mode 100644 sdk/python/feast/specs/ImportJobSpecs_pb2.py delete mode 100644 sdk/python/feast/types/Granularity_pb2.py delete mode 100644 serving/src/test/java/feast/serving/testutil/TimeUtil.java diff --git a/charts/dist/feast-0.1.0.tgz b/charts/dist/feast-0.1.0.tgz index 2ba968c8951f30021f93052446db3adee8c0176b..73ccb939713631a74e6f5489eb533fc15630cb27 100644 GIT binary patch delta 29728 zcmV)*K#9Mk?*WYO0gya@dw1KmmMA`d^Hc0A9rsdl5k9sECJfliSzsHci}|>eCT02Zqu4;wXsM%Hh_(djr|;9L|JElf>P$q&@}wF zukduc-R|!8w)nT(?Uw)T_IC1b@_lc!x6}JK_!_WOo{Tb-{#&8gCB4>1)rk}uoy^0845%Ao?{Wj5nN?MoJN>o z>O^RY`ylYkxwhN+Yig4P81CnB&orz!KLoBE{?L+JH z7g-diX`G&avpB^UvxK)oSrBKe{PCnxaT-T_DM>W$Yq)Vr;-C-P-L8h6M1&C(9^eqo z`mocLUs#GrM56HlVu)gnKqK<053Ld9O}8P8#}b(TK0i5joHk_wOWO({D*gmuHo$Fw z8 zXiW!`n2H8G&*M$o=ic!#q-sUT{+X6yQ z<2cOGd5S(A1$?LUc8WLWlVJlBf7X)prj~>>(VJeW@a8u~Zx9~iRPJ*eaX5lLj8I6i zbYxwM5;hE!5h8PX_SoqfLLo1AZ6pC%?f8=w~+uZ)c zv#OH+V|bc`h+*2foZv9QDfL+LV65N*{%?D;`?AFUZEbIFKJkB#@qGT=c>&jC+6Rgm zj7W(2QTBSur^TP(KD_9B`Qq?W&f%v7MS<9#eEUH>5`T-!py#M^E(CZ)BCHNekC$wL zTYN)V$iR~VhhyH1_mjMQfAEqg*Emf{fZ>)OOpy;eFU6l^dY+9&@Ss2dX*9?VZL~sBoNd(-U^D~yMi}H;3!N)*hh<%h%3^bl%_%Xl9%F&Jp z4g(4(#qt=y0Vi2(LMW^aXJRG}j?Z~nLFp3guY1nX2vQtkMAek}e{sYRiKx0ki+6;< z4GBXSV#p|mlL8XJD`*=gmY1+6myGhc)F1Qt)MXkAEH&XFEn0defbYLveoYNH^Ldc1 zcxQ5wWaS)I>+Q+_CMk*72wH!k?q6uD)SB#f6)bD|yM%pZtmAi^M@PE`l;Nue@ZP_Bat6wK@IkK>vaeyx@yG2qWRW>8uwWBR_);Ky5tL%S#g!W zmz7rFti0H&-eat{2JxF&@<05!Qd~l74wnkKH;T9ZK%`wR`db{~lrLj3NPQ>at?}^p)vcb~+C;jV0&onG&`0cr8l_@}y+iC`oGDwweX-<7pB{IAVRs)d+%)$&L(~_2r9OylJLf zs$F%wR>9Q;CO9Qb$SBPc=Ssr$<%`3Q>V6?sq_UvNFPQ>I*(_!J)EUp@6iA`+nJFHW zuvr2_L!2rlhLvv2;+`(2$5E9nF05~qmaXVDlCYR4ey}$nf>Lk7SmG7(U$arjMXxPhs&$v<$7$j(Zl0Ganh(h}jxeRC zX*|T{f1o%NZ!zmzO(cj-`p`+VnBSa|I}e-s6U-s|;o{=d_+aX`&aody0qsL?vuhNQ z8nNP|`Hg12k14&Fq?k_PFep?6P-sN*s@ui}Kml2KMkW}AZ1S%^?xbjSjZr*T#8`j} zZA_IOtNNa<h0Nq;!F|l`Lk#>p0A&_&tAJ zq9;OMb1G1))xpf~jQudqf=)@}LR)nk+yL3`TTow%4uV4XlPH{NjgvVHevvQ$#?iG6 zr|j|V+3Eh}>B-rJ)vmC7siEu5)T&j}_Q9LWA5P9|TmJmnxJY?_p&q^VJwMDS!>K2< ze@?As?u88;Qw<=?4AHu-%7G?xM|Zk+e*W{x*+I48@6&i{V>76$@{f3ShDYUZwQOVk zu*=vETCt~#y9uK7CQgGIpx>Rmy}bDQ>0!f~Sc|Pp4eF=#^z7un5BD!Fk7@>AgE+Yj zP5oS6ob4SQA05BFJlMO~J3qX*JUu)7fBxuymTJ1zVxnQq{`=(3<=Kbh@9gl0lMm;I%TGbx?!rkZ8ee4^iuNC9BBaUGJU|c6&Q8wGm!AUJ;==K(W>*=! zYWZ#>ue>b4*}FJDn0HJUP#{JFe{T{~Hn*u|4qDY#&EZ;FCVpWah4D?pvA%b)_x-z* zpD*VhMn=cFz5F~nSkUsjll?uuM$5L#Sr@)QODw_Ctfs-Not(UVcX+vXdiw52!gaZS zc6e}j%!zC*8qH%sJ6Cw-{gcL#swi1ngPV2l#Fa`m$7XN}>Hpl;u5cZJf0r}&sPeyB z`LE^)A4mYa!2V~m`*O1^|J~Z%eUkq^##8eDkn&z7#r*vVdw&;#T>unCF+AI2_Om{p^6Mu=0g(P9sj%$Ub6o=Snae8k>$=tp0TT`Z|=P{o0 z`L8Zu52^xQaQ^RZ^|ngq|4w)JN&ovO&sWj^+U94Er~)>fJANO`f5KcXu%4%{uL#b2 zy1Op8v56Jc!G4Mb>vTb;m@+g?`VeJdsJ=dove?9frPlGz`r=wtmrsq#SeKrwH7*xw zP#oLER?!{XH6K8ItQ%ZO#hbHDtGB*XY<3vsDV zHDhA7l3mQo%bAlu}$s<(s{W{U28E(Dq}Vf7{{N;ln*#_xAAkVlLTeXkAU(!lqXt zd#dR>W5PP({}O!18~!Wp zi2v&slW~XtPyeLXQKx8Vi2SQ85wUNm^TMMyiSxo6qATZx$EJz%;{SKr@DoZ&oKZMB zIHZoYsdGFMVAPT2()gd>B2kq4KgGXhB*jx4e=+KXv48cTeJtMpyQTgAa_8k!{HI5G z0-WF|z>!Zl>!!Jv!XLx2W9V_FgZ4X}BF?WXIC5#bi^f=Sz$hVpB4$W6Bws3EJG=>HEX2SKZ#$=JwXh&EfXe7TVt0+Sv?tws!+G+8&LD zf3Jpq?-d?yk2bf5FTca1?d_c|+Sx&P>$}};Jaon?5=794&2H~Ix4Y?fcQ1O~{`OXX zd)M3TZEtH&fD=e?STI$LlFsC^7L_pLI@B;Lqsu5 zQG~|g$N)N{3{PS09n>Y;w7i3Vs;)mnrgLhou-zuasw;XVDV8(<^vZ0U&7F>jg_ZnEibusJ%G-wSWhj6Mf4%44Ggq3~R2vDots88D7w5m@@U88!U51*&z~fAC1mcOUI! z6sPVuBLVJ+)@i4mq}WX|VfyIGcO3L~#n8@zJ^v`OUs^9*Lk0cI>`BJeNH`ub*>kCV7kb{*sH4fB!yE6-pQu5f%&q zrQ=N8-ZlVxV-H-Hx9|EB>|fDrT1C9(^F|=yFpP(tDWVLg#fIfW#}V_UL0iAOl>J|# zbX@JD<{Xw7o~0>{Scfvih~Gg;;>u>c37dw-kdMUMy>v|b`Ff%VP$-mW3=7M_APaG- zhTWnpPOn@=$NKlHVf0*Hv${?Zv@!1Aslwx7D3&Y0Tlw6g6O`iMO*R_gl(zcN+U|DWziEBBD~2d|TxEzW zDJ0{|$q)%ECTp_=F+?$wQ~=whAQ}HKM7-fX+`1y8fUwa66w3ctno*f)K-l5v*LfC! za)Ifr;G1WZfA4JQeYffLc3$z@ZfA1~ropxt!6`paDHHU8BMO2`*=#1N4X;uKm}7$+^q0cf{j zk8}MQ3d5OV{{sy#hwdjywmE0NA0jf9b~|fI={f(3JQ^P8O+tm91rwf&kX!VIxp?ov(}PD9XZc z1ERRnf0=M;0HVeMy!;0ElQ^cu(IunO<~`%o16Ug;n;aGPem>vGZ)YH+Hlm%kKOS!I zkMQqslHll^A^&P!d=!&!g3;&G-~b?Di2Xhea39Vv3Vu!r!zYo?7oLg~Ft6Sc{&zN1 z(WDBR;08n9gb`}0bBzQuY2H@&qIQxo|3|F1e`THlpT9WH6tk50)Y>jOl9Xa|Z_+r! zG{Rxt@+JP1#3>Ud83mJi-SxJ-9jC^}qLiGbWJzAOAO_{LSOxz_}_NYPn$L_+Cj)K;CoFD8+{ ze*{i6gPbkVF9AfJr9QOI58oZ`UqIjmh>7?Kdm@lE-hlnR^F#Rghr?sugpU{sbz{>kz2;r_+*b+{;1HbA-LNL5r@hr@U0htP$?;{y%>{|euq zoxE?T4guUua4JZJfSR8YB!ac)+K!_8e|!V-A3gp?Z z-tE0PeAlXcw!|{6VI_oO=WMX>V_(|lcTdGN%-FD(}&aD<2b;==0-?KO2?eG&yl_8qJk<#>PO#kBFs1gr!S4736(-g zJIL`VKQ@aM1jW+?x15Jbv~w}RplVFx0EdE&NRn{o!Os&M>7zqb*^n8VIHxGZb|P4K zoTk`kvNb;>{G1fFdh)at_aij`e?{D7%z7*)5>Dl2IR8KI_?0{q2W@_;#gT%7pZptU zSfsgO|;$m4W3r%M^<%z*x^G358*NbEsBTr#x%7#o8zv zM1iC%4zXn2sbn1E01%~QEN+~u-rdE15Uq7&Bb*&~ISO>z{Qufi>ml3ae>YP`T@(eb zF&KA<8FEJ$u`I<7FL|!b#Q520loa@mMk9V5o5_Q-4S6C3%`j+ZI{5d(iHvg*Lrfj{ z4@YQSO76{a4nS$?R_lZ{LL|&mY&dT1IEK4gU3=TL&ApTcc=lCio6q-N!5)Z}mh6yr zau&!SmdJC8w|ga)B8h|Ye=B*t$eR(HO531nkKQ}p@%V)%jYDw-Enw=ji)IU-VGhc& z^1mkeZ($LBd~$Jk?y*no&N9XY`k$?xiu`x?<@U=b`R`*q2L1Kry-yN!EM;+mp7V2& zUkUgw8O5BK@+*YON`fVQ=H!?}FpIMklpv+=EEnU`YqAip`p7xuf4no*JQ+tu?m^DD zp^#IKS$>VQL~KI(lgIJm+5H08g^bw0lpMW%PkYQG5nLp zQG(dyb?dW;1m{%-fAFORx4;?1oh@+P0Q+%({|eZj#Osb>u+0aj&Unr`h%x4@Wtx?7 z&5x&3PL$WO{Pv2Y>s$t`ScGAGjo0mE6uXv1wqOWD)CPJ`0{*R_kDVDR_>c#`(KS9KL6~|1WP{>0g=Azagg*Z3d(Lx zWqt`=ZZ3P9yI$ApdOd5D=KkPkb+s++ULvct;qVjoe=~;bsjRrnrLq2AT*Z{&O3IJm z^yJ|3c<=pTX_5urFSouSv`~_~Zkp|4rZduZf1fv=nA&Sos*- zQJmhOG*}MaXL`i2^n~|UTbkd%7j-fJfB%fY3;6$+-E#h?&F<#Qr}&?b@_fDgzm=W* zlmz&(f08vmWdi=~G6A1qmZkFkf|4YSlavS)%l)i{z)3NhQWLvi^VmFT-j2206i=^l zx*tyyj+?E&z3zEiZg&Izv*+!)-4=NJC_%#Vl3?m_v{tQnZwb?Y@%CEPAb>5$4P~#!d*t>cqWM0?0i$zITA0pMIF|)r>f4<97ictyf?^J&*V-=KrM1 zVTzJF8v`%0|KHu(D)WEaPx9Z#d1~W-NNw(Wlq`w>GB0n_{NNt7_qaqBxxG`~|GT?S{y&fMJaGKyKLkPGTSrK4^Q1R*O5mC} zKn4kbTOd(@BL-WN93=qOIMmmcw+qb_nt3pFfLM;TC9wuCOolfv@>%+0B

ErC+8P$&koNoKb%{W6CEfuyqVv{ z95wJV%^w{fT^zi*+&g=FzG^>(Z_?bJ-X82->{Z)dWW?TpizQ#Txnt9&1()9+S(9)b zDHkq}PHRDj&r;OSfz^LY?*pZLPU7I`6qB$WF9JQQli?jF26sHqInei$4jy+Dj(n@t z#(e&@mOJC8`sMk_hqL`dlbIeTf9c!H!~Z!wIlDMKyFC1#i^JoC%m4G?@a*rGr+XJa zG%*S_BF}j^*>D<`SH5x-e%E`od*=bx6O5vFdkhw>p0g2G*@|~cGgJqUbVw?Hxm(Jf zRob6Jul4jd((TH8l+rBMQ0_A6s(J`pNR@l-N~c}r;APpQj5l4bCAzF#vf;so*yO@T~vSz^#8rBos$38W^cRq6#x5Clh+?1e=R8# zwbm%%x8%pv(~*c*%>DoO!Z5x85{XzTrjNBFiK@l5yS7(+PPdozmnmQz8NxufItTKp znF&q3SMZzZLrZ$j)Eqo|_(>X1F`M9w0`*gr;JKq07m`2l39^{}C)fTx!$2;g|2x~| z_^-QL-R=|pe~hQ*R#JBhv%KUQe=k()!hTd96{mHVHK$~5_GhprsmT@9n&5Ei(MhKi zlB?CIbTVe#j_y?>e=`ds+w`G&t8Xg2zZAtcHr=_4lD2qlL3kt;Nvc^eOa5gw zIA<2jpj1ZDHTZ;xW4M_s|=RXArmj-Sp3Hmj|hPkjsZpx8tRaUx^=%)`B zIV)GXh2hQ?j{LGIuR&sJn*97(gy7u^lL6eYC2>%H1-)$!8ud@1Mp zv|>Qy!ii3gdFE%@#B{@9iY$-(7q0YisT44 zI2~o-2E+-DRA^3!o6`l!96=r&F_<6<(>TSzCMbg5Hq0263Tr-!Q@&q_^iH2`$OzyhVtAa2 z-$=;9MG#S8&L+Y*UcCG3!P#FOXD}EH5uG^CV1mP`$R2^h5M(kkV`_LomDz%&agYhQ zy5op^snWxH7{`pV6eWE6YA1Se4xn{2N@X1JhKdjsvY~QBf5~+n%L$6uX&-;U;k4*i zNu$bkN3O<~A7Fxx(o(I~Sp1eN+@ zL4z=m>9q!&DK-X5C=6tRt$|9gH84{i464v5W1K-fG)lC*5SoaDcbXXmjcH#yD7DSs z>1b9FzL9}eS*+r}dUep;6K~2wKXYz!L7)ywf2~2P0-|cvZFIYftCh zTG-&dvDor!9oTK_bP)B)vAza0wvu3AwOyBXjW67uh_;-g(gQ+cJBixRoT+41%7OH> ze>kBV8oWO=+HkYd$SsFLq#cS!kc|J4+e$V!D4>D0#$;9Nb0C$_RMq(d-L^=mCbBKD zSo7NQe$)sJOS8Le@ET}L+kwF?obt;;AX2>PPxWR5*4*4{r5ZG{1GvpILis{P8;@X! z{3`*{Px}pXBcNH!sNhSJ3DWvt)kYyke_US(O`$X&DnxtQa?Lh6^eExFBg{oO8@-ORT(D`Wu;;KoS+aL^Hy(JHZH_|~Y z-&5z=P{vl~tf#sJD41L>? zGTxY3)2dn0>gMM2mpeg{L!)D5)k)g+bQ~>;8NVY*t|CmGa0qsjowLzonE^B~aFM-@?o+}gypwi~=R^{WkqIamrBe?xWsAkYZCoJjg* zs@+(zW6`s8oywk^x{LfzofRuomgElbL+XFcs{0U8y z5O1uY1^czq#mnV8b-%zGst!kd3X`>B` zVF>W|I1_MX6kCQ0AB|>ySu=h}`;5~<0wfo$v>%qWb&(r_vN%<^yZzjZ_@cAR-C{E=_b=Q1~k;b#mak8L+L!su z>IkbsFtx?0C96h2n}ynjBG`DF#<4ha&~;1# zay=kIf5|%V09002lw!z|amrb*vD`*-uZjkSc*IXqYkbP;9jZ z(&|SEvJ46kC;5@U$skn{S}oDarLvrye=NkchUuhf2EHzLdR{vUl9)sc)`U#xyWXo^ zVOz;JtPsWEKoB^>L~_Yb3HD|FT1Db?JA6+3e>e=~XEl>~GRLbTnWNghRDdM@B2yBg z6cAO^o%$&mN)}15G3tzg)3VDvl+4mPDk}+mUa#5I-7>; ziaO`hE~F&1Faan_9A#G6LLkY{jbAg&G0uZ?j0>B!=M)CUF*Xn@D%6m20@Oi6`QfK# ze@4PUZGNqGCg#Y++Z?)R>_+%TM@Vw7336py8)@K4@vyVB6L?wnU5*%wotjXCXuMigrYn zWxn@PSD|7jz?#11Ca5iUuse%0e|Lz-BwF8a_|%IH+#s*xTUbcDJjdaDkzojU z-Sf8L-}yf~8-jdHGhx_BV-PvI;;1Cy8|GppN2GF-wD15&Gbr*y7K$nlhL(BpasfZu{e`xjj z4;oAR&VRJ}|7aP*`A6%^mzLxF%&DwB1$h!%mZH&!_<347Bj4RW`84L4!PUGA{7h2p z^W$5>vr)HjmT2>}l$vtFsHV^n$#ap6)RCvq<p_Pp1D#{BW`EHtAkL8M+`?&6p~6( zf%FYpCJj*r{+zeQDNvC|`wGwG?N{BxcsUY{=>|*?onX2lZLI|{q2eT_fBftWgaZ|$ zo`a(?iEtmrF-t;c5>Ig-)b>G$)h}@%d=ywm8v1WA!VlBOBL6=#7ohV0>vngZ=>KCp?KZp(;~@&+Kv00(dtyu|OJ_oy zC!~iP{9=mr^;0vvu}VsA?n@Z?&IEAFLJ}fCh*d8q1FsJWCSiwsNDxfy+U3Q53kT5M42Hja5;8 zZFEBtT}db$;ClIKrWV-x(V90@(W|4!*%l);V&Aq! zl&$GZ;rx$ycAiHqmVRqW(4%c39Ch91mDR;Ak6}(jbdC4pEMk4=IgWKn>zgP-Fol~5 z7Ghmgx-BXh^APLV(y-^fi3^@1+q)1Vd{J>42lDQQk|dmYQo5}0M^xDWaS^nyfXYi( zwr1S|1u>eSf9pc{Aq}XtMo!#T)gGkE42)m%MN9l9F96_Ev!RNK7I`BHv&?IxP$`-z zrdYPHRtAytWlY&+h|br&Vp**#uv!pD$(#3lU#v;<3Y=R7o0mZT)GLrLfSJ0#)Rd&W z3crtJP$<48g{F;9o}474^43%1US8-VbvTebn1SY3e;p?sBENBTXK(iQi@0#hj*U~{ zI8^cQ+if_@LQGB1pd+UIHl+A81R<$_}$ds)PZ2MO^Q%b?7Pptf5$D9u?dct9jW8Eno52$cLb8{y&=>QRB zQ=AfCUFme7F^y9O3>vX2r}WaNq9l1Pn_yR7FY^FdZXCHIl2YcT8D|5WJaU$vBwTV< zT9Y~?3Q3BuNt_jPvhK9OR$EENLz>xZe?~-we*&16oa9IvOAD~vD8VVY-{lxxXi=r6 zQ%rNAPE^Vi$53jh)C27{>=kF2*80`?YXl%QOmfRe%R&k@-a`42A*q70qu3CBF#6Qg zhh^Tg*2T!)wERs39u_pv(oBgtED-R$#JEB^&c8}D>}zlZqQ)t6bU>P&m%HDYAGL{q ze>Q-~*#~}3N?m(KCA+D)1zja15iaU&2u4rT<9sj%|aJ{H$P5*%Wev9M*9OmPyrL*!qD@z@nH zHGAF7t?95O!`66+)DvMTQfTO6Yr}z0=J#| z+({1)S#M>?YW9|}u-C!Cdf1=c%wEECO(_m+-CD-=e(ny6 z7;*J44MCp0yMY4g7=lR}Cpcx2vsbsc{3=pkh0PL)ZZhy+?p9;O*AwiFZbi##gdDJ9$k4ogb%93h3D7UZ`(C<2J-lkYv#fqt^)5|9U@#Ui^ z+`DJyfoFOSOCzo+;{2obb_tYr*teJm-KdZml*ezmHL+M)7V*B~D4ad;6zI0yFh+qp zR3Mmhfl8+-e-|3y9-a2<@ar@V;9RpoB5c4XCNNCCQg50F*@%jpf2%kLd+Kf&?Fma= z^-gxFJe^FFK6#C=UKx~C+lq&KjSXZj$TMy)4Z~1E3t_BjSeI$`Pgtno01(^Yh%w4S z88%GWPLk0Gr;Tiy@TW%M};=E51L4dE#D<0LYAz?a*;q)hI z#iKZb+`Q0J2Zl;Qf7)sl*vVlPm%r*#=-31zSb?r#-p#6k=>OAv^*5uAlDf6rR$aAL zYeVfy#o*H1B-OM`+$-i(PUcDu1TYV6gtNgZs7#`0yH1$$5a25-}wkj2;5Jv|x=GkzjN2u7* zd3;>P;*_9JD6bjEL>DDVs6;%SA)>^0^G_7BPFq;i^2>mJO|igL?G`&4n@b_(q zbBc~qCdjxdf1Zz57g&b6a*E2yp?Jm8I27!%0Z^x%=zg^DEH$CYp{OOJoKs3t5(}EC z5lu?h%WvgLZ;2w#Lv(E&@$I%U`YOApBH;HvBq&BMxo{LcNwK^nmu&tKXZNMSv^F(` zINbnZ?4k`x>_{Xc83UWdly3rIJ|~nD=4d(WLP)12f2;!hr8pIHyi4&ZPkhOrj4x4h z*=`WH$$M)q;i?rLy<0|DR1u{Jd3FU!@6N_AzH7@Ke&<*1cJ;E|u3@6uP1o9&UG=qe zv+e2kb=U58*YT}rnqHu-_E#E`PqpCfudd+{RYwg_nvj*`mz$B4K9HXw6NlU3b@N!rStFtKE3xD)+n+b0Uu`?TnF! ze(B`SEA6(U90~fhYWUomrOBSfwrGhtiAlTN9Vs5q9T!xP|8XC#yWZw2uPbj;hGm9n z1foy2J6LGk-A3c$*EdyAiZj-#iURo&Hz@K3M%Duuf+K1`vM?Bk@&QAD@|A=P=3nZ!(-B;$qQ1(1F zfBCP#vXH!%iD{MUidRaiBJYQa*CX5Wuf99y1W{Gx)S{8{aQ_i;q>Tp1<0 zR=SQL!-Iy5L|_wTAJI0PZ(u50%%vB+v2cj|tK4X)ptM@*kVPS90ku=NZdEYb*C^zK zK9^pYHWD`z#GX@mNstqwEK*=UPVvb(e|$8^x_0>F|E4mNK)vVm1cy^?yUS9HwP&67 z?5@sdn}tED#U56t8mZ)hKU8-$@7!L||BuE<{kqAxbDIB$Cpbf0V^= zmh&BoE8*Q&8?W*QE3*gMB!@G60Uj1yfiw)A&lEU@t%eBAJ0fA#0H~~%mu2~gv?|epm7K(+~*XzLFYm#J&NiE*(g%Fg+T zCA+S%|1VO%9iLnro_p*QyPu8ueEa`i_vL2U{(on4Yxl|i|1q8yFO+@12tIHDNuZq-);DWwQ45gwLKSn zfGuO_(kHehc!UYMmfX!WrVK)Ig_8+8B~iY{=4u0Ou*#>zFKpR(%*Az;P_mVtUDuAwV#h8jC~ZUR9YzL+=zJN`RPcP=uHSb?$Vhn9082Jteo2q zKdbke)D;Hkl<>zypAsd!h+&MG1SZG+G>eoDL$alCc6e}fetEige*W{x*}>~)lOsGS z5pH3KDBjru*A1{A2YB5Y@r9FJJRyxECISgQ5 zEqLK6BA0gm-BFY3JRcM1{!G%ga^Lss)gnX>jc+Q5*twG&JxYH*IXJvLIxW$1VW7Dj zaVrsO!tj)S?Ed0WhFC^D6a*bIZ*+mJ6yL8VPV{rhB{(P zV@51U%Thqf_&F^WUn3H7RQ8m#3gBlXIOd7SN69j|SNibWI_|IiC0FQX@ti-maPs`{ z?5CsslM+5Ve;%KV%hNKHLC-i}3wIQ!Hz*C3n|6BbcNR~5Bgfot=B59=I=8*eU9anP zyTEk`ma^NugyDX4Fpm zye6dlW&~WxVXij;FUk}hCMk*72wH!k?q6uj9)$c`h0RE9bEkTo4=v(yG*m)+dy(YiiU=$RT{46YQ^h z&e6!8r2Jr#6C}O-S#EJJrchkEIq(YFhUqI#7Pk=i>gCoS^Cj118Vf9i6Pm5(Kb{HT z`>&TrI|I&q9%L)lky^lNy|MPQ3dHN){3)wUleCw%Lig}BX+n1; zfAq+)Piygx6&7lY6Xm=-Op;Mv>-U;!?B02`%Zn=(&NaHvQFwkQ zHc~Ba)kJf%hkLcY>IN#d4y@_hu8z?xzSSJSR{p&?o2s~2m=#vqUFDh!cbQ$Ge|EP$ z(Cn@V74v1 zHOb}&bC`7%;kICF?7MZToN+FfyJ-Ak%G|5A`?WUupFdX`sVTQC)U|a5?JD^I)O2f= zSh_>ImguVNR#4NIU1aIbO4USfe^psI5jj>s+ z_tm{>e{P&GyO_CC?JrFq%zavM25Q`L-G}%Bm#eCeOM{qK5Nm39A=NdWe_p=%qDrGZRHN{#EIrhwYsiX0*|_Rf zz{=w$*^*t%Q?VEA8xK0Cw3ce5ac8_$PQiw*Mf&{9HS_+_=Zv}g*Kr!7NfAnuDPkG zJ4W7sG^(zzeaDMNJs|mYr}p0Qb>#nZ^*_0V{r!T$FUtS1-Q6wge}A^Nx1RJrkMR@| z+&*0QoGTIqedQqk9wp8cGZY|(_!%WF#+0LABG0O3(=o|KaJ;H5s8lO?NfF%wjd9=7 zU+X{=s*qAt^eZ}USwLpLSg7K5^nes3aWoPhBxUn6jr_WI;gFfSG)t0@;8NzvzGEo4 zi>kLkq5ww>wn_&if2g>+MfbdmfTVTGaw8_Ih=f!n?W$xca)K@>jVBhDm2=EzN{6=; z(7=~3&}ucD^ujMCBtEK+M21z;(jQ*tKvlhEP##U#rcH2yy9ReBI3c*ZySqC9f(-8N z5Hz^EySqEV-QDe8o_D|fw)R(d&(u`SpSh~~>f=0S;As2@dlKtyPhCq>u%DEqn78_( z&N!WGisn;A({*Nc%bkAh(}j18AJiS-Y<*fF3*u~R>!1Q zd4(Eao^5MgFm@TU@y!Q!oOPBHzhoD4A?vCeTubcvb0ne7rR2$ znquo`c+#)Rb^Y!1)(F-0mb7U|s*3)auuNa950J{hNK*Wkb<*q&pdWqhxQ*zm8?aWU zeN2^BIcaq)INRaR$k3l>OtZV#JKMQ=>U-r3F^_5YWjD&r5RBzMV1$V8wEL+IFF{+` zUtzw!_inX?(g_-q!8H~qGEUVftkcCUsxnAd;W`6Np zr93OuG({7Ak&NZk^W0UZItgl6@qBi=J%0OSY`kI=N^OO;ES+Fg$p{8m2=;J{Y7oY< zhD-ldO_=Px36}-?StLEhs{JIJb+RB0+WSQiMq~7`2V{PEkN*~K5 z?8|(R#lOA2rRgJf3HMVa%AO7SK+!ajjw=EbUAUcL^sM@twBG3IwFt?jVMz+Ak z0|RDB0~kV9(&EvZMrL$(Lofk|O zmb%4i7e}tSzQ9c_Ysoe%AmJ(B>7dIOno5Np)K6EMZv28smd*GNi^@QnACDDm|1&C96!m5;(*16V#2M!^%mV!z(2ioN}B)Ga^BBuZ6- z@NUdTrdV!$DqGa{>1LlDq|BkXj&wjrp!-PoPonnG69&dyZ6T<$?gaHc+=B%nR@JAV z3lbm5oegUMrQ|?aPnWOwxn8%BwxXw$+z+VQ#$^>R6bGp2d)S}4mff%X%FwmScLo;J z2zrBS*dY{JOIF72FDx$Gu^ORvPJ$`ynq?W?WGns6*CWadPWt3@(VX}}i))z0mw(q@ z9xZv#@tFjVxq<#VR^HmK@ogH}^CI)zFF@s$Ou^u!Uv`r_dsTizuEf3CUxYGQcL)?( zfdm=ISoud@-#|pSB;G`tVnJKcJ;3^@vT23Yy_dJ#merfd>HEQa#}b2pV%j%6uIN{p zFryKx>RoqH-lqV#3!)>M;F@l!B!D_kF~-ggR;s zG;9o1j-R+Y$|4Tq3kZj5bZ)2wTv6jTaJ|D0focH+i$H{-+TxhMS+f1W66S% zn+$8qwR)Ex+Gcf2Ny>jTH9~`DM?a{e!BNwd_C*q}{U%tgS}5jO)Li97#}@{qH7czu zwm&f;Y?ZG zY<@rf*oF5zoTEfuA#IQMtpU=qE6X-5=ETQ6l`T@P|1=A z%b{l_a)`)59Xd%)kHQOQC~N%8!>c`cQsUF?R&Q2T;)P6nVe+n8@f#Voe6I?+R+HQC zbg@mR#*eb(XW zc5@5tZ@9R>(R30&* zLRMcc@;EE6oLJQcbulWg54Y7|Wy1JAFLI>CeWMmVQ%$S45aJ53T|y@vMbnWk);i*+43-Z#;--jKQ1miZ z`fE!MsOUhoB!5kP^6ajB7VG`Z>}(tO=d6g`R8{X8Dz%W^<#<=Uw#AOGAcfn_1e^3g z()M-?>K6-(Q?rbystpoDu}CdTuG({AN+!zq+d&ITt{!P5#EUXFGt-?uJlWjAaK{TU zt&QGTthSx5cvv9IP@pOx%KQ%*HLbGmCB8THew2Y-4u41md$7#u7dq>b%Zf;6$$opI zUZEA&;WOv#VCtk>YbM8lrrt&8S0e45tP}a2KgA7S7#<@*0K2A&>`SiMjJo$N`@f2N zX(Zdcs|mCm*OYv-E^MP}iHGmluJ;B2*VndM6Wr4iMkKYsP7g*IvqLkd8%(6A)UU(o z(NQ;tcsH3ue-{EK0_8Wo@FyF&mlN{$(gWQi1Nf@h`fgFJ<0iXyY~~z%UU-6I;6}qy zox;PzDcfTp)$2G!kS5TTGB? z%uS^?9b*xi@A#eHk%!Yl3#%f+=1*GI@N>$?f)2gWps&(bv819t#M?21zoVbLc|!Ca*bJ!rRM8Y^egLN8amt%MXaJ{ z=EIDMJ$?2(yJ^3)dy`wCKdL27!2o7UPI7sdU8d$BOo5Ii`)q9M4*Z~XukMeFg9^L% z;dn3>uSuo;D|gDnPAm8-ccDJ1k;)|#HuZ{>K|_hZ*D#9j7zL4`{X#*&wcE8Snb2Qp z27gPRe=+tdP5$ponVG&=FTgK1+H0aV`3L<^p1$2}iqqtJA^%l>rR|Wh^T$26`^ zl@#N{S`A&rSu<->Yp}CrSEi74JX^7(i+o{gzsoU=mPKtvNxl_U!TqI&ck=1@HInW^ zKr!x52IH1ZI}no~h&vy6{0i3P`{Cv`dJX#B_D@^>8X(#)X_?Cx-aE~}UHst(Yl&q< z`_SUy5_(B(oWMl^LW8UwZY@Dds^e;(3%ju;pfJ)pF_G9$^Gqy98snF~0Fi`&qeWec zL1UeR*4IBX?=c$(=%7bx`C~Tr#}8W1wMeK@&KLYJCSdEGFa*%hoXZKMK3yBcbpk^m zZGKakwt132bJiP$ieOi#Y`9Ig`rg?vGKKc4&z}FCZQT!&eifwMW(wNem>i=5ZHjx0 z-pmh7ZW9dG;1Z~K<)n$N$10mg4K`~eK z(fBVo*IiKI-T*$Nz*m=16J-(RiUiI%9V^{Dy~?tMPpOOs&CGD;a3(d6N?4SQbMs4Y znP`s|OL~i*Qo+P?p;6lAmtV>h%F))^?$jL6O{$`cJxe9$4$nPU`P99amCt53& z(G&1yiQ3H{ZdSK3=jJOOU&O6g_MgTYxxAQvIT^l_gac`#jCkDYZ9L7-y zNZ9N6(HucnAT*csljii4Jx5g3meP82)A+%f$7b?zsa~4X1i84zgWbjy5gx`6rUDWA zkFw4Q+5^D0=)UugOTa(IyjWnZjXBRvvuQ<=mHkrjMnR9tye;mB@TsBEk`KUxYkDO7rV+>mVBdA z^&bev@Z;yrlUlhwSyz89WX7g_fc+SxwRDdVQ=9Vb6AvopGVVB`$s`Sdr7r^$t5yY+NWUIcBD!Hk2yW^YXX;s&X^y^efbI%X~0fsl7dZV+i+PYX`n#@}akJY6qQ z+bXyx$=IQ=U!s>?)m(vZY1*$FS!5pR{skGrHX6w{h-v!!sJg;?95ifM`sSdi0#NYB zE1!3{4QTz}`Txg*uzwI0*3UW_Ek&Oy`E~J4(I;X*?cxE_PG0dj`mAS1sL28J8sPe` zNG2ho(@^$?&2>R!n^@HUzPF|SQo?}pH2f|&w34_fe#izG*S*L-Y;JsZ<-bSoVDKYE zGpvHv&y_wxHWn`)4*tgsfRBDqzrA<(DVuS|xrTqEmfIe(37vyw{S5XUAFDkLd09RD zU$9<<$oUbd`gdxZbPL$sXH9~JJXRejQ_THHTiV21Yy*JNgs#iQPe!Qzih5_@XG#Jtdh#KjD>JfRjl1Cw_RS zUG^(qc)ea|3fWAmT{q+mTp>OR+UOCq0{sV35dzG~P;51gRI#t|cWDQ_!ChuT)U}4Qy!GI`(0zS^LnEPL6FY&xax_AE!ytA-IaQ{gz4oha)Ct3u1SKJf^Sl<9A^V1i8~`o`p2L8LBa?4v zd-}O7!8GC1I7L~z?QsIbB*RdIpOP=6{P)x&LW(e!M7n!11T)LX3ZWJWQkHr8uMT*V zPs5UIQi{=?RyF-nr+^2l0@tKyMjjYImm-gl0#ZRJ-U=~f&TKgu$kE@@fQ!>n$6y>? z%sGU*l%%2B^HOZ7dFHwGiHbYLR5hM|`3ax)bBn>Je&fA%vC*drBFO`4{{SP3f?8C0 zTper~lBHv>l@N(|ScyB+$f1^X=o!D=-BBGu>2+sP!^;FywcwWdxaOx!ZXBv7kgO&(QnXTCtCaQ)n+6K7kh>h7omJM)&By6G&*BrDZ6JS*tHP%>7ejIoW;*mk584N z?Qf=@cYGat;J0p78xr}~&=_)zP)7wN`r{5^Qa-f)Vdj2j8J1Qw73D^3Lwc}`ztqAc z#A$j;zMH|e8;u!EQQAU?)Tq(ZUdS>tkz_C(B;(mzU^|gvh%tZDN^_$jI(i}IV6C>U zG*znJVi61^NE6XyesbX@jot`em(iDjxw?BfK0MAP0C~F+h&am}S`g;xij)E})$(GH zFixSi?#A-4u50bzrG{LH)!cK=jP@fi=<4Bkp9CBWSVNU{$$)_woIKwtoeXHM{W2o` zT{a3o5i4$Jf*Kb}r!)IdHI=a^bm^H#lAZNBjb%yjui&VTUxS!;^q*inOL_T{eHG-4 z9w>-afQij0nbz$5)LpiCFTwP`xJCUZ!;^;5;=c3JA5A#u3ga}8RSd{NQ>o2PVDo-Q ziWSL_Do67$U#A+`vQX1^+cr|cwfog@B>em(cl23(ew!L) zM|}G!M1HqLU|h^S%dI#|z>~|?S6|oG*0ReziAt5o7L58Fc>Gm$Tg=AAgs>u@)P7SS z*1psBse~aH(Kk{`In|!=jRr1EUW045JnfKcGXXYe+Xezw8*xy5J%u@AWC6oPj3GD? z5x71l7JT}oCuioAsN}H-!J;T!$^3sL?QT_E!?8Ui;cY?VmIC&EWJ@1mhwe_Z8Oq8uqkIYgvZ! zaN50akXGn7OH4s97KK*7+GMltx5=>FN6JhdUoAnaqsK%- z(I7c<^pH=;W6A!@(B@iDHQaWFhe>^*k%W}Lhef_r0B_e~KU~z^)-1Ua=5;%}0v20> zdblD(B+?i%w^?%OGZFD9FDt0wVRnKG9OSUu z)>$Jni359dlsRJ zhtXO?lsvG|%x;_O50WG81xGuZH`w%HFHFG3sC7ZGX_?s}x_j<R-SH+3 zBFX53u%wabIJxy+z?uNaG9|>Z9UYAi-zo23#(^uc8G0jgo)N1bYgM*%;uCCDgECWYxUEu!PjferLm$gxt ztA+uhM^yC05MY6x(E?|2DFJ(}ml$b;UG6`~g1UL$d&+dDtS_DF|G&A_dTRTtVAXrx^l+eCo z5G{KaObkEjpz$M+fdu-#llCv5yBlj1R*f#<{S!tQb8)teU&SxPTg}&YA1Id;W>}bi zzl`w+3Gk+I;^FByZBgC*uF{fvmXsWNNvA4*r8390V@|mDS@vM_S)_AyR8+B$KmdvQ z*e_H(jgG{%kYU(z)7h(03UUnSLi@RIsckShf!z3fDQKlpdGLp6`#dCR@tBxS5!(Te z?s!FYdUk+tzBQ9sPMclaFet(4nsj{CI z(K{-aybCzxaCFeFWrgf#4T(X+jlfbA(&f}iaxg{oBIL8!pC*TRYMxX8%97d$Axc4_ zjyk^3wya0l?^B6f%|>t7=xjT}YPTKo1T9v=CI)Bj69{_Nm1O-xf+0*#uC$ZxtWR=O(fH()=qcfqJ{;4Xksx8!rZO9Si@g=p6<>a%Ee< zLMjvpkp``?b#kOA#=^wjeR&X3jHLm(daL=e3^Cp^(1PVjiI}L^02>{Ko2pjt-|&Zf z`a%i}#euz@RZnZZTMODwOj64k)@}-Sq^#=)9w`0${cOcvSB3*Ke)U~mp@s`Zr{7@P z_>x*V=@75Vc#$3L7%)uPM!!#v)9nh6uxxcM4YwLRBED2`*G5*cG{WGjwMullYX|@I zzFJqgaHzg@dTe`j0B~E(``Y!&4KX_rN7DpaDJ|tQO8>xo1inW~_JM8N{VktEAF+hi zJMZSP#PX<*KGn4J!t4rboqVVt3tW63>&;kFxMxp!B1RzP@7CJvdUg2d>-nZhIhXvbEXl zx~GAKUp4_7i?O7#jrs#sYuScRZW&OxZWM=syZU#ioh%}ldZK^**sW`en_P6Ho*`zR z*KVIbzJ@-G7w{s;dbFqNvyW)TG8^;#dgju~$D#olzt5ddk(pYahwJmXTl#ao8hA02 zI1MSJlDKD>#TyN6A0RIM?N#;$(dOp-T4;R>mE(B=d@CdtD&+2XnCHR0GNt2&mX@&A zCaoMjc5#Pv?LvUW4d35zK;GyL;oN)G0N3*ybsI0WD$uLtlvB^DokZK}qaF~=#P%pu z$n2@F?HfMS)Ce2HUcE4qE-ipq84r~*Tz6THJcT924`=GO1n#Xsg$;JCQ5id>vaZb^g2S`6c>I$%JMeM$(w{z*(C`zeGWK z@g=dL8ff}FrW}cx+xw9`eQ(Wyo)fx``ecXZZp}u+j+&(uZu|psBW{dId`Q!y%%nnN zP}BzQd&Qgd+DZJ6BdgR11oU-z4(;>!q+1GfQN4oG=(k3L65-%hX`!IdIfthoP9Bb% zd-zP6wP1O7Oc9YIL7)ET^|%z?{V}5wuAH3=ZlKFhWOHvn?-&u+}YRvOeyHdWde z6eE^|YQ>7Ymzp1QZn7}PpPsOwLN-xH5m$m2ommU!aA$h(d&j_k*chwG`tc~*J9$f+L;xvOEk`fPKqq z7VTQXV$^IGGU`lUi*ofCJsc{Sy#^s#>$qi6P_IZ?( zmkDNwMA1{pfjjjw%aHyGSN<6*S0hawM1>q4PYe^}=sseFLO8?B_g6H_QQdN&Eg5Rjoi4|1N@qL#@8V=am{4Tu?%fr{#%LH#IZG*dX88b&_6izuGZPw*96$C z$PQcWaNf?%QwsAAC$9&V=e?OxZ3W7X(HCbKIEe=BQ?~1SVdGzJ~qZSFh%4@e!c@x|1EKEi=s?W;rI|!)yf9fldS(C zB(z~1c0}-tP1Nv>NGSVPRO_VL0PtnJxcx8`Yc$fu3$Je&-3Ol!1!~A1^Lq#4a8vtE zNUH^0y*SwykeUg@=5t;@SiP)PC406%*ZC#PO)grG4alGF@kW<7WbDIXbgxof{aN{Y z9NYyc#}d&y?i(VI%9F**`xiYL0c46oXnwv$bEEQTk;CxHFJfct>H%x>F>Z)igR zlFHMoPBHH9qaAD2MpjjW0V>U+;wrbugary~H{6d*lgz)J8!U2aDdp>?Pj2z9;ZB}I z7+Y>5{#uA2wC}*VXHrRi#a={EEO!fmlq;~QLyiY|kwZkSL^`kZL2RnDZtO-AWhOm%L;p0+-xxC{4 z>&u8)i^q32=+SqHoXKSV^b|uXUT!w9Q3+p;nI(BYS|pk*_|08JG+>qJp)!Osujg4k zfJLyIzZ#_k-;T91`lAjFIYFm%!WW=WRx_PJL};Fl^vi;NEP(@I26)9${nY z%h*?~h5qe|HZ8gE8m3RXLNnLi;_HTja{{S-5{5cToS1X(0@^@rI=jHA72K}|$x@3l zQzu7yGpAH#4~7snHI*-aL&U~$25pk3BSl*wP#?aFv^msciL#lh^Zq=A`5j`Uv#usnPnYv>|CQ z{fpv1Gk?ZQ?JV-Zl4HMdnIA`&7V+)MbTiOzE9$b~TaM5RGF{6pJ$9RVqc?t30c|{9 zF`(DRp}De1$>&B+Mg4pMh54qAk$+jjlm9ldfd~IB&W6-7;hdp{m{$~_)plD<`p93Z z;&+bMD7eP27G~Geh3x7bXh;Q2I?jaDF;$@@dX7BOa4*%pFL=Zy)}{-ms(Dw*o{X>` z+ldKu*9zOq&?dW4JA6JmMmlBT+oI}F>QcqtjI(c+?q*yWG^7oGpTYY`T`Z_l8U=U= z%dfSrlhP31*)mW@<#_)9=qEIMfeRxR@ho^%HKA!|r+pR`IB7)J903=OkK~tAy0$ zWELIzC3U}gYgU;S{aXK1Yo<4nGinXNIO%Y6HfI@m&4^5rfB5ejcz%jJqFt)hDWJR> zQJbDSo zr!xNb`|6I{A@4Q-SeUq#w-*qy%=!J%8`&BxzS6?S1W#aUX@teIa<}bnARM;#l3Fr^ z;P#d@htC3sa}ZKWHRTw?{I^>Yf8oSOH-C4fVtcG4lWZipN%v7`T#4$53+}y@6TN93mH`!#4|>yk~R zP31&)GP&jh*HhrHHia&N8L2VE51W{wycvspdy~2#VjAV%m(4KMq+l-NzI~@7gu@@~ z8CGl|>7--S0Lm5~%RGR(hr09BIl7F99aJt+f#lCKhQq?u%6z3b5M2;gBDm-#{>Hfu7I?1v`%3E++k|iqdJUrMn2j;+nP(-zgu{o~ym!-bQ3M~G1 zwKl~({i3tQUd%R(hwH<5YDU>+?K2fa3gnuH@=BFb;J4tRv#UGPg4~R~i?3Q7YPo2f zYBVvaGZOJ>>k*?=9HwlSy{u)ikX7KCw0Z7sSPlE`P7T&==Mi7_k@;ltIO5y)jKe-$ zqLT(JZmmO9TWaW=`vIsoe!Qp?P|P6q{-%TX+x?N(9h7C<_<6Pv9EEQ!=*u8Vv#x-cUQjz+Z@YiY8jo2*KIczWLw*-dtEW_>u0k^IexfD7gLGBm z(|8X0ll<_3{rGvNHHdw<{sBTEnrBQTI4<_4R?`y1Krj)^!Gz3?)cRW@H_Qb~;3a)~ zH&j^n`o#b7-Zd(s0~T$FZ)5QI!QX`w)mfemfUsi-U0*@MUcC-f(La|jyNhPjOM{c4 zrA|wl4K>Z^VH?);3*d3v(31nyTVLpDz!XW$=#mHNGaX&mVUaCVnJZC5cEtrrVf%>Z z)hPvw6Sc7O0r1s5{%MvRM9?x(g>^|bD@Qhpv9~71oZm06seY$aL^}8LzRx~Ou2mv{ zKd^?vL9(ciq#pIpQ;60c7ItMeFGFuTCh3D~k@%MI9WZh1HewXkEbq3$5-*KFmEEN_ zb_z-_?No(xQtX|xJ`)vC`GbD5V8DhGQ)9bpkiT4_zDN|w5112hv=Adtv%mUYwIr~N zQ+W7R9>I9`qJ_ocp8Vp}eGrVA?Fa^RbZGhkeeTk@gt%DP1U4N3;DKF2;7mUm- z8!*VEr>68<-8J|TBEAJ*1k;d7r(;<(&OdCok{N8J^;B$Nw6*3Xg-X{AC@A{(i++Ar z(;1rOO}V)o2|v zQA!47No9 z`uzpF=Br=4@KKrpG{}|2$ws9njd7H3Rg_B7=Vgb;bs%N8amO~882 zuO-0+8eZ6~ie@lFxR!3xTSqqz>7lgMJr*8TT9ntz7ONT={Fh=_4|4(hWZ6?>3}*F) zrqXxau4}77Xi+V^{B3n|-}dKkQ5B5zKA00-v_*YiRUp*+G_%9KwtEK*4{Q~FkWr6- zF=j@J$Hh|~BV5R(Yn7tv=8R({xRr%rf`?(l7OrlL<0*fg4EZWZWe4<7%( z6#e=mT1CfCz@bG;Mwb}hB+rLIb<9aMOoqkoelw8w<>u6Wok;f6`3MEuF#erfzTs5z zudjvNgVaU6cjbX`4gKfOZX~9l5RY2Cm(xKBlF}XVd{sQ zljN`Gd+GFVt9dBEt8FdAcd-JAC@)i`lk?VeICf%g=|xN`@xl2@xhm!h+4Ec#d~)`u z0P!>b5fwCbMl~YXv@$F!(FESr=6qqLG&T0xB43lh zx7fou{v{Bx5;~vCIZpQxDSqH*oNR%6Cw+WI6<`az0t8dTj}%pXSPo;hGE2UXSe8 zIS6}eSmFb~io&mP8*QaRHgQQ-InI`{*DN^qmRX0Jbx&0(&l+05I~`?5vL`c^Y37H{ zh_UJ%ta_Pc4!V;c%coss6b`rV{x=@!hW>;qZ?Qc#jEk+8!xHa5_*g*%MvBRXfWGZ7 z@Z7Q7D(&>bUWnxo5(97xU4{3*Hp{UzqYt1%ax6vxC%g&ViHLGu=Fo`Z@0Nik;4$1x zj8hfA_j_`Gx5#qg&9WWFJ)8YD=(IQX$BHIFr%T?ajd)IXdJP0=4WSY+bTyPy-v~`nA7&;f)h;=pU{Hk-R$!^LQ6#-w zn-Ki=f*O@Ay08`gLSuT>ue4H?&O+Af^fW1S#F`B5*d48o!`hJY-*VbyW#Fwt`8`i& zYH^ie8&A{N40!$5P70;#E#VDxO-lwxZC$Zs0KwKlL(vwmqy!sft>35{nSuNdHe8Et z^?ac5&2xt7FnOk+m@{!D)cp=z$%mq z>E1#YOSkqIJeHMr&J+q5US_Obl}>f(VGgdHT;!LNKsMpqz_R8Rlx$+J&j=yY`?`l1 z@Neq3;?$!?)zT8aYDCBEV0&v%<>YI)%A-H8OZ&Syq=~l$Ekk_1yi&~!L9u9{cKH12 z)T>D&N-tiR%$=`orYVFFpT$tqStdFzz6EA@}UIC_$xGCc#7&y z>u9l^+gKGL{C~6tVzSRTW4g#^2zO3@o*yqH3w}i zNnBH0)b~ht+2Vu)1aM~-XlDk~w-crKBF^OF_xn()sglCC4AgoR)qF%r%dRpKtYR1v zHiIoK?JRm7FcVY>&=~jABm_c#@qEV43%?md-{#rK`Ip8C7VRgC|H3)3@M-D>C>6DM z6-x<%&h2c`{LG+n>H79E%85(K-I1Ur&>Lh#7OB=&z`gghd3CEJ%j%>h8vIF^z;z%m zNOE3~8owJK1zvE*^XmNmv~Of)Tw^D&-f0AD%UBkNYvm;^5#Id`dpGan0XN%^68#O- zcy+x*GI;H*IyaZ_hY)&DuayV`A=)Yn4iSXk>{QkJDq3BRW(tu{jtBg{HSiNGD!WA) z-bZF@9@|*=_1CZT=304bTV0r&oR_9>(;`MQn_0xnN0wM+B0R28jitP>QDv{z+paIF z7EKrNZW|h-xps1??^P zb#y0IybOSKy0QM81nCjcsToM^JBIj15SN2>R?W1QOZme#F3KlKO}GrRc`|65AW+Mm zIdsZPEUV+>4<{@I&W6*71o-ysTwvi+D{$}J>XUW4mL^5}4`r^JGQVh1dCC)>@Vvu* z=x{e{gNEre%(AoGIcL@G7-qj zz?$1{TY8t)A{hbxSl{U7y8?kuo%GfEIqx8=^m!tMX!6NQR6`2z38^t|ThZO(3Ly!z zn@pIv@@<7cxm1PR(wA;cH(=C}Xna*qDEEEPGycBD%Gn^V%Q-2pOshgQLBozj<>ARv z5JEF=(r=;gRN0}XFgx4tgV`nn(LDWHXY z!1e!3wLY-1Qj%Y&b2a3}g%%jZj2kvHVuhT9S&>lPFWBC?6P!fdXiT%+r8G-+km^Xk z4U48oFYd>?wwUNLYW}~|0r$-xgKc9Jr%ofSI<<@`*cM_QCs+60bosPXrXETT>Pnr6+ccNl9K)>Pf$GY*)%tiS1`QRnxz8 zJqscs32lmG5uhD)oZn}E2O9}+(PBHU%{-?n7K!BoxVX5u>j)#tItLS!GH-^a;lF)_ zr`zpz_jY!~zuj)P{BO6nyYp|ot?lit?rwK$yZdk5-sa}s&c8u__iF%Celp5X`fuF_ z_f;O;@8m%V`4Oj-#8Dq^dJamG{Abted0i*K)K5vm#H)Sy9*0x#Il2Igfkc#{Fof?o z7C{`rbvDFlgc+tzgr>L;0>7LayUkyF(9ci)$o+qV!VJ@|MF1A;|E=xqZh8OjY;8X8 z|EG9d*9FIqQ`~od0FpHRC-&KK(1+I8Ph9oeB|*ypAX7BPeQ*E(k}M3*;*j{WJ{*ru zVs@5diX$dIrg%&!OJ{u;`>99b&h^NZbU=PhVoF%dzl*mn?_PXjXxxWZZ*yy#_m|=% zBtANbvxxO!OT3|Rmim|~C_*wNO#cAzldKP~)Q>5iiY{J%y;Z&bl3`jdyjd!|%?qb- z7?NoGAqf!Ekh3VlX&(+rihXu2 zeu)pUpysrH53SE%WKo=^aeBex6kpB~-U?+woU!u9(@Mo@9Py#9Mo@T&Lp1BdZdZO`DIyVx#)pU@ia7#}$frKEMwB<*hAC+WPX6 zw;)!V58Lb`#Q~u;9ZX^>8t^=iH_6wQ|J>c$`<6H7Br#><6w_ZqvHnRK-w=6VvNXeb z?_Kik@vXP(^?DU_ljyRuy(RI}5s>gTF^*wZysT;8(SzY_`EfKN;{xxGB1YKk45x1p z^Cx|O*whOCN_%^rn`CG|`)0sqp6SiryeR#;M%rIO1>weHfvTV#jI2k$3}f z6wZK*U>0XEK{pss1o#u7j6`D%F(BXPlPn5N_k>Dn$;w774px93_gLvdQZCz_$ zR+1ae#A@~3RjrEe*Lc?O*}cXy$Ds@{8t}33t=WfG82cy`+tkVDqPMy8hi6qK|Htq& z2@%7zb2Y(Xf>Y|TPw{;I-1!D>$g~d>GZ>K&^P}v4 z^^{MGKf!(Yrt{^C!%I0wpAr-WVt?}O2k}V!EiQweqsqAu;1P+iIxIb2vITDO4P_w% zPYN7Pcr)IQ^76q;p5EXzB>{#zelSHo>~4ua$@C%{jmRfxxkY0*3dG+Mx`PxWh5_Y$ zjIuDC!7mvK$%x9T6M`6i|xg zF@OV3ve<-BSR2m7OdOtE@Unu^CD>p0oZ}IsIK+smDf8oqAret_gBI@ygIf}YFvO5i z4kra9fLG8qOe`;9Pc9kdbE!Y(^Qp@;7FcS+Lt3=-LIB@?z5JRQaOU$MTk+0+tmr60s4q{zBcq&{nB6+3zY?*7SD?`^s3y?>0}4b`2=QVJU4$F%NIUc!aE(r-FL|sEVJS&e=94kz*%{* zRlTQJZw=z_X378X=Sp!2tvOtOD&*cM-ueTPcDd+pafDO8jKv`JorJeOckX1l6Jz%B zIV*Nm?Oc27`R>qHrVH8WJj^tfoU><2)XtY%EW5%XSt>+B95$a1yiBWcmd*2B{;gC6 za3}80$j8w7j~2A9S}RTeugwDgsh{%wj}vk?z)2X-9!(uE|NhtO?QCs-m+pT%+dI$q zzbAQ2{U83saK!(lotxeeGt_ggNfh+qP~gK9N6eXGh62P;AzJ+W88rITFq$5xU9zZ> zP=Eex6KwHXmJ;Mi$+S_D)V6Ik3*g7oB#v;z`jD#;1Rawd88qw57q@uROu1CM>Ugb! zs|!qUN|=yQnk6ojgzL+H7l$9!{X(orWkHi)G6jyZS<3jSGoHyQkV55iQ#>ePvjm2Q zI8{mvE8UpIJzY+ZqbgfmSl=iuThVJIVKGreNKK7ZU14RLR(Gk@(UUJ55gZ9la~RxJ zP*>?Nq&&J+o*ePs(?V$Y@}<*MLJOkd%a?wA8HUF2MJZPDKFpsno)5KrgJTFx=ACem!VM@=^c!~y0Nzv*Wqj;`}u>cp^m?}M1 z^*vq5^OLMRX05wfqP8q*H=uM#t3V^G+z%F2w+cz=0&go>%z`&@m`(9}{<=g@gudoX zpjNAcnco@vVVnh>lE#I$>NdCmvfa0!z7`z>h480QIMW(`CvzD5B4GfGqZ=De+2h;u zvxBR%)ALKKU19lBL)V+BRja1$!#7vopI+3q{Q0wSk@EgRJ$mDNewa~)Q%`7}TFcxE z8#tyKK$aPzbzPMMP3Dg7Z2#ior_=MpYQx{9@zlm$2@a!Cq%HL|)#`6@$b4<{!_=gUmMKjR_wq`gS} zB=F{bIz9j4;%xt5sVNmL=5{H~jh>!e9-p3EEZwAkvZ{bPVDW#`IBIP8{prc^<>~p= z+0ps&>EYG;+Aba$F)Sx%&) zasBLGT%MjEU0ocV|H#p8iLGF@>9NKQVK=!&}SZ;Kkc6%eSiAl z;%NDQDahMhI0;4Lt4u@D{v%CDo9=+XK4>G{R-Qy^PhIDXaaDuY)ouRijk=l#oz z!+Gaz(IH|q@Fp>3bDLV`#8hq7oRXy#;Wyt=7~eLW&-<7A-@QBi>1zI|VsxzA%MX^r z1uefjJ=o{Vv244X2jOc`#j}OT*T^#(T-WJ;>DzZlSNmsY?~Wy0R|n@uhes!zTGnFB zJWjH6jc49JX&kAFk~cNjQuki$rQ}y^PNfjz&u#4**HLOYbHBE1Z<(i7{;PSyM-l)p zu>aZYZf%z3zq`GyXZi0_JSG1RDeqNM%-^1{_ctNf1wc_0GbGJ>C19LddE+=9huHIf z<7p?1QQ!_yh$0`SuCfk3ju=iO6t;5Nu>9~YM!_3Z)~Qfg>pF@*vE#G-Kym4zQStb! zS&C!Vn%BPB;)M_)oTF%rX|+G|YhSgR@?ZC9*cG4`;N>Sz^N(DzyN^yvN7N-wZl+4`=zcyuhexBkfpa1HL_NXf0 z1?T_XcCW1e-QDay`+q*k^Hub}w)xo;s(?-Bj^75eFjot#=jrP!g7cp4uM2K$VnubZ zpJKr}U6LuL3{8_hL|GWBuaBcFHt}Gob-cH}xE9srQ=>B0rRQpm%Y_;g$9A!QRdmO8 z%|}ol>jqa+@#bvP>g_M3q0ae1_<9QD6#-jCUAp@_Rv~}IY3X%;Yg**G{oSZZww0P% zeY<26xf0iGrBR^RFisjZK{p)u}`xlqF+JLY1a)Q@|xKyW_u{&E@^}Cpm zHFtJ!dVbU>m2R!32Chn08dSi43&44R&U=nNSa$G!|9`Gd-dtUNe}1&TP&KM2Ryr4Z z71!Z~Jsq7~9$)_b>g?$J>dpS;!S}0oX!|kG?eP5Q^8EPd$NhI#m&fmqPCqP-H>x*9 zTo&|kczkhnc6$0wjPK(3|Etx^%;_a}vi6iK^(y@xAHI7~f5$=C)ZcG^TtTBpCEb}l z<;mQjq251wdvtzvbh7{E-OeVy%}@4tUoXJ;ry(?)g8pHkx0B>#C#70^Qc z&t5tH&+gXtbNrttc^Di^TaBkGF!T|7HEFO1-(3`E@-v3A446~td*%{ z@|RG|EG6Ywtc|L@(A=Z@+Dr9ARYRuW>{&bXCwS!f?@e$xCF3Yg@mJ_$!TI0a=~d!? z?sR+4=l@eYZ8$@WpGj22^Asomw-X$}Fe70=qA?`MzeZzBJ*N$SmlHyPW{DuTKqoj1 zVI0OonDR?CiN+h8Y#>H%u#gxU?@<&uZHVw#TAQsU!h9JAa7)+({@1z(r%^bAI1)8E zNRZ$ZLK0!m@eVJpLBzs?eW5&MQZ-+1&kalY|}=-T=Kpuv@FPk|oKZMFJfe;^#dACo zVAPT2()gdhN1`bAe~N#}NQ$R8V$=&`|N2q;SiJxDO8b9nZ~NK*_eq`rCpZdl{F;*NfN(kj#umfO!#{T`*>*jsvV1$p={I1va zx=ujGm~uvDf;M+{`~GnIRkydjxwE~sIo#RaMmyWvyPLu8&R&2 z=FV{ITRhs?+3lj;U4*y4-P^%KXRIPY1bx`-_P%wyn{IdSve)hJZ1;EeyuIGe_P0B) zc6$HOebw)O{thD>*X;jCj{>}C|8H%U@BdqSJKg8~{}hkC|BL&JMFEUm_J7qbhEj0N z)_+lkh9PdL8$}cl88Tcqk4=oxQ#1{r@D78SwvGZ`a$j1OA^5MI>Z@$qpW|^{(sDj0Q2{MueY~Vk^gUQKFj}~gj z<$idJX7nYHQ63u|2!)qIi^X1U$bfMai@?$^&8XooEl{n?hDTz)`)DVlICaMv32;ZW zPCM-+#cq-b(??gn0=)0wt66cMo7Z74z;YMRc+&A%MVfOQ3JbBcBUKyli)3Z*Ct;wkjH zLOX+ehEt#{CDGWZcBWkQ*iXbAwXC z8A2zcSXcj!gt!Pv-i8kp!%-1@T!hkpGn$i@sL>eW2n(PXOf$+<-12Ia0l3C9ZKcSo zu?dC%QdHplV?Qh%gR>q?O@xO2im{#wqf#2r9Z1*g1K)jnnHeMgc#z<^MWu-71MAHy&|n zps1Ab;Vtj)%r>0d>RL#ZW)UMYtmuf0AwemcVun*HgNO#iXB&`Fz6X3X#o+;>SgrtX z<#UTpP>O>$*=U4Q+Ui4Vr`vu1ruF5%7^2{Dl_63IOAchh3Tp%kRbdo=r;v=VCPO5w zn5@kf#1O?yQUPq2f@J*t5b=ilaOaAQ0>VZQP$>UnX+~wH0bz%uU*}l_$_1vgf^VKt zzO$kC?WWh;eZ_CPoy~2S20LN|XZ$>+Owb38CnWDF`+gO z9piQwXHg&$$7uR1VCXBz^XV7!S_S$-$`?7{lvV*!;z#jS`h@>~r}^q{c@4#JwA@x* zrB-VLZYRW_KtL#;0Sdt^LQ~=wIa#FoRkoHz3IbS@hmAnpb-pgDqbLi*4T$1OXTqfc zh#Cv<@*Ch!;+PgkmyAlA_l#2yU~Qaia#Yy=>0%?loq>?rh<4uoaJ0cc!oS05f};zD z{Ofh`QB1xGMxRfAg9CtsA@=(?z0&0FrkO+;Sc!(az-rgQ_u&0~`uAB1yuT2R}`4q>m0!WkY6c;+&!s+lgS|ahhVE$=3Xk z@N-hw>dDhq+>g`%6mgd^>#>+fIFp;<;{UwkSMpFCwE3wPM+ypl@^6@BMccf&gmL~Y z!oCaR7Hgwy5CxL5IK+~5 zr;>4u13;9LvAA)rdUqH5LA2J9jc|6{lp58b?xocHn$}W@a(J34xjJ+f;|u`E!iRM-IA8rXF#B4qU|yxtu9wC}pot@C{D+?56GxKcP@pxHk- zNUqC4Zka1q{M~|oD9jea%=4$2?4K5ztiu{d?D{emud}W!yj@pvIj>tQ4f8eUloVt| zEI`-QtKqtk&|gAuKY)?srqC8u!VQ0*BEGDekIAx#w_M{|J`OTL&R3a#%afSlK3tH< z$0gRBP|b^1IBGHXOVUDX<{d5D5;)GR9U~G%nZ#R>$)z`0AhJS-Gr4!-j8V?w@tyU< zD_W57!hbSP?9|eCnpQa|q8%^v?Ctlb7ndcvB-pAYNu@z=UVion!&CaP`?K)$%&3Qg zAms;@c&C1RdAY9pK07^sFZWGv{I5n3Uw-yzf~6mcfJooBdR$S)NSbs0BVoGo&z=pmb~oTZd)}UZ+iih&fD$AuFA1g|M{CuJ_m(gX7;mpt4FcG5Ox~r`)Bjz$12rBJ zjlJtvLgsa&yI7Qj^)XUi8Z-MV<-07U7?t4uhO&$P+DFp!^Muc0{!gkLrYO0$G4LY$ z|Gn+)GXJ;xEdPC)r#AkF)aJfN$)X4#^YS*$5AIQW&nrcL_^2(dh_?55!94W%N}Q8Y z-g}nmgKSj3(nb?vUr2!CzoTimpoKg5C5stu2xwsdklUcC2p5HcniUtKDAv^Sy!Bta z{|jx?eF?w<`~U5&o!!#@@AZ1m{y$IhJaYW!KLkPGTSrK4^ISP~O5mC}Kn4kbJ0MYj zBL>?hf0aj*`x_wvN0S{KD<8?1N?n;I97Q3^u%o6+tJZzGHSJ^aGN$!NO{Zp^`;%uJ zNm>`zc(wsAgb|%7KMIJ?D(XGSla|ADx^@Whm!Ch2R^bk4gs!nK^?|SXYuU}0FE8tZ zKiV&w*|5gGCqr~4)cPICsdY?`g1KE<0h9Bzqo)^_Z_ke|u0C8?lL{RuHNKhO#T+&8 zGR+^K9A6&3x!ON}d$DRig>TZ_p57krU+!1iUS!1HfQuzxx4C1}rUh5u9b1!U9Vr*D zj?ZdAhtE>f&w&pFT!lLa1k6rOym)y91O zwU#^Mr~1{!>4)=!Ba@IGCx6-7tE2xpJ3YTVI=?#lpUb0@!>j-E;pqJDS7-Z|-#0M| zH6qV>IN5L-mRG)V6n@)#wRi6U))S1PcY6#Lt)8khP zL$CGpH`49Oe3Y^?*HG><>8g4NTS%3A?MkOz<=|@|6Mu1{xyjmAO(@@BxLZMYYTwsO zj4_@jY<5VbKTQL681}tQlyXT-Ux3fP*4~&?b1#!#9~&IWyvPp zQj5l4bCAzF#-C=Bl^-S(eN=!8^#8r>-ELX`x4ZQm|NBXk&mSRwdr~NBtx>{n$xo@L zBN4Bd`~UBUVSEcD60uTDA8SVvRf}tPZLj#8ZZGLCQ@}Vfgn@2#4&+la6PkLj;5XBU zmh_ydIe7H&lQf=UHo+MM>Zd5db4M>OB!A!&WHJ3uZu|#^fm}rYcX!J1U-x!;+t2j> zDV~~JN!=~X@{((RyilzR`%!sRoYh^{oRYcOpTU}>CRbEzg2SmtC!JDAu2!Sc$(V6F zx>t?-%`A*;(}(J|5bd^2b=+m=zq7nS@Hkc zeYXF5lBb5Lt9l1CBnjl7{}d!#8n~S%=$8x|=E4@aDMzwZS?NlmpFUXRtX%01hP&H1 z^2?^Y28pR@^7CgAf_FbmCg>y+fq0u6s*spp8v~!8U-Mal{(p7;f3NKS%lZGEXZrsX z&qMOR{vl|8`!dSw3Ikur5Ihohq!_`HJ=g`LD53Rj@6}$bj+d6_OF7r46$2s{PAmg& zow_x#^{Zx*(W00oGHEbjs!wQYL^3EhZ`IeVw+PLGf2kjF5n)8dXIT>pwn||gV`F_i z*U#l47V99?AEE3aQrpwilAY~OZG-Slga-5IFkDu3cfI>S!`#KORk|W&Wbd-e~5GOcN zp*bOLP8TF|1bJ}GV1g)2;}ip%pa^<9FvBRN5RU-)B6DbfZ*UkVc`QaGVpQV@Q0h+z z!#>M@WNq0NMrnitAPf*tmd4Q-Qk-NetobNT`Fcyac z-?+vyC=e87n^yf&L`jk{(d7DAp}AD#L}S|Yil~{;0zWIbt0%(veze~GywqF(agTo>!clC44bEk7%ELFN(*|`I^{*i4rd~B?;Vhy`Ghh`v5JO=e69q3JeIjE$u zYM2|N#(m~ypv_OK)oY+(Xgs$;7>LYQtAk=eQy8)i%(jnZG|DUtL8bm!&>#$CdaVIx zij9F13Imy7YoHQr4a}4WgDN!27-vuqjS?*{geD^4y=F#1W7^jaN^SFZI+|64Z)KoW z7OVKLUL7dPNFt6=PH?%av(i_D^BQ!1|JTM zHr%Wt<%X@}wwB;$YNwvx>a3TR-hFRa|5||b<3YpR z2x!(aD)`c5g0wzZwNZ$F5!V+&Qz*>``AVsbSBQ!%O5&0fr$t7kGBj4IrIlu>3QZvZ z9f)(PCh1pUJ5W0}pTFF5p=wr&{L4BSbbcCvxN4H-HVA`PZ^@(KjdT#p_tZr;l(CgL z>!~gQ3MdwfC zrWerUMkaN%`0PXHy~r6<&05rv8H-9a-m45l1Ce5Au)&+?!=TsgP8UHWpeae~EpF?A ziQ6Gi5@6RyscERFQ+AbB>Ve#rR!;?biJO} zyU*lQp|S2&6=)WJUaanFu1WWUri#r5Gz~ahh32ua*m6%&^k%;)QC(Am59f=kN|!+~ zH_90K_)J7Km$|5*QGRjFxc7Nx}Q+MhK9`&T!!ZQ zRi3L%C?Exq5pq7EYg8F^2U-<3xPw6+t$t;YENHYE&br-ySt-?IpjksBNT>k`JPO{_ zx;m(~RVFq&#lyS?A54qqSkm)j^DvHHGO@8qitUta{AOjQOM471-3%2~#v3zhS~W{r z-Q0ZsaxX}7XmqTsI!W8Uj-y2};}?cSFg+X^Bcj!y49#W!ZMhk`gO5Ky5Sn>GrUt5$ zho`1qyx$~$?>8qNNKTaB>GW};w2plNfrw>c&dp%A`>;F1+`uA(T4+?6e0B_IcJ2#I zJv8O$B!e1gR2g7-2$N!N9tE2Eh$0$q&h6}eO-uD?bJG-=WKauD!&2SBVC#{gsV-F& znueu%gt@T-7}~O}x;%!(txareyTJ!jzuHikgQcK z5k_?`Z%+C?YN&FR`4Xyk559|2X}3V4QJhXys!^#LPVq=84upRFKJBa9L=gKzqOA3@ zI-@;C{-hYEq0Aq+F27V?uWv-R`E}T2ks^%G*HrlWlh5&z$GXYmd zv1O?6(P-x9HRFf0&p0b2KyuMa`(asI7r7xQi&Hgw#v%(zmbBb4boCUF)K@JQVrh|> zQsK)WG@ddt5Q=&sN|VMqWpQV{sZAw=brG<{Rk2Vy>k>jjA&h;)&_fC32+b2!${ikm z%px=;zW$!u(~xNF!Lc$isFjh^Nt}g&&>9T2NspZ)V?!sHm?M?<9F_D=~-+Gy-I{m2Aj(Z^`TFd~1DU_9j<6~OQ(K%` zvT6jhS*UF&f{nLn9E&pt-NYm?KD?wbAwhs6(Cu;3C~^>zh~adEg>jM~*8?Jdl&k{} zKxK7BDTXW=r=0Z~%WWk0s%T({NBnfwX<-IkZ`?{70+* zkCri;q~Do$d5%FoU~I8ZU_IXD`V2=`$e zvm|sT@f7z#UC#tUb3*A5pN~c#RYSt~wMrb%i1KDSlp+6GZviqL%fDRl+TGmje%jpH z_L6AqT;mzPaL~Tvf+*vN(0F1x95)viI8#K5JPt14DI#ItNbc}o1`xS_iqLh)y8>M*#>w_cTNvQJR8ZOi@C8~4%ud6QMvg(b-h+VJib^rK8@K5~I z(0_vwew;oQ`Tz01Oa6bo&CO@}{}fNV4R6DEh(b6N6d?DW7!%6UnGoj*>EQ;ym|}hX z)C_N|Q~Gjd%C0K!P)cqgKG6-3VRnUJONdo) zIy~sP>a8!XM2dXEIY(p#k0NILbBagB8Nex_qSolD7=);ATO0O2gaK{#Ym0Vw`~z+YdTXn{{x;~ z ziv27lY$jwP1RIrZa5^kk$SAJ1$~Snq;Mm7r7IET1LOjMYsJI*eeEeDZ=TgIeaSLT^ zf+MB}Z1{aC4CE6hFh8%dHjLQ>q_kFEyg1ERcS^?tDxA>V+zCxOKt$OTr^HuRIvr?C zqz;KflHwZ@ zXT_YXJ8iJlR+90MX7<{j5mBLk0H!4;Ig-ZG0xUO5a7ylXIYt*+RH^9{(_E+%l`_RK zlo~4aK)Vh5#Tllxes%sD0Z0v#+%nR#kV1{OP<~`cs-WyBHiRFHJ~j1WnfI)9F|s!; ze-nX+1r4+`Q(_Ja1biampMWkY;yl?_2YuHWAQ&1`s*>z|Tpk zYtN`;H#N7QtAr%NMZFEd$SO@hKc_${Cls_gX%=~O(lR?Ylnz#-qY#Z(1w_=T2cpgA zk)`N6UPMg|xX^OlDf%=OR^Hmj;zme1*Ix;A^in}c-h;45sUe-WdaW(*nuv5ayfG%YD`66b516GHq zV1ey3wOxu3E6>5H@?;Q>4Eb9>$@h0WrwT{MYRatI2rFxc|!J2q&KMFDmhGm4fB#O+xniTJKS8m>oE4%5m zjmqN9x*lFl2Bh*a(va?MM@AR3)6U74<(zi%tG2`!tqfVs z-Vzq}IyjhIXfus+)ueny%3vw)JF*#p1W_E|PMiN<_x`wlEprTGQe~6ON`(x#!&JQh{uZV*VZ#>V>-9Zr}uKuMV$g_7h zP(U3+FiGPCr%ZD8>K2z@Me3`tSt8L*2L8+4YK-`Lf}PQ=XjzSrL)eZo+LG{HjDj~P zM3Jv}_1vL<#`N(~iF{XCQf&w27PbQVU8l|46l<$kF*S91`9vVTd=!Oy_sl%-OwVCy z#5F~nf7ISCfzl597W1GR6*7bJ_${|47E8+_-d7xjvqzo+-L@OXC~$`g1amG>=``i< zLId36vwj_Zoy7rMXf{ZM4fw7cFU%7$dTr2e39y zHX)8+|ECK+3KRt>{BDZff(Sa3NK`tQq z7w*HWw?u8L55jVl-!>4#a-Kxj~1S#CNwz|wPciYN=Zs$K{GX?N$Gm|tvu;1 zQN%@vZmc7|-Bw0lW%pDB{N9HI#mFTWj-n?imY3v`%|GJozBHKDrlt_58$gU*v>}Nd zi9{r0V3U~gO(4wYgmS_hEr(qQ>9mA@Re-+~r(%wGDL&}QdnHJBRs=Nvuo|Q$jtMFX=eW@FJ%&80+QX{X{KvW$|Bggs=5+CY2`QwL zV$P)SQV}Lwl-XV+%q$wMxoNfQ?pjTFTfT3#8*g0Yo>yW{I2R1iSMj)nID*n_yEc^*@B5DJ{+HI=oSixx)5`G z8-^&lR`8XN`Z7zP(y%Q|b3F6YO8R*LX(zFrEE?XB*SmDWCBl@T_2gM3tiZ%3K)Ao~I^%{}osklGidZ ztx{d_N=cQ3ogfjedoNQhSgO(N`*Dh6%kzt0RFITEs~zw@j!2#>qeRzA*AZlR(2$V` zY@+NV+J^HDOl6C?^ny1Q4v~MI8x0kdR!beSDC8`lcIwux3TFETg`Cjm(hJi@;&y`A zODZo3azd0v3LL~KKD~f{j|N%S4xjwrRAv&W_ne;KaH?&0S&Fgttka&|)%k3*Fi5r7 z!|GHcm0a-0>aOOU+bjD2AsbrWgDzC&>(y!&e7 zRsLXQ_CTBDaE33y!-6Z2hN1JB0>`k`5TSWTB&-?$mDTdHEFY2lYl@OS+__@%o2?$g zG!y+@%&5;ou@DD(9r$}~l91qZ6sJ?5vYbRAP6~zCH{jIe2Tp3kBHq-Rq%YF1mQ1DP znRI);MQOy#tUIz~y<-4fjt&QTsUX)BgF%@yV0hIvoj-+|4wm3_@~^lL0#=QLe`3Y6EVu z%BRFHY}t6s?;Clf4|+?OG_GppHQ5C9)FW}J1PV4h)0rzA0gS$^oZF8-s}Gvg6$a>(@W(`-5+%HhVT_psCdd9XiS;VB}Q_Tb%de;FDot{)uD z{E&bUt-y*M3OaZ1{!G%ga^DZ@)gnZXjc+Q5*oBi6JxVe;Jv_QP zJ}c33VW7DjaVrsO!tj)S?EdUghFC^D6a*atp`DF=+`;(A87y`Ud zld3%%MVJZm0fw$IAQ<6Wn35=CMl49nQb5Z1ITqg_5^_}bl(Y)qCnPxLiO5IEGPzg! z@X|W&Z~P@!=yvg(zqD}j;^_Rxvg?e{mdqhG@F(4>(`d2{NA3q^1rJI4ilVGk0lQY1+qZ? zx3#;oS(5+lZSL+p%m1F@`TV)_4cw4vUs$Y-NQi}qYRad@pWr@x(-9l2^Nn-#DN%p9 zYb_^o8I#yi=otHAX6*+lA%y269FZOfYg6;e!_ABM0v)ifx|*!h-Lg*8c(g9$5L=Xa2QCd zQX^WQP{OXMQ7a*b@O4hGzwSB5BYTqagGo-1^zvu9#l4t9ap~s3D`*?0uQ*xULg1^H zTYt=#T$gDquoO;cwx0iZA%O3{ULNfXIP-astyo8D0ju?PWdO?vWwj>zU1fhGH2p2H z?;MMummCXFlB98x5`kR_(O4^!tBSjs)SuO?Z0VktSGLxe;^_@e58`RUak2Hc*FA6B z?QXz-_Pjl}%Xbb+WIBF=smIY+wc@=cth|TBBtff)y3o1l4KYK#d6Sw0N}{oM{Yoeh zZ+i2mtTIi~UfK%X!`Gw<-IagPBga0i#XC`0s4-5I^YSoBMtQB@YpRiFw9e}=G_Drp zFYkcG7c%OCs|mQsv$yH>dO8)q42NN)rV(*4r2)5yIG_U+n8_1R;a3CHj}yG%3^N9* z@o$CbVAJb$A&4=JUa}(ZKidEr!z~sT8N&Wq$C=fZBCc4;Fdh@%bM}8j8VenSk9ade z+6R7u%Lv2{Ct zp?$G?=hZGRu2?wN=sri``JLEEwYXIi&CMPj)cUF$sMtENrf<7CMzi= z;$mS|SZQ~aYcAYnc7=c1-S$YcyUL6b%eX{YhPX*#@XVYNf~~Rd)}?aBxm@m|@rx;QuiEa{+US4& zTxq1H+_F&D))lm?`eHCi@D;HY2$4W)f zXVnX$rq6PrrTeT@6n)}P*q0V&+UQAnRnm$2!}Zl4oe4%En{*Nqjjx2#^a@46l@^ap z>SRDmY}e3AvNnIWS}r!mX0_f|_p1H5al-6k=1#T0G<`7lX~h|+ahuvrth7rPS8dv) zcFiUC=)6jeThxE9wCoO@TWih+wd*avKNr-Svpwya%kR$l^?oCJGdDu=l^@RB-C1(b zG4^NaOLd2q3p7q!xuCgA%io$mnLD+{ttfXN;tO1^sy;3aVqQV4so{lG*LePU`R0o% zjrLHD!mqOQP@AqHD+Xobs#^goPnu*)b}>)It}O08`9OcyWIW@m4tz=Qcmmilsm zqRiqOO0g8$gxXk)!Wz5grlRf`c>~g@y1w>3FBuudILn+1cwp>wli&DI~akxam39BntYp04oH7cadnICc^3gm>y+hHonvq&LBpZ|i>dRQ}6-96Pa*L}5w+)s4) za+x)?QI<T_t=)dPcgVwCN-idz!oeCjGocE3T6eeHkeqRiX%_%A#Vyj*0xp*>ra+ zTJysob7y;G$mE2mmzt_SY3io}jn~%pd)|W1C*3ADn_!!zr~$wkxc?Qh`dyo*p`_0rAS+}@Xsw>lCKy4HF^+EuB~-(02 z`q2tFP|g7QV6A>tqLbJ+;QnpyInJWh(o$Vg;u>9xiEV)4S-&>_Gq`ly0xbwi-ZCUp z!+cF#p)b=1$mC$9DZJ#JH2Vta$6numjThbxTK}MZN|IGQZF4L>-~Eh8>MSv)+g;wj z-Jb6Ezj8_*DR%g?8|7yS$8jIlKqPe8b!s3=(bf!9<7_T@jn6UwdeWlP%d8zzXX47` z#usbYpEVUr@9ioSpZlx4+JKNAE{R{a`77)6#+`m?QL zXXmTWhv49kE=lUe7Sd`q?%)dam|a{HXQXs*)2kF*xS?3|{>87{$?sdCOWrLRAfSAwxdduZAfBO zQYrNFAlL(d?KWZQH{+EGd7=IG@ynWT-5+O8reTPd9`(hPgHpBh@@Tq%!{^^^28brJ z%ntSrE5`06*_M3u^}WB}x$;zGOY&#F8R7rr0@jxZ3+~Ex@9k_kkI|`l_gRA5tK7t^`iIv^ zC6DusH~xal9wjQqM?KTp9U1C#Yf434bjqQNlpWx3Sb5Tv5F@4D#6o?c?J~q;Ul#IO z$Zi34pY`=~JU&C>rIz$A9Bw@4)4ylfd1VtQ$b`e9C1Uj^>?@89WW;a1P*1V<^sXRI z2p_+YSHRxgzR?K;Jc7J4j-2Iv;uFveI$s83Mm4|CM<7PRW#%B=(2#K*T7@E3Ccna= zWJ#E{KpC2FiBuk{eVM_(bnw&tg(4XJy8zJJtS_1~@tkUAv(subz}l#6B2TF47xg4G&*gMHkQNZMz zi|)IGHmSUnY1*@yRVpmPtl`Kt)7`HWqt)N6Z^KzTYb<-o2i7!kZ+u{~5A02qFA@c`A>V}5p77f60rmBn5f z+oWS{+cM6W)r52H`1Pv9-ZeE~QW#=@xRMy(wZY|e!Qx#Y22s8uk*mz`mU!$ZM?i^) z{>j9*SF(Dr2(cDlq7BCBoTD|PZ_5ftrq_G2p-*(%gqlu#tvv95vMLqSD_->&SM+J1 zrS~WbB*6Rxds0-hEBcygXR%T{g`JH%H0&nuNuyk_S)+eN@tcdR`V;&ENMGqiUkmx2 zrvaMebIWJ#TkFV{7{&T{b9_x1IODA;v?zm;|x)2b)vx9IbmqAH{y+^ptyd zS2w04SO(WGW#5EiffL8dN|i#lLd!saH&o~_E*0OUlWCj`GS7)-{oFu&oWv5NtK&ei z+FYy=Ic_#FzS(Sb3b>pdPt;&NpsQPnD{ndOKTjhb^*dp?NWq?1AWU`8tmbGH^uHkP zio9xFOHF}&rk&bEwmDvA4O7u@0<6mpVGi*{BMkP6C0za*1O*<;^+EI|qAjtd)b{^6 zsLDBo^-;8L$rwWg+tpjrU#vj&?~(bi`kI-H*<#l@jzzHC2CN?2c+wG?#{3Ms3}x3E zXVhnv9M>J!W;ZEYC7X;KCC$G=ijcY&Wcgc;Msz!z?f6Rm?fDsfqwE}=T+VOXe0^Hf z_II*R)_r4C?GI(G=U>O>ptp;`v*wgDO0u7RbXT9R0u|Vs8u0=cXX@2mAe>Td*9|3f zwIQ%}&1*5I3b3G)HV*X-g>=V<{S94BU9%L6FRUEqgs7FxWk{4gC9l`Re8(GN->`U& zz<~QPcz4LaM9Htq;8CevO%1~&;A|l9u{^Dht&rIvIo84Gv+Fvz^ah`4{OaN7@_6}d z0J?p?-#>jnF^uljia>r(jy|f8y*FnowZ@(LXm>9a1ayfy1>4`e_wrt5G7Q%;mc=dI z6sDOfquIm?YDrs=TU?C>B-$y)+lHi`qMgRtR^-@4#${ZX-r2~EA^jy#)JKv@K-K;7%s1|2k?jO+&eV$=YUIqJo zS>pGopVp6`2g&;WmRh2z0)7;kXNNz$(w!yUNR4t9IJ$KVa4^4xjjO5-9?k^HkP~29 zB`bQa!uL+t=OCh=QuJ}<38p~aiGZ4!S#4uBDgo3^cEev*ST%UAdTR?RTF|OcjX`Mf zCp=Y5G*Ozh8!CHEj&iPs9kk}0EY9+&^81-uQ`Tpu6`{^Kz&G2d6QA03%O6<>2bY!C z^%R%oALL!pS_Csr)Bix~iQ zQGw9HIy%{u`kKpK1M%a`qdpTiJ4!13otf}*|EiM}YM_R6>mR8W|BBr6{bcKoO=b|k zw2Ylp_%PQor0jP-l6vfuH@YbfUvB87{@{%1Er+A@mu!X8;@Gt3S-L0kGh-q2+1dxc-Y8&k6XnyfoLW6>D1XQKp`M zubtlSr8v#F+rPv4w^luKVYIKG9>}JG@F5Zh?_5c{)!vb|*u6t{7oHVi@lui|Rh={5 z2|m`QV|hdrpL)B7 zT!dz}TDz7Sj-<$Tk0yxa2JbdWYe;a!HFeBDb)H(4`%wFS3PY5psmQ_9q?znx{frZj zXf^tBwv=w_qcoQM^BZzGU6(-zka_DlX#VUm z=b}4j-_z$RXDfbCJ@FDSy~=rgzFatZu6Eumr}?wuJY3V%us5}Fia%BS?uJk;J(ov0 z%b$LVF<)G2kw4l}l2DMnAb%OwZQDYDeAzl^SvAD{RoH?pay$`c-h)y z7zUkPTwDqagM1T84;2zo;%lAFG_Ih?iZsK^SB?&6NxKT8*&PsSxr}nP( zsK&1LsL|(Ds6=U}Hxf-;H9j<*K&aAR;w*dwS?er7R~Hw_|GFou%|EWBbLK3qF5lH2 zf8!x0+237R*Dj(x{br(0j zvgDXP0^xSoqDKDHTcFO(OEYfJ@ZsU%-3O3h?zk~i8V+Jm0?wa=!HR@R$Fd1{S zMMNDO+HR>B;nBEsMzuOY-^~~F7H7P|($u9}U#$1+jdDU<2I<*;)W6g7ZuQh_$#H)) zFrbh`W$S2sinWYN8h*iC%Tp=tBTxoTOmo^ueYhjbKQ7VMXTR; zUhxk#qY=H@*k68z^R3$T(KwxYG_*OMq&7h+>y^IkCE6HHR>?mZ%Sn28z@I57;7DvH zxny}|C*nAIwO%VGXnA-Jk2n`9U3tEB+nxtWL1>n8$t@&;~kYaL8anI(Os}WAK zlI)w&rWN0pz@F^~jho9TuBnOf*rVhe^($(lp&aGqWunCKG8a@qK|C`Vp0?eQuorxa zD_g;+)xM|=)&$gQ)xQJNrF7R3eL2-fa8*9haiY+A=tDOh^XeG`#pm{0j+)#g^IEfc zDc?&%`1i-2ch{^slJmqsWl8{&zCFb3x0NopY%11#a-$*gE!o-hWzNH-cCMd%>V)ZL z=P5r{Zg!6!q=Lb;fPVLn+*oZ88&hywW`mg+iiZCdbn2U9lK&X=}25o^Lxg9pr*0nYy{X?LwzJ1o&Q3A zKxow8kYqpdMyq&?Pu41(fAhxk3NXHN-fy9v7@NLc{nfK~zDxhl$Y&2^C_~3U<#zF# z5$pNvZbHN}K(OYfm3+7q{)M7HhS-pMB#`2|`AW!514{S$|C0N*0Syd|m6lZE%5|S1 z36})D`Gfa41nclX?dY}Jx+Q2czsM73&q^JuCgzBGx zELOrY64AaA>U@ZpQTkp>+}M5(IsReG)}7l;ebi|pb7ovP<8%%kquzdMLw$gI7@+MgNy&`YW$6e5U5bXzx`DbKDtZpp3?Uu z*Mp@%_m8^OG{!_C=^owznvSzj*9sF|rQ(3-A>0yb5#ubcm^%pw(y*^M9^s^gTyc-a z1>Q;ILrOJ^1T1N;j@WoYckC&X<1A7B{SdGXz8IDki@UkS8X7Z7eW@n=ep0ywS zjyva)QghFt4k#&w#wlp}j7L6*xhD0mc+!PP)p!skMW0Qah-Mv*PZ=4~7oUCSKVkYSdMumCv5& zO&u%+aEk!R5dH^3aYPdM}9^8YG(;1bz8Iz@P7e?7^W zO8)OKQ=sEp5;#dKfX|>-#)+2L*Pft#AS#w|C^({<0QML@6q7OaeDu5=# zaX+s$_#Y;rP!>!5jxyU1_65g0bq@o4VPryqU#H0=3E)W5!772}u)~EangY~*$);F& zF$#>+AV0THD9-5*IOMv7^|sviF{p6($4^z95K29={W*MoOs2ZTNgB!k0&0?Ol~24; z2uA=c33ZJNjAMoMek~Wi@t5oLU-Mw4J`Z~YStaCgQVhfX-%_T;V&0KwBdzcNBbpaR zu?}x+L|$nj;jd&0`DhYvh#&*SIe;zdcip2zfgzOigoz<41?;_4V%LG3Reyt!t z(kP*gN*`EvrRAh-dWM$H_x#C51Nw$#O6KLzGzasLqQpFY=5z#Xdr^k>q3yv*1qTv* zHq?3t;4trNPcO`~lnQgqG8=))o9vKct=zeDeW7G{FtLsS61yZ1l}v^xRgzhn+v6jL zJO42EVQ|6_Gg&UPZa(csWOU3#p{vrv1p(acWGk{@Ger}4oU$dt-DHR$1?deuqInkk z9pMo#Mm}?vEtp|t?{h^vbbq>zys-I#k<=<0u!VvVOj2ox$Y_w9Ln;Cb*7d4?ns##di%@5a@@W{kaT(AeyK z4IIyQeeeejaknD(#CO%FM)tE%09yz4n-Pt;U1oDkxw$u!*`~=qLEm+;um#*5!awm^ zv1?Vu>uVH>o=P?6%MGlYWCXz>=V_EHAd43_>JutVTAkx2{v--l^9*nQMm9K5+i9GV zE2w1;Yv!qh1%(Gc^h{I^g;DUU&;5a%>y%m$_*mHALH(6~^G&ELufO+*BzB4^Mp3bA!@e2esob*?pnx41 z7a{Yl?mBgb4SK)Eo=nlD)SVW-gQ8ZB0C0s%N{Qy!|}*k6FhL9(SURX21VHA!N4ncLy$`Vl^t>{Z_1^7AA0X8gta z2+wOld{FSRl#PgClt{UEz3>PuWTY*ZYg}!lmV^=Q8^|*7Th00e#uP4)j_H4v#mL7Q zY8>Yd=sGCb7j^}#j0L-W{=zEL5fxB}m+l}9arLQK)bH0hCGHiYI>-Wbw2O80Qy~Yj z&wM7F5^0-5C(V4FN5n|(4&+^Z5t;|eFf(9V?{#%%8sS@Tu>vev5@mdf?hd$;OnB6- zI@h9ot(h3igEaa>Qey)o=4brDBMYHU)*97_6kA1~@&uc}A40_|c0yE+*f1h#Oj=%W zKLt#hW%_sNe~5l}7YPH<%jaXZ)?`JE>hZ@RLyg>@IZD5Tw8;^0VYNuR_On9RGxDKS zRJwkcf$|*Ay%w6iR;-c>wx$lqrp0|WhV_!TiBJSSI(WJ%LW{-JWLbuBZqQhSJvD&I-M~;oUw+SCBBb58 zjqRrO&E&v9y3HpY$}XNN&;Q+N*Y@cWT-Sj|whm=i)Va0}hmVn@V&Ro#8Jy%CmT7)5 zWRF1fQ#T6C!tgVqS;NVg8q$mLz!N&*O2h}7V!6RoIkq(iLdZ*0%QBOzCx+n_9IKOM z=W_r{#O%Yi%j>(;*NIT>D!!b*S%w!lv}(nd!8GWL!#Ot)i{SMlyw70q-(tW9ILEGy3NdVXiQa&Y)6t}tx=#NTSA6H1Gd_&$em z<6C?RpJRnD`8E;c^V#hMu$j>${i+!2+Ytl~tyURDAS+W7UeWLZn&t5I{sag2_1xRG z!8$*|B{7@#AE}5mJG$)fHR@22|7a)>v19SMCRW^YSqketoVxU8UGWcUMD@kJZ`(W} zWbv7_>P+=e9sc(hejdzGe^ZH%4;soGy4^eDdR;l|G`kc(h~027&u+0Pv@@jvZS@go zo89Tx+?=Z$n9caQWpRh1fGG}pSNj!NJSra@!4g}1uF00&gZ7#q0wX1qV#3Xi-gAh^ zhh(ddI(dkCCu*AI;YZT8k8`!@mwtzm#WezEl_fXPef_)J!Dszbk;$YzluPy+&wg5Z zr6|27h0MXeul?*b+G_<)7he@_PN0i`mDtiBgwR|XTmFV-GThW={yu^VA^Dl?D;-45 zTDj45y4`b@ojDT%G2^7%)*gLp-&^s=@9J~H(4L1(O4<8)WcLU#48|93su;V^jgim5 zy(?Lkt^e?sM+rGR?gcP+B(fJlzh&@v7IJC$Ow=$}d5^;rR9{(np*zMybCT`@=9pCw ziJy>JUudLJc!-#cjx#YWROdYQXE=pEnXt1>3ul%2Yx!4-?+kRRkmInwh_95of`ltw z^+)I5_$sj}I^sxraPTzt;`7#xg5Eu@ErM8h3E|}7Z>}6qax;gP4_Ogz!4d>|x~LIY z53ba4mxi6A0&`XRt}XZX4uV60=t}VqwN}Ne?Z7u#xC-cGrIjbFTwdHy5e%^ye5UgS zN~E4i$vPiu;P0$7nB-W2DlVR+$JAJssu!0|NWZg$C(8}^|`>V6rVQJkx>y3*LL%rPPqx7NolU$|5t-kW3MY!LtF z&U#)Je35m+<3M?v=eFX?bQ&8H+9l-}4}W}?+4dqzRiatETH=ZM(8ueXp@51Z|%I9nuLmbU(H&Y;cqB^B6oZl3x4qWbUF4@mBqokpY) zb+`PUS+l-x{{$94+mJK>h3`7LRHv{DmI;+Gk1R%mC;OTy>2bO3Y#okwOYc4b8veWZ z_}kX_%x9_gE4@J?IQ@8zG-b78qY#I!dl0$pOp`W!0-31xy@tEVELw5;Z{GJS9KNt3 z>hzX_dOm5p1JqHC6fxp1!kj0nqSY@DTPI{Zz=pOtD~Bd_TF)VX+p0eyuZ`!YJnv~&W#Dr+#nr;O#GisT~CAK7(60LxY0`-00kd2FgPvdmOHo@K? z(LNTZ0tU5iv=#?!D~z72C{}VQf7rS6Jo$&RW1Q)j2~j{3HqMS2EW#gMOe8u#j0 z<|ez{uCQG_OTH-PAwwH;f#xC0I29VE=~-#Qpox7qf>xyc52st1!hL3uc@K}Wjl{NP zje;%~Ckn?ixkmeTJ*yZS@0Jl9i!7#p$9{A$v7K^@YZL<-w-`t_mz(em%sfx&h?w_K zrYDC@%K8q(*5WC*Hn2L0&Q7RZd_k9_H;(Ue{7D`>MgHK7aB}keklPYWlhaV)S2O>xXcNxnEoTvWu1gF5v@7@5}9NuP1N4Cu*eyxqj8G% zo6hX?7lLC9*E6M^hnws1VW%kDM(EC3(ubq0-p?>Z5@gzM3ueS4q|zjrqZOq98&zd7 zz=u}JF+irU7cQS6`Nf9Jn*Y6!Ta)1+U?TyMnqZ5IsQ_H_?1W@;)QW~-Kdva&^*&@W zCyhwL!9&e^Xg4To+b#;KwXrdv`M3NWO1+>R)Um?=LtFJ=^4AwPxH;s%?|d6DBg&lb z9eRz}p!2U)491R#jf-Cu`lOho25$Dqfe}4J!X;x0P2^p2D-%sej5y3tLyUN}tke9u zNL>*czIyc}s8zbCe(9e<*WsZz=nS_}oduj(ChXzA$|jd?_Q*ajK%mfokNb;2``5!? zUJc33@sv$}`qKJml%-ur_>KR!_PaiM|G4qs@~>S(*3d2tN5T@^nnFrD;N3pzg#YVm zSi&fFLa*f<#)Lb1Fn3U5ZX@M_yfUDEr5E0Mx}|gQvB8v+v@&9ALjFf4k|UG08)gH_ zZ6;9wCdY{$WW(h#Coevg&0HspiLXF+dglMzx@CWuan)(F`gwaM=;u=3n2D@R-xskB z6EAJQTB~CVC$a?sZkFqGqrIO`mp4~SSE0Jj>Lw9>fr4(Gj-Jp7v4S6e5RPX3g=~q| z)El~voopPPbctN7YV+S3o&V(LNpSTUbHE!MOM^OCW?~i0&cnf98P5R`xT58oN2iyy z+7$2K{~7}l=O>nU`v+ss_W7c!lG6_1zx1q8T%WBLoMZsqiHXp$yQ`5MMw(*@th&3* z4Jg!C7V*uSV;-_q-qBbLM?B!?Uoke2|MI&JQd)cCeb8#M>o2pMo)PVuR3|i7BY}Q4 z#3<`up;1-JFFmrpX^-+q_;!kBRdHypk6+tUoZ%ikqR{l%Cqg&le+-I$Og+%*N^6hd zI5Rpqp(6mETTxU9P{(9R%aHZr2Yp?-*MKeRiATDB1;i;X+p2y614?i zE2jmo&5PR|vWP*b^+18I9ZeZyO|ZetL^A3)6kz||0X)EUX~o!mjyoShw@5Vg6|Qec zJYzH_6FPOVH0by+Z4SK#!y{pu1t)Z4Yr=x&hl9|Cdj#vyMhWa1G>G4+D2OTl z#AO2DvL*E61q8Sb`ClO|hu{T>-;yhwN9pbTIjWK4IKYY+N;M&?O z2`nP2gT(jsVn#EdL2exR;X8h(oIub>F=7rd8p6J}Dgvqd;@3>cv#rKTA-{*3(Td+F zqCdaiS^7dL{g;)7k(4a@kUT~9%ed7N7)>R`$^m70L3PULAbM^jbLp;3Fk6BuctM!R z1l%_cd5a;4#Q-qxlR{lCjT^GGx^XW46K|_9dwN`^R?Y!#RKBY)gcYto#K<{u-J>92 z>eXZr5dtnsiI`D+>4BCXGjaD!x=rn-_DZ9zp`tl?C<8ox-7JdnDw9hZE^!_*J>6_Z zaE|;RwR!>k^Y>bO@18xjigd4f*Pzv|&ZGlKNf>w6=bZN^Q6>eZ@00^!e>MEj35Asf z?}tddu(?URfREL?j;(#88k7M1Wr0gT^tdH}fFU1|w)r&R7XHp)^+0F2Iu+=PBtR2V ztJ;t&(4fa`Gm0n1(%T;=dzt~0wxz4osznHC-Ehu%Laofwi3md&=J-lbi}mEVhq0%X zjKuVt-l#k{;|O9fw-0l2#|6EdV)6(VJ70v?*bt#;uNS|XTt7WyMsX58|9k)#$IFk| z71hX&qQonlMfjB#(Zv(>Ga>#Dy1nlf)s}s=$DwB-xMx7~3D#bT7&=7I7;@odErcGT z1<6Mwj-$wTPo5nXh7wIansv|GeW-S&JlZmiAZ0f~DvIpK+{9o-k*1>&XjBZMj&G{A ztM0H%Fq#Yrm>P9nRr@T_D8UG;9U~#332H7+XGg<#txN|J&rPh)o1R&~06=ZxUe!TC zGCLoTsW-O0R(7p`hsBr8^oJD=-^oM1rx|~Q&PQg^5R%(hxb%(-8e==ShPjy^dxEJhqp zOxhx2%t$Njg;c66)F+)4YE;t&{o`TSb~|4rpW|5)dX}-sE45L)$Zv=^Zd*bICx~Wx zr3Y!fan^y0u!3OKWlDOisP4nvNVOQvoL+}s#F|TRY$vG@k2VrODird!9#8^>Bsve= z_-59hrQsBiHEY40(i@BI9ryLXV)3Skl)z|iAwbut-pwDaA!Cv6(tK4>EKg`>vI;d| z&>dQqtCb7dCe;TEZ>3>kJdz16M&Ok_L;Ou#V#5vE$=--Yc;_+YgsE96sf!g%6&n-@ zYZw2C_45|adb$FTDQ>`V;YbX761n{HAc1Ish1VO2qj>;FMivWhqKmaDPSa3u@)S*j ztR@u_=WWQ~!pFj4ro;Ec=x@;FU5KAqRHRNnQ@!R~Z>a9@kZvHuGEo_jCWq@RN=Q{b$5Ml9sZw$?-gEHU30|O@Rk^;9Alx5i#S@1k_LhQ?PKQ>(4NS=l} zL8*AL2t5FcO%Srp7N}G)w7>D?AIQaH?E>`r{QU8g(7SFcK(UZX4VeNtY3YaGWP_YG zn$@3f6`VOYDtcHZ*9S`#umX{R6#8flp6CNt2EIHs;dT+UXcB1n=_h`DT|DnDRV&-L z4;G6=kkGI{>zgrBEfsR^Q+jcVWtF!z|k zD%URfDU&%pf^CkK6_{>@WdM5XIKS8zL1r!kWqKcF`d%w+=!Re5@`YcGjUqBC+SCx7 zgfU9hR}f=IIEMFc47hNk>ufI!idM>7he)CXfl?w4ZYtDS_Sd+9?{vmo*!!ey1tgE~ zvH+Bb*~t2Z%W4?Aj61%QV{z~Yt8s4wn4p6HG8;|tnU~yoNnLTSOk!b6FMb-E09}OI zu{9(+{xk9>GNuGJ8^erE`FX#qWKR!E5J4x6nJcU;akAMjC6&Md=yCe467KIB_W31C zRM;Zr3Tj3I?~W1{+-pSs`^O$s8}UVpZ?SYND8i8M88) zS_-01A*-TuT;JO5hZ+wNxFKSZ?A{UJXS9*uTOuW7moCR7;$MC+USrX3~*5O z(;@T$POaCrRj!B8L2nbqZ&4&I-Txx*cgW26>j4syoPS871b!MjwpAQ;7d3WDt|j7i zwdD3k{gEjq&!*HSBeOQAZ35PvSVT>i{-@O~E|ojok^9)9CWZ4X4jig=mj8fHh(m|a?5|hz_rL_C?YU74Tq(;0 zg`E!*?o!Ko15G})gY^Saqgt9uR4>D7qQ&S~iG2@Y0rLtnpS%+|5B!N;IbdJ-r<7*T zykOMBSG;G^kcM)wLugH_{wBZufjjHCJLC1)2smP9N^51zoNCi}&_{&#wx{P`dJ7pQ z&Hd}|W4$7|#5MM|zg>gba35o|xr`dQOqEF-^zY$IQ90%tWUf_e6ff?xugOA)Bx!4e zzW&x>Xk(0nZ;0(Mea=)RH#ME(XALR~3PKVpGfWZi%-2zbfEM#@P^{W%y^!QpAhHLTLK3f2re~{}poD=J*$(LUvNML^xCt4iw^xB{=CWt&q?Z>|`-l z>XGzx*LUM_^0b)~wLT_DbWrbzg~y8rOFXA0Q>qVGJN+v*93_>sLI=PHYN1@Ur&KrJ zStbryvCp=RXQB0|IH1c`v0qwD8V4WMY^dPkW~Fp@^@?z`cS0>5`&!;ZLTH2|SCaJG zsVl0KY(e)rI+ba=QK<1n$Xz{AUG^ol5giRWRq5mX?ZJ9lkJ^6e8aah;Zd9Qza@;4| ztY9~(RAmY+f1$j_*bTs};K~&m!vfEJW6C@v8=w?f5vN_2W=XJ0K+|!#_h7j5-wxCr z@a!z)HFf2VfQG9+-G(f0_Ck4a2sZrj!b^&%HiCV+l2JKt%0gMVqg2lYrtEx3{s2#) zgCeE4&)0r>LGg>1EK}yjN15ESLr%j$3nS&L-e?KFx0h*Um?JP>9tX=yBZu5kPU@dl znGCfSDr?ftp{%zo4_%TDmI?2BX87BwV6?JGVj0Q*l#O&CK9HBv{Fta_zkbXzxN?zg zhQ%|GrJ$L9j0Lav!3&@4ul}UkMy0*t;@|5G7n7= zALJ@Xsll6r?l$1TZHpd9Gv*NHSDrzi6aED5G(+MaHIUILGG4AwbB>C2q4esgNKdXo{`41M#l2B~MCY8Eb`&l>a zI^yF#b=-)8Z~7OsGId6N3s0-isoT_>_A=~XE=n-br`A9|)7B?g6U@r);`L(kYAh-a zKmVs;nvWL|yB`BviB>2KON@5gNGqP?URHRqc3!`nCD|Ebrn!M3hq5vy>~j_sDX!?i zh|@rR>?XIoG3!MHE#u`EukmlU3bG=!IbyYwT4|${ZY2$LJA8yRoeC6`71k>?V;2og zV8wy<;i|yKYn;2-Pkh-8iPU&fQ9QyF)pQA~Q)np8dbn#*#9lLT5xYbB#htVpn;}+{ zgr%>A+-S>zJ~wF#Hmxe)joLWK*Zj_QUTCG{M$5s)%Z@`osv~>W^O_{bM%Dpi|JDQ? zq=gruY*H{b9?DL=KL)1L!rygragi6`Ekj@6wH<**QUT%TuOMUnm-dIKhTMn1oU?kp z+FFhZ4DXN#&*mWA>L;x7Q)o{U&sRmbk(U`&EpqAHBA;q*rC(D2`P(dbm3WfnY)n2k z3t8ftf(+;B@`sOtk763|rYg^~6#LG7eUSO_@AS3nE&VO}t;yu1Jxifhp+aZka!LaQ z?h-i0_^B#6tv18&2(c%)lIC+atzp}MA8xue5Vs5WW>Y9O7SDp4WV(@pq0%btEehre5#7N3SOG}DB&TQ-uXfcI{JVh26tlV!t8n_#(q1p^ z5s7C~g_PLX5LlM9ZOklmT4vhUC=gN2FeBN8RqC8FczeDpQe!ak3k)Z~7jC30s9KJI zC;v5?95wT=gp(wd`P<+=5_`%A%n$sEf+358CfH6pBamiejK8L*ZTw$d`mJgKV?;pz z_v3l(*>smE@onqr%m<9x;#QvIE5V`~P{??}+jP?TSuLkyGDsfJ$~%YZyoRQ$Tw0n9 zjkN3y^_x2x*q`Vc+!n_PlVK!rzc)$J4yiUPytJPu9?0ruB+A;ZHQAk2BwnfHT-VJ; z@NAvQ70nc}{Oc-hFkd1%%83-{#RS%ga8*75!305*{vGZLY7*B|OyZM!C%OL&S?yuQ zQd+nNc4~xC+)TMvWdKt6zRCDmo45 z*Q?QV_k1(Gvo<#5*1ybjuYEA3vafRPb4@9C$+fycHY}&#{-Y<`iI<3D`soyqE*AJc zZ>hYU^D_mN+1mi3Sk(|208}@AaCCp%zT@dv!^w_bF}vCDw%^eoF_X_u?wauPHZ@rRh&Az z*)ms@)O~yM^+5`f-<6hV#A^Da zr@RB{vj=<{4&_k91F&v~Kz8oWC(wz7%i003Ic~nCn>iCvmlC!*P}u9heExW?{^WQL zY@7{#$e-_1h4Bv@w*q6L^M_Mx&JZIWz@IW!?$NZLhF$g_j% zM@7#e59ocA`Rbmp)owimIWjr>166UUGUo~nJ=)RAB7c+Rj_-jnF}P$xRRYXNw)lc< z7@U}e#IIud^J1(H#z}RMWOj#a0JYMyq>S1#yl3zC$9dk{zedaW1z+k3|xt}>_@+`qsAWfu@VtPb{%>0m#ce1o2i?p z+J?Xm@RM*_Z2bjzR}i{>pHDU;&lNlWwdW}%cE{v}z<0P?*>&2k?6*VZ89J+>h<20W a+bn%`A$h$6z`?*kphNeQFtAGqu>S+636A;z diff --git a/charts/dist/index.yaml b/charts/dist/index.yaml index dc07348c221..0594c1d59c2 100755 --- a/charts/dist/index.yaml +++ b/charts/dist/index.yaml @@ -3,11 +3,11 @@ entries: feast: - apiVersion: v1 appVersion: 0.1.0 - created: 2019-03-15T19:35:11.848995-07:00 + created: 2019-03-21T18:38:21.941744229+08:00 description: A Helm chart to install Feast on kubernetes - digest: cbc97b4be6a84a33055900b620d2f6b6176ee81d86eb2e8bfcc9096ea040ff47 + digest: 68584c58f742f3ed88040ae8658d549a2ee643334f01c684bad83e2937eae19e name: feast urls: - feast-0.1.0.tgz version: 0.1.0 -generated: 2019-03-15T19:35:11.844526-07:00 +generated: 2019-03-21T18:38:21.937413876+08:00 diff --git a/charts/feast/templates/core-deploy.yaml b/charts/feast/templates/core-deploy.yaml index dca2c5ea502..532f50f90f9 100644 --- a/charts/feast/templates/core-deploy.yaml +++ b/charts/feast/templates/core-deploy.yaml @@ -91,6 +91,7 @@ spec: value: "{{ .Values.core.jobs.monitoring.period }}" - name: JOB_MONITOR_INITIAL_DELAY_MS value: "{{ .Values.core.jobs.monitoring.initialDelay }}" + {{- if .Values.store }} {{- if .Values.store.serving }} - name: STORE_SERVING_TYPE value: {{ .Values.store.serving.type }} @@ -109,6 +110,7 @@ spec: - name: STORE_ERRORS_OPTIONS value: {{ .Values.store.errors.options | toJson}} {{- end }} + {{- end }} - name: STATSD_HOST value: {{ .Values.statsd.host }} - name: STATSD_PORT diff --git a/cli/feast/pkg/parse/yaml.go b/cli/feast/pkg/parse/yaml.go index f02698171e1..b8d9ac252cc 100644 --- a/cli/feast/pkg/parse/yaml.go +++ b/cli/feast/pkg/parse/yaml.go @@ -34,7 +34,6 @@ func YamlToFeatureSpec(in []byte) (*specs.FeatureSpec, error) { if err != nil { return nil, err } - ymlMap["granularity"] = granularityOf(ymlMap["granularity"].(string)) ymlMap["valueType"] = valueTypeOf(ymlMap["valueType"].(string)) if err != nil { return nil, err @@ -131,10 +130,6 @@ func YamlToImportSpec(in []byte) (*specs.ImportSpec, error) { return &is, err } -func granularityOf(str string) types.Granularity_Enum { - return types.Granularity_Enum(types.Granularity_Enum_value[str]) -} - func valueTypeOf(str string) types.ValueType_Enum { return types.ValueType_Enum(types.ValueType_Enum_value[str]) } diff --git a/cli/feast/pkg/parse/yaml_test.go b/cli/feast/pkg/parse/yaml_test.go index 00331668ea3..0c7eaf7c012 100644 --- a/cli/feast/pkg/parse/yaml_test.go +++ b/cli/feast/pkg/parse/yaml_test.go @@ -34,13 +34,12 @@ func TestYamlToFeatureSpec(t *testing.T) { }{ { name: "valid yaml", - input: []byte(`id: test.none.test_feature_two + input: []byte(`id: test.test_feature_two name: test_feature_two entity: test owner: bob@example.com description: testing feature valueType: INT64 -granularity: NONE uri: https://github.com/bob/example dataStores: serving: @@ -48,12 +47,11 @@ dataStores: warehouse: id: BIGQUERY`), expected: &specs.FeatureSpec{ - Id: "test.none.test_feature_two", + Id: "test.test_feature_two", Owner: "bob@example.com", Name: "test_feature_two", Description: "testing feature", Uri: "https://github.com/bob/example", - Granularity: types.Granularity_NONE, ValueType: types.ValueType_INT64, Entity: "test", DataStores: &specs.DataStores{ @@ -257,7 +255,7 @@ schema: - name: timestamp - name: driver_id - name: last_opportunity - featureId: driver.none.last_opportunity`), + featureId: driver.last_opportunity`), expected: &specs.ImportSpec{ Type: "file", JobOptions: map[string]string{ @@ -272,7 +270,7 @@ schema: Fields: []*specs.Field{ {Name: "timestamp"}, {Name: "driver_id"}, - {Name: "last_opportunity", FeatureId: "driver.none.last_opportunity"}, + {Name: "last_opportunity", FeatureId: "driver.last_opportunity"}, }, EntityIdColumn: "driver_id", Timestamp: &specs.Schema_TimestampValue{ diff --git a/cli/feast/pkg/printer/printer_test.go b/cli/feast/pkg/printer/printer_test.go index a3ff2a263d0..07332324c11 100644 --- a/cli/feast/pkg/printer/printer_test.go +++ b/cli/feast/pkg/printer/printer_test.go @@ -23,12 +23,11 @@ func TestPrintFeature(t *testing.T) { name: "with storage", input: &core.UIServiceTypes_FeatureDetail{ Spec: &specs.FeatureSpec{ - Id: "test.none.test_feature_two", + Id: "test.test_feature_two", Owner: "bob@example.com", Name: "test_feature_two", Description: "testing feature", Uri: "https://github.com/bob/example", - Granularity: types.Granularity_NONE, ValueType: types.ValueType_INT64, Entity: "test", DataStores: &specs.DataStores{ @@ -45,7 +44,7 @@ func TestPrintFeature(t *testing.T) { LastUpdated: ×tamp.Timestamp{Seconds: 1}, Created: ×tamp.Timestamp{Seconds: 1}, }, - expected: fmt.Sprintf(`Id: test.none.test_feature_two + expected: fmt.Sprintf(`Id: test.test_feature_two Entity: test Owner: bob@example.com Description: testing feature @@ -65,12 +64,11 @@ Related Jobs: name: "no storage", input: &core.UIServiceTypes_FeatureDetail{ Spec: &specs.FeatureSpec{ - Id: "test.none.test_feature_two", + Id: "test.test_feature_two", Owner: "bob@example.com", Name: "test_feature_two", Description: "testing feature", Uri: "https://github.com/bob/example", - Granularity: types.Granularity_NONE, ValueType: types.ValueType_INT64, Entity: "test", }, @@ -79,7 +77,7 @@ Related Jobs: LastUpdated: ×tamp.Timestamp{Seconds: 1}, Created: ×tamp.Timestamp{Seconds: 1}, }, - expected: fmt.Sprintf(`Id: test.none.test_feature_two + expected: fmt.Sprintf(`Id: test.test_feature_two Entity: test Owner: bob@example.com Description: testing feature diff --git a/core/pom.xml b/core/pom.xml index 65565286b34..0fd82260ff0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -16,8 +16,8 @@ ~ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 feast @@ -31,8 +31,45 @@ - org.springframework.boot - spring-boot-maven-plugin + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + + + META-INF/spring.handlers + + + META-INF/spring.factories + + + META-INF/spring.schemas + + + META-INF/spring.components + + + feast.core.CoreApplication + + + + + + + + package + + shade + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${springBootVersion} + + org.xolstice.maven.plugins diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 2787f3e262c..16732412a64 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -283,7 +283,7 @@ public void applyFeature( } public FeatureSpec applyDefaultStores(FeatureSpec featureSpec) { - DataStores.Builder dataStoreBuilder = DataStores.newBuilder(); + DataStores.Builder dataStoreBuilder = featureSpec.getDataStores().toBuilder(); if (Strings.isNullOrEmpty(featureSpec.getDataStores().getServing().getId())) { log.info("Feature has no serving store specified using default"); if (storageSpecs.getServingStorageSpec() != null) { diff --git a/core/src/main/java/feast/core/job/StatsdMetricPusher.java b/core/src/main/java/feast/core/job/StatsdMetricPusher.java index 17e4558bc4f..7b8c193e338 100644 --- a/core/src/main/java/feast/core/job/StatsdMetricPusher.java +++ b/core/src/main/java/feast/core/job/StatsdMetricPusher.java @@ -43,7 +43,7 @@ public void pushMetrics(List metrics) { // `row:result` <- all // `scope:scope_id:result` <- per scope, either feature or entity // for examples: - // 1. feature:driver.minute.ping_sequence_weak_dir_change_mean:error + // 1. feature:driver.ping_sequence_weak_dir_change_mean:error // 2. entity:driver:stored // currently there are 3 results: // 1. valid diff --git a/core/src/main/java/feast/core/model/FeatureInfo.java b/core/src/main/java/feast/core/model/FeatureInfo.java index d2ac1ed68d6..ac497019d3d 100644 --- a/core/src/main/java/feast/core/model/FeatureInfo.java +++ b/core/src/main/java/feast/core/model/FeatureInfo.java @@ -23,7 +23,6 @@ import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.ValueType; import lombok.AllArgsConstructor; import lombok.Getter; @@ -62,9 +61,6 @@ public class FeatureInfo extends AbstractTimestampEntity { @Column(name = "uri", nullable = false) private String uri; - @Enumerated(EnumType.STRING) - private Granularity.Enum granularity; - @Enumerated(EnumType.STRING) private ValueType.Enum valueType; @@ -120,7 +116,6 @@ public FeatureInfo( this.owner = spec.getOwner(); this.description = spec.getDescription(); this.uri = spec.getUri(); - this.granularity = spec.getGranularity(); this.valueType = spec.getValueType(); this.entity = entityInfo; this.featureGroup = featureGroupInfo; @@ -144,7 +139,6 @@ public FeatureInfo(FeatureInfo other) { this.owner = other.owner; this.description = other.description; this.uri = other.uri; - this.granularity = other.granularity; this.valueType = other.valueType; this.entity = other.entity; this.featureGroup = other.featureGroup; @@ -183,7 +177,6 @@ public FeatureSpec getFeatureSpec() { .setOwner(owner) .setDescription(description) .setUri(uri) - .setGranularity(granularity) .setValueType(valueType) .setEntity(entity.getName()) .addAllTags(convertTagStringToList(tags)) @@ -250,8 +243,8 @@ private String createBigqueryViewLink(StorageInfo warehouseStore) { String dataset = opts.get(BigQueryStorageManager.OPT_BIGQUERY_DATASET); return String.format( - "https://bigquery.cloud.google.com/table/%s:%s.%s_%s_view", - projectId, dataset, entity.getName(), granularity.toString().toLowerCase()); + "https://bigquery.cloud.google.com/table/%s:%s.%s_view", + projectId, dataset, entity.getName()); } /** @@ -276,7 +269,6 @@ private boolean isLegalUpdate(FeatureSpec update) { FeatureSpec spec = this.getFeatureSpec(); return spec.getName().equals(update.getName()) && spec.getEntity().equals(update.getEntity()) - && spec.getGranularity().equals(update.getGranularity()) && spec.getValueType().equals(update.getValueType()) && spec.getGroup().equals(update.getGroup()) && spec.getOptionsMap().equals(update.getOptionsMap()) diff --git a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java index 197e0016aeb..00f64f8a86e 100644 --- a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java +++ b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java @@ -191,8 +191,7 @@ private Field createFeatureField(FeatureSpec featureSpec) { private String createTableName(FeatureSpec featureSpec) { String entityName = featureSpec.getEntity().toLowerCase(); - String granularity = featureSpec.getGranularity().toString().toLowerCase(); - return String.format("%s_%s", entityName, granularity); + return String.format("%s", entityName); } private void createOrUpdateView(String tableName, List features) { diff --git a/core/src/main/java/feast/core/storage/PostgresStorageManager.java b/core/src/main/java/feast/core/storage/PostgresStorageManager.java index 9fef9f0d2cb..8875510d174 100644 --- a/core/src/main/java/feast/core/storage/PostgresStorageManager.java +++ b/core/src/main/java/feast/core/storage/PostgresStorageManager.java @@ -91,7 +91,6 @@ private String createFieldType(FeatureSpec featureSpec) { private String createTableName(FeatureSpec featureSpec) { String entityName = featureSpec.getEntity().toLowerCase(); - String granularity = featureSpec.getGranularity().toString().toLowerCase(); - return String.format("%s_%s", entityName, granularity); + return String.format("%s", entityName); } } diff --git a/core/src/main/java/feast/core/training/BigQueryDatasetTemplater.java b/core/src/main/java/feast/core/training/BigQueryDatasetTemplater.java index 201ad220ca0..ea73d89d44e 100644 --- a/core/src/main/java/feast/core/training/BigQueryDatasetTemplater.java +++ b/core/src/main/java/feast/core/training/BigQueryDatasetTemplater.java @@ -16,8 +16,6 @@ */ package feast.core.training; -import static java.util.stream.Collectors.groupingBy; - import com.google.protobuf.Timestamp; import com.hubspot.jinjava.Jinjava; import feast.core.DatasetServiceProto.FeatureSet; @@ -26,13 +24,10 @@ import feast.core.model.StorageInfo; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,8 +41,6 @@ public class BigQueryDatasetTemplater { private final Jinjava jinjava; private final String template; private final DateTimeFormatter formatter; - private Comparator featureGroupComparator = - new FeatureGroupTemplateComparator().reversed(); public BigQueryDatasetTemplater( Jinjava jinjava, String templateString, FeatureInfoRepository featureInfoRepository) { @@ -77,40 +70,29 @@ public String createQuery( throw new NoSuchElementException("features not found: " + featureIds); } - List features = toFeatureTemplates(featureInfos); - List featureGroups = groupFeatureTemplate(features); + String tableId = getBqTableId(featureInfos.get(0)); + Features features = new Features(featureIds, tableId); String startDateStr = formatDateString(startDate); String endDateStr = formatDateString(endDate); String limitStr = (limit != 0) ? String.valueOf(limit) : null; - return renderTemplate(featureGroups, startDateStr, endDateStr, limitStr); + return renderTemplate(features, startDateStr, endDateStr, limitStr); } private String renderTemplate( - List featureGroups, String startDateStr, String endDateStr, String limitStr) { + Features features, String startDateStr, String endDateStr, String limitStr) { Map context = new HashMap<>(); - featureGroups.sort(featureGroupComparator); - context.put("feature_groups", featureGroups); + context.put("feature_set", features); context.put("start_date", startDateStr); context.put("end_date", endDateStr); context.put("limit", limitStr); return jinjava.render(template, context); } - private List toFeatureTemplates(List featureInfos) { - return featureInfos - .stream() - .map( - fi -> { - StorageInfo whStorage = fi.getWarehouseStore(); - String tableId = getBqTableId(fi.getFeatureSpec(), whStorage); - return new Feature(fi.getId(), fi.getGranularity(), tableId); - }) - .collect(Collectors.toList()); - } + private String getBqTableId(FeatureInfo featureInfo) { + StorageInfo whStorage = featureInfo.getWarehouseStore(); - private String getBqTableId(FeatureSpec featureSpec, StorageInfo whStorage) { String type = whStorage.getType(); if (!"bigquery".equals(type)) { throw new IllegalArgumentException( @@ -121,23 +103,8 @@ private String getBqTableId(FeatureSpec featureSpec, StorageInfo whStorage) { Map options = storageSpec.getOptionsMap(); String projectId = options.get("project"); String dataset = options.get("dataset"); - String entityName = featureSpec.getEntity().toLowerCase(); - String granularity = featureSpec.getGranularity().toString().toLowerCase(); - return String.format("%s.%s.%s_%s", projectId, dataset, entityName, granularity); - } - - private List groupFeatureTemplate(List features) { - Map> groupedFeature = - features.stream().collect(groupingBy(Feature::getTableId)); - List featureGroups = new ArrayList<>(); - for (Map.Entry> entry : groupedFeature.entrySet()) { - String tableId = entry.getKey(); - Granularity.Enum granularity = entry.getValue().get(0).granularity; - FeatureGroup group = new FeatureGroup(tableId, granularity, entry.getValue()); - - featureGroups.add(group); - } - return featureGroups; + String entityName = featureInfo.getFeatureSpec().getEntity().toLowerCase(); + return String.format("%s.%s.%s", projectId, dataset, entityName); } private String formatDateString(Timestamp timestamp) { @@ -146,55 +113,16 @@ private String formatDateString(Timestamp timestamp) { } @Getter - static final class FeatureGroup { - final String tableId; - final Granularity.Enum granularity; - final List features; - - public FeatureGroup(String tableId, Granularity.Enum granularity, List features) { - this.tableId = tableId; - this.granularity = granularity; - this.features = features; - } - - public String getTempTable() { - return tableId.replaceAll("[^a-zA-Z0-9]", "_"); - } - - public String getGranularityStr() { - return granularity.toString().toLowerCase(); - } - } - - @Getter - static final class Feature { - final String featureId; + static final class Features { + final List columns; final String tableId; - final Granularity.Enum granularity; - public Feature(String featureId, Granularity.Enum granularity, String tableId) { - this.featureId = featureId; + public Features(List featureIds, String tableId) { + this.columns = featureIds.stream() + .map(f -> f.replace(".", "_")) + .collect(Collectors.toList()); this.tableId = tableId; - this.granularity = granularity; - } - - public String getName() { - return featureId.split("\\.")[2]; - } - - public String getColumn() { - return featureId.replace(".", "_"); } } - private static final class FeatureGroupTemplateComparator implements Comparator { - @Override - public int compare(FeatureGroup o1, FeatureGroup o2) { - if (o1.granularity != o2.granularity) { - return o1.granularity.getNumber() - o2.granularity.getNumber(); - } - - return o1.tableId.compareTo(o2.tableId); - } - } } diff --git a/core/src/main/java/feast/core/validators/SpecValidator.java b/core/src/main/java/feast/core/validators/SpecValidator.java index 7ec2f33d46b..3e70e9922d0 100644 --- a/core/src/main/java/feast/core/validators/SpecValidator.java +++ b/core/src/main/java/feast/core/validators/SpecValidator.java @@ -96,16 +96,13 @@ public void validateFeatureSpec(FeatureSpec spec) throws IllegalArgumentExceptio // check id validity String[] idSplit = spec.getId().split("\\."); - checkArgument(idSplit.length == 3, "Id must contain entity, granularity, name"); + checkArgument(idSplit.length == 2, "Id must contain entity, name"); checkArgument( idSplit[0].equals(spec.getEntity()), - "Id must be in format entity.granularity.name, entity in Id does not match entity provided."); + "Id must be in format entity.name, entity in Id does not match entity provided."); checkArgument( - idSplit[1].equals(spec.getGranularity().toString().toLowerCase()), - "Id must be in format entity.granularity.name, granularity in Id does not match granularity provided."); - checkArgument( - idSplit[2].equals(spec.getName()), - "Id must be in format entity.granularity.name, name in Id does not match name provided."); + idSplit[1].equals(spec.getName()), + "Id must be in format entity.name, name in Id does not match name provided."); // check if referenced objects exist checkArgument( diff --git a/core/src/main/resources/templates/bq_training.tmpl b/core/src/main/resources/templates/bq_training.tmpl index faad4f18a40..0a7ce6d3222 100644 --- a/core/src/main/resources/templates/bq_training.tmpl +++ b/core/src/main/resources/templates/bq_training.tmpl @@ -1,41 +1,12 @@ -WITH - {% for feature_group in feature_groups -%} - raw_{{ feature_group.tempTable }} AS - (SELECT - ROW_NUMBER() OVER (PARTITION BY id, event_timestamp ORDER BY created_timestamp DESC) rownum, - id, event_timestamp, created_timestamp - {% for feature in feature_group.features -%} - , FIRST_VALUE({{feature.name}} IGNORE NULLS) OVER w AS {{feature.column}} - {% endfor -%} - FROM `{{feature_group.table_id}}` - {% if feature_group.granularityStr != "none" -%} - WHERE event_timestamp >= TIMESTAMP("{{ start_date }}") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("{{ end_date }}", INTERVAL 1 DAY)) - {%- endif %} - WINDOW w AS ( PARTITION BY id, event_timestamp ORDER BY event_timestamp DESC )), - {{ feature_group.temp_table }} AS ( - SELECT * FROM raw_{{ feature_group.temp_table }} WHERE rownum = 1 ) {{ "," if not loop.last }} - {% endfor %} SELECT - {{ feature_groups[0].temp_table }}.id, - {{ feature_groups[0].temp_table }}.event_timestamp - {% for feature_group in feature_groups -%} - {% for feature in feature_group.features -%} - ,{{ feature_group.temp_table }}.{{ feature.column }} - {%- endfor %} + {{ feature_set.tableId }}.id, + {{ feature_set.tableId }}.event_timestamp + {% for feature in feature_set.columns -%} + ,{{ feature }} {%- endfor %} FROM - {{ feature_groups[0].temp_table }} -{% for feature_group in feature_groups -%} -{% if loop.index > 1 %} -LEFT JOIN - {{ feature_group.temp_table }} -ON - {{ feature_groups[0].temp_table }}.id = {{ feature_group.temp_table }}.id - {% if feature_group.granularityStr != "none" -%} - AND TIMESTAMP_TRUNC({{feature_groups[0].temp_table}}.event_timestamp, {{ feature_group.granularityStr|upper }}) = {{ feature_group.temp_table }}.event_timestamp - {%- endif %} -{%- endif %} -{%- endfor %} + {{ feature_set.tableId }} +WHERE event_timestamp >= TIMESTAMP("{{ start_date }}") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("{{ end_date }}", INTERVAL 1 DAY)) {% if limit is not none -%} LIMIT {{ limit }} {%- endif %} \ No newline at end of file diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java index 42e22b25660..d80b5b7b42c 100644 --- a/core/src/test/java/feast/core/CoreApplicationTest.java +++ b/core/src/test/java/feast/core/CoreApplicationTest.java @@ -22,7 +22,6 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -94,20 +93,19 @@ public void test_withProperties_systemServingAndWarehouseStoresRegistered() thro EntitySpec entitySpec = EntitySpec.newBuilder().setName("test").build(); FeatureSpec featureSpec = FeatureSpec.newBuilder() - .setId("test.none.int64") + .setId("test.int64") .setName("int64") .setEntity("test") - .setGranularity(Enum.NONE) .setValueType(ValueType.Enum.INT64) .setOwner("hermione@example.com") .setDescription("Test is a test") - .setUri("http://example.com/test.none.int64").build(); + .setUri("http://example.com/test.int64").build(); ImportSpec importSpec = ImportSpec.newBuilder() .setSchema(Schema.newBuilder() .setEntityIdColumn("id") .setTimestampValue(Timestamp.getDefaultInstance()) .addFields(Field.newBuilder().setName("id")) - .addFields(Field.newBuilder().setName("a").setFeatureId("test.none.int64"))) + .addFields(Field.newBuilder().setName("a").setFeatureId("test.int64"))) .addEntities("test") .setType("file.csv") .putSourceOptions("path", "/tmp/foobar").build(); diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java index 93298c76045..61f55202ad9 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java @@ -23,7 +23,6 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -84,20 +83,19 @@ public void test_withProperties_systemServingAndWarehouseStoresRegistered() thro EntitySpec entitySpec = EntitySpec.newBuilder().setName("test").build(); FeatureSpec featureSpec = FeatureSpec.newBuilder() - .setId("test.none.int64") + .setId("test.int64") .setName("int64") .setEntity("test") - .setGranularity(Enum.NONE) .setValueType(ValueType.Enum.INT64) .setOwner("hermione@example.com") .setDescription("Test is a test") - .setUri("http://example.com/test.none.int64").build(); + .setUri("http://example.com/test.int64").build(); ImportSpec importSpec = ImportSpec.newBuilder() .setSchema(Schema.newBuilder() .setEntityIdColumn("id") .setTimestampValue(Timestamp.getDefaultInstance()) .addFields(Field.newBuilder().setName("id")) - .addFields(Field.newBuilder().setName("a").setFeatureId("test.none.int64"))) + .addFields(Field.newBuilder().setName("a").setFeatureId("test.int64"))) .addEntities("test") .setType("file.csv") .putSourceOptions("path", "/tmp/foobar").build(); diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java index 5b4c0801e91..6c8ee325aac 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java @@ -23,7 +23,6 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -92,20 +91,19 @@ public void test_withProperties_systemServingAndWarehouseStoresRegistered() thro EntitySpec entitySpec = EntitySpec.newBuilder().setName("test").build(); FeatureSpec featureSpec = FeatureSpec.newBuilder() - .setId("test.none.int64") + .setId("test.int64") .setName("int64") .setEntity("test") - .setGranularity(Enum.NONE) .setValueType(ValueType.Enum.INT64) .setOwner("hermione@example.com") .setDescription("Test is a test") - .setUri("http://example.com/test.none.int64").build(); + .setUri("http://example.com/test.int64").build(); ImportSpec importSpec = ImportSpec.newBuilder() .setSchema(Schema.newBuilder() .setEntityIdColumn("id") .setTimestampValue(Timestamp.getDefaultInstance()) .addFields(Field.newBuilder().setName("id")) - .addFields(Field.newBuilder().setName("a").setFeatureId("test.none.int64"))) + .addFields(Field.newBuilder().setName("a").setFeatureId("test.int64"))) .addEntities("test") .setType("file.csv") .putSourceOptions("path", "/tmp/foobar").build(); diff --git a/core/src/test/java/feast/core/grpc/DatasetServiceImplTest.java b/core/src/test/java/feast/core/grpc/DatasetServiceImplTest.java index 5740648848a..5bbe7f0d487 100644 --- a/core/src/test/java/feast/core/grpc/DatasetServiceImplTest.java +++ b/core/src/test/java/feast/core/grpc/DatasetServiceImplTest.java @@ -63,8 +63,8 @@ public void setUp() throws Exception { validFeatureSet = FeatureSet.newBuilder() .setEntityName("myentity") - .addFeatureIds("myentity.none.feature1") - .addFeatureIds("myentity.second.feature2") + .addFeatureIds("myentity.feature1") + .addFeatureIds("myentity.feature2") .build(); } @@ -169,8 +169,8 @@ public void shouldThrowExceptionIfFeatureSetHasDifferentEntity() { FeatureSet emptyFeatureSet = FeatureSet.newBuilder() .setEntityName("myentity") - .addFeatureIds("myentity.none.feature1") - .addFeatureIds("driver.none.feature2") + .addFeatureIds("myentity.feature1") + .addFeatureIds("driver.feature2") .build(); CreateDatasetRequest request = diff --git a/core/src/test/java/feast/core/grpc/UIServiceImplTest.java b/core/src/test/java/feast/core/grpc/UIServiceImplTest.java index cca090bec70..00f02f9af62 100644 --- a/core/src/test/java/feast/core/grpc/UIServiceImplTest.java +++ b/core/src/test/java/feast/core/grpc/UIServiceImplTest.java @@ -174,7 +174,7 @@ public void listEntities_shouldReturnClearErrorMessageForAnyFailure() { @Test public void getFeature_shouldReturnCorrectFeatureDetail() { - String featureId = "entity.granularity.feature"; + String featureId = "entity.feature"; FeatureInfo featureInfo = createFeatureInfo(featureId); when(specService.getFeatures(Collections.singletonList(featureId))) @@ -221,8 +221,8 @@ public void getFeature_shouldReturnErrorForAnyFailure() { @Test public void listFeature_shouldReturnAllFeatures() { - String featureId1 = "entity.granularity.feature1"; - String featureId2 = "entity.granularity.feature2"; + String featureId1 = "entity.feature1"; + String featureId2 = "entity.feature2"; FeatureInfo featureInfo1 = createFeatureInfo(featureId1); FeatureInfo featureInfo2 = createFeatureInfo(featureId2); diff --git a/core/src/test/java/feast/core/model/FeatureInfoTest.java b/core/src/test/java/feast/core/model/FeatureInfoTest.java index a85b99711f3..6e619a5791c 100644 --- a/core/src/test/java/feast/core/model/FeatureInfoTest.java +++ b/core/src/test/java/feast/core/model/FeatureInfoTest.java @@ -22,7 +22,6 @@ import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.ValueType; import org.junit.Before; import org.junit.Rule; @@ -49,12 +48,11 @@ public void setUp() { entityInfo.setName("entity"); featureInfo = new FeatureInfo(); - featureInfo.setId("entity.NONE.name"); + featureInfo.setId("entity.name"); featureInfo.setName("name"); featureInfo.setOwner("owner"); featureInfo.setDescription("desc"); featureInfo.setUri("uri"); - featureInfo.setGranularity(Granularity.Enum.NONE); featureInfo.setValueType(ValueType.Enum.BYTES); featureInfo.setEntity(entityInfo); featureInfo.setOptions("{}"); @@ -82,13 +80,12 @@ public void setUp() { featureSpec = FeatureSpec.newBuilder() - .setId("entity.NONE.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("desc") .setEntity("entity") .setUri("uri") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.BYTES) .addTags("tag1") .addTags("tag2") @@ -133,12 +130,11 @@ public void shouldBuildCorrespondingResolvedSpec() { featureGroupInfo.setWarehouseStore(warehouseStorage); featureGroupInfo.setTags("inherited"); FeatureInfo featureInfo = new FeatureInfo(); - featureInfo.setId("entity.NONE.name"); + featureInfo.setId("entity.name"); featureInfo.setName("name"); featureInfo.setOwner("owner"); featureInfo.setDescription("desc"); featureInfo.setUri("uri"); - featureInfo.setGranularity(Granularity.Enum.NONE); featureInfo.setValueType(ValueType.Enum.BYTES); featureInfo.setEntity(entityInfo); featureInfo.setOptions("{}"); @@ -157,14 +153,13 @@ public void shouldBuildCorrespondingResolvedSpec() { FeatureSpec expected = FeatureSpec.newBuilder() - .setId("entity.NONE.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("desc") .setEntity("entity") .setUri("uri") .setGroup("testGroup") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.BYTES) .addTags("tag1") .addTags("tag2") @@ -187,13 +182,12 @@ public void shouldUpdateMutableFields() { FeatureSpec update = FeatureSpec.newBuilder() - .setId("entity.NONE.name") + .setId("entity.name") .setName("name") .setOwner("owner2") .setDescription("overwrite") .setEntity("entity") .setUri("new_uri") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.BYTES) .addTags("new_tag") .setDataStores(dataStores) @@ -216,13 +210,12 @@ public void shouldThrowExceptionIfImmutableFieldsChanged() { FeatureSpec update = FeatureSpec.newBuilder() - .setId("entity.NONE.name") + .setId("entity.name") .setName("name") .setOwner("owner2") .setDescription("overwrite") .setEntity("entity") .setUri("new_uri") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.INT32) .addTags("new_tag") .setDataStores(dataStores) @@ -238,13 +231,12 @@ public void shouldThrowExceptionIfImmutableFieldsChanged() { public void shouldThrowExceptionIfImmutableFieldsChangedToNull() { FeatureSpec update = FeatureSpec.newBuilder() - .setId("entity.NONE.name") + .setId("entity.name") .setName("name") .setOwner("owner2") .setDescription("overwrite") .setEntity("entity") .setUri("new_uri") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.BYTES) .addTags("new_tag") .build(); diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 43f1450f7b5..bf7d7e7a7ae 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -36,7 +36,6 @@ import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.ValueType; import org.junit.Before; import org.junit.Rule; @@ -86,12 +85,11 @@ private StorageInfo newTestStorageInfo(String id, String type) { private FeatureInfo newTestFeatureInfo(String name) { FeatureInfo feature = new FeatureInfo(); - feature.setId(Strings.lenientFormat("entity.NONE.%s", name)); + feature.setId(Strings.lenientFormat("entity.%s", name)); feature.setName(name); feature.setEntity(newTestEntityInfo("entity")); feature.setDescription(""); feature.setOwner("@test"); - feature.setGranularity(Granularity.Enum.NONE); feature.setValueType(ValueType.Enum.BOOL); feature.setUri(""); feature.setWarehouseStore(newTestStorageInfo("BIGQUERY1", "BIGQUERY")); @@ -180,7 +178,7 @@ public void shouldGetFeaturesMatchingIds() { FeatureInfo feature1 = newTestFeatureInfo("feature1"); FeatureInfo feature2 = newTestFeatureInfo("feature2"); - ArrayList ids = Lists.newArrayList("entity.none.feature1", "entity.none.feature2"); + ArrayList ids = Lists.newArrayList("entity.feature1", "entity.feature2"); when(featureInfoRepository.findAllById(any(Iterable.class))).thenReturn(Lists.newArrayList(feature1, feature2)); SpecService specService = new SpecService( @@ -199,7 +197,7 @@ public void shouldDeduplicateGetFeature() { FeatureInfo feature1 = newTestFeatureInfo("feature1"); FeatureInfo feature2 = newTestFeatureInfo("feature2"); - ArrayList ids = Lists.newArrayList("entity.none.feature1", "entity.none.feature2", "entity.none.feature2"); + ArrayList ids = Lists.newArrayList("entity.feature1", "entity.feature2", "entity.feature2"); when(featureInfoRepository.findAllById(any(Iterable.class))).thenReturn(Lists.newArrayList(feature1, feature2)); SpecService specService = new SpecService( @@ -217,7 +215,7 @@ public void shouldDeduplicateGetFeature() { public void shouldThrowRetrievalExceptionIfAnyFeatureNotFound() { FeatureInfo feature2 = newTestFeatureInfo("feature2"); - ArrayList ids = Lists.newArrayList("entity.none.feature1", "entity.none.feature2"); + ArrayList ids = Lists.newArrayList("entity.feature1", "entity.feature2"); when(featureInfoRepository.findAllById(ids)).thenReturn(Lists.newArrayList(feature2)); SpecService specService = new SpecService( @@ -340,14 +338,13 @@ public void shouldRegisterFeatureWithGroupInheritance() { FeatureSpec spec = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("desc") .setEntity("entity") .setUri("uri") .setGroup("testGroup") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.BYTES) .build(); @@ -361,14 +358,13 @@ public void shouldRegisterFeatureWithGroupInheritance() { FeatureSpec resolvedSpec = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("desc") .setEntity("entity") .setUri("uri") .setGroup("testGroup") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.BYTES) .setDataStores(dataStores) .build(); diff --git a/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java b/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java index 3cbc7069a07..4e4408307b1 100644 --- a/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java +++ b/core/src/test/java/feast/core/storage/BigQueryStorageManagerTest.java @@ -40,8 +40,6 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.ViewDefinition; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import java.util.ArrayList; import java.util.List; @@ -77,8 +75,7 @@ public void shouldCreateNewDatasetAndTableAndViewIfNotExist() throws Interrupted when(bigQuery.getDataset(any(String.class))).thenReturn(null); String featureName = "my_feature"; String entityName = "my_entity"; - Granularity.Enum granularity = Enum.DAY; - String featureId = createFeatureId(entityName, granularity, featureName); + String featureId = createFeatureId(entityName, featureName); String description = "my feature is awesome"; ValueType.Enum type = ValueType.Enum.INT64; @@ -87,7 +84,6 @@ public void shouldCreateNewDatasetAndTableAndViewIfNotExist() throws Interrupted .setEntity(entityName) .setId(featureId) .setName(featureName) - .setGranularity(granularity) .setDescription(description) .setValueType(type) .build(); @@ -113,7 +109,7 @@ public void shouldCreateNewDatasetAndTableAndViewIfNotExist() throws Interrupted TableInfo actualTable = capturedValues.get(0); assertThat( actualTable.getTableId().getTable(), - equalTo(String.format("%s_%s", entityName, granularity.toString().toLowerCase()))); + equalTo(String.format("%s", entityName))); List fields = actualTable.getDefinition().getSchema().getFields(); assertThat(fields.size(), equalTo(5)); Field idField = fields.get(0); @@ -133,21 +129,20 @@ public void shouldCreateNewDatasetAndTableAndViewIfNotExist() throws Interrupted TableInfo actualView = capturedValues.get(1); assertThat( actualView.getTableId().getTable(), - equalTo(String.format("%s_%s_view", entityName, granularity.toString().toLowerCase()))); + equalTo(String.format("%s_view", entityName))); ViewDefinition actualDefinition = actualView.getDefinition(); assertThat( actualDefinition.getQuery(), equalTo( String.format( - "%s_%s.%s", entityName, granularity.toString().toLowerCase(), featureName))); + "%s.%s", entityName, featureName))); } @Test public void shouldNotUpdateTableIfColumnExists() { String featureName = "my_feature"; String entityName = "my_entity"; - Granularity.Enum granularity = Enum.DAY; - String featureId = createFeatureId(entityName, granularity, featureName); + String featureId = createFeatureId(entityName, featureName); String description = "my feature is awesome"; ValueType.Enum type = ValueType.Enum.BOOL; LegacySQLTypeName sqlType = LegacySQLTypeName.BOOLEAN; @@ -169,7 +164,6 @@ public void shouldNotUpdateTableIfColumnExists() { .setEntity(entityName) .setId(featureId) .setName(featureName) - .setGranularity(granularity) .setDescription(description) .setValueType(type) .build(); @@ -184,8 +178,7 @@ public void shouldNotUpdateTableIfColumnExists() { public void shouldUpdateTableAndViewIfColumnNotExists() { String newFeatureName = "my_feature"; String entityName = "my_entity"; - Granularity.Enum granularity = Enum.DAY; - String featureId = createFeatureId(entityName, granularity, newFeatureName); + String featureId = createFeatureId(entityName, newFeatureName); String description = "my feature is awesome"; ValueType.Enum type = ValueType.Enum.BOOL; LegacySQLTypeName sqlType = LegacySQLTypeName.BOOLEAN; @@ -203,7 +196,6 @@ public void shouldUpdateTableAndViewIfColumnNotExists() { .setEntity(entityName) .setId(featureId) .setName(newFeatureName) - .setGranularity(granularity) .setDescription(description) .setValueType(type) .build(); @@ -230,22 +222,21 @@ public void shouldUpdateTableAndViewIfColumnNotExists() { TableInfo actualView = capturedArgs.get(1); assertThat( actualView.getTableId().getTable(), - equalTo(String.format("%s_%s_view", entityName, granularity.toString().toLowerCase()))); + equalTo(String.format("%s_view", entityName))); ViewDefinition actualDefinition = actualView.getDefinition(); assertThat( actualDefinition.getQuery(), equalTo( String.format( - "%s_%s.%s.%s", + "%s.%s.%s", entityName, - granularity.toString().toLowerCase(), existingFeatureName, newFeatureName))); } private String createFeatureId( - String entityName, Granularity.Enum granualrity, String featureName) { - return String.format("%s.%s.%s", entityName, granualrity.toString(), featureName).toLowerCase(); + String entityName, String featureName) { + return String.format("%s.%s", entityName, featureName).toLowerCase(); } private Table createTable(FeatureSchema... featureSchemas) { diff --git a/core/src/test/java/feast/core/training/BigQueryDatasetCreatorTest.java b/core/src/test/java/feast/core/training/BigQueryDatasetCreatorTest.java index bde137c9c7e..9642399534c 100644 --- a/core/src/test/java/feast/core/training/BigQueryDatasetCreatorTest.java +++ b/core/src/test/java/feast/core/training/BigQueryDatasetCreatorTest.java @@ -63,7 +63,7 @@ public void shouldCreateCorrectDatasetIfPrefixNotSpecified() { FeatureSet featureSet = FeatureSet.newBuilder() .setEntityName(entityName) - .addAllFeatureIds(Arrays.asList("myentity.none.feature1", "myentity.none.feature2")) + .addAllFeatureIds(Arrays.asList("myentity.feature1", "myentity.feature2")) .build(); Timestamp startDate = @@ -90,7 +90,7 @@ public void shouldCreateCorrectDatasetIfPrefixIsSpecified() { FeatureSet featureSet = FeatureSet.newBuilder() .setEntityName(entityName) - .addAllFeatureIds(Arrays.asList("myentity.none.feature1", "myentity.none.feature2")) + .addAllFeatureIds(Arrays.asList("myentity.feature1", "myentity.feature2")) .build(); Timestamp startDate = @@ -115,7 +115,7 @@ public void shouldPassArgumentToTemplater() { FeatureSet featureSet = FeatureSet.newBuilder() .setEntityName("myentity") - .addAllFeatureIds(Arrays.asList("myentity.none.feature1", "myentity.none.feature2")) + .addAllFeatureIds(Arrays.asList("myentity.feature1", "myentity.feature2")) .build(); Timestamp startDate = Timestamps.fromSeconds(0); diff --git a/core/src/test/java/feast/core/training/BigQueryDatasetTemplaterTest.java b/core/src/test/java/feast/core/training/BigQueryDatasetTemplaterTest.java index 239d44d66e9..d8b4595d3e0 100644 --- a/core/src/test/java/feast/core/training/BigQueryDatasetTemplaterTest.java +++ b/core/src/test/java/feast/core/training/BigQueryDatasetTemplaterTest.java @@ -33,14 +33,12 @@ import feast.core.model.EntityInfo; import feast.core.model.FeatureInfo; import feast.core.model.StorageInfo; -import feast.core.training.BigQueryDatasetTemplater.FeatureGroup; +import feast.core.training.BigQueryDatasetTemplater.Features; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity; -import feast.types.GranularityProto.Granularity.Enum; import java.io.InputStream; import java.io.InputStreamReader; import java.time.Instant; @@ -82,7 +80,7 @@ public void shouldThrowNoSuchElementExceptionIfFeatureNotFound() { FeatureSet fs = FeatureSet.newBuilder() .setEntityName("myentity") - .addAllFeatureIds(Arrays.asList("myentity.day.feature1", "myentity.day.feature2")) + .addAllFeatureIds(Arrays.asList("myentity.feature1", "myentity.feature2")) .build(); templater.createQuery(fs, Timestamps.fromSeconds(0), Timestamps.fromSeconds(1), 0); } @@ -97,11 +95,11 @@ public void shouldPassCorrectArgumentToTemplateEngine() { Timestamp endDate = Timestamps.fromSeconds(Instant.parse("2019-01-01T00:00:00.00Z").getEpochSecond()); int limit = 100; - String featureId = "myentity.day.feature1"; - String tableId = "project.dataset.myentity_day"; + String featureId = "myentity.feature1"; + String tableId = "project.dataset.myentity"; when(featureInfoRespository.findAllById(any(List.class))) - .thenReturn(Collections.singletonList(createFeatureInfo(featureId, Enum.DAY, tableId))); + .thenReturn(Collections.singletonList(createFeatureInfo(featureId, tableId))); FeatureSet fs = FeatureSet.newBuilder() @@ -123,27 +121,24 @@ public void shouldPassCorrectArgumentToTemplateEngine() { assertThat(actualContext.get("end_date"), equalTo("2019-01-01")); assertThat(actualContext.get("limit"), equalTo(String.valueOf(limit))); - List featureGroups = (List) actualContext.get("feature_groups"); - assertThat(featureGroups.size(), equalTo(1)); - assertThat(featureGroups.get(0).granularity, equalTo(Enum.DAY)); - assertThat(featureGroups.get(0).features.size(), equalTo(1)); - assertThat(featureGroups.get(0).features.get(0).featureId, equalTo(featureId)); - assertThat(featureGroups.get(0).features.get(0).granularity, equalTo(Enum.DAY)); - assertThat(featureGroups.get(0).features.get(0).tableId, equalTo(tableId)); + Features features = (Features) actualContext.get("feature_set"); + assertThat(features.getColumns().size(), equalTo(1)); + assertThat(features.getColumns().get(0), equalTo(featureId.replace(".", "_"))); + assertThat(features.getTableId(), equalTo(tableId)); } @Test public void shouldRenderCorrectQuery1() throws Exception { - String tableId1 = "project.dataset.myentity_day"; - String featureId1 = "myentity.day.feature1"; - String featureId2 = "myentity.day.feature2"; + String tableId1 = "project.dataset.myentity"; + String featureId1 = "myentity.feature1"; + String featureId2 = "myentity.feature2"; - FeatureInfo featureInfo1 = createFeatureInfo(featureId1, Enum.DAY, tableId1); - FeatureInfo featureInfo2 = createFeatureInfo(featureId2, Enum.DAY, tableId1); + FeatureInfo featureInfo1 = createFeatureInfo(featureId1, tableId1); + FeatureInfo featureInfo2 = createFeatureInfo(featureId2, tableId1); - String tableId2 = "project.dataset.myentity_none"; - String featureId3 = "myentity.none.feature3"; - FeatureInfo featureInfo3 = createFeatureInfo(featureId3, Enum.NONE, tableId2); + String tableId2 = "project.dataset.myentity"; + String featureId3 = "myentity.feature3"; + FeatureInfo featureInfo3 = createFeatureInfo(featureId3, tableId2); when(featureInfoRespository.findAllById(any(List.class))) .thenReturn(Arrays.asList(featureInfo1, featureInfo2, featureInfo3)); @@ -166,52 +161,14 @@ public void shouldRenderCorrectQuery1() throws Exception { @Test public void shouldRenderCorrectQuery2() throws Exception { - String tableId1 = "project.dataset.myentity_day"; - String featureId1 = "myentity.day.feature1"; - String featureId2 = "myentity.day.feature2"; - - FeatureInfo featureInfo1 = createFeatureInfo(featureId1, Enum.DAY, tableId1); - FeatureInfo featureInfo2 = createFeatureInfo(featureId2, Enum.DAY, tableId1); - - String tableId2 = "project.dataset.myentity_none"; - String featureId3 = "myentity.none.feature3"; - FeatureInfo featureInfo3 = createFeatureInfo(featureId3, Enum.NONE, tableId2); - - when(featureInfoRespository.findAllById(any(List.class))) - .thenReturn(Arrays.asList(featureInfo1, featureInfo2, featureInfo3)); - - FeatureSet fs = - FeatureSet.newBuilder() - .setEntityName("myentity") - .addAllFeatureIds(Arrays.asList(featureId1, featureId2, featureId3)) - .build(); - Timestamp startDate = - Timestamps.fromSeconds(Instant.parse("2018-01-02T00:00:00.00Z").getEpochSecond()); - Timestamp endDate = - Timestamps.fromSeconds(Instant.parse("2018-01-30T12:11:11.00Z").getEpochSecond()); - int limit = 0; - - String query = templater.createQuery(fs, startDate, endDate, limit); - - checkExpectedQuery(query, "expQuery2.sql"); - } - - @Test - public void shouldRenderCorrectQuery3() throws Exception { List featureInfos = new ArrayList<>(); List featureIds = new ArrayList<>(); - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - String granularityStr = granularity.toString().toLowerCase(); - String tableId = "project.dataset.myentity_" + granularityStr; - String featureId = "myentity." + granularityStr + ".feature1"; + String tableId = "project.dataset.myentity"; + String featureId = "myentity.feature1"; - featureInfos.add(createFeatureInfo(featureId, granularity, tableId)); - featureIds.add(featureId); - } + featureInfos.add(createFeatureInfo(featureId, tableId)); + featureIds.add(featureId); when(featureInfoRespository.findAllById(any(List.class))).thenReturn(featureInfos); @@ -224,7 +181,7 @@ public void shouldRenderCorrectQuery3() throws Exception { String query = templater.createQuery(featureSet, startDate, endDate, 1000); - checkExpectedQuery(query, "expQuery3.sql"); + checkExpectedQuery(query, "expQuery2.sql"); } private void checkExpectedQuery(String query, String pathToExpQuery) throws Exception { @@ -240,7 +197,7 @@ private void checkExpectedQuery(String query, String pathToExpQuery) throws Exce assertThat(query, equalTo(expQuery)); } - private FeatureInfo createFeatureInfo(String id, Granularity.Enum granularity, String tableId) { + private FeatureInfo createFeatureInfo(String id, String tableId) { StorageSpec storageSpec = StorageSpec.newBuilder() .setId("BQ") @@ -253,7 +210,6 @@ private FeatureInfo createFeatureInfo(String id, Granularity.Enum granularity, S FeatureSpec fs = FeatureSpec.newBuilder() .setId(id) - .setGranularity(granularity) .setDataStores(DataStores.newBuilder().setWarehouse(DataStore.newBuilder().setId("BQ"))) .build(); diff --git a/core/src/test/java/feast/core/validators/SpecValidatorTest.java b/core/src/test/java/feast/core/validators/SpecValidatorTest.java index 9bb2cf461e5..1170a036b42 100644 --- a/core/src/test/java/feast/core/validators/SpecValidatorTest.java +++ b/core/src/test/java/feast/core/validators/SpecValidatorTest.java @@ -38,7 +38,6 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity; import java.util.Optional; import org.junit.Before; import org.junit.Rule; @@ -171,10 +170,9 @@ public void featureSpecWithIdWithoutThreeWordsShouldThrowIllegalArgumentExceptio .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(1)) .build(); exception.expect(IllegalArgumentException.class); - exception.expectMessage("Id must contain entity, granularity, name"); + exception.expectMessage("Id must contain entity, name"); validator.validateFeatureSpec(input); } @@ -188,44 +186,19 @@ public void featureSpecWithIdWithoutMatchingEntityShouldThrowIllegalArgumentExce featureInfoRepository); FeatureSpec input = FeatureSpec.newBuilder() - .setId("notentity.granularity.name") + .setId("notentity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(1)) .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage( - "Id must be in format entity.granularity.name, " + "Id must be in format entity.name, " + "entity in Id does not match entity provided."); validator.validateFeatureSpec(input); } - @Test - public void featureSpecWithIdWithoutMatchingGranularityShouldThrowIllegalArgumentException() { - SpecValidator validator = - new SpecValidator( - storageInfoRepository, - entityInfoRepository, - featureGroupInfoRepository, - featureInfoRepository); - FeatureSpec input = - FeatureSpec.newBuilder() - .setId("entity.granularity.name") - .setName("name") - .setOwner("owner") - .setDescription("dasdad") - .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) - .build(); - exception.expect(IllegalArgumentException.class); - exception.expectMessage( - "Id must be in format entity.granularity.name, " - + "granularity in Id does not match granularity provided."); - validator.validateFeatureSpec(input); - } - @Test public void featureSpecWithIdWithoutMatchingNameShouldThrowIllegalArgumentException() { SpecValidator validator = @@ -236,16 +209,15 @@ public void featureSpecWithIdWithoutMatchingNameShouldThrowIllegalArgumentExcept featureInfoRepository); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.notname") + .setId("entity.notname") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage( - "Id must be in format entity.granularity.name, " + "Id must be in format entity.name, " + "name in Id does not match name provided."); validator.validateFeatureSpec(input); } @@ -261,12 +233,11 @@ public void featureSpecWithoutExistingEntityShouldThrowIllegalArgumentException( featureInfoRepository); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("Entity with name entity does not exist"); @@ -285,12 +256,11 @@ public void featureSpecWithInvalidFeatureGroupShouldThrowIllegalArgumentExceptio featureInfoRepository); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .setGroup("group") .build(); exception.expect(IllegalArgumentException.class); @@ -312,12 +282,11 @@ public void featureSpecWithoutExistingServingStoreShouldThrowIllegalArgumentExce DataStores dataStores = DataStores.newBuilder().setServing(servingStore).build(); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .setDataStores(dataStores) .build(); exception.expect(IllegalArgumentException.class); @@ -348,13 +317,12 @@ public void featureSpecWithoutServingStoreShouldInheritServingStoreIdFromGroup() DataStores dataStores = DataStores.newBuilder().setWarehouse(warehouseStore).build(); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") .setGroup("group") - .setGranularity(Granularity.Enum.forNumber(0)) .setDataStores(dataStores) .build(); exception.expect(IllegalArgumentException.class); @@ -387,12 +355,11 @@ public void featureSpecWithoutExistingWarehouseStoreShouldThrowIllegalArgumentEx DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .setDataStores(dataStores) .build(); exception.expect(IllegalArgumentException.class); @@ -423,12 +390,11 @@ public void featureSpecWithoutWarehouseStoreShouldBeAllowed() { DataStores.newBuilder().setServing(servingStore).build(); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .setDataStores(dataStores) .build(); validator.validateFeatureSpec(input); @@ -464,12 +430,11 @@ public void featureSpecWithUnsupportedWarehouseStoreShouldThrowIllegalArgumentEx DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .setDataStores(dataStores) .build(); exception.expect(IllegalArgumentException.class); @@ -511,12 +476,11 @@ public void featureSpecWithUnsupportedServingStoreShouldThrowIllegalArgumentExce DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); FeatureSpec input = FeatureSpec.newBuilder() - .setId("entity.none.name") + .setId("entity.name") .setName("name") .setOwner("owner") .setDescription("dasdad") .setEntity("entity") - .setGranularity(Granularity.Enum.forNumber(0)) .setDataStores(dataStores) .build(); exception.expect(IllegalArgumentException.class); diff --git a/core/src/test/resources/sql/expQuery1.sql b/core/src/test/resources/sql/expQuery1.sql index 83a91f9502f..97d21c3fcb3 100644 --- a/core/src/test/resources/sql/expQuery1.sql +++ b/core/src/test/resources/sql/expQuery1.sql @@ -1,59 +1,11 @@ -WITH raw_project_dataset_myentity_day AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_day_feature1 , - FIRST_VALUE(feature2 IGNORE NULLS) OVER w AS myentity_day_feature2 -FROM - `project.dataset.myentity_day` +SELECT + project.dataset.myentity.id, + project.dataset.myentity.event_timestamp , + myentity_feature1, + myentity_feature2, + myentity_feature3 +FROM + project.dataset.myentity WHERE event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_day AS ( SELECT - * -FROM - raw_project_dataset_myentity_day -WHERE - rownum = 1 ) , raw_project_dataset_myentity_none AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature3 IGNORE NULLS) OVER w AS myentity_none_feature3 -FROM - `project.dataset.myentity_none` WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_none AS ( SELECT - * -FROM - raw_project_dataset_myentity_none -WHERE - rownum = 1 ) SELECT - project_dataset_myentity_day.id, - project_dataset_myentity_day.event_timestamp , - project_dataset_myentity_day.myentity_day_feature1, - project_dataset_myentity_day.myentity_day_feature2, - project_dataset_myentity_none.myentity_none_feature3 -FROM - project_dataset_myentity_day LEFT -JOIN - project_dataset_myentity_none - ON project_dataset_myentity_day.id = project_dataset_myentity_none.id LIMIT 100 \ No newline at end of file + AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) LIMIT 100 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQuery2.sql b/core/src/test/resources/sql/expQuery2.sql index 17a06b18fdd..9b9f74a81d1 100644 --- a/core/src/test/resources/sql/expQuery2.sql +++ b/core/src/test/resources/sql/expQuery2.sql @@ -1,59 +1,9 @@ -WITH raw_project_dataset_myentity_day AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_day_feature1 , - FIRST_VALUE(feature2 IGNORE NULLS) OVER w AS myentity_day_feature2 +SELECT + project.dataset.myentity.id, + project.dataset.myentity.event_timestamp , + myentity_feature1 FROM - `project.dataset.myentity_day` + project.dataset.myentity WHERE event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_day AS ( SELECT - * -FROM - raw_project_dataset_myentity_day -WHERE - rownum = 1 ) , raw_project_dataset_myentity_none AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature3 IGNORE NULLS) OVER w AS myentity_none_feature3 -FROM - `project.dataset.myentity_none` WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_none AS ( SELECT - * -FROM - raw_project_dataset_myentity_none -WHERE - rownum = 1 ) SELECT - project_dataset_myentity_day.id, - project_dataset_myentity_day.event_timestamp , - project_dataset_myentity_day.myentity_day_feature1, - project_dataset_myentity_day.myentity_day_feature2, - project_dataset_myentity_none.myentity_none_feature3 -FROM - project_dataset_myentity_day LEFT -JOIN - project_dataset_myentity_none - ON project_dataset_myentity_day.id = project_dataset_myentity_none.id \ No newline at end of file + AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) LIMIT 1000 \ No newline at end of file diff --git a/core/src/test/resources/sql/expQuery3.sql b/core/src/test/resources/sql/expQuery3.sql deleted file mode 100644 index 2c5405dd6ad..00000000000 --- a/core/src/test/resources/sql/expQuery3.sql +++ /dev/null @@ -1,150 +0,0 @@ -WITH raw_project_dataset_myentity_second AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_second_feature1 -FROM - `project.dataset.myentity_second` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_second AS ( SELECT - * -FROM - raw_project_dataset_myentity_second -WHERE - rownum = 1 ) , raw_project_dataset_myentity_minute AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_minute_feature1 -FROM - `project.dataset.myentity_minute` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_minute AS ( SELECT - * -FROM - raw_project_dataset_myentity_minute -WHERE - rownum = 1 ) , raw_project_dataset_myentity_hour AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_hour_feature1 -FROM - `project.dataset.myentity_hour` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_hour AS ( SELECT - * -FROM - raw_project_dataset_myentity_hour -WHERE - rownum = 1 ) , raw_project_dataset_myentity_day AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_day_feature1 -FROM - `project.dataset.myentity_day` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_day AS ( SELECT - * -FROM - raw_project_dataset_myentity_day -WHERE - rownum = 1 ) , raw_project_dataset_myentity_none AS (SELECT - ROW_NUMBER() OVER (PARTITION -BY - id, - event_timestamp -ORDER BY - created_timestamp DESC) rownum, - id, - event_timestamp, - created_timestamp , - FIRST_VALUE(feature1 IGNORE NULLS) OVER w AS myentity_none_feature1 -FROM - `project.dataset.myentity_none` WINDOW w AS ( PARTITION -BY - id, - event_timestamp -ORDER BY - event_timestamp DESC )), project_dataset_myentity_none AS ( SELECT - * -FROM - raw_project_dataset_myentity_none -WHERE - rownum = 1 ) SELECT - project_dataset_myentity_second.id, - project_dataset_myentity_second.event_timestamp , - project_dataset_myentity_second.myentity_second_feature1, - project_dataset_myentity_minute.myentity_minute_feature1, - project_dataset_myentity_hour.myentity_hour_feature1, - project_dataset_myentity_day.myentity_day_feature1, - project_dataset_myentity_none.myentity_none_feature1 -FROM - project_dataset_myentity_second LEFT -JOIN - project_dataset_myentity_minute - ON project_dataset_myentity_second.id = project_dataset_myentity_minute.id - AND TIMESTAMP_TRUNC(project_dataset_myentity_second.event_timestamp, - MINUTE) = project_dataset_myentity_minute.event_timestamp LEFT -JOIN - project_dataset_myentity_hour - ON project_dataset_myentity_second.id = project_dataset_myentity_hour.id - AND TIMESTAMP_TRUNC(project_dataset_myentity_second.event_timestamp, - HOUR) = project_dataset_myentity_hour.event_timestamp LEFT -JOIN - project_dataset_myentity_day - ON project_dataset_myentity_second.id = project_dataset_myentity_day.id - AND TIMESTAMP_TRUNC(project_dataset_myentity_second.event_timestamp, - DAY) = project_dataset_myentity_day.event_timestamp LEFT -JOIN - project_dataset_myentity_none - ON project_dataset_myentity_second.id = project_dataset_myentity_none.id LIMIT 1000 diff --git a/ingestion/example/core_specs/feature/product.day.completed_orders.json b/ingestion/example/core_specs/feature/product.day.completed_orders.json index 1e5c7e2665c..a4582848e02 100644 --- a/ingestion/example/core_specs/feature/product.day.completed_orders.json +++ b/ingestion/example/core_specs/feature/product.day.completed_orders.json @@ -1,7 +1,6 @@ { - "id": "product.day.completed_orders", + "id": "product.completed_orders", "entity": "product", - "granularity": "DAY", "name": "completed_orders", "owner": "feast@example.com", "description": "This feature represents a product's completed orders per day", diff --git a/ingestion/example/core_specs/feature/user.none.age.json b/ingestion/example/core_specs/feature/user.none.age.json index c09957e132d..9b8f8a68cf6 100644 --- a/ingestion/example/core_specs/feature/user.none.age.json +++ b/ingestion/example/core_specs/feature/user.none.age.json @@ -1,7 +1,6 @@ { - "id": "user.none.age", + "id": "user.age", "entity": "user", - "granularity": "NONE", "name": "age", "owner": "feast@example.com", "description": "This feature represents a user's age", diff --git a/ingestion/example/core_specs/feature/user.none.completed_orders.json b/ingestion/example/core_specs/feature/user.none.completed_orders.json index 8b62f6065e0..0280cf74831 100644 --- a/ingestion/example/core_specs/feature/user.none.completed_orders.json +++ b/ingestion/example/core_specs/feature/user.none.completed_orders.json @@ -1,7 +1,6 @@ { - "id": "user.none.completed_orders", + "id": "user.completed_orders", "entity": "user", - "granularity": "NONE", "name": "completed_orders", "owner": "feast@example.com", "description": "This feature represents a user's total completed orders", diff --git a/ingestion/example/import_products.yaml b/ingestion/example/import_products.yaml index 9e20e270d34..cb52580cf0d 100644 --- a/ingestion/example/import_products.yaml +++ b/ingestion/example/import_products.yaml @@ -11,5 +11,5 @@ schema: fields: - name: id - name: timestamp - - featureId: product.day.completed_orders + - featureId: product.completed_orders diff --git a/ingestion/example/import_users.yaml b/ingestion/example/import_users.yaml index bd35e877001..5e2d6e553ac 100644 --- a/ingestion/example/import_users.yaml +++ b/ingestion/example/import_users.yaml @@ -11,6 +11,6 @@ schema: fields: - name: id - name: name - - featureId: user.none.age - - featureId: user.none.completed_orders + - featureId: user.age + - featureId: user.completed_orders diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index e05362c15c7..a77d07b4cfe 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -38,7 +38,6 @@ import feast.ingestion.transform.WarehouseStoreTransform; import feast.ingestion.transform.fn.ConvertTypesDoFn; import feast.ingestion.transform.fn.LoggerDoFn; -import feast.ingestion.transform.fn.RoundEventTimestampsDoFn; import feast.ingestion.values.PFeatureRows; import feast.options.OptionsParser; import feast.specs.ImportJobSpecsProto.ImportJobSpecs; @@ -168,7 +167,7 @@ public void expand() { "A sample of size 1 of incoming rows from MAIN and ERRORS will logged every 30 seconds for visibility"); logNRows(pFeatureRows, "Output sample", 1, Duration.standardSeconds(30)); - PFeatureRows warehouseRows = roundTimestamps("Round timestamps for warehouse", pFeatureRows); + PFeatureRows warehouseRows = pFeatureRows; PFeatureRows servingRows = pFeatureRows; if (jobOptions.isCoalesceRowsEnabled()) { // Should we merge and dedupe rows before writing to the serving store? @@ -176,7 +175,6 @@ public void expand() { jobOptions.getCoalesceRowsDelaySeconds(), jobOptions.getCoalesceRowsTimeoutSeconds())); } - servingRows = roundTimestamps("Round timestamps for serving", servingRows); if (!dryRun) { servingRows.apply("Write to Serving Stores", servingStoreTransform); @@ -185,16 +183,6 @@ public void expand() { } } - public PFeatureRows roundTimestamps(String name, PFeatureRows pFeatureRows) { - return - PFeatureRows.of( - pFeatureRows - .getMain() - .apply(name, - ParDo.of(new RoundEventTimestampsDoFn())), - pFeatureRows.getErrors()); - } - public PipelineResult run() { PipelineResult result = pipeline.run(); log.info(String.format("FeastImportJobId:%s", this.retrieveId(result))); diff --git a/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java b/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java index c357454dbdf..d42be02fe38 100644 --- a/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java +++ b/ingestion/src/main/java/feast/ingestion/metrics/FeastMetrics.java @@ -17,12 +17,11 @@ package feast.ingestion.metrics; -import feast.ingestion.util.DateUtil; +import com.google.protobuf.util.Timestamps; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto; import feast.types.FeatureRowProto.FeatureRow; -import java.time.Instant; import lombok.AllArgsConstructor; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.transforms.DoFn; @@ -31,8 +30,7 @@ public class FeastMetrics { public static final String FEAST_NAMESPACE = "feast"; - private FeastMetrics() { - } + private FeastMetrics() {} private static void inc(String name) { Metrics.counter(FeastMetrics.FEAST_NAMESPACE, name).inc(); @@ -75,19 +73,13 @@ public void processElement( public static class CalculateLagMetricFunc extends DoFn { @ProcessElement - public void processElement(@Element FeatureRowExtended element, - OutputReceiver out) { + public void processElement( + @Element FeatureRowExtended element, OutputReceiver out) { FeatureRowProto.FeatureRow row = element.getRow(); com.google.protobuf.Timestamp eventTimestamp = row.getEventTimestamp(); - Instant now = Instant.now(); - com.google.protobuf.Timestamp roundedCurrentTimestamp = - DateUtil.roundToGranularity( - com.google.protobuf.Timestamp.newBuilder() - .setSeconds(now.getEpochSecond()) - .setNanos(now.getNano()) - .build(), - row.getGranularity()); - long lagSeconds = roundedCurrentTimestamp.getSeconds() - eventTimestamp.getSeconds(); + + com.google.protobuf.Timestamp now = Timestamps.fromMillis(System.currentTimeMillis()); + long lagSeconds = now.getSeconds() - eventTimestamp.getSeconds(); FeastMetrics.update("row:lag", lagSeconds); out.output(element); } diff --git a/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java b/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java index a7c53d21c65..aa8a1754c2f 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java +++ b/ingestion/src/main/java/feast/ingestion/transform/CoalesceFeatureRows.java @@ -59,16 +59,18 @@ * Takes FeatureRow, and merges them if they have the same FeatureRowKey, so that the latest values * will be emitted. It emits only once for batch. * - * For streaming we emits after a delay of 10 seconds (event time) by default we keep the previous - * state around for merging with future events. These timeout after 30 minutes by default. + *

For streaming we emits after a delay of 10 seconds (event time) by default we keep the + * previous state around for merging with future events. These timeout after 30 minutes by default. */ -public class CoalesceFeatureRows extends - PTransform, PCollection> { +public class CoalesceFeatureRows + extends PTransform, PCollection> { - private static final SerializableFunction KEY_FUNCTION = (row) -> - CoalesceKey.newBuilder() - .setEntityName(row.getEntityName()) - .setEntityKey(row.getEntityKey()).build(); + private static final SerializableFunction KEY_FUNCTION = + (row) -> + CoalesceKey.newBuilder() + .setEntityName(row.getEntityName()) + .setEntityKey(row.getEntityKey()) + .build(); private static final Duration DEFAULT_DELAY = Duration.standardSeconds(10); private static final Duration DEFAULT_TIMEOUT = Duration.ZERO; @@ -89,16 +91,15 @@ public CoalesceFeatureRows(Duration delay, Duration timeout) { this.timeout = (timeout.isEqual(Duration.ZERO)) ? DEFAULT_TIMEOUT : timeout; } - /** - * Return a FeatureRow of the new features accumulated since the given timestamp - */ + /** Return a FeatureRow of the new features accumulated since the given timestamp */ public static FeatureRow toFeatureRow(CoalesceAccum accum, long counter) { - Preconditions.checkArgument(counter <= - accum.getCounter(), "Accumulator has no features at or newer than the provided counter"); - FeatureRow.Builder builder = FeatureRow.newBuilder() - .setEntityName(accum.getEntityName()) - .setGranularity(accum.getGranularity()) - .setEntityKey(accum.getEntityKey()); + Preconditions.checkArgument( + counter <= accum.getCounter(), + "Accumulator has no features at or newer than the provided counter"); + FeatureRow.Builder builder = + FeatureRow.newBuilder() + .setEntityName(accum.getEntityName()) + .setEntityKey(accum.getEntityKey()); // This will be the latest timestamp if (accum.hasEventTimestamp()) { builder.setEventTimestamp(accum.getEventTimestamp()); @@ -108,12 +109,15 @@ public static FeatureRow toFeatureRow(CoalesceAccum accum, long counter) { if (counter <= 0) { builder.addAllFeatures(features.values()); } else { - List featureList = accum.getFeatureMarksMap().entrySet().stream() - .filter((e) -> e.getValue() > counter) - .map((e) -> features.get(e.getKey())) - .collect(Collectors.toList()); + List featureList = + accum + .getFeatureMarksMap() + .entrySet() + .stream() + .filter((e) -> e.getValue() > counter) + .map((e) -> features.get(e.getKey())) + .collect(Collectors.toList()); builder.addAllFeatures(featureList); - } return builder.build(); } @@ -129,8 +133,7 @@ public static CoalesceAccum combineFeatureRows(CoalesceAccum seed, Iterable expand(PCollection input) { - PCollection> kvs = input - .apply(WithKeys.of(KEY_FUNCTION).withKeyType(TypeDescriptor.of(CoalesceKey.class))) - .setCoder(KvCoder.of(ProtoCoder.of(CoalesceKey.class), ProtoCoder.of(FeatureRow.class))); + PCollection> kvs = + input + .apply(WithKeys.of(KEY_FUNCTION).withKeyType(TypeDescriptor.of(CoalesceKey.class))) + .setCoder( + KvCoder.of(ProtoCoder.of(CoalesceKey.class), ProtoCoder.of(FeatureRow.class))); if (kvs.isBounded().equals(IsBounded.UNBOUNDED)) { - return kvs.apply("Configure window", Window.>configure() - .withAllowedLateness(Duration.ZERO) - .discardingFiredPanes() - .triggering(AfterProcessingTime.pastFirstElementInPane())) + return kvs.apply( + "Configure window", + Window.>configure() + .withAllowedLateness(Duration.ZERO) + .discardingFiredPanes() + .triggering(AfterProcessingTime.pastFirstElementInPane())) .apply(ParDo.of(new CombineStateDoFn(delay, timeout))) .apply(Values.create()); } else { @@ -185,22 +191,25 @@ public PCollection expand(PCollection input) { } } - @Slf4j @AllArgsConstructor - public static class CombineStateDoFn extends - DoFn, KV> { + public static class CombineStateDoFn + extends DoFn, KV> { @StateId("lastKnownAccumValue") private final StateSpec> lastKnownAccumValueSpecs = StateSpecs.value(ProtoCoder.of(CoalesceAccum.class)); + @StateId("newElementsBag") private final StateSpec> newElementsBag = StateSpecs.bag(ProtoCoder.of(FeatureRow.class)); + @StateId("lastTimerTimestamp") private final StateSpec> lastTimerTimestamp = StateSpecs.value(); + @TimerId("bufferTimer") private final TimerSpec bufferTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME); + @TimerId("timeoutTimer") private final TimerSpec timeoutTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME); @@ -208,7 +217,8 @@ public static class CombineStateDoFn extends private Duration timeout; @ProcessElement - public void processElement(ProcessContext context, + public void processElement( + ProcessContext context, @StateId("newElementsBag") BagState newElementsBag, @TimerId("bufferTimer") Timer bufferTimer, @TimerId("timeoutTimer") Timer timeoutTimer, @@ -222,7 +232,8 @@ public void processElement(ProcessContext context, // We never timeout the state if a timeout is not set. timeoutTimer.offset(timeout).setRelative(); } - if (lastTimerTimestamp == null || lastTimerTimestamp.isBefore(contextTimestamp) + if (lastTimerTimestamp == null + || lastTimerTimestamp.isBefore(contextTimestamp) || lastTimerTimestamp.equals(contextTimestamp)) { lastTimerTimestamp = context.timestamp().plus(delay); log.debug("Setting timer for key {} to {}", context.element().getKey(), lastTimerTimestamp); @@ -233,7 +244,8 @@ public void processElement(ProcessContext context, @OnTimer("bufferTimer") public void bufferOnTimer( - OnTimerContext context, OutputReceiver> out, + OnTimerContext context, + OutputReceiver> out, @StateId("newElementsBag") BagState newElementsBag, @StateId("lastKnownAccumValue") ValueState lastKnownAccumValue) { log.debug("bufferOnTimer triggered {}", context.timestamp()); @@ -242,7 +254,8 @@ public void bufferOnTimer( @OnTimer("timeoutTimer") public void timeoutOnTimer( - OnTimerContext context, OutputReceiver> out, + OnTimerContext context, + OutputReceiver> out, @StateId("newElementsBag") BagState newElementsBag, @StateId("lastKnownAccumValue") ValueState lastKnownAccumValue) { log.debug("timeoutOnTimer triggered {}", context.timestamp()); diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java index cd19a913ff9..c5ed560eeaf 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ConvertTypesDoFn.java @@ -41,12 +41,10 @@ public void processElementImpl(ProcessContext context) { rowBuilder.setEventTimestamp(row.getEventTimestamp()); } - for (Feature feature : row.getFeaturesList()) { String featureId = feature.getId(); FeatureSpec featureSpec = specs.getFeatureSpec(featureId); - rowBuilder.setGranularity(featureSpec.getGranularity()); rowBuilder.addFeatures( Feature.newBuilder() .setId(featureId) diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/RoundEventTimestampsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/RoundEventTimestampsDoFn.java deleted file mode 100644 index 7170e65915b..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/RoundEventTimestampsDoFn.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.transform.fn; - -import org.apache.beam.sdk.transforms.DoFn; -import feast.ingestion.util.DateUtil; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; - -public class RoundEventTimestampsDoFn extends DoFn { - @ProcessElement - public void processElement(ProcessContext context) { - FeatureRowExtended rowExtended = context.element(); - FeatureRow row = rowExtended.getRow(); - com.google.protobuf.Timestamp timestamp = rowExtended.getRow().getEventTimestamp(); - row = - row.toBuilder() - .setEventTimestamp(DateUtil.roundToGranularity(timestamp, row.getGranularity())) - .build(); - context.output(rowExtended.toBuilder().setRow(row).build()); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/SplitFeaturesDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/SplitFeaturesDoFn.java index 8fdb226029e..5c46a5dd5df 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/SplitFeaturesDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/SplitFeaturesDoFn.java @@ -50,7 +50,6 @@ public void processElement(ProcessContext context) { if (builder == null) { builder = FeatureRow.newBuilder() - .setGranularity(row.getGranularity()) .setEventTimestamp(row.getEventTimestamp()) .setEntityName(row.getEntityName()) .setEntityKey(row.getEntityKey()); diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java index 8336c8dd323..06332eb8bf6 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java @@ -37,7 +37,6 @@ import feast.store.warehouse.FeatureWarehouseFactoryService; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import java.util.ArrayList; import java.util.HashSet; @@ -79,7 +78,6 @@ public void processElementImpl(ProcessContext context) { FeatureRow row = context.element().getRow(); EntitySpec entitySpec = specs.getEntitySpec(row.getEntityName()); Preconditions.checkNotNull(entitySpec, "Entity spec not found for " + row.getEntityName()); - ImportSpec importSpec = specs.getImportSpec(); try { checkArgument(!row.getEntityKey().isEmpty(), "Entity key must not be empty"); @@ -90,10 +88,6 @@ public void processElementImpl(ProcessContext context) { String.format( "Row entity not found in import spec entities. entity=%s", row.getEntityName())); - checkArgument( - !row.getGranularity().equals(Enum.UNRECOGNIZED), - String.format("Unrecognised granularity %s", row.getGranularity())); - checkArgument(row.hasEventTimestamp(), "Must have eventTimestamp set"); checkArgument(row.getFeaturesCount() > 0, "Must have at least one feature set"); @@ -124,11 +118,6 @@ public void processElementImpl(ProcessContext context) { "Feature must have same entity as row. featureId=%s FeatureRow.entityName=%s FeatureSpec.entity=%s", feature.getId(), row.getEntityName(), featureSpec.getEntity())); - checkArgument( - featureSpec.getGranularity().equals(row.getGranularity()), - String.format( - "Feature must have same granularity as entity, featureId=%s", feature.getId())); - ValueType.Enum expectedType = featureSpec.getValueType(); ValueType.Enum actualType = Values.toValueType(feature.getValue()); checkArgument( diff --git a/ingestion/src/main/java/feast/ingestion/util/DateUtil.java b/ingestion/src/main/java/feast/ingestion/util/DateUtil.java index 15c4cc4d465..ad779dbea33 100644 --- a/ingestion/src/main/java/feast/ingestion/util/DateUtil.java +++ b/ingestion/src/main/java/feast/ingestion/util/DateUtil.java @@ -20,11 +20,12 @@ import com.google.protobuf.Timestamp; import java.time.Instant; import org.joda.time.DateTime; -import org.joda.time.DateTimeField; import org.joda.time.DateTimeZone; -import org.joda.time.MutableDateTime; -import org.joda.time.format.*; -import feast.types.GranularityProto.Granularity; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.DateTimeFormatterBuilder; +import org.joda.time.format.DateTimeParser; +import org.joda.time.format.ISODateTimeFormat; public class DateUtil { @@ -34,12 +35,12 @@ public class DateUtil { DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); DateTimeFormatter base = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); DateTimeFormatter zone = DateTimeFormat.forPattern(" ZZZ"); - DateTimeParser fractionSecondParser = new DateTimeFormatterBuilder() - .appendLiteral(".") - .appendFractionOfSecond(1,6) - .toParser(); + DateTimeParser fractionSecondParser = + new DateTimeFormatterBuilder().appendLiteral(".").appendFractionOfSecond(1, 6).toParser(); - FALLBACK_TIMESTAMP_FORMAT = formatterBuilder.append(base) + FALLBACK_TIMESTAMP_FORMAT = + formatterBuilder + .append(base) .appendOptional(fractionSecondParser) .append(zone) .toFormatter(); @@ -93,30 +94,4 @@ public static Timestamp maxTimestamp(Timestamp a, Timestamp b) { public static long toMillis(Timestamp timestamp) { return toDateTime(timestamp).getMillis(); } - - public static Timestamp roundToGranularity(Timestamp timestamp, Granularity.Enum granularity) { - MutableDateTime dt = new MutableDateTime(DateTimeZone.UTC); - DateTimeField roundingField; - switch (granularity) { - case DAY: - roundingField = dt.getChronology().dayOfMonth(); - break; - case HOUR: - roundingField = dt.getChronology().hourOfDay(); - break; - case MINUTE: - roundingField = dt.getChronology().minuteOfHour(); - break; - case SECOND: - roundingField = dt.getChronology().secondOfMinute(); - break; - case NONE: - return Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); - default: - throw new RuntimeException("Unrecognised time series granularity"); - } - dt.setRounding(roundingField, MutableDateTime.ROUND_FLOOR); - dt.setMillis(toDateTime(timestamp).getMillis()); - return toTimestamp(dt.toDateTime()); - } } diff --git a/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java index 1d14b1201eb..6a9330c7c2b 100644 --- a/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java @@ -60,8 +60,7 @@ public class FeatureRowToBigTableMutationDoFn this.specs = specs; } - public static BigTableRowKey makeBigTableRowKey( - String entityKey) { + public static BigTableRowKey makeBigTableRowKey(String entityKey) { return BigTableRowKey.newBuilder() .setSha1Prefix(DigestUtils.sha1Hex(entityKey).substring(0, 7)) @@ -89,15 +88,14 @@ private String getTableName(FeatureRow row) { /** * Given an row and a feature info service, build a BigTable Put mutation * - *

bigtable row key = {sha1(row.key), row.key, row.timestamp} family = {feature.group, - * granularity} qualifier = {feature.name} value = {feature.value} + *

bigtable row key = {sha1(row.key), row.key, row.timestamp} family = {feature.group} + * qualifier = {feature.name} value = {feature.value} */ public Put makePut(FeatureRowExtended rowExtended) { FeatureRow row = rowExtended.getRow(); // We always additinally overwrite a None granularity row so that it is trivial to retrieve the // latest across all features. - Put latestPut = - new Put(makeBigTableRowKey(row.getEntityKey()).toByteArray()); + Put latestPut = new Put(makeBigTableRowKey(row.getEntityKey()).toByteArray()); for (Feature feature : row.getFeaturesList()) { FeatureSpec featureSpec = specs.getFeatureSpec(feature.getId()); BigTableFeatureOptions options = servingOptionsCache.get(featureSpec); diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java index dbe57bff9fa..01f779bff4c 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java @@ -54,13 +54,6 @@ public FeatureRowToRedisMutationDoFn(Specs specs) { this.random = new Random(); } - // TODO when extracting to a storage module, use reasonable defaults per granularity. - // TODO check is greater than 1 - static long getBucketId(com.google.protobuf.Timestamp eventTimestamp, Duration bucketSize) { - checkArgument(bucketSize.getStandardSeconds() > 0, "BucketSize must be greater than zero"); - return eventTimestamp.getSeconds() / bucketSize.getStandardSeconds(); - } - static RedisBucketKey getRedisBucketKey( String entityId, String featureIdSha1Prefix, long bucketId) { return RedisBucketKey.newBuilder() @@ -91,13 +84,10 @@ public void processElement(ProcessContext context) { RedisFeatureOptions options = servingOptionsCache.get(featureSpec); - com.google.protobuf.Timestamp roundedTimestamp = - DateUtil.roundToGranularity(row.getEventTimestamp(), featureSpec.getGranularity()); - RedisBucketValue value = RedisBucketValue.newBuilder() .setValue(feature.getValue()) - .setEventTimestamp(roundedTimestamp) + .setEventTimestamp(row.getEventTimestamp()) .build(); RedisBucketKey keyForLatest = getRedisBucketKey(entityKey, featureIdHash, 0L); diff --git a/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java index c8ec8a7825e..06180f364ae 100644 --- a/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java @@ -21,13 +21,10 @@ import com.google.common.base.Strings; import com.google.inject.Inject; import feast.ingestion.model.Specs; -import feast.store.FeatureStoreWrite; -import feast.ingestion.transform.SplitFeatures.SingleOutputSplit; import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.store.FeatureStoreWrite; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; @@ -38,6 +35,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.WriteResult; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.ValueInSingleWindow; import org.joda.time.Duration; @@ -60,11 +58,6 @@ public Write(BigQueryStoreOptions bigQueryOptions, Specs specs) { @Override public PDone expand(PCollection input) { - SingleOutputSplit granularitySplitter = - new SingleOutputSplit<>(FeatureSpec::getGranularity, specs); - PCollection features = - input.apply("Split by granularity", granularitySplitter); - FeatureRowToBigQueryTableRowDoFn toTableRowDoFn = new FeatureRowToBigQueryTableRowDoFn(specs); BigQueryIO.Write write = BigQueryIO.write() @@ -75,9 +68,7 @@ public String getDestination(ValueInSingleWindow element) { FeatureRow row = featureRowExtended.getRow(); EntitySpec entityInfo = specs.getEntitySpec(row.getEntityName()); - Granularity.Enum granularity = row.getGranularity(); - String tableName = - entityInfo.getName() + "_" + granularity.name().toLowerCase(); + String tableName = entityInfo.getName(); return bigQueryOptions.project + ":" + bigQueryOptions.dataset @@ -105,16 +96,16 @@ public TableSchema getSchema(String destination) { write = write.withCustomGcsTempLocation(StaticValueProvider.of(bigQueryOptions.tempLocation)); } - switch (input.isBounded()) { - case UNBOUNDED: - write = - write - .withTriggeringFrequency(triggerFrequency) - // this is apparently supposed to be the default according to beam code comments. - .withNumFileShards(100); - } - WriteResult result = features.apply(write); + if (input.isBounded() == IsBounded.UNBOUNDED) { + write = + write + .withTriggeringFrequency(triggerFrequency) + // this is apparently supposed to be the default according to beam code + // comments. + .withNumFileShards(100); + } + WriteResult result = input.apply(write); return PDone.in(input.getPipeline()); } } diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto index f8e82a59521..3e7ce90cbc0 100644 --- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto +++ b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto @@ -18,7 +18,6 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; import "feast/types/Feature.proto"; -import "feast/types/Granularity.proto"; option java_package = "feast_ingestion.types"; option java_outer_classname = "CoalesceAccumProto"; @@ -28,7 +27,6 @@ message CoalesceAccum { string entityKey = 1; google.protobuf.Timestamp eventTimestamp = 3; string entityName = 4; - feast.types.Granularity.Enum granularity = 5; map features = 6; // map of features to their counter values when they were last added to accumulator diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index 67e8ed8e862..356e023347a 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -27,7 +27,7 @@ import com.google.common.io.Resources; import com.google.inject.Guice; import com.google.inject.Injector; -import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; import feast.ToOrderedFeatureRows; import feast.ingestion.boot.ImportJobModule; import feast.ingestion.boot.TestPipelineModule; @@ -46,11 +46,11 @@ import feast.store.warehouse.FeatureWarehouseFactoryService; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.ParseException; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -87,7 +87,7 @@ public ImportJobPipelineOptions initOptions() { } @Test - public void testImportCSV() throws IOException { + public void testImportCSV() throws IOException, ParseException { ImportSpec importSpec = ProtoUtil.decodeProtoYaml( "---\n" @@ -101,8 +101,8 @@ public void testImportCSV() throws IOException { + " timestampValue: 2018-09-25T00:00:00.000Z\n" + " fields:\n" + " - name: id\n" - + " - featureId: testEntity.none.testInt32\n" - + " - featureId: testEntity.none.testString\n" + + " - featureId: testEntity.testInt32\n" + + " - featureId: testEntity.testString\n" + "\n", ImportSpec.getDefaultInstance()); @@ -142,33 +142,32 @@ public void testImportCSV() throws IOException { Lists.newArrayList( normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEventTimestamp(Timestamps.parse("2018-09-25T00:00:00.000Z")) .setEntityKey("1") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) - .addFeatures(Features.of("testEntity.none.testString", Values.ofString("a"))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(101))) + .addFeatures(Features.of("testEntity.testString", Values.ofString("a"))) .build()), normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEventTimestamp(Timestamps.parse("2018-09-25T00:00:00.000Z")) .setEntityKey("2") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(202))) - .addFeatures(Features.of("testEntity.none.testString", Values.ofString("b"))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(202))) + .addFeatures(Features.of("testEntity.testString", Values.ofString("b"))) .build()), normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEventTimestamp(Timestamps.parse("2018-09-25T00:00:00.000Z")) .setEntityKey("3") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(303))) - .addFeatures(Features.of("testEntity.none.testString", Values.ofString("c"))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(303))) + .addFeatures(Features.of("testEntity.testString", Values.ofString("c"))) .build())); PAssert.that(writtenToErrors).satisfies(hasCount(0)); + PAssert.that(writtenToServing).satisfies(hasCount(3)); + PAssert.that(writtenToWarehouse).satisfies(hasCount(3)); PAssert.that(writtenToServing.apply("serving toFeatureRows", new ToOrderedFeatureRows())) .containsInAnyOrder(expectedRows); @@ -180,7 +179,7 @@ public void testImportCSV() throws IOException { } @Test - public void testImportFileJson() throws IOException { + public void testImportFileJson() throws IOException, ParseException { ImportSpec importSpec = ProtoUtil.decodeProtoYaml( "---\n" @@ -195,7 +194,7 @@ public void testImportFileJson() throws IOException { + " fields:\n" + " - name: id\n" + " - name: x\n" - + " featureId: testEntity.none.testInt32\n" + + " featureId: testEntity.testInt32\n" + "\n", ImportSpec.getDefaultInstance()); @@ -234,19 +233,17 @@ public void testImportFileJson() throws IOException { Lists.newArrayList( normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEventTimestamp(Timestamps.parse("2018-09-25T00:00:00.000Z")) .setEntityKey("1") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(101))) .build()), normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(Timestamp.getDefaultInstance()) + .setEventTimestamp(Timestamps.parse("2018-09-25T00:00:00.000Z")) .setEntityKey("2") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(202))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(202))) .build())); PAssert.that(writtenToErrors).satisfies(hasCount(0)); @@ -277,8 +274,8 @@ public void testImportCSV_withSample1() throws IOException { + " timestampValue: 2018-09-25T00:00:00.000Z\n" + " fields:\n" + " - name: id\n" - + " - featureId: testEntity.none.testInt32\n" - + " - featureId: testEntity.none.testString\n" + + " - featureId: testEntity.testInt32\n" + + " - featureId: testEntity.testString\n" + "\n", ImportSpec.getDefaultInstance()); @@ -322,7 +319,7 @@ public void testImportCSV_withSample1() throws IOException { } @Test - public void testImportCSV_withCoalesceRows() throws IOException { + public void testImportCSV_withCoalesceRows() throws IOException, ParseException { ImportSpec importSpec = ProtoUtil.decodeProtoYaml( "---\n" @@ -339,8 +336,8 @@ public void testImportCSV_withCoalesceRows() throws IOException { + " fields:\n" + " - name: id\n" + " - name: timestamp\n" - + " - featureId: testEntity.none.testInt32\n" - + " - featureId: testEntity.none.testString\n" + + " - featureId: testEntity.testInt32\n" + + " - featureId: testEntity.testString\n" + "\n", ImportSpec.getDefaultInstance()); @@ -383,29 +380,29 @@ public void testImportCSV_withCoalesceRows() throws IOException { .containsInAnyOrder( normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) .setEntityKey("1") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) - .addFeatures(Features.of("testEntity.none.testString", Values.ofString("b"))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(101))) + .addFeatures(Features.of("testEntity.testString", Values.ofString("b"))) + .setEventTimestamp(Timestamps.parse("2018-09-26T00:00:00.000Z")) .build())); PAssert.that(writtenToWarehouse.apply("warehouse toFeatureRows", new ToOrderedFeatureRows())) .containsInAnyOrder( normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) .setEntityKey("1") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testInt32", Values.ofInt32(101))) - .addFeatures(Features.of("testEntity.none.testString", Values.ofString("a"))) + .addFeatures(Features.of("testEntity.testInt32", Values.ofInt32(101))) + .addFeatures(Features.of("testEntity.testString", Values.ofString("a"))) + .setEventTimestamp(Timestamps.parse("2018-09-25T00:00:00.000Z")) .build()), normalize( FeatureRow.newBuilder() - .setGranularity(Granularity.Enum.NONE) .setEntityKey("1") .setEntityName("testEntity") - .addFeatures(Features.of("testEntity.none.testString", Values.ofString("b"))) + .addFeatures(Features.of("testEntity.testString", Values.ofString("b"))) + .setEventTimestamp(Timestamps.parse("2018-09-26T00:00:00.000Z")) .build())); testPipeline.run(); @@ -429,9 +426,9 @@ public void testImportCSVUnknownServingStoreError() throws IOException { + " timestampValue: 2018-09-25T00:00:00.000Z\n" + " fields:\n" + " - name: id\n" - + " - featureId: testEntity.none.unknownInt32\n" + + " - featureId: testEntity.unknownInt32\n" // Unknown store is not available - + " - featureId: testEntity.none.testString\n" + + " - featureId: testEntity.testString\n" + "\n", ImportSpec.getDefaultInstance()); @@ -484,8 +481,8 @@ public void testImportWithErrors() throws IOException { + " timestampValue: 2018-09-25T00:00:00.000Z\n" + " fields:\n" + " - name: id\n" - + " - featureId: testEntity.none.testString\n" - + " - featureId: testEntity.none.testInt32\n" + + " - featureId: testEntity.testString\n" + + " - featureId: testEntity.testInt32\n" + "\n", ImportSpec.getDefaultInstance()); @@ -552,8 +549,8 @@ public void testImportWithoutWarehouseStore() throws IOException { + " timestampValue: 2018-09-25T00:00:00.000Z\n" + " fields:\n" + " - name: id\n" - + " - featureId: testEntity.none.testInt64NoWarehouse\n" - + " - featureId: testEntity.none.testStringNoWarehouse\n" + + " - featureId: testEntity.testInt64NoWarehouse\n" + + " - featureId: testEntity.testStringNoWarehouse\n" + "\n", ImportSpec.getDefaultInstance()); @@ -617,8 +614,8 @@ public void testImportWithoutWarehouseStoreSetByFeature() throws IOException { + " timestampValue: 2018-09-25T00:00:00.000Z\n" + " fields:\n" + " - name: id\n" - + " - featureId: testEntity.none.testInt64NoWarehouse\n" - + " - featureId: testEntity.none.testStringNoWarehouse\n" + + " - featureId: testEntity.testInt64NoWarehouse\n" + + " - featureId: testEntity.testStringNoWarehouse\n" + "\n", ImportSpec.getDefaultInstance()); diff --git a/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java b/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java index 3494e4c41bb..357df7d6a72 100644 --- a/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java +++ b/ingestion/src/test/java/feast/ingestion/config/ImportJobSpecsSupplierTest.java @@ -29,7 +29,6 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.ValueType.Enum; import java.io.File; import java.io.IOException; @@ -41,8 +40,7 @@ public class ImportJobSpecsSupplierTest { - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); String importSpecYaml = "---\n" + "servingStorageSpecs:\n" @@ -58,9 +56,8 @@ public class ImportJobSpecsSupplierTest { + " description: This is a test entity\n" + " tags: []\n" + "featureSpecs:\n" - + " - id: testEntity.day.testInt64\n" + + " - id: testEntity.testInt64\n" + " entity: testEntity\n" - + " granularity: DAY\n" + " name: testInt64\n" + " owner: feast@example.com\n" + " description: This is test feature of type integer\n" @@ -81,7 +78,7 @@ public class ImportJobSpecsSupplierTest { + " - name: timestamp\n" + " - name: driver_id\n" + " - name: trips_completed\n" - + " featureId: driver.none.trips_completed\n" + + " featureId: driver.trips_completed\n" + "\n"; @Test @@ -93,48 +90,51 @@ public void testSupplierImportSpecYamlFile() throws IOException { ImportJobSpecs importJobSpecs = new ImportJobSpecsSupplier(yamlFile.getParent()).get(); Specs specs = new Specs("", importJobSpecs); - System.out - .println(JsonFormat.printer().omittingInsignificantWhitespace().print(importJobSpecs)); - assertEquals(ImportSpec.newBuilder() - .setType("file.csv") - .putSourceOptions("path", "data.csv") - .addEntities("driver") - .setSchema( - Schema.newBuilder() - .addFields(Field.newBuilder().setName("timestamp")) - .addFields(Field.newBuilder().setName("driver_id")) - .addFields( - Field.newBuilder() - .setName("trips_completed") - .setFeatureId("driver.none.trips_completed")) - .setEntityIdColumn("driver_id") - .setTimestampValue(DateUtil.toTimestamp("2018-09-25T00:00:00.000Z"))) - .build(), importJobSpecs.getImportSpec()); + System.out.println( + JsonFormat.printer().omittingInsignificantWhitespace().print(importJobSpecs)); + assertEquals( + ImportSpec.newBuilder() + .setType("file.csv") + .putSourceOptions("path", "data.csv") + .addEntities("driver") + .setSchema( + Schema.newBuilder() + .addFields(Field.newBuilder().setName("timestamp")) + .addFields(Field.newBuilder().setName("driver_id")) + .addFields( + Field.newBuilder() + .setName("trips_completed") + .setFeatureId("driver.trips_completed")) + .setEntityIdColumn("driver_id") + .setTimestampValue(DateUtil.toTimestamp("2018-09-25T00:00:00.000Z"))) + .build(), + importJobSpecs.getImportSpec()); - assertEquals(StorageSpec.newBuilder() - .setId("TEST_SERVING") - .setType("serving.mock") - .build(), importJobSpecs.getServingStorageSpecs(0)); + assertEquals( + StorageSpec.newBuilder().setId("TEST_SERVING").setType("serving.mock").build(), + importJobSpecs.getServingStorageSpecs(0)); - assertEquals(StorageSpec.newBuilder() - .setId("TEST_WAREHOUSE") - .setType("warehouse.mock") - .build(), importJobSpecs.getWarehouseStorageSpecs(0)); + assertEquals( + StorageSpec.newBuilder().setId("TEST_WAREHOUSE").setType("warehouse.mock").build(), + importJobSpecs.getWarehouseStorageSpecs(0)); - assertEquals(EntitySpec.newBuilder() - .setName("testEntity") - .setDescription("This is a test entity") - .build(), specs.getEntitySpec("testEntity")); + assertEquals( + EntitySpec.newBuilder() + .setName("testEntity") + .setDescription("This is a test entity") + .build(), + specs.getEntitySpec("testEntity")); - assertEquals(FeatureSpec.newBuilder() - .setId("testEntity.day.testInt64") - .setEntity("testEntity") - .setName("testInt64") - .setOwner("feast@example.com") - .setUri("https://example.com/") - .setValueType(Enum.INT64) - .setGranularity(Granularity.Enum.DAY) - .setDescription("This is test feature of type integer") - .build(), specs.getFeatureSpec("testEntity.day.testInt64")); + assertEquals( + FeatureSpec.newBuilder() + .setId("testEntity.testInt64") + .setEntity("testEntity") + .setName("testInt64") + .setOwner("feast@example.com") + .setUri("https://example.com/") + .setValueType(Enum.INT64) + .setDescription("This is test feature of type integer") + .build(), + specs.getFeatureSpec("testEntity.testInt64")); } } diff --git a/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java b/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java index b6f2d5ed984..7bea79405b1 100644 --- a/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java +++ b/ingestion/src/test/java/feast/ingestion/model/SpecsTest.java @@ -52,7 +52,7 @@ public void testSingleFeatureAndEntity() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); @@ -65,7 +65,7 @@ public void testSingleFeatureAndEntity() { assertTrue(specs.getEntitySpecs().containsKey("testEntity")); assertEquals(1, specs.getFeatureSpecs().size()); - assertTrue(specs.getFeatureSpecs().containsKey("testEntity.none.testInt32")); + assertTrue(specs.getFeatureSpecs().containsKey("testEntity.testInt32")); assertTrue(specs.getServingStorageSpecs().containsKey("TEST_SERVING")); } @@ -75,7 +75,7 @@ public void testErrorOnUnknownEntity() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); @@ -89,7 +89,7 @@ public void testErrorOnUnknownFeature() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); @@ -103,14 +103,14 @@ public void testGetFeatureSpec() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); specs.validate(); assertEquals( - "testEntity.none.testInt32", specs.getFeatureSpec("testEntity.none.testInt32").getId()); + "testEntity.testInt32", specs.getFeatureSpec("testEntity.testInt32").getId()); } @Test @@ -118,7 +118,7 @@ public void testGetEntitySpec() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); @@ -132,7 +132,7 @@ public void testGetStorageSpec() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("testEntity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); @@ -147,7 +147,7 @@ public void testFeatureSpecReferencesUnknownEntity() { ImportJobSpecs importJobSpecs = this.importJobSpecs.toBuilder() .setImportSpec(ImportSpec.newBuilder() .addEntities("totally_different_entity") - .setSchema(Schema.newBuilder().addFields(newField("testEntity.none.testInt32"))) + .setSchema(Schema.newBuilder().addFields(newField("testEntity.testInt32"))) ).build(); Specs specs = Specs.of("testjob", importJobSpecs); diff --git a/ingestion/src/test/java/feast/ingestion/transform/fn/FilterFeatureRowDoFnTest.java b/ingestion/src/test/java/feast/ingestion/transform/fn/FilterFeatureRowDoFnTest.java index 0068748969e..e8fcc5818a9 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/fn/FilterFeatureRowDoFnTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/fn/FilterFeatureRowDoFnTest.java @@ -34,9 +34,9 @@ public class FilterFeatureRowDoFnTest { @Test public void shouldIgnoreUnspecifiedFeatureID() { - String featureId1 = "testentity.none.feature1"; - String featureId2 = "testentity.hour.feature2"; - String featureId3 = "testentity.day.feature3"; + String featureId1 = "testentity.feature1"; + String featureId2 = "testentity.feature2"; + String featureId3 = "testentity.feature3"; List specifiedFeatureIds = Arrays.asList(featureId1, featureId2, featureId3); FilterFeatureRowDoFn doFn = new FilterFeatureRowDoFn(specifiedFeatureIds); @@ -52,7 +52,7 @@ public void shouldIgnoreUnspecifiedFeatureID() { .addFeatures( Feature.newBuilder().setId(featureId3).setValue(Value.newBuilder().setInt64Val(12))) // this feature should be ignored - .addFeatures(Feature.newBuilder().setId("testEntity.none.unknown_feature")) + .addFeatures(Feature.newBuilder().setId("testEntity.unknown_feature")) .build(); PCollection output = testPipeline.apply(Create.of(row)) diff --git a/ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java b/ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java index a81b29a8024..9a2d57a23c3 100644 --- a/ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java +++ b/ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java @@ -25,7 +25,6 @@ import com.google.protobuf.Timestamp; import junit.framework.TestCase; import org.joda.time.DateTime; -import feast.types.GranularityProto.Granularity.Enum; public class DateUtilTest extends TestCase { public void testStringToTimestamp() { @@ -63,63 +62,4 @@ public void testTimestampToDateTime() { assertThat(34, is(equalTo(datetime.getSecondOfMinute()))); assertThat(123, is(equalTo(datetime.getMillisOfSecond()))); } - - public void testRoundToSecondGranularity() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - Timestamp timestamp2 = DateUtil.roundToGranularity(timestamp1, Enum.SECOND); - DateTime actual = DateUtil.toDateTime(timestamp2); - assertThat(2018, is(equalTo(actual.getYear()))); - assertThat(7, is(equalTo(actual.getMonthOfYear()))); - assertThat(3, is(equalTo(actual.getDayOfMonth()))); - assertThat(15, is(equalTo(actual.getHourOfDay()))); - assertThat(9, is(equalTo(actual.getMinuteOfHour()))); - assertThat(34, is(equalTo(actual.getSecondOfMinute()))); - assertThat(0, is(equalTo(actual.getMillisOfSecond()))); - } - - public void testRoundToMinuteGranularity() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - Timestamp timestamp2 = DateUtil.roundToGranularity(timestamp1, Enum.MINUTE); - DateTime actual = DateUtil.toDateTime(timestamp2); - assertThat(2018, is(equalTo(actual.getYear()))); - assertThat(7, is(equalTo(actual.getMonthOfYear()))); - assertThat(3, is(equalTo(actual.getDayOfMonth()))); - assertThat(15, is(equalTo(actual.getHourOfDay()))); - assertThat(9, is(equalTo(actual.getMinuteOfHour()))); - assertThat(0, is(equalTo(actual.getSecondOfMinute()))); - assertThat(0, is(equalTo(actual.getMillisOfSecond()))); - } - - public void testRoundToHourGranularity() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - Timestamp timestamp2 = DateUtil.roundToGranularity(timestamp1, Enum.HOUR); - DateTime actual = DateUtil.toDateTime(timestamp2); - assertThat(2018, is(equalTo(actual.getYear()))); - assertThat(7, is(equalTo(actual.getMonthOfYear()))); - assertThat(3, is(equalTo(actual.getDayOfMonth()))); - assertThat(15, is(equalTo(actual.getHourOfDay()))); - assertThat(0, is(equalTo(actual.getMinuteOfHour()))); - assertThat(0, is(equalTo(actual.getSecondOfMinute()))); - assertThat(0, is(equalTo(actual.getMillisOfSecond()))); - } - - public void testRoundToDayGranularity() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - Timestamp timestamp2 = DateUtil.roundToGranularity(timestamp1, Enum.DAY); - DateTime actual = DateUtil.toDateTime(timestamp2); - assertThat(2018, is(equalTo(actual.getYear()))); - assertThat(7, is(equalTo(actual.getMonthOfYear()))); - assertThat(3, is(equalTo(actual.getDayOfMonth()))); - assertThat(0, is(equalTo(actual.getHourOfDay()))); - assertThat(0, is(equalTo(actual.getMinuteOfHour()))); - assertThat(0, is(equalTo(actual.getSecondOfMinute()))); - assertThat(0, is(equalTo(actual.getMillisOfSecond()))); - } - - public void testRoundToNoneGranularity() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - Timestamp timestamp2 = DateUtil.roundToGranularity(timestamp1, Enum.NONE); - DateTime actual = DateUtil.toDateTime(timestamp2); - assertThat(0L, is(equalTo(actual.getMillis()))); - } } diff --git a/ingestion/src/test/java/feast/source/bigquery/BQToFeatureRowFnTest.java b/ingestion/src/test/java/feast/source/bigquery/BQToFeatureRowFnTest.java index 4c809fb5cca..e33bcc0249e 100644 --- a/ingestion/src/test/java/feast/source/bigquery/BQToFeatureRowFnTest.java +++ b/ingestion/src/test/java/feast/source/bigquery/BQToFeatureRowFnTest.java @@ -26,7 +26,6 @@ import com.google.cloud.bigquery.LegacySQLTypeName; import com.google.common.collect.Lists; import com.google.protobuf.Timestamp; -import feast.source.bigquery.BigQueryToFeatureRowFn; import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; import org.joda.time.DateTime; @@ -62,7 +61,7 @@ public void testStringEntityKey() { .addFields( Field.newBuilder() .setName("bq_value") - .setFeatureId("testEntity.day.testInt64"))) + .setFeatureId("testEntity.testInt64"))) .build(); GenericRecord record = mock(GenericRecord.class); @@ -86,7 +85,7 @@ public void testStringEntityKey() { row.getFeaturesList(), equalTo( Lists.newArrayList( - Features.of("testEntity.day.testInt64", Values.ofInt64(Long.MAX_VALUE))))); + Features.of("testEntity.testInt64", Values.ofInt64(Long.MAX_VALUE))))); } @Test @@ -105,7 +104,7 @@ public void testInt64EntityKey() { .addFields( Field.newBuilder() .setName("bq_value") - .setFeatureId("testEntity.day.testInt64"))) + .setFeatureId("testEntity.testInt64"))) .build(); GenericRecord record = mock(GenericRecord.class); @@ -129,6 +128,6 @@ public void testInt64EntityKey() { row.getFeaturesList(), equalTo( Lists.newArrayList( - Features.of("testEntity.day.testInt64", Values.ofInt64(Long.MAX_VALUE))))); + Features.of("testEntity.testInt64", Values.ofInt64(Long.MAX_VALUE))))); } } diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java index 940778ffdbe..39d452ce46b 100644 --- a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals; import com.google.common.io.Resources; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import feast.ingestion.config.ImportJobSpecsSupplier; import feast.ingestion.model.Features; @@ -39,8 +38,6 @@ import feast.store.FeatureStoreWrite; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.GranularityProto.Granularity; -import feast.types.ValueProto.Value; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -58,10 +55,10 @@ public class FeatureRowRedisIOWriteTest { - private static final String featureNoneInt32 = "testEntity.none.redisInt32"; - private static final String featureNoneString = "testEntity.none.redisString"; - private static final String featureHourInt32 = "testEntity.hour.redisInt32"; - private static final String featureHourString = "testEntity.hour.redisString"; + private static final String featureNoneInt32 = "testEntity.redisInt32"; + private static final String featureNoneString = "testEntity.redisString"; + private static final String featureHourInt32 = "testEntity.redisInt32"; + private static final String featureHourString = "testEntity.redisString"; private static int REDIS_PORT = 51234; private static Redis redis; @@ -108,7 +105,7 @@ Specs getSpecs() { } @Test - public void testWriteNoneGranularity() throws IOException { + public void testWrite() throws IOException { Specs specs = getSpecs(); specs.validate(); @@ -117,14 +114,15 @@ public void testWriteNoneGranularity() throws IOException { new FeatureRowRedisIO.Write( RedisStoreOptions.builder().host("localhost").port(REDIS_PORT).build(), specs); + Timestamp now = DateUtil.toTimestamp(DateTime.now()); + FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() .setRow( FeatureRow.newBuilder() .setEntityName("testEntity") .setEntityKey("1") - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(DateUtil.toTimestamp(DateTime.now())) + .setEventTimestamp(now) .addFeatures(Features.of(featureNoneInt32, Values.ofInt32(1))) .addFeatures(Features.of(featureNoneString, Values.ofString("a")))) .build(); @@ -146,27 +144,25 @@ public void testWriteNoneGranularity() throws IOException { RedisBucketValue.parseFrom(jedis.get(featureStringKey.toByteArray())); assertEquals(Values.ofInt32(1), featureInt32Value.getValue()); - // Timestamp is 0 for NONE granularity - assertEquals(Timestamp.getDefaultInstance(), featureInt32Value.getEventTimestamp()); + assertEquals(now, featureInt32Value.getEventTimestamp()); assertEquals(Values.ofString("a"), featureStringValue.getValue()); - // Timestamp is 0 for NONE granularity - assertEquals(Timestamp.getDefaultInstance(), featureStringValue.getEventTimestamp()); + assertEquals(now, featureStringValue.getEventTimestamp()); } @Test - public void testWriteNoneGranularityFromOptions() throws IOException { + public void testWriteFromOptions() throws IOException { Specs specs = getSpecs(); FeatureStoreWrite write = new RedisServingFactory() .create(specs.getServingStorageSpecs().get("REDIS1"), specs); + Timestamp now = DateUtil.toTimestamp(DateTime.now()); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() .setRow( FeatureRow.newBuilder() .setEntityName("testEntity") .setEntityKey("1") - .setGranularity(Granularity.Enum.NONE) - .setEventTimestamp(DateUtil.toTimestamp(DateTime.now())) + .setEventTimestamp(now) .addFeatures(Features.of(featureNoneInt32, Values.ofInt32(1))) .addFeatures(Features.of(featureNoneString, Values.ofString("a")))) .build(); @@ -188,53 +184,8 @@ public void testWriteNoneGranularityFromOptions() throws IOException { RedisBucketValue.parseFrom(jedis.get(featureStringKey.toByteArray())); assertEquals(Values.ofInt32(1), featureInt32Value.getValue()); - // Timestamp is 0 for NONE granularity - assertEquals(Timestamp.getDefaultInstance(), featureInt32Value.getEventTimestamp()); + assertEquals(now, featureInt32Value.getEventTimestamp()); assertEquals(Values.ofString("a"), featureStringValue.getValue()); - // Timestamp is 0 for NONE granularity - assertEquals(Timestamp.getDefaultInstance(), featureStringValue.getEventTimestamp()); - } - - @Test - public void testWriteHourGranularity() throws IOException { - Specs specs = getSpecs(); - FeatureStoreWrite write = new RedisServingFactory() - .create(specs.getServingStorageSpecs().get("REDIS1"), specs); - - FeatureRowExtended rowExtended = - FeatureRowExtended.newBuilder() - .setRow( - FeatureRow.newBuilder() - .setEntityName("testEntity") - .setEntityKey("1") - .setGranularity(Granularity.Enum.HOUR) - .setEventTimestamp(DateUtil.toTimestamp(DateTime.now())) - .addFeatures(Features.of(featureHourInt32, Values.ofInt32(1))) - .addFeatures(Features.of(featureHourString, Values.ofString("a")))) - .build(); - - PCollection input = testPipeline.apply(Create.of(rowExtended)); - - input.apply("write to embedded redis", write); - - testPipeline.run(); - - Timestamp rowTimestamp = rowExtended.getRow().getEventTimestamp(); - Timestamp roundedTimestamp = DateUtil.roundToGranularity(rowTimestamp, Granularity.Enum.HOUR); - - RedisBucketKey featureInt32LatestKey = - getRedisBucketKey("1", getFeatureIdSha1Prefix(featureHourInt32), 0L); - RedisBucketKey featureStringLatestKey = - getRedisBucketKey("1", getFeatureIdSha1Prefix(featureHourString), 0L); - - checkRedisValue(featureInt32LatestKey, Values.ofInt32(1), roundedTimestamp); - checkRedisValue(featureStringLatestKey, Values.ofString("a"), roundedTimestamp); - } - - void checkRedisValue(RedisBucketKey key, Value expectedValue, Timestamp expectedTimestamp) - throws InvalidProtocolBufferException { - RedisBucketValue featureInt32Value = RedisBucketValue.parseFrom(jedis.get(key.toByteArray())); - assertEquals(expectedValue, featureInt32Value.getValue()); - assertEquals(expectedTimestamp, featureInt32Value.getEventTimestamp()); + assertEquals(now, featureStringValue.getEventTimestamp()); } } diff --git a/ingestion/src/test/resources/import-specs/csv_to_store1.yaml b/ingestion/src/test/resources/import-specs/csv_to_store1.yaml index c17e5f80d65..57c8d4be5f9 100644 --- a/ingestion/src/test/resources/import-specs/csv_to_store1.yaml +++ b/ingestion/src/test/resources/import-specs/csv_to_store1.yaml @@ -11,6 +11,6 @@ schema: fields: - name: timestamp - name: id - - featureId: testEntity.none.testInt32 - - featureId: testEntity.none.testString + - featureId: testEntity.testInt32 + - featureId: testEntity.testString diff --git a/ingestion/src/test/resources/specs/importJobSpecs.yaml b/ingestion/src/test/resources/specs/importJobSpecs.yaml index cbc48b95582..a6b01b2d7f8 100644 --- a/ingestion/src/test/resources/specs/importJobSpecs.yaml +++ b/ingestion/src/test/resources/specs/importJobSpecs.yaml @@ -16,9 +16,8 @@ entitySpecs: description: This is a test entity tags: [] featureSpecs: - - id: testEntity.day.testInt64 + - id: testEntity.testInt64 entity: testEntity - granularity: DAY name: testInt64 owner: feast@example.com description: This is test feature of type integer @@ -31,38 +30,8 @@ featureSpecs: id: TEST_SERVING warehouse: id: TEST_WAREHOUSE - - id: testEntity.hour.redisInt32 + - id: testEntity.redisInt32 entity: testEntity - granularity: HOUR - name: redisInt32 - owner: feast@example.com - description: This is test feature of type integer that goes to redis - uri: https://example.com/ - valueType: INT32 - tags: [] - options: {} - dataStores: - serving: - id: REDIS1 - options: - expiry: PT1H - bucketSize: PT10H - - id: testEntity.hour.redisString - entity: testEntity - granularity: HOUR - name: redisString - owner: feast@example.com - description: This is test feature of type integer that goes to redis - uri: https://example.com/ - valueType: STRING - tags: [] - options: {} - dataStores: - serving: - id: REDIS1 - - id: testEntity.none.redisInt32 - entity: testEntity - granularity: NONE name: redisInt32 owner: feast@example.com description: This is test feature of type integer the goes to redis @@ -75,9 +44,8 @@ featureSpecs: id: REDIS1 options: expiry: PT1H - - id: testEntity.none.redisString + - id: testEntity.redisString entity: testEntity - granularity: NONE name: redisString owner: feast@example.com description: This is test feature of type integer that goes to redis @@ -88,9 +56,8 @@ featureSpecs: dataStores: serving: id: REDIS1 - - id: testEntity.none.testInt32 + - id: testEntity.testInt32 entity: testEntity - granularity: NONE name: testInt32 owner: feast@example.com description: This is test feature of type integer @@ -103,9 +70,8 @@ featureSpecs: id: TEST_SERVING warehouse: id: TEST_WAREHOUSE - - id: testEntity.none.testInt64 + - id: testEntity.testInt64 entity: testEntity - granularity: NONE name: testInt64 owner: feast@example.com description: This is test feature of type integer @@ -118,9 +84,8 @@ featureSpecs: id: TEST_SERVING warehouse: id: TEST_WAREHOUSE - - id: testEntity.none.testInt64NoWarehouse + - id: testEntity.testInt64NoWarehouse entity: testEntity - granularity: NONE name: testInt64NoWarehouse owner: feast@example.com description: This is test feature of type integer @@ -131,9 +96,8 @@ featureSpecs: dataStores: serving: id: TEST_SERVING - - id: testEntity.none.testString + - id: testEntity.testString entity: testEntity - granularity: NONE name: testString owner: feast@example.com description: This is test feature of type integer @@ -144,9 +108,8 @@ featureSpecs: id: TEST_SERVING warehouse: id: TEST_WAREHOUSE - - id: testEntity.none.testStringNoWarehouse + - id: testEntity.testStringNoWarehouse entity: testEntity - granularity: NONE name: testStringNoWarehouse owner: feast@example.com description: This is test feature of type integer @@ -155,9 +118,8 @@ featureSpecs: dataStores: serving: id: TEST_SERVING - - id: testEntity.none.unknownInt32 + - id: testEntity.unknownInt32 entity: testEntity - granularity: NONE name: unknownInt32 owner: feast@example.com description: This is test feature of type integer that goes to an unknown serving diff --git a/protos/feast/specs/FeatureSpec.proto b/protos/feast/specs/FeatureSpec.proto index 20246670818..97681d463d2 100644 --- a/protos/feast/specs/FeatureSpec.proto +++ b/protos/feast/specs/FeatureSpec.proto @@ -18,7 +18,6 @@ syntax = "proto3"; import "feast/specs/EntitySpec.proto"; import "feast/specs/StorageSpec.proto"; -import "feast/types/Granularity.proto"; import "feast/types/Value.proto"; package feast.specs; @@ -33,7 +32,6 @@ message FeatureSpec { string owner = 3; string description = 4; string uri = 5; - feast.types.Granularity.Enum granularity = 6; feast.types.ValueType.Enum valueType = 7; string entity = 8; string group = 9; diff --git a/protos/feast/storage/BigTable.proto b/protos/feast/storage/BigTable.proto index 1a6f994d8db..61cbed798b3 100644 --- a/protos/feast/storage/BigTable.proto +++ b/protos/feast/storage/BigTable.proto @@ -26,6 +26,5 @@ message BigTableRowKey { // This should be the first 7 characters of a sha1 of the entityKey proto encoded string sha1Prefix = 1; string entityKey = 2; - // This should be String.valueOf(int64.MAX_VALUE - roundToEntityGranularity(eventTimestamp)) string reversedMillis = 3; } diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto index 55e9a5acca0..b17ad7bbc9b 100644 --- a/protos/feast/types/FeatureRow.proto +++ b/protos/feast/types/FeatureRow.proto @@ -18,7 +18,6 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; import "feast/types/Feature.proto"; -import "feast/types/Granularity.proto"; package feast.types; @@ -30,7 +29,6 @@ message FeatureRowKey { string entityKey = 1; google.protobuf.Timestamp eventTimestamp = 3; string entityName = 4; - Granularity.Enum granularity = 5; } message FeatureRow { @@ -38,5 +36,4 @@ message FeatureRow { repeated Feature features = 2; google.protobuf.Timestamp eventTimestamp = 3; string entityName = 4; - Granularity.Enum granularity = 5; } diff --git a/protos/feast/types/Granularity.proto b/protos/feast/types/Granularity.proto deleted file mode 100644 index c45dfc23fbd..00000000000 --- a/protos/feast/types/Granularity.proto +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -package feast.types; - -option java_package = "feast.types"; -option java_outer_classname = "GranularityProto"; -option go_package = "github.com/gojek/feast/protos/generated/go/feast/types"; - -message Granularity { - enum Enum { - NONE = 0; - DAY = 1; - HOUR = 2; - MINUTE = 3; - SECOND = 4; - } -} diff --git a/protos/generated/go/feast/core/CoreService.pb.go b/protos/generated/go/feast/core/CoreService.pb.go index 76f38974337..4037da58af9 100644 --- a/protos/generated/go/feast/core/CoreService.pb.go +++ b/protos/generated/go/feast/core/CoreService.pb.go @@ -35,7 +35,7 @@ func (m *CoreServiceTypes) Reset() { *m = CoreServiceTypes{} } func (m *CoreServiceTypes) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes) ProtoMessage() {} func (*CoreServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0} } func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes.Unmarshal(m, b) @@ -66,7 +66,7 @@ func (m *CoreServiceTypes_GetEntitiesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetEntitiesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 0} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 0} } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Unmarshal(m, b) @@ -104,7 +104,7 @@ func (m *CoreServiceTypes_GetEntitiesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 1} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 1} } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Unmarshal(m, b) @@ -142,7 +142,7 @@ func (m *CoreServiceTypes_ListEntitiesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 2} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 2} } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -181,7 +181,7 @@ func (m *CoreServiceTypes_GetFeaturesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 3} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 3} } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Unmarshal(m, b) @@ -219,7 +219,7 @@ func (m *CoreServiceTypes_GetFeaturesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 4} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 4} } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Unmarshal(m, b) @@ -257,7 +257,7 @@ func (m *CoreServiceTypes_ListFeaturesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 5} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 5} } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -296,7 +296,7 @@ func (m *CoreServiceTypes_GetStorageRequest) Reset() { *m = CoreServiceT func (m *CoreServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageRequest) ProtoMessage() {} func (*CoreServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 6} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 6} } func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -334,7 +334,7 @@ func (m *CoreServiceTypes_GetStorageResponse) Reset() { *m = CoreService func (m *CoreServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 7} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 7} } func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -372,7 +372,7 @@ func (m *CoreServiceTypes_ListStorageResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 8} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 8} } func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -411,7 +411,7 @@ func (m *CoreServiceTypes_ApplyEntityResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ApplyEntityResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyEntityResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 9} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 9} } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Unmarshal(m, b) @@ -450,7 +450,7 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyFeatureResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyFeatureResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 10} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 10} } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Unmarshal(m, b) @@ -493,7 +493,7 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) String() string { } func (*CoreServiceTypes_ApplyFeatureGroupResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 11} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 11} } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Unmarshal(m, b) @@ -532,7 +532,7 @@ func (m *CoreServiceTypes_ApplyStorageResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_f716411f9ac886b5, []int{0, 12} + return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 12} } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Unmarshal(m, b) @@ -1006,10 +1006,10 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_f716411f9ac886b5) + proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_a0e9a1504f969203) } -var fileDescriptor_CoreService_f716411f9ac886b5 = []byte{ +var fileDescriptor_CoreService_a0e9a1504f969203 = []byte{ // 602 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5b, 0x6f, 0x12, 0x41, 0x14, 0x86, 0x34, 0x69, 0xe0, 0x40, 0x4c, 0x3b, 0x34, 0x8a, 0x23, 0x35, 0xcd, 0x26, 0x36, 0x7d, diff --git a/protos/generated/go/feast/core/DatasetService.pb.go b/protos/generated/go/feast/core/DatasetService.pb.go index 61b0a09975f..d2d3b01c932 100644 --- a/protos/generated/go/feast/core/DatasetService.pb.go +++ b/protos/generated/go/feast/core/DatasetService.pb.go @@ -34,7 +34,7 @@ func (m *DatasetServiceTypes) Reset() { *m = DatasetServiceTypes{} } func (m *DatasetServiceTypes) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes) ProtoMessage() {} func (*DatasetServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{0} + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0} } func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes.Unmarshal(m, b) @@ -77,7 +77,7 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) Reset() { func (m *DatasetServiceTypes_CreateDatasetRequest) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetRequest) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{0, 0} + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 0} } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Unmarshal(m, b) @@ -146,7 +146,7 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) Reset() { func (m *DatasetServiceTypes_CreateDatasetResponse) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetResponse) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{0, 1} + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 1} } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Unmarshal(m, b) @@ -188,7 +188,7 @@ func (m *FeatureSet) Reset() { *m = FeatureSet{} } func (m *FeatureSet) String() string { return proto.CompactTextString(m) } func (*FeatureSet) ProtoMessage() {} func (*FeatureSet) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{1} + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{1} } func (m *FeatureSet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSet.Unmarshal(m, b) @@ -237,7 +237,7 @@ func (m *DatasetInfo) Reset() { *m = DatasetInfo{} } func (m *DatasetInfo) String() string { return proto.CompactTextString(m) } func (*DatasetInfo) ProtoMessage() {} func (*DatasetInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_9ba96186e7cec93b, []int{2} + return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{2} } func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetInfo.Unmarshal(m, b) @@ -354,10 +354,10 @@ var _DatasetService_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_9ba96186e7cec93b) + proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_37ae639a8c7b5dd5) } -var fileDescriptor_DatasetService_9ba96186e7cec93b = []byte{ +var fileDescriptor_DatasetService_37ae639a8c7b5dd5 = []byte{ // 414 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, diff --git a/protos/generated/go/feast/core/JobService.pb.go b/protos/generated/go/feast/core/JobService.pb.go index e830edfe5fa..6a3a4e6dc0d 100644 --- a/protos/generated/go/feast/core/JobService.pb.go +++ b/protos/generated/go/feast/core/JobService.pb.go @@ -36,7 +36,7 @@ func (m *JobServiceTypes) Reset() { *m = JobServiceTypes{} } func (m *JobServiceTypes) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes) ProtoMessage() {} func (*JobServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0} } func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes.Unmarshal(m, b) @@ -70,7 +70,7 @@ func (m *JobServiceTypes_SubmitImportJobRequest) Reset() { func (m *JobServiceTypes_SubmitImportJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobRequest) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 0} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 0} } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Unmarshal(m, b) @@ -117,7 +117,7 @@ func (m *JobServiceTypes_SubmitImportJobResponse) Reset() { func (m *JobServiceTypes_SubmitImportJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobResponse) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 1} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 1} } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Unmarshal(m, b) @@ -155,7 +155,7 @@ func (m *JobServiceTypes_ListJobsResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_ListJobsResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_ListJobsResponse) ProtoMessage() {} func (*JobServiceTypes_ListJobsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 2} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 2} } func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Unmarshal(m, b) @@ -193,7 +193,7 @@ func (m *JobServiceTypes_GetJobRequest) Reset() { *m = JobServiceTypes_G func (m *JobServiceTypes_GetJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobRequest) ProtoMessage() {} func (*JobServiceTypes_GetJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 3} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 3} } func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Unmarshal(m, b) @@ -231,7 +231,7 @@ func (m *JobServiceTypes_GetJobResponse) Reset() { *m = JobServiceTypes_ func (m *JobServiceTypes_GetJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobResponse) ProtoMessage() {} func (*JobServiceTypes_GetJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 4} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 4} } func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Unmarshal(m, b) @@ -269,7 +269,7 @@ func (m *JobServiceTypes_AbortJobRequest) Reset() { *m = JobServiceTypes func (m *JobServiceTypes_AbortJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobRequest) ProtoMessage() {} func (*JobServiceTypes_AbortJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 5} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 5} } func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Unmarshal(m, b) @@ -307,7 +307,7 @@ func (m *JobServiceTypes_AbortJobResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_AbortJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobResponse) ProtoMessage() {} func (*JobServiceTypes_AbortJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 6} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 6} } func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Unmarshal(m, b) @@ -356,7 +356,7 @@ func (m *JobServiceTypes_JobDetail) Reset() { *m = JobServiceTypes_JobDe func (m *JobServiceTypes_JobDetail) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_JobDetail) ProtoMessage() {} func (*JobServiceTypes_JobDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_c970f49698845549, []int{0, 7} + return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 7} } func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_JobDetail.Unmarshal(m, b) @@ -639,10 +639,10 @@ var _JobService_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_JobService_c970f49698845549) + proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_JobService_edcd183b773c9f62) } -var fileDescriptor_JobService_c970f49698845549 = []byte{ +var fileDescriptor_JobService_edcd183b773c9f62 = []byte{ // 621 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0xdb, 0x4c, 0x10, 0x55, 0x62, 0x08, 0xf1, 0xf0, 0x7d, 0x80, 0x56, 0x15, 0x58, 0x4b, 0x25, 0x52, 0xa4, 0x4a, diff --git a/protos/generated/go/feast/core/UIService.pb.go b/protos/generated/go/feast/core/UIService.pb.go index d3cacf7afd1..772c548c875 100644 --- a/protos/generated/go/feast/core/UIService.pb.go +++ b/protos/generated/go/feast/core/UIService.pb.go @@ -36,7 +36,7 @@ func (m *UIServiceTypes) Reset() { *m = UIServiceTypes{} } func (m *UIServiceTypes) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes) ProtoMessage() {} func (*UIServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0} + return fileDescriptor_UIService_04866529701c634c, []int{0} } func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes.Unmarshal(m, b) @@ -70,7 +70,7 @@ func (m *UIServiceTypes_EntityDetail) Reset() { *m = UIServiceTypes_Enti func (m *UIServiceTypes_EntityDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_EntityDetail) ProtoMessage() {} func (*UIServiceTypes_EntityDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 0} + return fileDescriptor_UIService_04866529701c634c, []int{0, 0} } func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_EntityDetail.Unmarshal(m, b) @@ -122,7 +122,7 @@ func (m *UIServiceTypes_GetEntityRequest) Reset() { *m = UIServiceTypes_ func (m *UIServiceTypes_GetEntityRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityRequest) ProtoMessage() {} func (*UIServiceTypes_GetEntityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 1} + return fileDescriptor_UIService_04866529701c634c, []int{0, 1} } func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Unmarshal(m, b) @@ -160,7 +160,7 @@ func (m *UIServiceTypes_GetEntityResponse) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetEntityResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityResponse) ProtoMessage() {} func (*UIServiceTypes_GetEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 2} + return fileDescriptor_UIService_04866529701c634c, []int{0, 2} } func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Unmarshal(m, b) @@ -198,7 +198,7 @@ func (m *UIServiceTypes_ListEntitiesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*UIServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 3} + return fileDescriptor_UIService_04866529701c634c, []int{0, 3} } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -242,7 +242,7 @@ func (m *UIServiceTypes_FeatureDetail) Reset() { *m = UIServiceTypes_Fea func (m *UIServiceTypes_FeatureDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 4} + return fileDescriptor_UIService_04866529701c634c, []int{0, 4} } func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Unmarshal(m, b) @@ -315,7 +315,7 @@ func (m *UIServiceTypes_GetFeatureRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetFeatureRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 5} + return fileDescriptor_UIService_04866529701c634c, []int{0, 5} } func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Unmarshal(m, b) @@ -354,7 +354,7 @@ func (m *UIServiceTypes_GetFeatureResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetFeatureResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 6} + return fileDescriptor_UIService_04866529701c634c, []int{0, 6} } func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Unmarshal(m, b) @@ -399,7 +399,7 @@ func (m *UIServiceTypes_ListFeaturesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 7} + return fileDescriptor_UIService_04866529701c634c, []int{0, 7} } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -439,7 +439,7 @@ func (m *UIServiceTypes_FeatureGroupDetail) Reset() { *m = UIServiceType func (m *UIServiceTypes_FeatureGroupDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureGroupDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureGroupDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 8} + return fileDescriptor_UIService_04866529701c634c, []int{0, 8} } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Unmarshal(m, b) @@ -484,7 +484,7 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) Reset() { *m = UIService func (m *UIServiceTypes_GetFeatureGroupRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 9} + return fileDescriptor_UIService_04866529701c634c, []int{0, 9} } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Unmarshal(m, b) @@ -524,7 +524,7 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) Reset() { func (m *UIServiceTypes_GetFeatureGroupResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 10} + return fileDescriptor_UIService_04866529701c634c, []int{0, 10} } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Unmarshal(m, b) @@ -564,7 +564,7 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) Reset() { func (m *UIServiceTypes_ListFeatureGroupsResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeatureGroupsResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeatureGroupsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 11} + return fileDescriptor_UIService_04866529701c634c, []int{0, 11} } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Unmarshal(m, b) @@ -604,7 +604,7 @@ func (m *UIServiceTypes_StorageDetail) Reset() { *m = UIServiceTypes_Sto func (m *UIServiceTypes_StorageDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_StorageDetail) ProtoMessage() {} func (*UIServiceTypes_StorageDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 12} + return fileDescriptor_UIService_04866529701c634c, []int{0, 12} } func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_StorageDetail.Unmarshal(m, b) @@ -649,7 +649,7 @@ func (m *UIServiceTypes_GetStorageRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageRequest) ProtoMessage() {} func (*UIServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 13} + return fileDescriptor_UIService_04866529701c634c, []int{0, 13} } func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -687,7 +687,7 @@ func (m *UIServiceTypes_GetStorageResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageResponse) ProtoMessage() {} func (*UIServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 14} + return fileDescriptor_UIService_04866529701c634c, []int{0, 14} } func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -725,7 +725,7 @@ func (m *UIServiceTypes_ListStorageResponse) Reset() { *m = UIServiceTyp func (m *UIServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListStorageResponse) ProtoMessage() {} func (*UIServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_07f94aecf154b17a, []int{0, 15} + return fileDescriptor_UIService_04866529701c634c, []int{0, 15} } func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -1120,10 +1120,10 @@ var _UIService_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_UIService_07f94aecf154b17a) + proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_UIService_04866529701c634c) } -var fileDescriptor_UIService_07f94aecf154b17a = []byte{ +var fileDescriptor_UIService_04866529701c634c = []byte{ // 784 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xd3, 0x40, 0x10, 0xcd, 0x47, 0x49, 0x9a, 0x69, 0x5a, 0xe8, 0x82, 0xda, 0xb0, 0x50, 0x51, 0x99, 0x03, 0x91, diff --git a/protos/generated/go/feast/serving/Serving.pb.go b/protos/generated/go/feast/serving/Serving.pb.go index 7b00a6cd413..6f61517bee9 100644 --- a/protos/generated/go/feast/serving/Serving.pb.go +++ b/protos/generated/go/feast/serving/Serving.pb.go @@ -44,7 +44,7 @@ func (m *QueryFeaturesRequest) Reset() { *m = QueryFeaturesRequest{} } func (m *QueryFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*QueryFeaturesRequest) ProtoMessage() {} func (*QueryFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{0} + return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{0} } func (m *QueryFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_QueryFeaturesRequest.Unmarshal(m, b) @@ -99,7 +99,7 @@ func (m *QueryFeaturesResponse) Reset() { *m = QueryFeaturesResponse{} } func (m *QueryFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*QueryFeaturesResponse) ProtoMessage() {} func (*QueryFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{1} + return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{1} } func (m *QueryFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_QueryFeaturesResponse.Unmarshal(m, b) @@ -145,7 +145,7 @@ func (m *Entity) Reset() { *m = Entity{} } func (m *Entity) String() string { return proto.CompactTextString(m) } func (*Entity) ProtoMessage() {} func (*Entity) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{2} + return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{2} } func (m *Entity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Entity.Unmarshal(m, b) @@ -186,7 +186,7 @@ func (m *FeatureValue) Reset() { *m = FeatureValue{} } func (m *FeatureValue) String() string { return proto.CompactTextString(m) } func (*FeatureValue) ProtoMessage() {} func (*FeatureValue) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_fa8a820cedf8e8f2, []int{3} + return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{3} } func (m *FeatureValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureValue.Unmarshal(m, b) @@ -304,10 +304,10 @@ var _ServingAPI_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_Serving_fa8a820cedf8e8f2) + proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_Serving_f91320f9a3f0c4cf) } -var fileDescriptor_Serving_fa8a820cedf8e8f2 = []byte{ +var fileDescriptor_Serving_f91320f9a3f0c4cf = []byte{ // 429 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xd1, 0x8a, 0xd3, 0x40, 0x14, 0x75, 0x5a, 0x5c, 0x36, 0x77, 0x0d, 0xca, 0xe0, 0x62, 0xc8, 0x8a, 0x96, 0xac, 0x0f, 0x01, diff --git a/protos/generated/go/feast/specs/EntitySpec.pb.go b/protos/generated/go/feast/specs/EntitySpec.pb.go index 8543c6c2684..0f4374c679e 100644 --- a/protos/generated/go/feast/specs/EntitySpec.pb.go +++ b/protos/generated/go/feast/specs/EntitySpec.pb.go @@ -31,7 +31,7 @@ func (m *EntitySpec) Reset() { *m = EntitySpec{} } func (m *EntitySpec) String() string { return proto.CompactTextString(m) } func (*EntitySpec) ProtoMessage() {} func (*EntitySpec) Descriptor() ([]byte, []int) { - return fileDescriptor_EntitySpec_df39fb3786bb4912, []int{0} + return fileDescriptor_EntitySpec_b8950ded39b854cb, []int{0} } func (m *EntitySpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_EntitySpec.Unmarshal(m, b) @@ -77,10 +77,10 @@ func init() { } func init() { - proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_EntitySpec_df39fb3786bb4912) + proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_EntitySpec_b8950ded39b854cb) } -var fileDescriptor_EntitySpec_df39fb3786bb4912 = []byte{ +var fileDescriptor_EntitySpec_b8950ded39b854cb = []byte{ // 177 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0xcd, 0x2b, 0xc9, 0x2c, 0xa9, 0x0c, 0x2e, diff --git a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go index a8f4c6c1336..8349e5a53aa 100644 --- a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go @@ -31,7 +31,7 @@ func (m *FeatureGroupSpec) Reset() { *m = FeatureGroupSpec{} } func (m *FeatureGroupSpec) String() string { return proto.CompactTextString(m) } func (*FeatureGroupSpec) ProtoMessage() {} func (*FeatureGroupSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureGroupSpec_1fd10deeef20baa7, []int{0} + return fileDescriptor_FeatureGroupSpec_21c67ee01edd412c, []int{0} } func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureGroupSpec.Unmarshal(m, b) @@ -77,10 +77,10 @@ func init() { } func init() { - proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_FeatureGroupSpec_1fd10deeef20baa7) + proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_FeatureGroupSpec_21c67ee01edd412c) } -var fileDescriptor_FeatureGroupSpec_1fd10deeef20baa7 = []byte{ +var fileDescriptor_FeatureGroupSpec_21c67ee01edd412c = []byte{ // 203 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4a, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0x75, diff --git a/protos/generated/go/feast/specs/FeatureSpec.pb.go b/protos/generated/go/feast/specs/FeatureSpec.pb.go index 293a5303b0d..34ca43cc6bc 100644 --- a/protos/generated/go/feast/specs/FeatureSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureSpec.pb.go @@ -20,28 +20,27 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type FeatureSpec struct { - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Owner string `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` - Description string `protobuf:"bytes,4,opt,name=description,proto3" json:"description,omitempty"` - Uri string `protobuf:"bytes,5,opt,name=uri,proto3" json:"uri,omitempty"` - Granularity types.Granularity_Enum `protobuf:"varint,6,opt,name=granularity,proto3,enum=feast.types.Granularity_Enum" json:"granularity,omitempty"` - ValueType types.ValueType_Enum `protobuf:"varint,7,opt,name=valueType,proto3,enum=feast.types.ValueType_Enum" json:"valueType,omitempty"` - Entity string `protobuf:"bytes,8,opt,name=entity,proto3" json:"entity,omitempty"` - Group string `protobuf:"bytes,9,opt,name=group,proto3" json:"group,omitempty"` - Tags []string `protobuf:"bytes,10,rep,name=tags,proto3" json:"tags,omitempty"` - Options map[string]string `protobuf:"bytes,11,rep,name=options,proto3" json:"options,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - DataStores *DataStores `protobuf:"bytes,12,opt,name=dataStores,proto3" json:"dataStores,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Owner string `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` + Description string `protobuf:"bytes,4,opt,name=description,proto3" json:"description,omitempty"` + Uri string `protobuf:"bytes,5,opt,name=uri,proto3" json:"uri,omitempty"` + ValueType types.ValueType_Enum `protobuf:"varint,7,opt,name=valueType,proto3,enum=feast.types.ValueType_Enum" json:"valueType,omitempty"` + Entity string `protobuf:"bytes,8,opt,name=entity,proto3" json:"entity,omitempty"` + Group string `protobuf:"bytes,9,opt,name=group,proto3" json:"group,omitempty"` + Tags []string `protobuf:"bytes,10,rep,name=tags,proto3" json:"tags,omitempty"` + Options map[string]string `protobuf:"bytes,11,rep,name=options,proto3" json:"options,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + DataStores *DataStores `protobuf:"bytes,12,opt,name=dataStores,proto3" json:"dataStores,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *FeatureSpec) Reset() { *m = FeatureSpec{} } func (m *FeatureSpec) String() string { return proto.CompactTextString(m) } func (*FeatureSpec) ProtoMessage() {} func (*FeatureSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_97d610673462505c, []int{0} + return fileDescriptor_FeatureSpec_de24bbc3c8f2bed6, []int{0} } func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSpec.Unmarshal(m, b) @@ -96,13 +95,6 @@ func (m *FeatureSpec) GetUri() string { return "" } -func (m *FeatureSpec) GetGranularity() types.Granularity_Enum { - if m != nil { - return m.Granularity - } - return types.Granularity_NONE -} - func (m *FeatureSpec) GetValueType() types.ValueType_Enum { if m != nil { return m.ValueType @@ -157,7 +149,7 @@ func (m *DataStores) Reset() { *m = DataStores{} } func (m *DataStores) String() string { return proto.CompactTextString(m) } func (*DataStores) ProtoMessage() {} func (*DataStores) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_97d610673462505c, []int{1} + return fileDescriptor_FeatureSpec_de24bbc3c8f2bed6, []int{1} } func (m *DataStores) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStores.Unmarshal(m, b) @@ -203,7 +195,7 @@ func (m *DataStore) Reset() { *m = DataStore{} } func (m *DataStore) String() string { return proto.CompactTextString(m) } func (*DataStore) ProtoMessage() {} func (*DataStore) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_97d610673462505c, []int{2} + return fileDescriptor_FeatureSpec_de24bbc3c8f2bed6, []int{2} } func (m *DataStore) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStore.Unmarshal(m, b) @@ -246,39 +238,38 @@ func init() { } func init() { - proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_FeatureSpec_97d610673462505c) -} - -var fileDescriptor_FeatureSpec_97d610673462505c = []byte{ - // 479 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0x4d, 0x6b, 0xdb, 0x40, - 0x10, 0x45, 0x56, 0x62, 0x57, 0xa3, 0x10, 0xc2, 0x52, 0x92, 0xc5, 0x6d, 0x40, 0xb8, 0x14, 0x7c, - 0x92, 0x8a, 0x5b, 0xfa, 0x11, 0x28, 0x81, 0x50, 0xb7, 0xc7, 0x16, 0xa5, 0xa4, 0xd0, 0xdb, 0xc6, - 0x9a, 0x2a, 0x6a, 0x62, 0xad, 0xd8, 0x5d, 0x25, 0xe8, 0x8f, 0xf4, 0x6f, 0xf6, 0x2f, 0x94, 0x9d, - 0xb5, 0xa3, 0x35, 0x71, 0x4f, 0xb9, 0xcd, 0xec, 0x7b, 0x33, 0x9a, 0xf7, 0x46, 0x03, 0xc7, 0xbf, - 0x50, 0x68, 0x93, 0xe9, 0x06, 0x17, 0x3a, 0xfb, 0x8c, 0xc2, 0xb4, 0x0a, 0xcf, 0x1b, 0x5c, 0xa4, - 0x8d, 0x92, 0x46, 0xb2, 0x98, 0xe0, 0x94, 0xe0, 0xf1, 0x73, 0x9f, 0x3b, 0xaf, 0x4d, 0x65, 0xba, - 0x9e, 0x3a, 0xde, 0xe8, 0x74, 0x6e, 0xa4, 0x12, 0x25, 0x3e, 0x84, 0x4d, 0xd7, 0xa0, 0xce, 0xbe, - 0x28, 0x51, 0xb7, 0x37, 0x42, 0x55, 0xa6, 0x5b, 0xc1, 0x47, 0x3e, 0x7c, 0x21, 0x6e, 0x5a, 0x74, - 0xc0, 0xe4, 0x6f, 0x08, 0xb1, 0x37, 0x17, 0xdb, 0x87, 0x41, 0x55, 0xf0, 0x20, 0x09, 0xa6, 0x51, - 0x3e, 0xa8, 0x0a, 0xc6, 0x60, 0xa7, 0x16, 0x4b, 0xe4, 0x03, 0x7a, 0xa1, 0x98, 0x3d, 0x85, 0x5d, - 0x79, 0x57, 0xa3, 0xe2, 0x21, 0x3d, 0xba, 0x84, 0x25, 0x10, 0x17, 0xa8, 0x17, 0xaa, 0x6a, 0x4c, - 0x25, 0x6b, 0xbe, 0x43, 0x98, 0xff, 0xc4, 0x0e, 0x20, 0x6c, 0x55, 0xc5, 0x77, 0x09, 0xb1, 0x21, - 0x3b, 0x85, 0xb8, 0xec, 0x67, 0xe5, 0xc3, 0x24, 0x98, 0xee, 0xcf, 0x8e, 0x53, 0xe7, 0x0a, 0x0d, - 0x9b, 0xfa, 0x5a, 0xe6, 0x75, 0xbb, 0xcc, 0xfd, 0x0a, 0xf6, 0x01, 0xa2, 0x5b, 0xab, 0xe6, 0x7b, - 0xd7, 0x20, 0x1f, 0x51, 0xf9, 0xb3, 0x8d, 0xf2, 0x8b, 0x35, 0xea, 0x8a, 0x7b, 0x36, 0x3b, 0x84, - 0x21, 0x92, 0xc9, 0xfc, 0x09, 0x0d, 0xb4, 0xca, 0xac, 0xba, 0x52, 0xc9, 0xb6, 0xe1, 0x91, 0x53, - 0x47, 0x89, 0xf5, 0xc1, 0x88, 0x52, 0x73, 0x48, 0x42, 0xeb, 0x83, 0x8d, 0xd9, 0x29, 0x8c, 0x24, - 0x29, 0xd3, 0x3c, 0x4e, 0xc2, 0x69, 0x3c, 0x7b, 0x99, 0x7a, 0xfb, 0x4c, 0xfd, 0x75, 0x7f, 0x75, - 0xbc, 0x79, 0x6d, 0x54, 0x97, 0xaf, 0xab, 0xd8, 0x3b, 0x80, 0x42, 0x18, 0x61, 0xb7, 0x89, 0x9a, - 0xef, 0x25, 0xc1, 0x34, 0x9e, 0x1d, 0x6d, 0xf4, 0xf8, 0x74, 0x0f, 0xe7, 0x1e, 0x75, 0x7c, 0x02, - 0x7b, 0x7e, 0x47, 0xeb, 0xec, 0x35, 0x76, 0xab, 0xb5, 0xd9, 0xd0, 0xaa, 0x20, 0xa9, 0xab, 0xc5, - 0xb9, 0xe4, 0x64, 0xf0, 0x3e, 0x98, 0x18, 0x80, 0xbe, 0x2b, 0x7b, 0x05, 0x23, 0x8d, 0xea, 0xb6, - 0xaa, 0x4b, 0xaa, 0x8e, 0x67, 0x87, 0xdb, 0xbf, 0x9f, 0xaf, 0x69, 0xec, 0x0d, 0x44, 0x77, 0x42, - 0xe1, 0x95, 0x6c, 0xb5, 0xeb, 0xfe, 0xff, 0x9a, 0x9e, 0x38, 0xf9, 0x13, 0x40, 0x74, 0x0f, 0x3c, - 0xf8, 0xcb, 0x3e, 0xf6, 0x4e, 0x0e, 0xc8, 0xc9, 0x17, 0xdb, 0x3b, 0x6e, 0xf7, 0xf1, 0x31, 0x76, - 0x9c, 0xfd, 0x00, 0xff, 0x08, 0xcf, 0x0e, 0xbc, 0xad, 0x7d, 0xb3, 0x17, 0xf2, 0xf3, 0x6d, 0x59, - 0x99, 0xab, 0xf6, 0x32, 0x5d, 0xc8, 0x65, 0x56, 0xca, 0xdf, 0x78, 0x9d, 0xb9, 0x5b, 0xa2, 0xfb, - 0xd1, 0x59, 0x89, 0x35, 0x2a, 0x61, 0xb0, 0xc8, 0x4a, 0x99, 0x79, 0x37, 0x7a, 0x39, 0x24, 0xc2, - 0xeb, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x13, 0x00, 0x0c, 0x12, 0x03, 0x04, 0x00, 0x00, + proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_FeatureSpec_de24bbc3c8f2bed6) +} + +var fileDescriptor_FeatureSpec_de24bbc3c8f2bed6 = []byte{ + // 451 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0xdf, 0x6b, 0xd4, 0x40, + 0x10, 0x26, 0x49, 0xdb, 0x33, 0x93, 0x52, 0xca, 0x22, 0xed, 0x72, 0x2a, 0x84, 0x13, 0xe1, 0x9e, + 0x12, 0x39, 0xc5, 0x1f, 0x05, 0x11, 0x8a, 0xe7, 0xab, 0x92, 0x4a, 0x05, 0xdf, 0xb6, 0xc9, 0x98, + 0xc6, 0x7a, 0xd9, 0xb0, 0xbb, 0x69, 0xc9, 0x3f, 0xe2, 0x9f, 0xe1, 0xdf, 0x28, 0x3b, 0x7b, 0xd7, + 0xec, 0xe1, 0xf9, 0xe4, 0xdb, 0xcc, 0x7e, 0xdf, 0x7e, 0x3b, 0xf3, 0xed, 0x0c, 0x3c, 0xf9, 0x8e, + 0x42, 0x9b, 0x5c, 0x77, 0x58, 0xea, 0xfc, 0x23, 0x0a, 0xd3, 0x2b, 0xbc, 0xe8, 0xb0, 0xcc, 0x3a, + 0x25, 0x8d, 0x64, 0x09, 0xc1, 0x19, 0xc1, 0xd3, 0xc7, 0x3e, 0x77, 0xd9, 0x9a, 0xc6, 0x0c, 0x23, + 0x75, 0xba, 0xa5, 0x74, 0x61, 0xa4, 0x12, 0xb5, 0xa7, 0x34, 0x3d, 0x75, 0xb0, 0x19, 0x3a, 0xd4, + 0xf9, 0xa5, 0xf8, 0xd9, 0xa3, 0x03, 0x66, 0xbf, 0x23, 0x48, 0xbc, 0x87, 0xd9, 0x11, 0x84, 0x4d, + 0xc5, 0x83, 0x34, 0x98, 0xc7, 0x45, 0xd8, 0x54, 0x8c, 0xc1, 0x5e, 0x2b, 0x56, 0xc8, 0x43, 0x3a, + 0xa1, 0x98, 0x3d, 0x84, 0x7d, 0x79, 0xd7, 0xa2, 0xe2, 0x11, 0x1d, 0xba, 0x84, 0xa5, 0x90, 0x54, + 0xa8, 0x4b, 0xd5, 0x74, 0xa6, 0x91, 0x2d, 0xdf, 0x23, 0xcc, 0x3f, 0x62, 0xc7, 0x10, 0xf5, 0xaa, + 0xe1, 0xfb, 0x84, 0xd8, 0x90, 0xbd, 0x85, 0xf8, 0xd6, 0x16, 0xf3, 0x65, 0xe8, 0x90, 0x4f, 0xd2, + 0x60, 0x7e, 0xb4, 0x78, 0x94, 0xb9, 0xa6, 0xa9, 0xd4, 0xec, 0x72, 0x83, 0x66, 0xcb, 0xb6, 0x5f, + 0x15, 0x23, 0x9b, 0x9d, 0xc0, 0x01, 0x92, 0x09, 0xfc, 0x01, 0xe9, 0xad, 0x33, 0x5b, 0x5c, 0xad, + 0x64, 0xdf, 0xf1, 0xd8, 0x15, 0x47, 0x89, 0x6d, 0xc3, 0x88, 0x5a, 0x73, 0x48, 0x23, 0xdb, 0x86, + 0x8d, 0xd9, 0x7b, 0x98, 0x48, 0x2a, 0x4c, 0xf3, 0x24, 0x8d, 0xe6, 0xc9, 0xe2, 0x59, 0xe6, 0xf9, + 0x9d, 0xf9, 0xdf, 0xf1, 0xc9, 0xf1, 0x96, 0xad, 0x51, 0x43, 0xb1, 0xb9, 0xc5, 0x5e, 0x03, 0x54, + 0xc2, 0x08, 0xeb, 0x36, 0x6a, 0x7e, 0x98, 0x06, 0xf3, 0x64, 0x71, 0xba, 0xa5, 0xf1, 0xe1, 0x1e, + 0x2e, 0x3c, 0xea, 0xf4, 0x0c, 0x0e, 0x7d, 0x45, 0x6b, 0xcc, 0x0d, 0x0e, 0x6b, 0xd7, 0x6d, 0x68, + 0xbb, 0xa0, 0x56, 0xd7, 0xbe, 0xbb, 0xe4, 0x2c, 0x7c, 0x13, 0xcc, 0x0c, 0xc0, 0xa8, 0xca, 0x9e, + 0xc3, 0x44, 0xa3, 0xba, 0x6d, 0xda, 0x9a, 0x6e, 0x27, 0x8b, 0x93, 0xdd, 0xef, 0x17, 0x1b, 0x1a, + 0x7b, 0x09, 0xf1, 0x9d, 0x50, 0x78, 0x2d, 0x7b, 0xed, 0xd4, 0xff, 0x7d, 0x67, 0x24, 0xce, 0x7e, + 0x05, 0x10, 0xdf, 0x03, 0x7f, 0x0d, 0xc9, 0xbb, 0xd1, 0xc9, 0x90, 0x9c, 0x7c, 0xba, 0x5b, 0x71, + 0xb7, 0x8f, 0xff, 0x63, 0xc7, 0xf9, 0x57, 0xf0, 0x97, 0xe4, 0xfc, 0xd8, 0xfb, 0xb5, 0xcf, 0x76, + 0xc0, 0xbf, 0xbd, 0xaa, 0x1b, 0x73, 0xdd, 0x5f, 0x65, 0xa5, 0x5c, 0xe5, 0xb5, 0xfc, 0x81, 0x37, + 0xb9, 0x5b, 0x05, 0x1a, 0x7f, 0x9d, 0xd7, 0xd8, 0xa2, 0x12, 0x06, 0xab, 0xbc, 0x96, 0xb9, 0xb7, + 0x43, 0x57, 0x07, 0x44, 0x78, 0xf1, 0x27, 0x00, 0x00, 0xff, 0xff, 0xca, 0xfb, 0xc4, 0x86, 0xa3, + 0x03, 0x00, 0x00, } diff --git a/protos/generated/go/feast/specs/ImportJobSpecs.pb.go b/protos/generated/go/feast/specs/ImportJobSpecs.pb.go new file mode 100644 index 00000000000..d4c940e947b --- /dev/null +++ b/protos/generated/go/feast/specs/ImportJobSpecs.pb.go @@ -0,0 +1,137 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: feast/specs/ImportJobSpecs.proto + +package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type ImportJobSpecs struct { + JobId string `protobuf:"bytes,1,opt,name=jobId,proto3" json:"jobId,omitempty"` + ImportSpec *ImportSpec `protobuf:"bytes,2,opt,name=importSpec,proto3" json:"importSpec,omitempty"` + EntitySpecs []*EntitySpec `protobuf:"bytes,3,rep,name=entitySpecs,proto3" json:"entitySpecs,omitempty"` + FeatureSpecs []*FeatureSpec `protobuf:"bytes,4,rep,name=featureSpecs,proto3" json:"featureSpecs,omitempty"` + ServingStorageSpecs []*StorageSpec `protobuf:"bytes,5,rep,name=servingStorageSpecs,proto3" json:"servingStorageSpecs,omitempty"` + WarehouseStorageSpecs []*StorageSpec `protobuf:"bytes,6,rep,name=warehouseStorageSpecs,proto3" json:"warehouseStorageSpecs,omitempty"` + ErrorsStorageSpec *StorageSpec `protobuf:"bytes,7,opt,name=errorsStorageSpec,proto3" json:"errorsStorageSpec,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ImportJobSpecs) Reset() { *m = ImportJobSpecs{} } +func (m *ImportJobSpecs) String() string { return proto.CompactTextString(m) } +func (*ImportJobSpecs) ProtoMessage() {} +func (*ImportJobSpecs) Descriptor() ([]byte, []int) { + return fileDescriptor_ImportJobSpecs_c867f2e818ce78e5, []int{0} +} +func (m *ImportJobSpecs) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ImportJobSpecs.Unmarshal(m, b) +} +func (m *ImportJobSpecs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ImportJobSpecs.Marshal(b, m, deterministic) +} +func (dst *ImportJobSpecs) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportJobSpecs.Merge(dst, src) +} +func (m *ImportJobSpecs) XXX_Size() int { + return xxx_messageInfo_ImportJobSpecs.Size(m) +} +func (m *ImportJobSpecs) XXX_DiscardUnknown() { + xxx_messageInfo_ImportJobSpecs.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportJobSpecs proto.InternalMessageInfo + +func (m *ImportJobSpecs) GetJobId() string { + if m != nil { + return m.JobId + } + return "" +} + +func (m *ImportJobSpecs) GetImportSpec() *ImportSpec { + if m != nil { + return m.ImportSpec + } + return nil +} + +func (m *ImportJobSpecs) GetEntitySpecs() []*EntitySpec { + if m != nil { + return m.EntitySpecs + } + return nil +} + +func (m *ImportJobSpecs) GetFeatureSpecs() []*FeatureSpec { + if m != nil { + return m.FeatureSpecs + } + return nil +} + +func (m *ImportJobSpecs) GetServingStorageSpecs() []*StorageSpec { + if m != nil { + return m.ServingStorageSpecs + } + return nil +} + +func (m *ImportJobSpecs) GetWarehouseStorageSpecs() []*StorageSpec { + if m != nil { + return m.WarehouseStorageSpecs + } + return nil +} + +func (m *ImportJobSpecs) GetErrorsStorageSpec() *StorageSpec { + if m != nil { + return m.ErrorsStorageSpec + } + return nil +} + +func init() { + proto.RegisterType((*ImportJobSpecs)(nil), "feast.specs.ImportJobSpecs") +} + +func init() { + proto.RegisterFile("feast/specs/ImportJobSpecs.proto", fileDescriptor_ImportJobSpecs_c867f2e818ce78e5) +} + +var fileDescriptor_ImportJobSpecs_c867f2e818ce78e5 = []byte{ + // 306 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0x41, 0x4b, 0xc3, 0x40, + 0x10, 0x85, 0xa9, 0xb5, 0x15, 0xb7, 0x22, 0xb8, 0x55, 0x5c, 0x44, 0x21, 0x78, 0xea, 0x29, 0x0b, + 0x0a, 0x8a, 0xe0, 0xa9, 0x60, 0xa1, 0x3d, 0x88, 0xa4, 0xb7, 0xde, 0x92, 0x76, 0xba, 0x4d, 0xa5, + 0x99, 0xb0, 0xbb, 0x51, 0xfc, 0x6f, 0xfe, 0x38, 0xc9, 0x06, 0x9a, 0x59, 0x92, 0xe2, 0x71, 0xf3, + 0xbe, 0x37, 0xcc, 0xcb, 0x3c, 0x16, 0xac, 0x21, 0x36, 0x56, 0x9a, 0x1c, 0x96, 0x46, 0x4e, 0x77, + 0x39, 0x6a, 0x3b, 0xc3, 0x64, 0x5e, 0x3e, 0xc3, 0x5c, 0xa3, 0x45, 0x3e, 0x70, 0x44, 0xe8, 0x88, + 0x9b, 0xdb, 0x26, 0x5e, 0xb2, 0x15, 0xea, 0xab, 0x6f, 0x99, 0x4d, 0xed, 0x0f, 0x51, 0xef, 0xa8, + 0x3a, 0x81, 0xd8, 0x16, 0x1a, 0x0e, 0xc9, 0x73, 0x8b, 0x3a, 0x56, 0x44, 0xbe, 0xff, 0xed, 0xb2, + 0x73, 0x7f, 0x3f, 0x7e, 0xc9, 0x7a, 0x5b, 0x4c, 0xa6, 0x2b, 0xd1, 0x09, 0x3a, 0xa3, 0xd3, 0xa8, + 0x7a, 0xf0, 0x67, 0xc6, 0xd2, 0xfd, 0x62, 0xe2, 0x28, 0xe8, 0x8c, 0x06, 0x0f, 0xd7, 0x21, 0x09, + 0x11, 0xd6, 0x7b, 0x47, 0x04, 0xe5, 0x2f, 0x6c, 0x00, 0xfb, 0x9d, 0x8d, 0xe8, 0x06, 0xdd, 0x86, + 0xb3, 0xce, 0x14, 0x51, 0x96, 0xbf, 0xb2, 0xb3, 0x75, 0x1d, 0xc8, 0x88, 0x63, 0xe7, 0x15, 0x9e, + 0x97, 0x24, 0x8e, 0x3c, 0x9a, 0xcf, 0xd8, 0xd0, 0x80, 0xfe, 0x4a, 0x33, 0x45, 0x62, 0x1b, 0xd1, + 0x6b, 0x19, 0x42, 0x80, 0xa8, 0xcd, 0xc4, 0xdf, 0xd9, 0xd5, 0x77, 0xac, 0x61, 0x83, 0x85, 0x01, + 0x6f, 0x5a, 0xff, 0x9f, 0x69, 0xed, 0x36, 0x3e, 0x61, 0x17, 0xa0, 0x35, 0x6a, 0x43, 0xbe, 0x8a, + 0x13, 0xf7, 0x53, 0x0f, 0xcf, 0x6a, 0x5a, 0xc6, 0x0b, 0x46, 0x7b, 0x34, 0x1e, 0xfa, 0xa7, 0xfc, + 0x28, 0x4f, 0xbc, 0x78, 0x52, 0xa9, 0xdd, 0x14, 0x49, 0xb8, 0xc4, 0x9d, 0x54, 0xb8, 0x85, 0x4f, + 0x59, 0x55, 0xc2, 0x15, 0xc0, 0x48, 0x05, 0x19, 0xe8, 0xd8, 0xc2, 0x4a, 0x2a, 0x94, 0xa4, 0x2c, + 0x49, 0xdf, 0x01, 0x8f, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xdd, 0xf2, 0xf3, 0xa2, 0xcc, 0x02, + 0x00, 0x00, +} diff --git a/protos/generated/go/feast/specs/ImportSpec.pb.go b/protos/generated/go/feast/specs/ImportSpec.pb.go index 9bcb9b3ae77..d16f9fa07bd 100644 --- a/protos/generated/go/feast/specs/ImportSpec.pb.go +++ b/protos/generated/go/feast/specs/ImportSpec.pb.go @@ -34,7 +34,7 @@ func (m *ImportSpec) Reset() { *m = ImportSpec{} } func (m *ImportSpec) String() string { return proto.CompactTextString(m) } func (*ImportSpec) ProtoMessage() {} func (*ImportSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_a027fec3e71fec8d, []int{0} + return fileDescriptor_ImportSpec_673bc4f248a91137, []int{0} } func (m *ImportSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportSpec.Unmarshal(m, b) @@ -107,7 +107,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_a027fec3e71fec8d, []int{1} + return fileDescriptor_ImportSpec_673bc4f248a91137, []int{1} } func (m *Schema) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Schema.Unmarshal(m, b) @@ -260,7 +260,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_a027fec3e71fec8d, []int{2} + return fileDescriptor_ImportSpec_673bc4f248a91137, []int{2} } func (m *Field) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Field.Unmarshal(m, b) @@ -303,10 +303,10 @@ func init() { } func init() { - proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_a027fec3e71fec8d) + proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_673bc4f248a91137) } -var fileDescriptor_ImportSpec_a027fec3e71fec8d = []byte{ +var fileDescriptor_ImportSpec_673bc4f248a91137 = []byte{ // 440 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8f, 0x93, 0x40, 0x14, 0x5d, 0xca, 0x16, 0xed, 0x6d, 0xdc, 0x9a, 0xab, 0x0f, 0x84, 0x6c, 0x62, 0xd3, 0x07, 0x6d, diff --git a/protos/generated/go/feast/specs/StorageSpec.pb.go b/protos/generated/go/feast/specs/StorageSpec.pb.go index 8c76b758847..fefe56a1be5 100644 --- a/protos/generated/go/feast/specs/StorageSpec.pb.go +++ b/protos/generated/go/feast/specs/StorageSpec.pb.go @@ -36,7 +36,7 @@ func (m *StorageSpec) Reset() { *m = StorageSpec{} } func (m *StorageSpec) String() string { return proto.CompactTextString(m) } func (*StorageSpec) ProtoMessage() {} func (*StorageSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_StorageSpec_ceee6ec6c0eb3849, []int{0} + return fileDescriptor_StorageSpec_bfb8a5e5cf34de95, []int{0} } func (m *StorageSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StorageSpec.Unmarshal(m, b) @@ -83,10 +83,10 @@ func init() { } func init() { - proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_StorageSpec_ceee6ec6c0eb3849) + proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_StorageSpec_bfb8a5e5cf34de95) } -var fileDescriptor_StorageSpec_ceee6ec6c0eb3849 = []byte{ +var fileDescriptor_StorageSpec_bfb8a5e5cf34de95 = []byte{ // 227 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x0f, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x0d, diff --git a/protos/generated/go/feast/storage/BigTable.pb.go b/protos/generated/go/feast/storage/BigTable.pb.go index 6dc2c35c800..e65b5e35fce 100644 --- a/protos/generated/go/feast/storage/BigTable.pb.go +++ b/protos/generated/go/feast/storage/BigTable.pb.go @@ -33,7 +33,7 @@ func (m *BigTableRowKey) Reset() { *m = BigTableRowKey{} } func (m *BigTableRowKey) String() string { return proto.CompactTextString(m) } func (*BigTableRowKey) ProtoMessage() {} func (*BigTableRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_BigTable_967f8b41f64b081b, []int{0} + return fileDescriptor_BigTable_e25bdeffe0669ae7, []int{0} } func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BigTableRowKey.Unmarshal(m, b) @@ -79,10 +79,10 @@ func init() { } func init() { - proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_967f8b41f64b081b) + proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_e25bdeffe0669ae7) } -var fileDescriptor_BigTable_967f8b41f64b081b = []byte{ +var fileDescriptor_BigTable_e25bdeffe0669ae7 = []byte{ // 193 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x8f, 0xb1, 0x0b, 0x82, 0x40, 0x18, 0x47, 0xb1, 0x20, 0xf0, 0x40, 0x87, 0x9b, 0x1c, 0x24, 0xa2, 0x21, 0x9a, 0x3c, 0xa2, 0xa5, diff --git a/protos/generated/go/feast/storage/Redis.pb.go b/protos/generated/go/feast/storage/Redis.pb.go index c277a8f03e9..49f80a061e3 100644 --- a/protos/generated/go/feast/storage/Redis.pb.go +++ b/protos/generated/go/feast/storage/Redis.pb.go @@ -43,7 +43,7 @@ func (m *RedisBucketKey) Reset() { *m = RedisBucketKey{} } func (m *RedisBucketKey) String() string { return proto.CompactTextString(m) } func (*RedisBucketKey) ProtoMessage() {} func (*RedisBucketKey) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_749687aada0bf97b, []int{0} + return fileDescriptor_Redis_cef62c817c1622ce, []int{0} } func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketKey.Unmarshal(m, b) @@ -99,7 +99,7 @@ func (m *RedisBucketValue) Reset() { *m = RedisBucketValue{} } func (m *RedisBucketValue) String() string { return proto.CompactTextString(m) } func (*RedisBucketValue) ProtoMessage() {} func (*RedisBucketValue) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_749687aada0bf97b, []int{1} + return fileDescriptor_Redis_cef62c817c1622ce, []int{1} } func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValue.Unmarshal(m, b) @@ -147,7 +147,7 @@ func (m *RedisBucketValueList) Reset() { *m = RedisBucketValueList{} } func (m *RedisBucketValueList) String() string { return proto.CompactTextString(m) } func (*RedisBucketValueList) ProtoMessage() {} func (*RedisBucketValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_749687aada0bf97b, []int{2} + return fileDescriptor_Redis_cef62c817c1622ce, []int{2} } func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValueList.Unmarshal(m, b) @@ -180,9 +180,9 @@ func init() { proto.RegisterType((*RedisBucketValueList)(nil), "feast.storage.RedisBucketValueList") } -func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_Redis_749687aada0bf97b) } +func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_Redis_cef62c817c1622ce) } -var fileDescriptor_Redis_749687aada0bf97b = []byte{ +var fileDescriptor_Redis_cef62c817c1622ce = []byte{ // 325 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcd, 0x4f, 0xf2, 0x40, 0x10, 0xc6, 0xd3, 0x97, 0x57, 0x22, 0x4b, 0x24, 0x66, 0x35, 0xb1, 0x36, 0x26, 0x34, 0x9c, 0x7a, diff --git a/protos/generated/go/feast/types/Feature.pb.go b/protos/generated/go/feast/types/Feature.pb.go index 55c70f46b2e..51ed41fb54f 100644 --- a/protos/generated/go/feast/types/Feature.pb.go +++ b/protos/generated/go/feast/types/Feature.pb.go @@ -30,7 +30,7 @@ func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_Feature_9650e908dedbbf49, []int{0} + return fileDescriptor_Feature_c2a5d99d9bf3ca9c, []int{0} } func (m *Feature) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Feature.Unmarshal(m, b) @@ -68,9 +68,9 @@ func init() { proto.RegisterType((*Feature)(nil), "feast.types.Feature") } -func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_9650e908dedbbf49) } +func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_c2a5d99d9bf3ca9c) } -var fileDescriptor_Feature_9650e908dedbbf49 = []byte{ +var fileDescriptor_Feature_c2a5d99d9bf3ca9c = []byte{ // 173 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0xd5, diff --git a/protos/generated/go/feast/types/FeatureRow.pb.go b/protos/generated/go/feast/types/FeatureRow.pb.go index 5e9d6b0e74a..dc6099ba754 100644 --- a/protos/generated/go/feast/types/FeatureRow.pb.go +++ b/protos/generated/go/feast/types/FeatureRow.pb.go @@ -23,7 +23,6 @@ type FeatureRowKey struct { EntityKey string `protobuf:"bytes,1,opt,name=entityKey,proto3" json:"entityKey,omitempty"` EventTimestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=eventTimestamp,proto3" json:"eventTimestamp,omitempty"` EntityName string `protobuf:"bytes,4,opt,name=entityName,proto3" json:"entityName,omitempty"` - Granularity Granularity_Enum `protobuf:"varint,5,opt,name=granularity,proto3,enum=feast.types.Granularity_Enum" json:"granularity,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -33,7 +32,7 @@ func (m *FeatureRowKey) Reset() { *m = FeatureRowKey{} } func (m *FeatureRowKey) String() string { return proto.CompactTextString(m) } func (*FeatureRowKey) ProtoMessage() {} func (*FeatureRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_44b5ccd93feeaf74, []int{0} + return fileDescriptor_FeatureRow_3072498459678ede, []int{0} } func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowKey.Unmarshal(m, b) @@ -74,19 +73,11 @@ func (m *FeatureRowKey) GetEntityName() string { return "" } -func (m *FeatureRowKey) GetGranularity() Granularity_Enum { - if m != nil { - return m.Granularity - } - return Granularity_NONE -} - type FeatureRow struct { EntityKey string `protobuf:"bytes,1,opt,name=entityKey,proto3" json:"entityKey,omitempty"` Features []*Feature `protobuf:"bytes,2,rep,name=features,proto3" json:"features,omitempty"` EventTimestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=eventTimestamp,proto3" json:"eventTimestamp,omitempty"` EntityName string `protobuf:"bytes,4,opt,name=entityName,proto3" json:"entityName,omitempty"` - Granularity Granularity_Enum `protobuf:"varint,5,opt,name=granularity,proto3,enum=feast.types.Granularity_Enum" json:"granularity,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -96,7 +87,7 @@ func (m *FeatureRow) Reset() { *m = FeatureRow{} } func (m *FeatureRow) String() string { return proto.CompactTextString(m) } func (*FeatureRow) ProtoMessage() {} func (*FeatureRow) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_44b5ccd93feeaf74, []int{1} + return fileDescriptor_FeatureRow_3072498459678ede, []int{1} } func (m *FeatureRow) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRow.Unmarshal(m, b) @@ -144,42 +135,32 @@ func (m *FeatureRow) GetEntityName() string { return "" } -func (m *FeatureRow) GetGranularity() Granularity_Enum { - if m != nil { - return m.Granularity - } - return Granularity_NONE -} - func init() { proto.RegisterType((*FeatureRowKey)(nil), "feast.types.FeatureRowKey") proto.RegisterType((*FeatureRow)(nil), "feast.types.FeatureRow") } func init() { - proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_44b5ccd93feeaf74) -} - -var fileDescriptor_FeatureRow_44b5ccd93feeaf74 = []byte{ - // 305 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x92, 0x41, 0x4b, 0xfb, 0x40, - 0x10, 0xc5, 0x49, 0xfb, 0xff, 0x8b, 0x9d, 0x60, 0x85, 0xe0, 0x21, 0x96, 0x56, 0x43, 0x4f, 0x39, - 0xed, 0x4a, 0x04, 0xaf, 0x42, 0x40, 0x3d, 0x14, 0x44, 0x82, 0x78, 0xf0, 0xb6, 0xd1, 0xc9, 0x1a, - 0x6d, 0xb2, 0x21, 0x99, 0x28, 0x39, 0xfa, 0x15, 0xfd, 0x44, 0xe2, 0xae, 0x69, 0x56, 0x11, 0x3c, - 0x7b, 0x9d, 0xf7, 0x66, 0xe6, 0x37, 0xc3, 0x83, 0x79, 0x86, 0xa2, 0x21, 0x4e, 0x5d, 0x85, 0x0d, - 0x3f, 0x47, 0x41, 0x6d, 0x8d, 0x89, 0x7a, 0x61, 0x55, 0xad, 0x48, 0x79, 0xae, 0x56, 0x99, 0x56, - 0x67, 0x87, 0x52, 0x29, 0xb9, 0x46, 0xae, 0xa5, 0xb4, 0xcd, 0x38, 0xe5, 0x05, 0x36, 0x24, 0x8a, - 0xca, 0xb8, 0x67, 0xfb, 0x3f, 0xcc, 0xfa, 0x94, 0x16, 0xb6, 0x74, 0x51, 0x8b, 0xb2, 0x5d, 0x8b, - 0x3a, 0xa7, 0xce, 0xc8, 0xcb, 0x37, 0x07, 0x76, 0x86, 0xe5, 0x2b, 0xec, 0xbc, 0x39, 0x4c, 0xb0, - 0xa4, 0x9c, 0xba, 0x15, 0x76, 0xbe, 0x13, 0x38, 0xe1, 0x24, 0x19, 0x0a, 0x5e, 0x0c, 0x53, 0x7c, - 0xc6, 0x92, 0xae, 0x7b, 0x02, 0x7f, 0x1c, 0x38, 0xa1, 0x1b, 0xcd, 0x98, 0x61, 0x64, 0x3d, 0x23, - 0xdb, 0x38, 0x92, 0x6f, 0x1d, 0xde, 0x01, 0x80, 0x19, 0x78, 0x29, 0x0a, 0xf4, 0xff, 0xe9, 0x15, - 0x56, 0xc5, 0x3b, 0x05, 0x57, 0x0e, 0xa0, 0xfe, 0xff, 0xc0, 0x09, 0xa7, 0xd1, 0x82, 0x59, 0x1f, - 0x61, 0xf6, 0x21, 0x67, 0x65, 0x5b, 0x24, 0x76, 0xc7, 0xf2, 0x75, 0x04, 0x30, 0x1c, 0xf5, 0xcb, - 0x45, 0x47, 0xb0, 0x9d, 0x19, 0x6f, 0xe3, 0x8f, 0x82, 0x71, 0xe8, 0x46, 0x7b, 0x5f, 0x56, 0xf5, - 0x83, 0x36, 0xae, 0x3f, 0xf1, 0x83, 0xf8, 0x06, 0xec, 0x08, 0xc5, 0xbb, 0xc3, 0x3f, 0xae, 0x3e, - 0xe8, 0x6e, 0x4f, 0x64, 0x4e, 0x0f, 0x6d, 0xca, 0xee, 0x54, 0xc1, 0xa5, 0x7a, 0xc4, 0x27, 0x6e, - 0x82, 0xa2, 0xd9, 0x1b, 0x2e, 0xb1, 0xc4, 0x5a, 0x10, 0xde, 0x73, 0xa9, 0xb8, 0x15, 0xa1, 0x74, - 0x4b, 0x1b, 0x8e, 0xdf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xb5, 0xfe, 0x14, 0xc9, 0xbf, 0x02, 0x00, - 0x00, + proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_3072498459678ede) +} + +var fileDescriptor_FeatureRow_3072498459678ede = []byte{ + // 266 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x91, 0xc1, 0x4a, 0xc4, 0x30, + 0x10, 0x86, 0x89, 0x2b, 0xe2, 0xce, 0xa2, 0x42, 0xf1, 0x50, 0xcb, 0xa2, 0x65, 0x4f, 0x3d, 0x65, + 0x64, 0x05, 0x1f, 0xa0, 0x07, 0x2f, 0x0b, 0x22, 0x45, 0x3c, 0x78, 0x4b, 0x75, 0x1a, 0xab, 0xb6, + 0x29, 0xcd, 0x54, 0xe9, 0x63, 0xf8, 0x46, 0x3e, 0x9a, 0x98, 0xd0, 0x6d, 0x11, 0xc1, 0x9b, 0xd7, + 0xf9, 0xff, 0x7c, 0xf3, 0x4f, 0x7e, 0x58, 0x16, 0xa4, 0x2c, 0x23, 0xf7, 0x0d, 0x59, 0xbc, 0x22, + 0xc5, 0x5d, 0x4b, 0x99, 0x79, 0x97, 0x4d, 0x6b, 0xd8, 0x04, 0x0b, 0xa7, 0x4a, 0xa7, 0x46, 0x67, + 0xda, 0x18, 0xfd, 0x4a, 0xe8, 0xa4, 0xbc, 0x2b, 0x90, 0xcb, 0x8a, 0x2c, 0xab, 0xaa, 0xf1, 0xee, + 0xe8, 0xe4, 0x17, 0x96, 0x97, 0x56, 0x1f, 0x02, 0x0e, 0x46, 0xfa, 0x86, 0xfa, 0x60, 0x09, 0x73, + 0xaa, 0xb9, 0xe4, 0x7e, 0x43, 0x7d, 0x28, 0x62, 0x91, 0xcc, 0xb3, 0x71, 0x10, 0xa4, 0x70, 0x48, + 0x6f, 0x54, 0xf3, 0xed, 0xb0, 0x22, 0x9c, 0xc5, 0x22, 0x59, 0xac, 0x23, 0xe9, 0x43, 0xc8, 0x21, + 0x84, 0xdc, 0x3a, 0xb2, 0x1f, 0x2f, 0x82, 0x53, 0x00, 0x0f, 0xbc, 0x56, 0x15, 0x85, 0xbb, 0x6e, + 0xc5, 0x64, 0xb2, 0xfa, 0x14, 0x00, 0x63, 0xa6, 0x3f, 0x02, 0x9d, 0xc3, 0x7e, 0xe1, 0xbd, 0x36, + 0xdc, 0x89, 0x67, 0xc9, 0x62, 0x7d, 0x2c, 0x27, 0x9f, 0x23, 0x07, 0xd0, 0xd6, 0xf5, 0x1f, 0x27, + 0xa4, 0x77, 0x30, 0x6d, 0x28, 0x3d, 0x1a, 0xcf, 0xb9, 0xf9, 0x86, 0xdf, 0x5f, 0xea, 0x92, 0x9f, + 0xba, 0x5c, 0x3e, 0x98, 0x0a, 0xb5, 0x79, 0xa6, 0x17, 0xf4, 0x15, 0xb9, 0xd5, 0x16, 0x35, 0xd5, + 0xd4, 0x2a, 0xa6, 0x47, 0xd4, 0x06, 0x27, 0xe5, 0xe5, 0x7b, 0xce, 0x70, 0xf1, 0x15, 0x00, 0x00, + 0xff, 0xff, 0xa8, 0x83, 0xb6, 0x9f, 0x1e, 0x02, 0x00, 0x00, } diff --git a/protos/generated/go/feast/types/FeatureRowExtended.pb.go b/protos/generated/go/feast/types/FeatureRowExtended.pb.go index a01715d8db2..ad89f27db23 100644 --- a/protos/generated/go/feast/types/FeatureRowExtended.pb.go +++ b/protos/generated/go/feast/types/FeatureRowExtended.pb.go @@ -33,7 +33,7 @@ func (m *Error) Reset() { *m = Error{} } func (m *Error) String() string { return proto.CompactTextString(m) } func (*Error) ProtoMessage() {} func (*Error) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_f06d01003da6c18e, []int{0} + return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{0} } func (m *Error) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Error.Unmarshal(m, b) @@ -93,7 +93,7 @@ func (m *Attempt) Reset() { *m = Attempt{} } func (m *Attempt) String() string { return proto.CompactTextString(m) } func (*Attempt) ProtoMessage() {} func (*Attempt) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_f06d01003da6c18e, []int{1} + return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{1} } func (m *Attempt) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attempt.Unmarshal(m, b) @@ -140,7 +140,7 @@ func (m *FeatureRowExtended) Reset() { *m = FeatureRowExtended{} } func (m *FeatureRowExtended) String() string { return proto.CompactTextString(m) } func (*FeatureRowExtended) ProtoMessage() {} func (*FeatureRowExtended) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_f06d01003da6c18e, []int{2} + return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{2} } func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowExtended.Unmarshal(m, b) @@ -188,10 +188,10 @@ func init() { } func init() { - proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_FeatureRowExtended_f06d01003da6c18e) + proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_FeatureRowExtended_bfd3c37956d1a040) } -var fileDescriptor_FeatureRowExtended_f06d01003da6c18e = []byte{ +var fileDescriptor_FeatureRowExtended_bfd3c37956d1a040 = []byte{ // 338 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x6b, 0xea, 0x40, 0x10, 0xc6, 0xf1, 0xf9, 0xf2, 0x7c, 0x4e, 0x6e, 0x8b, 0x60, 0x08, 0xd2, 0x16, 0xe9, 0xc1, 0x5e, diff --git a/protos/generated/go/feast/types/Granularity.pb.go b/protos/generated/go/feast/types/Granularity.pb.go index a47e55ec373..5166f7bbc2e 100644 --- a/protos/generated/go/feast/types/Granularity.pb.go +++ b/protos/generated/go/feast/types/Granularity.pb.go @@ -47,7 +47,7 @@ func (x Granularity_Enum) String() string { return proto.EnumName(Granularity_Enum_name, int32(x)) } func (Granularity_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Granularity_1271850b4f4bfac4, []int{0, 0} + return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0, 0} } type Granularity struct { @@ -60,7 +60,7 @@ func (m *Granularity) Reset() { *m = Granularity{} } func (m *Granularity) String() string { return proto.CompactTextString(m) } func (*Granularity) ProtoMessage() {} func (*Granularity) Descriptor() ([]byte, []int) { - return fileDescriptor_Granularity_1271850b4f4bfac4, []int{0} + return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0} } func (m *Granularity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Granularity.Unmarshal(m, b) @@ -86,10 +86,10 @@ func init() { } func init() { - proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_Granularity_1271850b4f4bfac4) + proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_Granularity_4b57756f7a751fdb) } -var fileDescriptor_Granularity_1271850b4f4bfac4 = []byte{ +var fileDescriptor_Granularity_4b57756f7a751fdb = []byte{ // 183 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x2f, 0x4a, 0xcc, 0x2b, 0xcd, 0x49, 0x2c, diff --git a/protos/generated/go/feast/types/Value.pb.go b/protos/generated/go/feast/types/Value.pb.go index 2b0af9e51c7..b9c26f8c4b8 100644 --- a/protos/generated/go/feast/types/Value.pb.go +++ b/protos/generated/go/feast/types/Value.pb.go @@ -60,7 +60,7 @@ func (x ValueType_Enum) String() string { return proto.EnumName(ValueType_Enum_name, int32(x)) } func (ValueType_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{0, 0} + return fileDescriptor_Value_0680a2f024df1112, []int{0, 0} } type ValueType struct { @@ -73,7 +73,7 @@ func (m *ValueType) Reset() { *m = ValueType{} } func (m *ValueType) String() string { return proto.CompactTextString(m) } func (*ValueType) ProtoMessage() {} func (*ValueType) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{0} + return fileDescriptor_Value_0680a2f024df1112, []int{0} } func (m *ValueType) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueType.Unmarshal(m, b) @@ -113,7 +113,7 @@ func (m *Value) Reset() { *m = Value{} } func (m *Value) String() string { return proto.CompactTextString(m) } func (*Value) ProtoMessage() {} func (*Value) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{1} + return fileDescriptor_Value_0680a2f024df1112, []int{1} } func (m *Value) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Value.Unmarshal(m, b) @@ -427,7 +427,7 @@ func (m *ValueList) Reset() { *m = ValueList{} } func (m *ValueList) String() string { return proto.CompactTextString(m) } func (*ValueList) ProtoMessage() {} func (*ValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{2} + return fileDescriptor_Value_0680a2f024df1112, []int{2} } func (m *ValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueList.Unmarshal(m, b) @@ -761,7 +761,7 @@ func (m *BytesList) Reset() { *m = BytesList{} } func (m *BytesList) String() string { return proto.CompactTextString(m) } func (*BytesList) ProtoMessage() {} func (*BytesList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{3} + return fileDescriptor_Value_0680a2f024df1112, []int{3} } func (m *BytesList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BytesList.Unmarshal(m, b) @@ -799,7 +799,7 @@ func (m *StringList) Reset() { *m = StringList{} } func (m *StringList) String() string { return proto.CompactTextString(m) } func (*StringList) ProtoMessage() {} func (*StringList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{4} + return fileDescriptor_Value_0680a2f024df1112, []int{4} } func (m *StringList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StringList.Unmarshal(m, b) @@ -837,7 +837,7 @@ func (m *Int32List) Reset() { *m = Int32List{} } func (m *Int32List) String() string { return proto.CompactTextString(m) } func (*Int32List) ProtoMessage() {} func (*Int32List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{5} + return fileDescriptor_Value_0680a2f024df1112, []int{5} } func (m *Int32List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int32List.Unmarshal(m, b) @@ -875,7 +875,7 @@ func (m *Int64List) Reset() { *m = Int64List{} } func (m *Int64List) String() string { return proto.CompactTextString(m) } func (*Int64List) ProtoMessage() {} func (*Int64List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{6} + return fileDescriptor_Value_0680a2f024df1112, []int{6} } func (m *Int64List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int64List.Unmarshal(m, b) @@ -913,7 +913,7 @@ func (m *DoubleList) Reset() { *m = DoubleList{} } func (m *DoubleList) String() string { return proto.CompactTextString(m) } func (*DoubleList) ProtoMessage() {} func (*DoubleList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{7} + return fileDescriptor_Value_0680a2f024df1112, []int{7} } func (m *DoubleList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DoubleList.Unmarshal(m, b) @@ -951,7 +951,7 @@ func (m *FloatList) Reset() { *m = FloatList{} } func (m *FloatList) String() string { return proto.CompactTextString(m) } func (*FloatList) ProtoMessage() {} func (*FloatList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{8} + return fileDescriptor_Value_0680a2f024df1112, []int{8} } func (m *FloatList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FloatList.Unmarshal(m, b) @@ -989,7 +989,7 @@ func (m *BoolList) Reset() { *m = BoolList{} } func (m *BoolList) String() string { return proto.CompactTextString(m) } func (*BoolList) ProtoMessage() {} func (*BoolList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{9} + return fileDescriptor_Value_0680a2f024df1112, []int{9} } func (m *BoolList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BoolList.Unmarshal(m, b) @@ -1027,7 +1027,7 @@ func (m *TimestampList) Reset() { *m = TimestampList{} } func (m *TimestampList) String() string { return proto.CompactTextString(m) } func (*TimestampList) ProtoMessage() {} func (*TimestampList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_8f69b75784c97601, []int{10} + return fileDescriptor_Value_0680a2f024df1112, []int{10} } func (m *TimestampList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TimestampList.Unmarshal(m, b) @@ -1069,9 +1069,9 @@ func init() { proto.RegisterEnum("feast.types.ValueType_Enum", ValueType_Enum_name, ValueType_Enum_value) } -func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_Value_8f69b75784c97601) } +func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_Value_0680a2f024df1112) } -var fileDescriptor_Value_8f69b75784c97601 = []byte{ +var fileDescriptor_Value_0680a2f024df1112 = []byte{ // 626 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xd1, 0x6f, 0x9a, 0x50, 0x14, 0xc6, 0xb9, 0x22, 0x0a, 0xc7, 0x36, 0x21, 0x37, 0xd9, 0xda, 0x34, 0x6d, 0x47, 0x7c, 0xe2, diff --git a/rfcs/0001-python-sdk.md b/rfcs/0001-python-sdk.md index 2f7b7a55cf7..328127a7c8d 100644 --- a/rfcs/0001-python-sdk.md +++ b/rfcs/0001-python-sdk.md @@ -143,14 +143,13 @@ schema: fields: - name: start_time - name: s2id - - featureId: s2id.minute.surge_factor + - featureId: s2id.surge_factor name: surge_factor timestampColumn: start_time -id: s2id.minute.surge_factor +id: s2id.surge_factor name: surge_factor owner: user@website.com -granularity: MINUTE valueType: DOUBLE entity: s2id dataStores: {} diff --git a/sdk/python/examples/quickstart/Quickstart.ipynb b/sdk/python/examples/quickstart/Quickstart.ipynb index 5b9d6f3e8b9..9fd1680ea9c 100644 --- a/sdk/python/examples/quickstart/Quickstart.ipynb +++ b/sdk/python/examples/quickstart/Quickstart.ipynb @@ -13,7 +13,7 @@ "\n", "from feast.sdk.resources.entity import Entity\n", "from feast.sdk.resources.storage import Storage\n", - "from feast.sdk.resources.feature import Feature, Datastore, ValueType, Granularity\n", + "from feast.sdk.resources.feature import Feature, Datastore, ValueType\n", "from feast.sdk.resources.feature_set import FeatureSet, FileType\n", "import feast.specs.FeatureSpec_pb2 as feature_pb\n", "\n", diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index a8d146be89b..59b4ddd9f6f 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -1,17 +1,16 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Main interface for users to interact with the Core API. """ @@ -77,9 +76,10 @@ def core_url(self): if self._core_url is None: self._core_url = os.getenv(FEAST_CORE_URL_ENV_KEY) if self._core_url is None: - raise ValueError("Core API URL not set. Either set the " + - "environment variable {} or set it explicitly." - .format(FEAST_CORE_URL_ENV_KEY)) + raise ValueError( + "Core API URL not set. Either set the " + + "environment variable {} or set it explicitly.".format( + FEAST_CORE_URL_ENV_KEY)) return self._core_url @core_url.setter @@ -91,9 +91,10 @@ def serving_url(self): if self._serving_url is None: self._serving_url = os.getenv(FEAST_SERVING_URL_ENV_KEY) if self._serving_url is None: - raise ValueError("Serving API URL not set. Either set the " + - "environment variable {} or set it explicitly." - .format(FEAST_SERVING_URL_ENV_KEY)) + raise ValueError( + "Serving API URL not set. Either set the " + + "environment variable {} or set it explicitly.".format( + FEAST_SERVING_URL_ENV_KEY)) return self._serving_url @serving_url.setter @@ -127,8 +128,11 @@ def apply(self, obj): else: return self._apply(obj) - def run(self, importer, name_override=None, - apply_entity=False, apply_features=False): + def run(self, + importer, + name_override=None, + apply_entity=False, + apply_features=False): """ Run an import job Args: @@ -143,8 +147,7 @@ def run(self, importer, name_override=None, (str) job ID of the import job """ request = JobServiceTypes.SubmitImportJobRequest( - importSpec=importer.spec - ) + importSpec=importer.spec) if name_override is not None: request.name = name_override @@ -155,18 +158,22 @@ def run(self, importer, name_override=None, self._apply_feature(importer.features[feature]) if importer.require_staging: - print("Staging file to remote path {}" - .format(importer.remote_path)) + print("Staging file to remote path {}".format( + importer.remote_path)) importer.stage() - print("Submitting job with spec:\n {}" - .format(spec_to_yaml(importer.spec))) + print("Submitting job with spec:\n {}".format( + spec_to_yaml(importer.spec))) self._connect_core() response = self._job_service_stub.SubmitJob(request) print("Submitted job with id: {}".format(response.jobId)) return response.jobId - def create_dataset(self, feature_set, start_date, end_date, - limit=None, name_prefix=None): + def create_dataset(self, + feature_set, + start_date, + end_date, + limit=None, + name_prefix=None): """ Create training dataset for a feature set. The training dataset will be bounded by event timestamp between start_date and end_date. @@ -187,16 +194,15 @@ def create_dataset(self, feature_set, start_date, end_date, feast.resources.feature_set.DatasetInfo: DatasetInfo containing the information of training dataset """ - self._check_create_dataset_args(feature_set, start_date, - end_date, limit) + self._check_create_dataset_args(feature_set, start_date, end_date, + limit) req = DatasetServiceTypes.CreateDatasetRequest( featureSet=feature_set.proto, startDate=_timestamp_from_datetime(_parse_date(start_date)), endDate=_timestamp_from_datetime(_parse_date(end_date)), limit=limit, - namePrefix=name_prefix - ) + namePrefix=name_prefix) if self.verbose: print("creating training dataset for features: " + str(feature_set.features)) @@ -238,10 +244,14 @@ def get_serving_data(self, feature_set, entity_keys, ts_range=None): request = self._build_serving_request(feature_set, entity_keys) self._connect_serving() - return self._response_to_df(feature_set, self._serving_service_stub - .QueryFeatures(request), start, end) - - def download_dataset(self, dataset_info, dest, staging_location, + return self._response_to_df( + feature_set, self._serving_service_stub.QueryFeatures(request), + start, end) + + def download_dataset(self, + dataset_info, + dest, + staging_location, file_type=FileType.CSV): """ Download training dataset as file @@ -257,10 +267,7 @@ def download_dataset(self, dataset_info, dest, staging_location, str: path to the downloaded file """ return self._table_downloader.download_table_as_file( - dataset_info.table_id, - dest, - staging_location, - file_type) + dataset_info.table_id, dest, staging_location, file_type) def download_dataset_to_df(self, dataset_info, staging_location): """ @@ -275,8 +282,7 @@ def download_dataset_to_df(self, dataset_info, staging_location): """ return self._table_downloader.download_table_as_df( - dataset_info.table_id, - staging_location) + dataset_info.table_id, staging_location) def close(self): """ @@ -304,9 +310,10 @@ def _connect_serving(self): def _build_serving_request(self, feature_set, entity_keys): """Helper function to build serving service request.""" - return QueryFeaturesRequest(entityName=feature_set.entity, - entityId=entity_keys, - featureId=feature_set.features) + return QueryFeaturesRequest( + entityName=feature_set.entity, + entityId=entity_keys, + featureId=feature_set.features) def _response_to_df(self, feature_set, response, start=None, end=None): is_filter_time = start is not None and end is not None @@ -316,8 +323,7 @@ def _response_to_df(self, feature_set, response, start=None, end=None): row = {response.entityName: entity_id} for feature_id in feature_map: v = feature_map[feature_id].value - if is_filter_time and not _is_granularity_none( - feature_id): + if is_filter_time: ts = feature_map[feature_id].timestamp.ToDatetime() if ts < start or ts > end: continue @@ -353,9 +359,9 @@ def _apply_feature(self, feature): """ self._connect_core() response = self._core_service_stub.ApplyFeature(feature.spec) - if self.verbose: print( - "Successfully applied feature with id: {}\n---\n{}" - .format(response.featureId, feature)) + if self.verbose: + print("Successfully applied feature with id: {}\n---\n{}".format( + response.featureId, feature)) return response.featureId def _apply_entity(self, entity): @@ -367,8 +373,8 @@ def _apply_entity(self, entity): self._connect_core() response = self._core_service_stub.ApplyEntity(entity.spec) if self.verbose: - print("Successfully applied entity with name: {}\n---\n{}" - .format(response.entityName, entity)) + print("Successfully applied entity with name: {}\n---\n{}".format( + response.entityName, entity)) return response.entityName def _apply_feature_group(self, feature_group): @@ -381,9 +387,9 @@ def _apply_feature_group(self, feature_group): self._connect_core() response = self._core_service_stub.ApplyFeatureGroup( feature_group.spec) - if self.verbose: print("Successfully applied feature group with id: " + - "{}\n---\n{}".format(response.featureGroupId, - feature_group)) + if self.verbose: + print("Successfully applied feature group with id: " + + "{}\n---\n{}".format(response.featureGroupId, feature_group)) return response.featureGroupId def _apply_storage(self, storage): @@ -394,12 +400,13 @@ def _apply_storage(self, storage): """ self._connect_core() response = self._core_service_stub.ApplyStorage(storage.spec) - if self.verbose: print("Successfully applied storage with id: " + - "{}\n{}".format(response.storageId, storage)) + if self.verbose: + print("Successfully applied storage with id: " + + "{}\n{}".format(response.storageId, storage)) return response.storageId - def _check_create_dataset_args(self, feature_set, start_date, - end_date, limit): + def _check_create_dataset_args(self, feature_set, start_date, end_date, + limit): if len(feature_set.features) < 1: raise ValueError("feature set is empty") @@ -431,7 +438,3 @@ def _timestamp_from_datetime(dt): ts = Timestamp() ts.FromDatetime(dt) return ts - - -def _is_granularity_none(feature_id): - return feature_id.split(".")[1] == "none" diff --git a/sdk/python/feast/sdk/env.py b/sdk/python/feast/sdk/env.py index b9ff4470f7b..28c012aba7f 100644 --- a/sdk/python/feast/sdk/env.py +++ b/sdk/python/feast/sdk/env.py @@ -1,17 +1,16 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" diff --git a/sdk/python/feast/sdk/importer.py b/sdk/python/feast/sdk/importer.py index f9a97aad31b..d8444be3ab6 100644 --- a/sdk/python/feast/sdk/importer.py +++ b/sdk/python/feast/sdk/importer.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -73,9 +73,17 @@ def entity(self): return self._specs.get("entity") @classmethod - def from_csv(cls, path, entity, granularity, owner, staging_location=None, - id_column=None, feature_columns=None, timestamp_column=None, - timestamp_value=None, serving_store=None, warehouse_store=None, + def from_csv(cls, + path, + entity, + owner, + staging_location=None, + id_column=None, + feature_columns=None, + timestamp_column=None, + timestamp_value=None, + serving_store=None, + warehouse_store=None, job_options={}): """Creates an importer from a given csv dataset. This file can be either local or remote (in gcs). If it's a local file @@ -84,7 +92,6 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, Args: path (str): path to csv file entity (str): entity id - granularity (Granularity): granularity of data owner (str): owner staging_location (str, optional): Defaults to None. Staging location for ingesting a local csv file. @@ -115,7 +122,7 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, else: df = pd.read_csv(path) schema, features = \ - _detect_schema_and_feature(entity, granularity, owner, id_column, + _detect_schema_and_feature(entity, owner, id_column, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) @@ -129,9 +136,17 @@ def from_csv(cls, path, entity, granularity, owner, staging_location=None, return cls(specs, df, props) @classmethod - def from_bq(cls, bq_path, entity, granularity, owner, limit=10, - id_column=None, feature_columns=None, timestamp_column=None, - timestamp_value=None, serving_store=None, warehouse_store=None, + def from_bq(cls, + bq_path, + entity, + owner, + limit=10, + id_column=None, + feature_columns=None, + timestamp_column=None, + timestamp_value=None, + serving_store=None, + warehouse_store=None, job_options={}): """Creates an importer from a given bigquery table. @@ -139,7 +154,6 @@ def from_bq(cls, bq_path, entity, granularity, owner, limit=10, bq_path (str): path to bigquery table, in the format project.dataset.table entity (str): entity id - granularity (Granularity): granularity of data owner (str): owner limit (int, optional): Defaults to 10. The maximum number of rows to read into the importer df. @@ -175,21 +189,29 @@ def from_bq(cls, bq_path, entity, granularity, owner, limit=10, } df = head(cli, table, limit) schema, features = \ - _detect_schema_and_feature(entity, granularity, owner, id_column, + _detect_schema_and_feature(entity, owner, id_column, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) - iport_spec = _create_import("bigquery", source_options, - job_options, entity, schema) + iport_spec = _create_import("bigquery", source_options, job_options, + entity, schema) props = _properties("bigquery", table.num_rows, False, None) specs = _specs(iport_spec, Entity(name=entity), features) return cls(specs, df, props) @classmethod - def from_df(cls, df, entity, granularity, owner, staging_location, - id_column=None, feature_columns=None, timestamp_column=None, - timestamp_value=None, serving_store=None, warehouse_store=None, + def from_df(cls, + df, + entity, + owner, + staging_location, + id_column=None, + feature_columns=None, + timestamp_column=None, + timestamp_value=None, + serving_store=None, + warehouse_store=None, job_options={}): """Creates an importer from a given pandas dataframe. To import a file from a dataframe, the data will have to be staged. @@ -197,7 +219,6 @@ def from_df(cls, df, entity, granularity, owner, staging_location, Args: path (str): path to csv file entity (str): entity id - granularity (Granularity): granularity of data owner (str): owner staging_location (str): Defaults to None. Staging location for ingesting a local csv file. @@ -219,20 +240,20 @@ def from_df(cls, df, entity, granularity, owner, staging_location, Returns: Importer: the importer for the dataset provided. """ - tmp_file_name = ("tmp_{}_{}.csv" - .format(entity, int(round(time.time() * 1000)))) - source_options = { - "format": "csv" - } - source_options["path"], require_staging = ( - _get_remote_location(tmp_file_name, staging_location)) + tmp_file_name = ("tmp_{}_{}.csv".format(entity, + int(round( + time.time() * 1000)))) + src_type = "file.csv" + source_options = {} + source_options["path"], require_staging = (_get_remote_location( + tmp_file_name, staging_location)) schema, features = \ - _detect_schema_and_feature(entity, granularity, owner, id_column, + _detect_schema_and_feature(entity, owner, id_column, feature_columns, timestamp_column, timestamp_value, serving_store, warehouse_store, df) - iport_spec = _create_import("file", source_options, - job_options, entity, schema) + iport_spec = _create_import(src_type, source_options, job_options, + entity, schema) props = _properties("dataframe", len(df.index), require_staging, source_options["path"]) @@ -302,11 +323,7 @@ def _specs(iport, entity, features): [type] -- [description] """ - return { - "import": iport, - "features": features, - "entity": entity - } + return {"import": iport, "features": features, "entity": entity} def _get_remote_location(path, staging_location): @@ -330,16 +347,13 @@ def _get_remote_location(path, staging_location): return staging_location + "/" + filename, True -def _detect_schema_and_feature(entity, granularity, owner, id_column, - feature_columns, timestamp_column, - timestamp_value, serving_store, - warehouse_store, df): +def _detect_schema_and_feature(entity, owner, id_column, feature_columns, + timestamp_column, timestamp_value, + serving_store, warehouse_store, df): """Create schema object for import spec. Args: entity (str): entity name - granularity (feast.types.Granularity_pb2.Granularity): granularity of - the feature id_column (str): column name of entity id timestamp_column (str): column name of timestamp timestamp_value (datetime.datetime): timestamp to apply to all @@ -374,19 +388,19 @@ def _detect_schema_and_feature(entity, granularity, owner, id_column, ts = Timestamp() ts.GetCurrentTime() else: - ts = Timestamp(seconds= - int((timestamp_value - datetime.datetime(1970, 1, 1)) - .total_seconds())) + ts = Timestamp( + seconds=int((timestamp_value - + datetime.datetime(1970, 1, 1)).total_seconds())) schema.timestampValue.CopyFrom(ts) features = {} if feature_columns is not None: - # check if all column exist and create feature accordingly + # check if all column exist and create feature accordingly for column in feature_columns: if column not in df.columns: - raise ValueError("Column with name {} is not found".format(column)) - features[column] = _create_feature(df[column], entity, - granularity, owner, + raise ValueError( + "Column with name {} is not found".format(column)) + features[column] = _create_feature(df[column], entity, owner, serving_store, warehouse_store) else: # get all column except entity id and timestampColumn @@ -394,8 +408,7 @@ def _detect_schema_and_feature(entity, granularity, owner, id_column, _remove_safely(feature_columns, schema.entityIdColumn) _remove_safely(feature_columns, schema.timestampColumn) for column in feature_columns: - features[column] = _create_feature(df[column], entity, - granularity, owner, + features[column] = _create_feature(df[column], entity, owner, serving_store, warehouse_store) for col in df.columns: @@ -411,15 +424,12 @@ def _detect_schema_and_feature(entity, granularity, owner, id_column, return schema, features_dict -def _create_feature(column, entity, granularity, owner, - serving_store, warehouse_store): +def _create_feature(column, entity, owner, serving_store, warehouse_store): """Create Feature object. Args: column (pandas.Series): data column entity (str): entity name - granularity (feast.types.Granularity_pb2.Granularity): granularity of - the feature owner (str): owner of the feature serving_store (feast.sdk.resources.feature.DataStore): Defaults to None. Serving store to write the features in this instance to. @@ -433,7 +443,6 @@ def _create_feature(column, entity, granularity, owner, feature = Feature( name=column.name, entity=entity, - granularity=granularity, owner=owner, value_type=dtype_to_value_type(column.dtype)) if serving_store is not None: diff --git a/sdk/python/feast/sdk/resources/entity.py b/sdk/python/feast/sdk/resources/entity.py index 59560840bfc..ad9b566c8e5 100644 --- a/sdk/python/feast/sdk/resources/entity.py +++ b/sdk/python/feast/sdk/resources/entity.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,8 +34,7 @@ def __init__(self, name="", description="", tags=[]): tags (list[str], optional): defaults to []. list of tags for this entity """ - self.__spec = EntitySpec(name=name, description=description, - tags=tags) + self.__spec = EntitySpec(name=name, description=description, tags=tags) @property def spec(self): @@ -77,20 +76,15 @@ def from_yaml(cls, path): content = yaml.safe_load(file.read()) entity = cls() entity.__spec = Parse( - json.dumps(content), - EntitySpec(), - ignore_unknown_fields=False) + json.dumps(content), EntitySpec(), ignore_unknown_fields=False) return entity - def create_feature(self, name, granularity, value_type, owner, - description): + def create_feature(self, name, value_type, owner, description): """Create a feature related to this entity Args: name (str): feature name - granularity (feast.types.Granularity_pb2.Granularity): granularity - of the feature. e.g.: Granularity.NONE, Granularity.SECOND, etc - value_type (feast.types.ValueType_pb2.ValueType): value type of + value_type (feast.types.ValueType_pb2.ValueType): value type of the feature owner (str): owner of the feature description (str): feature's description @@ -114,4 +108,4 @@ def dump(self, path): """ with open(path, 'w') as file: file.write(str(self)) - print("Saved spec to {}".format(path)) \ No newline at end of file + print("Saved spec to {}".format(path)) diff --git a/sdk/python/feast/sdk/resources/feature.py b/sdk/python/feast/sdk/resources/feature.py index 5a88ae53692..714a1404511 100644 --- a/sdk/python/feast/sdk/resources/feature.py +++ b/sdk/python/feast/sdk/resources/feature.py @@ -1,35 +1,25 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import yaml -import json import enum -from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore -from feast.sdk.utils.print_utils import spec_to_yaml -from feast.types.Granularity_pb2 import Granularity as Granularity_pb2 -from google.protobuf.json_format import Parse +import json +import yaml +from google.protobuf.json_format import Parse -class Granularity(enum.Enum): - """ - Feature's Granularity - """ - NONE = 0 - DAY = 1 - HOUR = 2 - MINUTE = 3 - SECOND = 4 +from feast.sdk.utils.print_utils import spec_to_yaml +from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore class ValueType(enum.Enum): @@ -51,16 +41,24 @@ class Feature: """ Wrapper class for feast feature """ - def __init__(self, name='', entity='', granularity=Granularity.NONE, - owner='', value_type=ValueType.DOUBLE, description='', uri='', - warehouse_store=None, serving_store=None, group='', tags=[], options={}): + + def __init__(self, + name='', + entity='', + owner='', + value_type=ValueType.DOUBLE, + description='', + uri='', + warehouse_store=None, + serving_store=None, + group='', + tags=[], + options={}): """Create feast feature instance. Args: name (str): name of feature, in lower snake case entity (str): entity the feature belongs to, in lower case - granularity (feast.sdk.resources.feature.Granularity): - granularity of the feature owner (str): owner of the feature value_type (feast.sdk.resources.feature.ValueType): defaults to ValueType.DOUBLE. value type of the feature @@ -75,21 +73,27 @@ def __init__(self, name='', entity='', granularity=Granularity.NONE, tags (list[str], optional): tags assigned to the feature options (dic, optional): additional options for the feature """ - id = '.'.join([entity, - Granularity_pb2.Enum.Name(granularity.value), name]).lower() - + id = '{}.{}'.format(entity, name).lower() warehouse_store_spec = None serving_store_spec = None - if (serving_store is not None): + if serving_store is not None: serving_store_spec = serving_store.spec - if (warehouse_store is not None): + if warehouse_store is not None: warehouse_store_spec = warehouse_store.spec - data_stores = DataStores(serving = serving_store_spec, - warehouse = warehouse_store_spec) - self.__spec = FeatureSpec(id=id, granularity=granularity.value, - name=name, entity=entity, owner=owner, dataStores=data_stores, - description=description, uri=uri, valueType=value_type.value, - group=group, tags=tags, options=options) + data_stores = DataStores( + serving=serving_store_spec, warehouse=warehouse_store_spec) + self.__spec = FeatureSpec( + id=id, + name=name, + entity=entity, + owner=owner, + dataStores=data_stores, + description=description, + uri=uri, + valueType=value_type.value, + group=group, + tags=tags, + options=options) @property def spec(self): @@ -107,18 +111,7 @@ def name(self): def name(self, value): self.__spec.name = value id_split = self.id.split('.') - id_split[2] = value - self.__spec.id = '.'.join(id_split) - - @property - def granularity(self): - return Granularity(self.__spec.granularity) - - @granularity.setter - def granularity(self, value): - self.__spec.granularity = value.value - id_split = self.id.split('.') - id_split[1] = Granularity_pb2.Enum.Name(self.__spec.granularity).lower() + id_split[1] = value self.__spec.id = '.'.join(id_split) @property @@ -226,7 +219,7 @@ def from_yaml(cls, path): FeatureSpec(), ignore_unknown_fields=False) return feature - + def __str__(self): """Print the feature in yaml format @@ -249,7 +242,7 @@ def dump(self, path): class Datastore: def __init__(self, id, options={}): - self.__spec = DataStore(id = id, options = options) + self.__spec = DataStore(id=id, options=options) def __str__(self): """Print the datastore in yaml format diff --git a/sdk/python/feast/sdk/resources/feature_group.py b/sdk/python/feast/sdk/resources/feature_group.py index 39d2c2b38ae..3f73c571ca4 100644 --- a/sdk/python/feast/sdk/resources/feature_group.py +++ b/sdk/python/feast/sdk/resources/feature_group.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,6 +25,7 @@ class FeatureGroup(): """ Wrapper class for feast feature group """ + def __init__(self, id, tags=[], warehouse_store=None, serving_store=None): """Create FeatureGroup instance. @@ -43,10 +44,10 @@ def __init__(self, id, tags=[], warehouse_store=None, serving_store=None): serving_store_spec = serving_store.spec if (warehouse_store is not None): warehouse_store_spec = warehouse_store.spec - data_stores = DataStores(serving=serving_store_spec, - warehouse = warehouse_store_spec) - self.__spec = FeatureGroupSpec(id=id, tags=tags, - dataStores=data_stores) + data_stores = DataStores( + serving=serving_store_spec, warehouse=warehouse_store_spec) + self.__spec = FeatureGroupSpec( + id=id, tags=tags, dataStores=data_stores) @property def spec(self): @@ -55,11 +56,11 @@ def spec(self): @property def id(self): return self.__spec.id - + @id.setter def id(self, value): self.__spec.id = value - + @property def warehouse_store(self): return self.__spec.dataStores.warehouse @@ -75,7 +76,7 @@ def serving_store(self): @serving_store.setter def serving_store(self, value): self.__spec.dataStores.warehouse.CopyFrom(value) - + @property def tags(self): return self.__spec.tags diff --git a/sdk/python/feast/sdk/resources/feature_set.py b/sdk/python/feast/sdk/resources/feature_set.py index b39979b1554..1895f06c48b 100644 --- a/sdk/python/feast/sdk/resources/feature_set.py +++ b/sdk/python/feast/sdk/resources/feature_set.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/sdk/python/feast/sdk/resources/storage.py b/sdk/python/feast/sdk/resources/storage.py index 942d1a477ef..601aad52061 100644 --- a/sdk/python/feast/sdk/resources/storage.py +++ b/sdk/python/feast/sdk/resources/storage.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,8 @@ class Storage: """ Wrapper class for feast storage """ - def __init__(self, id = "", type = "", options={}): + + def __init__(self, id="", type="", options={}): """Create Storage instance. Args: @@ -32,7 +33,7 @@ def __init__(self, id = "", type = "", options={}): type (str): storage type options (dict, optional) : map of storage options """ - self.__spec = StorageSpec(id = id, type = type, options = options) + self.__spec = StorageSpec(id=id, type=type, options=options) @property def spec(self): diff --git a/sdk/python/feast/sdk/utils/bq_util.py b/sdk/python/feast/sdk/utils/bq_util.py index 535d56a14cc..8cbc4b090bd 100644 --- a/sdk/python/feast/sdk/utils/bq_util.py +++ b/sdk/python/feast/sdk/utils/bq_util.py @@ -1,18 +1,17 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os import time @@ -106,15 +105,14 @@ def download_table_as_file(self, table_id, dest, staging_location, if not is_gs_path(staging_location): raise ValueError("staging_uri must be a directory in GCS") - temp_file_name = 'temp_{}'.format( - int(round(time.time() * 1000))) + temp_file_name = 'temp_{}'.format(int(round(time.time() * 1000))) staging_file_path = os.path.join(staging_location, temp_file_name) job_config = ExtractJobConfig() job_config.destination_format = file_type src_table = Table.from_string(table_id) - job = self.bq.extract_table(src_table, staging_file_path, - job_config=job_config) + job = self.bq.extract_table( + src_table, staging_file_path, job_config=job_config) # await completion job.result() @@ -139,15 +137,15 @@ def download_table_as_df(self, table_id, staging_location): if not is_gs_path(staging_location): raise ValueError("staging_uri must be a directory in GCS") - temp_file_name = 'temp_{}'.format( - int(round(time.time() * 1000))) + temp_file_name = 'temp_{}'.format(int(round(time.time() * 1000))) staging_file_path = os.path.join(staging_location, temp_file_name) job_config = ExtractJobConfig() job_config.destination_format = DestinationFormat.CSV - job = self.bq.extract_table(Table.from_string(table_id), - staging_file_path, - job_config=job_config) + job = self.bq.extract_table( + Table.from_string(table_id), + staging_file_path, + job_config=job_config) # await completion job.result() diff --git a/sdk/python/feast/sdk/utils/gs_utils.py b/sdk/python/feast/sdk/utils/gs_utils.py index 4a37fb416de..2561a115a84 100644 --- a/sdk/python/feast/sdk/utils/gs_utils.py +++ b/sdk/python/feast/sdk/utils/gs_utils.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/sdk/python/feast/sdk/utils/print_utils.py b/sdk/python/feast/sdk/utils/print_utils.py index 8399f2fb52c..926e4eeac53 100644 --- a/sdk/python/feast/sdk/utils/print_utils.py +++ b/sdk/python/feast/sdk/utils/print_utils.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,4 +39,4 @@ def _dict_representer(dumper, data): def _dict_constructor(loader, node): - return OrderedDict(loader.construct_pairs(node)) \ No newline at end of file + return OrderedDict(loader.construct_pairs(node)) diff --git a/sdk/python/feast/sdk/utils/types.py b/sdk/python/feast/sdk/utils/types.py index 3f441a94fdc..cf72425c257 100644 --- a/sdk/python/feast/sdk/utils/types.py +++ b/sdk/python/feast/sdk/utils/types.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/sdk/python/feast/specs/FeatureSpec_pb2.py b/sdk/python/feast/specs/FeatureSpec_pb2.py index 28b10a527cc..ce50ca533a9 100644 --- a/sdk/python/feast/specs/FeatureSpec_pb2.py +++ b/sdk/python/feast/specs/FeatureSpec_pb2.py @@ -14,7 +14,6 @@ from feast.specs import EntitySpec_pb2 as feast_dot_specs_dot_EntitySpec__pb2 from feast.specs import StorageSpec_pb2 as feast_dot_specs_dot_StorageSpec__pb2 -from feast.types import Granularity_pb2 as feast_dot_types_dot_Granularity__pb2 from feast.types import Value_pb2 as feast_dot_types_dot_Value__pb2 @@ -23,9 +22,9 @@ package='feast.specs', syntax='proto3', serialized_options=_b('\n\013feast.specsB\020FeatureSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specs'), - serialized_pb=_b('\n\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\x1a\x1d\x66\x65\x61st/types/Granularity.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\xfe\x02\n\x0b\x46\x65\x61tureSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05owner\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x0b\n\x03uri\x18\x05 \x01(\t\x12\x32\n\x0bgranularity\x18\x06 \x01(\x0e\x32\x1d.feast.types.Granularity.Enum\x12.\n\tvalueType\x18\x07 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x0e\n\x06\x65ntity\x18\x08 \x01(\t\x12\r\n\x05group\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12\x36\n\x07options\x18\x0b \x03(\x0b\x32%.feast.specs.FeatureSpec.OptionsEntry\x12+\n\ndataStores\x18\x0c \x01(\x0b\x32\x17.feast.specs.DataStores\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n\nDataStores\x12\'\n\x07serving\x18\x01 \x01(\x0b\x32\x16.feast.specs.DataStore\x12)\n\twarehouse\x18\x02 \x01(\x0b\x32\x16.feast.specs.DataStore\"}\n\tDataStore\x12\n\n\x02id\x18\x01 \x01(\t\x12\x34\n\x07options\x18\x02 \x03(\x0b\x32#.feast.specs.DataStore.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42W\n\x0b\x66\x65\x61st.specsB\x10\x46\x65\x61tureSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') + serialized_pb=_b('\n\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\xca\x02\n\x0b\x46\x65\x61tureSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05owner\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x0b\n\x03uri\x18\x05 \x01(\t\x12.\n\tvalueType\x18\x07 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x0e\n\x06\x65ntity\x18\x08 \x01(\t\x12\r\n\x05group\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12\x36\n\x07options\x18\x0b \x03(\x0b\x32%.feast.specs.FeatureSpec.OptionsEntry\x12+\n\ndataStores\x18\x0c \x01(\x0b\x32\x17.feast.specs.DataStores\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n\nDataStores\x12\'\n\x07serving\x18\x01 \x01(\x0b\x32\x16.feast.specs.DataStore\x12)\n\twarehouse\x18\x02 \x01(\x0b\x32\x16.feast.specs.DataStore\"}\n\tDataStore\x12\n\n\x02id\x18\x01 \x01(\t\x12\x34\n\x07options\x18\x02 \x03(\x0b\x32#.feast.specs.DataStore.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42W\n\x0b\x66\x65\x61st.specsB\x10\x46\x65\x61tureSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') , - dependencies=[feast_dot_specs_dot_EntitySpec__pb2.DESCRIPTOR,feast_dot_specs_dot_StorageSpec__pb2.DESCRIPTOR,feast_dot_types_dot_Granularity__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) + dependencies=[feast_dot_specs_dot_EntitySpec__pb2.DESCRIPTOR,feast_dot_specs_dot_StorageSpec__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) @@ -63,8 +62,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=500, - serialized_end=546, + serialized_start=417, + serialized_end=463, ) _FEATURESPEC = _descriptor.Descriptor( @@ -110,49 +109,42 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='granularity', full_name='feast.specs.FeatureSpec.granularity', index=5, - number=6, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='valueType', full_name='feast.specs.FeatureSpec.valueType', index=6, + name='valueType', full_name='feast.specs.FeatureSpec.valueType', index=5, number=7, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='entity', full_name='feast.specs.FeatureSpec.entity', index=7, + name='entity', full_name='feast.specs.FeatureSpec.entity', index=6, number=8, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='group', full_name='feast.specs.FeatureSpec.group', index=8, + name='group', full_name='feast.specs.FeatureSpec.group', index=7, number=9, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='tags', full_name='feast.specs.FeatureSpec.tags', index=9, + name='tags', full_name='feast.specs.FeatureSpec.tags', index=8, number=10, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='options', full_name='feast.specs.FeatureSpec.options', index=10, + name='options', full_name='feast.specs.FeatureSpec.options', index=9, number=11, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='dataStores', full_name='feast.specs.FeatureSpec.dataStores', index=11, + name='dataStores', full_name='feast.specs.FeatureSpec.dataStores', index=10, number=12, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -170,8 +162,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=164, - serialized_end=546, + serialized_start=133, + serialized_end=463, ) @@ -208,8 +200,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=548, - serialized_end=644, + serialized_start=465, + serialized_end=561, ) @@ -246,8 +238,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=500, - serialized_end=546, + serialized_start=417, + serialized_end=463, ) _DATASTORE = _descriptor.Descriptor( @@ -283,12 +275,11 @@ extension_ranges=[], oneofs=[ ], - serialized_start=646, - serialized_end=771, + serialized_start=563, + serialized_end=688, ) _FEATURESPEC_OPTIONSENTRY.containing_type = _FEATURESPEC -_FEATURESPEC.fields_by_name['granularity'].enum_type = feast_dot_types_dot_Granularity__pb2._GRANULARITY_ENUM _FEATURESPEC.fields_by_name['valueType'].enum_type = feast_dot_types_dot_Value__pb2._VALUETYPE_ENUM _FEATURESPEC.fields_by_name['options'].message_type = _FEATURESPEC_OPTIONSENTRY _FEATURESPEC.fields_by_name['dataStores'].message_type = _DATASTORES diff --git a/sdk/python/feast/specs/ImportJobSpecs_pb2.py b/sdk/python/feast/specs/ImportJobSpecs_pb2.py new file mode 100644 index 00000000000..da84644da8c --- /dev/null +++ b/sdk/python/feast/specs/ImportJobSpecs_pb2.py @@ -0,0 +1,123 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: feast/specs/ImportJobSpecs.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from feast.specs import ImportSpec_pb2 as feast_dot_specs_dot_ImportSpec__pb2 +from feast.specs import EntitySpec_pb2 as feast_dot_specs_dot_EntitySpec__pb2 +from feast.specs import FeatureSpec_pb2 as feast_dot_specs_dot_FeatureSpec__pb2 +from feast.specs import StorageSpec_pb2 as feast_dot_specs_dot_StorageSpec__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='feast/specs/ImportJobSpecs.proto', + package='feast.specs', + syntax='proto3', + serialized_options=_b('\n\013feast.specsB\023ImportJobSpecsProtoZ6github.com/gojek/feast/protos/generated/go/feast/specs'), + serialized_pb=_b('\n feast/specs/ImportJobSpecs.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/ImportSpec.proto\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\"\xcf\x02\n\x0eImportJobSpecs\x12\r\n\x05jobId\x18\x01 \x01(\t\x12+\n\nimportSpec\x18\x02 \x01(\x0b\x32\x17.feast.specs.ImportSpec\x12,\n\x0b\x65ntitySpecs\x18\x03 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x12.\n\x0c\x66\x65\x61tureSpecs\x18\x04 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x12\x35\n\x13servingStorageSpecs\x18\x05 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x12\x37\n\x15warehouseStorageSpecs\x18\x06 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x12\x33\n\x11\x65rrorsStorageSpec\x18\x07 \x01(\x0b\x32\x18.feast.specs.StorageSpecBZ\n\x0b\x66\x65\x61st.specsB\x13ImportJobSpecsProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') + , + dependencies=[feast_dot_specs_dot_ImportSpec__pb2.DESCRIPTOR,feast_dot_specs_dot_EntitySpec__pb2.DESCRIPTOR,feast_dot_specs_dot_FeatureSpec__pb2.DESCRIPTOR,feast_dot_specs_dot_StorageSpec__pb2.DESCRIPTOR,]) + + + + +_IMPORTJOBSPECS = _descriptor.Descriptor( + name='ImportJobSpecs', + full_name='feast.specs.ImportJobSpecs', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='jobId', full_name='feast.specs.ImportJobSpecs.jobId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='importSpec', full_name='feast.specs.ImportJobSpecs.importSpec', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='entitySpecs', full_name='feast.specs.ImportJobSpecs.entitySpecs', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='featureSpecs', full_name='feast.specs.ImportJobSpecs.featureSpecs', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='servingStorageSpecs', full_name='feast.specs.ImportJobSpecs.servingStorageSpecs', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='warehouseStorageSpecs', full_name='feast.specs.ImportJobSpecs.warehouseStorageSpecs', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='errorsStorageSpec', full_name='feast.specs.ImportJobSpecs.errorsStorageSpec', index=6, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=172, + serialized_end=507, +) + +_IMPORTJOBSPECS.fields_by_name['importSpec'].message_type = feast_dot_specs_dot_ImportSpec__pb2._IMPORTSPEC +_IMPORTJOBSPECS.fields_by_name['entitySpecs'].message_type = feast_dot_specs_dot_EntitySpec__pb2._ENTITYSPEC +_IMPORTJOBSPECS.fields_by_name['featureSpecs'].message_type = feast_dot_specs_dot_FeatureSpec__pb2._FEATURESPEC +_IMPORTJOBSPECS.fields_by_name['servingStorageSpecs'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC +_IMPORTJOBSPECS.fields_by_name['warehouseStorageSpecs'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC +_IMPORTJOBSPECS.fields_by_name['errorsStorageSpec'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC +DESCRIPTOR.message_types_by_name['ImportJobSpecs'] = _IMPORTJOBSPECS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ImportJobSpecs = _reflection.GeneratedProtocolMessageType('ImportJobSpecs', (_message.Message,), dict( + DESCRIPTOR = _IMPORTJOBSPECS, + __module__ = 'feast.specs.ImportJobSpecs_pb2' + # @@protoc_insertion_point(class_scope:feast.specs.ImportJobSpecs) + )) +_sym_db.RegisterMessage(ImportJobSpecs) + + +DESCRIPTOR._options = None +# @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/types/FeatureRow_pb2.py b/sdk/python/feast/types/FeatureRow_pb2.py index c9ed5a29bc0..4096955df82 100644 --- a/sdk/python/feast/types/FeatureRow_pb2.py +++ b/sdk/python/feast/types/FeatureRow_pb2.py @@ -14,7 +14,6 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from feast.types import Feature_pb2 as feast_dot_types_dot_Feature__pb2 -from feast.types import Granularity_pb2 as feast_dot_types_dot_Granularity__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -22,9 +21,9 @@ package='feast.types', syntax='proto3', serialized_options=_b('\n\013feast.typesB\017FeatureRowProtoZ6github.com/gojek/feast/protos/generated/go/feast/types'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/types/FeatureRow.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/types/Feature.proto\x1a\x1d\x66\x65\x61st/types/Granularity.proto\"\x9e\x01\n\rFeatureRowKey\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\t\x12\x32\n\x0bgranularity\x18\x05 \x01(\x0e\x32\x1d.feast.types.Granularity.Enum\"\xc3\x01\n\nFeatureRow\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12&\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x14.feast.types.Feature\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\t\x12\x32\n\x0bgranularity\x18\x05 \x01(\x0e\x32\x1d.feast.types.Granularity.EnumBV\n\x0b\x66\x65\x61st.typesB\x0f\x46\x65\x61tureRowProtoZ6github.com/gojek/feast/protos/generated/go/feast/typesb\x06proto3') + serialized_pb=_b('\n\x1c\x66\x65\x61st/types/FeatureRow.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/types/Feature.proto\"j\n\rFeatureRowKey\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\t\"\x8f\x01\n\nFeatureRow\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12&\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x14.feast.types.Feature\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\tBV\n\x0b\x66\x65\x61st.typesB\x0f\x46\x65\x61tureRowProtoZ6github.com/gojek/feast/protos/generated/go/feast/typesb\x06proto3') , - dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Feature__pb2.DESCRIPTOR,feast_dot_types_dot_Granularity__pb2.DESCRIPTOR,]) + dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Feature__pb2.DESCRIPTOR,]) @@ -57,13 +56,6 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='granularity', full_name='feast.types.FeatureRowKey.granularity', index=3, - number=5, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -76,8 +68,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=137, - serialized_end=295, + serialized_start=105, + serialized_end=211, ) @@ -116,13 +108,6 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='granularity', full_name='feast.types.FeatureRow.granularity', index=4, - number=5, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -135,15 +120,13 @@ extension_ranges=[], oneofs=[ ], - serialized_start=298, - serialized_end=493, + serialized_start=214, + serialized_end=357, ) _FEATUREROWKEY.fields_by_name['eventTimestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -_FEATUREROWKEY.fields_by_name['granularity'].enum_type = feast_dot_types_dot_Granularity__pb2._GRANULARITY_ENUM _FEATUREROW.fields_by_name['features'].message_type = feast_dot_types_dot_Feature__pb2._FEATURE _FEATUREROW.fields_by_name['eventTimestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -_FEATUREROW.fields_by_name['granularity'].enum_type = feast_dot_types_dot_Granularity__pb2._GRANULARITY_ENUM DESCRIPTOR.message_types_by_name['FeatureRowKey'] = _FEATUREROWKEY DESCRIPTOR.message_types_by_name['FeatureRow'] = _FEATUREROW _sym_db.RegisterFileDescriptor(DESCRIPTOR) diff --git a/sdk/python/feast/types/Granularity_pb2.py b/sdk/python/feast/types/Granularity_pb2.py deleted file mode 100644 index 5ccb51cff4e..00000000000 --- a/sdk/python/feast/types/Granularity_pb2.py +++ /dev/null @@ -1,99 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: feast/types/Granularity.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='feast/types/Granularity.proto', - package='feast.types', - syntax='proto3', - serialized_options=_b('\n\013feast.typesB\020GranularityProtoZ6github.com/gojek/feast/protos/generated/go/feast/types'), - serialized_pb=_b('\n\x1d\x66\x65\x61st/types/Granularity.proto\x12\x0b\x66\x65\x61st.types\"J\n\x0bGranularity\";\n\x04\x45num\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03\x44\x41Y\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\n\n\x06MINUTE\x10\x03\x12\n\n\x06SECOND\x10\x04\x42W\n\x0b\x66\x65\x61st.typesB\x10GranularityProtoZ6github.com/gojek/feast/protos/generated/go/feast/typesb\x06proto3') -) - - - -_GRANULARITY_ENUM = _descriptor.EnumDescriptor( - name='Enum', - full_name='feast.types.Granularity.Enum', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='NONE', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DAY', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HOUR', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MINUTE', index=3, number=3, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SECOND', index=4, number=4, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=61, - serialized_end=120, -) -_sym_db.RegisterEnumDescriptor(_GRANULARITY_ENUM) - - -_GRANULARITY = _descriptor.Descriptor( - name='Granularity', - full_name='feast.types.Granularity', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _GRANULARITY_ENUM, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=46, - serialized_end=120, -) - -_GRANULARITY_ENUM.containing_type = _GRANULARITY -DESCRIPTOR.message_types_by_name['Granularity'] = _GRANULARITY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Granularity = _reflection.GeneratedProtocolMessageType('Granularity', (_message.Message,), dict( - DESCRIPTOR = _GRANULARITY, - __module__ = 'feast.types.Granularity_pb2' - # @@protoc_insertion_point(class_scope:feast.types.Granularity) - )) -_sym_db.RegisterMessage(Granularity) - - -DESCRIPTOR._options = None -# @@protoc_insertion_point(module_scope) diff --git a/sdk/python/tests/sample/valid_feature.yaml b/sdk/python/tests/sample/valid_feature.yaml index ffde1ee1946..a54915641bc 100644 --- a/sdk/python/tests/sample/valid_feature.yaml +++ b/sdk/python/tests/sample/valid_feature.yaml @@ -1,10 +1,9 @@ -id: myentity.none.feature_bool_redis1 +id: myentity.feature_bool_redis1 name: feature_bool_redis1 entity: myentity owner: bob@example.com description: test entity. valueType: BOOL -granularity: NONE uri: https://github.com/bob/example dataStores: serving: diff --git a/sdk/python/tests/sdk/resources/test_entity.py b/sdk/python/tests/sdk/resources/test_entity.py index 6bf9d70de44..65a10caa1da 100644 --- a/sdk/python/tests/sdk/resources/test_entity.py +++ b/sdk/python/tests/sdk/resources/test_entity.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +18,12 @@ class TestEntity(object): def test_read_from_yaml(self): - entity_no_tag = Entity.from_yaml("tests/sample/valid_entity_no_tag.yaml") + entity_no_tag = Entity.from_yaml( + "tests/sample/valid_entity_no_tag.yaml") assert entity_no_tag.name == "myentity" assert entity_no_tag.description == "test entity without tag" assert len(entity_no_tag.tags) == 0 - + entity = Entity.from_yaml("tests/sample/valid_entity.yaml") assert entity.name == "myentity" assert entity.description == "test entity with tag" @@ -39,4 +40,4 @@ def test_dump(self): assert t1 == t2 #cleanup - os.remove("myentity.yaml") \ No newline at end of file + os.remove("myentity.yaml") diff --git a/sdk/python/tests/sdk/resources/test_feature.py b/sdk/python/tests/sdk/resources/test_feature.py index a942c805f1e..7f296db2357 100644 --- a/sdk/python/tests/sdk/resources/test_feature.py +++ b/sdk/python/tests/sdk/resources/test_feature.py @@ -1,28 +1,33 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from feast.sdk.resources.feature import Feature, Datastore, Granularity, \ - ValueType +from feast.sdk.resources.feature import Feature, Datastore, ValueType class TestFeature(object): def dummy_feature(self): - warehouse_data_store = Datastore(id = "BIGQUERY1", options = {}) - serving_data_store = Datastore(id = "REDIS1", options = {}) - my_feature = Feature(name = "my_feature", entity = "my_entity", granularity = Granularity.NONE, value_type = ValueType.BYTES, - owner = "feast@web.com", description = "test feature", uri = "github.com/feature_repo", warehouse_store = warehouse_data_store, - serving_store = serving_data_store) + warehouse_data_store = Datastore(id="BIGQUERY1", options={}) + serving_data_store = Datastore(id="REDIS1", options={}) + my_feature = Feature( + name="my_feature", + entity="my_entity", + value_type=ValueType.BYTES, + owner="feast@web.com", + description="test feature", + uri="github.com/feature_repo", + warehouse_store=warehouse_data_store, + serving_store=serving_data_store) return my_feature def test_set_name(self): @@ -30,30 +35,23 @@ def test_set_name(self): new_name = "my_feature_new" my_feature.name = new_name assert my_feature.name == new_name - assert my_feature.id == "my_entity.none.my_feature_new" - - def test_set_granularity(self): - my_feature = self.dummy_feature() - my_feature.granularity = Granularity.DAY - assert my_feature.granularity == Granularity.DAY - assert my_feature.id == "my_entity.day.my_feature" - + assert my_feature.id == "my_entity.my_feature_new" + def test_set_entity(self): my_feature = self.dummy_feature() new_entity = "new_entity" my_feature.entity = new_entity assert my_feature.entity == new_entity - assert my_feature.id == "new_entity.none.my_feature" - + assert my_feature.id == "new_entity.my_feature" + def test_read_from_yaml(self): feature = Feature.from_yaml("tests/sample/valid_feature.yaml") - assert feature.id == "myentity.none.feature_bool_redis1" + assert feature.id == "myentity.feature_bool_redis1" assert feature.name == "feature_bool_redis1" assert feature.entity == "myentity" assert feature.owner == "bob@example.com" assert feature.description == "test entity." assert feature.value_type == ValueType.BOOL - assert feature.granularity == Granularity.NONE assert feature.uri == "https://github.com/bob/example" assert feature.serving_store.id == "REDIS1" assert feature.warehouse_store.id == "BIGQUERY1" diff --git a/sdk/python/tests/sdk/resources/test_feature_group.py b/sdk/python/tests/sdk/resources/test_feature_group.py index 1e431bf1356..244211d1924 100644 --- a/sdk/python/tests/sdk/resources/test_feature_group.py +++ b/sdk/python/tests/sdk/resources/test_feature_group.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,11 +14,12 @@ from feast.sdk.resources.feature_group import FeatureGroup + class TestFeatureGroupSpec(object): - def test_read_from_yaml(self): + def test_read_from_yaml(self): feature_group = FeatureGroup.from_yaml( "tests/sample/valid_feature_group.yaml") assert feature_group.id == "my_fg" assert feature_group.serving_store.id == "REDIS1" assert feature_group.warehouse_store.id == "BIGQUERY1" - assert feature_group.tags == ["tag1", "tag2"] \ No newline at end of file + assert feature_group.tags == ["tag1", "tag2"] diff --git a/sdk/python/tests/sdk/resources/test_feature_set.py b/sdk/python/tests/sdk/resources/test_feature_set.py index ddb66e38c7d..3c6c97eba2b 100644 --- a/sdk/python/tests/sdk/resources/test_feature_set.py +++ b/sdk/python/tests/sdk/resources/test_feature_set.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +19,7 @@ class TestFeatureSet(object): def test_features(self): entity_name = "driver" - features = ["driver.hour.feature1", "driver.hour.feature2"] + features = ["driver.feature1", "driver.feature2"] feature_set = FeatureSet(entity_name, features) assert len(feature_set.features) == 2 @@ -28,9 +28,10 @@ def test_features(self): def test_different_entity(self): entity_name = "driver" - features = ["customer.hour.feature1", "driver.day.feature1"] - with pytest.raises(ValueError, - match="feature set has different entity: customer"): + features = ["customer.feature1", "driver.feature1"] + with pytest.raises( + ValueError, + match="feature set has different entity: customer"): FeatureSet(entity_name, features) @@ -41,4 +42,3 @@ def test_creation(self): dataset = DatasetInfo(name, table_id) assert dataset.name == name assert dataset.table_id == table_id - diff --git a/sdk/python/tests/sdk/resources/test_storage.py b/sdk/python/tests/sdk/resources/test_storage.py index c9e6e9358ae..75b9d1a8f8a 100644 --- a/sdk/python/tests/sdk/resources/test_storage.py +++ b/sdk/python/tests/sdk/resources/test_storage.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,7 @@ class TestStorage(object): def test_update_options(self): storage = Storage(id="storage1", type="redis") assert storage.options == {} - myDict = {'key':'value'} + myDict = {'key': 'value'} storage.options = myDict assert storage.options == myDict @@ -28,16 +28,18 @@ def test_from_yaml(self): storage = Storage.from_yaml("tests/sample/valid_storage.yaml") assert storage.id == "BIGQUERY1" assert storage.type == "bigquery" - expDict = {"dataset" : "feast", - "project" : "gcp-project", - "tempLocation" : "gs://feast-storage"} + expDict = { + "dataset": "feast", + "project": "gcp-project", + "tempLocation": "gs://feast-storage" + } assert storage.options == expDict def test_dump(self): - opt = {"option1" : "value1", "option2" : "value2"} + opt = {"option1": "value1", "option2": "value2"} storage = Storage("storage1", "redis", opt) - storage.dump("storage.yaml") + storage.dump("storage.yaml") storage2 = Storage.from_yaml("storage.yaml") assert storage.id == storage2.id @@ -46,4 +48,4 @@ def test_dump(self): assert storage2.options == opt #cleanup - os.remove("storage.yaml") \ No newline at end of file + os.remove("storage.yaml") diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index c13c350ed43..f0d789afd07 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,24 +15,24 @@ from datetime import datetime import grpc +import numpy as np import pandas as pd import pytest -import numpy as np from google.protobuf.timestamp_pb2 import Timestamp from pandas.util.testing import assert_frame_equal import feast.core.CoreService_pb2_grpc as core -import feast.core.JobService_pb2_grpc as jobs import feast.core.DatasetService_pb2_grpc as training +import feast.core.JobService_pb2_grpc as jobs import feast.serving.Serving_pb2 as serving_pb from feast.core.CoreService_pb2 import CoreServiceTypes -from feast.core.JobService_pb2 import JobServiceTypes -from feast.core.DatasetService_pb2 import DatasetServiceTypes from feast.core.DatasetService_pb2 import DatasetInfo as DatasetInfo_pb +from feast.core.DatasetService_pb2 import DatasetServiceTypes +from feast.core.JobService_pb2 import JobServiceTypes from feast.sdk.client import Client, _parse_date, _timestamp_from_datetime from feast.sdk.importer import Importer from feast.sdk.resources.entity import Entity -from feast.sdk.resources.feature import Feature, Granularity +from feast.sdk.resources.feature import Feature from feast.sdk.resources.feature_group import FeatureGroup from feast.sdk.resources.feature_set import FeatureSet, DatasetInfo, FileType from feast.sdk.resources.storage import Storage @@ -54,24 +54,27 @@ def client(self, mocker): return cli def test_apply_single_feature(self, client, mocker): - my_feature = Feature(name="test", - entity="test", granularity=Granularity.NONE) + my_feature = Feature(name="test", entity="test") grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - with mocker.patch.object(grpc_stub, 'ApplyFeature', - return_value=CoreServiceTypes.ApplyFeatureResponse( - featureId="test.none.test")): + with mocker.patch.object( + grpc_stub, + 'ApplyFeature', + return_value=CoreServiceTypes.ApplyFeatureResponse( + featureId="test.test")): client._core_service_stub = grpc_stub id = client.apply(my_feature) - assert id == "test.none.test" + assert id == "test.test" def test_apply_single_entity(self, client, mocker): my_entity = Entity(name="test") grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - with mocker.patch.object(grpc_stub, 'ApplyEntity', - return_value=CoreServiceTypes.ApplyEntityResponse( - entityName="test")): + with mocker.patch.object( + grpc_stub, + 'ApplyEntity', + return_value=CoreServiceTypes.ApplyEntityResponse( + entityName="test")): client._core_service_stub = grpc_stub name = client.apply(my_entity) assert name == "test" @@ -80,9 +83,11 @@ def test_apply_single_feature_group(self, client, mocker): my_feature_group = FeatureGroup(id="test") grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - with mocker.patch.object(grpc_stub, 'ApplyFeatureGroup', - return_value=CoreServiceTypes.ApplyFeatureGroupResponse( - featureGroupId="test")): + with mocker.patch.object( + grpc_stub, + 'ApplyFeatureGroup', + return_value=CoreServiceTypes.ApplyFeatureGroupResponse( + featureGroupId="test")): client._core_service_stub = grpc_stub name = client.apply(my_feature_group) assert name == "test" @@ -91,9 +96,11 @@ def test_apply_single_storage(self, client, mocker): my_storage = Storage(id="TEST", type="redis") grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - with mocker.patch.object(grpc_stub, 'ApplyStorage', - return_value=CoreServiceTypes.ApplyStorageResponse( - storageId="TEST")): + with mocker.patch.object( + grpc_stub, + 'ApplyStorage', + return_value=CoreServiceTypes.ApplyStorageResponse( + storageId="TEST")): client._core_service_stub = grpc_stub name = client.apply(my_storage) assert name == "TEST" @@ -111,15 +118,21 @@ def test_apply_multiple(self, client, mocker): grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - mocker.patch.object(grpc_stub, 'ApplyStorage', - return_value=CoreServiceTypes.ApplyStorageResponse( - storageId="TEST")) - mocker.patch.object(grpc_stub, 'ApplyFeatureGroup', - return_value=CoreServiceTypes.ApplyFeatureGroupResponse( - featureGroupId="test")) - mocker.patch.object(grpc_stub, 'ApplyEntity', - return_value=CoreServiceTypes.ApplyEntityResponse( - entityName="test")) + mocker.patch.object( + grpc_stub, + 'ApplyStorage', + return_value=CoreServiceTypes.ApplyStorageResponse( + storageId="TEST")) + mocker.patch.object( + grpc_stub, + 'ApplyFeatureGroup', + return_value=CoreServiceTypes.ApplyFeatureGroupResponse( + featureGroupId="test")) + mocker.patch.object( + grpc_stub, + 'ApplyEntity', + return_value=CoreServiceTypes.ApplyEntityResponse( + entityName="test")) client._core_service_stub = grpc_stub ids = client.apply([my_storage, my_entity, my_feature_group]) @@ -128,59 +141,58 @@ def test_apply_multiple(self, client, mocker): def test_run_job_no_staging(self, client, mocker): grpc_stub = jobs.JobServiceStub(grpc.insecure_channel("")) - mocker.patch.object(grpc_stub, 'SubmitJob', - return_value=JobServiceTypes.SubmitImportJobResponse( - jobId="myjob12312")) + mocker.patch.object( + grpc_stub, + 'SubmitJob', + return_value=JobServiceTypes.SubmitImportJobResponse( + jobId="myjob12312")) client._job_service_stub = grpc_stub - importer = Importer( - {"import": ImportSpec()}, - None, - {"require_staging": False}) + importer = Importer({"import": ImportSpec()}, None, + {"require_staging": False}) job_id = client.run(importer) assert job_id == "myjob12312" def test_create_dataset_invalid_args(self, client): - feature_set = FeatureSet("entity", ["entity.none.feature1"]) + feature_set = FeatureSet("entity", ["entity.feature1"]) # empty feature set with pytest.raises(ValueError, match="feature set is empty"): inv_feature_set = FeatureSet("entity", []) - client.create_dataset(inv_feature_set, "2018-12-01", - "2018-12-02") + client.create_dataset(inv_feature_set, "2018-12-01", "2018-12-02") # invalid start date - with pytest.raises(ValueError, - match="Incorrect date format, should be YYYY-MM-DD"): - client.create_dataset(feature_set, "20181201", - "2018-12-02") + with pytest.raises( + ValueError, + match="Incorrect date format, should be YYYY-MM-DD"): + client.create_dataset(feature_set, "20181201", "2018-12-02") # invalid end date - with pytest.raises(ValueError, - match="Incorrect date format, should be YYYY-MM-DD"): - client.create_dataset(feature_set, "2018-12-01", - "20181202") + with pytest.raises( + ValueError, + match="Incorrect date format, should be YYYY-MM-DD"): + client.create_dataset(feature_set, "2018-12-01", "20181202") # start date > end date with pytest.raises(ValueError, match="end_date is before start_date"): - client.create_dataset(feature_set, "2018-12-02", - "2018-12-01") + client.create_dataset(feature_set, "2018-12-02", "2018-12-01") # invalid limit - with pytest.raises(ValueError, - match="limit is not a positive integer"): - client.create_dataset(feature_set, "2018-12-01", - "2018-12-02", -1) + with pytest.raises( + ValueError, match="limit is not a positive integer"): + client.create_dataset(feature_set, "2018-12-01", "2018-12-02", -1) def test_create_dataset(self, client, mocker): entity_name = "myentity" - feature_ids = ["myentity.none.feature1", "myentity.second.feature2"] + feature_ids = ["myentity.feature1", "myentity.feature2"] fs = FeatureSet(entity_name, feature_ids) start_date = "2018-01-02" end_date = "2018-12-31" - ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + ds_pb = DatasetInfo_pb( + name="dataset_name", tableUrl="project.dataset.table") mock_trn_stub = training.DatasetServiceStub(grpc.insecure_channel("")) - mocker.patch.object(mock_trn_stub, "CreateDataset", - return_value=DatasetServiceTypes - .CreateDatasetResponse(datasetInfo=ds_pb)) + mocker.patch.object( + mock_trn_stub, + "CreateDataset", + return_value=DatasetServiceTypes.CreateDatasetResponse( + datasetInfo=ds_pb)) client._dataset_service_stub = mock_trn_stub ds = client.create_dataset(fs, start_date, end_date) @@ -193,29 +205,28 @@ def test_create_dataset(self, client, mocker): startDate=_timestamp_from_datetime(_parse_date(start_date)), endDate=_timestamp_from_datetime(_parse_date(end_date)), limit=None, - namePrefix=None - ) - ) + namePrefix=None)) def test_create_dataset_with_limit(self, client, mocker): entity_name = "myentity" - feature_ids = ["myentity.none.feature1", "myentity.second.feature2"] + feature_ids = ["myentity.feature1", "myentity.feature2"] fs = FeatureSet(entity_name, feature_ids) start_date = "2018-01-02" end_date = "2018-12-31" limit = 100 - ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + ds_pb = DatasetInfo_pb( + name="dataset_name", tableUrl="project.dataset.table") mock_trn_stub = training.DatasetServiceStub(grpc.insecure_channel("")) - mocker.patch.object(mock_trn_stub, "CreateDataset", - return_value=DatasetServiceTypes - .CreateDatasetResponse(datasetInfo=ds_pb)) + mocker.patch.object( + mock_trn_stub, + "CreateDataset", + return_value=DatasetServiceTypes.CreateDatasetResponse( + datasetInfo=ds_pb)) client._dataset_service_stub = mock_trn_stub - ds = client.create_dataset(fs, start_date, end_date, - limit=limit) + ds = client.create_dataset(fs, start_date, end_date, limit=limit) assert "dataset_name" == ds.name assert "project.dataset.table" == ds.table_id @@ -225,32 +236,31 @@ def test_create_dataset_with_limit(self, client, mocker): startDate=_timestamp_from_datetime(_parse_date(start_date)), endDate=_timestamp_from_datetime(_parse_date(end_date)), limit=limit, - namePrefix=None - ) - ) + namePrefix=None)) def test_create_dataset_with_name_prefix(self, client, mocker): entity_name = "myentity" - feature_ids = ["myentity.none.feature1", "myentity.second.feature2"] + feature_ids = ["myentity.feature1", "myentity.feature2"] fs = FeatureSet(entity_name, feature_ids) start_date = "2018-01-02" end_date = "2018-12-31" limit = 100 name_prefix = "feast" - ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + ds_pb = DatasetInfo_pb( + name="dataset_name", tableUrl="project.dataset.table") mock_dssvc_stub = training.DatasetServiceStub( grpc.insecure_channel("")) - mocker.patch.object(mock_dssvc_stub, "CreateDataset", - return_value=DatasetServiceTypes - .CreateDatasetResponse(datasetInfo=ds_pb)) + mocker.patch.object( + mock_dssvc_stub, + "CreateDataset", + return_value=DatasetServiceTypes.CreateDatasetResponse( + datasetInfo=ds_pb)) client._dataset_service_stub = mock_dssvc_stub - ds = client.create_dataset(fs, start_date, end_date, - limit=limit, - name_prefix=name_prefix) + ds = client.create_dataset( + fs, start_date, end_date, limit=limit, name_prefix=name_prefix) assert "dataset_name" == ds.name assert "project.dataset.table" == ds.table_id @@ -260,28 +270,31 @@ def test_create_dataset_with_name_prefix(self, client, mocker): startDate=_timestamp_from_datetime(_parse_date(start_date)), endDate=_timestamp_from_datetime(_parse_date(end_date)), limit=limit, - namePrefix=name_prefix - ) - ) + namePrefix=name_prefix)) def test_build_serving_request(self, client): - feature_set = FeatureSet("entity", - ["entity.none.feat1", "entity.none.feat2"]) + feature_set = FeatureSet("entity", ["entity.feat1", "entity.feat2"]) req = client._build_serving_request(feature_set, ["1", "2", "3"]) - expected = QueryFeaturesRequest(entityName="entity", - entityId=["1", "2", "3"], - featureId=feature_set.features) + expected = QueryFeaturesRequest( + entityName="entity", + entityId=["1", "2", "3"], + featureId=feature_set.features) assert req == expected def test_serving_response_to_df(self, client): response = self._create_query_features_response( entity_name="entity", - entities={"1": {"entity.feat1": (1, Timestamp(seconds=1)), - "entity.feat2": (2, Timestamp(seconds=2))}, - "2": {"entity.feat1": (3, Timestamp(seconds=3)), - "entity.feat2": (4, Timestamp(seconds=4))}} - ) + entities={ + "1": { + "entity.feat1": (1, Timestamp(seconds=1)), + "entity.feat2": (2, Timestamp(seconds=2)) + }, + "2": { + "entity.feat1": (3, Timestamp(seconds=3)), + "entity.feat2": (4, Timestamp(seconds=4)) + } + }) expected_df = pd.DataFrame({'entity': ["1", "2"], 'entity.feat1': [1, 3], 'entity.feat2': [2, 4]}) \ @@ -291,18 +304,25 @@ def test_serving_response_to_df(self, client): response) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df, - check_dtype=False, - check_column_type=False, - check_index_type=False) + assert_frame_equal( + df, + expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_response_to_df_with_missing_value(self, client): response = self._create_query_features_response( entity_name="entity", - entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, - "2": {"entity.feat1": (3, Timestamp(seconds=3)), - "entity.feat2": (4, Timestamp(seconds=4))}} - ) + entities={ + "1": { + "entity.feat1": (1, Timestamp(seconds=1)) + }, + "2": { + "entity.feat1": (3, Timestamp(seconds=3)), + "entity.feat2": (4, Timestamp(seconds=4)) + } + }) expected_df = pd.DataFrame({'entity': ["1", "2"], 'entity.feat1': [1, 3], 'entity.feat2': [np.NaN, 4]}) \ @@ -312,17 +332,24 @@ def test_serving_response_to_df_with_missing_value(self, client): response) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df, - check_dtype=False, - check_column_type=False, - check_index_type=False) + assert_frame_equal( + df, + expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_response_to_df_with_missing_feature(self, client): response = self._create_query_features_response( entity_name="entity", - entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, - "2": {"entity.feat1": (3, Timestamp(seconds=3))}} - ) + entities={ + "1": { + "entity.feat1": (1, Timestamp(seconds=1)) + }, + "2": { + "entity.feat1": (3, Timestamp(seconds=3)) + } + }) expected_df = pd.DataFrame({'entity': ["1", "2"], 'entity.feat1': [1, 3], 'entity.feat2': [np.NaN, np.NaN]}) \ @@ -332,30 +359,37 @@ def test_serving_response_to_df_with_missing_feature(self, client): response) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df, - check_dtype=False, - check_column_type=False, - check_index_type=False) + assert_frame_equal( + df, + expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_response_to_df_no_data(self, client): response = QueryFeaturesResponse(entityName="entity") - expected_df = pd.DataFrame(columns= - ['entity', 'entity.day.feat1', - 'entity.day.feat2']) - df = client._response_to_df(FeatureSet("entity", - ["entity.day.feat1", - "entity.day.feat2"]), response) - assert_frame_equal(df, expected_df, - check_dtype=False, - check_column_type=False, - check_index_type=False) + expected_df = pd.DataFrame( + columns=['entity', 'entity.feat1', 'entity.feat2']) + df = client._response_to_df( + FeatureSet("entity", ["entity.feat1", "entity.feat2"]), response) + assert_frame_equal( + df, + expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_response_to_df_with_time_filter(self, client): response = self._create_query_features_response( entity_name="entity", - entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, - "2": {"entity.feat1": (3, Timestamp(seconds=3))}} - ) + entities={ + "1": { + "entity.feat1": (1, Timestamp(seconds=1)) + }, + "2": { + "entity.feat1": (3, Timestamp(seconds=3)) + } + }) expected_df = pd.DataFrame({'entity': ["1", "2"], 'entity.feat1': [np.NaN, 3], 'entity.feat2': [np.NaN, np.NaN]}) \ @@ -367,68 +401,45 @@ def test_serving_response_to_df_with_time_filter(self, client): response, start, end) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - print(df) - assert_frame_equal(df, expected_df, - check_dtype=False, - check_column_type=False, - check_index_type=False) - - def test_serving_response_to_df_with_time_filter_granularity_none(self, - client): - response = self._create_query_features_response( - entity_name="entity", - entities={"1": {"entity.none.feat1": (1, Timestamp(seconds=1))}, - "2": {"entity.none.feat1": (3, Timestamp(seconds=3))}} - ) - expected_df = pd.DataFrame({'entity': ["1", "2"], - 'entity.none.feat1': [1, 3], - 'entity.none.feat2': [np.NaN, np.NaN]}) \ - .reset_index(drop=True) - start = datetime.utcfromtimestamp(2) - end = datetime.utcfromtimestamp(5) - df = client._response_to_df(FeatureSet("entity", ["entity.none.feat1", - "entity.none.feat2"]), - response, start, end) \ - .sort_values(['entity']) \ - .reset_index(drop=True)[expected_df.columns] - print(df) - assert_frame_equal(df, expected_df, - check_dtype=False, - check_column_type=False, - check_index_type=False) + assert_frame_equal( + df, + expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_invalid_type(self, client): start = "2018-01-01T01:01:01" end = "2018-01-01T01:01:01" ts_range = [start, end] - with pytest.raises(TypeError, match="start and end must be datetime " - "type"): - client.get_serving_data(FeatureSet("entity", ["entity.none.feat1", - "entity.none.feat2"]), - ["1234", "5678"], - ts_range) + with pytest.raises( + TypeError, match="start and end must be datetime " + "type"): + client.get_serving_data( + FeatureSet("entity", ["entity.feat1", "entity.feat2"]), + ["1234", "5678"], ts_range) def test_download_dataset_as_file(self, client, mocker): destination = "/tmp/dest_file" table_dlder = TableDownloader() - mocker.patch.object(table_dlder, "download_table_as_file", - return_value=destination) + mocker.patch.object( + table_dlder, "download_table_as_file", return_value=destination) client._table_downloader = table_dlder table_id = "project.dataset.table" staging_location = "gs://gcs_bucket/" dataset = DatasetInfo("mydataset", table_id) - result = client.download_dataset(dataset, destination, - staging_location=staging_location, - file_type=FileType.CSV) + result = client.download_dataset( + dataset, + destination, + staging_location=staging_location, + file_type=FileType.CSV) assert result == destination - table_dlder.download_table_as_file.assert_called_once_with(table_id, - destination, - staging_location, - FileType.CSV) + table_dlder.download_table_as_file.assert_called_once_with( + table_id, destination, staging_location, FileType.CSV) def _create_query_features_response(self, entity_name, entities): response = QueryFeaturesResponse(entityName=entity_name) @@ -446,9 +457,13 @@ def _create_query_features_response(self, entity_name, entities): def _create_feature_spec(self, feature_id, wh_id): wh_store = DataStore(id=wh_id) datastores = DataStores(warehouse=wh_store) - return FeatureSpec(id=feature_id, - dataStores=datastores) + return FeatureSpec(id=feature_id, dataStores=datastores) def _create_bq_spec(self, id, project, dataset): - return StorageSpec(id=id, type="bigquery", options={ - "project": project, "dataset": dataset}) + return StorageSpec( + id=id, + type="bigquery", + options={ + "project": project, + "dataset": dataset + }) diff --git a/sdk/python/tests/sdk/test_importer.py b/sdk/python/tests/sdk/test_importer.py index 7bf1c05c3f5..b1049831e73 100644 --- a/sdk/python/tests/sdk/test_importer.py +++ b/sdk/python/tests/sdk/test_importer.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,124 +15,132 @@ import pandas as pd import pytest import ntpath -from feast.sdk.resources.feature import Feature, Granularity, ValueType, \ +from feast.sdk.resources.feature import Feature, ValueType, \ Datastore from feast.sdk.importer import _create_feature, Importer from feast.sdk.utils.gs_utils import is_gs_path -from feast.types.Granularity_pb2 import Granularity as Granularity_pb2 class TestImporter(object): def test_from_csv(self): csv_path = "tests/data/driver_features.csv" entity_name = "driver" - feature_granularity = Granularity.DAY owner = "owner@feast.com" staging_location = "gs://test-bucket" id_column = "driver_id" - feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + feature_columns = [ + "avg_distance_completed", "avg_customer_distance_completed" + ] timestamp_column = "ts" - importer = Importer.from_csv(path=csv_path, - entity=entity_name, - granularity=feature_granularity, - owner=owner, - staging_location=staging_location, - id_column=id_column, - feature_columns=feature_columns, - timestamp_column=timestamp_column) + importer = Importer.from_csv( + path=csv_path, + entity=entity_name, + owner=owner, + staging_location=staging_location, + id_column=id_column, + feature_columns=feature_columns, + timestamp_column=timestamp_column) self._validate_csv_importer(importer, csv_path, entity_name, - feature_granularity, owner, - staging_location, id_column, + owner, staging_location, id_column, feature_columns, timestamp_column) def test_from_csv_id_column_not_specified(self): - with pytest.raises(ValueError, - match="Column with name driver is not found"): - feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + with pytest.raises( + ValueError, match="Column with name driver is not found"): + feature_columns = [ + "avg_distance_completed", "avg_customer_distance_completed" + ] csv_path = "tests/data/driver_features.csv" - Importer.from_csv(path=csv_path, - entity="driver", - granularity=Granularity.DAY, - owner="owner@feast.com", - staging_location="gs://test-bucket", - feature_columns=feature_columns, - timestamp_column="ts") + Importer.from_csv( + path=csv_path, + entity="driver", + owner="owner@feast.com", + staging_location="gs://test-bucket", + feature_columns=feature_columns, + timestamp_column="ts") def test_from_csv_timestamp_column_not_specified(self): - feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed", - "avg_distance_cancelled"] + feature_columns = [ + "avg_distance_completed", "avg_customer_distance_completed", + "avg_distance_cancelled" + ] csv_path = "tests/data/driver_features.csv" entity_name = "driver" - granularity = Granularity.DAY owner = "owner@feast.com" staging_location = "gs://test-bucket" id_column = "driver_id" - importer = Importer.from_csv(path=csv_path, - entity=entity_name, - granularity=granularity, - owner=owner, - staging_location=staging_location, - id_column=id_column, - feature_columns=feature_columns) + importer = Importer.from_csv( + path=csv_path, + entity=entity_name, + owner=owner, + staging_location=staging_location, + id_column=id_column, + feature_columns=feature_columns) - self._validate_csv_importer(importer, csv_path, entity_name, - granularity, owner, - staging_location=staging_location, - id_column=id_column, - feature_columns=feature_columns) + self._validate_csv_importer( + importer, + csv_path, + entity_name, + owner, + staging_location=staging_location, + id_column=id_column, + feature_columns=feature_columns) def test_from_csv_feature_columns_not_specified(self): csv_path = "tests/data/driver_features.csv" entity_name = "driver" - granularity = Granularity.DAY owner = "owner@feast.com" staging_location = "gs://test-bucket" id_column = "driver_id" timestamp_column = "ts" - importer = Importer.from_csv(path=csv_path, - entity=entity_name, - granularity=granularity, - owner=owner, - staging_location=staging_location, - id_column=id_column, - timestamp_column=timestamp_column) + importer = Importer.from_csv( + path=csv_path, + entity=entity_name, + owner=owner, + staging_location=staging_location, + id_column=id_column, + timestamp_column=timestamp_column) - self._validate_csv_importer(importer, csv_path, entity_name, - granularity, owner, - staging_location=staging_location, - id_column=id_column, - timestamp_column=timestamp_column) + self._validate_csv_importer( + importer, + csv_path, + entity_name, + owner, + staging_location=staging_location, + id_column=id_column, + timestamp_column=timestamp_column) def test_from_csv_staging_location_not_specified(self): - with pytest.raises(ValueError, - match="Specify staging_location for importing local file/dataframe"): - feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + with pytest.raises( + ValueError, + match= + "Specify staging_location for importing local file/dataframe"): + feature_columns = [ + "avg_distance_completed", "avg_customer_distance_completed" + ] csv_path = "tests/data/driver_features.csv" - Importer.from_csv(path=csv_path, - entity="driver", - granularity=Granularity.DAY, - owner="owner@feast.com", - feature_columns=feature_columns, - timestamp_column="ts") - - with pytest.raises(ValueError, - match="Staging location must be in GCS") as e_info: - feature_columns = ["avg_distance_completed", - "avg_customer_distance_completed"] + Importer.from_csv( + path=csv_path, + entity="driver", + owner="owner@feast.com", + feature_columns=feature_columns, + timestamp_column="ts") + + with pytest.raises( + ValueError, match="Staging location must be in GCS") as e_info: + feature_columns = [ + "avg_distance_completed", "avg_customer_distance_completed" + ] csv_path = "tests/data/driver_features.csv" - Importer.from_csv(path=csv_path, - entity="driver", - granularity=Granularity.DAY, - owner="owner@feast.com", - staging_location="/home", - feature_columns=feature_columns, - timestamp_column="ts") + Importer.from_csv( + path=csv_path, + entity="driver", + owner="owner@feast.com", + staging_location="/home", + feature_columns=feature_columns, + timestamp_column="ts") def test_from_df(self): csv_path = "tests/data/driver_features.csv" @@ -140,58 +148,61 @@ def test_from_df(self): staging_location = "gs://test-bucket" entity = "driver" - importer = Importer.from_df(df=df, - entity=entity, - granularity=Granularity.DAY, - owner="owner@feast.com", - staging_location=staging_location, - id_column="driver_id", - timestamp_column="ts") + importer = Importer.from_df( + df=df, + entity=entity, + owner="owner@feast.com", + staging_location=staging_location, + id_column="driver_id", + timestamp_column="ts") assert importer.require_staging == True - assert ("{}/tmp_{}".format(staging_location, entity) - in importer.remote_path) + assert ("{}/tmp_{}".format(staging_location, + entity) in importer.remote_path) for feature in importer.features.values(): assert feature.name in df.columns - assert feature.id == "driver.day." + feature.name + assert feature.id == "driver." + feature.name import_spec = importer.spec assert import_spec.type == "file" - assert import_spec.sourceOptions == {"format": "csv", - "path": importer.remote_path} + assert import_spec.sourceOptions == { + "format": "csv", + "path": importer.remote_path + } assert import_spec.entities == ["driver"] schema = import_spec.schema assert schema.entityIdColumn == "driver_id" assert schema.timestampValue is not None - feature_columns = ["completed", "avg_distance_completed", - "avg_customer_distance_completed", - "avg_distance_cancelled"] + feature_columns = [ + "completed", "avg_distance_completed", + "avg_customer_distance_completed", "avg_distance_cancelled" + ] for col, field in zip(df.columns.values, schema.fields): assert col == field.name if col in feature_columns: - assert field.featureId == "driver.day." + col + assert field.featureId == "driver." + col def _validate_csv_importer(self, - importer, csv_path, entity_name, - feature_granularity, owner, - staging_location=None, id_column=None, + importer, + csv_path, + entity_name, + owner, + staging_location=None, + id_column=None, feature_columns=None, - timestamp_column=None, timestamp_value=None): + timestamp_column=None, + timestamp_value=None): df = pd.read_csv(csv_path) assert not importer.require_staging == is_gs_path(csv_path) if importer.require_staging: - assert importer.remote_path == "{}/{}".format(staging_location, - ntpath.basename( - csv_path)) + assert importer.remote_path == "{}/{}".format( + staging_location, ntpath.basename(csv_path)) # check features created for feature in importer.features.values(): assert feature.name in df.columns - assert feature.id == "{}.{}.{}".format(entity_name, - Granularity_pb2.Enum.Name( - feature_granularity.value).lower(), - feature.name) + assert feature.id == "{}.{}".format(entity_name, feature.name) import_spec = importer.spec assert import_spec.type == "file.csv" @@ -215,37 +226,33 @@ def _validate_csv_importer(self, for col, field in zip(df.columns.values, schema.fields): assert col == field.name if col in feature_columns: - assert field.featureId == \ - "{}.{}.{}".format(entity_name, - Granularity_pb2.Enum.Name( - feature_granularity.value).lower(), - col) + assert field.featureId == '{}.{}'.format(entity_name, + col).lower() class TestHelpers: def test_create_feature(self): col = pd.Series([1] * 3, dtype='int32', name="test") - expected = Feature(name="test", - entity="test", - granularity=Granularity.NONE, - owner="person", - value_type=ValueType.INT32) - actual = _create_feature(col, "test", Granularity.NONE, "person", None, - None) + expected = Feature( + name="test", + entity="test", + owner="person", + value_type=ValueType.INT32) + actual = _create_feature(col, "test", "person", None, None) assert actual.id == expected.id assert actual.value_type == expected.value_type assert actual.owner == expected.owner def test_create_feature_with_stores(self): col = pd.Series([1] * 3, dtype='int32', name="test") - expected = Feature(name="test", - entity="test", - granularity=Granularity.NONE, - owner="person", - value_type=ValueType.INT32, - serving_store=Datastore(id="SERVING"), - warehouse_store=Datastore(id="WAREHOUSE")) - actual = _create_feature(col, "test", Granularity.NONE, "person", + expected = Feature( + name="test", + entity="test", + owner="person", + value_type=ValueType.INT32, + serving_store=Datastore(id="SERVING"), + warehouse_store=Datastore(id="WAREHOUSE")) + actual = _create_feature(col, "test", "person", Datastore(id="SERVING"), Datastore(id="WAREHOUSE")) assert actual.id == expected.id diff --git a/sdk/python/tests/sdk/utils/test_bq_utils.py b/sdk/python/tests/sdk/utils/test_bq_utils.py index f45202ee784..206180197cc 100644 --- a/sdk/python/tests/sdk/utils/test_bq_utils.py +++ b/sdk/python/tests/sdk/utils/test_bq_utils.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,19 +25,23 @@ def test_get_table_name(): project_name = "my_project" dataset_name = "my_dataset" - feature_id = "myentity.none.feature1" - storage_spec = StorageSpec(id="BIGQUERY1", type="bigquery", - options={"project": project_name, - "dataset": dataset_name}) + feature_id = "myentity.feature1" + storage_spec = StorageSpec( + id="BIGQUERY1", + type="bigquery", + options={ + "project": project_name, + "dataset": dataset_name + }) assert get_table_name(feature_id, storage_spec) == \ "my_project.my_dataset.myentity_none" def test_get_table_name_not_bq(): - feature_id = "myentity.none.feature1" + feature_id = "myentity.feature1" storage_spec = StorageSpec(id="REDIS1", type="redis") - with pytest.raises(ValueError, - match="storage spec is not BigQuery storage spec"): + with pytest.raises( + ValueError, match="storage spec is not BigQuery storage spec"): get_table_name(feature_id, storage_spec) @@ -45,8 +49,7 @@ class TestTableDownloader(object): def test_download_table_as_df(self, mocker): self._stop_time(mocker) mocked_gcs_to_df = mocker.patch( - "feast.sdk.utils.bq_util.gcs_to_df", - return_value=None) + "feast.sdk.utils.bq_util.gcs_to_df", return_value=None) staging_path = "gs://temp/" staging_file_name = "temp_0" @@ -56,14 +59,13 @@ def test_download_table_as_df(self, mocker): exp_staging_path = os.path.join(staging_path, staging_file_name) table_dldr._bq = _Mock_BQ_Client() - mocker.patch.object(table_dldr._bq, "extract_table", - return_value=_Job()) + mocker.patch.object( + table_dldr._bq, "extract_table", return_value=_Job()) - table_dldr.download_table_as_df(table_id, - staging_location=staging_path) + table_dldr.download_table_as_df( + table_id, staging_location=staging_path) - assert len( - table_dldr._bq.extract_table.call_args_list) == 1 + assert len(table_dldr._bq.extract_table.call_args_list) == 1 args, kwargs = \ table_dldr._bq.extract_table.call_args_list[0] assert args[0].full_table_id == Table.from_string( @@ -87,19 +89,16 @@ def test_download_json(self, mocker): def test_download_invalid_staging_url(self): table_id = "project_id.dataset_id.table_id" table_dldr = TableDownloader() - with pytest.raises(ValueError, - match="staging_uri must be a directory in " - "GCS"): - table_dldr.download_table_as_file(table_id, - "/tmp/dst", - "/local/directory", - FileType.CSV) - - with pytest.raises(ValueError, - match="staging_uri must be a directory in " - "GCS"): - table_dldr.download_table_as_df(table_id, - "/local/directory") + with pytest.raises( + ValueError, match="staging_uri must be a directory in " + "GCS"): + table_dldr.download_table_as_file(table_id, "/tmp/dst", + "/local/directory", FileType.CSV) + + with pytest.raises( + ValueError, match="staging_uri must be a directory in " + "GCS"): + table_dldr.download_table_as_df(table_id, "/local/directory") def _test_download_file(self, mocker, type): staging_path = "gs://temp/" @@ -111,23 +110,21 @@ def _test_download_file(self, mocker, type): mock_blob = _Blob() mocker.patch.object(mock_blob, "download_to_filename") table_dldr._bq = _Mock_BQ_Client() - mocker.patch.object(table_dldr._bq, "extract_table", - return_value=_Job()) + mocker.patch.object( + table_dldr._bq, "extract_table", return_value=_Job()) table_dldr._gcs = _Mock_GCS_Client() - mocker.patch.object(table_dldr._gcs, "get_bucket", - return_value=_Bucket(mock_blob)) + mocker.patch.object( + table_dldr._gcs, "get_bucket", return_value=_Bucket(mock_blob)) - table_dldr.download_table_as_file(table_id, - dst_path, - staging_location=staging_path, - file_type=type) + table_dldr.download_table_as_file( + table_id, dst_path, staging_location=staging_path, file_type=type) exp_staging_path = os.path.join(staging_path, staging_file_name) - assert len( - table_dldr._bq.extract_table.call_args_list) == 1 + assert len(table_dldr._bq.extract_table.call_args_list) == 1 args, kwargs = \ table_dldr._bq.extract_table.call_args_list[0] - assert args[0].full_table_id == Table.from_string(table_id).full_table_id + assert args[0].full_table_id == Table.from_string( + table_id).full_table_id assert args[1] == exp_staging_path assert kwargs['job_config'].destination_format == str(type) @@ -136,14 +133,17 @@ def _test_download_file(self, mocker, type): def _stop_time(self, mocker): mocker.patch('time.time', return_value=0) + class _Mock_BQ_Client: def extract_table(self): pass + class _Mock_GCS_Client: def get_bucket(self): pass + class _Job: def result(self): return None diff --git a/sdk/python/tests/sdk/utils/test_gs_utils.py b/sdk/python/tests/sdk/utils/test_gs_utils.py index 402e6b03550..5e5a2e13bd1 100644 --- a/sdk/python/tests/sdk/utils/test_gs_utils.py +++ b/sdk/python/tests/sdk/utils/test_gs_utils.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,6 +15,7 @@ import pytest from feast.sdk.utils.gs_utils import is_gs_path + def test_is_gs_path(): assert is_gs_path("gs://valid/gs/file.csv") == True - assert is_gs_path("local/path/file.csv") == False \ No newline at end of file + assert is_gs_path("local/path/file.csv") == False diff --git a/sdk/python/tests/sdk/utils/test_types.py b/sdk/python/tests/sdk/utils/test_types.py index edfc741a3de..021e51e9eb8 100644 --- a/sdk/python/tests/sdk/utils/test_types.py +++ b/sdk/python/tests/sdk/utils/test_types.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,23 +14,26 @@ import pytest import pandas as pd -import numpy as np +import numpy as np from feast.sdk.utils.types import dtype_to_value_type, ValueType + def test_convert_dtype_to_value_type(): - dft = pd.DataFrame(dict(A = np.random.rand(3), - B = 1, - C = 'foo', - D = pd.Timestamp('20010102'), - E = pd.Series([1.0]*3).astype('float32'), - F = False, - G = pd.Series([1]*3,dtype='int8'))) - + dft = pd.DataFrame( + dict( + A=np.random.rand(3), + B=1, + C='foo', + D=pd.Timestamp('20010102'), + E=pd.Series([1.0] * 3).astype('float32'), + F=False, + G=pd.Series([1] * 3, dtype='int8'))) + assert dtype_to_value_type(dft['A'].dtype) == ValueType.DOUBLE assert dtype_to_value_type(dft['B'].dtype) == ValueType.INT64 assert dtype_to_value_type(dft['C'].dtype) == ValueType.STRING assert dtype_to_value_type(dft['D'].dtype) == ValueType.TIMESTAMP assert dtype_to_value_type(dft['E'].dtype) == ValueType.FLOAT assert dtype_to_value_type(dft['F'].dtype) == ValueType.BOOL - assert dtype_to_value_type(dft['G'].dtype) == ValueType.INT32 \ No newline at end of file + assert dtype_to_value_type(dft['G'].dtype) == ValueType.INT32 diff --git a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java index a9909858392..379e1b6aa03 100644 --- a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java @@ -140,7 +140,7 @@ private List createGets(Collection entityIds, FeatureSpec featureSp } /** - * Create BigTableRowKey based on entityId, timestamp, and granularity. + * Create BigTableRowKey based on entityId, and timestamp. * * @param entityIdPrefix hash prefix of entity ID. * @param entityId entity ID value diff --git a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java index 3ba9c2b7354..ffdf66fd99e 100644 --- a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java +++ b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java @@ -66,7 +66,7 @@ public void shouldReturnSameEntityNameAsRequest() { QueryFeaturesRequest request = QueryFeaturesRequest.newBuilder() .setEntityName(entityName) - .addFeatureId("driver.day.total_completed_booking") + .addFeatureId("driver.total_completed_booking") .build(); QueryFeaturesResponse response = feast.queryFeatures(request); @@ -79,7 +79,7 @@ public void shouldReturnSameEntityNameAsRequest() { public void shouldPassValidRequestToFeatureRetrievalDispatcher() { String entityName = "driver"; Collection entityIds = Arrays.asList("entity1", "entity2", "entity3"); - Collection featureIds = Arrays.asList("driver.day.total_completed_booking"); + Collection featureIds = Arrays.asList("driver.total_completed_booking"); QueryFeaturesRequest request = QueryFeaturesRequest.newBuilder() .setEntityName(entityName) diff --git a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java index 75c80f8aa28..d7eff6af4d8 100644 --- a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java +++ b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java @@ -55,7 +55,7 @@ public void setUp() throws Exception { .setEntityName("driver") .addAllEntityId(Arrays.asList("driver1", "driver2", "driver3")) .addAllFeatureId( - Arrays.asList("driver.day.completed_booking", "driver.none.last_opportunity")) + Arrays.asList("driver.completed_booking", "driver.last_opportunity")) .build(); Tracer tracer = Configuration.fromEnv("dummy").getTracer(); @@ -102,7 +102,7 @@ public void shouldCallOnErrorIfFeatureIdsIsNotSet() { public void shouldCallOnErrorIfFeatureIdsContainsDifferentEntity() { QueryFeaturesRequest differentEntityReq = QueryFeaturesRequest.newBuilder(validRequest) - .addFeatureId("customer.day.order_made") + .addFeatureId("customer.order_made") .build(); service.queryFeatures(differentEntityReq, mockStreamObserver); diff --git a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java index f8905e6342a..6212a6e29d2 100644 --- a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java +++ b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java @@ -23,8 +23,6 @@ import feast.serving.model.FeatureValue; import feast.serving.testutil.BigTablePopulator; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import java.util.ArrayList; import java.util.Arrays; @@ -66,22 +64,20 @@ public void tearDown() throws Exception { } @Test - public void getFeatures_shouldWorkForGranularityNone() { + public void getFeatures_shouldReturnLastValue() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_1") + .setId("test_entity.feature_1") .setName("feature_1") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.STRING) .build(); FeatureSpec featureSpec2 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_2") + .setId("test_entity.feature_2") .setName("feature_2") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.STRING) .build(); @@ -97,18 +93,16 @@ public void getFeatures_shouldGracefullyHandleMissingEntity() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_1") + .setId("test_entity.feature_1") .setName("feature_granularity_none") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.STRING) .build(); FeatureSpec featureSpec2 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_2") + .setId("test_entity.feature_2") .setName("feature_2") - .setGranularity(Granularity.Enum.NONE) .setValueType(ValueType.Enum.STRING) .build(); @@ -121,41 +115,6 @@ public void getFeatures_shouldGracefullyHandleMissingEntity() { bigTablePopulator.validate(results, entityIds, featureSpecs); } - @Test - public void getFeatures_shouldWorkForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); - - List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); - - List result = featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs); - - bigTablePopulator.validate(result, entityIds, featureSpecs); - } - } - - private FeatureSpec createFeatureSpec( - String featureName, Enum granularity, ValueType.Enum valType) { - String entityName = ENTITY_NAME; - String featureId = - String.format("%s.%s.%s", entityName, granularity.toString().toLowerCase(), featureName); - FeatureSpec spec = - FeatureSpec.newBuilder() - .setEntity(entityName) - .setId(featureId) - .setName(featureName) - .setGranularity(granularity) - .setValueType(valType) - .build(); - - return spec; - } - private List createEntityIds(int count) { List entityIds = new ArrayList<>(); for (int i = 0; i < count; i++) { diff --git a/serving/src/test/java/feast/serving/service/CoreServiceTest.java b/serving/src/test/java/feast/serving/service/CoreServiceTest.java index ef62ea7b25c..cc0551948a5 100644 --- a/serving/src/test/java/feast/serving/service/CoreServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CoreServiceTest.java @@ -17,8 +17,29 @@ package feast.serving.service; +import static org.hamcrest.CoreMatchers.everyItem; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.collection.IsIn.isIn; +import static org.junit.Assert.assertThat; + import com.google.protobuf.ByteString; import com.google.protobuf.Empty; +import feast.core.CoreServiceGrpc.CoreServiceImplBase; +import feast.core.CoreServiceProto.CoreServiceTypes.GetEntitiesRequest; +import feast.core.CoreServiceProto.CoreServiceTypes.GetEntitiesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.GetFeaturesRequest; +import feast.core.CoreServiceProto.CoreServiceTypes.GetFeaturesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.GetStorageRequest; +import feast.core.CoreServiceProto.CoreServiceTypes.GetStorageResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ListEntitiesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ListFeaturesResponse; +import feast.core.CoreServiceProto.CoreServiceTypes.ListStorageResponse; +import feast.serving.exception.SpecRetrievalException; +import feast.specs.EntitySpecProto.EntitySpec; +import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.specs.StorageSpecProto.StorageSpec; +import feast.types.ValueProto.ValueType; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import io.grpc.inprocess.InProcessChannelBuilder; @@ -26,20 +47,6 @@ import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; import io.grpc.util.MutableHandlerRegistry; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import feast.core.CoreServiceGrpc.CoreServiceImplBase; -import feast.core.CoreServiceProto.CoreServiceTypes.*; -import feast.serving.exception.SpecRetrievalException; -import feast.specs.EntitySpecProto.EntitySpec; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity.Enum; -import feast.types.ValueProto.ValueType; - import java.util.Arrays; import java.util.List; import java.util.Map; @@ -47,12 +54,11 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static org.hamcrest.CoreMatchers.everyItem; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.collection.IsIn.isIn; -import static org.junit.Assert.assertThat; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; public class CoreServiceTest { @@ -175,7 +181,7 @@ public void getAllEntitySpecs_shouldThrowExceptionWhenErrorHappens() { @Test public void getFeatureSpecs_shouldSendCorrectRequest() { List featureIds = - Arrays.asList("driver.day.total_accepted_booking", "driver.second.ping_location"); + Arrays.asList("driver.total_accepted_booking", "driver.ping_location"); AtomicReference deliveredRequest = new AtomicReference<>(); CoreServiceImplBase service = new CoreServiceImplBase() { @@ -202,7 +208,7 @@ public void getFeatures( @Test public void getFeatureSpecs_shouldReturnRequestedFeatureSpecs() { List featureIds = - Arrays.asList("driver.day.total_accepted_booking", "driver.second.ping_location"); + Arrays.asList("driver.total_accepted_booking", "driver.ping_location"); AtomicReference deliveredRequest = new AtomicReference<>(); CoreServiceImplBase service = new CoreServiceImplBase() { @@ -232,7 +238,7 @@ public void getFeatureSpecs_shouldThrowSpecsRetrievalExceptionWhenErrorHappen() expectedException.expectCause(instanceOf(StatusRuntimeException.class)); List featureIds = - Arrays.asList("driver.day.total_accepted_booking", "driver.second.ping_location"); + Arrays.asList("driver.total_accepted_booking", "driver.ping_location"); client.getFeatureSpecs(featureIds); } @@ -363,21 +369,19 @@ public void getAllStorageSpecs_shouldThrowSpecsRetrievalExceptionWhenErrorHappen private Map getFakeFeatureSpecs() { FeatureSpec spec1 = FeatureSpec.newBuilder() - .setId("driver.day.total_accepted_booking") + .setId("driver.total_accepted_booking") .setName("total_accepted_booking") .setOwner("dummy@go-jek.com") .setDescription("awesome feature") - .setGranularity(Enum.DAY) .setValueType(ValueType.Enum.STRING) .build(); FeatureSpec spec2 = FeatureSpec.newBuilder() - .setId("driver.second.ping") + .setId("driver.ping") .setName("ping") .setOwner("dummy@go-jek.com") .setDescription("awesome feature") - .setGranularity(Enum.SECOND) .setValueType(ValueType.Enum.INT64) .build(); diff --git a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java index 11b3f8fa87c..2103be849b5 100644 --- a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java +++ b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java @@ -69,7 +69,7 @@ public void shouldUseCurrentThreadIfRequestIsSmallEnough() { .thenReturn(Collections.emptyList()); when(featureStorageRegistry.get(any(String.class))).thenReturn(featureStorage); - String featureId = "entity.none.feature_1"; + String featureId = "entity.feature_1"; FeatureSpec featureSpec = FeatureSpec.newBuilder().setId(featureId).build(); dispatcher.dispatchFeatureRetrieval( entityName, entityIds, Collections.singletonList(featureSpec)); @@ -90,7 +90,7 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { .thenReturn(Collections.emptyList()); String entityName = "entity"; - String featureId1 = "entity.none.feature_1"; + String featureId1 = "entity.feature_1"; FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setId(featureId1) @@ -98,7 +98,7 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - String featureId2 = "entity.none.feature_2"; + String featureId2 = "entity.feature_2"; FeatureSpec featureSpec2 = FeatureSpec.newBuilder() .setId(featureId2) @@ -131,7 +131,7 @@ public void shouldUseExecutorServiceIfRequestFromMoreThanOneStorage() { .thenReturn(Collections.emptyList()); String entityName = "entity"; - String featureId1 = "entity.none.feature_1"; + String featureId1 = "entity.feature_1"; FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setId(featureId1) @@ -139,7 +139,7 @@ public void shouldUseExecutorServiceIfRequestFromMoreThanOneStorage() { DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - String featureId2 = "entity.none.feature_2"; + String featureId2 = "entity.feature_2"; FeatureSpec featureSpec2 = FeatureSpec.newBuilder() .setId(featureId2) diff --git a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java index be70fee48db..2a70b789835 100644 --- a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java +++ b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java @@ -25,8 +25,6 @@ import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; import io.opentracing.util.GlobalTracer; import java.util.ArrayList; @@ -78,17 +76,15 @@ public void setUp() throws Exception { public void getFeatures_shouldNotReturnMissingValue() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() - .setId("entity.none.feature_1") + .setId("entity.feature_1") .setEntity(entityName) - .setGranularity(Enum.NONE) .setValueType(ValueType.Enum.STRING) .build(); FeatureSpec featureSpec2 = FeatureSpec.newBuilder() - .setId("entity.none.feature_2") + .setId("entity.feature_2") .setEntity(entityName) - .setGranularity(Enum.NONE) .setValueType(ValueType.Enum.STRING) .build(); @@ -104,39 +100,32 @@ public void getFeatures_shouldNotReturnMissingValue() { @Test public void getFeatures_shouldReturnLastValue() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = createFeatureSpec("feature_1", Enum.NONE); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); + FeatureSpec spec1 = createFeatureSpec("feature_1"); + FeatureSpec spec2 = createFeatureSpec("feature_2"); + List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, now); + redisPopulator.populate(entityName, entityIds, featureSpecs, now); - List result = redisFs.getFeature(entityName, entityIds, featureSpecs); + List result = redisFs.getFeature(entityName, entityIds, featureSpecs); - redisPopulator.validate(result, entityIds, featureSpecs); - } + redisPopulator.validate(result, entityIds, featureSpecs); } - private FeatureSpec createFeatureSpec(String featureName, Enum granularity) { + private FeatureSpec createFeatureSpec(String featureName) { DataStore servingDatastoreSpec = DataStore.newBuilder().setId("REDIS").build(); - return createFeatureSpec(featureName, granularity, ValueType.Enum.STRING, servingDatastoreSpec); + return createFeatureSpec(featureName, ValueType.Enum.STRING, servingDatastoreSpec); } private FeatureSpec createFeatureSpec( - String featureName, Enum granularity, ValueType.Enum valType, DataStore dataStoreSpec) { + String featureName, ValueType.Enum valType, DataStore dataStoreSpec) { String entityName = "entity"; - String featureId = - String.format("%s.%s.%s", entityName, granularity.toString().toLowerCase(), featureName); + String featureId = String.format("%s.%s", entityName, featureName); FeatureSpec spec = FeatureSpec.newBuilder() .setDataStores(DataStores.newBuilder().setServing(dataStoreSpec)) .setEntity(entityName) .setId(featureId) .setName(featureName) - .setGranularity(granularity) .setValueType(valType) .build(); diff --git a/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java b/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java index 9f7b2292b38..f0040864286 100644 --- a/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java +++ b/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java @@ -100,9 +100,7 @@ private void populateTableWithFakeData( try (Table table = connection.getTable(tableName)) { for (FeatureSpec featureSpec : featureSpecs) { for (String entityId : entityIds) { - Timestamp roundedTimestamp = - TimeUtil.roundFloorTimestamp(timestamp, featureSpec.getGranularity()); - Put put = makePut(entityId, featureSpec, roundedTimestamp); + Put put = makePut(entityId, featureSpec, timestamp); table.put(put); } } diff --git a/serving/src/test/java/feast/serving/testutil/FakeSpecStorage.java b/serving/src/test/java/feast/serving/testutil/FakeSpecStorage.java index 331835dc5eb..3c110cda830 100644 --- a/serving/src/test/java/feast/serving/testutil/FakeSpecStorage.java +++ b/serving/src/test/java/feast/serving/testutil/FakeSpecStorage.java @@ -25,7 +25,6 @@ import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.ValueType; import java.util.Collections; @@ -44,9 +43,9 @@ public class FakeSpecStorage implements SpecStorage { public FakeSpecStorage() { // populate with hardcoded value String bigTableId = "BIGTABLE1"; - String lastOpportunityId = "driver.none.last_opportunity"; + String lastOpportunityId = "driver.last_opportunity"; String lastOpportunityName = "last_opportunity"; - String dailyCompletedBookingId = "driver.day.total_completed_booking"; + String dailyCompletedBookingId = "driver.total_completed_booking"; String dailyCompletedBookingName = "total_completed_booking"; DataStore bigTable = DataStore.newBuilder().setId(bigTableId).build(); @@ -59,7 +58,6 @@ public FakeSpecStorage() { FeatureSpec lastOpportunity = FeatureSpec.newBuilder() - .setGranularity(Granularity.Enum.NONE) .setId(lastOpportunityId) .setName(lastOpportunityName) .setValueType(ValueType.Enum.INT64) @@ -68,7 +66,6 @@ public FakeSpecStorage() { FeatureSpec totalCompleted = FeatureSpec.newBuilder() - .setGranularity(Granularity.Enum.DAY) .setId(dailyCompletedBookingId) .setName(dailyCompletedBookingName) .setValueType(ValueType.Enum.INT64) diff --git a/serving/src/test/java/feast/serving/testutil/RedisPopulator.java b/serving/src/test/java/feast/serving/testutil/RedisPopulator.java index 68340405e6b..21235a9bfa0 100644 --- a/serving/src/test/java/feast/serving/testutil/RedisPopulator.java +++ b/serving/src/test/java/feast/serving/testutil/RedisPopulator.java @@ -40,8 +40,7 @@ public void populate( Timestamp timestamp) { for (FeatureSpec fs : featureSpecs) { for (String entityId : entityIds) { - Timestamp roundedTimestamp = TimeUtil.roundFloorTimestamp(timestamp, fs.getGranularity()); - addData(entityId, fs, roundedTimestamp); + addData(entityId, fs, timestamp); } } } diff --git a/serving/src/test/java/feast/serving/testutil/TimeUtil.java b/serving/src/test/java/feast/serving/testutil/TimeUtil.java deleted file mode 100644 index 77e4edea978..00000000000 --- a/serving/src/test/java/feast/serving/testutil/TimeUtil.java +++ /dev/null @@ -1,60 +0,0 @@ -package feast.serving.testutil; - -import com.google.protobuf.Timestamp; -import feast.types.GranularityProto.Granularity.Enum; -import org.joda.time.DateTime; -import org.joda.time.DateTimeField; -import org.joda.time.DateTimeZone; -import org.joda.time.MutableDateTime; - -public class TimeUtil { - public static final int NANO_IN_MILLI = 1000000; - public static final int NANO_IN_MICRO = 1000; - public static final int MILLI_IN_SECOND = 1000; - - /** - * Round down timestamp to the nearest granularity. - * - * @param timestamp original timestamp. - * @param granularity granularity of the rounded timestamp. - * @return - */ - public static Timestamp roundFloorTimestamp(Timestamp timestamp, Enum granularity) { - MutableDateTime dt = new MutableDateTime(DateTimeZone.UTC); - DateTimeField roundingField; - switch (granularity) { - case DAY: - roundingField = dt.getChronology().dayOfMonth(); - break; - case HOUR: - roundingField = dt.getChronology().hourOfDay(); - break; - case MINUTE: - roundingField = dt.getChronology().minuteOfHour(); - break; - case SECOND: - roundingField = dt.getChronology().secondOfMinute(); - break; - case NONE: - return Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); - default: - throw new RuntimeException("Unrecognised time series granularity"); - } - dt.setMillis(timestamp.getSeconds() * 1000 + timestamp.getNanos() / 1000000); - dt.setRounding(roundingField, MutableDateTime.ROUND_FLOOR); - return dateTimeToTimestamp(dt.toDateTime()); - } - - /** - * Convert {@link DateTime} into {@link Timestamp} - * - * @param dateTime - * @return - */ - private static Timestamp dateTimeToTimestamp(DateTime dateTime) { - return Timestamp.newBuilder() - .setSeconds(dateTime.getMillis() / MILLI_IN_SECOND) - .setNanos((int) (dateTime.getMillis() % MILLI_IN_SECOND) * NANO_IN_MILLI) - .build(); - } -} diff --git a/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java b/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java index e2cf1938765..596f54c1d9c 100644 --- a/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java +++ b/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java @@ -208,7 +208,7 @@ private void validate( private List createFeatureIds(int count) { List featureIds = new ArrayList<>(); for (int i = 0; i < count; i++) { - featureIds.add("entity.none.feature_" + i); + featureIds.add("entity.feature_" + i); } return featureIds; } diff --git a/ui/src/components/Feature/SearchTable.vue b/ui/src/components/Feature/SearchTable.vue index 07a1675fd03..ac82c7ae8ec 100644 --- a/ui/src/components/Feature/SearchTable.vue +++ b/ui/src/components/Feature/SearchTable.vue @@ -14,8 +14,6 @@ {{ column.name }} - @@ -45,11 +43,6 @@ {{ item.spec.entity }} - - - {{ item.spec.granularity }} - - @@ -143,7 +136,6 @@ list: [], columns: [ {name: "Entity", sortable: true, reverse: false}, - {name: "Granularity", sortable: true, reverse: false}, {name: "Name", sortable: true, reverse: false}, {name: "Description", sortable: false}, {name: "Owner", sortable: true, reverse: false}, @@ -157,7 +149,7 @@ empty: false, loading: false, search: [], - searchEntities: ["entity", "granularity", "name", "owner","tags"], + searchEntities: ["entity", "name", "owner","tags"], searchSeparator: ':', selectedFeature: {}, options: [], @@ -208,8 +200,6 @@ if (typeof response.body['features'] === "undefined") { this.empty = true; } else { - // TODO: Handle nulls better. - response.body['features'].forEach(b => filters.fillGranularity(b)); this.list = response.body['features']; this.populate(); } @@ -238,11 +228,7 @@ updateRoute (route) { // TODO: Handle arbitrary search queries if (typeof route['entity'] !== "undefined") { - if (typeof route['granularity'] !== "undefined") { - this.$router.push({path: '/features/' + route['entity'] + '/' + route['granularity'], params: route}); - } else { - this.$router.push({path: '/features/' + route['entity'], params: route}); - } + this.$router.push({path: '/features/' + route['entity'], params: route}); } else { this.$router.push({path: '/features', params: route}); } diff --git a/ui/src/components/Job/SearchTable.vue b/ui/src/components/Job/SearchTable.vue index 07a1675fd03..ac82c7ae8ec 100644 --- a/ui/src/components/Job/SearchTable.vue +++ b/ui/src/components/Job/SearchTable.vue @@ -14,8 +14,6 @@ {{ column.name }} - @@ -45,11 +43,6 @@ {{ item.spec.entity }} - - - {{ item.spec.granularity }} - - @@ -143,7 +136,6 @@ list: [], columns: [ {name: "Entity", sortable: true, reverse: false}, - {name: "Granularity", sortable: true, reverse: false}, {name: "Name", sortable: true, reverse: false}, {name: "Description", sortable: false}, {name: "Owner", sortable: true, reverse: false}, @@ -157,7 +149,7 @@ empty: false, loading: false, search: [], - searchEntities: ["entity", "granularity", "name", "owner","tags"], + searchEntities: ["entity", "name", "owner","tags"], searchSeparator: ':', selectedFeature: {}, options: [], @@ -208,8 +200,6 @@ if (typeof response.body['features'] === "undefined") { this.empty = true; } else { - // TODO: Handle nulls better. - response.body['features'].forEach(b => filters.fillGranularity(b)); this.list = response.body['features']; this.populate(); } @@ -238,11 +228,7 @@ updateRoute (route) { // TODO: Handle arbitrary search queries if (typeof route['entity'] !== "undefined") { - if (typeof route['granularity'] !== "undefined") { - this.$router.push({path: '/features/' + route['entity'] + '/' + route['granularity'], params: route}); - } else { - this.$router.push({path: '/features/' + route['entity'], params: route}); - } + this.$router.push({path: '/features/' + route['entity'], params: route}); } else { this.$router.push({path: '/features', params: route}); } diff --git a/ui/src/filters.js b/ui/src/filters.js index b2a93a66313..f335d77e059 100644 --- a/ui/src/filters.js +++ b/ui/src/filters.js @@ -2,12 +2,6 @@ export default { formatTimestamp (timestamp) { let datetime = new Date(timestamp); return datetime.toLocaleTimeString('en-US'); - }, - fillGranularity (feature) { - if (typeof feature.spec['granularity'] === "undefined") { - feature.spec['granularity'] = "NONE"; - } - return feature; } } diff --git a/ui/src/router.js b/ui/src/router.js index 9cc13e9ca5d..9d7fec8d6bf 100644 --- a/ui/src/router.js +++ b/ui/src/router.js @@ -25,11 +25,6 @@ export default new Router({ name: 'features-by-entity', component: FeatureList }, - { - path: '/features/:entity/:granularity', - name: 'features-by-entity-granularity', - component: FeatureList - }, { path: '/feature/:id', name: 'feature-details', diff --git a/ui/src/views/Feature/Details.vue b/ui/src/views/Feature/Details.vue index ba405310911..45a97e13cbc 100644 --- a/ui/src/views/Feature/Details.vue +++ b/ui/src/views/Feature/Details.vue @@ -8,11 +8,6 @@ {{ feature.spec.entity }} -

  • - - {{ feature.spec.granularity }} - -
  • {{ feature.spec.name }}
    • @@ -35,10 +30,6 @@ - - Granularity: - {{ feature.spec.granularity }} - Value type: {{ feature.spec.valueType }} @@ -55,10 +46,6 @@ URI:     {{ feature.spec.uri }} - - BigQuery view: - {{ feature.bigqueryView }} - Created: {{ feature.created }} @@ -75,6 +62,14 @@ + + Warehouse store: + + + {{ feature.spec.dataStores.warehouse.id }} + + + Status: OK @@ -168,7 +163,6 @@ name: "", owner: "", description: "", - granularity: 'NONE', valueType: "", entity: "", group: "", @@ -205,9 +199,7 @@ return `{ "entityName": "${ this.feature.spec.entity }", "entityId": [${ '"' + this.entityList.join('","') + '"' }], - "requestDetails": [{ - "featureId": "${ this.feature.spec.id }" - }] + "featureId": ["${ this.feature.spec.id }"] }` } }, @@ -216,7 +208,7 @@ fetchData() { let featureId = this.$route.params.id; this.$http.get(process.env.VUE_APP_ROOT_API + '/features/'+featureId).then(response => { - this.feature = filters.fillGranularity(response.body['feature']); + this.feature = response.body['feature']; this.yaml = json2yaml.stringify(response.body['rawSpec']); }, response => { this.error = response.statusText; From 61a93096911289ba5bb7b87603d21930cc2d1487 Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Thu, 28 Mar 2019 14:53:28 +0800 Subject: [PATCH 18/25] Add davidheryanto to OWNER file (#165) --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 756bd1cef14..3153f0c86bb 100644 --- a/OWNERS +++ b/OWNERS @@ -4,9 +4,11 @@ approvers: - woop - tims - thirteen37 + - davidheryanto reviewers: - zhilingc - pradithya - woop - tims - thirteen37 + - davidheryanto From 5b6c7db0dae5b1890648863d768661eed74481f5 Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Thu, 28 Mar 2019 15:35:28 +0800 Subject: [PATCH 19/25] Check validity of event timestamp in ValidateFeatureRowDoFn (#164) --- .../feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java index 06332eb8bf6..67aec7d255a 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.protobuf.util.Timestamps; import feast.ingestion.exceptions.ValidationException; import feast.ingestion.metrics.FeastMetrics; import feast.ingestion.model.Specs; @@ -89,6 +90,8 @@ public void processElementImpl(ProcessContext context) { "Row entity not found in import spec entities. entity=%s", row.getEntityName())); checkArgument(row.hasEventTimestamp(), "Must have eventTimestamp set"); + Timestamps.checkValid(row.getEventTimestamp()); + checkArgument(row.getFeaturesCount() > 0, "Must have at least one feature set"); for (Feature feature : row.getFeaturesList()) { From a0111bb4a3bf20966c364e0e17eb4ae62ea7d88f Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Fri, 29 Mar 2019 12:01:28 +0800 Subject: [PATCH 20/25] Prevent throwing RuntimeException when invalid proto is received (#166) --- .../errors/json/JsonFileErrorsWrite.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java index ef95bf634bf..08df8158646 100644 --- a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java @@ -39,18 +39,24 @@ public class JsonFileErrorsWrite extends FeatureStoreWrite { @Override public PDone expand(PCollection input) { - return input.apply("Map to strings", MapElements.into(kvs(strings(), strings())).via( - (rowExtended) -> { - try { - return KV.of( - rowExtended.getRow().getEntityName(), - JsonFormat.printer().omittingInsignificantWhitespace() - .print(rowExtended) - ); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - )).apply("Write Error Json Files", new TextFileDynamicIO.Write(options, ".json")); + return input + .apply( + "Map to strings", + MapElements.into(kvs(strings(), strings())) + .via( + (rowExtended) -> { + try { + return KV.of( + rowExtended.getRow().getEntityName(), + JsonFormat.printer() + .omittingInsignificantWhitespace() + .print(rowExtended)); + } catch (InvalidProtocolBufferException e) { + return KV.of( + rowExtended.getRow().getEntityName(), + e.toString()); + } + })) + .apply("Write Error Json Files", new TextFileDynamicIO.Write(options, ".json")); } } From 2e7d7174a1b686a0d264de3f94b833f59523047a Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Fri, 29 Mar 2019 12:07:28 +0800 Subject: [PATCH 21/25] Add --name flag to submit job (#168) --- cli/feast/cmd/jobs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/feast/cmd/jobs.go b/cli/feast/cmd/jobs.go index 5eb3e401ce8..8f44d9593ed 100644 --- a/cli/feast/cmd/jobs.go +++ b/cli/feast/cmd/jobs.go @@ -29,6 +29,7 @@ import ( var ( waitJobComplete = false + jobName = "feastimport" ) // jobsCmd represents the jobs command @@ -69,6 +70,7 @@ var jobsAbortCmd = &cobra.Command{ func init() { jobsRunCmd.Flags().BoolVar(&waitJobComplete, "wait", false, "wait for job to run to completion") + jobsRunCmd.Flags().StringVar(&jobName, "name", "feastimport", "job name to be submitted") jobsCmd.AddCommand(jobsRunCmd) jobsCmd.AddCommand(jobsAbortCmd) rootCmd.AddCommand(jobsCmd) @@ -86,6 +88,7 @@ func runJob(ctx context.Context, path string) error { initConn() jobsClient := core.NewJobServiceClient(coreConn) out, err := jobsClient.SubmitJob(ctx, &core.JobServiceTypes_SubmitImportJobRequest{ + Name: jobName, ImportSpec: is, }) if err != nil { From 79abea3dde21c2856d23cba8866a1e06b6166939 Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Fri, 29 Mar 2019 14:17:28 +0800 Subject: [PATCH 22/25] Proto error (#169) * Prevent throwing RuntimeException when invalid proto is received * Catch all kind of exception to avoid retrying --- .../main/java/feast/store/errors/json/JsonFileErrorsWrite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java index 08df8158646..2ae55fa3b46 100644 --- a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java @@ -51,7 +51,7 @@ public PDone expand(PCollection input) { JsonFormat.printer() .omittingInsignificantWhitespace() .print(rowExtended)); - } catch (InvalidProtocolBufferException e) { + } catch (Exception e) { return KV.of( rowExtended.getRow().getEntityName(), e.toString()); From 022fbd6da48876d94f4c96c6def81c2b5f85ccfc Mon Sep 17 00:00:00 2001 From: pradithya aria Date: Mon, 1 Apr 2019 23:33:30 +0700 Subject: [PATCH 23/25] Catch all kind of exception to avoid retrying (#171) --- .../src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java index 8811faa173c..d641c690a29 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java @@ -17,7 +17,6 @@ package feast.ingestion.transform.fn; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat.Printer; @@ -46,7 +45,7 @@ public void processElement(ProcessContext context) { String message; try { message = prefix + printer.print(context.element()); - } catch (InvalidProtocolBufferException e) { + } catch (Exception e) { log.error(e.getMessage(), e); message = prefix + context.element().toString(); } From 6193aa9f6a125ee3c2cead437248cb69d6f4b8ec Mon Sep 17 00:00:00 2001 From: Tim Sell <63295+tims@users.noreply.github.com> Date: Fri, 5 Apr 2019 13:08:31 +0800 Subject: [PATCH 24/25] kafka test the tests the source class, and remove unnecessary FeatureRowKey class (#160) --- .../FeatureRowDeserializer.java | 2 +- .../source/kafka/KafkaFeatureSource.java | 38 +++-- .../FeatureRowKeyDeserializer.java | 48 ------- .../source/kafka/KafkaFeatureSourceTest.java | 125 ++++++++++++++++ .../KafkaFeatureRowDeserializerTest.java | 133 ------------------ protos/feast/types/FeatureRow.proto | 6 - 6 files changed, 149 insertions(+), 203 deletions(-) rename ingestion/src/main/java/feast/source/kafka/{deserializer => }/FeatureRowDeserializer.java (97%) delete mode 100644 ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowKeyDeserializer.java create mode 100644 ingestion/src/test/java/feast/source/kafka/KafkaFeatureSourceTest.java delete mode 100644 ingestion/src/test/java/feast/source/kafka/deserializer/KafkaFeatureRowDeserializerTest.java diff --git a/ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowDeserializer.java b/ingestion/src/main/java/feast/source/kafka/FeatureRowDeserializer.java similarity index 97% rename from ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowDeserializer.java rename to ingestion/src/main/java/feast/source/kafka/FeatureRowDeserializer.java index e75a17e64f8..d62b959adea 100644 --- a/ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowDeserializer.java +++ b/ingestion/src/main/java/feast/source/kafka/FeatureRowDeserializer.java @@ -15,7 +15,7 @@ * */ -package feast.source.kafka.deserializer; +package feast.source.kafka; import com.google.protobuf.InvalidProtocolBufferException; import feast.types.FeatureRowProto.FeatureRow; diff --git a/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java b/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java index a91add6924a..5c8ca713fda 100644 --- a/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java +++ b/ingestion/src/main/java/feast/source/kafka/KafkaFeatureSource.java @@ -21,17 +21,15 @@ import com.google.auto.service.AutoService; import com.google.common.base.Strings; +import feast.ingestion.options.JobOptions; import feast.ingestion.transform.fn.FilterFeatureRowDoFn; import feast.options.Options; import feast.options.OptionsParser; import feast.source.FeatureSource; import feast.source.FeatureSourceFactory; -import feast.source.kafka.deserializer.FeatureRowDeserializer; -import feast.source.kafka.deserializer.FeatureRowKeyDeserializer; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.types.FeatureRowProto.FeatureRow; -import feast.types.FeatureRowProto.FeatureRowKey; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -43,6 +41,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; /** * Transform for reading {@link feast.types.FeatureRowProto.FeatureRow FeatureRow} proto messages @@ -60,29 +59,35 @@ public PCollection expand(PInput input) { KafkaReadOptions options = OptionsParser.parse(importSpec.getSourceOptionsMap(), KafkaReadOptions.class); + JobOptions jobOptions = OptionsParser.parse(importSpec.getJobOptionsMap(), JobOptions.class); List topicsList = new ArrayList<>(Arrays.asList(options.topics.split(","))); - PCollection> featureRowRecord = - input.getPipeline().apply( KafkaIO.read() - .withBootstrapServers(options.server) - .withTopics(topicsList) - .withKeyDeserializer(FeatureRowKeyDeserializer.class) - .withValueDeserializer(FeatureRowDeserializer.class)); + KafkaIO.Read read = KafkaIO.read() + .withBootstrapServers(options.server) + .withTopics(topicsList) + .withKeyDeserializer(ByteArrayDeserializer.class) + .withValueDeserializer(FeatureRowDeserializer.class); + if (jobOptions.getSampleLimit() > 0) { + read = read.withMaxNumRecords(jobOptions.getSampleLimit()); + } + + PCollection> featureRowRecord = + input.getPipeline().apply(read); - PCollection featureRow = featureRowRecord.apply( + PCollection featureRow = featureRowRecord.apply( ParDo.of( - new DoFn, FeatureRow>() { + new DoFn, FeatureRow>() { @ProcessElement public void processElement(ProcessContext processContext) { - KafkaRecord record = processContext.element(); + KafkaRecord record = processContext.element(); processContext.output(record.getKV().getValue()); } })); if (options.discardUnknownFeatures) { List featureIds = new ArrayList<>(); - for(Field field: importSpec.getSchema().getFieldsList()) { + for (Field field : importSpec.getSchema().getFieldsList()) { String featureId = field.getFeatureId(); if (!Strings.isNullOrEmpty(featureId)) { featureIds.add(featureId); @@ -94,8 +99,11 @@ public void processElement(ProcessContext processContext) { } public static class KafkaReadOptions implements Options { - @NotEmpty public String server; - @NotEmpty public String topics; + + @NotEmpty + public String server; + @NotEmpty + public String topics; public boolean discardUnknownFeatures = false; } diff --git a/ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowKeyDeserializer.java b/ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowKeyDeserializer.java deleted file mode 100644 index 4e77d7c99da..00000000000 --- a/ingestion/src/main/java/feast/source/kafka/deserializer/FeatureRowKeyDeserializer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.source.kafka.deserializer; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.types.FeatureRowProto.FeatureRowKey; -import java.util.Map; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; - -/** - * Deserializer for Kafka to deserialize Protocol Buffers messages - * - * @param Protobuf message type - */ -public class FeatureRowKeyDeserializer implements Deserializer { - - @Override - public void configure(Map configs, boolean isKey) {} - - @Override - public FeatureRowKey deserialize(String topic, byte[] data) { - try { - return FeatureRowKey.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new SerializationException( - "Error deserializing FeatureRowKey from Protobuf message", e); - } - } - - @Override - public void close() {} -} diff --git a/ingestion/src/test/java/feast/source/kafka/KafkaFeatureSourceTest.java b/ingestion/src/test/java/feast/source/kafka/KafkaFeatureSourceTest.java new file mode 100644 index 00000000000..50ffa1e87fa --- /dev/null +++ b/ingestion/src/test/java/feast/source/kafka/KafkaFeatureSourceTest.java @@ -0,0 +1,125 @@ +package feast.source.kafka; + +import feast.specs.ImportSpecProto.ImportSpec; +import feast.types.FeatureRowProto.FeatureRow; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollection.IsBounded; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.Serializer; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +@Slf4j +@RunWith(SpringRunner.class) +@SpringBootTest +@DirtiesContext +public class KafkaFeatureSourceTest { + + @ClassRule + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, "TEST_TOPIC"); + @Rule + public TestPipeline pipeline = TestPipeline.create(); + @Autowired + private KafkaTemplate template; + + + public void send(FeatureRow... rows) { + for (FeatureRow row : rows) { + try { + log.info("Sent: " + template.send("TEST_TOPIC", row).get().toString()); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + } + + @Test + public void testFoo() throws ExecutionException, InterruptedException { + String server = embeddedKafka.getEmbeddedKafka().getBrokerAddresses()[0].toString(); + ImportSpec importSpec = ImportSpec.newBuilder().setType("kafka") + .addEntities("testEntity") + .putSourceOptions("topics", "TEST_TOPIC") + .putSourceOptions("server", server) + .putJobOptions("sample.limit", "1") + .build(); + FeatureRow row = FeatureRow.newBuilder().setEntityKey("key").build(); + ScheduledExecutorService scheduler = + Executors.newScheduledThreadPool(1); + // we keep sending on loop because beam will only start consuming rows that were sent after startup. + scheduler.scheduleAtFixedRate(() -> send(row), 0, 1, TimeUnit.SECONDS); + + PCollection rows = pipeline.apply(new KafkaFeatureSource(importSpec)); + Assert.assertEquals(IsBounded.BOUNDED, rows.isBounded()); + PAssert.that(rows).containsInAnyOrder(row); + pipeline.run(); + } + + public Producer getProducer() { + Map producerProps = + KafkaTestUtils.producerProps(embeddedKafka.getEmbeddedKafka()); + return new DefaultKafkaProducerFactory<>(producerProps, new ByteArraySerializer(), + new FeatureRowSerializer()).createProducer(); + } + + + public static class FeatureRowSerializer implements Serializer { + + @Override + public void configure(Map configs, boolean isKey) { + + } + + @Override + public byte[] serialize(String topic, FeatureRow data) { + return data.toByteArray(); + } + + @Override + public void close() { + + } + } + + @Configuration + static class ContextConfiguration { + + @Bean + ProducerFactory producerFactory() { + Map producerProps = + KafkaTestUtils.producerProps(embeddedKafka.getEmbeddedKafka()); + + return new DefaultKafkaProducerFactory<>( + producerProps, new ByteArraySerializer(), new FeatureRowSerializer()); + } + + @Bean + KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory(), true); + } + } +} \ No newline at end of file diff --git a/ingestion/src/test/java/feast/source/kafka/deserializer/KafkaFeatureRowDeserializerTest.java b/ingestion/src/test/java/feast/source/kafka/deserializer/KafkaFeatureRowDeserializerTest.java deleted file mode 100644 index 3436b8d8b7a..00000000000 --- a/ingestion/src/test/java/feast/source/kafka/deserializer/KafkaFeatureRowDeserializerTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.source.kafka.deserializer; - -import com.google.protobuf.MessageLite; -import feast.types.FeatureRowProto.FeatureRow; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.LinkedBlockingQueue; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.Deserializer; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.core.ProducerFactory; -import org.springframework.kafka.listener.ContainerProperties; -import org.springframework.kafka.listener.KafkaMessageListenerContainer; -import org.springframework.kafka.listener.MessageListener; -import org.springframework.kafka.listener.MessageListenerContainer; -import org.springframework.kafka.support.SendResult; -import org.springframework.kafka.test.rule.EmbeddedKafkaRule; -import org.springframework.kafka.test.utils.ContainerTestUtils; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.util.concurrent.ListenableFuture; - -@RunWith(SpringRunner.class) -@SpringBootTest -@DirtiesContext -public class KafkaFeatureRowDeserializerTest { - - private static final String topic = "TEST_TOPIC"; - - @ClassRule public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, topic); - @Autowired private KafkaTemplate template; - - private void deserialize(MessageType input) { - - Deserializer deserializer = new FeatureRowDeserializer(); - - Map consumerProps = - KafkaTestUtils.consumerProps("testGroup", "false", embeddedKafka.getEmbeddedKafka()); - ConsumerFactory consumerFactory = - new DefaultKafkaConsumerFactory<>(consumerProps, deserializer, deserializer); - - BlockingQueue> records = new LinkedBlockingQueue<>(); - ContainerProperties containerProps = new ContainerProperties(topic); - containerProps.setMessageListener((MessageListener) records::add); - - MessageListenerContainer container = - new KafkaMessageListenerContainer<>(consumerFactory, containerProps); - container.start(); - ContainerTestUtils.waitForAssignment( - container, embeddedKafka.getEmbeddedKafka().getPartitionsPerTopic()); - - byte[] data = input.toByteArray(); - ProducerRecord producerRecord = new ProducerRecord<>(topic, data, data); - ListenableFuture> producerFuture = template.send(producerRecord); - - try { - producerFuture.get(); - } catch (InterruptedException e) { - return; - } catch (ExecutionException e) { - throw new KafkaException("Error sending message to Kafka.", e.getCause()); - } - - ConsumerRecord consumerRecord; - try { - consumerRecord = records.take(); - } catch (InterruptedException e) { - return; - } - - FeatureRow key = consumerRecord.key(); - Assert.assertEquals(key, input); - - FeatureRow value = consumerRecord.value(); - Assert.assertEquals(value, input); - } - - @Test(timeout = 10000) - public void deserializeFeatureRowProto() { - FeatureRow message = FeatureRow.newBuilder().setEntityName("test").build(); - deserialize(message); - } - - @Configuration - static class ContextConfiguration { - @Bean - ProducerFactory producerFactory() { - Map producerProps = - KafkaTestUtils.producerProps(embeddedKafka.getEmbeddedKafka()); - - return new DefaultKafkaProducerFactory<>( - producerProps, new ByteArraySerializer(), new ByteArraySerializer()); - } - - @Bean - KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(producerFactory(), true); - } - } -} diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto index b17ad7bbc9b..755fe0f582e 100644 --- a/protos/feast/types/FeatureRow.proto +++ b/protos/feast/types/FeatureRow.proto @@ -25,12 +25,6 @@ option java_package = "feast.types"; option java_outer_classname = "FeatureRowProto"; option go_package = "github.com/gojek/feast/protos/generated/go/feast/types"; -message FeatureRowKey { - string entityKey = 1; - google.protobuf.Timestamp eventTimestamp = 3; - string entityName = 4; -} - message FeatureRow { string entityKey = 1; repeated Feature features = 2; From 5bbc277e132af9cd4b8a4811d1a0344a110e7095 Mon Sep 17 00:00:00 2001 From: Chen Zhiling Date: Tue, 9 Apr 2019 15:29:33 +0700 Subject: [PATCH 25/25] Integration test (#170) * Add wait flag for jobs, fix go proto path for dataset service * Rebase with master * Add integration test for redis-bq-dataflow * Use go modules, add tests for util, printer and cmd * Add HealthControllerTest * Add integration tests folder * Format dataframe dtypes returned from serving properly * Update helm templates for easier setting of LoadBalancerSourceRanges * Add unit test for UIServiceController * Update docker file to include argo * Add prow configuration for starting integration test * Remove warehouse and serving info in feature specs test data * Fix postsubmit name prefix * Update sourceOptions for testdata * Update location for staging data in feast integration * Update testdata to stage * Integration test: Fix incorrect path in import csv * Separate images for unit and integration tests --- .gitignore | 120 ++++- Gopkg.lock | 280 ------------ Gopkg.toml | 33 -- Makefile | 28 +- charts/feast/templates/core-service.yaml | 6 +- charts/feast/templates/serving-service.yaml | 6 +- charts/feast/values.yaml | 6 + cli/feast/cmd/apply_test.go | 146 ++++++ cli/feast/cmd/config_test.go | 42 ++ cli/feast/cmd/list.go | 4 +- cli/feast/cmd/mock/README.md | 11 + cli/feast/cmd/mock/core_service.go | 238 ++++++++++ cli/feast/cmd/testdata/invalid_entity.yaml | 5 + cli/feast/cmd/testdata/valid_entity.yaml | 5 + .../cmd/testdata/valid_entity_no_tag.yaml | 2 + cli/feast/cmd/testdata/valid_feature.yaml | 12 + .../cmd/testdata/valid_feature_group.yaml | 7 + cli/feast/cmd/testdata/valid_storage.yaml | 6 + cli/feast/pkg/printer/job.go | 4 +- cli/feast/pkg/printer/job_test.go | 66 +++ cli/feast/pkg/printer/specs.go | 10 +- .../{printer_test.go => specs_test.go} | 14 +- .../{util/utils.go => timeutil/timeutil.go} | 10 +- cli/feast/pkg/timeutil/timeutil_test.go | 49 ++ .../feast/core/http/HealthController.java | 3 +- .../java/feast/core/model/FeatureInfo.java | 1 + .../java/feast/core/CoreApplicationTest.java | 2 +- .../CoreApplicationWithNoServingTest.java | 2 +- .../CoreApplicationWithNoWarehouseTest.java | 2 +- .../feast/core/http/HealthControllerTest.java | 65 +++ .../core/http/UiServiceControllerTest.java | 177 +++++++ go.mod | 14 + go.sum | 76 +++ protos/Makefile | 2 +- .../generated/go/feast/core/CoreService.pb.go | 322 +++++++------ .../go/feast/core/DatasetService.pb.go | 126 ++--- .../generated/go/feast/core/JobService.pb.go | 193 ++++---- .../generated/go/feast/core/UIService.pb.go | 333 ++++++++------ .../generated/go/feast/serving/Serving.pb.go | 123 ++--- .../generated/go/feast/specs/EntitySpec.pb.go | 25 +- .../go/feast/specs/FeatureGroupSpec.pb.go | 25 +- .../go/feast/specs/FeatureSpec.pb.go | 41 +- .../go/feast/specs/ImportJobSpecs.pb.go | 25 +- .../generated/go/feast/specs/ImportSpec.pb.go | 109 +---- .../go/feast/specs/StorageSpec.pb.go | 25 +- .../generated/go/feast/storage/BigTable.pb.go | 30 +- protos/generated/go/feast/storage/Redis.pb.go | 49 +- protos/generated/go/feast/types/Feature.pb.go | 23 +- .../generated/go/feast/types/FeatureRow.pb.go | 34 +- .../go/feast/types/FeatureRowExtended.pb.go | 39 +- .../go/feast/types/Granularity.pb.go | 106 ----- protos/generated/go/feast/types/Value.pb.go | 432 +++--------------- sdk/python/feast/sdk/client.py | 131 +++--- sdk/python/feast/sdk/utils/bq_util.py | 132 +++++- sdk/python/feast/sdk/utils/types.py | 19 +- sdk/python/setup.py | 54 +-- .../austin_bikeshare.bikeshare_stations.avro | Bin 0 -> 7653 bytes sdk/python/tests/sdk/utils/test_bq_utils.py | 97 ++-- testing/Makefile | 4 +- testing/README.md | 16 +- .../docker-builder-image/Dockerfile | 25 + testing/docker-images/gcloud-argo/Dockerfile | 15 + testing/docker-images/gcloud-argo/run.sh | 51 +++ .../docker-images/terraform-image/Dockerfile | 19 + .../{ => docker-images}/test-image/Dockerfile | 2 - testing/{ => docker-images}/test-image/run.sh | 2 +- .../cloud-build/cloud-build.tf | 0 .../cloud-build/cloudbuild.yaml | 0 .../cloud-build/variables.tf | 0 testing/{tf => infrastructure}/gcs/backend.tf | 0 testing/{tf => infrastructure}/gcs/main.tf | 0 .../k8s-cluster/backend.tf | 0 .../k8s-cluster/main.tf | 4 + .../k8s-cluster/output.tf | 0 .../k8s-cluster/provider.tf | 0 .../k8s-cluster/variables.tf | 0 testing/integration/Makefile | 25 + testing/integration/README.md | 29 ++ .../tests/redis-bq-dataflow/.env.local | 7 + .../redis-bq-dataflow/.env.local.example | 7 + .../tests/redis-bq-dataflow/.gitignore | 3 + .../tests/redis-bq-dataflow/README.md | 9 + .../data/entity/myentity.yaml | 5 + .../feature/feature_minutedoubleredis1.yaml | 7 + .../feature/feature_minutefloatredis1.yaml | 7 + .../feature/feature_minuteint32redis1.yaml | 7 + .../feature/feature_minuteint64redis1.yaml | 7 + .../data/import/import_csv.yaml | 19 + .../redis-bq-dataflow/data/test_data.csv | 50 ++ .../tests/redis-bq-dataflow/main.py | 9 + .../tests/redis-bq-dataflow/test_feast.py | 133 ++++++ .../testdata/entity/myentity.yaml | 5 + .../feature/feature_double_redis.yaml | 7 + .../testdata/feature/feature_float_redis.yaml | 7 + .../testdata/feature/feature_int32_redis.yaml | 7 + .../testdata/feature/feature_int64_redis.yaml | 7 + .../testdata/import/import_csv.yaml | 19 + .../redis-bq-dataflow/testdata/myentity.avro | Bin 0 -> 2709 bytes .../redis-bq-dataflow/testdata/myentity.csv | 50 ++ .../tests/redis-bq-dataflow/tf/backend.tf | 6 + .../tests/redis-bq-dataflow/tf/data.tf | 1 + .../tests/redis-bq-dataflow/tf/input.tf | 3 + .../tests/redis-bq-dataflow/tf/main.tf | 70 +++ .../tests/redis-bq-dataflow/tf/provider.tf | 16 + .../redis-bq-dataflow/tf/scripts/empty-bq.sh | 10 + .../tf/scripts/wait-for-cluster.sh | 36 ++ .../tests/redis-bq-dataflow/tf/values.yaml | 101 ++++ .../tests/redis-bq-dataflow/workflow.yaml | 233 ++++++++++ .../integration/tf/modules/cluster/input.tf | 39 ++ .../integration/tf/modules/cluster/main.tf | 42 ++ .../integration/tf/modules/cluster/output.tf | 33 ++ .../tf/modules/feast-helm/input.tf | 61 +++ .../integration/tf/modules/feast-helm/main.tf | 100 ++++ .../tf/modules/feast-helm/output.tf | 11 + .../tf/modules/feast-helm/values.tmpl | 105 +++++ testing/prow/config.yaml | 17 +- testing/tf/cloud-build/cloudbuild_master.yaml | 19 - testing/tf/terraform.tfvars | 7 - 118 files changed, 3711 insertions(+), 1710 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml create mode 100644 cli/feast/cmd/apply_test.go create mode 100644 cli/feast/cmd/config_test.go create mode 100644 cli/feast/cmd/mock/README.md create mode 100644 cli/feast/cmd/mock/core_service.go create mode 100644 cli/feast/cmd/testdata/invalid_entity.yaml create mode 100644 cli/feast/cmd/testdata/valid_entity.yaml create mode 100644 cli/feast/cmd/testdata/valid_entity_no_tag.yaml create mode 100644 cli/feast/cmd/testdata/valid_feature.yaml create mode 100644 cli/feast/cmd/testdata/valid_feature_group.yaml create mode 100644 cli/feast/cmd/testdata/valid_storage.yaml create mode 100644 cli/feast/pkg/printer/job_test.go rename cli/feast/pkg/printer/{printer_test.go => specs_test.go} (89%) rename cli/feast/pkg/{util/utils.go => timeutil/timeutil.go} (75%) create mode 100644 cli/feast/pkg/timeutil/timeutil_test.go create mode 100644 core/src/test/java/feast/core/http/HealthControllerTest.java create mode 100644 core/src/test/java/feast/core/http/UiServiceControllerTest.java create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 protos/generated/go/feast/types/Granularity.pb.go create mode 100644 sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro create mode 100644 testing/docker-images/docker-builder-image/Dockerfile create mode 100644 testing/docker-images/gcloud-argo/Dockerfile create mode 100755 testing/docker-images/gcloud-argo/run.sh create mode 100644 testing/docker-images/terraform-image/Dockerfile rename testing/{ => docker-images}/test-image/Dockerfile (89%) rename testing/{ => docker-images}/test-image/run.sh (96%) rename testing/{tf => infrastructure}/cloud-build/cloud-build.tf (100%) rename testing/{tf => infrastructure}/cloud-build/cloudbuild.yaml (100%) rename testing/{tf => infrastructure}/cloud-build/variables.tf (100%) rename testing/{tf => infrastructure}/gcs/backend.tf (100%) rename testing/{tf => infrastructure}/gcs/main.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/backend.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/main.tf (69%) rename testing/{tf => infrastructure}/k8s-cluster/output.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/provider.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/variables.tf (100%) create mode 100644 testing/integration/Makefile create mode 100644 testing/integration/README.md create mode 100644 testing/integration/tests/redis-bq-dataflow/.env.local create mode 100644 testing/integration/tests/redis-bq-dataflow/.env.local.example create mode 100644 testing/integration/tests/redis-bq-dataflow/.gitignore create mode 100644 testing/integration/tests/redis-bq-dataflow/README.md create mode 100644 testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/data/test_data.csv create mode 100644 testing/integration/tests/redis-bq-dataflow/main.py create mode 100644 testing/integration/tests/redis-bq-dataflow/test_feast.py create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/entity/myentity.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_double_redis.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_float_redis.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int32_redis.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int64_redis.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/import/import_csv.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/myentity.avro create mode 100644 testing/integration/tests/redis-bq-dataflow/testdata/myentity.csv create mode 100644 testing/integration/tests/redis-bq-dataflow/tf/backend.tf create mode 100644 testing/integration/tests/redis-bq-dataflow/tf/data.tf create mode 100644 testing/integration/tests/redis-bq-dataflow/tf/input.tf create mode 100644 testing/integration/tests/redis-bq-dataflow/tf/main.tf create mode 100644 testing/integration/tests/redis-bq-dataflow/tf/provider.tf create mode 100755 testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh create mode 100755 testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh create mode 100755 testing/integration/tests/redis-bq-dataflow/tf/values.yaml create mode 100644 testing/integration/tests/redis-bq-dataflow/workflow.yaml create mode 100644 testing/integration/tf/modules/cluster/input.tf create mode 100644 testing/integration/tf/modules/cluster/main.tf create mode 100644 testing/integration/tf/modules/cluster/output.tf create mode 100644 testing/integration/tf/modules/feast-helm/input.tf create mode 100644 testing/integration/tf/modules/feast-helm/main.tf create mode 100644 testing/integration/tf/modules/feast-helm/output.tf create mode 100644 testing/integration/tf/modules/feast-helm/values.tmpl delete mode 100644 testing/tf/cloud-build/cloudbuild_master.yaml delete mode 100644 testing/tf/terraform.tfvars diff --git a/.gitignore b/.gitignore index 01df87f2dc0..f35e736b20e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,122 @@ vendor .direnv .terraform/ -*.tfvars \ No newline at end of file +*.tfvars + +# python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +.vscode diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 18004a22c77..00000000000 --- a/Gopkg.lock +++ /dev/null @@ -1,280 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" - name = "github.com/fsnotify/fsnotify" - packages = ["."] - pruneopts = "UT" - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - version = "v1.4.7" - -[[projects]] - digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" - name = "github.com/ghodss/yaml" - packages = ["."] - pruneopts = "UT" - revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" - version = "v1.0.0" - -[[projects]] - digest = "1:54a9254d3845e3a80d4957975abfc5e771a49d08379509ec4f9c99c1ccbe13a9" - name = "github.com/golang/protobuf" - packages = [ - "proto", - "protoc-gen-go", - "protoc-gen-go/descriptor", - "protoc-gen-go/generator", - "protoc-gen-go/generator/internal/remap", - "protoc-gen-go/grpc", - "protoc-gen-go/plugin", - "ptypes", - "ptypes/any", - "ptypes/duration", - "ptypes/empty", - "ptypes/timestamp", - ] - pruneopts = "UT" - revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" - version = "v1.2.0" - -[[projects]] - digest = "1:2e3c336fc7fde5c984d2841455a658a6d626450b1754a854b3b32e7a8f49a07a" - name = "github.com/google/go-cmp" - packages = [ - "cmp", - "cmp/internal/diff", - "cmp/internal/function", - "cmp/internal/value", - ] - pruneopts = "UT" - revision = "3af367b6b30c263d47e8895973edcca9a49cf029" - version = "v0.2.0" - -[[projects]] - digest = "1:c0d19ab64b32ce9fe5cf4ddceba78d5bc9807f0016db6b1183599da3dcc24d10" - name = "github.com/hashicorp/hcl" - packages = [ - ".", - "hcl/ast", - "hcl/parser", - "hcl/printer", - "hcl/scanner", - "hcl/strconv", - "hcl/token", - "json/parser", - "json/scanner", - "json/token", - ] - pruneopts = "UT" - revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" - version = "v1.0.0" - -[[projects]] - digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - pruneopts = "UT" - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7" - name = "github.com/magiconair/properties" - packages = ["."] - pruneopts = "UT" - revision = "c2353362d570a7bfa228149c62842019201cfb71" - version = "v1.8.0" - -[[projects]] - digest = "1:78bbb1ba5b7c3f2ed0ea1eab57bdd3859aec7e177811563edc41198a760b06af" - name = "github.com/mitchellh/go-homedir" - packages = ["."] - pruneopts = "UT" - revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4" - version = "v1.0.0" - -[[projects]] - digest = "1:53bc4cd4914cd7cd52139990d5170d6dc99067ae31c56530621b18b35fc30318" - name = "github.com/mitchellh/mapstructure" - packages = ["."] - pruneopts = "UT" - revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" - version = "v1.1.2" - -[[projects]] - digest = "1:95741de3af260a92cc5c7f3f3061e85273f5a81b5db20d4bd68da74bd521675e" - name = "github.com/pelletier/go-toml" - packages = ["."] - pruneopts = "UT" - revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" - version = "v1.2.0" - -[[projects]] - digest = "1:6a4a11ba764a56d2758899ec6f3848d24698d48442ebce85ee7a3f63284526cd" - name = "github.com/spf13/afero" - packages = [ - ".", - "mem", - ] - pruneopts = "UT" - revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd" - version = "v1.1.2" - -[[projects]] - digest = "1:516e71bed754268937f57d4ecb190e01958452336fa73dbac880894164e91c1f" - name = "github.com/spf13/cast" - packages = ["."] - pruneopts = "UT" - revision = "8965335b8c7107321228e3e3702cab9832751bac" - version = "v1.2.0" - -[[projects]] - digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939" - name = "github.com/spf13/cobra" - packages = ["."] - pruneopts = "UT" - revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" - version = "v0.0.3" - -[[projects]] - digest = "1:68ea4e23713989dc20b1bded5d9da2c5f9be14ff9885beef481848edd18c26cb" - name = "github.com/spf13/jwalterweatherman" - packages = ["."] - pruneopts = "UT" - revision = "4a4406e478ca629068e7768fc33f3f044173c0a6" - version = "v1.0.0" - -[[projects]] - digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2" - name = "github.com/spf13/pflag" - packages = ["."] - pruneopts = "UT" - revision = "298182f68c66c05229eb03ac171abe6e309ee79a" - version = "v1.0.3" - -[[projects]] - digest = "1:214775c11fd26da94a100111a62daa25339198a4f9c57cb4aab352da889f5b93" - name = "github.com/spf13/viper" - packages = ["."] - pruneopts = "UT" - revision = "2c12c60302a5a0e62ee102ca9bc996277c2f64f5" - version = "v1.2.1" - -[[projects]] - branch = "master" - digest = "1:505dbee0833715a72a529bb57c354826ad42a4496fad787fa143699b4de1a6d0" - name = "golang.org/x/net" - packages = [ - "context", - "http/httpguts", - "http2", - "http2/hpack", - "idna", - "internal/timeseries", - "trace", - ] - pruneopts = "UT" - revision = "04a2e542c03f1d053ab3e4d6e5abcd4b66e2be8e" - -[[projects]] - branch = "master" - digest = "1:d99359caa06128f971c1b1373e8c6c4221ed74f07840e66d660006f0c2df0ed6" - name = "golang.org/x/sys" - packages = ["unix"] - pruneopts = "UT" - revision = "8b8824e799c8fc8e6d900615cc6c761ce6c26c67" - -[[projects]] - digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" - name = "golang.org/x/text" - packages = [ - "collate", - "collate/build", - "internal/colltab", - "internal/gen", - "internal/tag", - "internal/triegen", - "internal/ucd", - "language", - "secure/bidirule", - "transform", - "unicode/bidi", - "unicode/cldr", - "unicode/norm", - "unicode/rangetable", - ] - pruneopts = "UT" - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - branch = "master" - digest = "1:56b0bca90b7e5d1facf5fbdacba23e4e0ce069d25381b8e2f70ef1e7ebfb9c1a" - name = "google.golang.org/genproto" - packages = ["googleapis/rpc/status"] - pruneopts = "UT" - revision = "94acd270e44e65579b9ee3cdab25034d33fed608" - -[[projects]] - digest = "1:ab8e92d746fb5c4c18846b0879842ac8e53b3d352449423d0924a11f1020ae1b" - name = "google.golang.org/grpc" - packages = [ - ".", - "balancer", - "balancer/base", - "balancer/roundrobin", - "codes", - "connectivity", - "credentials", - "encoding", - "encoding/proto", - "grpclog", - "internal", - "internal/backoff", - "internal/channelz", - "internal/envconfig", - "internal/grpcrand", - "internal/transport", - "keepalive", - "metadata", - "naming", - "peer", - "resolver", - "resolver/dns", - "resolver/passthrough", - "stats", - "status", - "tap", - ] - pruneopts = "UT" - revision = "8dea3dc473e90c8179e519d91302d0597c0ca1d1" - version = "v1.15.0" - -[[projects]] - digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202" - name = "gopkg.in/yaml.v2" - packages = ["."] - pruneopts = "UT" - revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" - version = "v2.2.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "github.com/ghodss/yaml", - "github.com/golang/protobuf/proto", - "github.com/golang/protobuf/protoc-gen-go", - "github.com/golang/protobuf/ptypes/empty", - "github.com/golang/protobuf/ptypes/timestamp", - "github.com/google/go-cmp/cmp", - "github.com/mitchellh/go-homedir", - "github.com/spf13/cobra", - "github.com/spf13/viper", - "golang.org/x/net/context", - "google.golang.org/grpc", - "gopkg.in/yaml.v2", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index c06fd9fc914..00000000000 --- a/Gopkg.toml +++ /dev/null @@ -1,33 +0,0 @@ -required = ["github.com/golang/protobuf/protoc-gen-go"] - -[[constraint]] - name = "github.com/ghodss/yaml" - version = "1.0.0" - -[[constraint]] - name = "github.com/golang/protobuf" - version = "1.2.0" - -[[constraint]] - name = "github.com/google/go-cmp" - version = "0.2.0" - -[[constraint]] - name = "github.com/mitchellh/go-homedir" - version = "1.0.0" - -[[constraint]] - name = "github.com/spf13/cobra" - version = "0.0.3" - -[[constraint]] - name = "github.com/spf13/viper" - version = "1.2.1" - -[[constraint]] - name = "google.golang.org/grpc" - version = "1.15.0" - -[prune] - go-tests = true - unused-packages = true diff --git a/Makefile b/Makefile index 888cfea4264..84b787c2c01 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,33 @@ -VERSION_FILE=VERSION -FEAST_VERSION=`cat $(VERSION_FILE)` +# +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +VERSION_FILE = VERSION +FEAST_VERSION = `cat $(VERSION_FILE)` test: mvn test -build-deps: +test-integration: + $(MAKE) -C testing/integration test-integration TYPE=$(TYPE) ID=$(ID) + +build-proto: $(MAKE) -C protos gen-go - dep ensure build-cli: - $(MAKE) build-deps + $(MAKE) build-proto $(MAKE) -C cli build-all build-java: diff --git a/charts/feast/templates/core-service.yaml b/charts/feast/templates/core-service.yaml index 965ee538d17..9f8f1209a23 100644 --- a/charts/feast/templates/core-service.yaml +++ b/charts/feast/templates/core-service.yaml @@ -14,12 +14,10 @@ spec: type: LoadBalancer {{- if .Values.core.service.extIPAdr }} loadBalancerIP: {{ .Values.core.service.extIPAdr }} + {{- end }} {{- if .Values.core.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: - {{- range .Values.core.service.loadBalancerSourceRanges }} - - {{ . }} - {{- end }} - {{- end }} +{{ toYaml .Values.core.service.loadBalancerSourceRanges | indent 2 }} {{- end }} ports: - name: http diff --git a/charts/feast/templates/serving-service.yaml b/charts/feast/templates/serving-service.yaml index ee1ecdfb0b5..81520ba4714 100644 --- a/charts/feast/templates/serving-service.yaml +++ b/charts/feast/templates/serving-service.yaml @@ -14,12 +14,10 @@ spec: type: LoadBalancer {{- if .Values.serving.service.extIPAdr }} loadBalancerIP: {{ .Values.serving.service.extIPAdr }} + {{- end }} {{- if .Values.serving.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: - {{- range .Values.serving.service.loadBalancerSourceRanges }} - - {{ . }} - {{- end }} - {{- end }} +{{ toYaml .Values.serving.service.loadBalancerSourceRanges | indent 2 }} {{- end }} ports: - name: grpc diff --git a/charts/feast/values.yaml b/charts/feast/values.yaml index fde0f7e7d90..f8cb3dfe442 100644 --- a/charts/feast/values.yaml +++ b/charts/feast/values.yaml @@ -24,6 +24,9 @@ core: http: port: 80 targetPort: 8080 + # loadBalancerSourceRanges sets the accepted IP ranges for firewall ingress rule + # this firewall rule is usually created when the service type is "LoadBalancer" + # loadBalancerSourceRanges: ["10.0.0.0/8"] jobs: workspace: "/tmp" runner: DirectRunner @@ -87,6 +90,9 @@ serving: http: port: 80 targetPort: 8080 + # loadBalancerSourceRanges sets the accepted IP ranges for firewall ingress rule + # this firewall rule is usually created when the service type is "LoadBalancer" + # loadBalancerSourceRanges: ["10.0.0.0/8"] jaeger: enabled: false diff --git a/cli/feast/cmd/apply_test.go b/cli/feast/cmd/apply_test.go new file mode 100644 index 00000000000..7a8de3aaab1 --- /dev/null +++ b/cli/feast/cmd/apply_test.go @@ -0,0 +1,146 @@ +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "github.com/gojek/feast/cli/feast/cmd/mock" + "github.com/golang/mock/gomock" + "testing" + + "github.com/gojek/feast/protos/generated/go/feast/core" +) + +func Test_apply(t *testing.T) { + mockCore := mock.NewMockCoreServiceClient(gomock.NewController(t)) + mockCore.EXPECT().ApplyEntity(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockCore.EXPECT().ApplyFeature(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockCore.EXPECT().ApplyFeatureGroup(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockCore.EXPECT().ApplyStorage(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + + type args struct { + ctx context.Context + coreCli core.CoreServiceClient + resource string + fileLocation string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test apply invalid resource", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "invalidResource", + fileLocation: "testdata/valid_entity.yaml", + }, + want: "", + wantErr: true, + }, + { + name: "test apply entity", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "entity", + fileLocation: "testdata/valid_entity.yaml", + }, + want: "myentity", + wantErr: false, + }, + { + name: "test apply entity with non-existent file", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "entity", + fileLocation: "testdata/file_not_exists.yaml", + }, + want: "", + wantErr: true, + }, + { + name: "test apply entity with no tag", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "entity", + fileLocation: "testdata/valid_entity_no_tag.yaml", + }, + want: "myentity", + wantErr: false, + }, + { + name: "test apply invalid syntax in entity yaml", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "entity", + fileLocation: "testdata/invalid_entity.yaml", + }, + want: "", + wantErr: true, + }, + { + name: "test apply feature", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "feature", + fileLocation: "testdata/valid_feature.yaml", + }, + want: "myentity.feature_bool_redis1", + wantErr: false, + }, + { + name: "test apply feature group", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "featureGroup", + fileLocation: "testdata/valid_feature_group.yaml", + }, + want: "my_fg", + wantErr: false, + }, + { + name: "test apply storage", + args: args{ + ctx: context.Background(), + coreCli: mockCore, + resource: "storage", + fileLocation: "testdata/valid_storage.yaml", + }, + want: "BIGQUERY1", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := apply(tt.args.ctx, tt.args.coreCli, tt.args.resource, tt.args.fileLocation) + if (err != nil) != tt.wantErr { + t.Errorf("apply() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("apply() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cli/feast/cmd/config_test.go b/cli/feast/cmd/config_test.go new file mode 100644 index 00000000000..5c6010c4e18 --- /dev/null +++ b/cli/feast/cmd/config_test.go @@ -0,0 +1,42 @@ +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/mitchellh/go-homedir" + "os" + "testing" +) + +func Test_initConfig(t *testing.T) { + tests := []struct { + name string + }{ + {"test creation of default config at $HOME/.feast"}, + } + + defaultFeastConfigFile, _ := homedir.Expand("~/.feast") + _ = os.Remove(defaultFeastConfigFile) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + initConfig() + _, err := os.Stat(defaultFeastConfigFile) + if os.IsNotExist(err) { + t.Errorf("initConfig should create config file at " + defaultFeastConfigFile) + } + }) + } +} diff --git a/cli/feast/cmd/list.go b/cli/feast/cmd/list.go index 684fb73ab4b..ac4a067e4ab 100644 --- a/cli/feast/cmd/list.go +++ b/cli/feast/cmd/list.go @@ -22,7 +22,7 @@ import ( "strings" "text/tabwriter" - "github.com/gojek/feast/cli/feast/pkg/util" + "github.com/gojek/feast/cli/feast/pkg/timeutil" "github.com/gojek/feast/protos/generated/go/feast/core" "github.com/golang/protobuf/ptypes/empty" @@ -144,7 +144,7 @@ func listJobs(ctx context.Context, cli core.JobServiceClient) error { fmt.Fprintf(w, "------\t----\t------\t------\t---\n") for _, job := range response.GetJobs() { fmt.Fprintf(w, strings.Join( - []string{job.Id, job.Type, job.Runner, job.Status, util.ParseAge(*job.Created)}, "\t")+"\n") + []string{job.Id, job.Type, job.Runner, job.Status, timeutil.DurationUntilNowInHumanFormat(*job.Created)}, "\t")+"\n") } w.Flush() return nil diff --git a/cli/feast/cmd/mock/README.md b/cli/feast/cmd/mock/README.md new file mode 100644 index 00000000000..3036d5856ab --- /dev/null +++ b/cli/feast/cmd/mock/README.md @@ -0,0 +1,11 @@ +Mock GRPC client is created using this library +https://github.com/golang/mock + +An example to generate `mock_core.go`: + +``` +cd $FEAST_REPO/cli/feast +mockgen --package mock github.com/gojek/feast/protos/generated/go/feast/core CoreServiceClient > cmd/mock/core_service.go +``` + +> We need to re-run `mockgen` whenever the protos for the services are updated \ No newline at end of file diff --git a/cli/feast/cmd/mock/core_service.go b/cli/feast/cmd/mock/core_service.go new file mode 100644 index 00000000000..702712c8479 --- /dev/null +++ b/cli/feast/cmd/mock/core_service.go @@ -0,0 +1,238 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/gojek/feast/protos/generated/go/feast/core (interfaces: CoreServiceClient) + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + core "github.com/gojek/feast/protos/generated/go/feast/core" + specs "github.com/gojek/feast/protos/generated/go/feast/specs" + gomock "github.com/golang/mock/gomock" + empty "github.com/golang/protobuf/ptypes/empty" + grpc "google.golang.org/grpc" + reflect "reflect" +) + +// MockCoreServiceClient is a mock of CoreServiceClient interface +type MockCoreServiceClient struct { + ctrl *gomock.Controller + recorder *MockCoreServiceClientMockRecorder +} + +// MockCoreServiceClientMockRecorder is the mock recorder for MockCoreServiceClient +type MockCoreServiceClientMockRecorder struct { + mock *MockCoreServiceClient +} + +// NewMockCoreServiceClient creates a new mock instance +func NewMockCoreServiceClient(ctrl *gomock.Controller) *MockCoreServiceClient { + mock := &MockCoreServiceClient{ctrl: ctrl} + mock.recorder = &MockCoreServiceClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockCoreServiceClient) EXPECT() *MockCoreServiceClientMockRecorder { + return m.recorder +} + +// ApplyEntity mocks base method +func (m *MockCoreServiceClient) ApplyEntity(arg0 context.Context, arg1 *specs.EntitySpec, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ApplyEntityResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ApplyEntity", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ApplyEntityResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyEntity indicates an expected call of ApplyEntity +func (mr *MockCoreServiceClientMockRecorder) ApplyEntity(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyEntity", reflect.TypeOf((*MockCoreServiceClient)(nil).ApplyEntity), varargs...) +} + +// ApplyFeature mocks base method +func (m *MockCoreServiceClient) ApplyFeature(arg0 context.Context, arg1 *specs.FeatureSpec, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ApplyFeatureResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ApplyFeature", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ApplyFeatureResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyFeature indicates an expected call of ApplyFeature +func (mr *MockCoreServiceClientMockRecorder) ApplyFeature(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyFeature", reflect.TypeOf((*MockCoreServiceClient)(nil).ApplyFeature), varargs...) +} + +// ApplyFeatureGroup mocks base method +func (m *MockCoreServiceClient) ApplyFeatureGroup(arg0 context.Context, arg1 *specs.FeatureGroupSpec, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ApplyFeatureGroupResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ApplyFeatureGroup", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ApplyFeatureGroupResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyFeatureGroup indicates an expected call of ApplyFeatureGroup +func (mr *MockCoreServiceClientMockRecorder) ApplyFeatureGroup(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyFeatureGroup", reflect.TypeOf((*MockCoreServiceClient)(nil).ApplyFeatureGroup), varargs...) +} + +// ApplyStorage mocks base method +func (m *MockCoreServiceClient) ApplyStorage(arg0 context.Context, arg1 *specs.StorageSpec, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ApplyStorageResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ApplyStorage", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ApplyStorageResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyStorage indicates an expected call of ApplyStorage +func (mr *MockCoreServiceClientMockRecorder) ApplyStorage(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyStorage", reflect.TypeOf((*MockCoreServiceClient)(nil).ApplyStorage), varargs...) +} + +// GetEntities mocks base method +func (m *MockCoreServiceClient) GetEntities(arg0 context.Context, arg1 *core.CoreServiceTypes_GetEntitiesRequest, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_GetEntitiesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetEntities", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_GetEntitiesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEntities indicates an expected call of GetEntities +func (mr *MockCoreServiceClientMockRecorder) GetEntities(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEntities", reflect.TypeOf((*MockCoreServiceClient)(nil).GetEntities), varargs...) +} + +// GetFeatures mocks base method +func (m *MockCoreServiceClient) GetFeatures(arg0 context.Context, arg1 *core.CoreServiceTypes_GetFeaturesRequest, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_GetFeaturesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetFeatures", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_GetFeaturesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFeatures indicates an expected call of GetFeatures +func (mr *MockCoreServiceClientMockRecorder) GetFeatures(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeatures", reflect.TypeOf((*MockCoreServiceClient)(nil).GetFeatures), varargs...) +} + +// GetStorage mocks base method +func (m *MockCoreServiceClient) GetStorage(arg0 context.Context, arg1 *core.CoreServiceTypes_GetStorageRequest, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_GetStorageResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetStorage", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_GetStorageResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStorage indicates an expected call of GetStorage +func (mr *MockCoreServiceClientMockRecorder) GetStorage(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStorage", reflect.TypeOf((*MockCoreServiceClient)(nil).GetStorage), varargs...) +} + +// ListEntities mocks base method +func (m *MockCoreServiceClient) ListEntities(arg0 context.Context, arg1 *empty.Empty, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ListEntitiesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListEntities", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ListEntitiesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListEntities indicates an expected call of ListEntities +func (mr *MockCoreServiceClientMockRecorder) ListEntities(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEntities", reflect.TypeOf((*MockCoreServiceClient)(nil).ListEntities), varargs...) +} + +// ListFeatures mocks base method +func (m *MockCoreServiceClient) ListFeatures(arg0 context.Context, arg1 *empty.Empty, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ListFeaturesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListFeatures", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ListFeaturesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListFeatures indicates an expected call of ListFeatures +func (mr *MockCoreServiceClientMockRecorder) ListFeatures(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFeatures", reflect.TypeOf((*MockCoreServiceClient)(nil).ListFeatures), varargs...) +} + +// ListStorage mocks base method +func (m *MockCoreServiceClient) ListStorage(arg0 context.Context, arg1 *empty.Empty, arg2 ...grpc.CallOption) (*core.CoreServiceTypes_ListStorageResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListStorage", varargs...) + ret0, _ := ret[0].(*core.CoreServiceTypes_ListStorageResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListStorage indicates an expected call of ListStorage +func (mr *MockCoreServiceClientMockRecorder) ListStorage(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListStorage", reflect.TypeOf((*MockCoreServiceClient)(nil).ListStorage), varargs...) +} diff --git a/cli/feast/cmd/testdata/invalid_entity.yaml b/cli/feast/cmd/testdata/invalid_entity.yaml new file mode 100644 index 00000000000..4fafacc87b0 --- /dev/null +++ b/cli/feast/cmd/testdata/invalid_entity.yaml @@ -0,0 +1,5 @@ +name: myentity +description: test entity with tag +tags: + - tag1 + tag2 \ No newline at end of file diff --git a/cli/feast/cmd/testdata/valid_entity.yaml b/cli/feast/cmd/testdata/valid_entity.yaml new file mode 100644 index 00000000000..97f215711f0 --- /dev/null +++ b/cli/feast/cmd/testdata/valid_entity.yaml @@ -0,0 +1,5 @@ +name : myentity +description: test entity with tag +tags: + - tag1 + - tag2 \ No newline at end of file diff --git a/cli/feast/cmd/testdata/valid_entity_no_tag.yaml b/cli/feast/cmd/testdata/valid_entity_no_tag.yaml new file mode 100644 index 00000000000..828a768b291 --- /dev/null +++ b/cli/feast/cmd/testdata/valid_entity_no_tag.yaml @@ -0,0 +1,2 @@ +name : myentity +description: test entity without tag \ No newline at end of file diff --git a/cli/feast/cmd/testdata/valid_feature.yaml b/cli/feast/cmd/testdata/valid_feature.yaml new file mode 100644 index 00000000000..a54915641bc --- /dev/null +++ b/cli/feast/cmd/testdata/valid_feature.yaml @@ -0,0 +1,12 @@ +id: myentity.feature_bool_redis1 +name: feature_bool_redis1 +entity: myentity +owner: bob@example.com +description: test entity. +valueType: BOOL +uri: https://github.com/bob/example +dataStores: + serving: + id: REDIS1 + warehouse: + id: BIGQUERY1 \ No newline at end of file diff --git a/cli/feast/cmd/testdata/valid_feature_group.yaml b/cli/feast/cmd/testdata/valid_feature_group.yaml new file mode 100644 index 00000000000..f877d209fec --- /dev/null +++ b/cli/feast/cmd/testdata/valid_feature_group.yaml @@ -0,0 +1,7 @@ +id: my_fg +tags: ["tag1", "tag2"] +dataStores: + serving: + id: "REDIS1" + warehouse: + id: "BIGQUERY1" \ No newline at end of file diff --git a/cli/feast/cmd/testdata/valid_storage.yaml b/cli/feast/cmd/testdata/valid_storage.yaml new file mode 100644 index 00000000000..96ae9e40f78 --- /dev/null +++ b/cli/feast/cmd/testdata/valid_storage.yaml @@ -0,0 +1,6 @@ +id: BIGQUERY1 +type: bigquery +options: + dataset: "feast" + project: "gcp-project" + tempLocation: "gs://feast-storage" \ No newline at end of file diff --git a/cli/feast/pkg/printer/job.go b/cli/feast/pkg/printer/job.go index 56f9ac8e8bf..45af68519bf 100644 --- a/cli/feast/pkg/printer/job.go +++ b/cli/feast/pkg/printer/job.go @@ -18,7 +18,7 @@ import ( "fmt" "strings" - "github.com/gojek/feast/cli/feast/pkg/util" + "github.com/gojek/feast/cli/feast/pkg/timeutil" "github.com/gojek/feast/protos/generated/go/feast/core" ) @@ -30,7 +30,7 @@ func PrintJobDetail(jobDetail *core.JobServiceTypes_JobDetail) string { fmt.Sprintf("%s:\t%s", "Type", jobDetail.GetType()), fmt.Sprintf("%s:\t%s", "Runner", jobDetail.GetRunner()), fmt.Sprintf("%s:\t%s", "Status", jobDetail.GetStatus()), - fmt.Sprintf("%s:\t%s", "Age", util.ParseAge(*jobDetail.GetCreated())), + fmt.Sprintf("%s:\t%s", "Age", timeutil.DurationUntilNowInHumanFormat(*jobDetail.GetCreated())), } lines = append(lines, "Metrics:") for k, v := range jobDetail.GetMetrics() { diff --git a/cli/feast/pkg/printer/job_test.go b/cli/feast/pkg/printer/job_test.go new file mode 100644 index 00000000000..4b930a56d7b --- /dev/null +++ b/cli/feast/pkg/printer/job_test.go @@ -0,0 +1,66 @@ +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package printer + +import ( + "testing" +) + +func Test_printEntity(t *testing.T) { + type args struct { + entityName string + metrics map[string]float64 + } + tests := []struct { + name string + args args + want string + }{ + {"1 feature 1 metric value", args{"entity1", map[string]float64{"entity:entity1:value1": 89.0}}, "- Name: entity1\n Metrics: \n value1: 89.0"}, + {"1 feature 2 metric value", args{"entity1", map[string]float64{"entity:entity1:value1": 89.0, "entity:entity1:value2": 90.0}}, "- Name: entity1\n Metrics: \n value1: 89.0\n value2: 90.0"}, + {"1 feature 0 metric value", args{"entity1", map[string]float64{"entity:entity2:value1": 89.0, "entity:entity3:value2": 90.0}}, "- Name: entity1\n Metrics: "}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := printEntity(tt.args.entityName, tt.args.metrics); got != tt.want { + t.Errorf("printEntity() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_printFeature(t *testing.T) { + type args struct { + featureName string + metrics map[string]float64 + } + tests := []struct { + name string + args args + want string + }{ + {"1 feature 1 metric value", args{"feature1", map[string]float64{"feature:feature1:value1": 89.0, "feature:feature2:value1": 90.0}}, "- Id: feature1\n Metrics: \n value1: 89.0"}, + {"1 feature 2 metric value", args{"feature1", map[string]float64{"feature:feature1:value1": 89.0, "feature:feature1:value2": 90.0}}, "- Id: feature1\n Metrics: \n value1: 89.0\n value2: 90.0"}, + {"1 feature 0 metric value", args{"feature1", map[string]float64{"feature:feature2:value1": 89.0, "feature:feature3:value1": 90.0}}, "- Id: feature1\n Metrics: "}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := printFeature(tt.args.featureName, tt.args.metrics); got != tt.want { + t.Errorf("printEntity() = %v, want %v", got, tt.want) + println(len(got), len(tt.want)) + } + }) + } +} diff --git a/cli/feast/pkg/printer/specs.go b/cli/feast/pkg/printer/specs.go index 8c88bffbfeb..6730938ba64 100644 --- a/cli/feast/pkg/printer/specs.go +++ b/cli/feast/pkg/printer/specs.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/gojek/feast/cli/feast/pkg/util" + "github.com/gojek/feast/cli/feast/pkg/timeutil" "github.com/gojek/feast/protos/generated/go/feast/core" ) @@ -29,8 +29,8 @@ func PrintFeatureDetail(featureDetail *core.UIServiceTypes_FeatureDetail) string lines = append(lines, fmt.Sprintf(" %s:\t%s", "Warehouse", wh.GetId())) } } - lines = append(lines, fmt.Sprintf("%s:\t%s", "Created", util.ParseTimestamp(*featureDetail.GetCreated()))) - lines = append(lines, fmt.Sprintf("%s:\t%s", "LastUpdated", util.ParseTimestamp(*featureDetail.GetLastUpdated()))) + lines = append(lines, fmt.Sprintf("%s:\t%s", "Created", timeutil.FormatToRFC3339(*featureDetail.GetCreated()))) + lines = append(lines, fmt.Sprintf("%s:\t%s", "LastUpdated", timeutil.FormatToRFC3339(*featureDetail.GetLastUpdated()))) if jobs := featureDetail.GetJobs(); len(jobs) > 0 { lines = append(lines, "Related Jobs:") for _, job := range jobs { @@ -56,7 +56,7 @@ func PrintEntityDetail(entityDetail *core.UIServiceTypes_EntityDetail) string { if tags := spec.GetTags(); len(tags) > 0 { lines = append(lines, fmt.Sprintf("Tags: %s", strings.Join(tags, ","))) } - lines = append(lines, fmt.Sprintf("%s:\t%s", "LastUpdated", util.ParseTimestamp(*entityDetail.GetLastUpdated()))) + lines = append(lines, fmt.Sprintf("%s:\t%s", "LastUpdated", timeutil.FormatToRFC3339(*entityDetail.GetLastUpdated()))) lines = append(lines, "Related Jobs:") for _, job := range entityDetail.GetJobs() { lines = append(lines, fmt.Sprintf("- %s", job)) @@ -78,7 +78,7 @@ func PrintStorageDetail(storageDetail *core.UIServiceTypes_StorageDetail) string for k, v := range spec.GetOptions() { lines = append(lines, fmt.Sprintf(" %s: %s", k, v)) } - lines = append(lines, fmt.Sprintf("%s:\t%s", "LastUpdated", util.ParseTimestamp(*storageDetail.GetLastUpdated()))) + lines = append(lines, fmt.Sprintf("%s:\t%s", "LastUpdated", timeutil.FormatToRFC3339(*storageDetail.GetLastUpdated()))) out := strings.Join(lines, "\n") fmt.Println(out) return out diff --git a/cli/feast/pkg/printer/printer_test.go b/cli/feast/pkg/printer/specs_test.go similarity index 89% rename from cli/feast/pkg/printer/printer_test.go rename to cli/feast/pkg/printer/specs_test.go index 07332324c11..60551332112 100644 --- a/cli/feast/pkg/printer/printer_test.go +++ b/cli/feast/pkg/printer/specs_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/gojek/feast/cli/feast/pkg/util" + "github.com/gojek/feast/cli/feast/pkg/timeutil" "github.com/golang/protobuf/ptypes/timestamp" @@ -58,8 +58,8 @@ LastUpdated: %s Related Jobs: - job1 - job2`, - util.ParseTimestamp(timestamp.Timestamp{Seconds: 1}), - util.ParseTimestamp(timestamp.Timestamp{Seconds: 1})), + timeutil.FormatToRFC3339(timestamp.Timestamp{Seconds: 1}), + timeutil.FormatToRFC3339(timestamp.Timestamp{Seconds: 1})), }, { name: "no storage", input: &core.UIServiceTypes_FeatureDetail{ @@ -88,8 +88,8 @@ LastUpdated: %s Related Jobs: - job1 - job2`, - util.ParseTimestamp(timestamp.Timestamp{Seconds: 1}), - util.ParseTimestamp(timestamp.Timestamp{Seconds: 1})), + timeutil.FormatToRFC3339(timestamp.Timestamp{Seconds: 1}), + timeutil.FormatToRFC3339(timestamp.Timestamp{Seconds: 1})), }, } @@ -121,7 +121,7 @@ LastUpdated: %s Related Jobs: - job1 - job2`, - util.ParseTimestamp(timestamp.Timestamp{Seconds: 1})) + timeutil.FormatToRFC3339(timestamp.Timestamp{Seconds: 1})) if out != expected { t.Errorf("Expected output:\n%s \nActual:\n%s \n", expected, out) } @@ -146,7 +146,7 @@ Options: option1: value1 option2: value2 LastUpdated: %s`, - util.ParseTimestamp(timestamp.Timestamp{Seconds: 1})) + timeutil.FormatToRFC3339(timestamp.Timestamp{Seconds: 1})) if out != expected { t.Errorf("Expected output:\n%s \nActual:\n%s \n", expected, out) } diff --git a/cli/feast/pkg/util/utils.go b/cli/feast/pkg/timeutil/timeutil.go similarity index 75% rename from cli/feast/pkg/util/utils.go rename to cli/feast/pkg/timeutil/timeutil.go index a48ef6d7415..379a3367e3c 100644 --- a/cli/feast/pkg/util/utils.go +++ b/cli/feast/pkg/timeutil/timeutil.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package timeutil import ( "fmt" @@ -22,9 +22,8 @@ import ( "github.com/golang/protobuf/ptypes/timestamp" ) -// ParseAge parses a given timestamp to a human readable duration. -func ParseAge(createdTimestamp timestamp.Timestamp) string { - timeSinceCreation := float64(time.Now().Unix() - createdTimestamp.GetSeconds()) +func DurationUntilNowInHumanFormat(timestamp timestamp.Timestamp) string { + timeSinceCreation := float64(time.Now().Unix() - timestamp.GetSeconds()) if days := math.Floor(timeSinceCreation / float64(8.64e+4)); days > 0 { return fmt.Sprintf("%dd", int(days)) } else if hours := math.Floor(timeSinceCreation / float64(3.6e+3)); hours > 0 { @@ -33,8 +32,7 @@ func ParseAge(createdTimestamp timestamp.Timestamp) string { return fmt.Sprintf("%dm", int(math.Floor(timeSinceCreation/float64(60)))) } -// ParseTimestamp parses a given timestamp to a human readable format. -func ParseTimestamp(ts timestamp.Timestamp) string { +func FormatToRFC3339(ts timestamp.Timestamp) string { t := time.Unix(ts.GetSeconds(), int64(ts.GetNanos())) return t.Format(time.RFC3339) } diff --git a/cli/feast/pkg/timeutil/timeutil_test.go b/cli/feast/pkg/timeutil/timeutil_test.go new file mode 100644 index 00000000000..907f60f5e61 --- /dev/null +++ b/cli/feast/pkg/timeutil/timeutil_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package timeutil + +import ( + "testing" + "time" + + "github.com/golang/protobuf/ptypes/timestamp" +) + +func TestDurationUntilNowInHumanFormat(t *testing.T) { + type args struct { + createdTimestamp timestamp.Timestamp + } + + tests := []struct { + name string + args args + want string + }{ + {"59 second before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-1 * time.Duration(time.Second)).Unix()}}, "0m"}, + {"1 minute before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-1 * time.Duration(time.Minute)).Unix()}}, "1m"}, + {"30 minutes before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-30 * time.Duration(time.Minute)).Unix()}}, "30m"}, + {"1 hour before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-1 * time.Duration(time.Hour)).Unix()}}, "1h"}, + {"2 hours before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-2 * time.Duration(time.Hour)).Unix()}}, "2h"}, + {"1 day before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-24 * time.Duration(time.Hour)).Unix()}}, "1d"}, + {"2 days before now", args{timestamp.Timestamp{Seconds: time.Now().Add(-48 * time.Duration(time.Hour)).Unix()}}, "2d"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := DurationUntilNowInHumanFormat(tt.args.createdTimestamp); got != tt.want { + t.Errorf("DurationUntilNowInHumanFormat() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/core/src/main/java/feast/core/http/HealthController.java b/core/src/main/java/feast/core/http/HealthController.java index 1372d09c1cd..e7183320705 100644 --- a/core/src/main/java/feast/core/http/HealthController.java +++ b/core/src/main/java/feast/core/http/HealthController.java @@ -63,7 +63,8 @@ public ResponseEntity healthz() { return ResponseEntity.ok("healthy"); } log.error("Unable to reach DB"); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Unable to establish connection with DB"); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body("Unable to establish connection with DB"); } catch (SQLException e) { log.error("Unable to reach DB: {}", e); return ResponseEntity.status(INTERNAL_SERVER_ERROR).body(e.getMessage()); diff --git a/core/src/main/java/feast/core/model/FeatureInfo.java b/core/src/main/java/feast/core/model/FeatureInfo.java index ac497019d3d..e10944df1d5 100644 --- a/core/src/main/java/feast/core/model/FeatureInfo.java +++ b/core/src/main/java/feast/core/model/FeatureInfo.java @@ -25,6 +25,7 @@ import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.ValueProto.ValueType; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; import lombok.Setter; diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java index d80b5b7b42c..ccd30ddfacc 100644 --- a/core/src/test/java/feast/core/CoreApplicationTest.java +++ b/core/src/test/java/feast/core/CoreApplicationTest.java @@ -48,7 +48,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java index 61f55202ad9..6dcef362a2a 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java @@ -49,7 +49,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java index 6c8ee325aac..46b2e151a05 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java @@ -49,7 +49,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.serving.type=redis", "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", diff --git a/core/src/test/java/feast/core/http/HealthControllerTest.java b/core/src/test/java/feast/core/http/HealthControllerTest.java new file mode 100644 index 00000000000..030033b789c --- /dev/null +++ b/core/src/test/java/feast/core/http/HealthControllerTest.java @@ -0,0 +1,65 @@ +package feast.core.http; + +import org.junit.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.*; + +public class HealthControllerTest { + @Test + public void ping() { + HealthController healthController = new HealthController(null); + assertEquals(ResponseEntity.ok("pong"), healthController.ping()); + } + + @Test + public void healthz() { + assertEquals(ResponseEntity.ok("healthy"), mockHealthyController().healthz()); + assertEquals( + ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body("Unable to establish connection with DB"), + mockUnhealthyControllerBecauseInvalidConn().healthz()); + assertEquals( + ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("mocked sqlexception"), + mockUnhealthyControllerBecauseSQLException().healthz()); + } + + private HealthController mockHealthyController() { + DataSource mockDataSource = mock(DataSource.class); + Connection mockConnection = mock(Connection.class); + try { + when(mockConnection.isValid(any(int.class))).thenReturn(Boolean.TRUE); + when(mockDataSource.getConnection()).thenReturn(mockConnection); + } catch (Exception e) { + e.printStackTrace(); + } + return new HealthController(mockDataSource); + } + + private HealthController mockUnhealthyControllerBecauseInvalidConn() { + DataSource mockDataSource = mock(DataSource.class); + Connection mockConnection = mock(Connection.class); + try { + when(mockConnection.isValid(any(int.class))).thenReturn(Boolean.FALSE); + when(mockDataSource.getConnection()).thenReturn(mockConnection); + } catch (Exception ignored) { + } + return new HealthController(mockDataSource); + } + + private HealthController mockUnhealthyControllerBecauseSQLException() { + DataSource mockDataSource = mock(DataSource.class); + Connection mockConnection = mock(Connection.class); + try { + when(mockDataSource.getConnection()).thenThrow(new SQLException("mocked sqlexception")); + } catch (SQLException ignored) { + } + return new HealthController(mockDataSource); + } +} diff --git a/core/src/test/java/feast/core/http/UiServiceControllerTest.java b/core/src/test/java/feast/core/http/UiServiceControllerTest.java new file mode 100644 index 00000000000..6a858940df6 --- /dev/null +++ b/core/src/test/java/feast/core/http/UiServiceControllerTest.java @@ -0,0 +1,177 @@ +package feast.core.http; + +import feast.core.UIServiceProto; +import feast.core.UIServiceProto.UIServiceTypes.EntityDetail; +import feast.core.UIServiceProto.UIServiceTypes.FeatureDetail; +import feast.core.UIServiceProto.UIServiceTypes.FeatureGroupDetail; +import feast.core.UIServiceProto.UIServiceTypes.StorageDetail; +import feast.core.model.EntityInfo; +import feast.core.model.FeatureGroupInfo; +import feast.core.model.FeatureInfo; +import feast.core.model.StorageInfo; +import feast.core.service.JobManagementService; +import feast.core.service.SpecService; +import feast.specs.FeatureSpecProto; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static feast.specs.FeatureSpecProto.*; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class UiServiceControllerTest { + private UiServiceController goodUiServiceController; + private UiServiceController badUiServiceController; + + @Before + public void setUp() throws Exception { + FeatureInfo mockFeatureInfo = mock(FeatureInfo.class); + when(mockFeatureInfo.getFeatureDetail()).thenReturn(FeatureDetail.getDefaultInstance()); + when(mockFeatureInfo.getFeatureSpec()).thenReturn(FeatureSpec.getDefaultInstance()); + when(mockFeatureInfo.resolve()).thenReturn(mockFeatureInfo); + + EntityInfo mockEntityInfo = mock(EntityInfo.class); + when(mockEntityInfo.getEntityDetail()).thenReturn(EntityDetail.getDefaultInstance()); + + FeatureGroupInfo mockFeatureGroupInfo = mock(FeatureGroupInfo.class); + when(mockFeatureGroupInfo.getFeatureGroupDetail()) + .thenReturn(FeatureGroupDetail.getDefaultInstance()); + + StorageInfo mockStorageInfo = mock(StorageInfo.class); + when(mockStorageInfo.getStorageDetail()).thenReturn(StorageDetail.getDefaultInstance()); + + SpecService goodMockSpecService = mock(SpecService.class); + when(goodMockSpecService.listFeatures()).thenReturn(Collections.singletonList(mockFeatureInfo)); + when(goodMockSpecService.getFeatures(Collections.singletonList("1"))) + .thenReturn(Collections.singletonList(mockFeatureInfo)); + when(goodMockSpecService.listFeatureGroups()) + .thenReturn(Collections.singletonList(mockFeatureGroupInfo)); + when(goodMockSpecService.getFeatureGroups(Collections.singletonList("1"))) + .thenReturn(Collections.singletonList(mockFeatureGroupInfo)); + when(goodMockSpecService.listEntities()).thenReturn(Collections.singletonList(mockEntityInfo)); + when(goodMockSpecService.getEntities(Collections.singletonList("1"))) + .thenReturn(Collections.singletonList(mockEntityInfo)); + when(goodMockSpecService.listStorage()).thenReturn(Collections.singletonList(mockStorageInfo)); + when(goodMockSpecService.getStorage(Collections.singletonList("1"))) + .thenReturn(Collections.singletonList(mockStorageInfo)); + + JobManagementService goodMockJobMangementService = mock(JobManagementService.class); + + goodUiServiceController = + new UiServiceController(goodMockSpecService, goodMockJobMangementService); + + SpecService badMockSpecService = mock(SpecService.class); + when(badMockSpecService.listFeatures()).thenReturn(null); + when(badMockSpecService.getFeatures(Collections.singletonList("1"))).thenReturn(null); + when(badMockSpecService.listFeatureGroups()).thenReturn(null); + when(badMockSpecService.getFeatureGroups(Collections.singletonList("1"))).thenReturn(null); + when(badMockSpecService.listEntities()).thenReturn(null); + when(badMockSpecService.getEntities(Collections.singletonList("1"))).thenReturn(null); + when(badMockSpecService.listStorage()).thenReturn(null); + when(badMockSpecService.getStorage(Collections.singletonList("1"))).thenReturn(null); + + JobManagementService badMockJobMangementService = mock(JobManagementService.class); + + badUiServiceController = + new UiServiceController(badMockSpecService, badMockJobMangementService); + } + + @Test + public void listFeatures() { + FeatureDetail expected = FeatureDetail.getDefaultInstance(); + FeatureDetail actual = goodUiServiceController.listFeatures().getFeaturesList().get(0); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void listFeaturesWithExecption() { + badUiServiceController.listFeatures(); + } + + @Test + public void getFeature() { + FeatureDetail expected = FeatureDetail.getDefaultInstance(); + FeatureDetail actual = goodUiServiceController.getFeature("1").getFeature(); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void getFeatureWithException() { + badUiServiceController.getFeature("1"); + } + + @Test + public void listFeatureGroups() { + FeatureGroupDetail expected = FeatureGroupDetail.getDefaultInstance(); + FeatureGroupDetail actual = goodUiServiceController.listFeatureGroups().getFeatureGroups(0); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void listFeatureGroupsWithException() { + badUiServiceController.listFeatureGroups(); + } + + @Test + public void getFeatureGroup() { + FeatureGroupDetail expected = FeatureGroupDetail.getDefaultInstance(); + FeatureGroupDetail actual = goodUiServiceController.getFeatureGroup("1").getFeatureGroup(); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void getFeatureGroupWithException() { + badUiServiceController.getFeatureGroup("1"); + } + + @Test + public void listEntities() { + EntityDetail expected = EntityDetail.getDefaultInstance(); + EntityDetail actual = goodUiServiceController.listEntities().getEntitiesList().get(0); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void listEntitiesWithException() { + badUiServiceController.listEntities(); + } + + @Test + public void getEntity() { + EntityDetail expected = EntityDetail.getDefaultInstance(); + EntityDetail actual = goodUiServiceController.getEntity("1").getEntity(); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void getEntityWithException() { + badUiServiceController.getEntity("1"); + } + + @Test + public void listStorage() { + StorageDetail expected = StorageDetail.getDefaultInstance(); + StorageDetail actual = goodUiServiceController.listStorage().getStorage(0); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void listStorageWithException() { + badUiServiceController.listStorage(); + } + + @Test + public void getStorage() { + StorageDetail expected = StorageDetail.getDefaultInstance(); + StorageDetail actual = goodUiServiceController.getStorage("1").getStorage(); + assertEquals(expected, actual); + } + + @Test(expected = Exception.class) + public void getStorageWithException() { + badUiServiceController.getStorage("1"); + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000000..84acd86f2b2 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module github.com/gojek/feast + +require ( + github.com/ghodss/yaml v1.0.0 + github.com/golang/mock v1.2.0 + github.com/golang/protobuf v1.3.1 + github.com/google/go-cmp v0.2.0 + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 + github.com/spf13/cobra v0.0.3 + github.com/spf13/viper v1.3.2 + google.golang.org/grpc v1.19.1 + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000000..c169d036193 --- /dev/null +++ b/go.sum @@ -0,0 +1,76 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.1 h1:TrBcJ1yqAl1G++wO39nD/qtgpsW9/1+QGrluyMGEYgM= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/protos/Makefile b/protos/Makefile index e2f61a8c37e..e49b8c32d6a 100644 --- a/protos/Makefile +++ b/protos/Makefile @@ -4,7 +4,7 @@ dirs = core serving specs storage types service_dirs = core serving gen-go: - @$(foreach dir,$(dirs),protoc -I/usr/local/include -I. --go_out=plugins=grpc:$$GOPATH/src feast/$(dir)/*.proto;) + @$(foreach dir,$(dirs),protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:generated/go feast/$(dir)/*.proto;) gen-python: pip install grpcio-tools diff --git a/protos/generated/go/feast/core/CoreService.pb.go b/protos/generated/go/feast/core/CoreService.pb.go index 4037da58af9..f903333006e 100644 --- a/protos/generated/go/feast/core/CoreService.pb.go +++ b/protos/generated/go/feast/core/CoreService.pb.go @@ -1,17 +1,18 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/core/CoreService.proto -package core // import "github.com/gojek/feast/protos/generated/go/feast/core" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import specs "github.com/gojek/feast/protos/generated/go/feast/specs" -import empty "github.com/golang/protobuf/ptypes/empty" +package core import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + specs "github.com/gojek/feast/protos/generated/go/feast/specs" + proto "github.com/golang/protobuf/proto" + empty "github.com/golang/protobuf/ptypes/empty" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -23,7 +24,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type CoreServiceTypes struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -35,16 +36,17 @@ func (m *CoreServiceTypes) Reset() { *m = CoreServiceTypes{} } func (m *CoreServiceTypes) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes) ProtoMessage() {} func (*CoreServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0} + return fileDescriptor_d9be266444105411, []int{0} } + func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes.Unmarshal(m, b) } func (m *CoreServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes.Merge(dst, src) +func (m *CoreServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes.Merge(m, src) } func (m *CoreServiceTypes) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes.Size(m) @@ -66,16 +68,17 @@ func (m *CoreServiceTypes_GetEntitiesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetEntitiesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 0} + return fileDescriptor_d9be266444105411, []int{0, 0} } + func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Unmarshal(m, b) } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(m, src) } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Size(m) @@ -104,16 +107,17 @@ func (m *CoreServiceTypes_GetEntitiesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 1} + return fileDescriptor_d9be266444105411, []int{0, 1} } + func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(m, src) } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Size(m) @@ -142,16 +146,17 @@ func (m *CoreServiceTypes_ListEntitiesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 2} + return fileDescriptor_d9be266444105411, []int{0, 2} } + func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(m, src) } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Size(m) @@ -181,16 +186,17 @@ func (m *CoreServiceTypes_GetFeaturesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 3} + return fileDescriptor_d9be266444105411, []int{0, 3} } + func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Unmarshal(m, b) } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(m, src) } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Size(m) @@ -219,16 +225,17 @@ func (m *CoreServiceTypes_GetFeaturesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 4} + return fileDescriptor_d9be266444105411, []int{0, 4} } + func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(m, src) } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Size(m) @@ -257,16 +264,17 @@ func (m *CoreServiceTypes_ListFeaturesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 5} + return fileDescriptor_d9be266444105411, []int{0, 5} } + func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(m, src) } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Size(m) @@ -296,16 +304,17 @@ func (m *CoreServiceTypes_GetStorageRequest) Reset() { *m = CoreServiceT func (m *CoreServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageRequest) ProtoMessage() {} func (*CoreServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 6} + return fileDescriptor_d9be266444105411, []int{0, 6} } + func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Unmarshal(m, b) } func (m *CoreServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(m, src) } func (m *CoreServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Size(m) @@ -334,16 +343,17 @@ func (m *CoreServiceTypes_GetStorageResponse) Reset() { *m = CoreService func (m *CoreServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 7} + return fileDescriptor_d9be266444105411, []int{0, 7} } + func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Size(m) @@ -372,16 +382,17 @@ func (m *CoreServiceTypes_ListStorageResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 8} + return fileDescriptor_d9be266444105411, []int{0, 8} } + func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Size(m) @@ -411,16 +422,17 @@ func (m *CoreServiceTypes_ApplyEntityResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ApplyEntityResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyEntityResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 9} + return fileDescriptor_d9be266444105411, []int{0, 9} } + func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Size(m) @@ -450,16 +462,17 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyFeatureResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyFeatureResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 10} + return fileDescriptor_d9be266444105411, []int{0, 10} } + func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Size(m) @@ -493,16 +506,17 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) String() string { } func (*CoreServiceTypes_ApplyFeatureGroupResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 11} + return fileDescriptor_d9be266444105411, []int{0, 11} } + func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Size(m) @@ -532,16 +546,17 @@ func (m *CoreServiceTypes_ApplyStorageResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 12} + return fileDescriptor_d9be266444105411, []int{0, 12} } + func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Unmarshal(m, b) } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Size(m) @@ -576,6 +591,50 @@ func init() { proto.RegisterType((*CoreServiceTypes_ApplyStorageResponse)(nil), "feast.core.CoreServiceTypes.ApplyStorageResponse") } +func init() { proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) } + +var fileDescriptor_d9be266444105411 = []byte{ + // 602 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5b, 0x6f, 0x12, 0x41, + 0x14, 0x86, 0x34, 0x69, 0xe0, 0x40, 0x4c, 0x3b, 0x34, 0x8a, 0x23, 0x35, 0xcd, 0x26, 0x36, 0x7d, + 0x9a, 0xa9, 0xbd, 0xf8, 0xe4, 0x8b, 0x6d, 0x2a, 0xd1, 0x36, 0xc6, 0x2c, 0xfa, 0x60, 0x8d, 0x0f, + 0x5c, 0x0e, 0xeb, 0x2a, 0x30, 0xeb, 0xce, 0x60, 0xc2, 0xb3, 0xff, 0xcf, 0xdf, 0x64, 0x98, 0x1d, + 0x66, 0x07, 0xd8, 0xed, 0x12, 0xc3, 0x1b, 0x9c, 0xdb, 0x37, 0xdf, 0x39, 0xf3, 0x9d, 0x59, 0x68, + 0x0d, 0xb1, 0x2b, 0x15, 0xef, 0x8b, 0x18, 0xf9, 0xb5, 0x88, 0xb1, 0x83, 0xf1, 0xef, 0xb0, 0x8f, + 0x2c, 0x8a, 0x85, 0x12, 0x04, 0xb4, 0x97, 0xcd, 0xbd, 0xd4, 0x44, 0xca, 0x08, 0xfb, 0x92, 0xdf, + 0x4c, 0x54, 0xa8, 0x66, 0x9d, 0x08, 0xfb, 0x49, 0x24, 0x3d, 0x74, 0xbd, 0x6f, 0xb1, 0xab, 0xa6, + 0x31, 0x3a, 0x6e, 0x2f, 0xc3, 0xdd, 0x8e, 0xc5, 0x34, 0xca, 0x2b, 0xd1, 0x51, 0x22, 0xee, 0x06, + 0x6e, 0x89, 0x67, 0x81, 0x10, 0xc1, 0x08, 0xb9, 0xfe, 0xd7, 0x9b, 0x0e, 0x39, 0x8e, 0x23, 0x35, + 0x4b, 0x9c, 0xde, 0xdf, 0x5d, 0xd8, 0x73, 0x8e, 0xff, 0x69, 0x16, 0xa1, 0xa4, 0xc7, 0x40, 0xda, + 0xa8, 0xf4, 0x51, 0x43, 0x94, 0x3e, 0xfe, 0x9a, 0xa2, 0x54, 0x64, 0x0f, 0x76, 0xc2, 0x81, 0x6c, + 0x96, 0x8f, 0x76, 0x4e, 0xaa, 0xfe, 0xfc, 0x27, 0x7d, 0x0f, 0x8d, 0xa5, 0x38, 0x19, 0x89, 0x89, + 0x44, 0x72, 0x0e, 0x15, 0x34, 0x36, 0x1d, 0x5d, 0x3b, 0x7b, 0xc2, 0x92, 0x7e, 0xe8, 0x23, 0xb2, + 0xb4, 0x07, 0xbe, 0x0d, 0xa4, 0xb7, 0x70, 0x70, 0x17, 0xca, 0x2d, 0x15, 0x4b, 0x08, 0x98, 0x76, + 0x3d, 0x40, 0xe0, 0x56, 0x13, 0x48, 0xe3, 0x0c, 0xe6, 0x05, 0x54, 0x86, 0xc6, 0x66, 0x30, 0x9b, + 0x4b, 0x98, 0xce, 0x98, 0x7c, 0x1b, 0x49, 0xef, 0x12, 0x06, 0x5b, 0xaa, 0xf6, 0x02, 0xf6, 0xdb, + 0xa8, 0xcc, 0x34, 0xf3, 0x19, 0xf8, 0x9a, 0xa9, 0x0d, 0x33, 0x90, 0xaf, 0xa1, 0x2e, 0xd3, 0x7b, + 0x90, 0x0d, 0xeb, 0x5c, 0x14, 0x7f, 0x29, 0x9a, 0x76, 0xa0, 0x31, 0x27, 0xb2, 0xdd, 0xa2, 0x97, + 0xd0, 0x78, 0x13, 0x45, 0xa3, 0x59, 0x32, 0x2f, 0x5b, 0xf4, 0x39, 0x80, 0x9e, 0xda, 0xec, 0x43, + 0x77, 0x8c, 0xcd, 0xf2, 0x51, 0xf9, 0xa4, 0xea, 0x3b, 0x16, 0x7a, 0x01, 0x07, 0x3a, 0xcd, 0x34, + 0xc9, 0xe6, 0xb5, 0xa0, 0x6a, 0x5a, 0xf5, 0x6e, 0x60, 0xd2, 0x52, 0x03, 0xbd, 0x86, 0xa7, 0x6e, + 0x96, 0x16, 0x8c, 0x4d, 0x3d, 0x86, 0x47, 0x43, 0xc7, 0x6e, 0xf3, 0x57, 0xac, 0x16, 0x7a, 0xb5, + 0x0f, 0x2d, 0xa8, 0x1a, 0x66, 0x29, 0xb4, 0x35, 0x9c, 0xfd, 0xa9, 0x40, 0xcd, 0x11, 0x14, 0x89, + 0xa1, 0xe6, 0x68, 0x84, 0x70, 0x96, 0x6e, 0x06, 0xb6, 0x2a, 0x3c, 0xb6, 0xae, 0x3a, 0x7a, 0xba, + 0x79, 0x42, 0x72, 0x3e, 0xaf, 0x44, 0xbe, 0x42, 0xdd, 0xd5, 0x12, 0x79, 0xcc, 0x92, 0x15, 0xc0, + 0x16, 0x2b, 0x80, 0xdd, 0xcc, 0x57, 0x00, 0x7d, 0xf9, 0x60, 0xed, 0x2c, 0x39, 0x7a, 0x25, 0x43, + 0x68, 0x71, 0xcb, 0x8b, 0x09, 0xad, 0xa8, 0xb0, 0x98, 0xd0, 0xaa, 0x80, 0x52, 0x42, 0x16, 0xf4, + 0xff, 0x09, 0x65, 0x14, 0x17, 0x00, 0xa9, 0x84, 0x08, 0x2b, 0x3a, 0xde, 0xb2, 0x24, 0x29, 0xdf, + 0x38, 0xde, 0x02, 0x7e, 0x81, 0x9a, 0xa3, 0xaf, 0x5c, 0x32, 0xa7, 0x85, 0x64, 0xd6, 0x4b, 0x7f, + 0x83, 0xba, 0x7b, 0xf1, 0x49, 0xee, 0xa6, 0x29, 0x68, 0x55, 0x96, 0xe6, 0xbc, 0x12, 0x19, 0xc1, + 0xfe, 0x9a, 0xae, 0xc8, 0x61, 0x16, 0x86, 0x7d, 0xa3, 0xe8, 0xab, 0x8d, 0x81, 0x96, 0x64, 0xea, + 0x95, 0xc8, 0x3d, 0xd4, 0x9c, 0x95, 0x41, 0xf2, 0xf6, 0x7e, 0x41, 0xa3, 0x32, 0xb6, 0x8e, 0xd3, + 0xa8, 0xc5, 0x10, 0x72, 0xd7, 0xd8, 0x26, 0x8d, 0x5a, 0x9b, 0xc3, 0xd5, 0x67, 0x70, 0xbe, 0x00, + 0xae, 0xdc, 0x17, 0xf6, 0xe3, 0x7c, 0xc8, 0xf7, 0x97, 0x41, 0xa8, 0xbe, 0x4f, 0x7b, 0xac, 0x2f, + 0xc6, 0x3c, 0x10, 0x3f, 0xf0, 0x27, 0x4f, 0xde, 0x70, 0x7d, 0x05, 0x24, 0x0f, 0x70, 0x82, 0x71, + 0x57, 0xe1, 0x80, 0x07, 0x82, 0xa7, 0x1f, 0x1a, 0xbd, 0x5d, 0xed, 0x3f, 0xff, 0x17, 0x00, 0x00, + 0xff, 0xff, 0x57, 0x65, 0xdd, 0x73, 0x7d, 0x08, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -589,44 +648,44 @@ const _ = grpc.SupportPackageIsVersion4 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type CoreServiceClient interface { // - // Get entities specified in request. - // This process returns a list of entity specs. + //Get entities specified in request. + //This process returns a list of entity specs. GetEntities(ctx context.Context, in *CoreServiceTypes_GetEntitiesRequest, opts ...grpc.CallOption) (*CoreServiceTypes_GetEntitiesResponse, error) // - // Get all entities - // This process returns a list of entity specs. + //Get all entities + //This process returns a list of entity specs. ListEntities(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*CoreServiceTypes_ListEntitiesResponse, error) // - // Get features specified in request. - // This process returns a list of feature specs. + //Get features specified in request. + //This process returns a list of feature specs. GetFeatures(ctx context.Context, in *CoreServiceTypes_GetFeaturesRequest, opts ...grpc.CallOption) (*CoreServiceTypes_GetFeaturesResponse, error) // - // Get all features. - // This process returns a list of entity specs. + //Get all features. + //This process returns a list of entity specs. ListFeatures(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*CoreServiceTypes_ListFeaturesResponse, error) // - // Get storage specs specified in request. - // This process returns a list of storage specs. + //Get storage specs specified in request. + //This process returns a list of storage specs. GetStorage(ctx context.Context, in *CoreServiceTypes_GetStorageRequest, opts ...grpc.CallOption) (*CoreServiceTypes_GetStorageResponse, error) // - // Get all storage specs. - // This process returns a list of storage specs. + //Get all storage specs. + //This process returns a list of storage specs. ListStorage(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*CoreServiceTypes_ListStorageResponse, error) // - // Register a new feature to the metadata store, or update an existing feature. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new feature to the metadata store, or update an existing feature. + //If any validation errors occur, only the first encountered error will be returned. ApplyFeature(ctx context.Context, in *specs.FeatureSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyFeatureResponse, error) // - // Register a new feature group to the metadata store, or update an existing feature group. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new feature group to the metadata store, or update an existing feature group. + //If any validation errors occur, only the first encountered error will be returned. ApplyFeatureGroup(ctx context.Context, in *specs.FeatureGroupSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyFeatureGroupResponse, error) // - // Register a new entity to the metadata store, or update an existing entity. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new entity to the metadata store, or update an existing entity. + //If any validation errors occur, only the first encountered error will be returned. ApplyEntity(ctx context.Context, in *specs.EntitySpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyEntityResponse, error) // - // Register a new storage spec to the metadata store, or update an existing storage. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new storage spec to the metadata store, or update an existing storage. + //If any validation errors occur, only the first encountered error will be returned. ApplyStorage(ctx context.Context, in *specs.StorageSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyStorageResponse, error) } @@ -731,47 +790,82 @@ func (c *coreServiceClient) ApplyStorage(ctx context.Context, in *specs.StorageS // CoreServiceServer is the server API for CoreService service. type CoreServiceServer interface { // - // Get entities specified in request. - // This process returns a list of entity specs. + //Get entities specified in request. + //This process returns a list of entity specs. GetEntities(context.Context, *CoreServiceTypes_GetEntitiesRequest) (*CoreServiceTypes_GetEntitiesResponse, error) // - // Get all entities - // This process returns a list of entity specs. + //Get all entities + //This process returns a list of entity specs. ListEntities(context.Context, *empty.Empty) (*CoreServiceTypes_ListEntitiesResponse, error) // - // Get features specified in request. - // This process returns a list of feature specs. + //Get features specified in request. + //This process returns a list of feature specs. GetFeatures(context.Context, *CoreServiceTypes_GetFeaturesRequest) (*CoreServiceTypes_GetFeaturesResponse, error) // - // Get all features. - // This process returns a list of entity specs. + //Get all features. + //This process returns a list of entity specs. ListFeatures(context.Context, *empty.Empty) (*CoreServiceTypes_ListFeaturesResponse, error) // - // Get storage specs specified in request. - // This process returns a list of storage specs. + //Get storage specs specified in request. + //This process returns a list of storage specs. GetStorage(context.Context, *CoreServiceTypes_GetStorageRequest) (*CoreServiceTypes_GetStorageResponse, error) // - // Get all storage specs. - // This process returns a list of storage specs. + //Get all storage specs. + //This process returns a list of storage specs. ListStorage(context.Context, *empty.Empty) (*CoreServiceTypes_ListStorageResponse, error) // - // Register a new feature to the metadata store, or update an existing feature. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new feature to the metadata store, or update an existing feature. + //If any validation errors occur, only the first encountered error will be returned. ApplyFeature(context.Context, *specs.FeatureSpec) (*CoreServiceTypes_ApplyFeatureResponse, error) // - // Register a new feature group to the metadata store, or update an existing feature group. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new feature group to the metadata store, or update an existing feature group. + //If any validation errors occur, only the first encountered error will be returned. ApplyFeatureGroup(context.Context, *specs.FeatureGroupSpec) (*CoreServiceTypes_ApplyFeatureGroupResponse, error) // - // Register a new entity to the metadata store, or update an existing entity. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new entity to the metadata store, or update an existing entity. + //If any validation errors occur, only the first encountered error will be returned. ApplyEntity(context.Context, *specs.EntitySpec) (*CoreServiceTypes_ApplyEntityResponse, error) // - // Register a new storage spec to the metadata store, or update an existing storage. - // If any validation errors occur, only the first encountered error will be returned. + //Register a new storage spec to the metadata store, or update an existing storage. + //If any validation errors occur, only the first encountered error will be returned. ApplyStorage(context.Context, *specs.StorageSpec) (*CoreServiceTypes_ApplyStorageResponse, error) } +// UnimplementedCoreServiceServer can be embedded to have forward compatible implementations. +type UnimplementedCoreServiceServer struct { +} + +func (*UnimplementedCoreServiceServer) GetEntities(ctx context.Context, req *CoreServiceTypes_GetEntitiesRequest) (*CoreServiceTypes_GetEntitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetEntities not implemented") +} +func (*UnimplementedCoreServiceServer) ListEntities(ctx context.Context, req *empty.Empty) (*CoreServiceTypes_ListEntitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListEntities not implemented") +} +func (*UnimplementedCoreServiceServer) GetFeatures(ctx context.Context, req *CoreServiceTypes_GetFeaturesRequest) (*CoreServiceTypes_GetFeaturesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeatures not implemented") +} +func (*UnimplementedCoreServiceServer) ListFeatures(ctx context.Context, req *empty.Empty) (*CoreServiceTypes_ListFeaturesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListFeatures not implemented") +} +func (*UnimplementedCoreServiceServer) GetStorage(ctx context.Context, req *CoreServiceTypes_GetStorageRequest) (*CoreServiceTypes_GetStorageResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStorage not implemented") +} +func (*UnimplementedCoreServiceServer) ListStorage(ctx context.Context, req *empty.Empty) (*CoreServiceTypes_ListStorageResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListStorage not implemented") +} +func (*UnimplementedCoreServiceServer) ApplyFeature(ctx context.Context, req *specs.FeatureSpec) (*CoreServiceTypes_ApplyFeatureResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ApplyFeature not implemented") +} +func (*UnimplementedCoreServiceServer) ApplyFeatureGroup(ctx context.Context, req *specs.FeatureGroupSpec) (*CoreServiceTypes_ApplyFeatureGroupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ApplyFeatureGroup not implemented") +} +func (*UnimplementedCoreServiceServer) ApplyEntity(ctx context.Context, req *specs.EntitySpec) (*CoreServiceTypes_ApplyEntityResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ApplyEntity not implemented") +} +func (*UnimplementedCoreServiceServer) ApplyStorage(ctx context.Context, req *specs.StorageSpec) (*CoreServiceTypes_ApplyStorageResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ApplyStorage not implemented") +} + func RegisterCoreServiceServer(s *grpc.Server, srv CoreServiceServer) { s.RegisterService(&_CoreService_serviceDesc, srv) } @@ -1004,49 +1098,3 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "feast/core/CoreService.proto", } - -func init() { - proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_a0e9a1504f969203) -} - -var fileDescriptor_CoreService_a0e9a1504f969203 = []byte{ - // 602 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5b, 0x6f, 0x12, 0x41, - 0x14, 0x86, 0x34, 0x69, 0xe0, 0x40, 0x4c, 0x3b, 0x34, 0x8a, 0x23, 0x35, 0xcd, 0x26, 0x36, 0x7d, - 0x9a, 0xa9, 0xbd, 0xf8, 0xe4, 0x8b, 0x6d, 0x2a, 0xd1, 0x36, 0xc6, 0x2c, 0xfa, 0x60, 0x8d, 0x0f, - 0x5c, 0x0e, 0xeb, 0x2a, 0x30, 0xeb, 0xce, 0x60, 0xc2, 0xb3, 0xff, 0xcf, 0xdf, 0x64, 0x98, 0x1d, - 0x66, 0x07, 0xd8, 0xed, 0x12, 0xc3, 0x1b, 0x9c, 0xdb, 0x37, 0xdf, 0x39, 0xf3, 0x9d, 0x59, 0x68, - 0x0d, 0xb1, 0x2b, 0x15, 0xef, 0x8b, 0x18, 0xf9, 0xb5, 0x88, 0xb1, 0x83, 0xf1, 0xef, 0xb0, 0x8f, - 0x2c, 0x8a, 0x85, 0x12, 0x04, 0xb4, 0x97, 0xcd, 0xbd, 0xd4, 0x44, 0xca, 0x08, 0xfb, 0x92, 0xdf, - 0x4c, 0x54, 0xa8, 0x66, 0x9d, 0x08, 0xfb, 0x49, 0x24, 0x3d, 0x74, 0xbd, 0x6f, 0xb1, 0xab, 0xa6, - 0x31, 0x3a, 0x6e, 0x2f, 0xc3, 0xdd, 0x8e, 0xc5, 0x34, 0xca, 0x2b, 0xd1, 0x51, 0x22, 0xee, 0x06, - 0x6e, 0x89, 0x67, 0x81, 0x10, 0xc1, 0x08, 0xb9, 0xfe, 0xd7, 0x9b, 0x0e, 0x39, 0x8e, 0x23, 0x35, - 0x4b, 0x9c, 0xde, 0xdf, 0x5d, 0xd8, 0x73, 0x8e, 0xff, 0x69, 0x16, 0xa1, 0xa4, 0xc7, 0x40, 0xda, - 0xa8, 0xf4, 0x51, 0x43, 0x94, 0x3e, 0xfe, 0x9a, 0xa2, 0x54, 0x64, 0x0f, 0x76, 0xc2, 0x81, 0x6c, - 0x96, 0x8f, 0x76, 0x4e, 0xaa, 0xfe, 0xfc, 0x27, 0x7d, 0x0f, 0x8d, 0xa5, 0x38, 0x19, 0x89, 0x89, - 0x44, 0x72, 0x0e, 0x15, 0x34, 0x36, 0x1d, 0x5d, 0x3b, 0x7b, 0xc2, 0x92, 0x7e, 0xe8, 0x23, 0xb2, - 0xb4, 0x07, 0xbe, 0x0d, 0xa4, 0xb7, 0x70, 0x70, 0x17, 0xca, 0x2d, 0x15, 0x4b, 0x08, 0x98, 0x76, - 0x3d, 0x40, 0xe0, 0x56, 0x13, 0x48, 0xe3, 0x0c, 0xe6, 0x05, 0x54, 0x86, 0xc6, 0x66, 0x30, 0x9b, - 0x4b, 0x98, 0xce, 0x98, 0x7c, 0x1b, 0x49, 0xef, 0x12, 0x06, 0x5b, 0xaa, 0xf6, 0x02, 0xf6, 0xdb, - 0xa8, 0xcc, 0x34, 0xf3, 0x19, 0xf8, 0x9a, 0xa9, 0x0d, 0x33, 0x90, 0xaf, 0xa1, 0x2e, 0xd3, 0x7b, - 0x90, 0x0d, 0xeb, 0x5c, 0x14, 0x7f, 0x29, 0x9a, 0x76, 0xa0, 0x31, 0x27, 0xb2, 0xdd, 0xa2, 0x97, - 0xd0, 0x78, 0x13, 0x45, 0xa3, 0x59, 0x32, 0x2f, 0x5b, 0xf4, 0x39, 0x80, 0x9e, 0xda, 0xec, 0x43, - 0x77, 0x8c, 0xcd, 0xf2, 0x51, 0xf9, 0xa4, 0xea, 0x3b, 0x16, 0x7a, 0x01, 0x07, 0x3a, 0xcd, 0x34, - 0xc9, 0xe6, 0xb5, 0xa0, 0x6a, 0x5a, 0xf5, 0x6e, 0x60, 0xd2, 0x52, 0x03, 0xbd, 0x86, 0xa7, 0x6e, - 0x96, 0x16, 0x8c, 0x4d, 0x3d, 0x86, 0x47, 0x43, 0xc7, 0x6e, 0xf3, 0x57, 0xac, 0x16, 0x7a, 0xb5, - 0x0f, 0x2d, 0xa8, 0x1a, 0x66, 0x29, 0xb4, 0x35, 0x9c, 0xfd, 0xa9, 0x40, 0xcd, 0x11, 0x14, 0x89, - 0xa1, 0xe6, 0x68, 0x84, 0x70, 0x96, 0x6e, 0x06, 0xb6, 0x2a, 0x3c, 0xb6, 0xae, 0x3a, 0x7a, 0xba, - 0x79, 0x42, 0x72, 0x3e, 0xaf, 0x44, 0xbe, 0x42, 0xdd, 0xd5, 0x12, 0x79, 0xcc, 0x92, 0x15, 0xc0, - 0x16, 0x2b, 0x80, 0xdd, 0xcc, 0x57, 0x00, 0x7d, 0xf9, 0x60, 0xed, 0x2c, 0x39, 0x7a, 0x25, 0x43, - 0x68, 0x71, 0xcb, 0x8b, 0x09, 0xad, 0xa8, 0xb0, 0x98, 0xd0, 0xaa, 0x80, 0x52, 0x42, 0x16, 0xf4, - 0xff, 0x09, 0x65, 0x14, 0x17, 0x00, 0xa9, 0x84, 0x08, 0x2b, 0x3a, 0xde, 0xb2, 0x24, 0x29, 0xdf, - 0x38, 0xde, 0x02, 0x7e, 0x81, 0x9a, 0xa3, 0xaf, 0x5c, 0x32, 0xa7, 0x85, 0x64, 0xd6, 0x4b, 0x7f, - 0x83, 0xba, 0x7b, 0xf1, 0x49, 0xee, 0xa6, 0x29, 0x68, 0x55, 0x96, 0xe6, 0xbc, 0x12, 0x19, 0xc1, - 0xfe, 0x9a, 0xae, 0xc8, 0x61, 0x16, 0x86, 0x7d, 0xa3, 0xe8, 0xab, 0x8d, 0x81, 0x96, 0x64, 0xea, - 0x95, 0xc8, 0x3d, 0xd4, 0x9c, 0x95, 0x41, 0xf2, 0xf6, 0x7e, 0x41, 0xa3, 0x32, 0xb6, 0x8e, 0xd3, - 0xa8, 0xc5, 0x10, 0x72, 0xd7, 0xd8, 0x26, 0x8d, 0x5a, 0x9b, 0xc3, 0xd5, 0x67, 0x70, 0xbe, 0x00, - 0xae, 0xdc, 0x17, 0xf6, 0xe3, 0x7c, 0xc8, 0xf7, 0x97, 0x41, 0xa8, 0xbe, 0x4f, 0x7b, 0xac, 0x2f, - 0xc6, 0x3c, 0x10, 0x3f, 0xf0, 0x27, 0x4f, 0xde, 0x70, 0x7d, 0x05, 0x24, 0x0f, 0x70, 0x82, 0x71, - 0x57, 0xe1, 0x80, 0x07, 0x82, 0xa7, 0x1f, 0x1a, 0xbd, 0x5d, 0xed, 0x3f, 0xff, 0x17, 0x00, 0x00, - 0xff, 0xff, 0x57, 0x65, 0xdd, 0x73, 0x7d, 0x08, 0x00, 0x00, -} diff --git a/protos/generated/go/feast/core/DatasetService.pb.go b/protos/generated/go/feast/core/DatasetService.pb.go index d2d3b01c932..e93f569c8ec 100644 --- a/protos/generated/go/feast/core/DatasetService.pb.go +++ b/protos/generated/go/feast/core/DatasetService.pb.go @@ -1,16 +1,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/core/DatasetService.proto -package core // import "github.com/gojek/feast/protos/generated/go/feast/core" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +package core import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -22,7 +23,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type DatasetServiceTypes struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -34,16 +35,17 @@ func (m *DatasetServiceTypes) Reset() { *m = DatasetServiceTypes{} } func (m *DatasetServiceTypes) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes) ProtoMessage() {} func (*DatasetServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0} + return fileDescriptor_3edc37a8b0d37b39, []int{0} } + func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes.Unmarshal(m, b) } func (m *DatasetServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes.Merge(dst, src) +func (m *DatasetServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes.Merge(m, src) } func (m *DatasetServiceTypes) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes.Size(m) @@ -77,16 +79,17 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) Reset() { func (m *DatasetServiceTypes_CreateDatasetRequest) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetRequest) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 0} + return fileDescriptor_3edc37a8b0d37b39, []int{0, 0} } + func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Unmarshal(m, b) } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(dst, src) +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(m, src) } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Size(m) @@ -146,16 +149,17 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) Reset() { func (m *DatasetServiceTypes_CreateDatasetResponse) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetResponse) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 1} + return fileDescriptor_3edc37a8b0d37b39, []int{0, 1} } + func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Unmarshal(m, b) } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(dst, src) +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(m, src) } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Size(m) @@ -188,16 +192,17 @@ func (m *FeatureSet) Reset() { *m = FeatureSet{} } func (m *FeatureSet) String() string { return proto.CompactTextString(m) } func (*FeatureSet) ProtoMessage() {} func (*FeatureSet) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{1} + return fileDescriptor_3edc37a8b0d37b39, []int{1} } + func (m *FeatureSet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSet.Unmarshal(m, b) } func (m *FeatureSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSet.Marshal(b, m, deterministic) } -func (dst *FeatureSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSet.Merge(dst, src) +func (m *FeatureSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSet.Merge(m, src) } func (m *FeatureSet) XXX_Size() int { return xxx_messageInfo_FeatureSet.Size(m) @@ -237,16 +242,17 @@ func (m *DatasetInfo) Reset() { *m = DatasetInfo{} } func (m *DatasetInfo) String() string { return proto.CompactTextString(m) } func (*DatasetInfo) ProtoMessage() {} func (*DatasetInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{2} + return fileDescriptor_3edc37a8b0d37b39, []int{2} } + func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetInfo.Unmarshal(m, b) } func (m *DatasetInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetInfo.Marshal(b, m, deterministic) } -func (dst *DatasetInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetInfo.Merge(dst, src) +func (m *DatasetInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetInfo.Merge(m, src) } func (m *DatasetInfo) XXX_Size() int { return xxx_messageInfo_DatasetInfo.Size(m) @@ -279,6 +285,38 @@ func init() { proto.RegisterType((*DatasetInfo)(nil), "feast.core.DatasetInfo") } +func init() { proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_3edc37a8b0d37b39) } + +var fileDescriptor_3edc37a8b0d37b39 = []byte{ + // 414 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, + 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, + 0xa5, 0xd2, 0x22, 0x38, 0x70, 0x29, 0x2b, 0xa4, 0x4a, 0x08, 0x55, 0x6e, 0x91, 0x10, 0x37, 0x67, + 0x33, 0x09, 0x81, 0x24, 0x0e, 0xf6, 0x04, 0xd1, 0x0b, 0xdf, 0xc0, 0x27, 0xf2, 0x11, 0x7c, 0x00, + 0x8a, 0xb3, 0xdb, 0x78, 0xab, 0x4a, 0x88, 0x9b, 0x3d, 0xef, 0xcd, 0x78, 0xde, 0xf8, 0x0d, 0x1c, + 0x16, 0xa8, 0x2c, 0x89, 0xb5, 0x36, 0x28, 0x56, 0x8a, 0x94, 0x45, 0xba, 0x40, 0xf3, 0xbd, 0x5a, + 0x23, 0xef, 0x8c, 0x26, 0xcd, 0xc0, 0x11, 0xf8, 0x40, 0x88, 0x0f, 0x4b, 0xad, 0xcb, 0x1a, 0x85, + 0x43, 0xb2, 0xbe, 0x10, 0x54, 0x35, 0x68, 0x49, 0x35, 0xdd, 0x48, 0x4e, 0x7f, 0xcf, 0xe0, 0xd1, + 0x6e, 0x95, 0xcb, 0xab, 0x0e, 0x6d, 0xfc, 0x27, 0x80, 0x83, 0x37, 0x06, 0x15, 0xe1, 0x06, 0x95, + 0xf8, 0xad, 0x47, 0x4b, 0xec, 0x05, 0x0c, 0xf5, 0xa9, 0x37, 0x78, 0x81, 0x14, 0x05, 0x49, 0xb0, + 0x5c, 0x1c, 0x3d, 0xe1, 0xd3, 0x93, 0xfc, 0xed, 0x35, 0x2a, 0x3d, 0x26, 0x7b, 0x09, 0xa1, 0x25, + 0x65, 0x68, 0xa5, 0x08, 0xa3, 0x99, 0x4b, 0x8b, 0xf9, 0xd8, 0x1d, 0xdf, 0x76, 0xc7, 0x2f, 0xb7, + 0xdd, 0xc9, 0x89, 0xcc, 0x8e, 0xe1, 0x1e, 0xb6, 0xb9, 0xcb, 0x9b, 0xff, 0x33, 0x6f, 0x4b, 0x65, + 0x07, 0xb0, 0x5f, 0x57, 0x4d, 0x45, 0xd1, 0x5e, 0x12, 0x2c, 0xe7, 0x72, 0xbc, 0xb0, 0x67, 0x00, + 0xad, 0x6a, 0xf0, 0xdc, 0x60, 0x51, 0xfd, 0x88, 0xf6, 0x93, 0x60, 0x19, 0x4a, 0x2f, 0x12, 0x4b, + 0x78, 0x7c, 0x43, 0xb5, 0xed, 0x74, 0x6b, 0x91, 0xbd, 0x82, 0x45, 0x3e, 0x86, 0xce, 0xda, 0x42, + 0x6f, 0x74, 0x3f, 0xf5, 0x75, 0xaf, 0x26, 0x58, 0xfa, 0xdc, 0xf4, 0x1d, 0xc0, 0x34, 0x93, 0xa1, + 0x03, 0x6c, 0xa9, 0xa2, 0xab, 0xf7, 0xaa, 0x41, 0x57, 0x27, 0x94, 0x5e, 0x64, 0xc0, 0x37, 0x53, + 0x3b, 0xcb, 0x6d, 0x34, 0x4b, 0xe6, 0x03, 0x3e, 0x45, 0xd2, 0xd7, 0xb0, 0xf0, 0x5e, 0x62, 0x0c, + 0xf6, 0xda, 0xa9, 0x90, 0x3b, 0xb3, 0x18, 0xee, 0x93, 0xca, 0x6a, 0xfc, 0x60, 0x6a, 0x37, 0xe9, + 0x50, 0x5e, 0xdf, 0x8f, 0x7e, 0x05, 0xf0, 0x70, 0xf7, 0xbf, 0xd9, 0x4f, 0x78, 0xb0, 0xa3, 0x99, + 0x1d, 0xdf, 0x22, 0xcb, 0x37, 0x07, 0xbf, 0xcd, 0x18, 0xf1, 0xc9, 0x7f, 0x66, 0x8d, 0x83, 0x4d, + 0xef, 0x9c, 0x7e, 0x04, 0xcf, 0xb1, 0xa7, 0x37, 0xdc, 0x78, 0x3e, 0x7c, 0xf1, 0xa7, 0x93, 0xb2, + 0xa2, 0xcf, 0x7d, 0xc6, 0xd7, 0xba, 0x11, 0xa5, 0xfe, 0x82, 0x5f, 0xc5, 0xb8, 0x04, 0xce, 0x00, + 0x56, 0x94, 0xd8, 0xa2, 0x51, 0x84, 0xb9, 0x28, 0xb5, 0x98, 0xd6, 0x23, 0xbb, 0xeb, 0xf0, 0xe7, + 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x9e, 0x4d, 0xe0, 0x33, 0x03, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -318,6 +356,14 @@ type DatasetServiceServer interface { CreateDataset(context.Context, *DatasetServiceTypes_CreateDatasetRequest) (*DatasetServiceTypes_CreateDatasetResponse, error) } +// UnimplementedDatasetServiceServer can be embedded to have forward compatible implementations. +type UnimplementedDatasetServiceServer struct { +} + +func (*UnimplementedDatasetServiceServer) CreateDataset(ctx context.Context, req *DatasetServiceTypes_CreateDatasetRequest) (*DatasetServiceTypes_CreateDatasetResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateDataset not implemented") +} + func RegisterDatasetServiceServer(s *grpc.Server, srv DatasetServiceServer) { s.RegisterService(&_DatasetService_serviceDesc, srv) } @@ -352,37 +398,3 @@ var _DatasetService_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "feast/core/DatasetService.proto", } - -func init() { - proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_37ae639a8c7b5dd5) -} - -var fileDescriptor_DatasetService_37ae639a8c7b5dd5 = []byte{ - // 414 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, - 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, - 0xa5, 0xd2, 0x22, 0x38, 0x70, 0x29, 0x2b, 0xa4, 0x4a, 0x08, 0x55, 0x6e, 0x91, 0x10, 0x37, 0x67, - 0x33, 0x09, 0x81, 0x24, 0x0e, 0xf6, 0x04, 0xd1, 0x0b, 0xdf, 0xc0, 0x27, 0xf2, 0x11, 0x7c, 0x00, - 0x8a, 0xb3, 0xdb, 0x78, 0xab, 0x4a, 0x88, 0x9b, 0x3d, 0xef, 0xcd, 0x78, 0xde, 0xf8, 0x0d, 0x1c, - 0x16, 0xa8, 0x2c, 0x89, 0xb5, 0x36, 0x28, 0x56, 0x8a, 0x94, 0x45, 0xba, 0x40, 0xf3, 0xbd, 0x5a, - 0x23, 0xef, 0x8c, 0x26, 0xcd, 0xc0, 0x11, 0xf8, 0x40, 0x88, 0x0f, 0x4b, 0xad, 0xcb, 0x1a, 0x85, - 0x43, 0xb2, 0xbe, 0x10, 0x54, 0x35, 0x68, 0x49, 0x35, 0xdd, 0x48, 0x4e, 0x7f, 0xcf, 0xe0, 0xd1, - 0x6e, 0x95, 0xcb, 0xab, 0x0e, 0x6d, 0xfc, 0x27, 0x80, 0x83, 0x37, 0x06, 0x15, 0xe1, 0x06, 0x95, - 0xf8, 0xad, 0x47, 0x4b, 0xec, 0x05, 0x0c, 0xf5, 0xa9, 0x37, 0x78, 0x81, 0x14, 0x05, 0x49, 0xb0, - 0x5c, 0x1c, 0x3d, 0xe1, 0xd3, 0x93, 0xfc, 0xed, 0x35, 0x2a, 0x3d, 0x26, 0x7b, 0x09, 0xa1, 0x25, - 0x65, 0x68, 0xa5, 0x08, 0xa3, 0x99, 0x4b, 0x8b, 0xf9, 0xd8, 0x1d, 0xdf, 0x76, 0xc7, 0x2f, 0xb7, - 0xdd, 0xc9, 0x89, 0xcc, 0x8e, 0xe1, 0x1e, 0xb6, 0xb9, 0xcb, 0x9b, 0xff, 0x33, 0x6f, 0x4b, 0x65, - 0x07, 0xb0, 0x5f, 0x57, 0x4d, 0x45, 0xd1, 0x5e, 0x12, 0x2c, 0xe7, 0x72, 0xbc, 0xb0, 0x67, 0x00, - 0xad, 0x6a, 0xf0, 0xdc, 0x60, 0x51, 0xfd, 0x88, 0xf6, 0x93, 0x60, 0x19, 0x4a, 0x2f, 0x12, 0x4b, - 0x78, 0x7c, 0x43, 0xb5, 0xed, 0x74, 0x6b, 0x91, 0xbd, 0x82, 0x45, 0x3e, 0x86, 0xce, 0xda, 0x42, - 0x6f, 0x74, 0x3f, 0xf5, 0x75, 0xaf, 0x26, 0x58, 0xfa, 0xdc, 0xf4, 0x1d, 0xc0, 0x34, 0x93, 0xa1, - 0x03, 0x6c, 0xa9, 0xa2, 0xab, 0xf7, 0xaa, 0x41, 0x57, 0x27, 0x94, 0x5e, 0x64, 0xc0, 0x37, 0x53, - 0x3b, 0xcb, 0x6d, 0x34, 0x4b, 0xe6, 0x03, 0x3e, 0x45, 0xd2, 0xd7, 0xb0, 0xf0, 0x5e, 0x62, 0x0c, - 0xf6, 0xda, 0xa9, 0x90, 0x3b, 0xb3, 0x18, 0xee, 0x93, 0xca, 0x6a, 0xfc, 0x60, 0x6a, 0x37, 0xe9, - 0x50, 0x5e, 0xdf, 0x8f, 0x7e, 0x05, 0xf0, 0x70, 0xf7, 0xbf, 0xd9, 0x4f, 0x78, 0xb0, 0xa3, 0x99, - 0x1d, 0xdf, 0x22, 0xcb, 0x37, 0x07, 0xbf, 0xcd, 0x18, 0xf1, 0xc9, 0x7f, 0x66, 0x8d, 0x83, 0x4d, - 0xef, 0x9c, 0x7e, 0x04, 0xcf, 0xb1, 0xa7, 0x37, 0xdc, 0x78, 0x3e, 0x7c, 0xf1, 0xa7, 0x93, 0xb2, - 0xa2, 0xcf, 0x7d, 0xc6, 0xd7, 0xba, 0x11, 0xa5, 0xfe, 0x82, 0x5f, 0xc5, 0xb8, 0x04, 0xce, 0x00, - 0x56, 0x94, 0xd8, 0xa2, 0x51, 0x84, 0xb9, 0x28, 0xb5, 0x98, 0xd6, 0x23, 0xbb, 0xeb, 0xf0, 0xe7, - 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x9e, 0x4d, 0xe0, 0x33, 0x03, 0x00, 0x00, -} diff --git a/protos/generated/go/feast/core/JobService.pb.go b/protos/generated/go/feast/core/JobService.pb.go index 6a3a4e6dc0d..2a6455e5c81 100644 --- a/protos/generated/go/feast/core/JobService.pb.go +++ b/protos/generated/go/feast/core/JobService.pb.go @@ -1,18 +1,19 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/core/JobService.proto -package core // import "github.com/gojek/feast/protos/generated/go/feast/core" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import specs "github.com/gojek/feast/protos/generated/go/feast/specs" -import empty "github.com/golang/protobuf/ptypes/empty" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +package core import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + specs "github.com/gojek/feast/protos/generated/go/feast/specs" + proto "github.com/golang/protobuf/proto" + empty "github.com/golang/protobuf/ptypes/empty" + timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -24,7 +25,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type JobServiceTypes struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -36,16 +37,17 @@ func (m *JobServiceTypes) Reset() { *m = JobServiceTypes{} } func (m *JobServiceTypes) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes) ProtoMessage() {} func (*JobServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0} + return fileDescriptor_7115affcfc7885c4, []int{0} } + func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes.Unmarshal(m, b) } func (m *JobServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes.Merge(dst, src) +func (m *JobServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes.Merge(m, src) } func (m *JobServiceTypes) XXX_Size() int { return xxx_messageInfo_JobServiceTypes.Size(m) @@ -70,16 +72,17 @@ func (m *JobServiceTypes_SubmitImportJobRequest) Reset() { func (m *JobServiceTypes_SubmitImportJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobRequest) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 0} + return fileDescriptor_7115affcfc7885c4, []int{0, 0} } + func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Unmarshal(m, b) } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(dst, src) +func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(m, src) } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Size(m) @@ -117,16 +120,17 @@ func (m *JobServiceTypes_SubmitImportJobResponse) Reset() { func (m *JobServiceTypes_SubmitImportJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobResponse) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 1} + return fileDescriptor_7115affcfc7885c4, []int{0, 1} } + func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Unmarshal(m, b) } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(dst, src) +func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(m, src) } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Size(m) @@ -155,16 +159,17 @@ func (m *JobServiceTypes_ListJobsResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_ListJobsResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_ListJobsResponse) ProtoMessage() {} func (*JobServiceTypes_ListJobsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 2} + return fileDescriptor_7115affcfc7885c4, []int{0, 2} } + func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Unmarshal(m, b) } func (m *JobServiceTypes_ListJobsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(dst, src) +func (m *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(m, src) } func (m *JobServiceTypes_ListJobsResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Size(m) @@ -193,16 +198,17 @@ func (m *JobServiceTypes_GetJobRequest) Reset() { *m = JobServiceTypes_G func (m *JobServiceTypes_GetJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobRequest) ProtoMessage() {} func (*JobServiceTypes_GetJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 3} + return fileDescriptor_7115affcfc7885c4, []int{0, 3} } + func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Unmarshal(m, b) } func (m *JobServiceTypes_GetJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(dst, src) +func (m *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(m, src) } func (m *JobServiceTypes_GetJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Size(m) @@ -231,16 +237,17 @@ func (m *JobServiceTypes_GetJobResponse) Reset() { *m = JobServiceTypes_ func (m *JobServiceTypes_GetJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobResponse) ProtoMessage() {} func (*JobServiceTypes_GetJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 4} + return fileDescriptor_7115affcfc7885c4, []int{0, 4} } + func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Unmarshal(m, b) } func (m *JobServiceTypes_GetJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(dst, src) +func (m *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(m, src) } func (m *JobServiceTypes_GetJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Size(m) @@ -269,16 +276,17 @@ func (m *JobServiceTypes_AbortJobRequest) Reset() { *m = JobServiceTypes func (m *JobServiceTypes_AbortJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobRequest) ProtoMessage() {} func (*JobServiceTypes_AbortJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 5} + return fileDescriptor_7115affcfc7885c4, []int{0, 5} } + func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Unmarshal(m, b) } func (m *JobServiceTypes_AbortJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(dst, src) +func (m *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(m, src) } func (m *JobServiceTypes_AbortJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Size(m) @@ -307,16 +315,17 @@ func (m *JobServiceTypes_AbortJobResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_AbortJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobResponse) ProtoMessage() {} func (*JobServiceTypes_AbortJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 6} + return fileDescriptor_7115affcfc7885c4, []int{0, 6} } + func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Unmarshal(m, b) } func (m *JobServiceTypes_AbortJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(dst, src) +func (m *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(m, src) } func (m *JobServiceTypes_AbortJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Size(m) @@ -356,16 +365,17 @@ func (m *JobServiceTypes_JobDetail) Reset() { *m = JobServiceTypes_JobDe func (m *JobServiceTypes_JobDetail) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_JobDetail) ProtoMessage() {} func (*JobServiceTypes_JobDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 7} + return fileDescriptor_7115affcfc7885c4, []int{0, 7} } + func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_JobDetail.Unmarshal(m, b) } func (m *JobServiceTypes_JobDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_JobDetail.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_JobDetail.Merge(dst, src) +func (m *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_JobDetail.Merge(m, src) } func (m *JobServiceTypes_JobDetail) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_JobDetail.Size(m) @@ -459,6 +469,51 @@ func init() { proto.RegisterMapType((map[string]float64)(nil), "feast.core.JobServiceTypes.JobDetail.MetricsEntry") } +func init() { proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_7115affcfc7885c4) } + +var fileDescriptor_7115affcfc7885c4 = []byte{ + // 621 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0xdb, 0x4c, + 0x10, 0x55, 0x62, 0x08, 0xf1, 0xf0, 0x7d, 0x80, 0x56, 0x15, 0x58, 0x4b, 0x25, 0x52, 0xa4, 0x4a, + 0xe9, 0x8f, 0x6c, 0x29, 0xb4, 0xa2, 0x45, 0xbd, 0x29, 0x2a, 0xaa, 0x82, 0x40, 0x42, 0x86, 0xde, + 0xf4, 0xa6, 0xb2, 0x9d, 0xc1, 0xdd, 0x10, 0x7b, 0x5d, 0xef, 0x1a, 0x35, 0xef, 0xd2, 0x37, 0xe8, + 0x13, 0xf5, 0x6d, 0xaa, 0xdd, 0xb5, 0x63, 0x63, 0xaa, 0x94, 0xde, 0xf9, 0xec, 0x9c, 0xd9, 0xb3, + 0x73, 0x3c, 0x33, 0xb0, 0x7b, 0x8d, 0x81, 0x90, 0x5e, 0xc4, 0x73, 0xf4, 0x4e, 0x79, 0x78, 0x89, + 0xf9, 0x2d, 0x8b, 0xd0, 0xcd, 0x72, 0x2e, 0x39, 0x01, 0x1d, 0x74, 0x55, 0x90, 0x3e, 0x36, 0x44, + 0x91, 0x61, 0x24, 0xbc, 0x71, 0x92, 0xf1, 0x5c, 0x5e, 0x66, 0x18, 0x19, 0x26, 0xdd, 0x8d, 0x39, + 0x8f, 0x67, 0xe8, 0x69, 0x14, 0x16, 0xd7, 0x1e, 0x26, 0x99, 0x9c, 0x97, 0xc1, 0xbd, 0x76, 0x50, + 0xb2, 0x04, 0x85, 0x0c, 0x92, 0xcc, 0x10, 0xf6, 0x7f, 0xf5, 0x60, 0xb3, 0x16, 0xbf, 0x9a, 0x67, + 0x28, 0x28, 0xc2, 0xf6, 0x65, 0x11, 0x26, 0x4c, 0x1a, 0xad, 0x53, 0x1e, 0xfa, 0xf8, 0xad, 0x40, + 0x21, 0xc9, 0x21, 0x00, 0x5b, 0xe8, 0x3b, 0x9d, 0x41, 0x67, 0xb8, 0x3e, 0xda, 0x71, 0xcd, 0x53, + 0xf5, 0xf3, 0xdc, 0xfa, 0x79, 0x7e, 0x83, 0x4a, 0x08, 0xac, 0xa4, 0x41, 0x82, 0x4e, 0x77, 0xd0, + 0x19, 0xda, 0xbe, 0xfe, 0xa6, 0x1e, 0xec, 0xdc, 0x93, 0x11, 0x19, 0x4f, 0x05, 0x92, 0x47, 0xb0, + 0x3a, 0xe5, 0xe1, 0x78, 0xa2, 0x25, 0x6c, 0xdf, 0x00, 0x7a, 0x0e, 0x5b, 0x67, 0x4c, 0x28, 0xa2, + 0x58, 0x30, 0xdf, 0xc2, 0xca, 0x94, 0x87, 0xc2, 0xe9, 0x0c, 0xac, 0xe1, 0xfa, 0xe8, 0xa9, 0x5b, + 0xdb, 0xe6, 0xb6, 0xca, 0x52, 0xf8, 0x03, 0xca, 0x80, 0xcd, 0x7c, 0x9d, 0x42, 0xf7, 0xe0, 0xff, + 0x8f, 0xd8, 0xac, 0x6e, 0x03, 0xba, 0xac, 0x92, 0xec, 0xb2, 0x09, 0x1d, 0xc3, 0x46, 0x45, 0x28, + 0xd5, 0x0e, 0xc1, 0x9a, 0xf2, 0xb0, 0x2c, 0xfc, 0x81, 0x62, 0x2a, 0x83, 0x3e, 0x81, 0xcd, 0xf7, + 0xe1, 0x5d, 0x2f, 0xdb, 0x6a, 0xfb, 0xb0, 0x55, 0x53, 0x4a, 0xbd, 0x36, 0xe7, 0xa7, 0x05, 0xf6, + 0xe2, 0xe6, 0x76, 0x54, 0xb9, 0x86, 0xdf, 0xe5, 0x78, 0x52, 0xba, 0x6c, 0x80, 0xb2, 0x5e, 0xce, + 0x33, 0x74, 0x2c, 0x63, 0xbd, 0xfa, 0x26, 0xdb, 0xd0, 0xcb, 0x8b, 0x34, 0xc5, 0xdc, 0x59, 0xd1, + 0xa7, 0x25, 0x52, 0xe7, 0x42, 0x06, 0xb2, 0x10, 0xce, 0xaa, 0x39, 0x37, 0x88, 0x50, 0xe8, 0x63, + 0x2a, 0x99, 0x64, 0x28, 0x9c, 0xde, 0xc0, 0x1a, 0xda, 0xfe, 0x02, 0xab, 0xd8, 0x35, 0x06, 0xb2, + 0xc8, 0x51, 0x38, 0x6b, 0x26, 0x56, 0x61, 0x72, 0x06, 0x6b, 0x09, 0xca, 0x9c, 0x45, 0xc2, 0xe9, + 0xeb, 0x1f, 0x34, 0x7a, 0x90, 0x67, 0xee, 0xb9, 0x49, 0x3a, 0x49, 0x65, 0x3e, 0xf7, 0xab, 0x2b, + 0xc8, 0x3b, 0x58, 0x9f, 0x05, 0x42, 0x7e, 0xca, 0x26, 0x81, 0xc4, 0x89, 0x63, 0xeb, 0xbf, 0x40, + 0x5d, 0xd3, 0xe2, 0x6e, 0xd5, 0xe2, 0xee, 0x55, 0xd5, 0xe2, 0x7e, 0x93, 0x4e, 0x5e, 0xc1, 0x5a, + 0x94, 0xa3, 0xce, 0x84, 0xbf, 0x66, 0x56, 0x54, 0x7a, 0x04, 0xff, 0x35, 0x1f, 0x43, 0xb6, 0xc0, + 0xba, 0xc1, 0x79, 0x69, 0xba, 0xfa, 0x54, 0xae, 0xdf, 0x06, 0xb3, 0xc2, 0xf4, 0x76, 0xc7, 0x37, + 0xe0, 0xa8, 0xfb, 0xa6, 0x33, 0xfa, 0x61, 0x01, 0xd4, 0x35, 0x12, 0x09, 0xb6, 0xe9, 0xf7, 0x53, + 0x1e, 0x92, 0xa5, 0x46, 0xfc, 0x79, 0xfa, 0xe8, 0xc1, 0x3f, 0xe5, 0x94, 0x2d, 0x74, 0x01, 0xfd, + 0x6a, 0x68, 0xc8, 0xf6, 0xbd, 0x8a, 0x4f, 0xd4, 0xae, 0xa0, 0x2f, 0x97, 0x5d, 0x7c, 0x6f, 0xe4, + 0xbe, 0x40, 0xcf, 0x8c, 0x05, 0x79, 0xb6, 0x2c, 0xef, 0xce, 0x6c, 0xd1, 0xe7, 0x0f, 0xa1, 0x96, + 0x02, 0x08, 0xfd, 0x6a, 0x12, 0xc8, 0x8b, 0x65, 0x79, 0xad, 0x91, 0x5a, 0x5e, 0x47, 0x7b, 0xb8, + 0x8e, 0xaf, 0xa0, 0xb1, 0x64, 0x8f, 0x1b, 0x5b, 0xf0, 0x42, 0xb9, 0xf3, 0xf9, 0x75, 0xcc, 0xe4, + 0xd7, 0x22, 0x74, 0x23, 0x9e, 0x78, 0x31, 0x9f, 0xe2, 0x8d, 0x67, 0xd6, 0xb0, 0xf6, 0x4e, 0x78, + 0x31, 0xa6, 0x98, 0xab, 0x16, 0xf1, 0x62, 0xee, 0xd5, 0x9b, 0x3c, 0xec, 0xe9, 0xf8, 0xc1, 0xef, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xf4, 0xb6, 0x7c, 0xde, 0x05, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -537,6 +592,23 @@ type JobServiceServer interface { AbortJob(context.Context, *JobServiceTypes_AbortJobRequest) (*JobServiceTypes_AbortJobResponse, error) } +// UnimplementedJobServiceServer can be embedded to have forward compatible implementations. +type UnimplementedJobServiceServer struct { +} + +func (*UnimplementedJobServiceServer) SubmitJob(ctx context.Context, req *JobServiceTypes_SubmitImportJobRequest) (*JobServiceTypes_SubmitImportJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubmitJob not implemented") +} +func (*UnimplementedJobServiceServer) ListJobs(ctx context.Context, req *empty.Empty) (*JobServiceTypes_ListJobsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListJobs not implemented") +} +func (*UnimplementedJobServiceServer) GetJob(ctx context.Context, req *JobServiceTypes_GetJobRequest) (*JobServiceTypes_GetJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetJob not implemented") +} +func (*UnimplementedJobServiceServer) AbortJob(ctx context.Context, req *JobServiceTypes_AbortJobRequest) (*JobServiceTypes_AbortJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AbortJob not implemented") +} + func RegisterJobServiceServer(s *grpc.Server, srv JobServiceServer) { s.RegisterService(&_JobService_serviceDesc, srv) } @@ -637,50 +709,3 @@ var _JobService_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "feast/core/JobService.proto", } - -func init() { - proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_JobService_edcd183b773c9f62) -} - -var fileDescriptor_JobService_edcd183b773c9f62 = []byte{ - // 621 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0xdb, 0x4c, - 0x10, 0x55, 0x62, 0x08, 0xf1, 0xf0, 0x7d, 0x80, 0x56, 0x15, 0x58, 0x4b, 0x25, 0x52, 0xa4, 0x4a, - 0xe9, 0x8f, 0x6c, 0x29, 0xb4, 0xa2, 0x45, 0xbd, 0x29, 0x2a, 0xaa, 0x82, 0x40, 0x42, 0x86, 0xde, - 0xf4, 0xa6, 0xb2, 0x9d, 0xc1, 0xdd, 0x10, 0x7b, 0x5d, 0xef, 0x1a, 0x35, 0xef, 0xd2, 0x37, 0xe8, - 0x13, 0xf5, 0x6d, 0xaa, 0xdd, 0xb5, 0x63, 0x63, 0xaa, 0x94, 0xde, 0xf9, 0xec, 0x9c, 0xd9, 0xb3, - 0x73, 0x3c, 0x33, 0xb0, 0x7b, 0x8d, 0x81, 0x90, 0x5e, 0xc4, 0x73, 0xf4, 0x4e, 0x79, 0x78, 0x89, - 0xf9, 0x2d, 0x8b, 0xd0, 0xcd, 0x72, 0x2e, 0x39, 0x01, 0x1d, 0x74, 0x55, 0x90, 0x3e, 0x36, 0x44, - 0x91, 0x61, 0x24, 0xbc, 0x71, 0x92, 0xf1, 0x5c, 0x5e, 0x66, 0x18, 0x19, 0x26, 0xdd, 0x8d, 0x39, - 0x8f, 0x67, 0xe8, 0x69, 0x14, 0x16, 0xd7, 0x1e, 0x26, 0x99, 0x9c, 0x97, 0xc1, 0xbd, 0x76, 0x50, - 0xb2, 0x04, 0x85, 0x0c, 0x92, 0xcc, 0x10, 0xf6, 0x7f, 0xf5, 0x60, 0xb3, 0x16, 0xbf, 0x9a, 0x67, - 0x28, 0x28, 0xc2, 0xf6, 0x65, 0x11, 0x26, 0x4c, 0x1a, 0xad, 0x53, 0x1e, 0xfa, 0xf8, 0xad, 0x40, - 0x21, 0xc9, 0x21, 0x00, 0x5b, 0xe8, 0x3b, 0x9d, 0x41, 0x67, 0xb8, 0x3e, 0xda, 0x71, 0xcd, 0x53, - 0xf5, 0xf3, 0xdc, 0xfa, 0x79, 0x7e, 0x83, 0x4a, 0x08, 0xac, 0xa4, 0x41, 0x82, 0x4e, 0x77, 0xd0, - 0x19, 0xda, 0xbe, 0xfe, 0xa6, 0x1e, 0xec, 0xdc, 0x93, 0x11, 0x19, 0x4f, 0x05, 0x92, 0x47, 0xb0, - 0x3a, 0xe5, 0xe1, 0x78, 0xa2, 0x25, 0x6c, 0xdf, 0x00, 0x7a, 0x0e, 0x5b, 0x67, 0x4c, 0x28, 0xa2, - 0x58, 0x30, 0xdf, 0xc2, 0xca, 0x94, 0x87, 0xc2, 0xe9, 0x0c, 0xac, 0xe1, 0xfa, 0xe8, 0xa9, 0x5b, - 0xdb, 0xe6, 0xb6, 0xca, 0x52, 0xf8, 0x03, 0xca, 0x80, 0xcd, 0x7c, 0x9d, 0x42, 0xf7, 0xe0, 0xff, - 0x8f, 0xd8, 0xac, 0x6e, 0x03, 0xba, 0xac, 0x92, 0xec, 0xb2, 0x09, 0x1d, 0xc3, 0x46, 0x45, 0x28, - 0xd5, 0x0e, 0xc1, 0x9a, 0xf2, 0xb0, 0x2c, 0xfc, 0x81, 0x62, 0x2a, 0x83, 0x3e, 0x81, 0xcd, 0xf7, - 0xe1, 0x5d, 0x2f, 0xdb, 0x6a, 0xfb, 0xb0, 0x55, 0x53, 0x4a, 0xbd, 0x36, 0xe7, 0xa7, 0x05, 0xf6, - 0xe2, 0xe6, 0x76, 0x54, 0xb9, 0x86, 0xdf, 0xe5, 0x78, 0x52, 0xba, 0x6c, 0x80, 0xb2, 0x5e, 0xce, - 0x33, 0x74, 0x2c, 0x63, 0xbd, 0xfa, 0x26, 0xdb, 0xd0, 0xcb, 0x8b, 0x34, 0xc5, 0xdc, 0x59, 0xd1, - 0xa7, 0x25, 0x52, 0xe7, 0x42, 0x06, 0xb2, 0x10, 0xce, 0xaa, 0x39, 0x37, 0x88, 0x50, 0xe8, 0x63, - 0x2a, 0x99, 0x64, 0x28, 0x9c, 0xde, 0xc0, 0x1a, 0xda, 0xfe, 0x02, 0xab, 0xd8, 0x35, 0x06, 0xb2, - 0xc8, 0x51, 0x38, 0x6b, 0x26, 0x56, 0x61, 0x72, 0x06, 0x6b, 0x09, 0xca, 0x9c, 0x45, 0xc2, 0xe9, - 0xeb, 0x1f, 0x34, 0x7a, 0x90, 0x67, 0xee, 0xb9, 0x49, 0x3a, 0x49, 0x65, 0x3e, 0xf7, 0xab, 0x2b, - 0xc8, 0x3b, 0x58, 0x9f, 0x05, 0x42, 0x7e, 0xca, 0x26, 0x81, 0xc4, 0x89, 0x63, 0xeb, 0xbf, 0x40, - 0x5d, 0xd3, 0xe2, 0x6e, 0xd5, 0xe2, 0xee, 0x55, 0xd5, 0xe2, 0x7e, 0x93, 0x4e, 0x5e, 0xc1, 0x5a, - 0x94, 0xa3, 0xce, 0x84, 0xbf, 0x66, 0x56, 0x54, 0x7a, 0x04, 0xff, 0x35, 0x1f, 0x43, 0xb6, 0xc0, - 0xba, 0xc1, 0x79, 0x69, 0xba, 0xfa, 0x54, 0xae, 0xdf, 0x06, 0xb3, 0xc2, 0xf4, 0x76, 0xc7, 0x37, - 0xe0, 0xa8, 0xfb, 0xa6, 0x33, 0xfa, 0x61, 0x01, 0xd4, 0x35, 0x12, 0x09, 0xb6, 0xe9, 0xf7, 0x53, - 0x1e, 0x92, 0xa5, 0x46, 0xfc, 0x79, 0xfa, 0xe8, 0xc1, 0x3f, 0xe5, 0x94, 0x2d, 0x74, 0x01, 0xfd, - 0x6a, 0x68, 0xc8, 0xf6, 0xbd, 0x8a, 0x4f, 0xd4, 0xae, 0xa0, 0x2f, 0x97, 0x5d, 0x7c, 0x6f, 0xe4, - 0xbe, 0x40, 0xcf, 0x8c, 0x05, 0x79, 0xb6, 0x2c, 0xef, 0xce, 0x6c, 0xd1, 0xe7, 0x0f, 0xa1, 0x96, - 0x02, 0x08, 0xfd, 0x6a, 0x12, 0xc8, 0x8b, 0x65, 0x79, 0xad, 0x91, 0x5a, 0x5e, 0x47, 0x7b, 0xb8, - 0x8e, 0xaf, 0xa0, 0xb1, 0x64, 0x8f, 0x1b, 0x5b, 0xf0, 0x42, 0xb9, 0xf3, 0xf9, 0x75, 0xcc, 0xe4, - 0xd7, 0x22, 0x74, 0x23, 0x9e, 0x78, 0x31, 0x9f, 0xe2, 0x8d, 0x67, 0xd6, 0xb0, 0xf6, 0x4e, 0x78, - 0x31, 0xa6, 0x98, 0xab, 0x16, 0xf1, 0x62, 0xee, 0xd5, 0x9b, 0x3c, 0xec, 0xe9, 0xf8, 0xc1, 0xef, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xf4, 0xb6, 0x7c, 0xde, 0x05, 0x00, 0x00, -} diff --git a/protos/generated/go/feast/core/UIService.pb.go b/protos/generated/go/feast/core/UIService.pb.go index 772c548c875..1d06d0c5ef7 100644 --- a/protos/generated/go/feast/core/UIService.pb.go +++ b/protos/generated/go/feast/core/UIService.pb.go @@ -1,18 +1,19 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/core/UIService.proto -package core // import "github.com/gojek/feast/protos/generated/go/feast/core" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import specs "github.com/gojek/feast/protos/generated/go/feast/specs" -import empty "github.com/golang/protobuf/ptypes/empty" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +package core import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + specs "github.com/gojek/feast/protos/generated/go/feast/specs" + proto "github.com/golang/protobuf/proto" + empty "github.com/golang/protobuf/ptypes/empty" + timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -24,7 +25,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type UIServiceTypes struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -36,16 +37,17 @@ func (m *UIServiceTypes) Reset() { *m = UIServiceTypes{} } func (m *UIServiceTypes) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes) ProtoMessage() {} func (*UIServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0} + return fileDescriptor_c13b3edb35d457e8, []int{0} } + func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes.Unmarshal(m, b) } func (m *UIServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes.Merge(dst, src) +func (m *UIServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes.Merge(m, src) } func (m *UIServiceTypes) XXX_Size() int { return xxx_messageInfo_UIServiceTypes.Size(m) @@ -70,16 +72,17 @@ func (m *UIServiceTypes_EntityDetail) Reset() { *m = UIServiceTypes_Enti func (m *UIServiceTypes_EntityDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_EntityDetail) ProtoMessage() {} func (*UIServiceTypes_EntityDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 0} + return fileDescriptor_c13b3edb35d457e8, []int{0, 0} } + func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_EntityDetail.Unmarshal(m, b) } func (m *UIServiceTypes_EntityDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_EntityDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(dst, src) +func (m *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(m, src) } func (m *UIServiceTypes_EntityDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_EntityDetail.Size(m) @@ -122,16 +125,17 @@ func (m *UIServiceTypes_GetEntityRequest) Reset() { *m = UIServiceTypes_ func (m *UIServiceTypes_GetEntityRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityRequest) ProtoMessage() {} func (*UIServiceTypes_GetEntityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 1} + return fileDescriptor_c13b3edb35d457e8, []int{0, 1} } + func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Unmarshal(m, b) } func (m *UIServiceTypes_GetEntityRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(dst, src) +func (m *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(m, src) } func (m *UIServiceTypes_GetEntityRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Size(m) @@ -160,16 +164,17 @@ func (m *UIServiceTypes_GetEntityResponse) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetEntityResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityResponse) ProtoMessage() {} func (*UIServiceTypes_GetEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 2} + return fileDescriptor_c13b3edb35d457e8, []int{0, 2} } + func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Unmarshal(m, b) } func (m *UIServiceTypes_GetEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(dst, src) +func (m *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(m, src) } func (m *UIServiceTypes_GetEntityResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Size(m) @@ -198,16 +203,17 @@ func (m *UIServiceTypes_ListEntitiesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*UIServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 3} + return fileDescriptor_c13b3edb35d457e8, []int{0, 3} } + func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Unmarshal(m, b) } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(dst, src) +func (m *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(m, src) } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Size(m) @@ -242,16 +248,17 @@ func (m *UIServiceTypes_FeatureDetail) Reset() { *m = UIServiceTypes_Fea func (m *UIServiceTypes_FeatureDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 4} + return fileDescriptor_c13b3edb35d457e8, []int{0, 4} } + func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Unmarshal(m, b) } func (m *UIServiceTypes_FeatureDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(dst, src) +func (m *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(m, src) } func (m *UIServiceTypes_FeatureDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Size(m) @@ -315,16 +322,17 @@ func (m *UIServiceTypes_GetFeatureRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetFeatureRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 5} + return fileDescriptor_c13b3edb35d457e8, []int{0, 5} } + func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Unmarshal(m, b) } func (m *UIServiceTypes_GetFeatureRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(m, src) } func (m *UIServiceTypes_GetFeatureRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Size(m) @@ -354,16 +362,17 @@ func (m *UIServiceTypes_GetFeatureResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetFeatureResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 6} + return fileDescriptor_c13b3edb35d457e8, []int{0, 6} } + func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Unmarshal(m, b) } func (m *UIServiceTypes_GetFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(m, src) } func (m *UIServiceTypes_GetFeatureResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Size(m) @@ -399,16 +408,17 @@ func (m *UIServiceTypes_ListFeaturesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 7} + return fileDescriptor_c13b3edb35d457e8, []int{0, 7} } + func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Unmarshal(m, b) } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(dst, src) +func (m *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(m, src) } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Size(m) @@ -439,16 +449,17 @@ func (m *UIServiceTypes_FeatureGroupDetail) Reset() { *m = UIServiceType func (m *UIServiceTypes_FeatureGroupDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureGroupDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureGroupDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 8} + return fileDescriptor_c13b3edb35d457e8, []int{0, 8} } + func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Unmarshal(m, b) } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(dst, src) +func (m *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(m, src) } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Size(m) @@ -484,16 +495,17 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) Reset() { *m = UIService func (m *UIServiceTypes_GetFeatureGroupRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 9} + return fileDescriptor_c13b3edb35d457e8, []int{0, 9} } + func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Unmarshal(m, b) } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(m, src) } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Size(m) @@ -524,16 +536,17 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) Reset() { func (m *UIServiceTypes_GetFeatureGroupResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 10} + return fileDescriptor_c13b3edb35d457e8, []int{0, 10} } + func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Unmarshal(m, b) } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(m, src) } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Size(m) @@ -564,16 +577,17 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) Reset() { func (m *UIServiceTypes_ListFeatureGroupsResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeatureGroupsResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeatureGroupsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 11} + return fileDescriptor_c13b3edb35d457e8, []int{0, 11} } + func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Unmarshal(m, b) } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(dst, src) +func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(m, src) } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Size(m) @@ -604,16 +618,17 @@ func (m *UIServiceTypes_StorageDetail) Reset() { *m = UIServiceTypes_Sto func (m *UIServiceTypes_StorageDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_StorageDetail) ProtoMessage() {} func (*UIServiceTypes_StorageDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 12} + return fileDescriptor_c13b3edb35d457e8, []int{0, 12} } + func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_StorageDetail.Unmarshal(m, b) } func (m *UIServiceTypes_StorageDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_StorageDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(dst, src) +func (m *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(m, src) } func (m *UIServiceTypes_StorageDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_StorageDetail.Size(m) @@ -649,16 +664,17 @@ func (m *UIServiceTypes_GetStorageRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageRequest) ProtoMessage() {} func (*UIServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 13} + return fileDescriptor_c13b3edb35d457e8, []int{0, 13} } + func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Unmarshal(m, b) } func (m *UIServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(dst, src) +func (m *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(m, src) } func (m *UIServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Size(m) @@ -687,16 +703,17 @@ func (m *UIServiceTypes_GetStorageResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageResponse) ProtoMessage() {} func (*UIServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 14} + return fileDescriptor_c13b3edb35d457e8, []int{0, 14} } + func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Unmarshal(m, b) } func (m *UIServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(dst, src) +func (m *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(m, src) } func (m *UIServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Size(m) @@ -725,16 +742,17 @@ func (m *UIServiceTypes_ListStorageResponse) Reset() { *m = UIServiceTyp func (m *UIServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListStorageResponse) ProtoMessage() {} func (*UIServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 15} + return fileDescriptor_c13b3edb35d457e8, []int{0, 15} } + func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Unmarshal(m, b) } func (m *UIServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(dst, src) +func (m *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(m, src) } func (m *UIServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Size(m) @@ -772,6 +790,61 @@ func init() { proto.RegisterType((*UIServiceTypes_ListStorageResponse)(nil), "feast.core.UIServiceTypes.ListStorageResponse") } +func init() { proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_c13b3edb35d457e8) } + +var fileDescriptor_c13b3edb35d457e8 = []byte{ + // 784 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xcd, 0x47, 0x49, 0x9a, 0x69, 0x5a, 0xe8, 0x82, 0xda, 0xb0, 0x50, 0x51, 0x99, 0x03, 0x91, + 0xda, 0xda, 0x6a, 0x28, 0x37, 0x24, 0xa4, 0xd2, 0x52, 0x21, 0x71, 0x00, 0xa7, 0x05, 0x0a, 0x5c, + 0x9c, 0x64, 0xe2, 0xba, 0x4d, 0x62, 0xd7, 0xde, 0x50, 0x05, 0x8e, 0x88, 0x1b, 0xfc, 0x26, 0xfe, + 0x16, 0x47, 0x94, 0xdd, 0xb5, 0xbd, 0x4e, 0x9a, 0xc4, 0xa9, 0xb8, 0x25, 0x3b, 0x33, 0x6f, 0x67, + 0xde, 0x9b, 0x19, 0x2f, 0xd0, 0x36, 0x5a, 0x01, 0x33, 0x9a, 0xae, 0x8f, 0xc6, 0xc9, 0xeb, 0x3a, + 0xfa, 0x5f, 0x9d, 0x26, 0xea, 0x9e, 0xef, 0x32, 0x97, 0x00, 0xb7, 0xe9, 0x43, 0x1b, 0x7d, 0x28, + 0xfc, 0x02, 0x0f, 0x9b, 0x81, 0x71, 0xd8, 0x63, 0x0e, 0x1b, 0xd4, 0x3d, 0x6c, 0x0a, 0x4f, 0xba, + 0xa1, 0x5a, 0x5f, 0xa1, 0xc5, 0xfa, 0x3e, 0x2a, 0x66, 0xed, 0x1a, 0xf3, 0x91, 0xef, 0xf6, 0xbd, + 0x49, 0x10, 0x75, 0xe6, 0xfa, 0x96, 0xad, 0x42, 0x3c, 0xb0, 0x5d, 0xd7, 0xee, 0xa0, 0xc1, 0xff, + 0x35, 0xfa, 0x6d, 0x03, 0xbb, 0x1e, 0x1b, 0x48, 0xe3, 0xa3, 0x51, 0x23, 0x73, 0xba, 0x18, 0x30, + 0xab, 0xeb, 0x09, 0x07, 0xed, 0x6f, 0x19, 0x56, 0xa2, 0xea, 0x8e, 0x07, 0x1e, 0x06, 0xf4, 0x77, + 0x16, 0xca, 0xa2, 0x8e, 0x03, 0x64, 0x96, 0xd3, 0x21, 0x5b, 0xb0, 0x30, 0xbc, 0xbc, 0x92, 0xdd, + 0xcc, 0x56, 0x97, 0x6a, 0xeb, 0xba, 0x28, 0x9e, 0xe7, 0xa3, 0xc7, 0x05, 0x9b, 0xdc, 0x89, 0x10, + 0x58, 0x38, 0x77, 0x1b, 0x41, 0x25, 0xb7, 0x99, 0xaf, 0x96, 0x4c, 0xfe, 0x9b, 0x3c, 0x87, 0xa5, + 0x8e, 0x15, 0xb0, 0x13, 0xaf, 0x65, 0x31, 0x6c, 0x55, 0xf2, 0x1c, 0x87, 0xea, 0x22, 0x37, 0x3d, + 0xcc, 0x4d, 0x3f, 0x0e, 0x73, 0x33, 0x55, 0x77, 0xaa, 0xc1, 0x9d, 0x23, 0x64, 0xe2, 0x22, 0x13, + 0x2f, 0xfb, 0x18, 0x30, 0xb2, 0x02, 0x39, 0xa7, 0xc5, 0x13, 0x2a, 0x99, 0x39, 0xa7, 0x45, 0x8f, + 0x61, 0x55, 0xf1, 0x09, 0x3c, 0xb7, 0x17, 0x20, 0x79, 0x01, 0x05, 0xe4, 0x27, 0x32, 0xf3, 0x27, + 0x7a, 0x2c, 0x9b, 0x9e, 0x2c, 0x5a, 0x57, 0x0b, 0x36, 0x65, 0x18, 0xfd, 0x0c, 0xf7, 0xde, 0x38, + 0x81, 0x80, 0x75, 0x30, 0x88, 0x80, 0x5f, 0xc2, 0x22, 0xca, 0xb3, 0x4a, 0x76, 0x33, 0x3f, 0x0f, + 0x74, 0x14, 0x48, 0x7f, 0xe4, 0x60, 0x59, 0x2a, 0x2e, 0x79, 0xde, 0x4e, 0xf0, 0x5c, 0x49, 0xf0, + 0xac, 0xb4, 0x8e, 0x24, 0x5a, 0x83, 0x72, 0xc3, 0xb1, 0x2f, 0xfb, 0xe8, 0x0f, 0xde, 0x3b, 0x78, + 0x55, 0xc9, 0x71, 0x32, 0x12, 0x67, 0xa4, 0x02, 0x45, 0xec, 0x59, 0x8d, 0x8e, 0x24, 0x7d, 0xd1, + 0x0c, 0xff, 0x46, 0x32, 0x2d, 0x4c, 0x96, 0xe9, 0xd6, 0x5c, 0x32, 0x91, 0x3d, 0x28, 0x36, 0x7d, + 0xe4, 0x91, 0x85, 0x99, 0x91, 0xa1, 0x2b, 0x7d, 0xcc, 0x85, 0x93, 0xd5, 0x4d, 0x52, 0xf7, 0x57, + 0x16, 0x88, 0xea, 0x25, 0x65, 0xd8, 0x87, 0x62, 0x5b, 0x1c, 0x49, 0xca, 0xaa, 0x53, 0x54, 0x48, + 0x50, 0x6d, 0x86, 0x81, 0xa4, 0x06, 0x45, 0xdf, 0xba, 0x1a, 0xd2, 0xca, 0x09, 0x9c, 0x46, 0x7b, + 0xe8, 0x48, 0xbf, 0x88, 0xb6, 0x90, 0xb6, 0xb8, 0x2d, 0x0e, 0x60, 0x51, 0xc2, 0x86, 0x6d, 0x91, + 0x3e, 0xa1, 0x28, 0x92, 0xfe, 0xcc, 0x02, 0x51, 0x37, 0x81, 0x6c, 0x8e, 0xdd, 0x44, 0x73, 0x6c, + 0x5c, 0x97, 0x65, 0xb4, 0x38, 0x64, 0x87, 0x8c, 0xe8, 0x99, 0x9b, 0x6f, 0xec, 0xaa, 0xb0, 0x16, + 0x73, 0xce, 0xa1, 0x27, 0xc9, 0xd3, 0x81, 0xf5, 0x31, 0x4f, 0x49, 0xc9, 0x3b, 0x28, 0xb7, 0x95, + 0x73, 0x99, 0xfd, 0xce, 0x6c, 0x5a, 0x94, 0xd2, 0xcd, 0x04, 0x04, 0xf5, 0xe0, 0xbe, 0xc2, 0x3e, + 0x3f, 0x8b, 0x25, 0xa8, 0xc3, 0xb2, 0xea, 0x1c, 0xea, 0x30, 0xe7, 0x85, 0x49, 0x0c, 0xfa, 0x1d, + 0x96, 0xe5, 0xda, 0x4d, 0x31, 0xa8, 0xca, 0x82, 0xfe, 0x2f, 0x32, 0x88, 0x01, 0x91, 0xa8, 0x93, + 0x14, 0xf8, 0xc8, 0xe7, 0x23, 0x72, 0x8a, 0xe7, 0x23, 0x10, 0x47, 0x29, 0xe6, 0x23, 0x51, 0xa1, + 0x19, 0x06, 0xd2, 0x53, 0xb8, 0x3b, 0x64, 0x7b, 0x2a, 0x74, 0xfe, 0x46, 0xd0, 0xb5, 0x3f, 0x05, + 0x28, 0x45, 0x9e, 0xe4, 0x0c, 0x4a, 0xd1, 0x06, 0x27, 0x5b, 0x53, 0xd0, 0x46, 0xbf, 0x05, 0x74, + 0x3b, 0x9d, 0xb3, 0xc8, 0x5c, 0xcb, 0x90, 0x53, 0x28, 0xab, 0x5b, 0x9d, 0xac, 0x8d, 0x49, 0x71, + 0x38, 0xfc, 0x82, 0x52, 0x63, 0x0a, 0xee, 0x75, 0x9f, 0x05, 0x2d, 0x43, 0x2e, 0x00, 0xe2, 0x49, + 0x20, 0x33, 0x12, 0x4b, 0x2e, 0x3d, 0xba, 0x93, 0xd2, 0x7b, 0xb4, 0x8e, 0x70, 0x0d, 0xdd, 0xb8, + 0x8e, 0xd1, 0x3d, 0xa6, 0x65, 0xc8, 0x37, 0xb8, 0x3d, 0x32, 0xd1, 0x64, 0x37, 0x55, 0x7a, 0xea, + 0x9e, 0xa0, 0xb5, 0x79, 0x42, 0xa2, 0xbb, 0x9b, 0xb0, 0x3a, 0x36, 0xdf, 0x13, 0x6b, 0xdb, 0x4b, + 0x57, 0x5b, 0x72, 0x4b, 0x44, 0x42, 0xc9, 0xc6, 0x9c, 0x25, 0x54, 0x72, 0xf8, 0x66, 0x09, 0x35, + 0x32, 0x2a, 0x5a, 0x86, 0x7c, 0x80, 0x25, 0x65, 0x86, 0x26, 0xd6, 0xa2, 0xcf, 0xa8, 0x65, 0x0c, + 0x78, 0xbf, 0x0e, 0xca, 0x43, 0x74, 0x3f, 0x7e, 0xc7, 0xbd, 0x1d, 0x02, 0x7f, 0x7a, 0x66, 0x3b, + 0xec, 0xac, 0xdf, 0xd0, 0x9b, 0x6e, 0xd7, 0xb0, 0xdd, 0x73, 0xbc, 0x30, 0xc4, 0x43, 0x92, 0x5f, + 0x1b, 0x18, 0x36, 0xf6, 0xd0, 0x1f, 0xae, 0x18, 0xc3, 0x76, 0x8d, 0xf8, 0xad, 0xdb, 0x28, 0x70, + 0xfb, 0xd3, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x90, 0x76, 0x0d, 0x62, 0x00, 0x0b, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -785,34 +858,34 @@ const _ = grpc.SupportPackageIsVersion4 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type UIServiceClient interface { // - // Get entity specified in request. - // This process returns a single of entity specs. + //Get entity specified in request. + //This process returns a single of entity specs. GetEntity(ctx context.Context, in *UIServiceTypes_GetEntityRequest, opts ...grpc.CallOption) (*UIServiceTypes_GetEntityResponse, error) // - // Get all entities. - // This process returns a list of entity specs. + //Get all entities. + //This process returns a list of entity specs. ListEntities(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*UIServiceTypes_ListEntitiesResponse, error) // - // Get feature specified in request. + //Get feature specified in request. GetFeature(ctx context.Context, in *UIServiceTypes_GetFeatureRequest, opts ...grpc.CallOption) (*UIServiceTypes_GetFeatureResponse, error) // - // Get all features. - // This process returns a list of feature specs. + //Get all features. + //This process returns a list of feature specs. ListFeatures(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*UIServiceTypes_ListFeaturesResponse, error) // - // Get feature group specified in request. + //Get feature group specified in request. GetFeatureGroup(ctx context.Context, in *UIServiceTypes_GetFeatureGroupRequest, opts ...grpc.CallOption) (*UIServiceTypes_GetFeatureGroupResponse, error) // - // Get all feature groups. - // This process returns a list of feature group specs. + //Get all feature groups. + //This process returns a list of feature group specs. ListFeatureGroups(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*UIServiceTypes_ListFeatureGroupsResponse, error) // - // Get storage spec specified in request. + //Get storage spec specified in request. // GetStorage(ctx context.Context, in *UIServiceTypes_GetStorageRequest, opts ...grpc.CallOption) (*UIServiceTypes_GetStorageResponse, error) // - // Get all storage specs. - // This process returns a list of storage specs. + //Get all storage specs. + //This process returns a list of storage specs. ListStorage(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*UIServiceTypes_ListStorageResponse, error) } @@ -899,37 +972,66 @@ func (c *uIServiceClient) ListStorage(ctx context.Context, in *empty.Empty, opts // UIServiceServer is the server API for UIService service. type UIServiceServer interface { // - // Get entity specified in request. - // This process returns a single of entity specs. + //Get entity specified in request. + //This process returns a single of entity specs. GetEntity(context.Context, *UIServiceTypes_GetEntityRequest) (*UIServiceTypes_GetEntityResponse, error) // - // Get all entities. - // This process returns a list of entity specs. + //Get all entities. + //This process returns a list of entity specs. ListEntities(context.Context, *empty.Empty) (*UIServiceTypes_ListEntitiesResponse, error) // - // Get feature specified in request. + //Get feature specified in request. GetFeature(context.Context, *UIServiceTypes_GetFeatureRequest) (*UIServiceTypes_GetFeatureResponse, error) // - // Get all features. - // This process returns a list of feature specs. + //Get all features. + //This process returns a list of feature specs. ListFeatures(context.Context, *empty.Empty) (*UIServiceTypes_ListFeaturesResponse, error) // - // Get feature group specified in request. + //Get feature group specified in request. GetFeatureGroup(context.Context, *UIServiceTypes_GetFeatureGroupRequest) (*UIServiceTypes_GetFeatureGroupResponse, error) // - // Get all feature groups. - // This process returns a list of feature group specs. + //Get all feature groups. + //This process returns a list of feature group specs. ListFeatureGroups(context.Context, *empty.Empty) (*UIServiceTypes_ListFeatureGroupsResponse, error) // - // Get storage spec specified in request. + //Get storage spec specified in request. // GetStorage(context.Context, *UIServiceTypes_GetStorageRequest) (*UIServiceTypes_GetStorageResponse, error) // - // Get all storage specs. - // This process returns a list of storage specs. + //Get all storage specs. + //This process returns a list of storage specs. ListStorage(context.Context, *empty.Empty) (*UIServiceTypes_ListStorageResponse, error) } +// UnimplementedUIServiceServer can be embedded to have forward compatible implementations. +type UnimplementedUIServiceServer struct { +} + +func (*UnimplementedUIServiceServer) GetEntity(ctx context.Context, req *UIServiceTypes_GetEntityRequest) (*UIServiceTypes_GetEntityResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetEntity not implemented") +} +func (*UnimplementedUIServiceServer) ListEntities(ctx context.Context, req *empty.Empty) (*UIServiceTypes_ListEntitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListEntities not implemented") +} +func (*UnimplementedUIServiceServer) GetFeature(ctx context.Context, req *UIServiceTypes_GetFeatureRequest) (*UIServiceTypes_GetFeatureResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented") +} +func (*UnimplementedUIServiceServer) ListFeatures(ctx context.Context, req *empty.Empty) (*UIServiceTypes_ListFeaturesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListFeatures not implemented") +} +func (*UnimplementedUIServiceServer) GetFeatureGroup(ctx context.Context, req *UIServiceTypes_GetFeatureGroupRequest) (*UIServiceTypes_GetFeatureGroupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeatureGroup not implemented") +} +func (*UnimplementedUIServiceServer) ListFeatureGroups(ctx context.Context, req *empty.Empty) (*UIServiceTypes_ListFeatureGroupsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListFeatureGroups not implemented") +} +func (*UnimplementedUIServiceServer) GetStorage(ctx context.Context, req *UIServiceTypes_GetStorageRequest) (*UIServiceTypes_GetStorageResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStorage not implemented") +} +func (*UnimplementedUIServiceServer) ListStorage(ctx context.Context, req *empty.Empty) (*UIServiceTypes_ListStorageResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListStorage not implemented") +} + func RegisterUIServiceServer(s *grpc.Server, srv UIServiceServer) { s.RegisterService(&_UIService_serviceDesc, srv) } @@ -1118,60 +1220,3 @@ var _UIService_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "feast/core/UIService.proto", } - -func init() { - proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_UIService_04866529701c634c) -} - -var fileDescriptor_UIService_04866529701c634c = []byte{ - // 784 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xcd, 0x47, 0x49, 0x9a, 0x69, 0x5a, 0xe8, 0x82, 0xda, 0xb0, 0x50, 0x51, 0x99, 0x03, 0x91, - 0xda, 0xda, 0x6a, 0x28, 0x37, 0x24, 0xa4, 0xd2, 0x52, 0x21, 0x71, 0x00, 0xa7, 0x05, 0x0a, 0x5c, - 0x9c, 0x64, 0xe2, 0xba, 0x4d, 0x62, 0xd7, 0xde, 0x50, 0x05, 0x8e, 0x88, 0x1b, 0xfc, 0x26, 0xfe, - 0x16, 0x47, 0x94, 0xdd, 0xb5, 0xbd, 0x4e, 0x9a, 0xc4, 0xa9, 0xb8, 0x25, 0x3b, 0x33, 0x6f, 0x67, - 0xde, 0x9b, 0x19, 0x2f, 0xd0, 0x36, 0x5a, 0x01, 0x33, 0x9a, 0xae, 0x8f, 0xc6, 0xc9, 0xeb, 0x3a, - 0xfa, 0x5f, 0x9d, 0x26, 0xea, 0x9e, 0xef, 0x32, 0x97, 0x00, 0xb7, 0xe9, 0x43, 0x1b, 0x7d, 0x28, - 0xfc, 0x02, 0x0f, 0x9b, 0x81, 0x71, 0xd8, 0x63, 0x0e, 0x1b, 0xd4, 0x3d, 0x6c, 0x0a, 0x4f, 0xba, - 0xa1, 0x5a, 0x5f, 0xa1, 0xc5, 0xfa, 0x3e, 0x2a, 0x66, 0xed, 0x1a, 0xf3, 0x91, 0xef, 0xf6, 0xbd, - 0x49, 0x10, 0x75, 0xe6, 0xfa, 0x96, 0xad, 0x42, 0x3c, 0xb0, 0x5d, 0xd7, 0xee, 0xa0, 0xc1, 0xff, - 0x35, 0xfa, 0x6d, 0x03, 0xbb, 0x1e, 0x1b, 0x48, 0xe3, 0xa3, 0x51, 0x23, 0x73, 0xba, 0x18, 0x30, - 0xab, 0xeb, 0x09, 0x07, 0xed, 0x6f, 0x19, 0x56, 0xa2, 0xea, 0x8e, 0x07, 0x1e, 0x06, 0xf4, 0x77, - 0x16, 0xca, 0xa2, 0x8e, 0x03, 0x64, 0x96, 0xd3, 0x21, 0x5b, 0xb0, 0x30, 0xbc, 0xbc, 0x92, 0xdd, - 0xcc, 0x56, 0x97, 0x6a, 0xeb, 0xba, 0x28, 0x9e, 0xe7, 0xa3, 0xc7, 0x05, 0x9b, 0xdc, 0x89, 0x10, - 0x58, 0x38, 0x77, 0x1b, 0x41, 0x25, 0xb7, 0x99, 0xaf, 0x96, 0x4c, 0xfe, 0x9b, 0x3c, 0x87, 0xa5, - 0x8e, 0x15, 0xb0, 0x13, 0xaf, 0x65, 0x31, 0x6c, 0x55, 0xf2, 0x1c, 0x87, 0xea, 0x22, 0x37, 0x3d, - 0xcc, 0x4d, 0x3f, 0x0e, 0x73, 0x33, 0x55, 0x77, 0xaa, 0xc1, 0x9d, 0x23, 0x64, 0xe2, 0x22, 0x13, - 0x2f, 0xfb, 0x18, 0x30, 0xb2, 0x02, 0x39, 0xa7, 0xc5, 0x13, 0x2a, 0x99, 0x39, 0xa7, 0x45, 0x8f, - 0x61, 0x55, 0xf1, 0x09, 0x3c, 0xb7, 0x17, 0x20, 0x79, 0x01, 0x05, 0xe4, 0x27, 0x32, 0xf3, 0x27, - 0x7a, 0x2c, 0x9b, 0x9e, 0x2c, 0x5a, 0x57, 0x0b, 0x36, 0x65, 0x18, 0xfd, 0x0c, 0xf7, 0xde, 0x38, - 0x81, 0x80, 0x75, 0x30, 0x88, 0x80, 0x5f, 0xc2, 0x22, 0xca, 0xb3, 0x4a, 0x76, 0x33, 0x3f, 0x0f, - 0x74, 0x14, 0x48, 0x7f, 0xe4, 0x60, 0x59, 0x2a, 0x2e, 0x79, 0xde, 0x4e, 0xf0, 0x5c, 0x49, 0xf0, - 0xac, 0xb4, 0x8e, 0x24, 0x5a, 0x83, 0x72, 0xc3, 0xb1, 0x2f, 0xfb, 0xe8, 0x0f, 0xde, 0x3b, 0x78, - 0x55, 0xc9, 0x71, 0x32, 0x12, 0x67, 0xa4, 0x02, 0x45, 0xec, 0x59, 0x8d, 0x8e, 0x24, 0x7d, 0xd1, - 0x0c, 0xff, 0x46, 0x32, 0x2d, 0x4c, 0x96, 0xe9, 0xd6, 0x5c, 0x32, 0x91, 0x3d, 0x28, 0x36, 0x7d, - 0xe4, 0x91, 0x85, 0x99, 0x91, 0xa1, 0x2b, 0x7d, 0xcc, 0x85, 0x93, 0xd5, 0x4d, 0x52, 0xf7, 0x57, - 0x16, 0x88, 0xea, 0x25, 0x65, 0xd8, 0x87, 0x62, 0x5b, 0x1c, 0x49, 0xca, 0xaa, 0x53, 0x54, 0x48, - 0x50, 0x6d, 0x86, 0x81, 0xa4, 0x06, 0x45, 0xdf, 0xba, 0x1a, 0xd2, 0xca, 0x09, 0x9c, 0x46, 0x7b, - 0xe8, 0x48, 0xbf, 0x88, 0xb6, 0x90, 0xb6, 0xb8, 0x2d, 0x0e, 0x60, 0x51, 0xc2, 0x86, 0x6d, 0x91, - 0x3e, 0xa1, 0x28, 0x92, 0xfe, 0xcc, 0x02, 0x51, 0x37, 0x81, 0x6c, 0x8e, 0xdd, 0x44, 0x73, 0x6c, - 0x5c, 0x97, 0x65, 0xb4, 0x38, 0x64, 0x87, 0x8c, 0xe8, 0x99, 0x9b, 0x6f, 0xec, 0xaa, 0xb0, 0x16, - 0x73, 0xce, 0xa1, 0x27, 0xc9, 0xd3, 0x81, 0xf5, 0x31, 0x4f, 0x49, 0xc9, 0x3b, 0x28, 0xb7, 0x95, - 0x73, 0x99, 0xfd, 0xce, 0x6c, 0x5a, 0x94, 0xd2, 0xcd, 0x04, 0x04, 0xf5, 0xe0, 0xbe, 0xc2, 0x3e, - 0x3f, 0x8b, 0x25, 0xa8, 0xc3, 0xb2, 0xea, 0x1c, 0xea, 0x30, 0xe7, 0x85, 0x49, 0x0c, 0xfa, 0x1d, - 0x96, 0xe5, 0xda, 0x4d, 0x31, 0xa8, 0xca, 0x82, 0xfe, 0x2f, 0x32, 0x88, 0x01, 0x91, 0xa8, 0x93, - 0x14, 0xf8, 0xc8, 0xe7, 0x23, 0x72, 0x8a, 0xe7, 0x23, 0x10, 0x47, 0x29, 0xe6, 0x23, 0x51, 0xa1, - 0x19, 0x06, 0xd2, 0x53, 0xb8, 0x3b, 0x64, 0x7b, 0x2a, 0x74, 0xfe, 0x46, 0xd0, 0xb5, 0x3f, 0x05, - 0x28, 0x45, 0x9e, 0xe4, 0x0c, 0x4a, 0xd1, 0x06, 0x27, 0x5b, 0x53, 0xd0, 0x46, 0xbf, 0x05, 0x74, - 0x3b, 0x9d, 0xb3, 0xc8, 0x5c, 0xcb, 0x90, 0x53, 0x28, 0xab, 0x5b, 0x9d, 0xac, 0x8d, 0x49, 0x71, - 0x38, 0xfc, 0x82, 0x52, 0x63, 0x0a, 0xee, 0x75, 0x9f, 0x05, 0x2d, 0x43, 0x2e, 0x00, 0xe2, 0x49, - 0x20, 0x33, 0x12, 0x4b, 0x2e, 0x3d, 0xba, 0x93, 0xd2, 0x7b, 0xb4, 0x8e, 0x70, 0x0d, 0xdd, 0xb8, - 0x8e, 0xd1, 0x3d, 0xa6, 0x65, 0xc8, 0x37, 0xb8, 0x3d, 0x32, 0xd1, 0x64, 0x37, 0x55, 0x7a, 0xea, - 0x9e, 0xa0, 0xb5, 0x79, 0x42, 0xa2, 0xbb, 0x9b, 0xb0, 0x3a, 0x36, 0xdf, 0x13, 0x6b, 0xdb, 0x4b, - 0x57, 0x5b, 0x72, 0x4b, 0x44, 0x42, 0xc9, 0xc6, 0x9c, 0x25, 0x54, 0x72, 0xf8, 0x66, 0x09, 0x35, - 0x32, 0x2a, 0x5a, 0x86, 0x7c, 0x80, 0x25, 0x65, 0x86, 0x26, 0xd6, 0xa2, 0xcf, 0xa8, 0x65, 0x0c, - 0x78, 0xbf, 0x0e, 0xca, 0x43, 0x74, 0x3f, 0x7e, 0xc7, 0xbd, 0x1d, 0x02, 0x7f, 0x7a, 0x66, 0x3b, - 0xec, 0xac, 0xdf, 0xd0, 0x9b, 0x6e, 0xd7, 0xb0, 0xdd, 0x73, 0xbc, 0x30, 0xc4, 0x43, 0x92, 0x5f, - 0x1b, 0x18, 0x36, 0xf6, 0xd0, 0x1f, 0xae, 0x18, 0xc3, 0x76, 0x8d, 0xf8, 0xad, 0xdb, 0x28, 0x70, - 0xfb, 0xd3, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x90, 0x76, 0x0d, 0x62, 0x00, 0x0b, 0x00, 0x00, -} diff --git a/protos/generated/go/feast/serving/Serving.pb.go b/protos/generated/go/feast/serving/Serving.pb.go index 6f61517bee9..c2366dafa9f 100644 --- a/protos/generated/go/feast/serving/Serving.pb.go +++ b/protos/generated/go/feast/serving/Serving.pb.go @@ -1,17 +1,18 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/serving/Serving.proto -package serving // import "github.com/gojek/feast/protos/generated/go/feast/serving" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import types "github.com/gojek/feast/protos/generated/go/feast/types" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +package serving import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + types "github.com/gojek/feast/protos/generated/go/feast/types" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -23,7 +24,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type QueryFeaturesRequest struct { // e.g. "driver", "customer", "city". @@ -44,16 +45,17 @@ func (m *QueryFeaturesRequest) Reset() { *m = QueryFeaturesRequest{} } func (m *QueryFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*QueryFeaturesRequest) ProtoMessage() {} func (*QueryFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{0} + return fileDescriptor_7609de6de542e6f0, []int{0} } + func (m *QueryFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_QueryFeaturesRequest.Unmarshal(m, b) } func (m *QueryFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_QueryFeaturesRequest.Marshal(b, m, deterministic) } -func (dst *QueryFeaturesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeaturesRequest.Merge(dst, src) +func (m *QueryFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesRequest.Merge(m, src) } func (m *QueryFeaturesRequest) XXX_Size() int { return xxx_messageInfo_QueryFeaturesRequest.Size(m) @@ -99,16 +101,17 @@ func (m *QueryFeaturesResponse) Reset() { *m = QueryFeaturesResponse{} } func (m *QueryFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*QueryFeaturesResponse) ProtoMessage() {} func (*QueryFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{1} + return fileDescriptor_7609de6de542e6f0, []int{1} } + func (m *QueryFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_QueryFeaturesResponse.Unmarshal(m, b) } func (m *QueryFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_QueryFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *QueryFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeaturesResponse.Merge(dst, src) +func (m *QueryFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesResponse.Merge(m, src) } func (m *QueryFeaturesResponse) XXX_Size() int { return xxx_messageInfo_QueryFeaturesResponse.Size(m) @@ -145,16 +148,17 @@ func (m *Entity) Reset() { *m = Entity{} } func (m *Entity) String() string { return proto.CompactTextString(m) } func (*Entity) ProtoMessage() {} func (*Entity) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{2} + return fileDescriptor_7609de6de542e6f0, []int{2} } + func (m *Entity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Entity.Unmarshal(m, b) } func (m *Entity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Entity.Marshal(b, m, deterministic) } -func (dst *Entity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Entity.Merge(dst, src) +func (m *Entity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Entity.Merge(m, src) } func (m *Entity) XXX_Size() int { return xxx_messageInfo_Entity.Size(m) @@ -186,16 +190,17 @@ func (m *FeatureValue) Reset() { *m = FeatureValue{} } func (m *FeatureValue) String() string { return proto.CompactTextString(m) } func (*FeatureValue) ProtoMessage() {} func (*FeatureValue) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_f91320f9a3f0c4cf, []int{3} + return fileDescriptor_7609de6de542e6f0, []int{3} } + func (m *FeatureValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureValue.Unmarshal(m, b) } func (m *FeatureValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureValue.Marshal(b, m, deterministic) } -func (dst *FeatureValue) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureValue.Merge(dst, src) +func (m *FeatureValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureValue.Merge(m, src) } func (m *FeatureValue) XXX_Size() int { return xxx_messageInfo_FeatureValue.Size(m) @@ -229,6 +234,39 @@ func init() { proto.RegisterType((*FeatureValue)(nil), "feast.serving.FeatureValue") } +func init() { proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_7609de6de542e6f0) } + +var fileDescriptor_7609de6de542e6f0 = []byte{ + // 429 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xd1, 0x8a, 0xd3, 0x40, + 0x14, 0x75, 0x5a, 0x5c, 0x36, 0x77, 0x0d, 0xca, 0xe0, 0x62, 0xc8, 0x8a, 0x96, 0xac, 0x0f, 0x01, + 0x61, 0x06, 0xe3, 0x4b, 0xf1, 0x45, 0x5c, 0x58, 0xa1, 0x2f, 0x4b, 0x8d, 0x22, 0x52, 0x7c, 0x49, + 0xed, 0x6d, 0x4c, 0xdb, 0x64, 0x62, 0x66, 0x52, 0xc8, 0xf7, 0xf8, 0x59, 0xfe, 0x8c, 0x74, 0x66, + 0xd2, 0x26, 0xa5, 0xe8, 0x3e, 0x25, 0x33, 0xe7, 0xde, 0x73, 0xce, 0xcc, 0x3d, 0x03, 0x57, 0x4b, + 0x4c, 0xa4, 0xe2, 0x12, 0xab, 0x6d, 0x56, 0xa4, 0xfc, 0xb3, 0xf9, 0xb2, 0xb2, 0x12, 0x4a, 0x50, + 0x57, 0x83, 0xcc, 0x82, 0xfe, 0xcb, 0x54, 0x88, 0x74, 0x83, 0x5c, 0x83, 0xf3, 0x7a, 0xc9, 0x55, + 0x96, 0xa3, 0x54, 0x49, 0x5e, 0x9a, 0x7a, 0xff, 0x99, 0x21, 0x53, 0x4d, 0x89, 0x92, 0x7f, 0x4d, + 0x36, 0x35, 0x1a, 0x20, 0x28, 0xe1, 0xe9, 0xa7, 0x1a, 0xab, 0xe6, 0x23, 0x26, 0xaa, 0xae, 0x50, + 0xc6, 0xf8, 0xab, 0x46, 0xa9, 0xe8, 0x0b, 0x00, 0x2c, 0x54, 0xa6, 0x9a, 0xbb, 0x24, 0x47, 0x8f, + 0x8c, 0x48, 0xe8, 0xc4, 0x9d, 0x1d, 0xea, 0xc3, 0xb9, 0x59, 0x4d, 0x16, 0xde, 0x60, 0x34, 0x0c, + 0x9d, 0x78, 0xbf, 0xa6, 0xcf, 0xc1, 0x59, 0x1a, 0xba, 0xc9, 0xc2, 0x1b, 0x6a, 0xf0, 0xb0, 0x11, + 0xfc, 0x21, 0x70, 0x79, 0x24, 0x29, 0x4b, 0x51, 0x48, 0xfc, 0xaf, 0xe6, 0x9d, 0xd5, 0xcc, 0x50, + 0x6a, 0xcd, 0x8b, 0x28, 0x62, 0xbd, 0x7b, 0x60, 0x27, 0x79, 0xd9, 0xad, 0x6d, 0xba, 0x2d, 0x54, + 0xd5, 0xc4, 0x7b, 0x0e, 0x3f, 0x06, 0xb7, 0x07, 0xd1, 0x27, 0x30, 0x5c, 0x63, 0x63, 0x95, 0x77, + 0xbf, 0xf4, 0x35, 0x3c, 0xdc, 0xee, 0x6e, 0xcb, 0x1b, 0x8c, 0x48, 0x78, 0x11, 0x5d, 0x1e, 0xe9, + 0xe9, 0xf6, 0x26, 0x36, 0x35, 0xef, 0x06, 0x63, 0x12, 0xfc, 0x26, 0x70, 0x66, 0x76, 0xe9, 0x7b, + 0x38, 0xb7, 0xa7, 0x96, 0x1e, 0xd1, 0x76, 0xaf, 0x4f, 0xb6, 0xb3, 0xd6, 0xb0, 0xf5, 0xd7, 0x36, + 0xf9, 0xdf, 0xc0, 0xed, 0x41, 0x27, 0xfc, 0xbd, 0xe9, 0xfb, 0xbb, 0x3a, 0x12, 0xb0, 0xed, 0x7a, + 0xe0, 0x5d, 0x97, 0x15, 0x3c, 0xea, 0x42, 0x34, 0x6c, 0x69, 0x88, 0xa6, 0xa1, 0x96, 0x46, 0xc7, + 0x85, 0x75, 0xbb, 0xe9, 0x18, 0x9c, 0x7d, 0xb6, 0xac, 0xa8, 0xcf, 0x4c, 0xfa, 0x58, 0x9b, 0x3e, + 0xf6, 0xa5, 0xad, 0x88, 0x0f, 0xc5, 0xd1, 0x0a, 0xc0, 0x66, 0xf8, 0xc3, 0x74, 0x42, 0xbf, 0x83, + 0xdb, 0x1b, 0x16, 0xbd, 0xfe, 0xf7, 0x28, 0x75, 0x2a, 0xfd, 0x57, 0xf7, 0x99, 0x77, 0xf0, 0xe0, + 0x66, 0x06, 0xfd, 0x07, 0x72, 0xf3, 0xf8, 0x20, 0x3d, 0xdd, 0xb9, 0x9c, 0x8d, 0xd3, 0x4c, 0xfd, + 0xac, 0xe7, 0xec, 0x87, 0xc8, 0x79, 0x2a, 0x56, 0xb8, 0xe6, 0xe6, 0x85, 0xe8, 0x33, 0x48, 0x9e, + 0x62, 0x81, 0x55, 0xa2, 0x70, 0xc1, 0x53, 0xc1, 0x7b, 0x0f, 0x71, 0x7e, 0xa6, 0x4b, 0xde, 0xfe, + 0x0d, 0x00, 0x00, 0xff, 0xff, 0x77, 0xb5, 0x0e, 0xd1, 0xa0, 0x03, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -268,6 +306,14 @@ type ServingAPIServer interface { QueryFeatures(context.Context, *QueryFeaturesRequest) (*QueryFeaturesResponse, error) } +// UnimplementedServingAPIServer can be embedded to have forward compatible implementations. +type UnimplementedServingAPIServer struct { +} + +func (*UnimplementedServingAPIServer) QueryFeatures(ctx context.Context, req *QueryFeaturesRequest) (*QueryFeaturesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QueryFeatures not implemented") +} + func RegisterServingAPIServer(s *grpc.Server, srv ServingAPIServer) { s.RegisterService(&_ServingAPI_serviceDesc, srv) } @@ -302,38 +348,3 @@ var _ServingAPI_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "feast/serving/Serving.proto", } - -func init() { - proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_Serving_f91320f9a3f0c4cf) -} - -var fileDescriptor_Serving_f91320f9a3f0c4cf = []byte{ - // 429 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xd1, 0x8a, 0xd3, 0x40, - 0x14, 0x75, 0x5a, 0x5c, 0x36, 0x77, 0x0d, 0xca, 0xe0, 0x62, 0xc8, 0x8a, 0x96, 0xac, 0x0f, 0x01, - 0x61, 0x06, 0xe3, 0x4b, 0xf1, 0x45, 0x5c, 0x58, 0xa1, 0x2f, 0x4b, 0x8d, 0x22, 0x52, 0x7c, 0x49, - 0xed, 0x6d, 0x4c, 0xdb, 0x64, 0x62, 0x66, 0x52, 0xc8, 0xf7, 0xf8, 0x59, 0xfe, 0x8c, 0x74, 0x66, - 0xd2, 0x26, 0xa5, 0xe8, 0x3e, 0x25, 0x33, 0xe7, 0xde, 0x73, 0xce, 0xcc, 0x3d, 0x03, 0x57, 0x4b, - 0x4c, 0xa4, 0xe2, 0x12, 0xab, 0x6d, 0x56, 0xa4, 0xfc, 0xb3, 0xf9, 0xb2, 0xb2, 0x12, 0x4a, 0x50, - 0x57, 0x83, 0xcc, 0x82, 0xfe, 0xcb, 0x54, 0x88, 0x74, 0x83, 0x5c, 0x83, 0xf3, 0x7a, 0xc9, 0x55, - 0x96, 0xa3, 0x54, 0x49, 0x5e, 0x9a, 0x7a, 0xff, 0x99, 0x21, 0x53, 0x4d, 0x89, 0x92, 0x7f, 0x4d, - 0x36, 0x35, 0x1a, 0x20, 0x28, 0xe1, 0xe9, 0xa7, 0x1a, 0xab, 0xe6, 0x23, 0x26, 0xaa, 0xae, 0x50, - 0xc6, 0xf8, 0xab, 0x46, 0xa9, 0xe8, 0x0b, 0x00, 0x2c, 0x54, 0xa6, 0x9a, 0xbb, 0x24, 0x47, 0x8f, - 0x8c, 0x48, 0xe8, 0xc4, 0x9d, 0x1d, 0xea, 0xc3, 0xb9, 0x59, 0x4d, 0x16, 0xde, 0x60, 0x34, 0x0c, - 0x9d, 0x78, 0xbf, 0xa6, 0xcf, 0xc1, 0x59, 0x1a, 0xba, 0xc9, 0xc2, 0x1b, 0x6a, 0xf0, 0xb0, 0x11, - 0xfc, 0x21, 0x70, 0x79, 0x24, 0x29, 0x4b, 0x51, 0x48, 0xfc, 0xaf, 0xe6, 0x9d, 0xd5, 0xcc, 0x50, - 0x6a, 0xcd, 0x8b, 0x28, 0x62, 0xbd, 0x7b, 0x60, 0x27, 0x79, 0xd9, 0xad, 0x6d, 0xba, 0x2d, 0x54, - 0xd5, 0xc4, 0x7b, 0x0e, 0x3f, 0x06, 0xb7, 0x07, 0xd1, 0x27, 0x30, 0x5c, 0x63, 0x63, 0x95, 0x77, - 0xbf, 0xf4, 0x35, 0x3c, 0xdc, 0xee, 0x6e, 0xcb, 0x1b, 0x8c, 0x48, 0x78, 0x11, 0x5d, 0x1e, 0xe9, - 0xe9, 0xf6, 0x26, 0x36, 0x35, 0xef, 0x06, 0x63, 0x12, 0xfc, 0x26, 0x70, 0x66, 0x76, 0xe9, 0x7b, - 0x38, 0xb7, 0xa7, 0x96, 0x1e, 0xd1, 0x76, 0xaf, 0x4f, 0xb6, 0xb3, 0xd6, 0xb0, 0xf5, 0xd7, 0x36, - 0xf9, 0xdf, 0xc0, 0xed, 0x41, 0x27, 0xfc, 0xbd, 0xe9, 0xfb, 0xbb, 0x3a, 0x12, 0xb0, 0xed, 0x7a, - 0xe0, 0x5d, 0x97, 0x15, 0x3c, 0xea, 0x42, 0x34, 0x6c, 0x69, 0x88, 0xa6, 0xa1, 0x96, 0x46, 0xc7, - 0x85, 0x75, 0xbb, 0xe9, 0x18, 0x9c, 0x7d, 0xb6, 0xac, 0xa8, 0xcf, 0x4c, 0xfa, 0x58, 0x9b, 0x3e, - 0xf6, 0xa5, 0xad, 0x88, 0x0f, 0xc5, 0xd1, 0x0a, 0xc0, 0x66, 0xf8, 0xc3, 0x74, 0x42, 0xbf, 0x83, - 0xdb, 0x1b, 0x16, 0xbd, 0xfe, 0xf7, 0x28, 0x75, 0x2a, 0xfd, 0x57, 0xf7, 0x99, 0x77, 0xf0, 0xe0, - 0x66, 0x06, 0xfd, 0x07, 0x72, 0xf3, 0xf8, 0x20, 0x3d, 0xdd, 0xb9, 0x9c, 0x8d, 0xd3, 0x4c, 0xfd, - 0xac, 0xe7, 0xec, 0x87, 0xc8, 0x79, 0x2a, 0x56, 0xb8, 0xe6, 0xe6, 0x85, 0xe8, 0x33, 0x48, 0x9e, - 0x62, 0x81, 0x55, 0xa2, 0x70, 0xc1, 0x53, 0xc1, 0x7b, 0x0f, 0x71, 0x7e, 0xa6, 0x4b, 0xde, 0xfe, - 0x0d, 0x00, 0x00, 0xff, 0xff, 0x77, 0xb5, 0x0e, 0xd1, 0xa0, 0x03, 0x00, 0x00, -} diff --git a/protos/generated/go/feast/specs/EntitySpec.pb.go b/protos/generated/go/feast/specs/EntitySpec.pb.go index 0f4374c679e..75ef9aa3186 100644 --- a/protos/generated/go/feast/specs/EntitySpec.pb.go +++ b/protos/generated/go/feast/specs/EntitySpec.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/specs/EntitySpec.proto -package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" +package specs -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type EntitySpec struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -31,16 +33,17 @@ func (m *EntitySpec) Reset() { *m = EntitySpec{} } func (m *EntitySpec) String() string { return proto.CompactTextString(m) } func (*EntitySpec) ProtoMessage() {} func (*EntitySpec) Descriptor() ([]byte, []int) { - return fileDescriptor_EntitySpec_b8950ded39b854cb, []int{0} + return fileDescriptor_a3230229c2278d5e, []int{0} } + func (m *EntitySpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_EntitySpec.Unmarshal(m, b) } func (m *EntitySpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_EntitySpec.Marshal(b, m, deterministic) } -func (dst *EntitySpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_EntitySpec.Merge(dst, src) +func (m *EntitySpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_EntitySpec.Merge(m, src) } func (m *EntitySpec) XXX_Size() int { return xxx_messageInfo_EntitySpec.Size(m) @@ -76,11 +79,9 @@ func init() { proto.RegisterType((*EntitySpec)(nil), "feast.specs.EntitySpec") } -func init() { - proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_EntitySpec_b8950ded39b854cb) -} +func init() { proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_a3230229c2278d5e) } -var fileDescriptor_EntitySpec_b8950ded39b854cb = []byte{ +var fileDescriptor_a3230229c2278d5e = []byte{ // 177 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0xcd, 0x2b, 0xc9, 0x2c, 0xa9, 0x0c, 0x2e, diff --git a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go index 8349e5a53aa..ed043326936 100644 --- a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/specs/FeatureGroupSpec.proto -package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" +package specs -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type FeatureGroupSpec struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` @@ -31,16 +33,17 @@ func (m *FeatureGroupSpec) Reset() { *m = FeatureGroupSpec{} } func (m *FeatureGroupSpec) String() string { return proto.CompactTextString(m) } func (*FeatureGroupSpec) ProtoMessage() {} func (*FeatureGroupSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureGroupSpec_21c67ee01edd412c, []int{0} + return fileDescriptor_df3b6a9e736b5719, []int{0} } + func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureGroupSpec.Unmarshal(m, b) } func (m *FeatureGroupSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureGroupSpec.Marshal(b, m, deterministic) } -func (dst *FeatureGroupSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureGroupSpec.Merge(dst, src) +func (m *FeatureGroupSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureGroupSpec.Merge(m, src) } func (m *FeatureGroupSpec) XXX_Size() int { return xxx_messageInfo_FeatureGroupSpec.Size(m) @@ -76,11 +79,9 @@ func init() { proto.RegisterType((*FeatureGroupSpec)(nil), "feast.specs.FeatureGroupSpec") } -func init() { - proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_FeatureGroupSpec_21c67ee01edd412c) -} +func init() { proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_df3b6a9e736b5719) } -var fileDescriptor_FeatureGroupSpec_21c67ee01edd412c = []byte{ +var fileDescriptor_df3b6a9e736b5719 = []byte{ // 203 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4a, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0x75, diff --git a/protos/generated/go/feast/specs/FeatureSpec.pb.go b/protos/generated/go/feast/specs/FeatureSpec.pb.go index 34ca43cc6bc..a8cbb886a8c 100644 --- a/protos/generated/go/feast/specs/FeatureSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureSpec.pb.go @@ -1,12 +1,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/specs/FeatureSpec.proto -package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" +package specs -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import types "github.com/gojek/feast/protos/generated/go/feast/types" +import ( + fmt "fmt" + types "github.com/gojek/feast/protos/generated/go/feast/types" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type FeatureSpec struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` @@ -40,16 +42,17 @@ func (m *FeatureSpec) Reset() { *m = FeatureSpec{} } func (m *FeatureSpec) String() string { return proto.CompactTextString(m) } func (*FeatureSpec) ProtoMessage() {} func (*FeatureSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_de24bbc3c8f2bed6, []int{0} + return fileDescriptor_b8f1468f11147fbe, []int{0} } + func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSpec.Unmarshal(m, b) } func (m *FeatureSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSpec.Marshal(b, m, deterministic) } -func (dst *FeatureSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSpec.Merge(dst, src) +func (m *FeatureSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSpec.Merge(m, src) } func (m *FeatureSpec) XXX_Size() int { return xxx_messageInfo_FeatureSpec.Size(m) @@ -149,16 +152,17 @@ func (m *DataStores) Reset() { *m = DataStores{} } func (m *DataStores) String() string { return proto.CompactTextString(m) } func (*DataStores) ProtoMessage() {} func (*DataStores) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_de24bbc3c8f2bed6, []int{1} + return fileDescriptor_b8f1468f11147fbe, []int{1} } + func (m *DataStores) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStores.Unmarshal(m, b) } func (m *DataStores) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStores.Marshal(b, m, deterministic) } -func (dst *DataStores) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStores.Merge(dst, src) +func (m *DataStores) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStores.Merge(m, src) } func (m *DataStores) XXX_Size() int { return xxx_messageInfo_DataStores.Size(m) @@ -195,16 +199,17 @@ func (m *DataStore) Reset() { *m = DataStore{} } func (m *DataStore) String() string { return proto.CompactTextString(m) } func (*DataStore) ProtoMessage() {} func (*DataStore) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_de24bbc3c8f2bed6, []int{2} + return fileDescriptor_b8f1468f11147fbe, []int{2} } + func (m *DataStore) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStore.Unmarshal(m, b) } func (m *DataStore) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStore.Marshal(b, m, deterministic) } -func (dst *DataStore) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStore.Merge(dst, src) +func (m *DataStore) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStore.Merge(m, src) } func (m *DataStore) XXX_Size() int { return xxx_messageInfo_DataStore.Size(m) @@ -237,11 +242,9 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.DataStore.OptionsEntry") } -func init() { - proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_FeatureSpec_de24bbc3c8f2bed6) -} +func init() { proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_b8f1468f11147fbe) } -var fileDescriptor_FeatureSpec_de24bbc3c8f2bed6 = []byte{ +var fileDescriptor_b8f1468f11147fbe = []byte{ // 451 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0xdf, 0x6b, 0xd4, 0x40, 0x10, 0x26, 0x49, 0xdb, 0x33, 0x93, 0x52, 0xca, 0x22, 0xed, 0x72, 0x2a, 0x84, 0x13, 0xe1, 0x9e, diff --git a/protos/generated/go/feast/specs/ImportJobSpecs.pb.go b/protos/generated/go/feast/specs/ImportJobSpecs.pb.go index d4c940e947b..924175e7bd8 100644 --- a/protos/generated/go/feast/specs/ImportJobSpecs.pb.go +++ b/protos/generated/go/feast/specs/ImportJobSpecs.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/specs/ImportJobSpecs.proto -package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" +package specs -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type ImportJobSpecs struct { JobId string `protobuf:"bytes,1,opt,name=jobId,proto3" json:"jobId,omitempty"` @@ -35,16 +37,17 @@ func (m *ImportJobSpecs) Reset() { *m = ImportJobSpecs{} } func (m *ImportJobSpecs) String() string { return proto.CompactTextString(m) } func (*ImportJobSpecs) ProtoMessage() {} func (*ImportJobSpecs) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportJobSpecs_c867f2e818ce78e5, []int{0} + return fileDescriptor_ca149f837727a759, []int{0} } + func (m *ImportJobSpecs) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportJobSpecs.Unmarshal(m, b) } func (m *ImportJobSpecs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ImportJobSpecs.Marshal(b, m, deterministic) } -func (dst *ImportJobSpecs) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportJobSpecs.Merge(dst, src) +func (m *ImportJobSpecs) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportJobSpecs.Merge(m, src) } func (m *ImportJobSpecs) XXX_Size() int { return xxx_messageInfo_ImportJobSpecs.Size(m) @@ -108,11 +111,9 @@ func init() { proto.RegisterType((*ImportJobSpecs)(nil), "feast.specs.ImportJobSpecs") } -func init() { - proto.RegisterFile("feast/specs/ImportJobSpecs.proto", fileDescriptor_ImportJobSpecs_c867f2e818ce78e5) -} +func init() { proto.RegisterFile("feast/specs/ImportJobSpecs.proto", fileDescriptor_ca149f837727a759) } -var fileDescriptor_ImportJobSpecs_c867f2e818ce78e5 = []byte{ +var fileDescriptor_ca149f837727a759 = []byte{ // 306 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0x41, 0x4b, 0xc3, 0x40, 0x10, 0x85, 0xa9, 0xb5, 0x15, 0xb7, 0x22, 0xb8, 0x55, 0x5c, 0x44, 0x21, 0x78, 0xea, 0x29, 0x0b, diff --git a/protos/generated/go/feast/specs/ImportSpec.pb.go b/protos/generated/go/feast/specs/ImportSpec.pb.go index d16f9fa07bd..d7788c13728 100644 --- a/protos/generated/go/feast/specs/ImportSpec.pb.go +++ b/protos/generated/go/feast/specs/ImportSpec.pb.go @@ -1,12 +1,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/specs/ImportSpec.proto -package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" +package specs -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type ImportSpec struct { Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` @@ -34,16 +36,17 @@ func (m *ImportSpec) Reset() { *m = ImportSpec{} } func (m *ImportSpec) String() string { return proto.CompactTextString(m) } func (*ImportSpec) ProtoMessage() {} func (*ImportSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{0} + return fileDescriptor_bf21bfd2215b3e0c, []int{0} } + func (m *ImportSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportSpec.Unmarshal(m, b) } func (m *ImportSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ImportSpec.Marshal(b, m, deterministic) } -func (dst *ImportSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportSpec.Merge(dst, src) +func (m *ImportSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportSpec.Merge(m, src) } func (m *ImportSpec) XXX_Size() int { return xxx_messageInfo_ImportSpec.Size(m) @@ -107,16 +110,17 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{1} + return fileDescriptor_bf21bfd2215b3e0c, []int{1} } + func (m *Schema) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Schema.Unmarshal(m, b) } func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Schema.Marshal(b, m, deterministic) } -func (dst *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(dst, src) +func (m *Schema) XXX_Merge(src proto.Message) { + xxx_messageInfo_Schema.Merge(m, src) } func (m *Schema) XXX_Size() int { return xxx_messageInfo_Schema.Size(m) @@ -178,76 +182,14 @@ func (m *Schema) GetEntityIdColumn() string { return "" } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Schema) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Schema_OneofMarshaler, _Schema_OneofUnmarshaler, _Schema_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*Schema) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*Schema_TimestampColumn)(nil), (*Schema_TimestampValue)(nil), } } -func _Schema_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Schema) - // timestamp - switch x := m.Timestamp.(type) { - case *Schema_TimestampColumn: - b.EncodeVarint(5<<3 | proto.WireBytes) - b.EncodeStringBytes(x.TimestampColumn) - case *Schema_TimestampValue: - b.EncodeVarint(6<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.TimestampValue); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("Schema.Timestamp has unexpected type %T", x) - } - return nil -} - -func _Schema_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Schema) - switch tag { - case 5: // timestamp.timestampColumn - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Timestamp = &Schema_TimestampColumn{x} - return true, err - case 6: // timestamp.timestampValue - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(timestamp.Timestamp) - err := b.DecodeMessage(msg) - m.Timestamp = &Schema_TimestampValue{msg} - return true, err - default: - return false, nil - } -} - -func _Schema_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Schema) - // timestamp - switch x := m.Timestamp.(type) { - case *Schema_TimestampColumn: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.TimestampColumn))) - n += len(x.TimestampColumn) - case *Schema_TimestampValue: - s := proto.Size(x.TimestampValue) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type Field struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` FeatureId string `protobuf:"bytes,2,opt,name=featureId,proto3" json:"featureId,omitempty"` @@ -260,16 +202,17 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{2} + return fileDescriptor_bf21bfd2215b3e0c, []int{2} } + func (m *Field) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Field.Unmarshal(m, b) } func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Field.Marshal(b, m, deterministic) } -func (dst *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(dst, src) +func (m *Field) XXX_Merge(src proto.Message) { + xxx_messageInfo_Field.Merge(m, src) } func (m *Field) XXX_Size() int { return xxx_messageInfo_Field.Size(m) @@ -302,11 +245,9 @@ func init() { proto.RegisterType((*Field)(nil), "feast.specs.Field") } -func init() { - proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_673bc4f248a91137) -} +func init() { proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_bf21bfd2215b3e0c) } -var fileDescriptor_ImportSpec_673bc4f248a91137 = []byte{ +var fileDescriptor_bf21bfd2215b3e0c = []byte{ // 440 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8f, 0x93, 0x40, 0x14, 0x5d, 0xca, 0x16, 0xed, 0x6d, 0xdc, 0x9a, 0xab, 0x0f, 0x84, 0x6c, 0x62, 0xd3, 0x07, 0x6d, diff --git a/protos/generated/go/feast/specs/StorageSpec.pb.go b/protos/generated/go/feast/specs/StorageSpec.pb.go index fefe56a1be5..180fcf14c5d 100644 --- a/protos/generated/go/feast/specs/StorageSpec.pb.go +++ b/protos/generated/go/feast/specs/StorageSpec.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/specs/StorageSpec.proto -package specs // import "github.com/gojek/feast/protos/generated/go/feast/specs" +package specs -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type StorageSpec struct { // unique identifier for this instance @@ -36,16 +38,17 @@ func (m *StorageSpec) Reset() { *m = StorageSpec{} } func (m *StorageSpec) String() string { return proto.CompactTextString(m) } func (*StorageSpec) ProtoMessage() {} func (*StorageSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_StorageSpec_bfb8a5e5cf34de95, []int{0} + return fileDescriptor_7783b72a0d689614, []int{0} } + func (m *StorageSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StorageSpec.Unmarshal(m, b) } func (m *StorageSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StorageSpec.Marshal(b, m, deterministic) } -func (dst *StorageSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_StorageSpec.Merge(dst, src) +func (m *StorageSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_StorageSpec.Merge(m, src) } func (m *StorageSpec) XXX_Size() int { return xxx_messageInfo_StorageSpec.Size(m) @@ -82,11 +85,9 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.StorageSpec.OptionsEntry") } -func init() { - proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_StorageSpec_bfb8a5e5cf34de95) -} +func init() { proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_7783b72a0d689614) } -var fileDescriptor_StorageSpec_bfb8a5e5cf34de95 = []byte{ +var fileDescriptor_7783b72a0d689614 = []byte{ // 227 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x0f, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x0d, diff --git a/protos/generated/go/feast/storage/BigTable.pb.go b/protos/generated/go/feast/storage/BigTable.pb.go index e65b5e35fce..7ad1e0d5b92 100644 --- a/protos/generated/go/feast/storage/BigTable.pb.go +++ b/protos/generated/go/feast/storage/BigTable.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/storage/BigTable.proto -package storage // import "github.com/gojek/feast/protos/generated/go/feast/storage" +package storage -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,13 +18,12 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type BigTableRowKey struct { // This should be the first 7 characters of a sha1 of the entityKey proto encoded - Sha1Prefix string `protobuf:"bytes,1,opt,name=sha1Prefix,proto3" json:"sha1Prefix,omitempty"` - EntityKey string `protobuf:"bytes,2,opt,name=entityKey,proto3" json:"entityKey,omitempty"` - // This should be String.valueOf(int64.MAX_VALUE - roundToEntityGranularity(eventTimestamp)) + Sha1Prefix string `protobuf:"bytes,1,opt,name=sha1Prefix,proto3" json:"sha1Prefix,omitempty"` + EntityKey string `protobuf:"bytes,2,opt,name=entityKey,proto3" json:"entityKey,omitempty"` ReversedMillis string `protobuf:"bytes,3,opt,name=reversedMillis,proto3" json:"reversedMillis,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -33,16 +34,17 @@ func (m *BigTableRowKey) Reset() { *m = BigTableRowKey{} } func (m *BigTableRowKey) String() string { return proto.CompactTextString(m) } func (*BigTableRowKey) ProtoMessage() {} func (*BigTableRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_BigTable_e25bdeffe0669ae7, []int{0} + return fileDescriptor_1d33ec3bd45c712c, []int{0} } + func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BigTableRowKey.Unmarshal(m, b) } func (m *BigTableRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BigTableRowKey.Marshal(b, m, deterministic) } -func (dst *BigTableRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_BigTableRowKey.Merge(dst, src) +func (m *BigTableRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_BigTableRowKey.Merge(m, src) } func (m *BigTableRowKey) XXX_Size() int { return xxx_messageInfo_BigTableRowKey.Size(m) @@ -78,11 +80,9 @@ func init() { proto.RegisterType((*BigTableRowKey)(nil), "feast.storage.BigTableRowKey") } -func init() { - proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_e25bdeffe0669ae7) -} +func init() { proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_1d33ec3bd45c712c) } -var fileDescriptor_BigTable_e25bdeffe0669ae7 = []byte{ +var fileDescriptor_1d33ec3bd45c712c = []byte{ // 193 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x8f, 0xb1, 0x0b, 0x82, 0x40, 0x18, 0x47, 0xb1, 0x20, 0xf0, 0x40, 0x87, 0x9b, 0x1c, 0x24, 0xa2, 0x21, 0x9a, 0x3c, 0xa2, 0xa5, diff --git a/protos/generated/go/feast/storage/Redis.pb.go b/protos/generated/go/feast/storage/Redis.pb.go index 49f80a061e3..4d42fa2fcca 100644 --- a/protos/generated/go/feast/storage/Redis.pb.go +++ b/protos/generated/go/feast/storage/Redis.pb.go @@ -1,13 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/storage/Redis.proto -package storage // import "github.com/gojek/feast/protos/generated/go/feast/storage" +package storage -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import types "github.com/gojek/feast/protos/generated/go/feast/types" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +import ( + fmt "fmt" + types "github.com/gojek/feast/protos/generated/go/feast/types" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -18,16 +20,16 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type RedisBucketKey struct { // Entity key from the FeatureRow EntityKey string `protobuf:"bytes,2,opt,name=entityKey,proto3" json:"entityKey,omitempty"` - // * + //* // This should be the first 7 characters of a sha1 of the featureId // This is just to save storage space as it's kept in memory. FeatureIdSha1Prefix string `protobuf:"bytes,3,opt,name=featureIdSha1Prefix,proto3" json:"featureIdSha1Prefix,omitempty"` - // * + //* // This groups a feature's values (for different eventTimestamps), // into buckets so many can be retrieved together. // @@ -43,16 +45,17 @@ func (m *RedisBucketKey) Reset() { *m = RedisBucketKey{} } func (m *RedisBucketKey) String() string { return proto.CompactTextString(m) } func (*RedisBucketKey) ProtoMessage() {} func (*RedisBucketKey) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{0} + return fileDescriptor_64e898a359fc9e5d, []int{0} } + func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketKey.Unmarshal(m, b) } func (m *RedisBucketKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketKey.Marshal(b, m, deterministic) } -func (dst *RedisBucketKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketKey.Merge(dst, src) +func (m *RedisBucketKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketKey.Merge(m, src) } func (m *RedisBucketKey) XXX_Size() int { return xxx_messageInfo_RedisBucketKey.Size(m) @@ -84,7 +87,7 @@ func (m *RedisBucketKey) GetBucketId() uint64 { return 0 } -// * +//* // Because in redis features are stored as a key per feature not per // feature row, we need the event timestamp in the value. type RedisBucketValue struct { @@ -99,16 +102,17 @@ func (m *RedisBucketValue) Reset() { *m = RedisBucketValue{} } func (m *RedisBucketValue) String() string { return proto.CompactTextString(m) } func (*RedisBucketValue) ProtoMessage() {} func (*RedisBucketValue) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{1} + return fileDescriptor_64e898a359fc9e5d, []int{1} } + func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValue.Unmarshal(m, b) } func (m *RedisBucketValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValue.Marshal(b, m, deterministic) } -func (dst *RedisBucketValue) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValue.Merge(dst, src) +func (m *RedisBucketValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValue.Merge(m, src) } func (m *RedisBucketValue) XXX_Size() int { return xxx_messageInfo_RedisBucketValue.Size(m) @@ -133,7 +137,7 @@ func (m *RedisBucketValue) GetEventTimestamp() *timestamp.Timestamp { return nil } -// * +//* // This allows us to group multiple bucket values together in a // single list to make it easier to keep sets together type RedisBucketValueList struct { @@ -147,16 +151,17 @@ func (m *RedisBucketValueList) Reset() { *m = RedisBucketValueList{} } func (m *RedisBucketValueList) String() string { return proto.CompactTextString(m) } func (*RedisBucketValueList) ProtoMessage() {} func (*RedisBucketValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{2} + return fileDescriptor_64e898a359fc9e5d, []int{2} } + func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValueList.Unmarshal(m, b) } func (m *RedisBucketValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValueList.Marshal(b, m, deterministic) } -func (dst *RedisBucketValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValueList.Merge(dst, src) +func (m *RedisBucketValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValueList.Merge(m, src) } func (m *RedisBucketValueList) XXX_Size() int { return xxx_messageInfo_RedisBucketValueList.Size(m) @@ -180,9 +185,9 @@ func init() { proto.RegisterType((*RedisBucketValueList)(nil), "feast.storage.RedisBucketValueList") } -func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_Redis_cef62c817c1622ce) } +func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_64e898a359fc9e5d) } -var fileDescriptor_Redis_cef62c817c1622ce = []byte{ +var fileDescriptor_64e898a359fc9e5d = []byte{ // 325 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcd, 0x4f, 0xf2, 0x40, 0x10, 0xc6, 0xd3, 0x97, 0x57, 0x22, 0x4b, 0x24, 0x66, 0x35, 0xb1, 0x36, 0x26, 0x34, 0x9c, 0x7a, diff --git a/protos/generated/go/feast/types/Feature.pb.go b/protos/generated/go/feast/types/Feature.pb.go index 51ed41fb54f..0ef6e65d556 100644 --- a/protos/generated/go/feast/types/Feature.pb.go +++ b/protos/generated/go/feast/types/Feature.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/types/Feature.proto -package types // import "github.com/gojek/feast/protos/generated/go/feast/types" +package types -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Feature struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` @@ -30,16 +32,17 @@ func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_Feature_c2a5d99d9bf3ca9c, []int{0} + return fileDescriptor_4e19474999533fc9, []int{0} } + func (m *Feature) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Feature.Unmarshal(m, b) } func (m *Feature) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Feature.Marshal(b, m, deterministic) } -func (dst *Feature) XXX_Merge(src proto.Message) { - xxx_messageInfo_Feature.Merge(dst, src) +func (m *Feature) XXX_Merge(src proto.Message) { + xxx_messageInfo_Feature.Merge(m, src) } func (m *Feature) XXX_Size() int { return xxx_messageInfo_Feature.Size(m) @@ -68,9 +71,9 @@ func init() { proto.RegisterType((*Feature)(nil), "feast.types.Feature") } -func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_c2a5d99d9bf3ca9c) } +func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_4e19474999533fc9) } -var fileDescriptor_Feature_c2a5d99d9bf3ca9c = []byte{ +var fileDescriptor_4e19474999533fc9 = []byte{ // 173 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0xd5, diff --git a/protos/generated/go/feast/types/FeatureRow.pb.go b/protos/generated/go/feast/types/FeatureRow.pb.go index dc6099ba754..9d482da4a4e 100644 --- a/protos/generated/go/feast/types/FeatureRow.pb.go +++ b/protos/generated/go/feast/types/FeatureRow.pb.go @@ -1,12 +1,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/types/FeatureRow.proto -package types // import "github.com/gojek/feast/protos/generated/go/feast/types" +package types -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type FeatureRowKey struct { EntityKey string `protobuf:"bytes,1,opt,name=entityKey,proto3" json:"entityKey,omitempty"` @@ -32,16 +34,17 @@ func (m *FeatureRowKey) Reset() { *m = FeatureRowKey{} } func (m *FeatureRowKey) String() string { return proto.CompactTextString(m) } func (*FeatureRowKey) ProtoMessage() {} func (*FeatureRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_3072498459678ede, []int{0} + return fileDescriptor_fbbea9c89787d1c7, []int{0} } + func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowKey.Unmarshal(m, b) } func (m *FeatureRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowKey.Marshal(b, m, deterministic) } -func (dst *FeatureRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowKey.Merge(dst, src) +func (m *FeatureRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowKey.Merge(m, src) } func (m *FeatureRowKey) XXX_Size() int { return xxx_messageInfo_FeatureRowKey.Size(m) @@ -87,16 +90,17 @@ func (m *FeatureRow) Reset() { *m = FeatureRow{} } func (m *FeatureRow) String() string { return proto.CompactTextString(m) } func (*FeatureRow) ProtoMessage() {} func (*FeatureRow) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_3072498459678ede, []int{1} + return fileDescriptor_fbbea9c89787d1c7, []int{1} } + func (m *FeatureRow) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRow.Unmarshal(m, b) } func (m *FeatureRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRow.Marshal(b, m, deterministic) } -func (dst *FeatureRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRow.Merge(dst, src) +func (m *FeatureRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRow.Merge(m, src) } func (m *FeatureRow) XXX_Size() int { return xxx_messageInfo_FeatureRow.Size(m) @@ -140,11 +144,9 @@ func init() { proto.RegisterType((*FeatureRow)(nil), "feast.types.FeatureRow") } -func init() { - proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_3072498459678ede) -} +func init() { proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_fbbea9c89787d1c7) } -var fileDescriptor_FeatureRow_3072498459678ede = []byte{ +var fileDescriptor_fbbea9c89787d1c7 = []byte{ // 266 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x91, 0xc1, 0x4a, 0xc4, 0x30, 0x10, 0x86, 0x89, 0x2b, 0xe2, 0xce, 0xa2, 0x42, 0xf1, 0x50, 0xcb, 0xa2, 0x65, 0x4f, 0x3d, 0x65, diff --git a/protos/generated/go/feast/types/FeatureRowExtended.pb.go b/protos/generated/go/feast/types/FeatureRowExtended.pb.go index ad89f27db23..e8371fcfcd6 100644 --- a/protos/generated/go/feast/types/FeatureRowExtended.pb.go +++ b/protos/generated/go/feast/types/FeatureRowExtended.pb.go @@ -1,12 +1,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/types/FeatureRowExtended.proto -package types // import "github.com/gojek/feast/protos/generated/go/feast/types" +package types -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Error struct { Cause string `protobuf:"bytes,1,opt,name=cause,proto3" json:"cause,omitempty"` @@ -33,16 +35,17 @@ func (m *Error) Reset() { *m = Error{} } func (m *Error) String() string { return proto.CompactTextString(m) } func (*Error) ProtoMessage() {} func (*Error) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{0} + return fileDescriptor_7823aa2c72575793, []int{0} } + func (m *Error) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Error.Unmarshal(m, b) } func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Error.Marshal(b, m, deterministic) } -func (dst *Error) XXX_Merge(src proto.Message) { - xxx_messageInfo_Error.Merge(dst, src) +func (m *Error) XXX_Merge(src proto.Message) { + xxx_messageInfo_Error.Merge(m, src) } func (m *Error) XXX_Size() int { return xxx_messageInfo_Error.Size(m) @@ -93,16 +96,17 @@ func (m *Attempt) Reset() { *m = Attempt{} } func (m *Attempt) String() string { return proto.CompactTextString(m) } func (*Attempt) ProtoMessage() {} func (*Attempt) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{1} + return fileDescriptor_7823aa2c72575793, []int{1} } + func (m *Attempt) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attempt.Unmarshal(m, b) } func (m *Attempt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Attempt.Marshal(b, m, deterministic) } -func (dst *Attempt) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attempt.Merge(dst, src) +func (m *Attempt) XXX_Merge(src proto.Message) { + xxx_messageInfo_Attempt.Merge(m, src) } func (m *Attempt) XXX_Size() int { return xxx_messageInfo_Attempt.Size(m) @@ -140,16 +144,17 @@ func (m *FeatureRowExtended) Reset() { *m = FeatureRowExtended{} } func (m *FeatureRowExtended) String() string { return proto.CompactTextString(m) } func (*FeatureRowExtended) ProtoMessage() {} func (*FeatureRowExtended) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{2} + return fileDescriptor_7823aa2c72575793, []int{2} } + func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowExtended.Unmarshal(m, b) } func (m *FeatureRowExtended) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowExtended.Marshal(b, m, deterministic) } -func (dst *FeatureRowExtended) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowExtended.Merge(dst, src) +func (m *FeatureRowExtended) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowExtended.Merge(m, src) } func (m *FeatureRowExtended) XXX_Size() int { return xxx_messageInfo_FeatureRowExtended.Size(m) @@ -188,10 +193,10 @@ func init() { } func init() { - proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_FeatureRowExtended_bfd3c37956d1a040) + proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_7823aa2c72575793) } -var fileDescriptor_FeatureRowExtended_bfd3c37956d1a040 = []byte{ +var fileDescriptor_7823aa2c72575793 = []byte{ // 338 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x6b, 0xea, 0x40, 0x10, 0xc6, 0xf1, 0xf9, 0xf2, 0x7c, 0x4e, 0x6e, 0x8b, 0x60, 0x08, 0xd2, 0x16, 0xe9, 0xc1, 0x5e, diff --git a/protos/generated/go/feast/types/Granularity.pb.go b/protos/generated/go/feast/types/Granularity.pb.go deleted file mode 100644 index 5166f7bbc2e..00000000000 --- a/protos/generated/go/feast/types/Granularity.pb.go +++ /dev/null @@ -1,106 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: feast/types/Granularity.proto - -package types // import "github.com/gojek/feast/protos/generated/go/feast/types" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -type Granularity_Enum int32 - -const ( - Granularity_NONE Granularity_Enum = 0 - Granularity_DAY Granularity_Enum = 1 - Granularity_HOUR Granularity_Enum = 2 - Granularity_MINUTE Granularity_Enum = 3 - Granularity_SECOND Granularity_Enum = 4 -) - -var Granularity_Enum_name = map[int32]string{ - 0: "NONE", - 1: "DAY", - 2: "HOUR", - 3: "MINUTE", - 4: "SECOND", -} -var Granularity_Enum_value = map[string]int32{ - "NONE": 0, - "DAY": 1, - "HOUR": 2, - "MINUTE": 3, - "SECOND": 4, -} - -func (x Granularity_Enum) String() string { - return proto.EnumName(Granularity_Enum_name, int32(x)) -} -func (Granularity_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0, 0} -} - -type Granularity struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Granularity) Reset() { *m = Granularity{} } -func (m *Granularity) String() string { return proto.CompactTextString(m) } -func (*Granularity) ProtoMessage() {} -func (*Granularity) Descriptor() ([]byte, []int) { - return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0} -} -func (m *Granularity) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Granularity.Unmarshal(m, b) -} -func (m *Granularity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Granularity.Marshal(b, m, deterministic) -} -func (dst *Granularity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Granularity.Merge(dst, src) -} -func (m *Granularity) XXX_Size() int { - return xxx_messageInfo_Granularity.Size(m) -} -func (m *Granularity) XXX_DiscardUnknown() { - xxx_messageInfo_Granularity.DiscardUnknown(m) -} - -var xxx_messageInfo_Granularity proto.InternalMessageInfo - -func init() { - proto.RegisterType((*Granularity)(nil), "feast.types.Granularity") - proto.RegisterEnum("feast.types.Granularity_Enum", Granularity_Enum_name, Granularity_Enum_value) -} - -func init() { - proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_Granularity_4b57756f7a751fdb) -} - -var fileDescriptor_Granularity_4b57756f7a751fdb = []byte{ - // 183 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, - 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x2f, 0x4a, 0xcc, 0x2b, 0xcd, 0x49, 0x2c, - 0xca, 0x2c, 0xa9, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x06, 0x4b, 0xeb, 0x81, 0xa5, - 0x95, 0xbc, 0xb8, 0xb8, 0x91, 0x54, 0x28, 0x59, 0x73, 0xb1, 0xb8, 0xe6, 0x95, 0xe6, 0x0a, 0x71, - 0x70, 0xb1, 0xf8, 0xf9, 0xfb, 0xb9, 0x0a, 0x30, 0x08, 0xb1, 0x73, 0x31, 0xbb, 0x38, 0x46, 0x0a, - 0x30, 0x82, 0x84, 0x3c, 0xfc, 0x43, 0x83, 0x04, 0x98, 0x84, 0xb8, 0xb8, 0xd8, 0x7c, 0x3d, 0xfd, - 0x42, 0x43, 0x5c, 0x05, 0x98, 0x41, 0xec, 0x60, 0x57, 0x67, 0x7f, 0x3f, 0x17, 0x01, 0x16, 0xa7, - 0x70, 0x2e, 0x64, 0xa3, 0x9d, 0x04, 0x90, 0x0c, 0x0e, 0x00, 0xd9, 0x1c, 0x65, 0x96, 0x9e, 0x59, - 0x92, 0x51, 0x9a, 0xa4, 0x97, 0x9c, 0x9f, 0xab, 0x9f, 0x9e, 0x9f, 0x95, 0x9a, 0xad, 0x0f, 0x71, - 0x29, 0xd8, 0x5d, 0xc5, 0xfa, 0xe9, 0xa9, 0x79, 0xa9, 0x45, 0x89, 0x25, 0xa9, 0x29, 0xfa, 0xe9, - 0xf9, 0xfa, 0x48, 0x7e, 0x48, 0x62, 0x03, 0x2b, 0x30, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xe9, - 0x56, 0x0c, 0xf2, 0xd9, 0x00, 0x00, 0x00, -} diff --git a/protos/generated/go/feast/types/Value.pb.go b/protos/generated/go/feast/types/Value.pb.go index b9c26f8c4b8..a7ef038e1d6 100644 --- a/protos/generated/go/feast/types/Value.pb.go +++ b/protos/generated/go/feast/types/Value.pb.go @@ -1,12 +1,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: feast/types/Value.proto -package types // import "github.com/gojek/feast/protos/generated/go/feast/types" +package types -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type ValueType_Enum int32 @@ -44,6 +46,7 @@ var ValueType_Enum_name = map[int32]string{ 7: "BOOL", 8: "TIMESTAMP", } + var ValueType_Enum_value = map[string]int32{ "UNKNOWN": 0, "BYTES": 1, @@ -59,8 +62,9 @@ var ValueType_Enum_value = map[string]int32{ func (x ValueType_Enum) String() string { return proto.EnumName(ValueType_Enum_name, int32(x)) } + func (ValueType_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{0, 0} + return fileDescriptor_47c504407d284ecc, []int{0, 0} } type ValueType struct { @@ -73,16 +77,17 @@ func (m *ValueType) Reset() { *m = ValueType{} } func (m *ValueType) String() string { return proto.CompactTextString(m) } func (*ValueType) ProtoMessage() {} func (*ValueType) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{0} + return fileDescriptor_47c504407d284ecc, []int{0} } + func (m *ValueType) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueType.Unmarshal(m, b) } func (m *ValueType) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueType.Marshal(b, m, deterministic) } -func (dst *ValueType) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueType.Merge(dst, src) +func (m *ValueType) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueType.Merge(m, src) } func (m *ValueType) XXX_Size() int { return xxx_messageInfo_ValueType.Size(m) @@ -113,16 +118,17 @@ func (m *Value) Reset() { *m = Value{} } func (m *Value) String() string { return proto.CompactTextString(m) } func (*Value) ProtoMessage() {} func (*Value) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{1} + return fileDescriptor_47c504407d284ecc, []int{1} } + func (m *Value) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Value.Unmarshal(m, b) } func (m *Value) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Value.Marshal(b, m, deterministic) } -func (dst *Value) XXX_Merge(src proto.Message) { - xxx_messageInfo_Value.Merge(dst, src) +func (m *Value) XXX_Merge(src proto.Message) { + xxx_messageInfo_Value.Merge(m, src) } func (m *Value) XXX_Size() int { return xxx_messageInfo_Value.Size(m) @@ -248,9 +254,9 @@ func (m *Value) GetTimestampVal() *timestamp.Timestamp { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Value) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Value_OneofMarshaler, _Value_OneofUnmarshaler, _Value_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*Value) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*Value_BytesVal)(nil), (*Value_StringVal)(nil), (*Value_Int32Val)(nil), @@ -262,151 +268,6 @@ func (*Value) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, } } -func _Value_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Value) - // val - switch x := m.Val.(type) { - case *Value_BytesVal: - b.EncodeVarint(1<<3 | proto.WireBytes) - b.EncodeRawBytes(x.BytesVal) - case *Value_StringVal: - b.EncodeVarint(2<<3 | proto.WireBytes) - b.EncodeStringBytes(x.StringVal) - case *Value_Int32Val: - b.EncodeVarint(3<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Int32Val)) - case *Value_Int64Val: - b.EncodeVarint(4<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Int64Val)) - case *Value_DoubleVal: - b.EncodeVarint(5<<3 | proto.WireFixed64) - b.EncodeFixed64(math.Float64bits(x.DoubleVal)) - case *Value_FloatVal: - b.EncodeVarint(6<<3 | proto.WireFixed32) - b.EncodeFixed32(uint64(math.Float32bits(x.FloatVal))) - case *Value_BoolVal: - t := uint64(0) - if x.BoolVal { - t = 1 - } - b.EncodeVarint(7<<3 | proto.WireVarint) - b.EncodeVarint(t) - case *Value_TimestampVal: - b.EncodeVarint(8<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.TimestampVal); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("Value.Val has unexpected type %T", x) - } - return nil -} - -func _Value_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Value) - switch tag { - case 1: // val.bytesVal - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeRawBytes(true) - m.Val = &Value_BytesVal{x} - return true, err - case 2: // val.stringVal - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Val = &Value_StringVal{x} - return true, err - case 3: // val.int32Val - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Val = &Value_Int32Val{int32(x)} - return true, err - case 4: // val.int64Val - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Val = &Value_Int64Val{int64(x)} - return true, err - case 5: // val.doubleVal - if wire != proto.WireFixed64 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed64() - m.Val = &Value_DoubleVal{math.Float64frombits(x)} - return true, err - case 6: // val.floatVal - if wire != proto.WireFixed32 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed32() - m.Val = &Value_FloatVal{math.Float32frombits(uint32(x))} - return true, err - case 7: // val.boolVal - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Val = &Value_BoolVal{x != 0} - return true, err - case 8: // val.timestampVal - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(timestamp.Timestamp) - err := b.DecodeMessage(msg) - m.Val = &Value_TimestampVal{msg} - return true, err - default: - return false, nil - } -} - -func _Value_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Value) - // val - switch x := m.Val.(type) { - case *Value_BytesVal: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.BytesVal))) - n += len(x.BytesVal) - case *Value_StringVal: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.StringVal))) - n += len(x.StringVal) - case *Value_Int32Val: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(x.Int32Val)) - case *Value_Int64Val: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(x.Int64Val)) - case *Value_DoubleVal: - n += 1 // tag and wire - n += 8 - case *Value_FloatVal: - n += 1 // tag and wire - n += 4 - case *Value_BoolVal: - n += 1 // tag and wire - n += 1 - case *Value_TimestampVal: - s := proto.Size(x.TimestampVal) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type ValueList struct { // Types that are valid to be assigned to ValueList: // *ValueList_BytesList @@ -427,16 +288,17 @@ func (m *ValueList) Reset() { *m = ValueList{} } func (m *ValueList) String() string { return proto.CompactTextString(m) } func (*ValueList) ProtoMessage() {} func (*ValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{2} + return fileDescriptor_47c504407d284ecc, []int{2} } + func (m *ValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueList.Unmarshal(m, b) } func (m *ValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueList.Marshal(b, m, deterministic) } -func (dst *ValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueList.Merge(dst, src) +func (m *ValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueList.Merge(m, src) } func (m *ValueList) XXX_Size() int { return xxx_messageInfo_ValueList.Size(m) @@ -562,9 +424,9 @@ func (m *ValueList) GetTimestampList() *TimestampList { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*ValueList) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _ValueList_OneofMarshaler, _ValueList_OneofUnmarshaler, _ValueList_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*ValueList) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*ValueList_BytesList)(nil), (*ValueList_StringList)(nil), (*ValueList_Int32List)(nil), @@ -576,180 +438,6 @@ func (*ValueList) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) err } } -func _ValueList_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*ValueList) - // valueList - switch x := m.ValueList.(type) { - case *ValueList_BytesList: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.BytesList); err != nil { - return err - } - case *ValueList_StringList: - b.EncodeVarint(2<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.StringList); err != nil { - return err - } - case *ValueList_Int32List: - b.EncodeVarint(3<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Int32List); err != nil { - return err - } - case *ValueList_Int64List: - b.EncodeVarint(4<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Int64List); err != nil { - return err - } - case *ValueList_DoubleList: - b.EncodeVarint(5<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.DoubleList); err != nil { - return err - } - case *ValueList_FloatList: - b.EncodeVarint(6<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.FloatList); err != nil { - return err - } - case *ValueList_BoolList: - b.EncodeVarint(7<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.BoolList); err != nil { - return err - } - case *ValueList_TimestampList: - b.EncodeVarint(8<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.TimestampList); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("ValueList.ValueList has unexpected type %T", x) - } - return nil -} - -func _ValueList_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*ValueList) - switch tag { - case 1: // valueList.bytesList - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(BytesList) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_BytesList{msg} - return true, err - case 2: // valueList.stringList - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(StringList) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_StringList{msg} - return true, err - case 3: // valueList.int32List - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(Int32List) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_Int32List{msg} - return true, err - case 4: // valueList.int64List - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(Int64List) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_Int64List{msg} - return true, err - case 5: // valueList.doubleList - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(DoubleList) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_DoubleList{msg} - return true, err - case 6: // valueList.floatList - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(FloatList) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_FloatList{msg} - return true, err - case 7: // valueList.boolList - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(BoolList) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_BoolList{msg} - return true, err - case 8: // valueList.timestampList - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(TimestampList) - err := b.DecodeMessage(msg) - m.ValueList = &ValueList_TimestampList{msg} - return true, err - default: - return false, nil - } -} - -func _ValueList_OneofSizer(msg proto.Message) (n int) { - m := msg.(*ValueList) - // valueList - switch x := m.ValueList.(type) { - case *ValueList_BytesList: - s := proto.Size(x.BytesList) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_StringList: - s := proto.Size(x.StringList) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_Int32List: - s := proto.Size(x.Int32List) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_Int64List: - s := proto.Size(x.Int64List) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_DoubleList: - s := proto.Size(x.DoubleList) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_FloatList: - s := proto.Size(x.FloatList) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_BoolList: - s := proto.Size(x.BoolList) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *ValueList_TimestampList: - s := proto.Size(x.TimestampList) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type BytesList struct { Val [][]byte `protobuf:"bytes,1,rep,name=val,proto3" json:"val,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -761,16 +449,17 @@ func (m *BytesList) Reset() { *m = BytesList{} } func (m *BytesList) String() string { return proto.CompactTextString(m) } func (*BytesList) ProtoMessage() {} func (*BytesList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{3} + return fileDescriptor_47c504407d284ecc, []int{3} } + func (m *BytesList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BytesList.Unmarshal(m, b) } func (m *BytesList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BytesList.Marshal(b, m, deterministic) } -func (dst *BytesList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BytesList.Merge(dst, src) +func (m *BytesList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BytesList.Merge(m, src) } func (m *BytesList) XXX_Size() int { return xxx_messageInfo_BytesList.Size(m) @@ -799,16 +488,17 @@ func (m *StringList) Reset() { *m = StringList{} } func (m *StringList) String() string { return proto.CompactTextString(m) } func (*StringList) ProtoMessage() {} func (*StringList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{4} + return fileDescriptor_47c504407d284ecc, []int{4} } + func (m *StringList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StringList.Unmarshal(m, b) } func (m *StringList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StringList.Marshal(b, m, deterministic) } -func (dst *StringList) XXX_Merge(src proto.Message) { - xxx_messageInfo_StringList.Merge(dst, src) +func (m *StringList) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringList.Merge(m, src) } func (m *StringList) XXX_Size() int { return xxx_messageInfo_StringList.Size(m) @@ -837,16 +527,17 @@ func (m *Int32List) Reset() { *m = Int32List{} } func (m *Int32List) String() string { return proto.CompactTextString(m) } func (*Int32List) ProtoMessage() {} func (*Int32List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{5} + return fileDescriptor_47c504407d284ecc, []int{5} } + func (m *Int32List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int32List.Unmarshal(m, b) } func (m *Int32List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int32List.Marshal(b, m, deterministic) } -func (dst *Int32List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int32List.Merge(dst, src) +func (m *Int32List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int32List.Merge(m, src) } func (m *Int32List) XXX_Size() int { return xxx_messageInfo_Int32List.Size(m) @@ -875,16 +566,17 @@ func (m *Int64List) Reset() { *m = Int64List{} } func (m *Int64List) String() string { return proto.CompactTextString(m) } func (*Int64List) ProtoMessage() {} func (*Int64List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{6} + return fileDescriptor_47c504407d284ecc, []int{6} } + func (m *Int64List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int64List.Unmarshal(m, b) } func (m *Int64List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int64List.Marshal(b, m, deterministic) } -func (dst *Int64List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int64List.Merge(dst, src) +func (m *Int64List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int64List.Merge(m, src) } func (m *Int64List) XXX_Size() int { return xxx_messageInfo_Int64List.Size(m) @@ -913,16 +605,17 @@ func (m *DoubleList) Reset() { *m = DoubleList{} } func (m *DoubleList) String() string { return proto.CompactTextString(m) } func (*DoubleList) ProtoMessage() {} func (*DoubleList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{7} + return fileDescriptor_47c504407d284ecc, []int{7} } + func (m *DoubleList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DoubleList.Unmarshal(m, b) } func (m *DoubleList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DoubleList.Marshal(b, m, deterministic) } -func (dst *DoubleList) XXX_Merge(src proto.Message) { - xxx_messageInfo_DoubleList.Merge(dst, src) +func (m *DoubleList) XXX_Merge(src proto.Message) { + xxx_messageInfo_DoubleList.Merge(m, src) } func (m *DoubleList) XXX_Size() int { return xxx_messageInfo_DoubleList.Size(m) @@ -951,16 +644,17 @@ func (m *FloatList) Reset() { *m = FloatList{} } func (m *FloatList) String() string { return proto.CompactTextString(m) } func (*FloatList) ProtoMessage() {} func (*FloatList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{8} + return fileDescriptor_47c504407d284ecc, []int{8} } + func (m *FloatList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FloatList.Unmarshal(m, b) } func (m *FloatList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FloatList.Marshal(b, m, deterministic) } -func (dst *FloatList) XXX_Merge(src proto.Message) { - xxx_messageInfo_FloatList.Merge(dst, src) +func (m *FloatList) XXX_Merge(src proto.Message) { + xxx_messageInfo_FloatList.Merge(m, src) } func (m *FloatList) XXX_Size() int { return xxx_messageInfo_FloatList.Size(m) @@ -989,16 +683,17 @@ func (m *BoolList) Reset() { *m = BoolList{} } func (m *BoolList) String() string { return proto.CompactTextString(m) } func (*BoolList) ProtoMessage() {} func (*BoolList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{9} + return fileDescriptor_47c504407d284ecc, []int{9} } + func (m *BoolList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BoolList.Unmarshal(m, b) } func (m *BoolList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BoolList.Marshal(b, m, deterministic) } -func (dst *BoolList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BoolList.Merge(dst, src) +func (m *BoolList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BoolList.Merge(m, src) } func (m *BoolList) XXX_Size() int { return xxx_messageInfo_BoolList.Size(m) @@ -1027,16 +722,17 @@ func (m *TimestampList) Reset() { *m = TimestampList{} } func (m *TimestampList) String() string { return proto.CompactTextString(m) } func (*TimestampList) ProtoMessage() {} func (*TimestampList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{10} + return fileDescriptor_47c504407d284ecc, []int{10} } + func (m *TimestampList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TimestampList.Unmarshal(m, b) } func (m *TimestampList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TimestampList.Marshal(b, m, deterministic) } -func (dst *TimestampList) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampList.Merge(dst, src) +func (m *TimestampList) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimestampList.Merge(m, src) } func (m *TimestampList) XXX_Size() int { return xxx_messageInfo_TimestampList.Size(m) @@ -1055,6 +751,7 @@ func (m *TimestampList) GetVal() []*timestamp.Timestamp { } func init() { + proto.RegisterEnum("feast.types.ValueType_Enum", ValueType_Enum_name, ValueType_Enum_value) proto.RegisterType((*ValueType)(nil), "feast.types.ValueType") proto.RegisterType((*Value)(nil), "feast.types.Value") proto.RegisterType((*ValueList)(nil), "feast.types.ValueList") @@ -1066,12 +763,11 @@ func init() { proto.RegisterType((*FloatList)(nil), "feast.types.FloatList") proto.RegisterType((*BoolList)(nil), "feast.types.BoolList") proto.RegisterType((*TimestampList)(nil), "feast.types.TimestampList") - proto.RegisterEnum("feast.types.ValueType_Enum", ValueType_Enum_name, ValueType_Enum_value) } -func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_Value_0680a2f024df1112) } +func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_47c504407d284ecc) } -var fileDescriptor_Value_0680a2f024df1112 = []byte{ +var fileDescriptor_47c504407d284ecc = []byte{ // 626 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xd1, 0x6f, 0x9a, 0x50, 0x14, 0xc6, 0xb9, 0x22, 0x0a, 0xc7, 0x36, 0x21, 0x37, 0xd9, 0xda, 0x34, 0x6d, 0x47, 0x7c, 0xe2, diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index 59b4ddd9f6f..1d835d37b5f 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -35,6 +35,7 @@ from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader from feast.sdk.utils.print_utils import spec_to_yaml +from feast.sdk.utils import types from feast.serving.Serving_pb2 import QueryFeaturesRequest from feast.serving.Serving_pb2_grpc import ServingAPIStub @@ -77,9 +78,11 @@ def core_url(self): self._core_url = os.getenv(FEAST_CORE_URL_ENV_KEY) if self._core_url is None: raise ValueError( - "Core API URL not set. Either set the " + - "environment variable {} or set it explicitly.".format( - FEAST_CORE_URL_ENV_KEY)) + "Core API URL not set. Either set the " + + "environment variable {} or set it explicitly.".format( + FEAST_CORE_URL_ENV_KEY + ) + ) return self._core_url @core_url.setter @@ -92,9 +95,11 @@ def serving_url(self): self._serving_url = os.getenv(FEAST_SERVING_URL_ENV_KEY) if self._serving_url is None: raise ValueError( - "Serving API URL not set. Either set the " + - "environment variable {} or set it explicitly.".format( - FEAST_SERVING_URL_ENV_KEY)) + "Serving API URL not set. Either set the " + + "environment variable {} or set it explicitly.".format( + FEAST_SERVING_URL_ENV_KEY + ) + ) return self._serving_url @serving_url.setter @@ -128,11 +133,9 @@ def apply(self, obj): else: return self._apply(obj) - def run(self, - importer, - name_override=None, - apply_entity=False, - apply_features=False): + def run( + self, importer, name_override=None, apply_entity=False, apply_features=False + ): """ Run an import job Args: @@ -146,8 +149,7 @@ def run(self, Returns: (str) job ID of the import job """ - request = JobServiceTypes.SubmitImportJobRequest( - importSpec=importer.spec) + request = JobServiceTypes.SubmitImportJobRequest(importSpec=importer.spec) if name_override is not None: request.name = name_override @@ -158,22 +160,17 @@ def run(self, self._apply_feature(importer.features[feature]) if importer.require_staging: - print("Staging file to remote path {}".format( - importer.remote_path)) + print("Staging file to remote path {}".format(importer.remote_path)) importer.stage() - print("Submitting job with spec:\n {}".format( - spec_to_yaml(importer.spec))) + print("Submitting job with spec:\n {}".format(spec_to_yaml(importer.spec))) self._connect_core() response = self._job_service_stub.SubmitJob(request) print("Submitted job with id: {}".format(response.jobId)) return response.jobId - def create_dataset(self, - feature_set, - start_date, - end_date, - limit=None, - name_prefix=None): + def create_dataset( + self, feature_set, start_date, end_date, limit=None, name_prefix=None + ): """ Create training dataset for a feature set. The training dataset will be bounded by event timestamp between start_date and end_date. @@ -194,24 +191,28 @@ def create_dataset(self, feast.resources.feature_set.DatasetInfo: DatasetInfo containing the information of training dataset """ - self._check_create_dataset_args(feature_set, start_date, end_date, - limit) + self._check_create_dataset_args(feature_set, start_date, end_date, limit) req = DatasetServiceTypes.CreateDatasetRequest( featureSet=feature_set.proto, startDate=_timestamp_from_datetime(_parse_date(start_date)), endDate=_timestamp_from_datetime(_parse_date(end_date)), limit=limit, - namePrefix=name_prefix) + namePrefix=name_prefix, + ) if self.verbose: - print("creating training dataset for features: " + - str(feature_set.features)) + print( + "creating training dataset for features: " + str(feature_set.features) + ) self._connect_core() resp = self._dataset_service_stub.CreateDataset(req) if self.verbose: - print("created dataset {}: {}".format(resp.datasetInfo.name, - resp.datasetInfo.tableUrl)) + print( + "created dataset {}: {}".format( + resp.datasetInfo.name, resp.datasetInfo.tableUrl + ) + ) return DatasetInfo(resp.datasetInfo.name, resp.datasetInfo.tableUrl) def get_serving_data(self, feature_set, entity_keys, ts_range=None): @@ -245,14 +246,12 @@ def get_serving_data(self, feature_set, entity_keys, ts_range=None): request = self._build_serving_request(feature_set, entity_keys) self._connect_serving() return self._response_to_df( - feature_set, self._serving_service_stub.QueryFeatures(request), - start, end) - - def download_dataset(self, - dataset_info, - dest, - staging_location, - file_type=FileType.CSV): + feature_set, self._serving_service_stub.QueryFeatures(request), start, end + ) + + def download_dataset( + self, dataset_info, dest, staging_location, file_type=FileType.CSV + ): """ Download training dataset as file Args: @@ -267,7 +266,8 @@ def download_dataset(self, str: path to the downloaded file """ return self._table_downloader.download_table_as_file( - dataset_info.table_id, dest, staging_location, file_type) + dataset_info.table_id, dest, staging_location, file_type + ) def download_dataset_to_df(self, dataset_info, staging_location): """ @@ -282,7 +282,8 @@ def download_dataset_to_df(self, dataset_info, staging_location): """ return self._table_downloader.download_table_as_df( - dataset_info.table_id, staging_location) + dataset_info.table_id, staging_location + ) def close(self): """ @@ -299,8 +300,7 @@ def _connect_core(self): self.__core_channel = grpc.insecure_channel(self.core_url) self._core_service_stub = CoreServiceStub(self.__core_channel) self._job_service_stub = JobServiceStub(self.__core_channel) - self._dataset_service_stub = DatasetServiceStub( - self.__core_channel) + self._dataset_service_stub = DatasetServiceStub(self.__core_channel) def _connect_serving(self): """Connect to serving api""" @@ -313,11 +313,13 @@ def _build_serving_request(self, feature_set, entity_keys): return QueryFeaturesRequest( entityName=feature_set.entity, entityId=entity_keys, - featureId=feature_set.features) + featureId=feature_set.features, + ) def _response_to_df(self, feature_set, response, start=None, end=None): is_filter_time = start is not None and end is not None df = pd.DataFrame(columns=[feature_set.entity] + feature_set.features) + dtypes = {} for entity_id in response.entities: feature_map = response.entities[entity_id].features row = {response.entityName: entity_id} @@ -327,10 +329,13 @@ def _response_to_df(self, feature_set, response, start=None, end=None): ts = feature_map[feature_id].timestamp.ToDatetime() if ts < start or ts > end: continue + feast_valuetype = v.WhichOneof("val") + if feast_valuetype not in dtypes: + dtypes[feature_id] = types.FEAST_VALUETYPE_TO_DTYPE[feast_valuetype] v = getattr(v, v.WhichOneof("val")) row[feature_id] = v df = df.append(row, ignore_index=True) - return df.reset_index(drop=True) + return df.astype(dtypes).reset_index(drop=True) def _apply(self, obj): """Applies a single object to feast core. @@ -348,8 +353,10 @@ def _apply(self, obj): elif isinstance(obj, Storage): return self._apply_storage(obj) else: - raise TypeError('Apply can only be passed one of the following \ - types: [Feature, Entity, FeatureGroup, Storage, Importer]') + raise TypeError( + "Apply can only be passed one of the following \ + types: [Feature, Entity, FeatureGroup, Storage, Importer]" + ) def _apply_feature(self, feature): """Apply the feature to the core API @@ -360,8 +367,11 @@ def _apply_feature(self, feature): self._connect_core() response = self._core_service_stub.ApplyFeature(feature.spec) if self.verbose: - print("Successfully applied feature with id: {}\n---\n{}".format( - response.featureId, feature)) + print( + "Successfully applied feature with id: {}\n---\n{}".format( + response.featureId, feature + ) + ) return response.featureId def _apply_entity(self, entity): @@ -373,8 +383,11 @@ def _apply_entity(self, entity): self._connect_core() response = self._core_service_stub.ApplyEntity(entity.spec) if self.verbose: - print("Successfully applied entity with name: {}\n---\n{}".format( - response.entityName, entity)) + print( + "Successfully applied entity with name: {}\n---\n{}".format( + response.entityName, entity + ) + ) return response.entityName def _apply_feature_group(self, feature_group): @@ -385,11 +398,12 @@ def _apply_feature_group(self, feature_group): feature group to apply """ self._connect_core() - response = self._core_service_stub.ApplyFeatureGroup( - feature_group.spec) + response = self._core_service_stub.ApplyFeatureGroup(feature_group.spec) if self.verbose: - print("Successfully applied feature group with id: " + - "{}\n---\n{}".format(response.featureGroupId, feature_group)) + print( + "Successfully applied feature group with id: " + + "{}\n---\n{}".format(response.featureGroupId, feature_group) + ) return response.featureGroupId def _apply_storage(self, storage): @@ -401,12 +415,13 @@ def _apply_storage(self, storage): self._connect_core() response = self._core_service_stub.ApplyStorage(storage.spec) if self.verbose: - print("Successfully applied storage with id: " + - "{}\n{}".format(response.storageId, storage)) + print( + "Successfully applied storage with id: " + + "{}\n{}".format(response.storageId, storage) + ) return response.storageId - def _check_create_dataset_args(self, feature_set, start_date, end_date, - limit): + def _check_create_dataset_args(self, feature_set, start_date, end_date, limit): if len(feature_set.features) < 1: raise ValueError("feature set is empty") diff --git a/sdk/python/feast/sdk/utils/bq_util.py b/sdk/python/feast/sdk/utils/bq_util.py index 8cbc4b090bd..b52dee92ca4 100644 --- a/sdk/python/feast/sdk/utils/bq_util.py +++ b/sdk/python/feast/sdk/utils/bq_util.py @@ -13,19 +13,25 @@ # limitations under the License. import os +import tempfile import time - +from datetime import datetime +import pytz +import fastavro import pandas as pd +from google.cloud import bigquery from google.cloud.bigquery.client import Client as BQClient from google.cloud.bigquery.job import ExtractJobConfig, DestinationFormat from google.cloud.bigquery.table import Table +from google.cloud.exceptions import NotFound from google.cloud.storage import Client as GCSClient +from google.cloud import storage from feast.sdk.utils.gs_utils import is_gs_path, split_gs_path, gcs_to_df def head(client, table, max_rows=10): - '''Get the head of the table. Retrieves rows from the given table at + """Get the head of the table. Retrieves rows from the given table at minimum cost Args: @@ -37,12 +43,13 @@ def head(client, table, max_rows=10): Returns: pandas.DataFrame: dataframe containing the head of rows - ''' + """ rows = client.list_rows(table, max_results=max_rows) rows = [x for x in rows] return pd.DataFrame( - data=[list(x.values()) for x in rows], columns=list(rows[0].keys())) + data=[list(x.values()) for x in rows], columns=list(rows[0].keys()) + ) def get_table_name(feature_id, storage_spec): @@ -71,6 +78,108 @@ def get_table_name(feature_id, storage_spec): return ".".join([project, dataset, table_name]) +def get_default_templocation(bigquery_client, project=None): + if project is None: + project = bigquery_client.project + assert isinstance(project, str) + assert len(project) > 0 + storage_client = storage.Client() + default_bucket_name = f"feast-templocation-{project}" + try: + storage_client.get_bucket(default_bucket_name) + except NotFound: + print( + f'Default bucket "{default_bucket_name}" not found. Attempting to create it.' + ) + storage_client.create_bucket(bucket_name=default_bucket_name, project=project) + return f"gs://{default_bucket_name}" + + +def query_to_dataframe( + query: str, + bigquery_client: bigquery.Client = None, + storage_client: storage.Client = None, + project: str = None, + templocation: str = None, +) -> pd.DataFrame: + """ + Run a query job on BigQuery and return the result in Pandas DataFrame format + + Args: + query: BigQuery query e.g. "SELECT * FROM dataset.table" + bigquery_client: + storage_client: + project: Google Cloud project id + templocation: Google Cloud Storage location to store intermediate files, must start with "gs://" + + Returns: Pandas DataFrame of the query result + + """ + if isinstance(templocation, str) and not templocation.startswith("gs://"): + raise RuntimeError('templocation must start with "gs://"') + + if bigquery_client is None: + bigquery_client = bigquery.Client(project=project) + + if project is None: + project = bigquery_client.project + + query_job = bigquery_client.query(query, project=project) + query_job_state = "" + + while not query_job.done(): + if query_job.state != query_job_state: + print(f"Query status: {query_job.state}") + query_job_state = query_job.state + time.sleep(5) + + if query_job.state != query_job_state: + print(f"Query status: {query_job.state}") + + if query_job.exception(): + raise query_job.exception() + + if not templocation: + templocation = get_default_templocation(bigquery_client, project=project) + + if templocation.endswith("/"): + templocation += templocation[:-1] + + destination_uri = ( + f"{templocation}/bq-{datetime.now(pytz.utc).strftime('%Y%m%dT%H%M%SZ')}.avro" + ) + extract_job_config = bigquery.job.ExtractJobConfig(destination_format="AVRO") + extract_job = bigquery_client.extract_table( + query_job.destination, destination_uri, job_config=extract_job_config + ) + + while not extract_job.done(): + time.sleep(5) + + if extract_job.exception(): + raise extract_job.exception() + + if not storage_client: + storage_client = storage.Client(project=project) + + print("Reading query result into DataFrame") + + bucket_name, blob_name = ( + destination_uri.split("/")[2], + "/".join(destination_uri.split("/")[3:]), + ) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.get_blob(blob_name) + downloaded_avro_filename = tempfile.NamedTemporaryFile().name + blob.download_to_filename(downloaded_avro_filename) + + with open(downloaded_avro_filename, "rb") as avro_file: + avro_reader = fastavro.reader(avro_file) + df = pd.DataFrame.from_records(avro_reader) + + return df + + class TableDownloader: def __init__(self): self._bq = None @@ -88,8 +197,7 @@ def bq(self): self._bq = BQClient() return self._bq - def download_table_as_file(self, table_id, dest, staging_location, - file_type): + def download_table_as_file(self, table_id, dest, staging_location, file_type): """ Download a bigquery table as file Args: @@ -105,14 +213,13 @@ def download_table_as_file(self, table_id, dest, staging_location, if not is_gs_path(staging_location): raise ValueError("staging_uri must be a directory in GCS") - temp_file_name = 'temp_{}'.format(int(round(time.time() * 1000))) + temp_file_name = "temp_{}".format(int(round(time.time() * 1000))) staging_file_path = os.path.join(staging_location, temp_file_name) job_config = ExtractJobConfig() job_config.destination_format = file_type src_table = Table.from_string(table_id) - job = self.bq.extract_table( - src_table, staging_file_path, job_config=job_config) + job = self.bq.extract_table(src_table, staging_file_path, job_config=job_config) # await completion job.result() @@ -137,15 +244,14 @@ def download_table_as_df(self, table_id, staging_location): if not is_gs_path(staging_location): raise ValueError("staging_uri must be a directory in GCS") - temp_file_name = 'temp_{}'.format(int(round(time.time() * 1000))) + temp_file_name = "temp_{}".format(int(round(time.time() * 1000))) staging_file_path = os.path.join(staging_location, temp_file_name) job_config = ExtractJobConfig() job_config.destination_format = DestinationFormat.CSV job = self.bq.extract_table( - Table.from_string(table_id), - staging_file_path, - job_config=job_config) + Table.from_string(table_id), staging_file_path, job_config=job_config + ) # await completion job.result() diff --git a/sdk/python/feast/sdk/utils/types.py b/sdk/python/feast/sdk/utils/types.py index cf72425c257..21752c8e248 100644 --- a/sdk/python/feast/sdk/utils/types.py +++ b/sdk/python/feast/sdk/utils/types.py @@ -13,9 +13,10 @@ # limitations under the License. from feast.sdk.resources.feature import ValueType +import numpy as np # mapping of pandas dtypes to feast value type strings -_DTYPE_TO_VALUE_TYPE_MAPPING = { +DTYPE_TO_VALUE_TYPE_MAPPING = { "float64": ValueType.DOUBLE, "float32": ValueType.FLOAT, "int64": ValueType.INT64, @@ -32,6 +33,20 @@ "object": ValueType.STRING } +# Mapping of feast value type to Pandas DataFrame dtypes +# Integer and floating values are all 64-bit for better integration +# with BigQuery data types +FEAST_VALUETYPE_TO_DTYPE = { + "bytesVal": np.byte, + "stringVal": np.object, + "int32Val": np.int64, + "int64Val": np.int64, + "doubleVal": np.float64, + "floatVal": np.float64, + "boolVal": np.bool, + "timestampVal": np.datetime64, +} + def dtype_to_value_type(dtype): """Returns the equivalent feast valueType for the given dtype @@ -42,4 +57,4 @@ def dtype_to_value_type(dtype): Returns: feast.types.ValueType2.ValueType: equivalent feast valuetype """ - return _DTYPE_TO_VALUE_TYPE_MAPPING[dtype.__str__()] + return DTYPE_TO_VALUE_TYPE_MAPPING[dtype.__str__()] diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 6e3fe62f6cf..b07157bc3f2 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -1,11 +1,11 @@ # Copyright 2018 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,23 +16,23 @@ import os from setuptools import find_packages, setup, Command -NAME = 'Feast' -DESCRIPTION = 'Python sdk for Feast' -URL = 'https://github.com/gojek/feast' -AUTHOR = 'Feast' -REQUIRES_PYTHON = '>=3.6.0' -VERSION = imp.load_source( - 'feast.version', os.path.join('feast', 'version.py')).VERSION +NAME = "Feast" +DESCRIPTION = "Python sdk for Feast" +URL = "https://github.com/gojek/feast" +AUTHOR = "Feast" +REQUIRES_PYTHON = ">=3.6.0" +VERSION = imp.load_source("feast.version", os.path.join("feast", "version.py")).VERSION REQUIRED = [ - 'google-api-core>=1.7.0', - 'google-auth>=1.6.0', - 'google-cloud-bigquery>=1.8.0', - 'google-cloud-storage>=1.13.0', - 'googleapis-common-protos>=1.5.5', - 'grpcio>=1.16.1', - 'pandas', - 'protobuf>=3.0.0', - 'PyYAML', + "google-api-core>=1.7.0", + "google-auth>=1.6.0", + "google-cloud-bigquery>=1.8.0", + "google-cloud-storage>=1.13.0", + "googleapis-common-protos>=1.5.5", + "grpcio>=1.16.1", + "pandas", + "protobuf>=3.0.0", + "PyYAML", + "fastavro>=0.21.19" ] setup( @@ -42,16 +42,16 @@ author=AUTHOR, python_requires=REQUIRES_PYTHON, url=URL, - packages=find_packages(exclude=('tests',)), + packages=find_packages(exclude=("tests",)), install_requires=REQUIRED, include_package_data=True, - license='Apache', + license="Apache", classifiers=[ # Trove classifiers # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - ] -) \ No newline at end of file + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + ], +) diff --git a/sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro b/sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro new file mode 100644 index 0000000000000000000000000000000000000000..0a44e3a3657cc2b0efe60015a68bb90fd6403209 GIT binary patch literal 7653 zcmb7JYj_l8mX;4j#YIqui3|f+WPo&lgsyuP0dnaCVlE^k2n?WXrMo0eI$cS1caQ*z zJP3#pSs#f(!({+b0UZ_@k$FILhs9A)W(P$E(9Jpu46f)nUIAgx>8|>|O2VJ@k53*t zZ_anld*1V&(-k%Idi3gVG}%^m&l{zEKo#KTr}P8dI&`sG>a2n;_)tmxeIdl!$U zA5MaG`&cY)hr4_bQ4_K5h}ZdO1paN92`l~v`lg`gK)f>#F>;ZLPjLfVA5#%)y%zR{UksUuP#3PhzCVi zB^O1&Xm;ihpTNiB_^Jle49zzdaCw!1GK$oGh}&fsBNG0HNW+rv(&(#HIg@P9A?Xgx zJyA%99Sg@7P>g4Pb^Pb8h*36hOJXXEMiXT>8c~+9&FiH z*ZXcjNV|OG;@gNYA{B+~h9N0xCRsWT!Vqoz{i?9VO|p$>h$Ut5k@oDz5ai3Z|5)&n zCk$24+@wf4yyoQ`lIOs9O3meQg{4(ob=(Mr6ZJH-Ce=wrfpS%3xKcb${+^mJ?m$&5i%XIGG%isFjIT0C|J3WIo3S|k{dpfZ zu3B9Pzt8m<_x+Fth`^Ujj-tB1F7uk6Nv=W&Qj##;LO8Sj_ouTF;@GMBJD>4{pongT zBvA#{t;!&2auTD!njdDl4c}lr+zaX7_jH=6cw*4qEXblLWRk-g8Khi4hW_atw{6E5 zW`4Wx)GAL5njpC45j04{k_<9PyFSMEiH`Q$F^uNN@7>&t7)~m5Hx&}EDNx~&L@6+A zXg#1kYE=xXTN;i=%>`UObhF#lm5)Ciy#q_L`ROU6_u{isyi?qW6rP8yv}6#e3mh^q z*#{m1KEL3t%&`T4Hh6x?O`o&qWKDG+sHu>oHXj+$1+LPvST|JsI&1CTBw=<;m_DnK zy;73hxKti+-qXpT0?V+Wt0Q$>C(eAes^g(i`IxBff#`CUY*~_NEP@PAyCRVR1-8*_ z#h|I6ax^vkocoq6#&YG4@`n-(K~ZT_OFT-<>d|C!z-S1^jR;q6#%+r$p2B5uMFyx2 zQpt#O1-50N^ic4hzJo$ycz^i5;qAtH`a)T8`K_;y&+;$ zY>IjSI`wU2htaX``v#VORd7iv@dBVO&mjH)2pCsvL?VD!W;7W=1JIy>DcFy9|Gwfa zEW+06=@X84-Y=;X4LUHB{7EO-o`?cw%&@sYt3WBuQS&4+HgF~WO)%HAX4r5jU6?Jl+qH7X^CIrGs%-aA|=V3 zbO)HNltU;t?4ZTqHrKQt?2Y6rcwkTCA)FH_gP`b&t4*R`g??G%BjY;bLE@%`gK^7V z03$6NVI|mnvFDmo*n`^XGi@6@m>QK`L6e}qe-O#YB&IuDL#7=yA|?y3<@Kdf5Ie4J z)OSyRgdLX(PnIQ@fxMzX-d+<)dJ@VRCAHz86{YDPq1^T1axCTAVvtN0k*rgV6GRmv z>c}8M5+*ebDor;coP!#~ zfpM(Swmtaq)U61(qc{ybnx<`$U{IlcYP9(%H{&uC@hqJ|Ti~{dUuq;S0tTyD3 zk!41m$>k?v@EQoFD4MkGG@eU^8;0Fm3Q)73_>XUWHign9(QP12#5A?%5X}o!5NQgb zuQAOAt|$~@0=)BDtZfV$$p0MQIw6iD#tBaIyTPdf&Z_+)89?>XJ0IS?ab4(BE&i997W}Jxj|Dl%m5~+9IH-OB$7>E5 zI5}YkO&qU+H-BHO8{T6c%Fl^Us&8mv@6`QNFC;;S1iqt@0Tcq}01aYx&soD~UWb4d_ zhwhlgz*O*4uCD4j4A0d*l3r;<>R~M9CqiLREw5V!>6^I-t9R3@CA&06dKpUQz{nz! zzQsqzR^A^;%mo~p?ddpyyH1Y_4;;-i@tN)Q(Xt8#si>@vBn^c14EV;%h|z4o0vCyc zjIKVjILK8uCJY<7neypW#At{YcDJ2egs@GGzq+-<1FO;zDQSM7{$|+f*dB!hg&--+ zqhB7{c_)g_sGmRh;zkCe!=C6iIQS{hFCXO0HNy~Bbv9RK)EkUj=u%$6>}L(V>zjzD zf|BHNOHR(7Wj^BfZe-WZTK`w>>+=~tul3xUA9*SO!sAXdr~qW`DMU+ZjmMtl2aS+f z&xG)B+R{N!e3m-CvEcY0L|2clT*Tik;to zIC~0z;KJBK{2Cre(9xF9djj&Hy*u-y-fv+>guW=rJ~(O(q@Jubs(#ihFwR4*wSPF z&+fn{J#w?QY@}zWlwH|UU=IfY%!RX)A5sDfuB%=Y#DB*rI~$`Y2wdxRCwmFZIx1K) z?nE!=l7aM?7J(5>A%2{Ew3){^=6-N)-Y8UJXM=~*I1dqc5c93MWKc@r0!Dp9jp=d4 zGp?QU&%xn)Id{d6Uwd)^WOpDG*ikacZoo*{PguurVlj?4qq{(^40zz;_}Sipptu-< zB*IHJdghss3hV3uqT%&#THDX`CI_jqB_W|qZ-99 zqgY6=ZpSa6Bnlq>L2}@FD+K zXBJ``Ki>Yx_%GNK@mS=f3ARs2(`&h;gw}eDcYg)ksUT+)!6?*QQRa`I+jP2gAZBlG z%|m}3&)%*GOoBBYI&D9^9sOvm)`F1*SHFm9#EdBHJJ2j9Q{i(j_TBf(2>|%~%Zpmi zu{}V78-)`q4_iDeI=Mt(fO)QEF1BKElkH72UMl7-4P*x$Yhci za>IYjxDbJ7&mh_PfdE{YI@y=hbaBAmnfm4nTTvxCPJA@(0}q(&$&Uyx z`xG=h1?Ffk!qAD}pq)H9eEVKRE1a^a;!_WrLX{6B2Z-=~2ANO*U#3{WI<7nsGZXdD zL>~W9URFwzHkfPM@)j;(Wf&s^)%>*MVE{si7bNEdk@`NMTNgP&;tstL6(o88^3cpL zxRiji8i@SI{kkr74#F<2{l_HVdrj+`wVOo*e0PZoH%K5=2Ov00Ngf`j=$lqt?VUXh zJLb@$1wFp=ASk*!A|xHtzD6Pa;Y)y+>C7o7g1#kf;mep4xo!FX{F;fo28URO*0O*` zW)tioj#CHqL;DFeqhb8<)uMH`Vk#k1k)X;TCMz?@kS;ErV$7a->!S-XEqVS) zPyDwhEvoF!Z}8+$n*Yr}G45*hX|F(t&drWO<8f^jLa)z$?9q3<(CoV)1+C;Mpk`;g zmdEk#7X}f_HbSs4Id9JDU0Hnu`Ci>RbhYRDt15Igf$J|2;Z#3ToB~C=fd-9QKYCW? zlWT>894p;1v@v;kc!Hmw#llr}dU+64oE^JHlI*FLRS);XDm-GObkW%?958X!XRXsv zcC_E#)?)=@iz516w#k|Rvh!y+_e{r`V?zd0j*p4gF`ip^08y6rgxTv6FG+EW19-r> pGyMqXF;~vX-HGMBEA#r^hIKf)=ciGlJoB5K|G{D@nO(nq{69IW`K|x} literal 0 HcmV?d00001 diff --git a/sdk/python/tests/sdk/utils/test_bq_utils.py b/sdk/python/tests/sdk/utils/test_bq_utils.py index 206180197cc..bc5fd72f181 100644 --- a/sdk/python/tests/sdk/utils/test_bq_utils.py +++ b/sdk/python/tests/sdk/utils/test_bq_utils.py @@ -14,13 +14,18 @@ import os +import fastavro +import pandas as pd import pytest from google.cloud.bigquery.table import Table +from google.cloud.exceptions import NotFound from feast.sdk.resources.feature_set import FileType -from feast.sdk.utils.bq_util import TableDownloader, get_table_name +from feast.sdk.utils.bq_util import TableDownloader, get_table_name, query_to_dataframe from feast.specs.StorageSpec_pb2 import StorageSpec +testdata_path = os.path.abspath(os.path.join(__file__, "..", "..", "..", "data")) + def test_get_table_name(): project_name = "my_project" @@ -29,27 +34,53 @@ def test_get_table_name(): storage_spec = StorageSpec( id="BIGQUERY1", type="bigquery", - options={ - "project": project_name, - "dataset": dataset_name - }) - assert get_table_name(feature_id, storage_spec) == \ - "my_project.my_dataset.myentity_none" + options={"project": project_name, "dataset": dataset_name}, + ) + assert ( + get_table_name(feature_id, storage_spec) + == "my_project.my_dataset.myentity_none" + ) def test_get_table_name_not_bq(): feature_id = "myentity.feature1" storage_spec = StorageSpec(id="REDIS1", type="redis") - with pytest.raises( - ValueError, match="storage spec is not BigQuery storage spec"): + with pytest.raises(ValueError, match="storage spec is not BigQuery storage spec"): get_table_name(feature_id, storage_spec) +@pytest.mark.skipif( + os.getenv("SKIP_BIGQUERY_TEST") is not None, + reason="SKIP_BIGQUERY_TEST is set in the environment", +) +def test_query_to_dataframe(): + with open( + os.path.join(testdata_path, "austin_bikeshare.bikeshare_stations.avro"), "rb" + ) as expected_file: + avro_reader = fastavro.reader(expected_file) + expected = pd.DataFrame.from_records(avro_reader) + + query = "SELECT * FROM `bigquery-public-data.austin_bikeshare.bikeshare_stations`" + actual = query_to_dataframe(query) + assert expected.equals(actual) + + +@pytest.mark.skipif( + os.getenv("SKIP_BIGQUERY_TEST") is not None, + reason="SKIP_BIGQUERY_TEST is set in the environment", +) +def test_query_to_dataframe_for_non_existing_dataset(): + query = "SELECT * FROM `bigquery-public-data.this_dataset_should_not_exists.bikeshare_stations`" + with pytest.raises(NotFound): + query_to_dataframe(query) + + class TestTableDownloader(object): def test_download_table_as_df(self, mocker): self._stop_time(mocker) mocked_gcs_to_df = mocker.patch( - "feast.sdk.utils.bq_util.gcs_to_df", return_value=None) + "feast.sdk.utils.bq_util.gcs_to_df", return_value=None + ) staging_path = "gs://temp/" staging_file_name = "temp_0" @@ -59,19 +90,15 @@ def test_download_table_as_df(self, mocker): exp_staging_path = os.path.join(staging_path, staging_file_name) table_dldr._bq = _Mock_BQ_Client() - mocker.patch.object( - table_dldr._bq, "extract_table", return_value=_Job()) + mocker.patch.object(table_dldr._bq, "extract_table", return_value=_Job()) - table_dldr.download_table_as_df( - table_id, staging_location=staging_path) + table_dldr.download_table_as_df(table_id, staging_location=staging_path) assert len(table_dldr._bq.extract_table.call_args_list) == 1 - args, kwargs = \ - table_dldr._bq.extract_table.call_args_list[0] - assert args[0].full_table_id == Table.from_string( - table_id).full_table_id + args, kwargs = table_dldr._bq.extract_table.call_args_list[0] + assert args[0].full_table_id == Table.from_string(table_id).full_table_id assert args[1] == exp_staging_path - assert kwargs['job_config'].destination_format == "CSV" + assert kwargs["job_config"].destination_format == "CSV" mocked_gcs_to_df.assert_called_once_with(exp_staging_path) def test_download_csv(self, mocker): @@ -90,14 +117,15 @@ def test_download_invalid_staging_url(self): table_id = "project_id.dataset_id.table_id" table_dldr = TableDownloader() with pytest.raises( - ValueError, match="staging_uri must be a directory in " - "GCS"): - table_dldr.download_table_as_file(table_id, "/tmp/dst", - "/local/directory", FileType.CSV) + ValueError, match="staging_uri must be a directory in " "GCS" + ): + table_dldr.download_table_as_file( + table_id, "/tmp/dst", "/local/directory", FileType.CSV + ) with pytest.raises( - ValueError, match="staging_uri must be a directory in " - "GCS"): + ValueError, match="staging_uri must be a directory in " "GCS" + ): table_dldr.download_table_as_df(table_id, "/local/directory") def _test_download_file(self, mocker, type): @@ -110,28 +138,27 @@ def _test_download_file(self, mocker, type): mock_blob = _Blob() mocker.patch.object(mock_blob, "download_to_filename") table_dldr._bq = _Mock_BQ_Client() - mocker.patch.object( - table_dldr._bq, "extract_table", return_value=_Job()) + mocker.patch.object(table_dldr._bq, "extract_table", return_value=_Job()) table_dldr._gcs = _Mock_GCS_Client() mocker.patch.object( - table_dldr._gcs, "get_bucket", return_value=_Bucket(mock_blob)) + table_dldr._gcs, "get_bucket", return_value=_Bucket(mock_blob) + ) table_dldr.download_table_as_file( - table_id, dst_path, staging_location=staging_path, file_type=type) + table_id, dst_path, staging_location=staging_path, file_type=type + ) exp_staging_path = os.path.join(staging_path, staging_file_name) assert len(table_dldr._bq.extract_table.call_args_list) == 1 - args, kwargs = \ - table_dldr._bq.extract_table.call_args_list[0] - assert args[0].full_table_id == Table.from_string( - table_id).full_table_id + args, kwargs = table_dldr._bq.extract_table.call_args_list[0] + assert args[0].full_table_id == Table.from_string(table_id).full_table_id assert args[1] == exp_staging_path - assert kwargs['job_config'].destination_format == str(type) + assert kwargs["job_config"].destination_format == str(type) mock_blob.download_to_filename.assert_called_once_with(dst_path) def _stop_time(self, mocker): - mocker.patch('time.time', return_value=0) + mocker.patch("time.time", return_value=0) class _Mock_BQ_Client: diff --git a/testing/Makefile b/testing/Makefile index 0eef0abab9a..0ff05f827fa 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -32,6 +32,6 @@ update-config: get-cluster-credentials update-plugins: get-cluster-credentials kubectl create configmap plugins --from-file=plugins.yaml=prow/plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f - -build-push: - docker build test-image/ -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) +build-push-test-image: + docker build -f docker-images/test-image/Dockerfile -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) . $(PUSH) "$(REGISTRY)/$(PROJECT)/test-image:$(VERSION)" \ No newline at end of file diff --git a/testing/README.md b/testing/README.md index c345c46e75e..df38d34a676 100644 --- a/testing/README.md +++ b/testing/README.md @@ -4,9 +4,9 @@ This folder contains the Feast test infrastructure. ## Components -* test-image/ - Base docker image and script for running tests. +* docker-images/ - Docker images for running tests * prow/ - Prow configuration (plugins and jobs) -* tf/ - Terraform modules to provision the base testing infrastructure on GCP +* infrastructure/ - Terraform modules to provision the base testing infrastructure on GCP ## Set up @@ -23,7 +23,7 @@ mv tf/gcs terraform import google_storage_bucket.kf-feast-terraform-state kf-feast-terraform-state ``` -4. Ensure that all variables are set correctly in `tf/terraform.tfvars`. It is likely that the GCP project will need to be updated. +4. Ensure that all variables are set correctly in `infrastructure/terraform.tfvars`. It is likely that the GCP project will need to be updated. 5. Create the primary Kubernetes cluster which will host Prow and Argo @@ -51,7 +51,11 @@ To update Prow jobs, plugins, or the Docker image used for testing, modify one o - `prow/config.yaml` - `prow/plugins.yaml` -- `test-image/Dockerfile` -- `test-image/run.sh` +- `docker-images/test-image/Dockerfile` +- `docker-images/test-image/run.sh` -After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. \ No newline at end of file +After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. + +## Installing Argo + +Argo is used to run integration tests. It follows the [standard installation process described in the argo quickstart](https://github.com/argoproj/argo/blob/master/demo.md), with the artifact repository [configured to use gcs](https://github.com/argoproj/argo/blob/master/ARTIFACT_REPO.md). \ No newline at end of file diff --git a/testing/docker-images/docker-builder-image/Dockerfile b/testing/docker-images/docker-builder-image/Dockerfile new file mode 100644 index 00000000000..45f2ce67b15 --- /dev/null +++ b/testing/docker-images/docker-builder-image/Dockerfile @@ -0,0 +1,25 @@ +FROM docker:18.09 + +# from https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/alpine/Dockerfile +ENV CLOUD_SDK_VERSION=236.0.0 + +ENV PATH /google-cloud-sdk/bin:$PATH +RUN apk --no-cache add \ + curl \ + python \ + py-crcmod \ + bash \ + libc6-compat \ + openssh-client \ + git \ + gnupg \ + && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + ln -s /lib /lib64 && \ + gcloud config set core/disable_usage_reporting true && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud config set metrics/environment github_docker_image && \ + gcloud --version + +RUN gcloud auth configure-docker diff --git a/testing/docker-images/gcloud-argo/Dockerfile b/testing/docker-images/gcloud-argo/Dockerfile new file mode 100644 index 00000000000..d84f56ea22f --- /dev/null +++ b/testing/docker-images/gcloud-argo/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:stretch + +RUN apt-get update && apt-get install -y make curl gnupg wget python + +# Install kubectl gcloud and argo +RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-241.0.0-linux-x86_64.tar.gz | tar xzf - \ + && /google-cloud-sdk/bin/gcloud -q components install kubectl \ + && /google-cloud-sdk/install.sh \ + && ln -s /google-cloud-sdk/bin/gcloud /usr/bin/gcloud +RUN curl -sSL -o /usr/local/bin/argo https://github.com/argoproj/argo/releases/download/v2.2.1/argo-linux-amd64 \ + && chmod +x /usr/local/bin/argo + +COPY run.sh /usr/local/bin/ + +ENTRYPOINT ["run.sh"] \ No newline at end of file diff --git a/testing/docker-images/gcloud-argo/run.sh b/testing/docker-images/gcloud-argo/run.sh new file mode 100755 index 00000000000..e7f501972f3 --- /dev/null +++ b/testing/docker-images/gcloud-argo/run.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e +printenv + +if [ $# -eq 0 ] +then + echo "" + echo "please specify commands as arguments, for example \"make test\"" + echo "" + exit 1 +fi + +echo "" +echo "cloning Feast repository..." +echo "" + +git clone https://github.com/gojek/feast +cd feast + +if [ -n "${PULL_NUMBER}" ] && [ "$JOB_TYPE" = "presubmit" ] +then + echo "" + echo "fetching PR ${PULL_NUMBER}..." + echo "" + git fetch origin pull/"${PULL_NUMBER}"/head:pull_"${PULL_NUMBER}" + + echo "" + echo "checking out PR ${PULL_NUMBER}..." + echo "" + git checkout pull_"${PULL_NUMBER}" +fi + +echo "connecting to test cluster..." +gcloud container clusters get-credentials primary-test-cluster --zone us-central1-a --project kf-feast +echo "testing connection to argo..." +argo list +if [ $? -ne 0 ] +then + echo "failed to connect to argo" + echo "exiting" + exit 1 +fi + +echo "sha:" +git rev-parse HEAD +echo "" +echo "running tests" +echo "" + +$@ \ No newline at end of file diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile new file mode 100644 index 00000000000..04b734fbef6 --- /dev/null +++ b/testing/docker-images/terraform-image/Dockerfile @@ -0,0 +1,19 @@ +FROM debian:stretch + +RUN apt-get update && apt-get install -y jq wget python zip netcat + +RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \ + && /google-cloud-sdk/bin/gcloud -q components install kubectl \ + && /google-cloud-sdk/bin/gcloud -q components install beta \ + && /google-cloud-sdk/bin/gcloud -q components install bq + +ENV PATH="/google-cloud-sdk/bin:${PATH}" + +RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ + && unzip terraform_0.11.11_linux_amd64.zip \ + && mv terraform /bin/terraform + +RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-arm64.tar.gz | tar xzf - \ + && mv linux-arm64/helm /bin/helm + +RUN helm init --client-only diff --git a/testing/test-image/Dockerfile b/testing/docker-images/test-image/Dockerfile similarity index 89% rename from testing/test-image/Dockerfile rename to testing/docker-images/test-image/Dockerfile index b3fcbdf4b61..8dc68c9edd8 100644 --- a/testing/test-image/Dockerfile +++ b/testing/docker-images/test-image/Dockerfile @@ -6,8 +6,6 @@ RUN git clone https://github.com/gojek/feast \ && cd feast && mvn test || true \ && cd .. && rm -rf feast -RUN apt-get install -y make - COPY run.sh /usr/local/bin/ ENTRYPOINT ["run.sh"] \ No newline at end of file diff --git a/testing/test-image/run.sh b/testing/docker-images/test-image/run.sh similarity index 96% rename from testing/test-image/run.sh rename to testing/docker-images/test-image/run.sh index f007da1e0d1..a001255897e 100755 --- a/testing/test-image/run.sh +++ b/testing/docker-images/test-image/run.sh @@ -34,7 +34,7 @@ fi echo "sha:" git rev-parse HEAD echo "" -echo "running unit tests" +echo "running tests" echo "" $@ \ No newline at end of file diff --git a/testing/tf/cloud-build/cloud-build.tf b/testing/infrastructure/cloud-build/cloud-build.tf similarity index 100% rename from testing/tf/cloud-build/cloud-build.tf rename to testing/infrastructure/cloud-build/cloud-build.tf diff --git a/testing/tf/cloud-build/cloudbuild.yaml b/testing/infrastructure/cloud-build/cloudbuild.yaml similarity index 100% rename from testing/tf/cloud-build/cloudbuild.yaml rename to testing/infrastructure/cloud-build/cloudbuild.yaml diff --git a/testing/tf/cloud-build/variables.tf b/testing/infrastructure/cloud-build/variables.tf similarity index 100% rename from testing/tf/cloud-build/variables.tf rename to testing/infrastructure/cloud-build/variables.tf diff --git a/testing/tf/gcs/backend.tf b/testing/infrastructure/gcs/backend.tf similarity index 100% rename from testing/tf/gcs/backend.tf rename to testing/infrastructure/gcs/backend.tf diff --git a/testing/tf/gcs/main.tf b/testing/infrastructure/gcs/main.tf similarity index 100% rename from testing/tf/gcs/main.tf rename to testing/infrastructure/gcs/main.tf diff --git a/testing/tf/k8s-cluster/backend.tf b/testing/infrastructure/k8s-cluster/backend.tf similarity index 100% rename from testing/tf/k8s-cluster/backend.tf rename to testing/infrastructure/k8s-cluster/backend.tf diff --git a/testing/tf/k8s-cluster/main.tf b/testing/infrastructure/k8s-cluster/main.tf similarity index 69% rename from testing/tf/k8s-cluster/main.tf rename to testing/infrastructure/k8s-cluster/main.tf index e5b01754f96..43b1de31b98 100644 --- a/testing/tf/k8s-cluster/main.tf +++ b/testing/infrastructure/k8s-cluster/main.tf @@ -16,6 +16,10 @@ resource "google_container_cluster" "primary-test-cluster" { node_config { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", ] machine_type = "n1-standard-4" diff --git a/testing/tf/k8s-cluster/output.tf b/testing/infrastructure/k8s-cluster/output.tf similarity index 100% rename from testing/tf/k8s-cluster/output.tf rename to testing/infrastructure/k8s-cluster/output.tf diff --git a/testing/tf/k8s-cluster/provider.tf b/testing/infrastructure/k8s-cluster/provider.tf similarity index 100% rename from testing/tf/k8s-cluster/provider.tf rename to testing/infrastructure/k8s-cluster/provider.tf diff --git a/testing/tf/k8s-cluster/variables.tf b/testing/infrastructure/k8s-cluster/variables.tf similarity index 100% rename from testing/tf/k8s-cluster/variables.tf rename to testing/infrastructure/k8s-cluster/variables.tf diff --git a/testing/integration/Makefile b/testing/integration/Makefile new file mode 100644 index 00000000000..7ce9feda305 --- /dev/null +++ b/testing/integration/Makefile @@ -0,0 +1,25 @@ +# +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: test-integration it-redis-bq-dataflow + +ifeq ($(TYPE), PR) +test-integration: + argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p prId=$(ID) +else +test-integration: + argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p gitRef=$(ID) +endif diff --git a/testing/integration/README.md b/testing/integration/README.md new file mode 100644 index 00000000000..fde86a35335 --- /dev/null +++ b/testing/integration/README.md @@ -0,0 +1,29 @@ +# Integration tests + +Integration tests for Feast are run on argo workflows. The tests follow the following steps: + +1. Provision infrastructure using Terraform +2. Test using pytest +3. Teardown infrastructure using Terraform + +## Terraform modules + +The `tf/modules` directory contains Terraform modules to set up the necessary infrastructure. Currently contains: + +- `cluster`: kubernetes cluster with necessary permissions to run Feast jobs +- `feast-helm`: Feast helm installation. + +## Tests + +The `tests` directory contains the integration tests. Each folder should contain the following: + +1. The terraform scripts to set up the necessary infra. +2. Data to run the tests on +3. A pytest file that executes the ingestion jobs and then tests for correctness. +4. Argo workflow yaml to orchestrate the entire process. + +Multiple tests can be run on the same infrastructure. + +### Adding new tests + +To add your own tests, either (1) create a new test case within an existing folder or (2) create a new folder with the resources mentioned above. \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/.env.local b/testing/integration/tests/redis-bq-dataflow/.env.local new file mode 100644 index 00000000000..99a58fd76d3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/.env.local @@ -0,0 +1,7 @@ +# Example environment variables that need to be set when running this test locally +# Export the variables like so: +# set -a; source .env.local +FEAST_CORE_URL=localhost:6565 +FEAST_SERVING_URL=localhost:6566 +PROJECT_ID=kf-feast +BUCKET_NAME=it-feast-storage diff --git a/testing/integration/tests/redis-bq-dataflow/.env.local.example b/testing/integration/tests/redis-bq-dataflow/.env.local.example new file mode 100644 index 00000000000..1ecb945e3ce --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/.env.local.example @@ -0,0 +1,7 @@ +# Example environment variables that need to be set when running this test locally +# Export the variables like so: +# set -a; source .env.local +FEAST_CORE_URL=localhost:6565 +FEAST_SERVING_URL=localhost:6566 +PROJECT_ID=google-cloud-project +BUCKET_NAME=bucket-name diff --git a/testing/integration/tests/redis-bq-dataflow/.gitignore b/testing/integration/tests/redis-bq-dataflow/.gitignore new file mode 100644 index 00000000000..781b3b25c5c --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/.gitignore @@ -0,0 +1,3 @@ +# local env files +.env.local +.env.*.local diff --git a/testing/integration/tests/redis-bq-dataflow/README.md b/testing/integration/tests/redis-bq-dataflow/README.md new file mode 100644 index 00000000000..248cc4e1d88 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/README.md @@ -0,0 +1,9 @@ +# redis-bq-dataflow + +This repository contains an end-to-end test that sets up the following infrastructure: + +1. Feast configured to run jobs on dataflow +2. Redis as serving store +3. BQ as warehouse store + +And then runs the tests in `test_feast.py`. \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml b/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml new file mode 100644 index 00000000000..8fa945a6774 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml @@ -0,0 +1,5 @@ +name: myentity +description: my test entity +tags: + - tag1 + - tag2 diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml new file mode 100644 index 00000000000..05b5279d0e4 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_double_redis +name: feature_double_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: DOUBLE +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml new file mode 100644 index 00000000000..3ff38358ce3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_float_redis +name: feature_float_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: FLOAT +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml new file mode 100644 index 00000000000..ebeb92dc999 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_int32_redis +name: feature_int32_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT32 +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml new file mode 100644 index 00000000000..dd6b7339bf3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_int64_redis +name: feature_int64_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT64 +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml new file mode 100644 index 00000000000..3eb170a23c4 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml @@ -0,0 +1,19 @@ +type: file.csv +sourceOptions: + path: gs://it-feast-storage/test-cases/test_data.csv +entities: + - myentity +schema: + entityIdColumn: entity + timestampColumn: ts + fields: + - name: entity + - name: ts + - name: feature1 + featureId: myentity.feature_double_redis + - name: feature2 + featureId: myentity.feature_float_redis + - name: feature3 + featureId: myentity.feature_int32_redis + - name: feature4 + featureId: myentity.feature_int64_redis diff --git a/testing/integration/tests/redis-bq-dataflow/data/test_data.csv b/testing/integration/tests/redis-bq-dataflow/data/test_data.csv new file mode 100644 index 00000000000..16906eef985 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/test_data.csv @@ -0,0 +1,50 @@ +0,2018-01-01T00:00:00,0.16654998178974967,0.0779124915810051,8,1 +0,2018-01-01T00:01:00,0.35055416999061695,0.3789967915556205,4,7 +0,2018-01-01T00:02:00,0.922617931213872,0.3179716588679623,8,6 +0,2018-01-01T00:03:00,0.06582437841319277,0.05565087947895475,9,1 +0,2018-01-01T00:04:00,0.8470568422759338,0.21347178570025072,2,10 +0,2018-01-01T00:05:00,0.11361651376595083,0.610881579742112,8,10 +0,2018-01-01T00:06:00,0.3034691430518833,0.19830856710384326,6,2 +0,2018-01-01T00:07:00,0.8346843022317924,0.08544530941545558,2,4 +0,2018-01-01T00:08:00,0.012367592836242891,0.5726466047910151,2,5 +0,2018-01-01T00:09:00,0.9887391081292758,0.9819570408280592,9,10 +1,2018-01-01T00:00:00,0.7503367819099366,0.946197160138704,10,1 +1,2018-01-01T00:01:00,0.9190195271598668,0.8836270958333856,10,4 +1,2018-01-01T00:02:00,0.07725768926853083,0.448071643977631,4,9 +1,2018-01-01T00:03:00,0.4965090410288343,0.9615236179532041,3,5 +1,2018-01-01T00:04:00,0.4838512691935104,0.355736320341819,7,2 +1,2018-01-01T00:05:00,0.7422860280929264,0.26936605638140254,7,6 +1,2018-01-01T00:06:00,0.007120660156963887,0.9457195989857463,5,4 +1,2018-01-01T00:07:00,0.5165841927879966,0.5734751201042606,1,4 +1,2018-01-01T00:08:00,0.0874458350429409,0.3284354801066641,3,8 +1,2018-01-01T00:09:00,0.8091561596661963,0.5176044026790632,1,9 +2,2018-01-01T00:00:00,0.5440258526780313,0.7936189500778384,3,5 +2,2018-01-01T00:01:00,0.9012681719733718,0.408129175696462,5,8 +2,2018-01-01T00:02:00,0.1357239530601766,0.2882272639567518,9,9 +2,2018-01-01T00:03:00,0.5482100359374814,0.22784885285759426,2,10 +2,2018-01-01T00:04:00,0.47915480013638667,0.5783117797172574,6,2 +2,2018-01-01T00:05:00,0.08954537587399436,0.6036422156720167,2,5 +2,2018-01-01T00:06:00,0.022254256710355302,0.8425123292474225,2,2 +2,2018-01-01T00:07:00,0.2981277839268307,0.7187791198561501,9,6 +2,2018-01-01T00:08:00,0.7483695369918476,0.24681429759605789,10,3 +2,2018-01-01T00:09:00,0.1357618414286158,0.887316489600256,3,10 +3,2018-01-01T00:00:00,0.24835796330732807,0.8871182399308823,3,2 +3,2018-01-01T00:01:00,0.18779166308706263,0.6635037319574626,4,10 +3,2018-01-01T00:02:00,0.00017995781865332017,0.9437703134427515,9,2 +3,2018-01-01T00:03:00,0.44519915810431854,0.03739113677349182,4,5 +3,2018-01-01T00:04:00,0.27156326671637077,0.06880899222977854,3,2 +3,2018-01-01T00:05:00,0.7647529493108539,0.14002956138831768,7,10 +3,2018-01-01T00:06:00,0.9954469835451967,0.7567907196365666,9,9 +3,2018-01-01T00:07:00,0.3883998745214877,0.18671618739049767,7,10 +3,2018-01-01T00:08:00,0.5448378857720906,0.38482161282383076,3,1 +3,2018-01-01T00:09:00,0.3550003641256,0.1818481760674323,1,9 +4,2018-01-01T00:00:00,0.5925452403226181,0.30212100652221296,5,10 +4,2018-01-01T00:01:00,0.18599533814587,0.5230622774213863,6,5 +4,2018-01-01T00:02:00,0.9599946262502064,0.26435543431553166,10,8 +4,2018-01-01T00:03:00,0.22755474767811312,0.6693510146123766,1,8 +4,2018-01-01T00:04:00,0.12063065422121622,0.7460800163440976,10,9 +4,2018-01-01T00:05:00,0.58262241980741,0.2714493926314999,9,5 +4,2018-01-01T00:06:00,0.26952820440313163,0.40841655204277727,5,1 +4,2018-01-01T00:07:00,0.7779897852625218,0.39955885342995734,9,3 +4,2018-01-01T00:08:00,0.9892858266244913,0.91265457567995,2,10 +4,2018-01-01T00:09:00,0.31825556885425366,0.6727958709215186,3,9 diff --git a/testing/integration/tests/redis-bq-dataflow/main.py b/testing/integration/tests/redis-bq-dataflow/main.py new file mode 100644 index 00000000000..7c77180bac2 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/main.py @@ -0,0 +1,9 @@ +import test_feast +from feast.sdk.client import Client + +# This main.py is for easier local debugging +# because running pytest often hides useful debugging output + +if __name__ == "__main__": + feast_client = Client(verbose=True) + test_feast.TestFeastIntegration().test_end_to_end(client=feast_client) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py new file mode 100644 index 00000000000..e7c3003fe01 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -0,0 +1,133 @@ +import os +import subprocess +from subprocess import PIPE +from time import sleep + +import fastavro +import feast.sdk.utils.gs_utils as utils +import numpy as np +import pandas as pd +import pytest +import yaml +from feast.sdk.client import Client +from feast.sdk.resources.entity import Entity +from feast.sdk.resources.feature import Feature +from feast.sdk.resources.feature_set import FeatureSet +from feast.sdk.utils import bq_util +from google.cloud import storage + + +@pytest.fixture +def client(): + return Client(verbose=True) + + +# Init the system by registering relevant resources. +def _register_resources(client, entities_fldr, features_fldr): + resources = [] + for ent_file in os.listdir(entities_fldr): + resources.append(Entity.from_yaml(os.path.join(entities_fldr, ent_file))) + for feat_file in os.listdir(features_fldr): + resources.append(Feature.from_yaml(os.path.join(features_fldr, feat_file))) + client.apply(resources) + + +# Run an import job given an import spec. +def _run_job_and_wait_for_completion(job_yaml): + out = subprocess.run( + "feast jobs run {}".format(job_yaml).split(" "), check=True, stdout=PIPE + ) + job_id = out.stdout.decode("utf-8").split(" ")[-1] + job_status, job_complete = "UNKNOWN", False + terminal_states = ["COMPLETED", "ABORTED", "ERROR", "UNKNOWN"] + while not job_complete: + out = subprocess.run( + "feast get job {}".format(job_id).split(" "), check=True, stdout=PIPE + ) + job_details = yaml.load(out.stdout.decode("utf-8").replace("\t", " ")) + if job_details["Status"] != job_status: + job_status = job_details["Status"] + print("Job id {} currently {}".format(job_id, job_status)) + if job_status in terminal_states: + break + sleep(10) + return job_status + + +def _stage_data(local, remote): + split = utils.split_gs_path(remote) + storage_client = storage.Client() + bucket = storage_client.get_bucket(split[0]) + blob = bucket.blob(split[1]) + + blob.upload_from_filename(local) + + +class TestFeastIntegration: + def test_end_to_end(self, client): + project_id = os.environ.get("PROJECT_ID") + bucket_name = os.environ.get("BUCKET_NAME") + testdata_path = os.path.abspath(os.path.join(__file__, "..", "testdata")) + + with open(os.path.join(testdata_path, "myentity.avro"), "rb") as myentity_file: + avro_reader = fastavro.reader(myentity_file) + expected = ( + pd.DataFrame.from_records(avro_reader) + .drop(columns=["created_timestamp"]) + .sort_values(["id", "event_timestamp"]) + .reset_index(drop=True) + ) + + self.run_batch_import(bucket_name, client) + self.validate_warehouse_data(project_id, expected) + self.validate_serving_data(client, expected) + + @staticmethod + def run_batch_import(bucket_name, client): + _stage_data( + "testdata/myentity.csv", + "gs://{}/test-cases/myentity.csv".format(bucket_name), + ) + _register_resources(client, "testdata/entity", "testdata/feature") + job_status = _run_job_and_wait_for_completion("testdata/import/import_csv.yaml") + assert job_status == "COMPLETED" + + @staticmethod + def validate_serving_data(client, expected): + features = { + "myentity.feature_double_redis", + "myentity.feature_float_redis", + "myentity.feature_int32_redis", + "myentity.feature_int64_redis", + } + + feature_set = FeatureSet(entity="myentity", features=[f for f in features]) + entity_keys = [str(id) for id in list(expected.id.unique())] + actual = client.get_serving_data(feature_set, entity_keys=entity_keys) + actual = actual.sort_values(["myentity"]).reset_index(drop=True) + + # Serving store returns only latest feature values so we update expected values + # to only have the feature values with latest event timestamp + expected = expected.loc[expected.groupby("id")["event_timestamp"].idxmax()] + # Ensure the columns in expected are the same as those in actual + # For example, in actual there is not event_timestamp column + expected = expected.rename(columns={"id": "myentity"}) + expected = expected[ + [c.replace("myentity.", "") for c in actual.columns] + ].reset_index(drop=True) + + np.testing.assert_array_equal(expected, actual) + + @staticmethod + def validate_warehouse_data(project_id, expected): + actual = ( + bq_util.query_to_dataframe( + f"SELECT * FROM `feast_it.myentity_view`", project=project_id + ) + # created_timestamp is not relevant for validating correctness of import + # and retrieval of feature values + .drop(columns=["created_timestamp"]) + .sort_values(["id", "event_timestamp"]) + .reset_index(drop=True) + ) + assert expected.equals(actual) diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/entity/myentity.yaml b/testing/integration/tests/redis-bq-dataflow/testdata/entity/myentity.yaml new file mode 100644 index 00000000000..8fa945a6774 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/entity/myentity.yaml @@ -0,0 +1,5 @@ +name: myentity +description: my test entity +tags: + - tag1 + - tag2 diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_double_redis.yaml b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_double_redis.yaml new file mode 100644 index 00000000000..1ddc2558a36 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_double_redis.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_double_redis +name: feature_double_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: DOUBLE +uri: https://github.com/bob/example \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_float_redis.yaml b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_float_redis.yaml new file mode 100644 index 00000000000..e5e7afa4704 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_float_redis.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_float_redis +name: feature_float_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: FLOAT +uri: https://github.com/bob/example \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int32_redis.yaml b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int32_redis.yaml new file mode 100644 index 00000000000..caacd5a055c --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int32_redis.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_int32_redis +name: feature_int32_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT32 +uri: https://github.com/bob/example \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int64_redis.yaml b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int64_redis.yaml new file mode 100644 index 00000000000..e085a62c022 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/feature/feature_int64_redis.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_int64_redis +name: feature_int64_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT64 +uri: https://github.com/bob/example \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/import/import_csv.yaml b/testing/integration/tests/redis-bq-dataflow/testdata/import/import_csv.yaml new file mode 100644 index 00000000000..04f45d54442 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/import/import_csv.yaml @@ -0,0 +1,19 @@ +type: file.csv +sourceOptions: + path: gs://it-feast-storage/test-cases/myentity.csv +entities: + - myentity +schema: + entityIdColumn: entity + timestampColumn: ts + fields: + - name: entity + - name: ts + - name: feature1 + featureId: myentity.feature_double_redis + - name: feature2 + featureId: myentity.feature_float_redis + - name: feature3 + featureId: myentity.feature_int32_redis + - name: feature4 + featureId: myentity.feature_int64_redis diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/myentity.avro b/testing/integration/tests/redis-bq-dataflow/testdata/myentity.avro new file mode 100644 index 0000000000000000000000000000000000000000..42dfb251ac9aa0affd02b58173c20dc992b94b3e GIT binary patch literal 2709 zcmcJReN0nV7{8uTG}giwp>?=y1D8d zZW_cXGQ;A;q(so-bScp~qSIZR2r|IBDe|S1Vfpe4MZh76JMTI7oK+2f?!Vsjcb@yc z@AJH;O-%djxu7V+I-5A%D4GOgNJf^$5>1{njjIK#!8>-N*8as>K|4ol6O5wGq>a&L z7_9KTR21#_X_Z;9m@>65iAI{&XJ;>|q^RrkgCRhDO^Rv|sQO~kF6dA=>Wq!O z8fy8AqUV~9J_OX8^N6a30IEOJ=Fy=bN#JfHLj)S7e5+8_PO zO^S}}cK$;wa4BsORm}{`U9jL|&54=v0sIloKoYZ{kDPPq=kt z{w*jV?ya%|`&PqTajiT94S}livJY^rVMto3a(y?|K~WyULkdca)zqECXfRq@^=V7p z`8JAj3#t^97^_({_Y$;qa`>9FUhbkOiwkcn65L9N9&-g$U5kD$ZAv>qQ5KiHYNahT z$w2(kR>*+2xw3Cc6;3&{#oUqxl>!K=nVkm{LiP8IS^IAE;Eywr#4O0G90{;M-u5+~ zYdD%%I?li0Fj#<5HY~nd#4Ua60FHhT^2Mq~C~J#HI~Z!!7yfV&EO_awhM$V> z!vwG@k4uSC0GX>ZNx%gLqX$(-yIiZ=Daz;Ce*Ofx2M8*KF2T_bLekid3AS4>EUe1n z!WV{GFjs}&szza@n|E%Fg5}02i%TCXfLP$WFtHgjkX!tDO6*mf96*@`vPY?vTCmvm zC#bs18bZ@kD=5n1!cnp?@*7j@pw`6kgOzu;D{V2i`1M9N)^EaZ4MxYmbzCpjgK8X- z*j%{)^Nd6_vfzciGitCKfU*p@ZvI&X*Rz`lJ&C3`*yz$=892P07uEfB&6my!34$`w_jQ4S5#R6zGg+0 z%r(K9ilx+O@1eHJtHp3BVpTQ)o-BfDZvTq!fpB!BV&0>&dYBtVnOl5VbPJZQ!NWo* zytVZEEgBqMKv@QOvIwe*yXl4H-jJVtaDF+C1E4Hf{5O`kmC(`F1!#!<{Pu(GXDG^y zbrufM<)&!vHe5h3>c0K>$B$OxA4X7R(dM@cu|VIy3~zK8(pTh*lLnOuu!WI^m0?kI WeLVgF2ttkhL%+>&-PJw)$KzkG5kcbs literal 0 HcmV?d00001 diff --git a/testing/integration/tests/redis-bq-dataflow/testdata/myentity.csv b/testing/integration/tests/redis-bq-dataflow/testdata/myentity.csv new file mode 100644 index 00000000000..16906eef985 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/testdata/myentity.csv @@ -0,0 +1,50 @@ +0,2018-01-01T00:00:00,0.16654998178974967,0.0779124915810051,8,1 +0,2018-01-01T00:01:00,0.35055416999061695,0.3789967915556205,4,7 +0,2018-01-01T00:02:00,0.922617931213872,0.3179716588679623,8,6 +0,2018-01-01T00:03:00,0.06582437841319277,0.05565087947895475,9,1 +0,2018-01-01T00:04:00,0.8470568422759338,0.21347178570025072,2,10 +0,2018-01-01T00:05:00,0.11361651376595083,0.610881579742112,8,10 +0,2018-01-01T00:06:00,0.3034691430518833,0.19830856710384326,6,2 +0,2018-01-01T00:07:00,0.8346843022317924,0.08544530941545558,2,4 +0,2018-01-01T00:08:00,0.012367592836242891,0.5726466047910151,2,5 +0,2018-01-01T00:09:00,0.9887391081292758,0.9819570408280592,9,10 +1,2018-01-01T00:00:00,0.7503367819099366,0.946197160138704,10,1 +1,2018-01-01T00:01:00,0.9190195271598668,0.8836270958333856,10,4 +1,2018-01-01T00:02:00,0.07725768926853083,0.448071643977631,4,9 +1,2018-01-01T00:03:00,0.4965090410288343,0.9615236179532041,3,5 +1,2018-01-01T00:04:00,0.4838512691935104,0.355736320341819,7,2 +1,2018-01-01T00:05:00,0.7422860280929264,0.26936605638140254,7,6 +1,2018-01-01T00:06:00,0.007120660156963887,0.9457195989857463,5,4 +1,2018-01-01T00:07:00,0.5165841927879966,0.5734751201042606,1,4 +1,2018-01-01T00:08:00,0.0874458350429409,0.3284354801066641,3,8 +1,2018-01-01T00:09:00,0.8091561596661963,0.5176044026790632,1,9 +2,2018-01-01T00:00:00,0.5440258526780313,0.7936189500778384,3,5 +2,2018-01-01T00:01:00,0.9012681719733718,0.408129175696462,5,8 +2,2018-01-01T00:02:00,0.1357239530601766,0.2882272639567518,9,9 +2,2018-01-01T00:03:00,0.5482100359374814,0.22784885285759426,2,10 +2,2018-01-01T00:04:00,0.47915480013638667,0.5783117797172574,6,2 +2,2018-01-01T00:05:00,0.08954537587399436,0.6036422156720167,2,5 +2,2018-01-01T00:06:00,0.022254256710355302,0.8425123292474225,2,2 +2,2018-01-01T00:07:00,0.2981277839268307,0.7187791198561501,9,6 +2,2018-01-01T00:08:00,0.7483695369918476,0.24681429759605789,10,3 +2,2018-01-01T00:09:00,0.1357618414286158,0.887316489600256,3,10 +3,2018-01-01T00:00:00,0.24835796330732807,0.8871182399308823,3,2 +3,2018-01-01T00:01:00,0.18779166308706263,0.6635037319574626,4,10 +3,2018-01-01T00:02:00,0.00017995781865332017,0.9437703134427515,9,2 +3,2018-01-01T00:03:00,0.44519915810431854,0.03739113677349182,4,5 +3,2018-01-01T00:04:00,0.27156326671637077,0.06880899222977854,3,2 +3,2018-01-01T00:05:00,0.7647529493108539,0.14002956138831768,7,10 +3,2018-01-01T00:06:00,0.9954469835451967,0.7567907196365666,9,9 +3,2018-01-01T00:07:00,0.3883998745214877,0.18671618739049767,7,10 +3,2018-01-01T00:08:00,0.5448378857720906,0.38482161282383076,3,1 +3,2018-01-01T00:09:00,0.3550003641256,0.1818481760674323,1,9 +4,2018-01-01T00:00:00,0.5925452403226181,0.30212100652221296,5,10 +4,2018-01-01T00:01:00,0.18599533814587,0.5230622774213863,6,5 +4,2018-01-01T00:02:00,0.9599946262502064,0.26435543431553166,10,8 +4,2018-01-01T00:03:00,0.22755474767811312,0.6693510146123766,1,8 +4,2018-01-01T00:04:00,0.12063065422121622,0.7460800163440976,10,9 +4,2018-01-01T00:05:00,0.58262241980741,0.2714493926314999,9,5 +4,2018-01-01T00:06:00,0.26952820440313163,0.40841655204277727,5,1 +4,2018-01-01T00:07:00,0.7779897852625218,0.39955885342995734,9,3 +4,2018-01-01T00:08:00,0.9892858266244913,0.91265457567995,2,10 +4,2018-01-01T00:09:00,0.31825556885425366,0.6727958709215186,3,9 diff --git a/testing/integration/tests/redis-bq-dataflow/tf/backend.tf b/testing/integration/tests/redis-bq-dataflow/tf/backend.tf new file mode 100644 index 00000000000..fb1f9c81b5b --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = "kf-feast-terraform-state" + prefix = "tf/k8s-cluster/integration-test" + } +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/data.tf b/testing/integration/tests/redis-bq-dataflow/tf/data.tf new file mode 100644 index 00000000000..a988612c59d --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/data.tf @@ -0,0 +1 @@ +data "google_client_config" "current" {} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/input.tf b/testing/integration/tests/redis-bq-dataflow/tf/input.tf new file mode 100644 index 00000000000..bb653c213a5 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/input.tf @@ -0,0 +1,3 @@ +variable "docker_tag" { + description = "Docker image to deploy" +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf new file mode 100644 index 00000000000..fadca6442a6 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -0,0 +1,70 @@ +locals { + project_name = "kf-feast" + region = "us-central1" + subnetwork = "regions/${local.region}/subnetworks/default" + network = "default" + cluster_name = "it-feast" + + job_runner_options = { + "project" = "${local.project_name}" + "region" = "${local.region}" + "tempLocation" = "gs://${local.cluster_name}-storage/tempJob" + "subnetwork" = "${local.subnetwork}" + "maxNumWorkers" = "64" + "autoscalingAlgorithm" = "THROUGHPUT_BASED" + } + + errors_store_options = { + "path" = "gs://${local.cluster_name}-storage/errors" + } +} + +module "cluster" { + source = "../../../tf/modules/cluster" + cluster_name = "${local.cluster_name}" + project_name = "${local.project_name}" + region = "${local.region}" + bucket_name = "${local.cluster_name}-storage" + network = "${local.network}" + subnetwork = "${local.subnetwork}" + gke_machine_type = "n1-standard-8" +} + +resource "null_resource" "wait_for_regional_cluster" { + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${local.project_name} ${local.cluster_name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${local.project_name} ${local.cluster_name}" + } + + depends_on = ["module.cluster"] +} + +module "feast" { + source = "../../../tf/modules/feast-helm" + project_name = "${local.project_name}" + region = "${local.region}" + subnetwork = "${local.subnetwork}" + docker_tag = "${var.docker_tag}" + core_address = "10.128.0.99" + serving_address = "10.128.0.100" + redis_address = "10.128.0.101" + load_balancer_source_range = "10.0.0.0/8" + job_runner = "DataflowRunner" + job_runner_options = "'${jsonencode(local.job_runner_options)}'" + bucket_name = "${local.cluster_name}-storage" + bq_dataset = "${google_bigquery_dataset.feast_bq_dataset.dataset_id}" + + depends_on = ["module.cluster.cluster_name", "null_resource.wait_for_regional_cluster"] +} + +resource "google_bigquery_dataset" "feast_bq_dataset" { + dataset_id = "feast_it" + description = "Feast integration test dataset" + default_table_expiration_ms = 36000000 + location = "US" + delete_contents_on_destroy = true +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/provider.tf b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf new file mode 100644 index 00000000000..91f4c3caca4 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf @@ -0,0 +1,16 @@ +provider "google" { + version = "2.1.0" + # credentials = "${file("~/.secrets/${local.project_name}.json")}" + project = "${local.project_name}" +} + +provider "helm" { + kubernetes { + host = "${module.cluster.endpoint}" + token = "${data.google_client_config.current.access_token}" + + client_certificate = "${base64decode(module.cluster.client_certificate)}" + client_key = "${base64decode(module.cluster.client_key)}" + cluster_ca_certificate = "${base64decode(module.cluster.cluster_ca_certificate)}" + } +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh b/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh new file mode 100755 index 00000000000..5abeb004abd --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +PROJECT_ID=$1 +DATASET_NAME=$2 +for i in $(bq ls $PROJECT_ID:$DATASET_NAME | sed 1,2d | awk "{print \$1}"); +do + if [ ! -z "$i" ]; then + bq rm -ft $PROJECT_ID:$DATASET_NAME.$i; + fi +done \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh b/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh new file mode 100755 index 00000000000..aba1dc2450c --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh @@ -0,0 +1,36 @@ + #!/bin/bash + + # Copyright 2018 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + +# from https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/blob/176ad6f47d2d92ac9a49ef112830da2f539b737d/scripts/wait-for-cluster.sh +set -e + +PROJECT=$1 +CLUSTER_NAME=$2 +gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" +jq_query=".[] | select(.name==\"$CLUSTER_NAME\") | .status" + +echo "Waiting for cluster $2 in project $1 to reconcile..." + +current_status=$($gcloud_command | jq -r "$jq_query") + +while [ "${current_status}" = "RECONCILING" ]; do + printf "." + sleep 5 + current_status=$($gcloud_command | jq -r "$jq_query") +done + +echo "Cluster is ready!" \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/tf/values.yaml b/testing/integration/tests/redis-bq-dataflow/tf/values.yaml new file mode 100755 index 00000000000..0a21749c675 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/values.yaml @@ -0,0 +1,101 @@ +--- +core: + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-core + tag: "integration-test" + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 1 + memory: 4G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6565 + targetPort: 6565 + http: + port: 80 + targetPort: 8080 + jobs: + workspace: "it-feast-storage/workspace" + runner: "DataflowRunner" + options: '{"autoscalingAlgorithm":"THROUGHPUT_BASED","maxNumWorkers":"64","project":"kf-feast","region":"us-central1","subnetwork":"regions/us-central1/subnetworks/default","tempLocation":"gs://it-feast-storage/tempJob"}' + monitoring: + period: 5000 + initialDelay: 60000 +dataflow: + projectID: "kf-feast" + location: "us-central1" + +postgresql: + name: feast-metadata + service: + port: 5432 + postgresPassword: password + +postgresql: + provision: true + imageTag: 9.6.11 + service: + port: 5432 + +redis: + name: feast-redis + provision: true + master: + service: + type: LoadBalancer + annotations: + "cloud.google.com/load-balancer-type": "Internal" + +serving: + config: + maxEntityPerBatch: 2000 + maxNumberOfThread: 1024 + redisPool: + maxIdle: 16 + maxSize: 1024 + timeout: 1 + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-serving + tag: "integration-test" + jaeger: + enabled: false + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 2 + memory: 2G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6566 + targetPort: 6566 + http: + port: 80 + targetPort: 8080 + +store: + errors: + type: "file.json" + options: '{"path":"gs://it-feast-storage/error-log"}' + warehouse: + type: "bigquery" + options: '{"dataset":"feast_it","project":"kf-feast"}' + serving: + type: "redis" + options: '{"host":"10.128.0.101","port":"6379"}' \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml new file mode 100644 index 00000000000..7ea30c49736 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -0,0 +1,233 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: feast-integration-test- +spec: + entrypoint: run + onExit: exit-handler + arguments: + parameters: + - name: revision + value: integration-test + - name: prId + value: "none" + - name: gitRef + value: "none" + templates: + - name: run + steps: + - - name: get-feast + template: get-feast + - - name: build-jars + template: build-jars + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - name: build-cli + template: build-cli + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - - name: build-push-docker + template: build-docker + arguments: + artifacts: + - name: feast + from: "{{steps.build-jars.outputs.artifacts.feast-build}}" + - - name: terraform-provision + template: terraform + arguments: + parameters: + - name: arg + value: "apply -var docker_tag=integration-test -auto-approve ." + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - name: feast-cli + from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" + - - name: test + template: run-test + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - name: feast-cli + from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" + + - name: exit-handler + steps: + - - name: terraform-destroy + template: terraform-destroy + arguments: + parameters: + - name: arg + value: "destroy -var docker_tag=integration-test -auto-approve ." + artifacts: + - name: feast + from: "{{workflow.outputs.artifacts.feast-repo}}" + - name: get-feast + steps: + - - name: get-feast-by-pullrequest + template: get-feast-by-pullrequest + when: "{{workflow.parameters.prId}} != none" + - name: get-feast-by-gitref + template: get-feast-by-gitref + when: "{{workflow.parameters.gitRef}} != none" + outputs: + artifacts: + - name: feast-repo + from: "{{workflow.outputs.artifacts.feast-repo}}" + globalName: feast-repo + - name: get-feast-by-pullrequest + container: + image: alpine/git + command: + - /bin/sh + - -c + - | + git clone https://github.com/gojek/feast; + cd feast; + git fetch origin pull/{{workflow.parameters.prId}}/head; + git checkout FETCH_HEAD + outputs: + artifacts: + - name: feast + path: /git/feast + globalName: feast-repo + - name: get-feast-by-gitref + container: + image: alpine/git + command: + - /bin/sh + - -c + - | + git clone https://github.com/gojek/feast; + cd feast; + git checkout {{workflow.parameters.gitRef}} + outputs: + artifacts: + - name: feast + path: /git/feast + globalName: feast-repo + - name: build-jars + inputs: + artifacts: + - name: feast + path: /feast + container: + image: maven:3.6.0-jdk-8-slim + command: ["/bin/bash", "-c"] + args: + - mvn package -DskipTests -Drevision=integration-test -T 6 -DdependencyLocationsEnabled=false + workingDir: /feast + outputs: + artifacts: + - name: feast-build + path: /feast + globalName: feast-build + - name: build-docker + inputs: + artifacts: + - name: feast + path: /feast + container: + image: gcr.io/kf-feast/docker:18.09-gcloud + command: [sh, -c] + args: [' + until docker ps; do sleep 3; done; + docker build -t gcr.io/kf-feast/feast-core:integration-test -f docker/core/Dockerfile --build-arg RELEASE=integration-test .; + docker build -t gcr.io/kf-feast/feast-serving:integration-test -f docker/serving/Dockerfile --build-arg RELEASE=integration-test .; + docker push gcr.io/kf-feast/feast-core:integration-test; + docker push gcr.io/kf-feast/feast-serving:integration-test + '] + env: + - name: DOCKER_HOST + value: 127.0.0.1 + workingDir: /feast + sidecars: + - name: dind + image: docker:stable-dind + securityContext: + privileged: true + mirrorVolumeMounts: true + - name: build-cli + inputs: + artifacts: + - name: feast + path: /build/feast + container: + image: golang:1.12 + command: ["/bin/bash", "-c"] + args: + - | + GOOS=linux go build -o feast-cli-v0.0.0-linux-amd64 ./cli/feast + GOOS=linux go build -o feast-cli-v0.0.0-darwin-amd64 ./cli/feast + # Fix the feast cli binary name for use by subsequent steps + cp feast-cli-v0.0.0-linux-amd64 /usr/local/bin/feast + workingDir: /build/feast + outputs: + artifacts: + - name: feast-cli + path: /usr/local/bin/feast + - name: terraform + inputs: + parameters: + - name: arg + artifacts: + - name: feast + path: /feast + - name: feast-cli + path: /bin/feast + container: + image: gcr.io/kf-feast/it-terraform:0.1.0 + workingDir: /feast/testing/integration/tests/redis-bq-dataflow/tf + command: ["sh", "-c"] + args: + - | + terraform init + terraform {{inputs.parameters.arg}} + - name: run-test + inputs: + artifacts: + - name: feast-cli + path: /bin/feast + - name: feast + path: /feast + container: + image: python:3.7.2 + command: ["/bin/bash", "-c"] + args: [' + feast config set coreURI 10.128.0.99:6565; + pip install pytest; + pip install -r /feast/sdk/python/test-requirements.txt; + pip install -e /feast/sdk/python; + pytest --capture=no + '] + workingDir: /feast/testing/integration/tests/redis-bq-dataflow + env: + - name: FEAST_CORE_URL + value: "10.128.0.99:6565" + - name: FEAST_SERVING_URL + value: "10.128.0.100:6566" + - name: PROJECT_ID + value: kf-feast + - name: BUCKET_NAME + value: it-feast-storage + - name: terraform-destroy + inputs: + parameters: + - name: arg + artifacts: + - name: feast + path: /feast + container: + image: gcr.io/kf-feast/it-terraform:0.1.0 + workingDir: /feast/testing/integration/tests/redis-bq-dataflow/tf + command: ["sh", "-c"] + args: [' + terraform init; + terraform {{inputs.parameters.arg}} + '] + diff --git a/testing/integration/tf/modules/cluster/input.tf b/testing/integration/tf/modules/cluster/input.tf new file mode 100644 index 00000000000..bccca70fbbc --- /dev/null +++ b/testing/integration/tf/modules/cluster/input.tf @@ -0,0 +1,39 @@ +variable "cluster_name" { + default = "feast" + description = "Cluster name" +} + +variable "project_name" { + description = "Project name" +} + +variable "region" { + description = "Subnet region" +} + +variable "network" { + default = "default" +} + +variable "subnetwork" { + default = "default" +} + +variable "gke_machine_type" { + description = "The machine type for the default node pool" + default = "n1-standard-4" +} + +variable "bucket_name" { + description = "Working storage for feast" +} + +variable "feast_node_pool_min_size" { + description = "Minimum number of nodes for node pool" + default = 1 +} + +variable "feast_node_pool_max_size" { + description = "Maximum number of nodes for node pool" + default = 2 +} diff --git a/testing/integration/tf/modules/cluster/main.tf b/testing/integration/tf/modules/cluster/main.tf new file mode 100644 index 00000000000..23b66241a42 --- /dev/null +++ b/testing/integration/tf/modules/cluster/main.tf @@ -0,0 +1,42 @@ +data "google_client_config" "current" {} + +resource "google_storage_bucket" "feast_storage_bucket" { + name = "${var.bucket_name}" + location = "${var.region}" + force_destroy = true +} + +resource "google_container_cluster" "feast_k8s" { + name = "${var.cluster_name}" + region = "${var.region}" + initial_node_count = 1 + + # node_version = "1.11.5-gke.5" + # min_master_version = "1.11.5-gke.5" + network = "${var.network}" + + subnetwork = "${var.subnetwork}" + + enable_legacy_abac = true + + node_config { + machine_type = "${var.gke_machine_type}" + disk_size_gb = "50" + + oauth_scopes = [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigtable.admin", + "https://www.googleapis.com/auth/bigtable.data", + ] + } + + provisioner "local-exec" { + when = "destroy" + command = "sleep 90" + } +} diff --git a/testing/integration/tf/modules/cluster/output.tf b/testing/integration/tf/modules/cluster/output.tf new file mode 100644 index 00000000000..6b7af14886f --- /dev/null +++ b/testing/integration/tf/modules/cluster/output.tf @@ -0,0 +1,33 @@ + + +output "endpoint" { + value = "${google_container_cluster.feast_k8s.endpoint}" +} + +output "cluster_name" { + value = "${var.cluster_name}" +} + +output "client_certificate" { + value = "${google_container_cluster.feast_k8s.master_auth.0.client_certificate}" +} + +output "client_key" { + value = "${google_container_cluster.feast_k8s.master_auth.0.client_key}" +} + +output "cluster_ca_certificate" { + value = "${google_container_cluster.feast_k8s.master_auth.0.cluster_ca_certificate}" +} + +output "username" { + value = "${google_container_cluster.feast_k8s.master_auth.0.username}" +} + +output "password" { + value = "${google_container_cluster.feast_k8s.master_auth.0.password}" +} + +output "feast_storage_bucket" { + value = "${google_storage_bucket.feast_storage_bucket.name}" +} \ No newline at end of file diff --git a/testing/integration/tf/modules/feast-helm/input.tf b/testing/integration/tf/modules/feast-helm/input.tf new file mode 100644 index 00000000000..6652bac56f4 --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/input.tf @@ -0,0 +1,61 @@ +variable "project_name" { + description = "Project name" +} + +variable "subnetwork" { + description = "Desired subnetwork" +} + +variable "region" { + description = "Subnet region" +} + +variable "docker_tag" { + default = "0.1.0" + description = "Feast build version" +} + +variable "core_address" { + description = "Core internal address" +} + +variable "serving_address" { + description = "Serving internal address" +} + +variable "redis_address" { + description = "Redis internal address" +} + +variable "statsd_host" { + description = "Statsd host to write metrics to" + default = "" +} + +variable "job_runner" { + description = "Desired job runner" + default = "Dataflow" +} + +variable "job_runner_options" { + description = "Job runner options as a json string" + default = "'{}'" +} + +variable "load_balancer_source_range" { + description = "ingress filter for google internal load balancer" +} + +variable "bq_dataset" { + description = "BigQuery dataset for warehouse" +} + +variable depends_on { + default = [] + + type = "list" +} + +variable "bucket_name" { + description = "Working storage for feast" +} \ No newline at end of file diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf new file mode 100644 index 00000000000..687d392fe98 --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -0,0 +1,100 @@ + +locals { + warehouse_option = { + "project" = "${var.project_name}" + "dataset" = "${var.bq_dataset}" + } + serving_option = { + "host" = "${var.redis_address}" + "port" = "6379" + } + store_error_option = { + "path" = "gs://${var.bucket_name}/error-log" + } +} + +resource "google_compute_address" "core_address" { + project = "${var.project_name}" + name = "feast-core-ip" + subnetwork = "${var.subnetwork}" + address_type = "INTERNAL" + address = "${var.core_address}" + region = "${var.region}" +} + +resource "google_compute_address" "serving_address" { + project = "${var.project_name}" + name = "feast-serving-ip" + subnetwork = "${var.subnetwork}" + address_type = "INTERNAL" + address = "${var.serving_address}" + region = "${var.region}" +} + +resource "template_file" "helm_values" { + template = "${file("${path.module}/values.tmpl")}" + + vars = { + docker_tag = "${var.docker_tag}" + job_runner = "${var.job_runner}" + job_runner_options = "${var.job_runner_options}" + store_warehouse_option = "'${jsonencode(local.warehouse_option)}'" + store_error_option = "'${jsonencode(local.store_error_option)}'" + store_serving_option = "'${jsonencode(local.serving_option)}'" + project_id = "${var.project_name}" + region = "${var.region}" + workspace = "gs://${var.bucket_name}/workspace" + } +} + +resource "local_file" "helm_values_output" { + content = "${template_file.helm_values.rendered}" + filename = "values.yaml" + + depends_on = ["template_file.helm_values"] +} + +resource "helm_release" "feast" { + name = "feast-it" + chart = "/feast/charts/feast" + + set { + name = "core.service.extIPAdr" + value = "${google_compute_address.core_address.address}" + } + + set { + name = "redis.master.service.loadBalancerIP" + value = "${var.redis_address}" + } + + set { + name = "core.service.loadBalancerSourceRanges[0]" + value = "${var.load_balancer_source_range}" + } + + set { + name = "serving.service.extIPAdr" + value = "${google_compute_address.serving_address.address}" + } + + set { + name = "serving.service.loadBalancerSourceRanges[0]" + value = "${var.load_balancer_source_range}" + } + + set { + name = "statsd.host" + value = "${var.statsd_host}" + } + + values = [ + "${template_file.helm_values.rendered}", + ] + + timeout = 600 + + depends_on = [ + "local_file.helm_values_output", + ] +} diff --git a/testing/integration/tf/modules/feast-helm/output.tf b/testing/integration/tf/modules/feast-helm/output.tf new file mode 100644 index 00000000000..2d0e8eef07e --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/output.tf @@ -0,0 +1,11 @@ +output "core_url" { + value = "${google_compute_address.core_address.address}" +} + +output "serving_url" { + value = "${google_compute_address.serving_address.address}" +} + +output "redis_url" { + value = "${var.redis_address}" +} \ No newline at end of file diff --git a/testing/integration/tf/modules/feast-helm/values.tmpl b/testing/integration/tf/modules/feast-helm/values.tmpl new file mode 100644 index 00000000000..ab1fe95ca37 --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/values.tmpl @@ -0,0 +1,105 @@ +--- +core: + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-core + tag: "${docker_tag}" + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 1 + memory: 4G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6565 + targetPort: 6565 + http: + port: 80 + targetPort: 8080 + jobs: + workspace: "${workspace}" + runner: "${job_runner}" + options: ${job_runner_options} + monitoring: + period: 5000 + initialDelay: 60000 + trainingDatasetPrefix: "fs" + projectId: "kf-feast" + logType: JSON +dataflow: + projectID: "${project_id}" + location: "${region}" + +postgresql: + name: feast-metadata + service: + port: 5432 + postgresPassword: password + +postgresql: + provision: true + imageTag: 9.6.11 + service: + port: 5432 + +redis: + name: feast-redis + provision: true + master: + service: + type: LoadBalancer + annotations: + "cloud.google.com/load-balancer-type": "Internal" + usePassword: false + +serving: + config: + maxEntityPerBatch: 2000 + maxNumberOfThread: 1024 + redisPool: + maxIdle: 16 + maxSize: 1024 + timeout: 1 + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-serving + tag: "${docker_tag}" + jaeger: + enabled: false + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 2 + memory: 2G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6566 + targetPort: 6566 + http: + port: 80 + targetPort: 8080 + +store: + errors: + type: "file.json" + options: ${store_error_option} + warehouse: + type: "bigquery" + options: ${store_warehouse_option} + serving: + type: "redis" + options: ${store_serving_option} \ No newline at end of file diff --git a/testing/prow/config.yaml b/testing/prow/config.yaml index 7289d1372a5..6ff78541f17 100644 --- a/testing/prow/config.yaml +++ b/testing/prow/config.yaml @@ -7,6 +7,14 @@ presubmits: - image: us.gcr.io/kf-feast/test-image:1.0.0 args: ["make", "test"] imagePullPolicy: Always + - name: presubmit-integration-test-batch + always_run: false + spec: + containers: + - image: us.gcr.io/kf-feast/gcloud-argo:1.0.0 + args: ["make", "test-integration", "ID=$PULL_NUMBER", "TYPE=PR"] + imagePullPolicy: Always + trigger: integration-batch postsubmits: gojek/feast: - name: postsubmit-unit-tests @@ -16,7 +24,14 @@ postsubmits: - image: us.gcr.io/kf-feast/test-image:1.0.0 args: ["make", "test"] imagePullPolicy: Always - + - name: postsubmit-integration-test-batch + always_run: true + spec: + containers: + - image: us.gcr.io/kf-feast/gcloud-argo:1.0.0 + args: ["make", "test-integration", "ID=$PULL_BASE_REF", "TYPE=GITREF"] + imagePullPolicy: Always + trigger: integration-batch tide: queries: - repos: diff --git a/testing/tf/cloud-build/cloudbuild_master.yaml b/testing/tf/cloud-build/cloudbuild_master.yaml deleted file mode 100644 index 743e7588a52..00000000000 --- a/testing/tf/cloud-build/cloudbuild_master.yaml +++ /dev/null @@ -1,19 +0,0 @@ -steps: -- name: maven:3.6.0-jdk-8-slim - args: ['mvn', 'clean', 'install', '-Drevision=$COMMIT_SHA', '-DskipTests'] - id: build-jars - -- name: "gcr.io/cloud-builders/docker" - args: ['build', '-t', 'gcr.io/$PROJECT_ID/feast-core:$COMMIT_SHA', '--build-arg', 'RELEASE=$COMMIT_SHA', '-f', './docker/core/Dockerfile', '.'] - id: docker-core - waitFor: - - build-jars - -- name: "gcr.io/cloud-builders/docker" - args: ['build', '-t', 'gcr.io/$PROJECT_ID/feast-serving:$COMMIT_SHA', '--build-arg', 'RELEASE=$COMMIT_SHA', '-f', './docker/serving/Dockerfile', '.'] - id: docker-serving - waitFor: - - build-jars - -timeout: 1200s -images: ['gcr.io/$PROJECT_ID/feast-core:$COMMIT_SHA', 'gcr.io/$PROJECT_ID/feast-serving:$COMMIT_SHA'] \ No newline at end of file diff --git a/testing/tf/terraform.tfvars b/testing/tf/terraform.tfvars deleted file mode 100644 index a8b45f64116..00000000000 --- a/testing/tf/terraform.tfvars +++ /dev/null @@ -1,7 +0,0 @@ -gcp_project = "kf-feast" - -default_network = "default" - -default_subnet = "default" - -default_region = "us-central1"