diff --git a/.prow/scripts/sync-helm-charts.sh b/.prow/scripts/sync-helm-charts.sh new file mode 100755 index 00000000000..50a27bb588f --- /dev/null +++ b/.prow/scripts/sync-helm-charts.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Script to sync local charts to remote helm repository in Google Cloud Storage +# Copied from: https://github.com/helm/charts/blob/master/test/repo-sync.sh + +set -o errexit +set -o nounset +set -o pipefail + +log_error() { + printf '\e[31mERROR: %s\n\e[39m' "$1" >&2 +} + +# Assume working directory is Feast repository root folder +repo_dir=charts +bucket=gs://feast-charts +repo_url=https://feast-charts.storage.googleapis.com/ +sync_dir=/tmp/syncdir +index_dir=/tmp/indexdir + +rm -rf $sync_dir $index_dir + +echo "Syncing repo '$repo_dir'..." + +mkdir -p "$sync_dir" +if ! gsutil cp "$bucket/index.yaml" "$index_dir/index.yaml"; then + log_error "Exiting because unable to copy index locally. Not safe to proceed." + exit 1 +fi + +exit_code=0 + +for dir in "$repo_dir"/*; do + if helm dependency build "$dir"; then + helm package --destination "$sync_dir" "$dir" + else + log_error "Problem building dependencies. Skipping packaging of '$dir'." + exit_code=1 + fi +done + +if helm repo index --url "$repo_url" --merge "$index_dir/index.yaml" "$sync_dir"; then + # Move updated index.yaml to sync folder so we don't push the old one again + mv -f "$sync_dir/index.yaml" "$index_dir/index.yaml" + + gsutil -m rsync "$sync_dir" "$bucket" + + # Make sure index.yaml is synced last + gsutil cp "$index_dir/index.yaml" "$bucket" +else + log_error "Exiting because unable to update index. Not safe to push update." + exit 1 +fi + +ls -l "$sync_dir" + +exit "$exit_code" \ No newline at end of file diff --git a/charts/feast/Chart.yaml b/charts/feast/Chart.yaml index 9eb45964a62..6cc6b68c837 100644 --- a/charts/feast/Chart.yaml +++ b/charts/feast/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "0.1.0" +appVersion: "0.1.7" description: A Helm chart to install Feast on kubernetes name: feast -version: 0.1.0 +version: 0.1.4 diff --git a/charts/README.md b/charts/feast/README.md similarity index 87% rename from charts/README.md rename to charts/feast/README.md index 88edc6f4ae0..1b75bc7c5e0 100644 --- a/charts/README.md +++ b/charts/feast/README.md @@ -4,8 +4,13 @@ This chart adds all the components required to run feast, sans the stores to whi ## Installing the chart ``` -helm dep update -helm install --name feast . +# Add feast-charts repository +helm repo add feast-charts https://feast-charts.storage.googleapis.com +helm repo update + +# Install Feast chart with release name "feast-release" and optional custom +# values "value-feast.yaml" +helm install --name feast-release feast-charts/feast [-f value-feast.yaml] ``` ## Requirements @@ -41,7 +46,6 @@ Components that Feast supports, but this installation will not include are: - Note that if you do not provision a metrics store, feast will only retain the latest metrics from your jobs. - [Jaeger tracing](www.jaegertracing.io) for serving performance. - Set `serving.jaeger.enabled` to `true`, and configure the following parameters: - - `serving.jaeger.host` - `serving.jaeger.port` - `serving.jaeger.options.samplerType` - `serving.jaeger.options.samplerParam` @@ -67,6 +71,10 @@ The following table lists the configurable parameters of the Feast chart and the | `core.jobs.options` | additional options to be provided to the beam job. Should be a char escaped json k-v object | {} | | `core.jobs.runner` | beam job runner - one of `DirectRunner`, `FlinkRunner` or `DataflowRunner` | DirectRunner | | `core.jobs.workspace` | workspace path for ingestion jobs, used for separate job workspaces to share importJobSpecs.yaml with ingestion and for writing errors to if no default errors store is configured | nil | +| `core.jobs.writeFeatureMetricsToInfluxDb` | specifies whether Feast import job will write feature metrics (such as feature lag and values summaries) to Influx DB for monitoring and alert purpose | false | +| `core.jobs.influxDbUrl` | Influx DB url e.g. http://localhost:8086 (required if `core.jobs.writeFeatureMetricsToInfluxDb = true`) | | +| `core.jobs.influxDbDatabase` | Influx DB database name (required if `core.jobs.writeFeatureMetricsToInfluxDb = true`) | | +| `core.jobs.influxDbMeasurement` | Influx DB [measurement name](https://docs.influxdata.com/influxdb/v1.7/concepts/key_concepts/#measurement) (required if `core.jobs.writeFeatureMetricsToInfluxDb = true`) | | | `core.replicaCount` | core deployment replica count | 3 | | `core.resources.limits.cpu` | core cpu limits | 1 | | `core.resources.limits.memory` | core memory limits | 2G | diff --git a/charts/feast/templates/core-deploy.yaml b/charts/feast/templates/core-deploy.yaml index 74b625de029..8825a3515bd 100644 --- a/charts/feast/templates/core-deploy.yaml +++ b/charts/feast/templates/core-deploy.yaml @@ -58,12 +58,7 @@ spec: timeoutSeconds: 3 failureThreshold: {{ .Values.core.readinessProbe.failureThreshold }} resources: - requests: - cpu: {{ .Values.core.resources.requests.cpu }} - memory: {{ .Values.core.resources.requests.memory }} - limits: - cpu: {{ .Values.core.resources.limits.cpu }} - memory: {{ .Values.core.resources.limits.memory }} +{{ toYaml .Values.core.resources | indent 10 }} {{- if .Values.serviceAccount }} volumeMounts: - name: "{{ .Values.serviceAccount.name }}" @@ -98,6 +93,18 @@ spec: value: "{{ .Values.core.jobs.monitoring.period }}" - name: JOB_MONITOR_INITIAL_DELAY_MS value: "{{ .Values.core.jobs.monitoring.initialDelay }}" + + {{- if .Values.core.jobs.writeFeatureMetricsToInfluxDb }} + - name: INGESTION_METRICS_ENABLED + value: "{{ .Values.core.jobs.writeFeatureMetricsToInfluxDb }}" + - name: INGESTION_METRICS_INFLUX_URL + value: {{ .Values.core.jobs.influxDbUrl }} + - name: INGESTION_METRICS_INFLUX_DB_NAME + value: {{ .Values.core.jobs.influxDbDatabase }} + - name: INGESTION_METRICS_INFLUX_DB_MEASUREMENT + value: {{ .Values.core.jobs.influxDbMeasurement }} + {{- end }} + {{- if .Values.store }} {{- if .Values.store.serving }} - name: STORE_SERVING_TYPE diff --git a/charts/feast/templates/serving-deploy.yaml b/charts/feast/templates/serving-deploy.yaml index b84aca8c716..470117fabff 100644 --- a/charts/feast/templates/serving-deploy.yaml +++ b/charts/feast/templates/serving-deploy.yaml @@ -59,12 +59,7 @@ spec: timeoutSeconds: 3 failureThreshold: {{ .Values.serving.readinessProbe.failureThreshold }} resources: - requests: - cpu: "{{ .Values.serving.resources.requests.cpu }}" - memory: "{{ .Values.serving.resources.requests.memory }}" - limits: - cpu: "{{ .Values.serving.resources.limits.cpu }}" - memory: "{{ .Values.serving.resources.limits.memory }}" +{{ toYaml .Values.serving.resources | indent 10 }} env: - name: FEAST_SERVING_HTTP_PORT value: "{{ .Values.serving.service.http.targetPort }}" @@ -96,7 +91,9 @@ spec: - name: JAEGER_ENABLED value: "{{ .Values.serving.jaeger.enabled }}" - name: JAEGER_AGENT_HOST - value: "{{ .Values.serving.jaeger.host }}" + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: JAEGER_AGENT_PORT value: "{{ .Values.serving.jaeger.port }}" - name: JAEGER_SAMPLER_TYPE diff --git a/charts/feast/values.yaml b/charts/feast/values.yaml index 82ed93c9778..b63d8b73053 100644 --- a/charts/feast/values.yaml +++ b/charts/feast/values.yaml @@ -1,16 +1,26 @@ ---- +# [required] +# global.postgresql.secretName is an existing Kubernetes secret name containing Postgresql password +# The secret needs to have this key: postgresql-password +# +# Example of how to generate this secret: +# kubectl create secret generic -postgresql \ +# --from-literal=postgresql-password= +# +# With the example above the secretName will be -postgresql +# +# global: +# postgresql: +# secretName: + core: projectId: "gcp-project-id" image: pullPolicy: IfNotPresent registry: gcr.io/kf-feast repository: feast-core - tag: "0.1.1" + tag: "0.1.7" replicaCount: 1 resources: - limits: - cpu: 4 - memory: 6G requests: cpu: 1 memory: 2G @@ -38,6 +48,14 @@ core: monitoring: period: 5000 initialDelay: 60000 + # writeFeatureMetricsToInfluxDb specifies whether Feast import job will write feature metrics (such as feature lag and values summaries) to Influx DB for monitoring and alert purpose + writeFeatureMetricsToInfluxDb: false + # influxDbUrl, influxDbDatabase and influxDbMeasurement sets the Influx DB configuration where Feast import job will write the feature metrics + # Uncomment the following 3 fields if writeFeatureMetricsToInfluxDb = true + # + # influxDbUrl: http://localhost:8086 + # influxDbDatabase: influx_db_database + # influxDbMeasurement: influx_db_measurement trainingDatasetPrefix: "fs" # logType: JSON livenessProbe: @@ -84,12 +102,9 @@ serving: pullPolicy: IfNotPresent registry: gcr.io/kf-feast repository: feast-serving - tag: "0.1.1" + tag: "0.1.7" replicaCount: 1 resources: - limits: - cpu: 2 - memory: 4G requests: cpu: 1 memory: 1G @@ -109,6 +124,9 @@ serving: # loadBalancerSourceRanges: ["10.0.0.0/8"] jaeger: enabled: false + # options: + # samplerType: constants + # samplerParam: 1 livenessProbe: initialDelaySeconds: 120 failureThreshold: 3 diff --git a/core/src/main/java/feast/core/config/ImportJobMetricsConfig.java b/core/src/main/java/feast/core/config/ImportJobMetricsConfig.java new file mode 100644 index 00000000000..7bbb3bbc72b --- /dev/null +++ b/core/src/main/java/feast/core/config/ImportJobMetricsConfig.java @@ -0,0 +1,13 @@ +package feast.core.config; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class ImportJobMetricsConfig { + private final boolean ingestionMetricsEnabled; + private final String influxDbUrl; + private final String influxDbName; + private final String influxDbMeasurementName; +} diff --git a/core/src/main/java/feast/core/config/InstrumentationConfig.java b/core/src/main/java/feast/core/config/InstrumentationConfig.java index eed24873556..d9b84b8b191 100644 --- a/core/src/main/java/feast/core/config/InstrumentationConfig.java +++ b/core/src/main/java/feast/core/config/InstrumentationConfig.java @@ -26,8 +26,17 @@ @Configuration public class InstrumentationConfig { @Bean - public StatsDClient getStatsDClient(@Value("${statsd.host}") String host, - @Value("${statsd.port}") int port) { + public StatsDClient getStatsDClient( + @Value("${statsd.host}") String host, @Value("${statsd.port}") int port) { return new NonBlockingStatsDClient("feast_core", host, port); } + + @Bean + public ImportJobMetricsConfig getImportJobMetricsConfig( + @Value("${ingestion.metrics.enabled}") boolean enabled, + @Value("${ingestion.metrics.influxUrl}") String influxDbUrl, + @Value("${ingestion.metrics.dbName}") String influxDbName, + @Value("${ingestion.metrics.dbMeasurement}") String influxDbMeasurementName) { + return new ImportJobMetricsConfig(enabled, influxDbUrl, influxDbName, influxDbMeasurementName); + } } diff --git a/core/src/main/java/feast/core/config/TrainingConfig.java b/core/src/main/java/feast/core/config/TrainingConfig.java index 9bcff052f1b..f560b13efff 100644 --- a/core/src/main/java/feast/core/config/TrainingConfig.java +++ b/core/src/main/java/feast/core/config/TrainingConfig.java @@ -14,6 +14,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; @@ -23,7 +24,7 @@ public class TrainingConfig { @Bean public BigQueryDatasetTemplater getBigQueryTrainingDatasetTemplater( - StorageSpecs storageSpecs, FeatureInfoRepository featureInfoRepository) throws IOException { + @Lazy StorageSpecs storageSpecs, FeatureInfoRepository featureInfoRepository) throws IOException { Resource resource = new ClassPathResource("templates/bq_training.tmpl"); InputStream resourceInputStream = resource.getInputStream(); String tmpl = CharStreams.toString(new InputStreamReader(resourceInputStream, Charsets.UTF_8)); diff --git a/core/src/main/java/feast/core/service/JobManagementService.java b/core/src/main/java/feast/core/service/JobManagementService.java index 5b2ee9217b6..003b0bcd53e 100644 --- a/core/src/main/java/feast/core/service/JobManagementService.java +++ b/core/src/main/java/feast/core/service/JobManagementService.java @@ -30,6 +30,7 @@ import com.google.protobuf.util.JsonFormat; import feast.core.JobServiceProto.JobServiceTypes.JobDetail; import feast.core.config.ImportJobDefaults; +import feast.core.config.ImportJobMetricsConfig; import feast.core.config.StorageConfig.StorageSpecs; import feast.core.dao.JobInfoRepository; import feast.core.dao.MetricsRepository; @@ -73,6 +74,7 @@ public class JobManagementService { private static final String JOB_PREFIX_DEFAULT = "feastimport"; private static final String UNKNOWN_EXT_JOB_ID = ""; private static final String IMPORT_JOB_SPECS_FILENAME = "importJobSpecs.yaml"; + private ImportJobMetricsConfig metricsConfig; private JobInfoRepository jobInfoRepository; private MetricsRepository metricsRepository; @@ -88,13 +90,15 @@ public JobManagementService( JobManager jobManager, ImportJobDefaults defaults, SpecService specService, - StorageSpecs storageSpecs) { + StorageSpecs storageSpecs, + ImportJobMetricsConfig metricsConfig) { this.jobInfoRepository = jobInfoRepository; this.metricsRepository = metricsRepository; this.jobManager = jobManager; this.defaults = defaults; this.specService = specService; this.storageSpecs = storageSpecs; + this.metricsConfig = metricsConfig; } public void writeImportJobSpecs(ImportJobSpecs importJobSpecs, Path workspace) { @@ -132,7 +136,11 @@ private ImportJobSpecs buildImportJobSpecs(ImportSpec importSpec, String jobId) .setJobId(jobId) .setImportSpec(importSpec) .addAllEntitySpecs(entitySpecs) - .addAllFeatureSpecs(featureSpecs); + .addAllFeatureSpecs(featureSpecs) + .setWriteFeatureMetricsToInfluxDb(metricsConfig.isIngestionMetricsEnabled()) + .setInfluxDbUrl(metricsConfig.getInfluxDbUrl()) + .setInfluxDbDatabase(metricsConfig.getInfluxDbName()) + .setInfluxDbMeasurement(metricsConfig.getInfluxDbMeasurementName()); if (storageSpecs.getServingStorageSpec() != null) { importJobSpecsBuilder.setServingStorageSpec(storageSpecs.getServingStorageSpec()); } diff --git a/core/src/main/resources/application.properties b/core/src/main/resources/application.properties index 58d8af33b3f..38553b5a13b 100644 --- a/core/src/main/resources/application.properties +++ b/core/src/main/resources/application.properties @@ -55,3 +55,8 @@ management.metrics.export.simple.enabled=false management.metrics.export.statsd.enabled=true management.metrics.export.statsd.host=${STATSD_HOST:localhost} management.metrics.export.statsd.port=${STATSD_PORT:8125} + +ingestion.metrics.enabled=${INGESTION_METRICS_ENABLED:false} +ingestion.metrics.influxUrl=${INGESTION_METRICS_INFLUX_URL:} +ingestion.metrics.dbName=${INGESTION_METRICS_INFLUX_DB_NAME:} +ingestion.metrics.dbMeasurement=${INGESTION_METRICS_INFLUX_DB_MEASUREMENT:} \ No newline at end of file diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java index 01f92fcd006..73113fcc821 100644 --- a/core/src/test/java/feast/core/CoreApplicationTest.java +++ b/core/src/test/java/feast/core/CoreApplicationTest.java @@ -52,7 +52,11 @@ "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", "feast.store.serving.type=redis", "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", - "feast.store.errors.type=stderr" + "feast.store.errors.type=stderr", + "ingestion.metrics.enabled=true", + "ingestion.metrics.influxUrl=localhost", + "ingestion.metrics.dbName=db", + "ingestion.metrics.dbMeasurement=measurement" }) @DirtiesContext public class CoreApplicationTest { @@ -138,6 +142,10 @@ public void test_withProperties_systemServingAndWarehouseStoresRegistered() thro .setId(DEFAULT_WAREHOUSE_ID) .setType("file.json") .putOptions("path", "/tmp/foobar")) + .setWriteFeatureMetricsToInfluxDb(true) + .setInfluxDbUrl("localhost") + .setInfluxDbDatabase("db") + .setInfluxDbMeasurement("measurement") .build(), args.get(0)); } diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java index 4b6cbb7838b..6e326d526c4 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java @@ -42,7 +42,11 @@ "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", - "feast.store.errors.type=stderr" + "feast.store.errors.type=stderr", + "ingestion.metrics.enabled=true", + "ingestion.metrics.influxUrl=localhost", + "ingestion.metrics.dbName=db", + "ingestion.metrics.dbMeasurement=measurement" }) @DirtiesContext public class CoreApplicationWithNoServingTest { diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java index 1acd238b8ba..d3b57226d84 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java @@ -51,7 +51,11 @@ "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.serving.type=redis", "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", - "feast.store.errors.type=stderr" + "feast.store.errors.type=stderr", + "ingestion.metrics.enabled=true", + "ingestion.metrics.influxUrl=localhost", + "ingestion.metrics.dbName=db", + "ingestion.metrics.dbMeasurement=measurement" }) @DirtiesContext public class CoreApplicationWithNoWarehouseTest { @@ -132,6 +136,10 @@ public void test_withProperties_systemServingAndWarehouseStoresRegistered() thro .setId(DEFAULT_SERVING_ID) .setType("redis") .putOptions("host", "localhost").putOptions("port", "1234")) + .setWriteFeatureMetricsToInfluxDb(true) + .setInfluxDbUrl("localhost") + .setInfluxDbDatabase("db") + .setInfluxDbMeasurement("measurement") .build(), args.get(0)); } diff --git a/core/src/test/java/feast/core/service/JobManagementServiceTest.java b/core/src/test/java/feast/core/service/JobManagementServiceTest.java index 71967f8da88..03d17d143d3 100644 --- a/core/src/test/java/feast/core/service/JobManagementServiceTest.java +++ b/core/src/test/java/feast/core/service/JobManagementServiceTest.java @@ -28,6 +28,7 @@ import com.google.protobuf.Timestamp; import feast.core.JobServiceProto.JobServiceTypes.JobDetail; import feast.core.config.ImportJobDefaults; +import feast.core.config.ImportJobMetricsConfig; import feast.core.config.StorageConfig.StorageSpecs; import feast.core.dao.JobInfoRepository; import feast.core.dao.MetricsRepository; @@ -62,6 +63,7 @@ public class JobManagementServiceTest { @Mock private SpecService specService; private StorageSpecs storageSpecs; + private ImportJobMetricsConfig metricsConfig; @Before public void setUp() { @@ -71,6 +73,8 @@ public void setUp() { .runner("DirectRunner").executable("/feast-import.jar").build(); storageSpecs = StorageSpecs.builder() .errorsStorageSpec(StorageSpec.newBuilder().setType("stderr").build()).build(); + + metricsConfig = new ImportJobMetricsConfig(true, "localhost", "db", "measurement"); } @Test @@ -108,7 +112,7 @@ public void shouldListAllJobDetails() { when(jobInfoRepository.findAll()).thenReturn(Lists.newArrayList(jobInfo1, jobInfo2)); JobManagementService jobManagementService = new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, - specService, storageSpecs); + specService, storageSpecs, metricsConfig); List actual = jobManagementService.listJobs(); List expected = Lists.newArrayList( @@ -147,7 +151,7 @@ public void shouldReturnDetailOfRequestedJobId() { when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(jobInfo1)); JobManagementService jobManagementService = new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, - specService, storageSpecs); + specService, storageSpecs, metricsConfig); JobDetail actual = jobManagementService.getJob("job1"); JobDetail expected = JobDetail.newBuilder() @@ -163,7 +167,7 @@ public void shouldReturnDetailOfRequestedJobId() { public void shouldThrowErrorIfJobIdNotFoundWhenGettingJob() { when(jobInfoRepository.findById("job1")).thenReturn(Optional.empty()); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs, metricsConfig); exception.expect(RetrievalException.class); exception.expectMessage("Unable to retrieve job with id job1"); jobManagementService.getJob("job1"); @@ -173,7 +177,7 @@ public void shouldThrowErrorIfJobIdNotFoundWhenGettingJob() { public void shouldThrowErrorIfJobIdNotFoundWhenAbortingJob() { when(jobInfoRepository.findById("job1")).thenReturn(Optional.empty()); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs, metricsConfig); exception.expect(RetrievalException.class); exception.expectMessage("Unable to retrieve job with id job1"); jobManagementService.abortJob("job1"); @@ -185,7 +189,7 @@ public void shouldThrowErrorIfJobInTerminalStateWhenAbortingJob() { job.setStatus(JobStatus.COMPLETED); when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(job)); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs, metricsConfig); exception.expect(IllegalStateException.class); exception.expectMessage("Unable to stop job already in terminal state"); jobManagementService.abortJob("job1"); @@ -198,7 +202,7 @@ public void shouldUpdateJobAfterAborting() { job.setExtId("extId1"); when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(job)); JobManagementService jobManagementService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs, metricsConfig); jobManagementService.abortJob("job1"); ArgumentCaptor jobCapture = ArgumentCaptor.forClass(JobInfo.class); verify(jobInfoRepository).saveAndFlush(jobCapture.capture()); @@ -212,7 +216,7 @@ public void shouldUpdateJobStatusIfExists() { ArgumentCaptor jobInfoArgumentCaptor = ArgumentCaptor.forClass(JobInfo.class); JobManagementService jobExecutionService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs, metricsConfig); jobExecutionService.updateJobStatus("jobid", JobStatus.PENDING); verify(jobInfoRepository, times(1)).save(jobInfoArgumentCaptor.capture()); @@ -229,7 +233,7 @@ public void shouldUpdateJobExtIdIfExists() { ArgumentCaptor jobInfoArgumentCaptor = ArgumentCaptor.forClass(JobInfo.class); JobManagementService jobExecutionService = - new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs); + new JobManagementService(jobInfoRepository, metricsRepository, jobManager, defaults, specService, storageSpecs, metricsConfig); jobExecutionService.updateJobExtId("jobid", "extid"); verify(jobInfoRepository, times(1)).save(jobInfoArgumentCaptor.capture()); diff --git a/ingestion/README.md b/ingestion/README.md new file mode 100644 index 00000000000..8b9a2b5659f --- /dev/null +++ b/ingestion/README.md @@ -0,0 +1,55 @@ +# Feast Ingestion + +Feast ingestion creates and runs Apache Beam pipeline to import Feature Row elements from various +sources to multiple stores in Feast. + +The configuration for the ingestion is specified in an import job, which can be referred from +[ImportJobSpecs.proto](../protos/feast/specs/ImportJobSpecs.proto). Import job configuration +can be passed as a YAML file following the format in `ImportJobSpecs.proto`. + +Example of an import job that listens to Kafka for Feature Row and writes them to Redis. +```yaml +importSpec: + type: kafka + sourceOptions: + server: localhost:9092 + topics: topic_1 + entities: + - entity_1 + schema: + fields: + - featureId: entity_1.feature_1 + - featureId: entity_1.feature_2 + +writeFeatureMetricsToInfluxDb: true +influxDbUrl: http://localhost:8086 +influxDbDatabase: my_influx_db +influxDbMeasurement: my_influx_measurement + +entitySpecs: +- name: entity_1 + description: This is a test entity + +featureSpecs: +- id: entity_1.feature_1 + entity: entity_1 + name: feature_1 + owner: feast + valueType: INT64 +- id: entity_1.feature_2 + entity: entity_1 + name: feature_2 + owner: feast + valueType: INT64 + +servingStorageSpec: + id: serving_store + type: redis + options: + host: localhost + port: 6379 + +errorsStorageSpec: + id: error_store + type: stdout +``` \ No newline at end of file diff --git a/ingestion/pom.xml b/ingestion/pom.xml index faf60b5a955..6908c19298c 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -349,5 +349,11 @@ ${org.apache.beam.version} runtime + + + org.influxdb + influxdb-java + 2.15 + diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index 5a8dec382b6..536db15ff78 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -38,6 +38,7 @@ import feast.ingestion.transform.ToFeatureRowExtended; import feast.ingestion.transform.ValidateTransform; import feast.ingestion.transform.WarehouseStoreTransform; +import feast.ingestion.transform.WriteFeatureMetricsToInfluxDB; import feast.ingestion.transform.fn.ConvertTypesDoFn; import feast.ingestion.transform.fn.LoggerDoFn; import feast.ingestion.values.PFeatureRows; @@ -56,6 +57,7 @@ import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder; +import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.Sample; @@ -110,6 +112,7 @@ public static void main(String[] args) { mainWithResult(args); } + @SuppressWarnings({"UnusedReturnValue", "WeakerAccess"}) public static PipelineResult mainWithResult(String[] args) { log.info("Arguments: " + Arrays.toString(args)); ImportJobPipelineOptions options = @@ -118,12 +121,30 @@ public static PipelineResult mainWithResult(String[] args) { options.setJobName(generateName()); } log.info("options: " + options.toString()); - ImportJobSpecs importJobSpecs = new ImportJobSpecsSupplier(options.getWorkspace()) - .get(); + ImportJobSpecs importJobSpecs = new ImportJobSpecsSupplier(options.getWorkspace()).get(); Injector injector = Guice.createInjector(new ImportJobModule(options, importJobSpecs), new PipelineModule()); ImportJob job = injector.getInstance(ImportJob.class); + // Validate Influx DB configuration if options to write feature metrics to Influx DB is enabled + if (importJobSpecs.getWriteFeatureMetricsToInfluxDb()) { + if (importJobSpecs.getInfluxDbUrl().isEmpty()) { + throw new IllegalArgumentException( + "Influx DB url is required to write feature metrics to " + + "Influx DB. Please set this value in the import job."); + } + if (importJobSpecs.getInfluxDbDatabase().isEmpty()) { + throw new IllegalArgumentException( + "Influx DB database is required to write feature metrics to " + + "Influx DB. Please set this value in the import job."); + } + if (importJobSpecs.getInfluxDbMeasurement().isEmpty()) { + throw new IllegalArgumentException( + "Influx DB measurement name is required to write feature metrics to " + + "Influx DB.Please set this value in the import job."); + } + } + job.expand(); return job.run(); } @@ -143,8 +164,8 @@ public void expand() { TypeDescriptor.of(FeatureRowExtended.class), ProtoCoder.of(FeatureRowExtended.class)); coderRegistry.registerCoderForType(TypeDescriptor.of(TableRow.class), TableRowJsonCoder.of()); - JobOptions jobOptions = OptionsParser - .parse(importJobSpecs.getImportSpec().getJobOptionsMap(), JobOptions.class); + JobOptions jobOptions = + OptionsParser.parse(importJobSpecs.getImportSpec().getJobOptionsMap(), JobOptions.class); try { log.info(JsonFormat.printer().print(importJobSpecs)); @@ -174,12 +195,41 @@ public void expand() { PCollection errorRows = pFeatureRows.getErrors(); if (jobOptions.isCoalesceRowsEnabled()) { // Should we merge and dedupe rows before writing to the serving store? - servingRows = servingRows.apply("Coalesce Rows", new CoalesceFeatureRowExtended( - jobOptions.getCoalesceRowsDelaySeconds(), - jobOptions.getCoalesceRowsTimeoutSeconds())); + servingRows = + servingRows.apply( + "Coalesce Rows", + new CoalesceFeatureRowExtended( + jobOptions.getCoalesceRowsDelaySeconds(), + jobOptions.getCoalesceRowsTimeoutSeconds())); } if (!dryRun) { + // Write feature metrics only if the source is unbounded. + // Feature metrics such as lag is usually relevant when we are consuming real-time + // data from unbounded sources, because we want to check how responsive or how fresh + // our processed data are in the streaming pipelines. + // + // When the source of the data is bounded, it is normal for users to provide + // data with event timestamp way behind the current timestamp. For example, when + // users are doing tests or performing a backfill. + // + // Combining feature metrics from these 2 different sources may confuse the interpretation + // of the metrics. For example, we may think our processing pipelines are lagging, + // when in fact someone is running a backfill of data in the past. + // + // TODO: Consider tagging the data point with data source name (e.g. Kafka, Pub/Sub) or + // data source type (bounded vs unbounded) in order to differentiate metrics coming + // from different sources. + // + // Kafka and Pub/Sub are the unbounded sources supported in Feast 0.1.x. + if (importJobSpecs.getImportSpec().getType().matches("(?i)kafka|pubsub")) { + servingRows.apply("Write feature metrics to Influx DB", + new WriteFeatureMetricsToInfluxDB( + importJobSpecs.getInfluxDbUrl(), + importJobSpecs.getInfluxDbDatabase(), + importJobSpecs.getInfluxDbMeasurement())); + } + servingRows.apply("Write to Serving Stores", servingStoreTransform); if (!Strings.isNullOrEmpty(importJobSpecs.getWarehouseStorageSpec().getId())) { warehouseRows.apply("Write to Warehouse Stores", warehouseStoreTransform); @@ -194,7 +244,8 @@ public PipelineResult run() { return result; } - public void logNRows(PFeatureRows pFeatureRows, String name, long limit, Duration period) { + @SuppressWarnings("SameParameterValue") + private void logNRows(PFeatureRows pFeatureRows, String name, long limit, Duration period) { PCollection main = pFeatureRows.getMain(); PCollection errors = pFeatureRows.getErrors(); diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java index 25dd62d4abc..cc88ee861c6 100644 --- a/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java +++ b/ingestion/src/main/java/feast/ingestion/options/ImportJobPipelineOptions.java @@ -23,11 +23,10 @@ import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; +import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.options.Validation.Required; -/** - * Options passed to Beam to influence the job's execution environment - */ +/** Options passed to Beam to influence the job's execution environment */ public interface ImportJobPipelineOptions extends PipelineOptions { @Description("Path to a workspace directory containing importJobSpecs.yaml") @@ -38,9 +37,9 @@ public interface ImportJobPipelineOptions extends PipelineOptions { @Description("If dry run is set, execute up to feature row validation") @Default.Boolean(false) - Boolean isDryRun(); + boolean isDryRun(); - void setDryRun(Boolean value); + void setDryRun(boolean value); @AutoService(PipelineOptionsRegistrar.class) class ImportJobPipelineOptionsRegistrar implements PipelineOptionsRegistrar { diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteFeatureMetricsToInfluxDB.java b/ingestion/src/main/java/feast/ingestion/transform/WriteFeatureMetricsToInfluxDB.java new file mode 100644 index 00000000000..0f679d93db2 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/WriteFeatureMetricsToInfluxDB.java @@ -0,0 +1,210 @@ +package feast.ingestion.transform; + +import feast.types.FeatureProto.Feature; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import java.util.DoubleSummaryStatistics; +import java.util.LongSummaryStatistics; +import java.util.concurrent.TimeUnit; +import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.coders.DefaultCoder; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.GroupByKey; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; +import org.influxdb.BatchOptions; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.Point; +import org.joda.time.Duration; + +public class WriteFeatureMetricsToInfluxDB + extends PTransform, PDone> { + + private static final int DEFAULT_WINDOW_DURATION_IN_SEC = 30; + private static final int DEFAULT_INFLUX_DB_JITTER_DURATION_IN_MS = 100; + private String influxDbUrl; + private String influxDbDatabase; + private String influxDbMeasurement; + + public WriteFeatureMetricsToInfluxDB( + String influxDbUrl, String influxDbDatabase, String influxDbMeasurement) { + this.influxDbUrl = influxDbUrl; + this.influxDbDatabase = influxDbDatabase; + this.influxDbMeasurement = influxDbMeasurement; + } + + @DefaultCoder(AvroCoder.class) + static class FeatureMetric { + // value is the numeric value of the feature, FeatureMetric only supports double type + // if the value is a timestamp type, value corresponds to epoch seconds + // if the value is a boolean type, value of 1 corresponds to "true" and 0 otherwise + // if the value type is other non number format, value will be set to 0 + double value; + // lagInSeconds is the delta between processing time in the Dataflow job + // and the event time of the FeatureRow containing this feature + long lagInSeconds; + String entityName; + + public FeatureMetric() {} + + FeatureMetric(double value, long lagInSeconds, String entityName) { + this.value = value; + this.lagInSeconds = lagInSeconds; + this.entityName = entityName; + } + } + + @Override + public PDone expand(PCollection input) { + input + .apply( + Window.into(FixedWindows.of(Duration.standardSeconds(DEFAULT_WINDOW_DURATION_IN_SEC)))) + .apply( + "Create feature metric keyed by feature id", + ParDo.of( + new DoFn>() { + @ProcessElement + public void processElement(ProcessContext c) { + FeatureRow featureRow = c.element().getRow(); + long lagInSeconds = + System.currentTimeMillis() / 1000L + - featureRow.getEventTimestamp().getSeconds(); + for (Feature feature : featureRow.getFeaturesList()) { + c.output( + KV.of( + feature.getId(), + new FeatureMetric( + getValue(feature), lagInSeconds, featureRow.getEntityName()))); + } + } + })) + .apply(GroupByKey.create()) + .apply( + ParDo.of( + new DoFn>, Void>() { + InfluxDB influxDB; + + @Setup + public void setup() { + try { + influxDB = InfluxDBFactory.connect(influxDbUrl); + influxDB.setDatabase(influxDbDatabase); + influxDB.enableBatch( + BatchOptions.DEFAULTS.jitterDuration( + DEFAULT_INFLUX_DB_JITTER_DURATION_IN_MS)); + } catch (Exception e) { + // Ignored because writing metrics is not a critical component of Feast + // and we do not want to get overwhelmed with connection error logs + // due to timeouts and downtime in upstream Influx DB server + } + } + + @Teardown + public void tearDown() { + if (influxDB != null) { + influxDB.close(); + } + } + + @ProcessElement + public void processElement(ProcessContext c) { + if (influxDB == null) { + // Influx DB client is not setup properly, skip writing metrics + return; + } + + DoubleSummaryStatistics statsForValue = new DoubleSummaryStatistics(); + LongSummaryStatistics statsForLagInSeconds = new LongSummaryStatistics(); + + String entityName = null; + String featureId = c.element().getKey(); + + for (FeatureMetric featureMetric : c.element().getValue()) { + statsForValue.accept(featureMetric.value); + statsForLagInSeconds.accept(featureMetric.lagInSeconds); + if (entityName == null) { + entityName = featureMetric.entityName; + } + } + + if (featureId == null || entityName == null) { + return; + } + + try { + influxDB.write( + Point.measurement(influxDbMeasurement) + .time(System.currentTimeMillis(), TimeUnit.MILLISECONDS) + .addField("lag_in_seconds_mean", statsForLagInSeconds.getAverage()) + .addField("lag_in_seconds_min", statsForLagInSeconds.getMin()) + .addField("lag_in_seconds_max", statsForLagInSeconds.getMax()) + .addField("value_mean", statsForValue.getAverage()) + .addField("value_min", statsForValue.getMin()) + .addField("value_max", statsForValue.getMax()) + .addField("value_count", statsForValue.getCount()) + .tag("feature_id", featureId) + .tag("entity_name", entityName) + .build()); + } catch (Exception e) { + // Ignored because writing metrics is not a critical component of Feast + // and we do not want to get overwhelmed with failed metric write logs + // due to timeouts and downtime in upstream Influx DB server + } + } + })); + + return PDone.in(input.getPipeline()); + } + + /** + * This method returns the numeric value of the feature in double type + * + *
    + *
  • if the value is a numeric type, it will be cast to double type + *
  • f the value is a timestamp type, value corresponds to epoch seconds + *
  • if the value is a boolean type, value of 1 corresponds to "true" and value of 0 + * corresponds to "false" + *
  • if the value type is other non number format, value will be set to 0 + *
+ * + * @param feature Feast feature instance of a feature row + * @return numeric value of the feature in double type + */ + private double getValue(Feature feature) { + double value; + Value featureValue = feature.getValue(); + + switch (featureValue.getValCase()) { + case INT32VAL: + value = featureValue.getInt32Val(); + break; + case INT64VAL: + value = featureValue.getInt64Val(); + break; + case DOUBLEVAL: + value = featureValue.getDoubleVal(); + break; + case FLOATVAL: + value = featureValue.getFloatVal(); + break; + case BOOLVAL: + value = featureValue.getBoolVal() ? 1.0 : 0.0; + break; + case TIMESTAMPVAL: + value = featureValue.getTimestampVal().getSeconds(); + break; + default: + value = 0.0; + break; + } + + return value; + } +} diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java index 4dc53467c70..6884b86b61a 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java @@ -138,12 +138,16 @@ public static class WriteDoFn extends DoFn { } public WriteDoFn withBatchSize(int batchSize) { - if (batchSize > 0) this.batchSize = batchSize; + if (batchSize > 0) { + this.batchSize = batchSize; + } return this; } public WriteDoFn withTimeout(int timeout) { - if (timeout > 0) this.timeout = timeout; + if (timeout > 0) { + this.timeout = timeout; + } return this; } @@ -155,7 +159,6 @@ public void setup() { @StartBundle public void startBundle() { pipeline = jedis.pipelined(); - pipeline.multi(); batchCount = 0; } @@ -168,9 +171,7 @@ public void processElement(ProcessContext context) { } batchCount++; if (batchCount >= batchSize) { - pipeline.exec(); pipeline.sync(); - pipeline.multi(); batchCount = 0; } } @@ -197,10 +198,7 @@ private Response writeRecord(RedisMutation mutation) { @FinishBundle public void finishBundle() { - if (pipeline.isInMulti()) { - pipeline.exec(); - pipeline.sync(); - } + pipeline.sync(); batchCount = 0; } diff --git a/ingestion/src/main/resources/logback.xml b/ingestion/src/main/resources/logback.xml index e9fc5ebbd5a..652a0d08341 100644 --- a/ingestion/src/main/resources/logback.xml +++ b/ingestion/src/main/resources/logback.xml @@ -24,4 +24,7 @@ + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8afd15d0a5d..9235ff70027 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ Feast Parent - 0.1.1 + 0.1.5 UTF-8 3.6.1 1.14.0 diff --git a/protos/Makefile b/protos/Makefile index e49b8c32d6a..367a1fa025a 100644 --- a/protos/Makefile +++ b/protos/Makefile @@ -8,5 +8,5 @@ gen-go: gen-python: pip install grpcio-tools - @$(foreach dir,$(dirs),python -m grpc_tools.protoc -I. --python_out=../sdk/python/ feast/$(dir)/*.proto;) + @$(foreach dir,$(dirs),python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;) @$(foreach dir,$(service_dirs),python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;) \ No newline at end of file diff --git a/protos/feast/specs/ImportJobSpecs.proto b/protos/feast/specs/ImportJobSpecs.proto index 5267e05e886..6dd3b17d095 100644 --- a/protos/feast/specs/ImportJobSpecs.proto +++ b/protos/feast/specs/ImportJobSpecs.proto @@ -35,4 +35,31 @@ message ImportJobSpecs { StorageSpec servingStorageSpec = 5; StorageSpec warehouseStorageSpec = 6; StorageSpec errorsStorageSpec = 7; + + // The following four fields below are only valid in Feast 0.1.x + // and will be backward incompatible for versions greater than 0.1.x + // ============================================================ + // + // (optional, default: false) + // write_feature_metrics_to_influx_db specifies if Feast should write feature metrics + // (such as lag and value summaries) to an existing Influx DB database. + // If set to true, the next three fields must all be set: + // - influx_db_url + // - influx_db_database + // - influx_db_measurement + // NOTE: + // When retrieving the data from Influx DB, + // - The field name for feature lag value in seconds unit is "lag_in_seconds" + // - The field name for value is "value" + // - Points will be tagged with "feature_id" and "entity_name" + bool write_feature_metrics_to_influx_db = 8; + // influx_db_url is the URL for Feast to connect to Influx DB server. + // For example: http://localhost:8086 + // Feast and the import jobs created by Feast must be able to reach this URL. + string influx_db_url = 9; + // influx_db_database is the name of the Influx DB database for the metrics. + string influx_db_database = 10; + // influx_db_measurement is the name of measurement that will contain the metrics data. + // Ref: https://docs.influxdata.com/influxdb/v1.7/concepts/key_concepts/#measurement + string influx_db_measurement = 11; } diff --git a/sdk/python/feast/core/CoreService_pb2.py b/sdk/python/feast/core/CoreService_pb2.py index 93212793dff..f65187823b8 100644 --- a/sdk/python/feast/core/CoreService_pb2.py +++ b/sdk/python/feast/core/CoreService_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/CoreService.proto @@ -24,7 +25,7 @@ package='feast.core', syntax='proto3', serialized_options=_b('\n\nfeast.coreB\020CoreServiceProtoZ5github.com/gojek/feast/protos/generated/go/feast/core'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/core/CoreService.proto\x12\nfeast.core\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x1a\"feast/specs/FeatureGroupSpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\x1a\x1bgoogle/protobuf/empty.proto\"\xc9\x05\n\x10\x43oreServiceTypes\x1a!\n\x12GetEntitiesRequest\x12\x0b\n\x03ids\x18\x01 \x03(\t\x1a@\n\x13GetEntitiesResponse\x12)\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x1a\x41\n\x14ListEntitiesResponse\x12)\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x1a!\n\x12GetFeaturesRequest\x12\x0b\n\x03ids\x18\x01 \x03(\t\x1a\x41\n\x13GetFeaturesResponse\x12*\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x1a\x42\n\x14ListFeaturesResponse\x12*\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x1a \n\x11GetStorageRequest\x12\x0b\n\x03ids\x18\x01 \x03(\t\x1a\x44\n\x12GetStorageResponse\x12.\n\x0cstorageSpecs\x18\x01 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x1a\x45\n\x13ListStorageResponse\x12.\n\x0cstorageSpecs\x18\x01 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x1a)\n\x13\x41pplyEntityResponse\x12\x12\n\nentityName\x18\x01 \x01(\t\x1a)\n\x14\x41pplyFeatureResponse\x12\x11\n\tfeatureId\x18\x01 \x01(\t\x1a\x33\n\x19\x41pplyFeatureGroupResponse\x12\x16\n\x0e\x66\x65\x61tureGroupId\x18\x01 \x01(\t\x1a)\n\x14\x41pplyStorageResponse\x12\x11\n\tstorageId\x18\x01 \x01(\t2\x83\x08\n\x0b\x43oreService\x12r\n\x0bGetEntities\x12/.feast.core.CoreServiceTypes.GetEntitiesRequest\x1a\x30.feast.core.CoreServiceTypes.GetEntitiesResponse\"\x00\x12[\n\x0cListEntities\x12\x16.google.protobuf.Empty\x1a\x31.feast.core.CoreServiceTypes.ListEntitiesResponse\"\x00\x12r\n\x0bGetFeatures\x12/.feast.core.CoreServiceTypes.GetFeaturesRequest\x1a\x30.feast.core.CoreServiceTypes.GetFeaturesResponse\"\x00\x12[\n\x0cListFeatures\x12\x16.google.protobuf.Empty\x1a\x31.feast.core.CoreServiceTypes.ListFeaturesResponse\"\x00\x12o\n\nGetStorage\x12..feast.core.CoreServiceTypes.GetStorageRequest\x1a/.feast.core.CoreServiceTypes.GetStorageResponse\"\x00\x12Y\n\x0bListStorage\x12\x16.google.protobuf.Empty\x1a\x30.feast.core.CoreServiceTypes.ListStorageResponse\"\x00\x12]\n\x0c\x41pplyFeature\x12\x18.feast.specs.FeatureSpec\x1a\x31.feast.core.CoreServiceTypes.ApplyFeatureResponse\"\x00\x12l\n\x11\x41pplyFeatureGroup\x12\x1d.feast.specs.FeatureGroupSpec\x1a\x36.feast.core.CoreServiceTypes.ApplyFeatureGroupResponse\"\x00\x12Z\n\x0b\x41pplyEntity\x12\x17.feast.specs.EntitySpec\x1a\x30.feast.core.CoreServiceTypes.ApplyEntityResponse\"\x00\x12]\n\x0c\x41pplyStorage\x12\x18.feast.specs.StorageSpec\x1a\x31.feast.core.CoreServiceTypes.ApplyStorageResponse\"\x00\x42U\n\nfeast.coreB\x10\x43oreServiceProtoZ5github.com/gojek/feast/protos/generated/go/feast/coreb\x06proto3') + serialized_pb=_b('\n\x1c\x66\x65\x61st/core/CoreService.proto\x12\nfeast.core\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x1a\"feast/specs/FeatureGroupSpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\x1a\x1bgoogle/protobuf/empty.proto\"\x9e\x05\n\x10\x43oreServiceTypes\x1a!\n\x12GetEntitiesRequest\x12\x0b\n\x03ids\x18\x01 \x03(\t\x1a@\n\x13GetEntitiesResponse\x12)\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x1a\x41\n\x14ListEntitiesResponse\x12)\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x1a!\n\x12GetFeaturesRequest\x12\x0b\n\x03ids\x18\x01 \x03(\t\x1a\x41\n\x13GetFeaturesResponse\x12*\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x1a\x42\n\x14ListFeaturesResponse\x12*\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x1a \n\x11GetStorageRequest\x12\x0b\n\x03ids\x18\x01 \x03(\t\x1a\x44\n\x12GetStorageResponse\x12.\n\x0cstorageSpecs\x18\x01 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x1a\x45\n\x13ListStorageResponse\x12.\n\x0cstorageSpecs\x18\x01 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x1a)\n\x13\x41pplyEntityResponse\x12\x12\n\nentityName\x18\x01 \x01(\t\x1a)\n\x14\x41pplyFeatureResponse\x12\x11\n\tfeatureId\x18\x01 \x01(\t\x1a\x33\n\x19\x41pplyFeatureGroupResponse\x12\x16\n\x0e\x66\x65\x61tureGroupId\x18\x01 \x01(\t2\xaa\x07\n\x0b\x43oreService\x12r\n\x0bGetEntities\x12/.feast.core.CoreServiceTypes.GetEntitiesRequest\x1a\x30.feast.core.CoreServiceTypes.GetEntitiesResponse\"\x00\x12[\n\x0cListEntities\x12\x16.google.protobuf.Empty\x1a\x31.feast.core.CoreServiceTypes.ListEntitiesResponse\"\x00\x12r\n\nGetStorage\x12..feast.core.CoreServiceTypes.GetStorageRequest\x1a/.feast.core.CoreServiceTypes.GetStorageResponse\"\x03\x88\x02\x01\x12\\\n\x0bListStorage\x12\x16.google.protobuf.Empty\x1a\x30.feast.core.CoreServiceTypes.ListStorageResponse\"\x03\x88\x02\x01\x12r\n\x0bGetFeatures\x12/.feast.core.CoreServiceTypes.GetFeaturesRequest\x1a\x30.feast.core.CoreServiceTypes.GetFeaturesResponse\"\x00\x12[\n\x0cListFeatures\x12\x16.google.protobuf.Empty\x1a\x31.feast.core.CoreServiceTypes.ListFeaturesResponse\"\x00\x12]\n\x0c\x41pplyFeature\x12\x18.feast.specs.FeatureSpec\x1a\x31.feast.core.CoreServiceTypes.ApplyFeatureResponse\"\x00\x12l\n\x11\x41pplyFeatureGroup\x12\x1d.feast.specs.FeatureGroupSpec\x1a\x36.feast.core.CoreServiceTypes.ApplyFeatureGroupResponse\"\x00\x12Z\n\x0b\x41pplyEntity\x12\x17.feast.specs.EntitySpec\x1a\x30.feast.core.CoreServiceTypes.ApplyEntityResponse\"\x00\x42U\n\nfeast.coreB\x10\x43oreServiceProtoZ5github.com/gojek/feast/protos/generated/go/feast/coreb\x06proto3') , dependencies=[feast_dot_specs_dot_EntitySpec__pb2.DESCRIPTOR,feast_dot_specs_dot_FeatureSpec__pb2.DESCRIPTOR,feast_dot_specs_dot_FeatureGroupSpec__pb2.DESCRIPTOR,feast_dot_specs_dot_StorageSpec__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) @@ -391,36 +392,6 @@ serialized_end=872, ) -_CORESERVICETYPES_APPLYSTORAGERESPONSE = _descriptor.Descriptor( - name='ApplyStorageResponse', - full_name='feast.core.CoreServiceTypes.ApplyStorageResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='storageId', full_name='feast.core.CoreServiceTypes.ApplyStorageResponse.storageId', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=874, - serialized_end=915, -) - _CORESERVICETYPES = _descriptor.Descriptor( name='CoreServiceTypes', full_name='feast.core.CoreServiceTypes', @@ -431,7 +402,7 @@ ], extensions=[ ], - nested_types=[_CORESERVICETYPES_GETENTITIESREQUEST, _CORESERVICETYPES_GETENTITIESRESPONSE, _CORESERVICETYPES_LISTENTITIESRESPONSE, _CORESERVICETYPES_GETFEATURESREQUEST, _CORESERVICETYPES_GETFEATURESRESPONSE, _CORESERVICETYPES_LISTFEATURESRESPONSE, _CORESERVICETYPES_GETSTORAGEREQUEST, _CORESERVICETYPES_GETSTORAGERESPONSE, _CORESERVICETYPES_LISTSTORAGERESPONSE, _CORESERVICETYPES_APPLYENTITYRESPONSE, _CORESERVICETYPES_APPLYFEATURERESPONSE, _CORESERVICETYPES_APPLYFEATUREGROUPRESPONSE, _CORESERVICETYPES_APPLYSTORAGERESPONSE, ], + nested_types=[_CORESERVICETYPES_GETENTITIESREQUEST, _CORESERVICETYPES_GETENTITIESRESPONSE, _CORESERVICETYPES_LISTENTITIESRESPONSE, _CORESERVICETYPES_GETFEATURESREQUEST, _CORESERVICETYPES_GETFEATURESRESPONSE, _CORESERVICETYPES_LISTFEATURESRESPONSE, _CORESERVICETYPES_GETSTORAGEREQUEST, _CORESERVICETYPES_GETSTORAGERESPONSE, _CORESERVICETYPES_LISTSTORAGERESPONSE, _CORESERVICETYPES_APPLYENTITYRESPONSE, _CORESERVICETYPES_APPLYFEATURERESPONSE, _CORESERVICETYPES_APPLYFEATUREGROUPRESPONSE, ], enum_types=[ ], serialized_options=None, @@ -441,7 +412,7 @@ oneofs=[ ], serialized_start=202, - serialized_end=915, + serialized_end=872, ) _CORESERVICETYPES_GETENTITIESREQUEST.containing_type = _CORESERVICETYPES @@ -462,7 +433,6 @@ _CORESERVICETYPES_APPLYENTITYRESPONSE.containing_type = _CORESERVICETYPES _CORESERVICETYPES_APPLYFEATURERESPONSE.containing_type = _CORESERVICETYPES _CORESERVICETYPES_APPLYFEATUREGROUPRESPONSE.containing_type = _CORESERVICETYPES -_CORESERVICETYPES_APPLYSTORAGERESPONSE.containing_type = _CORESERVICETYPES DESCRIPTOR.message_types_by_name['CoreServiceTypes'] = _CORESERVICETYPES _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -551,13 +521,6 @@ # @@protoc_insertion_point(class_scope:feast.core.CoreServiceTypes.ApplyFeatureGroupResponse) )) , - - ApplyStorageResponse = _reflection.GeneratedProtocolMessageType('ApplyStorageResponse', (_message.Message,), dict( - DESCRIPTOR = _CORESERVICETYPES_APPLYSTORAGERESPONSE, - __module__ = 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.CoreServiceTypes.ApplyStorageResponse) - )) - , DESCRIPTOR = _CORESERVICETYPES, __module__ = 'feast.core.CoreService_pb2' # @@protoc_insertion_point(class_scope:feast.core.CoreServiceTypes) @@ -575,7 +538,6 @@ _sym_db.RegisterMessage(CoreServiceTypes.ApplyEntityResponse) _sym_db.RegisterMessage(CoreServiceTypes.ApplyFeatureResponse) _sym_db.RegisterMessage(CoreServiceTypes.ApplyFeatureGroupResponse) -_sym_db.RegisterMessage(CoreServiceTypes.ApplyStorageResponse) DESCRIPTOR._options = None @@ -586,8 +548,8 @@ file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=918, - serialized_end=1945, + serialized_start=875, + serialized_end=1813, methods=[ _descriptor.MethodDescriptor( name='GetEntities', @@ -608,39 +570,39 @@ serialized_options=None, ), _descriptor.MethodDescriptor( - name='GetFeatures', - full_name='feast.core.CoreService.GetFeatures', + name='GetStorage', + full_name='feast.core.CoreService.GetStorage', index=2, containing_service=None, - input_type=_CORESERVICETYPES_GETFEATURESREQUEST, - output_type=_CORESERVICETYPES_GETFEATURESRESPONSE, - serialized_options=None, + input_type=_CORESERVICETYPES_GETSTORAGEREQUEST, + output_type=_CORESERVICETYPES_GETSTORAGERESPONSE, + serialized_options=_b('\210\002\001'), ), _descriptor.MethodDescriptor( - name='ListFeatures', - full_name='feast.core.CoreService.ListFeatures', + name='ListStorage', + full_name='feast.core.CoreService.ListStorage', index=3, containing_service=None, input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, - output_type=_CORESERVICETYPES_LISTFEATURESRESPONSE, - serialized_options=None, + output_type=_CORESERVICETYPES_LISTSTORAGERESPONSE, + serialized_options=_b('\210\002\001'), ), _descriptor.MethodDescriptor( - name='GetStorage', - full_name='feast.core.CoreService.GetStorage', + name='GetFeatures', + full_name='feast.core.CoreService.GetFeatures', index=4, containing_service=None, - input_type=_CORESERVICETYPES_GETSTORAGEREQUEST, - output_type=_CORESERVICETYPES_GETSTORAGERESPONSE, + input_type=_CORESERVICETYPES_GETFEATURESREQUEST, + output_type=_CORESERVICETYPES_GETFEATURESRESPONSE, serialized_options=None, ), _descriptor.MethodDescriptor( - name='ListStorage', - full_name='feast.core.CoreService.ListStorage', + name='ListFeatures', + full_name='feast.core.CoreService.ListFeatures', index=5, containing_service=None, input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, - output_type=_CORESERVICETYPES_LISTSTORAGERESPONSE, + output_type=_CORESERVICETYPES_LISTFEATURESRESPONSE, serialized_options=None, ), _descriptor.MethodDescriptor( @@ -670,15 +632,6 @@ output_type=_CORESERVICETYPES_APPLYENTITYRESPONSE, serialized_options=None, ), - _descriptor.MethodDescriptor( - name='ApplyStorage', - full_name='feast.core.CoreService.ApplyStorage', - index=9, - containing_service=None, - input_type=feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC, - output_type=_CORESERVICETYPES_APPLYSTORAGERESPONSE, - serialized_options=None, - ), ]) _sym_db.RegisterServiceDescriptor(_CORESERVICE) diff --git a/sdk/python/feast/core/CoreService_pb2.pyi b/sdk/python/feast/core/CoreService_pb2.pyi new file mode 100644 index 00000000000..5a85361af13 --- /dev/null +++ b/sdk/python/feast/core/CoreService_pb2.pyi @@ -0,0 +1,234 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.specs.EntitySpec_pb2 import ( + EntitySpec as feast___specs___EntitySpec_pb2___EntitySpec, +) + +from feast.specs.FeatureSpec_pb2 import ( + FeatureSpec as feast___specs___FeatureSpec_pb2___FeatureSpec, +) + +from feast.specs.StorageSpec_pb2 import ( + StorageSpec as feast___specs___StorageSpec_pb2___StorageSpec, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class CoreServiceTypes(google___protobuf___message___Message): + class GetEntitiesRequest(google___protobuf___message___Message): + ids = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + ids : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.GetEntitiesRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"ids"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"ids"]) -> None: ... + + class GetEntitiesResponse(google___protobuf___message___Message): + + @property + def entities(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___EntitySpec_pb2___EntitySpec]: ... + + def __init__(self, + entities : typing___Optional[typing___Iterable[feast___specs___EntitySpec_pb2___EntitySpec]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.GetEntitiesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entities"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entities"]) -> None: ... + + class ListEntitiesResponse(google___protobuf___message___Message): + + @property + def entities(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___EntitySpec_pb2___EntitySpec]: ... + + def __init__(self, + entities : typing___Optional[typing___Iterable[feast___specs___EntitySpec_pb2___EntitySpec]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.ListEntitiesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entities"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entities"]) -> None: ... + + class GetFeaturesRequest(google___protobuf___message___Message): + ids = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + ids : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.GetFeaturesRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"ids"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"ids"]) -> None: ... + + class GetFeaturesResponse(google___protobuf___message___Message): + + @property + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___FeatureSpec_pb2___FeatureSpec]: ... + + def __init__(self, + features : typing___Optional[typing___Iterable[feast___specs___FeatureSpec_pb2___FeatureSpec]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.GetFeaturesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"features"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"features"]) -> None: ... + + class ListFeaturesResponse(google___protobuf___message___Message): + + @property + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___FeatureSpec_pb2___FeatureSpec]: ... + + def __init__(self, + features : typing___Optional[typing___Iterable[feast___specs___FeatureSpec_pb2___FeatureSpec]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.ListFeaturesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"features"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"features"]) -> None: ... + + class GetStorageRequest(google___protobuf___message___Message): + ids = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + ids : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.GetStorageRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"ids"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"ids"]) -> None: ... + + class GetStorageResponse(google___protobuf___message___Message): + + @property + def storageSpecs(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___StorageSpec_pb2___StorageSpec]: ... + + def __init__(self, + storageSpecs : typing___Optional[typing___Iterable[feast___specs___StorageSpec_pb2___StorageSpec]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.GetStorageResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"storageSpecs"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"storageSpecs"]) -> None: ... + + class ListStorageResponse(google___protobuf___message___Message): + + @property + def storageSpecs(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___StorageSpec_pb2___StorageSpec]: ... + + def __init__(self, + storageSpecs : typing___Optional[typing___Iterable[feast___specs___StorageSpec_pb2___StorageSpec]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.ListStorageResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"storageSpecs"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"storageSpecs"]) -> None: ... + + class ApplyEntityResponse(google___protobuf___message___Message): + entityName = ... # type: typing___Text + + def __init__(self, + entityName : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.ApplyEntityResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entityName"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entityName"]) -> None: ... + + class ApplyFeatureResponse(google___protobuf___message___Message): + featureId = ... # type: typing___Text + + def __init__(self, + featureId : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.ApplyFeatureResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"featureId"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"featureId"]) -> None: ... + + class ApplyFeatureGroupResponse(google___protobuf___message___Message): + featureGroupId = ... # type: typing___Text + + def __init__(self, + featureGroupId : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes.ApplyFeatureGroupResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"featureGroupId"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"featureGroupId"]) -> None: ... + + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> CoreServiceTypes: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... diff --git a/sdk/python/feast/core/CoreService_pb2_grpc.py b/sdk/python/feast/core/CoreService_pb2_grpc.py index 6331551fda6..04557e86bef 100644 --- a/sdk/python/feast/core/CoreService_pb2_grpc.py +++ b/sdk/python/feast/core/CoreService_pb2_grpc.py @@ -5,7 +5,6 @@ from feast.specs import EntitySpec_pb2 as feast_dot_specs_dot_EntitySpec__pb2 from feast.specs import FeatureGroupSpec_pb2 as feast_dot_specs_dot_FeatureGroupSpec__pb2 from feast.specs import FeatureSpec_pb2 as feast_dot_specs_dot_FeatureSpec__pb2 -from feast.specs import StorageSpec_pb2 as feast_dot_specs_dot_StorageSpec__pb2 from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 @@ -29,16 +28,6 @@ def __init__(self, channel): request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListEntitiesResponse.FromString, ) - self.GetFeatures = channel.unary_unary( - '/feast.core.CoreService/GetFeatures', - request_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesRequest.SerializeToString, - response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesResponse.FromString, - ) - self.ListFeatures = channel.unary_unary( - '/feast.core.CoreService/ListFeatures', - request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListFeaturesResponse.FromString, - ) self.GetStorage = channel.unary_unary( '/feast.core.CoreService/GetStorage', request_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetStorageRequest.SerializeToString, @@ -49,6 +38,16 @@ def __init__(self, channel): request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListStorageResponse.FromString, ) + self.GetFeatures = channel.unary_unary( + '/feast.core.CoreService/GetFeatures', + request_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesRequest.SerializeToString, + response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesResponse.FromString, + ) + self.ListFeatures = channel.unary_unary( + '/feast.core.CoreService/ListFeatures', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListFeaturesResponse.FromString, + ) self.ApplyFeature = channel.unary_unary( '/feast.core.CoreService/ApplyFeature', request_serializer=feast_dot_specs_dot_FeatureSpec__pb2.FeatureSpec.SerializeToString, @@ -64,11 +63,6 @@ def __init__(self, channel): request_serializer=feast_dot_specs_dot_EntitySpec__pb2.EntitySpec.SerializeToString, response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ApplyEntityResponse.FromString, ) - self.ApplyStorage = channel.unary_unary( - '/feast.core.CoreService/ApplyStorage', - request_serializer=feast_dot_specs_dot_StorageSpec__pb2.StorageSpec.SerializeToString, - response_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ApplyStorageResponse.FromString, - ) class CoreServiceServicer(object): @@ -93,37 +87,37 @@ def ListEntities(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetFeatures(self, request, context): + def GetStorage(self, request, context): """ - Get features specified in request. - This process returns a list of feature specs. + Get storage specs specified in request. + This process returns a list of storage specs. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def ListFeatures(self, request, context): + def ListStorage(self, request, context): """ - Get all features. - This process returns a list of entity specs. + Get all storage specs. + This process returns a list of storage specs. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetStorage(self, request, context): + def GetFeatures(self, request, context): """ - Get storage specs specified in request. - This process returns a list of storage specs. + Get features specified in request. + This process returns a list of feature specs. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def ListStorage(self, request, context): + def ListFeatures(self, request, context): """ - Get all storage specs. - This process returns a list of storage specs. + Get all features. + This process returns a list of entity specs. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -156,15 +150,6 @@ def ApplyEntity(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def ApplyStorage(self, request, context): - """ - Register a new storage spec to the metadata store, or update an existing storage. - If any validation errors occur, only the first encountered error will be returned. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - def add_CoreServiceServicer_to_server(servicer, server): rpc_method_handlers = { @@ -178,16 +163,6 @@ def add_CoreServiceServicer_to_server(servicer, server): request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListEntitiesResponse.SerializeToString, ), - 'GetFeatures': grpc.unary_unary_rpc_method_handler( - servicer.GetFeatures, - request_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesRequest.FromString, - response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesResponse.SerializeToString, - ), - 'ListFeatures': grpc.unary_unary_rpc_method_handler( - servicer.ListFeatures, - request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListFeaturesResponse.SerializeToString, - ), 'GetStorage': grpc.unary_unary_rpc_method_handler( servicer.GetStorage, request_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetStorageRequest.FromString, @@ -198,6 +173,16 @@ def add_CoreServiceServicer_to_server(servicer, server): request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListStorageResponse.SerializeToString, ), + 'GetFeatures': grpc.unary_unary_rpc_method_handler( + servicer.GetFeatures, + request_deserializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesRequest.FromString, + response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.GetFeaturesResponse.SerializeToString, + ), + 'ListFeatures': grpc.unary_unary_rpc_method_handler( + servicer.ListFeatures, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ListFeaturesResponse.SerializeToString, + ), 'ApplyFeature': grpc.unary_unary_rpc_method_handler( servicer.ApplyFeature, request_deserializer=feast_dot_specs_dot_FeatureSpec__pb2.FeatureSpec.FromString, @@ -213,11 +198,6 @@ def add_CoreServiceServicer_to_server(servicer, server): request_deserializer=feast_dot_specs_dot_EntitySpec__pb2.EntitySpec.FromString, response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ApplyEntityResponse.SerializeToString, ), - 'ApplyStorage': grpc.unary_unary_rpc_method_handler( - servicer.ApplyStorage, - request_deserializer=feast_dot_specs_dot_StorageSpec__pb2.StorageSpec.FromString, - response_serializer=feast_dot_core_dot_CoreService__pb2.CoreServiceTypes.ApplyStorageResponse.SerializeToString, - ), } generic_handler = grpc.method_handlers_generic_handler( 'feast.core.CoreService', rpc_method_handlers) diff --git a/sdk/python/feast/core/DatasetService_pb2.py b/sdk/python/feast/core/DatasetService_pb2.py index 0c94041a4b1..d2722109df5 100644 --- a/sdk/python/feast/core/DatasetService_pb2.py +++ b/sdk/python/feast/core/DatasetService_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/DatasetService.proto diff --git a/sdk/python/feast/core/DatasetService_pb2.pyi b/sdk/python/feast/core/DatasetService_pb2.pyi new file mode 100644 index 00000000000..a630ee2eaa0 --- /dev/null +++ b/sdk/python/feast/core/DatasetService_pb2.pyi @@ -0,0 +1,140 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class DatasetServiceTypes(google___protobuf___message___Message): + class CreateDatasetRequest(google___protobuf___message___Message): + class FiltersEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DatasetServiceTypes.CreateDatasetRequest.FiltersEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + limit = ... # type: int + namePrefix = ... # type: typing___Text + + @property + def featureSet(self) -> FeatureSet: ... + + @property + def startDate(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + @property + def endDate(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + @property + def filters(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + def __init__(self, + featureSet : typing___Optional[FeatureSet] = None, + startDate : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + endDate : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + limit : typing___Optional[int] = None, + namePrefix : typing___Optional[typing___Text] = None, + filters : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DatasetServiceTypes.CreateDatasetRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"endDate",u"featureSet",u"startDate"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"endDate",u"featureSet",u"filters",u"limit",u"namePrefix",u"startDate"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"endDate",b"endDate",u"featureSet",b"featureSet",u"startDate",b"startDate"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"endDate",b"featureSet",b"filters",b"limit",b"namePrefix",b"startDate"]) -> None: ... + + class CreateDatasetResponse(google___protobuf___message___Message): + + @property + def datasetInfo(self) -> DatasetInfo: ... + + def __init__(self, + datasetInfo : typing___Optional[DatasetInfo] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DatasetServiceTypes.CreateDatasetResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"datasetInfo"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"datasetInfo"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"datasetInfo",b"datasetInfo"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"datasetInfo"]) -> None: ... + + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DatasetServiceTypes: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class FeatureSet(google___protobuf___message___Message): + entityName = ... # type: typing___Text + featureIds = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + entityName : typing___Optional[typing___Text] = None, + featureIds : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entityName",u"featureIds"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entityName",b"featureIds"]) -> None: ... + +class DatasetInfo(google___protobuf___message___Message): + name = ... # type: typing___Text + tableUrl = ... # type: typing___Text + + def __init__(self, + name : typing___Optional[typing___Text] = None, + tableUrl : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DatasetInfo: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"name",u"tableUrl"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"name",b"tableUrl"]) -> None: ... diff --git a/sdk/python/feast/core/JobService_pb2.py b/sdk/python/feast/core/JobService_pb2.py index fd208380484..fc4f8cf83d4 100644 --- a/sdk/python/feast/core/JobService_pb2.py +++ b/sdk/python/feast/core/JobService_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/JobService.proto diff --git a/sdk/python/feast/core/JobService_pb2.pyi b/sdk/python/feast/core/JobService_pb2.pyi new file mode 100644 index 00000000000..a60fcc02bbd --- /dev/null +++ b/sdk/python/feast/core/JobService_pb2.pyi @@ -0,0 +1,215 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.specs.ImportSpec_pb2 import ( + ImportSpec as feast___specs___ImportSpec_pb2___ImportSpec, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class JobServiceTypes(google___protobuf___message___Message): + class SubmitImportJobRequest(google___protobuf___message___Message): + name = ... # type: typing___Text + + @property + def importSpec(self) -> feast___specs___ImportSpec_pb2___ImportSpec: ... + + def __init__(self, + importSpec : typing___Optional[feast___specs___ImportSpec_pb2___ImportSpec] = None, + name : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.SubmitImportJobRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"importSpec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"importSpec",u"name"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"importSpec",b"importSpec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"importSpec",b"name"]) -> None: ... + + class SubmitImportJobResponse(google___protobuf___message___Message): + jobId = ... # type: typing___Text + + def __init__(self, + jobId : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.SubmitImportJobResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"jobId"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"jobId"]) -> None: ... + + class ListJobsResponse(google___protobuf___message___Message): + + @property + def jobs(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[JobServiceTypes.JobDetail]: ... + + def __init__(self, + jobs : typing___Optional[typing___Iterable[JobServiceTypes.JobDetail]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.ListJobsResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"jobs"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"jobs"]) -> None: ... + + class GetJobRequest(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.GetJobRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class GetJobResponse(google___protobuf___message___Message): + + @property + def job(self) -> JobServiceTypes.JobDetail: ... + + def __init__(self, + job : typing___Optional[JobServiceTypes.JobDetail] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.GetJobResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"job"]) -> None: ... + + class AbortJobRequest(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.AbortJobRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class AbortJobResponse(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.AbortJobResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class JobDetail(google___protobuf___message___Message): + class MetricsEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: float + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[float] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.JobDetail.MetricsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + id = ... # type: typing___Text + extId = ... # type: typing___Text + type = ... # type: typing___Text + runner = ... # type: typing___Text + status = ... # type: typing___Text + entities = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + features = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def metrics(self) -> typing___MutableMapping[typing___Text, float]: ... + + @property + def lastUpdated(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + @property + def created(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + id : typing___Optional[typing___Text] = None, + extId : typing___Optional[typing___Text] = None, + type : typing___Optional[typing___Text] = None, + runner : typing___Optional[typing___Text] = None, + status : typing___Optional[typing___Text] = None, + entities : typing___Optional[typing___Iterable[typing___Text]] = None, + features : typing___Optional[typing___Iterable[typing___Text]] = None, + metrics : typing___Optional[typing___Mapping[typing___Text, float]] = None, + lastUpdated : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + created : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes.JobDetail: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"created",u"lastUpdated"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"created",u"entities",u"extId",u"features",u"id",u"lastUpdated",u"metrics",u"runner",u"status",u"type"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"created",b"created",u"lastUpdated",b"lastUpdated"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"created",b"entities",b"extId",b"features",b"id",b"lastUpdated",b"metrics",b"runner",b"status",b"type"]) -> None: ... + + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> JobServiceTypes: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... diff --git a/sdk/python/feast/core/UIService_pb2.py b/sdk/python/feast/core/UIService_pb2.py index d4d477ac850..3c237639aa6 100644 --- a/sdk/python/feast/core/UIService_pb2.py +++ b/sdk/python/feast/core/UIService_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/UIService.proto diff --git a/sdk/python/feast/core/UIService_pb2.pyi b/sdk/python/feast/core/UIService_pb2.pyi new file mode 100644 index 00000000000..cb39928dce6 --- /dev/null +++ b/sdk/python/feast/core/UIService_pb2.pyi @@ -0,0 +1,362 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.specs.EntitySpec_pb2 import ( + EntitySpec as feast___specs___EntitySpec_pb2___EntitySpec, +) + +from feast.specs.FeatureGroupSpec_pb2 import ( + FeatureGroupSpec as feast___specs___FeatureGroupSpec_pb2___FeatureGroupSpec, +) + +from feast.specs.FeatureSpec_pb2 import ( + FeatureSpec as feast___specs___FeatureSpec_pb2___FeatureSpec, +) + +from feast.specs.StorageSpec_pb2 import ( + StorageSpec as feast___specs___StorageSpec_pb2___StorageSpec, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class UIServiceTypes(google___protobuf___message___Message): + class EntityDetail(google___protobuf___message___Message): + jobs = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def spec(self) -> feast___specs___EntitySpec_pb2___EntitySpec: ... + + @property + def lastUpdated(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + spec : typing___Optional[feast___specs___EntitySpec_pb2___EntitySpec] = None, + jobs : typing___Optional[typing___Iterable[typing___Text]] = None, + lastUpdated : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.EntityDetail: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"lastUpdated",u"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"jobs",u"lastUpdated",u"spec"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"lastUpdated",b"lastUpdated",u"spec",b"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"jobs",b"lastUpdated",b"spec"]) -> None: ... + + class GetEntityRequest(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetEntityRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class GetEntityResponse(google___protobuf___message___Message): + + @property + def entity(self) -> UIServiceTypes.EntityDetail: ... + + def __init__(self, + entity : typing___Optional[UIServiceTypes.EntityDetail] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetEntityResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"entity"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entity"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"entity",b"entity"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"entity"]) -> None: ... + + class ListEntitiesResponse(google___protobuf___message___Message): + + @property + def entities(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[UIServiceTypes.EntityDetail]: ... + + def __init__(self, + entities : typing___Optional[typing___Iterable[UIServiceTypes.EntityDetail]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.ListEntitiesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entities"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entities"]) -> None: ... + + class FeatureDetail(google___protobuf___message___Message): + bigqueryView = ... # type: typing___Text + enabled = ... # type: bool + jobs = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def spec(self) -> feast___specs___FeatureSpec_pb2___FeatureSpec: ... + + @property + def lastUpdated(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + @property + def created(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + spec : typing___Optional[feast___specs___FeatureSpec_pb2___FeatureSpec] = None, + bigqueryView : typing___Optional[typing___Text] = None, + enabled : typing___Optional[bool] = None, + jobs : typing___Optional[typing___Iterable[typing___Text]] = None, + lastUpdated : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + created : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.FeatureDetail: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"created",u"lastUpdated",u"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bigqueryView",u"created",u"enabled",u"jobs",u"lastUpdated",u"spec"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"created",b"created",u"lastUpdated",b"lastUpdated",u"spec",b"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"bigqueryView",b"created",b"enabled",b"jobs",b"lastUpdated",b"spec"]) -> None: ... + + class GetFeatureRequest(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetFeatureRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class GetFeatureResponse(google___protobuf___message___Message): + + @property + def feature(self) -> UIServiceTypes.FeatureDetail: ... + + @property + def rawSpec(self) -> feast___specs___FeatureSpec_pb2___FeatureSpec: ... + + def __init__(self, + feature : typing___Optional[UIServiceTypes.FeatureDetail] = None, + rawSpec : typing___Optional[feast___specs___FeatureSpec_pb2___FeatureSpec] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetFeatureResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"feature",u"rawSpec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature",u"rawSpec"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"feature",b"feature",u"rawSpec",b"rawSpec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"feature",b"rawSpec"]) -> None: ... + + class ListFeaturesResponse(google___protobuf___message___Message): + + @property + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[UIServiceTypes.FeatureDetail]: ... + + def __init__(self, + features : typing___Optional[typing___Iterable[UIServiceTypes.FeatureDetail]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.ListFeaturesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"features"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"features"]) -> None: ... + + class FeatureGroupDetail(google___protobuf___message___Message): + + @property + def spec(self) -> feast___specs___FeatureGroupSpec_pb2___FeatureGroupSpec: ... + + @property + def lastUpdated(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + spec : typing___Optional[feast___specs___FeatureGroupSpec_pb2___FeatureGroupSpec] = None, + lastUpdated : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.FeatureGroupDetail: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"lastUpdated",u"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"lastUpdated",u"spec"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"lastUpdated",b"lastUpdated",u"spec",b"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"lastUpdated",b"spec"]) -> None: ... + + class GetFeatureGroupRequest(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetFeatureGroupRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class GetFeatureGroupResponse(google___protobuf___message___Message): + + @property + def featureGroup(self) -> UIServiceTypes.FeatureGroupDetail: ... + + def __init__(self, + featureGroup : typing___Optional[UIServiceTypes.FeatureGroupDetail] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetFeatureGroupResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"featureGroup"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"featureGroup"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"featureGroup",b"featureGroup"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"featureGroup"]) -> None: ... + + class ListFeatureGroupsResponse(google___protobuf___message___Message): + + @property + def featureGroups(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[UIServiceTypes.FeatureGroupDetail]: ... + + def __init__(self, + featureGroups : typing___Optional[typing___Iterable[UIServiceTypes.FeatureGroupDetail]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.ListFeatureGroupsResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"featureGroups"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"featureGroups"]) -> None: ... + + class StorageDetail(google___protobuf___message___Message): + + @property + def spec(self) -> feast___specs___StorageSpec_pb2___StorageSpec: ... + + @property + def lastUpdated(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + spec : typing___Optional[feast___specs___StorageSpec_pb2___StorageSpec] = None, + lastUpdated : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.StorageDetail: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"lastUpdated",u"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"lastUpdated",u"spec"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"lastUpdated",b"lastUpdated",u"spec",b"spec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"lastUpdated",b"spec"]) -> None: ... + + class GetStorageRequest(google___protobuf___message___Message): + id = ... # type: typing___Text + + def __init__(self, + id : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetStorageRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id"]) -> None: ... + + class GetStorageResponse(google___protobuf___message___Message): + + @property + def storage(self) -> UIServiceTypes.StorageDetail: ... + + def __init__(self, + storage : typing___Optional[UIServiceTypes.StorageDetail] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.GetStorageResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"storage"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"storage"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"storage",b"storage"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"storage"]) -> None: ... + + class ListStorageResponse(google___protobuf___message___Message): + + @property + def storage(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[UIServiceTypes.StorageDetail]: ... + + def __init__(self, + storage : typing___Optional[typing___Iterable[UIServiceTypes.StorageDetail]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes.ListStorageResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"storage"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"storage"]) -> None: ... + + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> UIServiceTypes: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index 79db6575ff8..dc6f404ce2b 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -362,8 +362,6 @@ def _apply(self, obj): return self._apply_entity(obj) elif isinstance(obj, FeatureGroup): return self._apply_feature_group(obj) - elif isinstance(obj, Storage): - return self._apply_storage(obj) else: raise TypeError( "Apply can only be passed one of the following \ diff --git a/sdk/python/feast/sdk/importer.py b/sdk/python/feast/sdk/importer.py index 0a86f481203..b3b9d10c1fa 100644 --- a/sdk/python/feast/sdk/importer.py +++ b/sdk/python/feast/sdk/importer.py @@ -124,8 +124,7 @@ def from_csv(cls, schema, features = \ _detect_schema_and_feature(entity, owner, id_column, feature_columns, timestamp_column, - timestamp_value, serving_store, - warehouse_store, df) + timestamp_value,df) iport_spec = _create_import(src_type, source_options, job_options, entity, schema) @@ -133,6 +132,7 @@ def from_csv(cls, source_options["path"])) specs = _specs(iport_spec, Entity(name=entity), features) + return cls(specs, df, props) @classmethod @@ -191,8 +191,7 @@ def from_bq(cls, schema, features = \ _detect_schema_and_feature(entity, owner, id_column, feature_columns, timestamp_column, - timestamp_value, serving_store, - warehouse_store, df) + timestamp_value, df) iport_spec = _create_import("bigquery", source_options, job_options, entity, schema) @@ -250,8 +249,7 @@ def from_df(cls, schema, features = \ _detect_schema_and_feature(entity, owner, id_column, feature_columns, timestamp_column, - timestamp_value, serving_store, - warehouse_store, df) + timestamp_value, df) iport_spec = _create_import(src_type, source_options, job_options, entity, schema) @@ -349,8 +347,7 @@ def _get_remote_location(path, staging_location): def _detect_schema_and_feature(entity, owner, id_column, feature_columns, - timestamp_column, timestamp_value, - serving_store, warehouse_store, df): + timestamp_column, timestamp_value,df): """Create schema object for import spec. Args: @@ -361,10 +358,6 @@ def _detect_schema_and_feature(entity, owner, id_column, feature_columns, rows in dataset feature_columns (str): list of column to be extracted df (pandas.Dataframe): pandas dataframe of the data - serving_store (feast.sdk.resources.feature.DataStore): Defaults to None. - Serving store to write the features in this instance to. - warehouse_store (feast.sdk.resources.feature.DataStore): Defaults to None. - Warehouse store to write the features in this instance to. Returns: feast.specs.ImportSpec_pb2.Schema: schema of the data @@ -401,16 +394,14 @@ def _detect_schema_and_feature(entity, owner, id_column, feature_columns, if column not in df.columns: raise ValueError( "Column with name {} is not found".format(column)) - features[column] = _create_feature(df[column], entity, owner, - serving_store, warehouse_store) + features[column] = _create_feature(df[column], entity, owner) else: # get all column except entity id and timestampColumn feature_columns = list(df.columns.values) _remove_safely(feature_columns, schema.entityIdColumn) _remove_safely(feature_columns, schema.timestampColumn) for column in feature_columns: - features[column] = _create_feature(df[column], entity, owner, - serving_store, warehouse_store) + features[column] = _create_feature(df[column], entity, owner) for col in df.columns: field = schema.fields.add() @@ -425,7 +416,7 @@ def _detect_schema_and_feature(entity, owner, id_column, feature_columns, return schema, features_dict -def _create_feature(column, entity, owner, serving_store, warehouse_store): +def _create_feature(column, entity, owner): """Create Feature object. Args: @@ -446,10 +437,6 @@ def _create_feature(column, entity, owner, serving_store, warehouse_store): entity=entity, owner=owner, value_type=dtype_to_value_type(column.dtype)) - if serving_store is not None: - feature.serving_store = serving_store - if warehouse_store is not None: - feature.warehouse_store = warehouse_store return feature diff --git a/sdk/python/feast/sdk/resources/feature.py b/sdk/python/feast/sdk/resources/feature.py index 714a1404511..d9f746813c0 100644 --- a/sdk/python/feast/sdk/resources/feature.py +++ b/sdk/python/feast/sdk/resources/feature.py @@ -19,7 +19,7 @@ from google.protobuf.json_format import Parse from feast.sdk.utils.print_utils import spec_to_yaml -from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore +from feast.specs.FeatureSpec_pb2 import FeatureSpec class ValueType(enum.Enum): @@ -49,8 +49,6 @@ def __init__(self, value_type=ValueType.DOUBLE, description='', uri='', - warehouse_store=None, - serving_store=None, group='', tags=[], options={}): @@ -65,29 +63,16 @@ def __init__(self, description (str): defaults to "". description of the feature uri (str): defaults to "". uri pointing to the source code or origin of this feature - warehouse_store (feast.sdk.resources.feature.Datastore): - warehouse store id and options - serving_store (feast.sdk.resources.feature.Datastore): serving - store id and options group (str, optional): feature group to inherit from tags (list[str], optional): tags assigned to the feature options (dic, optional): additional options for the feature """ id = '{}.{}'.format(entity, name).lower() - warehouse_store_spec = None - serving_store_spec = None - if serving_store is not None: - serving_store_spec = serving_store.spec - if warehouse_store is not None: - warehouse_store_spec = warehouse_store.spec - data_stores = DataStores( - serving=serving_store_spec, warehouse=warehouse_store_spec) self.__spec = FeatureSpec( id=id, name=name, entity=entity, owner=owner, - dataStores=data_stores, description=description, uri=uri, valueType=value_type.value, @@ -133,24 +118,6 @@ def owner(self): def owner(self, value): self.__spec.owner = value - @property - def warehouse_store(self): - return self.__spec.dataStores.warehouse - - @warehouse_store.setter - def warehouse_store(self, value): - """Set warehouse store from given Datastore""" - self.__spec.dataStores.warehouse.CopyFrom(value.spec) - - @property - def serving_store(self): - return self.__spec.dataStores.serving - - @serving_store.setter - def serving_store(self, value): - """Set serving store from given Datastore""" - self.__spec.dataStores.serving.CopyFrom(value.spec) - @property def description(self): return self.__spec.description @@ -238,20 +205,3 @@ def dump(self, path): with open(path, 'w') as file: file.write(str(self)) print("Saved spec to {}".format(path)) - - -class Datastore: - def __init__(self, id, options={}): - self.__spec = DataStore(id=id, options=options) - - def __str__(self): - """Print the datastore in yaml format - - Returns: - str: yaml formatted representation of the Datastore - """ - return spec_to_yaml(self.__spec) - - @property - def spec(self): - return self.__spec diff --git a/sdk/python/feast/sdk/resources/feature_group.py b/sdk/python/feast/sdk/resources/feature_group.py index 3f73c571ca4..b2d68e253cd 100644 --- a/sdk/python/feast/sdk/resources/feature_group.py +++ b/sdk/python/feast/sdk/resources/feature_group.py @@ -15,7 +15,6 @@ import yaml import json -from feast.specs.FeatureSpec_pb2 import DataStores from feast.specs.FeatureGroupSpec_pb2 import FeatureGroupSpec from feast.sdk.utils.print_utils import spec_to_yaml from google.protobuf.json_format import Parse @@ -26,28 +25,16 @@ class FeatureGroup(): Wrapper class for feast feature group """ - def __init__(self, id, tags=[], warehouse_store=None, serving_store=None): + def __init__(self, id, tags=[]): """Create FeatureGroup instance. Args: id (str): id of feature group tags (list): Defaults to []. tags assigned to feature group as well as all children features. - warehouse_store (feast.sdk.resources.feature.Datastore): - warehouse store id and options - serving_store (feast.sdk.resources.feature.Datastore): - serving store id and options """ - warehouse_store_spec = None - serving_store_spec = None - if (serving_store is not None): - serving_store_spec = serving_store.spec - if (warehouse_store is not None): - warehouse_store_spec = warehouse_store.spec - data_stores = DataStores( - serving=serving_store_spec, warehouse=warehouse_store_spec) self.__spec = FeatureGroupSpec( - id=id, tags=tags, dataStores=data_stores) + id=id, tags=tags) @property def spec(self): @@ -61,22 +48,6 @@ def id(self): def id(self, value): self.__spec.id = value - @property - def warehouse_store(self): - return self.__spec.dataStores.warehouse - - @warehouse_store.setter - def warehouse_store(self, value): - self.__spec.dataStores.serving.CopyFrom(value) - - @property - def serving_store(self): - return self.__spec.dataStores.serving - - @serving_store.setter - def serving_store(self, value): - self.__spec.dataStores.warehouse.CopyFrom(value) - @property def tags(self): return self.__spec.tags diff --git a/sdk/python/feast/serving/Serving_pb2.py b/sdk/python/feast/serving/Serving_pb2.py index 4f7900e2c3e..3777a32c7b7 100644 --- a/sdk/python/feast/serving/Serving_pb2.py +++ b/sdk/python/feast/serving/Serving_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/serving/Serving.proto diff --git a/sdk/python/feast/serving/Serving_pb2.pyi b/sdk/python/feast/serving/Serving_pb2.pyi new file mode 100644 index 00000000000..88b92ee019b --- /dev/null +++ b/sdk/python/feast/serving/Serving_pb2.pyi @@ -0,0 +1,150 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.types.Value_pb2 import ( + Value as feast___types___Value_pb2___Value, +) + +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class QueryFeaturesRequest(google___protobuf___message___Message): + entityName = ... # type: typing___Text + entityId = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + featureId = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + entityName : typing___Optional[typing___Text] = None, + entityId : typing___Optional[typing___Iterable[typing___Text]] = None, + featureId : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> QueryFeaturesRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entityId",u"entityName",u"featureId"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entityId",b"entityName",b"featureId"]) -> None: ... + +class QueryFeaturesResponse(google___protobuf___message___Message): + class EntitiesEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + + @property + def value(self) -> Entity: ... + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[Entity] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> QueryFeaturesResponse.EntitiesEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + entityName = ... # type: typing___Text + + @property + def entities(self) -> typing___MutableMapping[typing___Text, Entity]: ... + + def __init__(self, + entityName : typing___Optional[typing___Text] = None, + entities : typing___Optional[typing___Mapping[typing___Text, Entity]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> QueryFeaturesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entities",u"entityName"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entities",b"entityName"]) -> None: ... + +class Entity(google___protobuf___message___Message): + class FeaturesEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + + @property + def value(self) -> FeatureValue: ... + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[FeatureValue] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Entity.FeaturesEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + + @property + def features(self) -> typing___MutableMapping[typing___Text, FeatureValue]: ... + + def __init__(self, + features : typing___Optional[typing___Mapping[typing___Text, FeatureValue]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Entity: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"features"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"features"]) -> None: ... + +class FeatureValue(google___protobuf___message___Message): + + @property + def value(self) -> feast___types___Value_pb2___Value: ... + + @property + def timestamp(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + value : typing___Optional[feast___types___Value_pb2___Value] = None, + timestamp : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureValue: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"timestamp",u"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"timestamp",u"value"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"timestamp",b"timestamp",u"value",b"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"timestamp",b"value"]) -> None: ... diff --git a/sdk/python/feast/specs/EntitySpec_pb2.py b/sdk/python/feast/specs/EntitySpec_pb2.py index 2d8dec931e3..8e0096cd83e 100644 --- a/sdk/python/feast/specs/EntitySpec_pb2.py +++ b/sdk/python/feast/specs/EntitySpec_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/specs/EntitySpec.proto diff --git a/sdk/python/feast/specs/EntitySpec_pb2.pyi b/sdk/python/feast/specs/EntitySpec_pb2.pyi new file mode 100644 index 00000000000..603e839e6b8 --- /dev/null +++ b/sdk/python/feast/specs/EntitySpec_pb2.pyi @@ -0,0 +1,39 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class EntitySpec(google___protobuf___message___Message): + name = ... # type: typing___Text + description = ... # type: typing___Text + tags = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + name : typing___Optional[typing___Text] = None, + description : typing___Optional[typing___Text] = None, + tags : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> EntitySpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"description",u"name",u"tags"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"description",b"name",b"tags"]) -> None: ... diff --git a/sdk/python/feast/specs/FeatureGroupSpec_pb2.py b/sdk/python/feast/specs/FeatureGroupSpec_pb2.py index e4f0f1a2756..6b600334025 100644 --- a/sdk/python/feast/specs/FeatureGroupSpec_pb2.py +++ b/sdk/python/feast/specs/FeatureGroupSpec_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/specs/FeatureGroupSpec.proto @@ -20,13 +21,50 @@ package='feast.specs', syntax='proto3', serialized_options=_b('\n\013feast.specsB\025FeatureGroupSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specs'), - serialized_pb=_b('\n\"feast/specs/FeatureGroupSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\"Y\n\x10\x46\x65\x61tureGroupSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12+\n\ndataStores\x18\x03 \x01(\x0b\x32\x17.feast.specs.DataStoresB\\\n\x0b\x66\x65\x61st.specsB\x15\x46\x65\x61tureGroupSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') + serialized_pb=_b('\n\"feast/specs/FeatureGroupSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\"\x99\x01\n\x10\x46\x65\x61tureGroupSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12;\n\x07options\x18\x03 \x03(\x0b\x32*.feast.specs.FeatureGroupSpec.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\\\n\x0b\x66\x65\x61st.specsB\x15\x46\x65\x61tureGroupSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') , dependencies=[feast_dot_specs_dot_FeatureSpec__pb2.DESCRIPTOR,]) +_FEATUREGROUPSPEC_OPTIONSENTRY = _descriptor.Descriptor( + name='OptionsEntry', + full_name='feast.specs.FeatureGroupSpec.OptionsEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='feast.specs.FeatureGroupSpec.OptionsEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='feast.specs.FeatureGroupSpec.OptionsEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=_b('8\001'), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=190, + serialized_end=236, +) + _FEATUREGROUPSPEC = _descriptor.Descriptor( name='FeatureGroupSpec', full_name='feast.specs.FeatureGroupSpec', @@ -49,16 +87,16 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='dataStores', full_name='feast.specs.FeatureGroupSpec.dataStores', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, + name='options', full_name='feast.specs.FeatureGroupSpec.options', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], - nested_types=[], + nested_types=[_FEATUREGROUPSPEC_OPTIONSENTRY, ], enum_types=[ ], serialized_options=None, @@ -67,21 +105,31 @@ extension_ranges=[], oneofs=[ ], - serialized_start=82, - serialized_end=171, + serialized_start=83, + serialized_end=236, ) -_FEATUREGROUPSPEC.fields_by_name['dataStores'].message_type = feast_dot_specs_dot_FeatureSpec__pb2._DATASTORES +_FEATUREGROUPSPEC_OPTIONSENTRY.containing_type = _FEATUREGROUPSPEC +_FEATUREGROUPSPEC.fields_by_name['options'].message_type = _FEATUREGROUPSPEC_OPTIONSENTRY DESCRIPTOR.message_types_by_name['FeatureGroupSpec'] = _FEATUREGROUPSPEC _sym_db.RegisterFileDescriptor(DESCRIPTOR) FeatureGroupSpec = _reflection.GeneratedProtocolMessageType('FeatureGroupSpec', (_message.Message,), dict( + + OptionsEntry = _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), dict( + DESCRIPTOR = _FEATUREGROUPSPEC_OPTIONSENTRY, + __module__ = 'feast.specs.FeatureGroupSpec_pb2' + # @@protoc_insertion_point(class_scope:feast.specs.FeatureGroupSpec.OptionsEntry) + )) + , DESCRIPTOR = _FEATUREGROUPSPEC, __module__ = 'feast.specs.FeatureGroupSpec_pb2' # @@protoc_insertion_point(class_scope:feast.specs.FeatureGroupSpec) )) _sym_db.RegisterMessage(FeatureGroupSpec) +_sym_db.RegisterMessage(FeatureGroupSpec.OptionsEntry) DESCRIPTOR._options = None +_FEATUREGROUPSPEC_OPTIONSENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/specs/FeatureGroupSpec_pb2.pyi b/sdk/python/feast/specs/FeatureGroupSpec_pb2.pyi new file mode 100644 index 00000000000..1274401977f --- /dev/null +++ b/sdk/python/feast/specs/FeatureGroupSpec_pb2.pyi @@ -0,0 +1,60 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class FeatureGroupSpec(google___protobuf___message___Message): + class OptionsEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureGroupSpec.OptionsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + id = ... # type: typing___Text + tags = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def options(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + def __init__(self, + id : typing___Optional[typing___Text] = None, + tags : typing___Optional[typing___Iterable[typing___Text]] = None, + options : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureGroupSpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id",u"options",u"tags"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id",b"options",b"tags"]) -> None: ... diff --git a/sdk/python/feast/specs/FeatureSpec_pb2.py b/sdk/python/feast/specs/FeatureSpec_pb2.py index ce50ca533a9..1032f34fcec 100644 --- a/sdk/python/feast/specs/FeatureSpec_pb2.py +++ b/sdk/python/feast/specs/FeatureSpec_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/specs/FeatureSpec.proto @@ -22,7 +23,7 @@ package='feast.specs', syntax='proto3', serialized_options=_b('\n\013feast.specsB\020FeatureSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specs'), - serialized_pb=_b('\n\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\xca\x02\n\x0b\x46\x65\x61tureSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05owner\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x0b\n\x03uri\x18\x05 \x01(\t\x12.\n\tvalueType\x18\x07 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x0e\n\x06\x65ntity\x18\x08 \x01(\t\x12\r\n\x05group\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12\x36\n\x07options\x18\x0b \x03(\x0b\x32%.feast.specs.FeatureSpec.OptionsEntry\x12+\n\ndataStores\x18\x0c \x01(\x0b\x32\x17.feast.specs.DataStores\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n\nDataStores\x12\'\n\x07serving\x18\x01 \x01(\x0b\x32\x16.feast.specs.DataStore\x12)\n\twarehouse\x18\x02 \x01(\x0b\x32\x16.feast.specs.DataStore\"}\n\tDataStore\x12\n\n\x02id\x18\x01 \x01(\t\x12\x34\n\x07options\x18\x02 \x03(\x0b\x32#.feast.specs.DataStore.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42W\n\x0b\x66\x65\x61st.specsB\x10\x46\x65\x61tureSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') + serialized_pb=_b('\n\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\x9d\x02\n\x0b\x46\x65\x61tureSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05owner\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x0b\n\x03uri\x18\x05 \x01(\t\x12.\n\tvalueType\x18\x07 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x0e\n\x06\x65ntity\x18\x08 \x01(\t\x12\r\n\x05group\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12\x36\n\x07options\x18\x0b \x03(\x0b\x32%.feast.specs.FeatureSpec.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42W\n\x0b\x66\x65\x61st.specsB\x10\x46\x65\x61tureSpecProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') , dependencies=[feast_dot_specs_dot_EntitySpec__pb2.DESCRIPTOR,feast_dot_specs_dot_StorageSpec__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) @@ -62,8 +63,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=417, - serialized_end=463, + serialized_start=372, + serialized_end=418, ) _FEATURESPEC = _descriptor.Descriptor( @@ -143,13 +144,6 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dataStores', full_name='feast.specs.FeatureSpec.dataStores', index=10, - number=12, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -163,133 +157,13 @@ oneofs=[ ], serialized_start=133, - serialized_end=463, -) - - -_DATASTORES = _descriptor.Descriptor( - name='DataStores', - full_name='feast.specs.DataStores', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='serving', full_name='feast.specs.DataStores.serving', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='warehouse', full_name='feast.specs.DataStores.warehouse', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=465, - serialized_end=561, -) - - -_DATASTORE_OPTIONSENTRY = _descriptor.Descriptor( - name='OptionsEntry', - full_name='feast.specs.DataStore.OptionsEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='feast.specs.DataStore.OptionsEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='feast.specs.DataStore.OptionsEntry.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=_b('8\001'), - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=417, - serialized_end=463, -) - -_DATASTORE = _descriptor.Descriptor( - name='DataStore', - full_name='feast.specs.DataStore', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='id', full_name='feast.specs.DataStore.id', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='options', full_name='feast.specs.DataStore.options', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_DATASTORE_OPTIONSENTRY, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=563, - serialized_end=688, + serialized_end=418, ) _FEATURESPEC_OPTIONSENTRY.containing_type = _FEATURESPEC _FEATURESPEC.fields_by_name['valueType'].enum_type = feast_dot_types_dot_Value__pb2._VALUETYPE_ENUM _FEATURESPEC.fields_by_name['options'].message_type = _FEATURESPEC_OPTIONSENTRY -_FEATURESPEC.fields_by_name['dataStores'].message_type = _DATASTORES -_DATASTORES.fields_by_name['serving'].message_type = _DATASTORE -_DATASTORES.fields_by_name['warehouse'].message_type = _DATASTORE -_DATASTORE_OPTIONSENTRY.containing_type = _DATASTORE -_DATASTORE.fields_by_name['options'].message_type = _DATASTORE_OPTIONSENTRY DESCRIPTOR.message_types_by_name['FeatureSpec'] = _FEATURESPEC -DESCRIPTOR.message_types_by_name['DataStores'] = _DATASTORES -DESCRIPTOR.message_types_by_name['DataStore'] = _DATASTORE _sym_db.RegisterFileDescriptor(DESCRIPTOR) FeatureSpec = _reflection.GeneratedProtocolMessageType('FeatureSpec', (_message.Message,), dict( @@ -307,30 +181,7 @@ _sym_db.RegisterMessage(FeatureSpec) _sym_db.RegisterMessage(FeatureSpec.OptionsEntry) -DataStores = _reflection.GeneratedProtocolMessageType('DataStores', (_message.Message,), dict( - DESCRIPTOR = _DATASTORES, - __module__ = 'feast.specs.FeatureSpec_pb2' - # @@protoc_insertion_point(class_scope:feast.specs.DataStores) - )) -_sym_db.RegisterMessage(DataStores) - -DataStore = _reflection.GeneratedProtocolMessageType('DataStore', (_message.Message,), dict( - - OptionsEntry = _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), dict( - DESCRIPTOR = _DATASTORE_OPTIONSENTRY, - __module__ = 'feast.specs.FeatureSpec_pb2' - # @@protoc_insertion_point(class_scope:feast.specs.DataStore.OptionsEntry) - )) - , - DESCRIPTOR = _DATASTORE, - __module__ = 'feast.specs.FeatureSpec_pb2' - # @@protoc_insertion_point(class_scope:feast.specs.DataStore) - )) -_sym_db.RegisterMessage(DataStore) -_sym_db.RegisterMessage(DataStore.OptionsEntry) - DESCRIPTOR._options = None _FEATURESPEC_OPTIONSENTRY._options = None -_DATASTORE_OPTIONSENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/specs/FeatureSpec_pb2.pyi b/sdk/python/feast/specs/FeatureSpec_pb2.pyi new file mode 100644 index 00000000000..6c703c2d4e3 --- /dev/null +++ b/sdk/python/feast/specs/FeatureSpec_pb2.pyi @@ -0,0 +1,78 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.types.Value_pb2 import ( + ValueType as feast___types___Value_pb2___ValueType, +) + +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class FeatureSpec(google___protobuf___message___Message): + class OptionsEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureSpec.OptionsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + id = ... # type: typing___Text + name = ... # type: typing___Text + owner = ... # type: typing___Text + description = ... # type: typing___Text + uri = ... # type: typing___Text + valueType = ... # type: feast___types___Value_pb2___ValueType.Enum + entity = ... # type: typing___Text + group = ... # type: typing___Text + tags = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def options(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + def __init__(self, + id : typing___Optional[typing___Text] = None, + name : typing___Optional[typing___Text] = None, + owner : typing___Optional[typing___Text] = None, + description : typing___Optional[typing___Text] = None, + uri : typing___Optional[typing___Text] = None, + valueType : typing___Optional[feast___types___Value_pb2___ValueType.Enum] = None, + entity : typing___Optional[typing___Text] = None, + group : typing___Optional[typing___Text] = None, + tags : typing___Optional[typing___Iterable[typing___Text]] = None, + options : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureSpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"description",u"entity",u"group",u"id",u"name",u"options",u"owner",u"tags",u"uri",u"valueType"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"description",b"entity",b"group",b"id",b"name",b"options",b"owner",b"tags",b"uri",b"valueType"]) -> None: ... diff --git a/sdk/python/feast/specs/ImportJobSpecs_pb2.py b/sdk/python/feast/specs/ImportJobSpecs_pb2.py index da84644da8c..205c4e5e8af 100644 --- a/sdk/python/feast/specs/ImportJobSpecs_pb2.py +++ b/sdk/python/feast/specs/ImportJobSpecs_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/specs/ImportJobSpecs.proto @@ -23,7 +24,7 @@ package='feast.specs', syntax='proto3', serialized_options=_b('\n\013feast.specsB\023ImportJobSpecsProtoZ6github.com/gojek/feast/protos/generated/go/feast/specs'), - serialized_pb=_b('\n feast/specs/ImportJobSpecs.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/ImportSpec.proto\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\"\xcf\x02\n\x0eImportJobSpecs\x12\r\n\x05jobId\x18\x01 \x01(\t\x12+\n\nimportSpec\x18\x02 \x01(\x0b\x32\x17.feast.specs.ImportSpec\x12,\n\x0b\x65ntitySpecs\x18\x03 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x12.\n\x0c\x66\x65\x61tureSpecs\x18\x04 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x12\x35\n\x13servingStorageSpecs\x18\x05 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x12\x37\n\x15warehouseStorageSpecs\x18\x06 \x03(\x0b\x32\x18.feast.specs.StorageSpec\x12\x33\n\x11\x65rrorsStorageSpec\x18\x07 \x01(\x0b\x32\x18.feast.specs.StorageSpecBZ\n\x0b\x66\x65\x61st.specsB\x13ImportJobSpecsProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') + serialized_pb=_b('\n feast/specs/ImportJobSpecs.proto\x12\x0b\x66\x65\x61st.specs\x1a\x1c\x66\x65\x61st/specs/ImportSpec.proto\x1a\x1c\x66\x65\x61st/specs/EntitySpec.proto\x1a\x1d\x66\x65\x61st/specs/FeatureSpec.proto\x1a\x1d\x66\x65\x61st/specs/StorageSpec.proto\"\xcb\x03\n\x0eImportJobSpecs\x12\r\n\x05jobId\x18\x01 \x01(\t\x12+\n\nimportSpec\x18\x02 \x01(\x0b\x32\x17.feast.specs.ImportSpec\x12,\n\x0b\x65ntitySpecs\x18\x03 \x03(\x0b\x32\x17.feast.specs.EntitySpec\x12.\n\x0c\x66\x65\x61tureSpecs\x18\x04 \x03(\x0b\x32\x18.feast.specs.FeatureSpec\x12\x34\n\x12servingStorageSpec\x18\x05 \x01(\x0b\x32\x18.feast.specs.StorageSpec\x12\x36\n\x14warehouseStorageSpec\x18\x06 \x01(\x0b\x32\x18.feast.specs.StorageSpec\x12\x33\n\x11\x65rrorsStorageSpec\x18\x07 \x01(\x0b\x32\x18.feast.specs.StorageSpec\x12*\n\"write_feature_metrics_to_influx_db\x18\x08 \x01(\x08\x12\x15\n\rinflux_db_url\x18\t \x01(\t\x12\x1a\n\x12influx_db_database\x18\n \x01(\t\x12\x1d\n\x15influx_db_measurement\x18\x0b \x01(\tBZ\n\x0b\x66\x65\x61st.specsB\x13ImportJobSpecsProtoZ6github.com/gojek/feast/protos/generated/go/feast/specsb\x06proto3') , dependencies=[feast_dot_specs_dot_ImportSpec__pb2.DESCRIPTOR,feast_dot_specs_dot_EntitySpec__pb2.DESCRIPTOR,feast_dot_specs_dot_FeatureSpec__pb2.DESCRIPTOR,feast_dot_specs_dot_StorageSpec__pb2.DESCRIPTOR,]) @@ -66,16 +67,16 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='servingStorageSpecs', full_name='feast.specs.ImportJobSpecs.servingStorageSpecs', index=4, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], + name='servingStorageSpec', full_name='feast.specs.ImportJobSpecs.servingStorageSpec', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='warehouseStorageSpecs', full_name='feast.specs.ImportJobSpecs.warehouseStorageSpecs', index=5, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], + name='warehouseStorageSpec', full_name='feast.specs.ImportJobSpecs.warehouseStorageSpec', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -86,6 +87,34 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='write_feature_metrics_to_influx_db', full_name='feast.specs.ImportJobSpecs.write_feature_metrics_to_influx_db', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='influx_db_url', full_name='feast.specs.ImportJobSpecs.influx_db_url', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='influx_db_database', full_name='feast.specs.ImportJobSpecs.influx_db_database', index=9, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='influx_db_measurement', full_name='feast.specs.ImportJobSpecs.influx_db_measurement', index=10, + number=11, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -99,14 +128,14 @@ oneofs=[ ], serialized_start=172, - serialized_end=507, + serialized_end=631, ) _IMPORTJOBSPECS.fields_by_name['importSpec'].message_type = feast_dot_specs_dot_ImportSpec__pb2._IMPORTSPEC _IMPORTJOBSPECS.fields_by_name['entitySpecs'].message_type = feast_dot_specs_dot_EntitySpec__pb2._ENTITYSPEC _IMPORTJOBSPECS.fields_by_name['featureSpecs'].message_type = feast_dot_specs_dot_FeatureSpec__pb2._FEATURESPEC -_IMPORTJOBSPECS.fields_by_name['servingStorageSpecs'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC -_IMPORTJOBSPECS.fields_by_name['warehouseStorageSpecs'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC +_IMPORTJOBSPECS.fields_by_name['servingStorageSpec'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC +_IMPORTJOBSPECS.fields_by_name['warehouseStorageSpec'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC _IMPORTJOBSPECS.fields_by_name['errorsStorageSpec'].message_type = feast_dot_specs_dot_StorageSpec__pb2._STORAGESPEC DESCRIPTOR.message_types_by_name['ImportJobSpecs'] = _IMPORTJOBSPECS _sym_db.RegisterFileDescriptor(DESCRIPTOR) diff --git a/sdk/python/feast/specs/ImportJobSpecs_pb2.pyi b/sdk/python/feast/specs/ImportJobSpecs_pb2.pyi new file mode 100644 index 00000000000..9643ccc25f7 --- /dev/null +++ b/sdk/python/feast/specs/ImportJobSpecs_pb2.pyi @@ -0,0 +1,85 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.specs.EntitySpec_pb2 import ( + EntitySpec as feast___specs___EntitySpec_pb2___EntitySpec, +) + +from feast.specs.FeatureSpec_pb2 import ( + FeatureSpec as feast___specs___FeatureSpec_pb2___FeatureSpec, +) + +from feast.specs.ImportSpec_pb2 import ( + ImportSpec as feast___specs___ImportSpec_pb2___ImportSpec, +) + +from feast.specs.StorageSpec_pb2 import ( + StorageSpec as feast___specs___StorageSpec_pb2___StorageSpec, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class ImportJobSpecs(google___protobuf___message___Message): + jobId = ... # type: typing___Text + write_feature_metrics_to_influx_db = ... # type: bool + influx_db_url = ... # type: typing___Text + influx_db_database = ... # type: typing___Text + influx_db_measurement = ... # type: typing___Text + + @property + def importSpec(self) -> feast___specs___ImportSpec_pb2___ImportSpec: ... + + @property + def entitySpecs(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___EntitySpec_pb2___EntitySpec]: ... + + @property + def featureSpecs(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___specs___FeatureSpec_pb2___FeatureSpec]: ... + + @property + def servingStorageSpec(self) -> feast___specs___StorageSpec_pb2___StorageSpec: ... + + @property + def warehouseStorageSpec(self) -> feast___specs___StorageSpec_pb2___StorageSpec: ... + + @property + def errorsStorageSpec(self) -> feast___specs___StorageSpec_pb2___StorageSpec: ... + + def __init__(self, + jobId : typing___Optional[typing___Text] = None, + importSpec : typing___Optional[feast___specs___ImportSpec_pb2___ImportSpec] = None, + entitySpecs : typing___Optional[typing___Iterable[feast___specs___EntitySpec_pb2___EntitySpec]] = None, + featureSpecs : typing___Optional[typing___Iterable[feast___specs___FeatureSpec_pb2___FeatureSpec]] = None, + servingStorageSpec : typing___Optional[feast___specs___StorageSpec_pb2___StorageSpec] = None, + warehouseStorageSpec : typing___Optional[feast___specs___StorageSpec_pb2___StorageSpec] = None, + errorsStorageSpec : typing___Optional[feast___specs___StorageSpec_pb2___StorageSpec] = None, + write_feature_metrics_to_influx_db : typing___Optional[bool] = None, + influx_db_url : typing___Optional[typing___Text] = None, + influx_db_database : typing___Optional[typing___Text] = None, + influx_db_measurement : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ImportJobSpecs: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"errorsStorageSpec",u"importSpec",u"servingStorageSpec",u"warehouseStorageSpec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entitySpecs",u"errorsStorageSpec",u"featureSpecs",u"importSpec",u"influx_db_database",u"influx_db_measurement",u"influx_db_url",u"jobId",u"servingStorageSpec",u"warehouseStorageSpec",u"write_feature_metrics_to_influx_db"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"errorsStorageSpec",b"errorsStorageSpec",u"importSpec",b"importSpec",u"servingStorageSpec",b"servingStorageSpec",u"warehouseStorageSpec",b"warehouseStorageSpec"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"entitySpecs",b"errorsStorageSpec",b"featureSpecs",b"importSpec",b"influx_db_database",b"influx_db_measurement",b"influx_db_url",b"jobId",b"servingStorageSpec",b"warehouseStorageSpec",b"write_feature_metrics_to_influx_db"]) -> None: ... diff --git a/sdk/python/feast/specs/ImportSpec_pb2.py b/sdk/python/feast/specs/ImportSpec_pb2.py index 60e5b4f400a..e04cb21549e 100644 --- a/sdk/python/feast/specs/ImportSpec_pb2.py +++ b/sdk/python/feast/specs/ImportSpec_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/specs/ImportSpec.proto diff --git a/sdk/python/feast/specs/ImportSpec_pb2.pyi b/sdk/python/feast/specs/ImportSpec_pb2.pyi new file mode 100644 index 00000000000..6c686a9e1d2 --- /dev/null +++ b/sdk/python/feast/specs/ImportSpec_pb2.pyi @@ -0,0 +1,137 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class ImportSpec(google___protobuf___message___Message): + class SourceOptionsEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ImportSpec.SourceOptionsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + class JobOptionsEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ImportSpec.JobOptionsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + type = ... # type: typing___Text + entities = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def sourceOptions(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + @property + def jobOptions(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + @property + def schema(self) -> Schema: ... + + def __init__(self, + type : typing___Optional[typing___Text] = None, + sourceOptions : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + jobOptions : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + entities : typing___Optional[typing___Iterable[typing___Text]] = None, + schema : typing___Optional[Schema] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ImportSpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"schema"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entities",u"jobOptions",u"schema",u"sourceOptions",u"type"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"schema",b"schema"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"entities",b"jobOptions",b"schema",b"sourceOptions",b"type"]) -> None: ... + +class Schema(google___protobuf___message___Message): + timestampColumn = ... # type: typing___Text + entityIdColumn = ... # type: typing___Text + + @property + def fields(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Field]: ... + + @property + def timestampValue(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + fields : typing___Optional[typing___Iterable[Field]] = None, + timestampColumn : typing___Optional[typing___Text] = None, + timestampValue : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + entityIdColumn : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Schema: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"timestamp",u"timestampColumn",u"timestampValue"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entityIdColumn",u"fields",u"timestamp",u"timestampColumn",u"timestampValue"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"timestamp",b"timestamp",u"timestampColumn",b"timestampColumn",u"timestampValue",b"timestampValue"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"entityIdColumn",b"fields",b"timestamp",b"timestampColumn",b"timestampValue"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"timestamp",b"timestamp"]) -> typing_extensions___Literal["timestampColumn","timestampValue"]: ... + +class Field(google___protobuf___message___Message): + name = ... # type: typing___Text + featureId = ... # type: typing___Text + + def __init__(self, + name : typing___Optional[typing___Text] = None, + featureId : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Field: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"featureId",u"name"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"featureId",b"name"]) -> None: ... diff --git a/sdk/python/feast/specs/StorageSpec_pb2.py b/sdk/python/feast/specs/StorageSpec_pb2.py index 5f4b89bf1af..eb5a44d2034 100644 --- a/sdk/python/feast/specs/StorageSpec_pb2.py +++ b/sdk/python/feast/specs/StorageSpec_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/specs/StorageSpec.proto diff --git a/sdk/python/feast/specs/StorageSpec_pb2.pyi b/sdk/python/feast/specs/StorageSpec_pb2.pyi new file mode 100644 index 00000000000..f305abeb771 --- /dev/null +++ b/sdk/python/feast/specs/StorageSpec_pb2.pyi @@ -0,0 +1,55 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class StorageSpec(google___protobuf___message___Message): + class OptionsEntry(google___protobuf___message___Message): + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> StorageSpec.OptionsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"key",b"value"]) -> None: ... + + id = ... # type: typing___Text + type = ... # type: typing___Text + + @property + def options(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + def __init__(self, + id : typing___Optional[typing___Text] = None, + type : typing___Optional[typing___Text] = None, + options : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> StorageSpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"id",u"options",u"type"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"id",b"options",b"type"]) -> None: ... diff --git a/sdk/python/feast/storage/BigTable_pb2.py b/sdk/python/feast/storage/BigTable_pb2.py index 4b59ff8b177..c8ec3095a10 100644 --- a/sdk/python/feast/storage/BigTable_pb2.py +++ b/sdk/python/feast/storage/BigTable_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/storage/BigTable.proto diff --git a/sdk/python/feast/storage/BigTable_pb2.pyi b/sdk/python/feast/storage/BigTable_pb2.pyi new file mode 100644 index 00000000000..1744f29485c --- /dev/null +++ b/sdk/python/feast/storage/BigTable_pb2.pyi @@ -0,0 +1,34 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class BigTableRowKey(google___protobuf___message___Message): + sha1Prefix = ... # type: typing___Text + entityKey = ... # type: typing___Text + reversedMillis = ... # type: typing___Text + + def __init__(self, + sha1Prefix : typing___Optional[typing___Text] = None, + entityKey : typing___Optional[typing___Text] = None, + reversedMillis : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> BigTableRowKey: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"entityKey",u"reversedMillis",u"sha1Prefix"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"entityKey",b"reversedMillis",b"sha1Prefix"]) -> None: ... diff --git a/sdk/python/feast/storage/Redis_pb2.py b/sdk/python/feast/storage/Redis_pb2.py index 2220469d497..389963980ab 100644 --- a/sdk/python/feast/storage/Redis_pb2.py +++ b/sdk/python/feast/storage/Redis_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/storage/Redis.proto diff --git a/sdk/python/feast/storage/Redis_pb2.pyi b/sdk/python/feast/storage/Redis_pb2.pyi new file mode 100644 index 00000000000..fa4dba708bf --- /dev/null +++ b/sdk/python/feast/storage/Redis_pb2.pyi @@ -0,0 +1,87 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.types.Value_pb2 import ( + Value as feast___types___Value_pb2___Value, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class RedisBucketKey(google___protobuf___message___Message): + entityKey = ... # type: typing___Text + featureIdSha1Prefix = ... # type: typing___Text + bucketId = ... # type: int + + def __init__(self, + entityKey : typing___Optional[typing___Text] = None, + featureIdSha1Prefix : typing___Optional[typing___Text] = None, + bucketId : typing___Optional[int] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> RedisBucketKey: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"bucketId",u"entityKey",u"featureIdSha1Prefix"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"bucketId",b"entityKey",b"featureIdSha1Prefix"]) -> None: ... + +class RedisBucketValue(google___protobuf___message___Message): + + @property + def value(self) -> feast___types___Value_pb2___Value: ... + + @property + def eventTimestamp(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + value : typing___Optional[feast___types___Value_pb2___Value] = None, + eventTimestamp : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> RedisBucketValue: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"eventTimestamp",u"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"eventTimestamp",u"value"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"eventTimestamp",b"eventTimestamp",u"value",b"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"eventTimestamp",b"value"]) -> None: ... + +class RedisBucketValueList(google___protobuf___message___Message): + + @property + def values(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[RedisBucketValue]: ... + + def __init__(self, + values : typing___Optional[typing___Iterable[RedisBucketValue]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> RedisBucketValueList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"values"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"values"]) -> None: ... diff --git a/sdk/python/feast/types/FeatureRowExtended_pb2.py b/sdk/python/feast/types/FeatureRowExtended_pb2.py index 21597cc3045..1fa25c673dd 100644 --- a/sdk/python/feast/types/FeatureRowExtended_pb2.py +++ b/sdk/python/feast/types/FeatureRowExtended_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/FeatureRowExtended.proto diff --git a/sdk/python/feast/types/FeatureRowExtended_pb2.pyi b/sdk/python/feast/types/FeatureRowExtended_pb2.pyi new file mode 100644 index 00000000000..6a02035dc4d --- /dev/null +++ b/sdk/python/feast/types/FeatureRowExtended_pb2.pyi @@ -0,0 +1,92 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.types.FeatureRow_pb2 import ( + FeatureRow as feast___types___FeatureRow_pb2___FeatureRow, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Error(google___protobuf___message___Message): + cause = ... # type: typing___Text + transform = ... # type: typing___Text + message = ... # type: typing___Text + stackTrace = ... # type: typing___Text + + def __init__(self, + cause : typing___Optional[typing___Text] = None, + transform : typing___Optional[typing___Text] = None, + message : typing___Optional[typing___Text] = None, + stackTrace : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Error: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"cause",u"message",u"stackTrace",u"transform"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"cause",b"message",b"stackTrace",b"transform"]) -> None: ... + +class Attempt(google___protobuf___message___Message): + attempts = ... # type: int + + @property + def error(self) -> Error: ... + + def __init__(self, + attempts : typing___Optional[int] = None, + error : typing___Optional[Error] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Attempt: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"error"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"attempts",u"error"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"error",b"error"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"attempts",b"error"]) -> None: ... + +class FeatureRowExtended(google___protobuf___message___Message): + + @property + def row(self) -> feast___types___FeatureRow_pb2___FeatureRow: ... + + @property + def lastAttempt(self) -> Attempt: ... + + @property + def firstSeen(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + row : typing___Optional[feast___types___FeatureRow_pb2___FeatureRow] = None, + lastAttempt : typing___Optional[Attempt] = None, + firstSeen : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureRowExtended: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"firstSeen",u"lastAttempt",u"row"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"firstSeen",u"lastAttempt",u"row"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"firstSeen",b"firstSeen",u"lastAttempt",b"lastAttempt",u"row",b"row"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"firstSeen",b"lastAttempt",b"row"]) -> None: ... diff --git a/sdk/python/feast/types/FeatureRow_pb2.py b/sdk/python/feast/types/FeatureRow_pb2.py index 4096955df82..9e8e9796df4 100644 --- a/sdk/python/feast/types/FeatureRow_pb2.py +++ b/sdk/python/feast/types/FeatureRow_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/FeatureRow.proto @@ -21,58 +22,13 @@ package='feast.types', syntax='proto3', serialized_options=_b('\n\013feast.typesB\017FeatureRowProtoZ6github.com/gojek/feast/protos/generated/go/feast/types'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/types/FeatureRow.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/types/Feature.proto\"j\n\rFeatureRowKey\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\t\"\x8f\x01\n\nFeatureRow\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12&\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x14.feast.types.Feature\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\tBV\n\x0b\x66\x65\x61st.typesB\x0f\x46\x65\x61tureRowProtoZ6github.com/gojek/feast/protos/generated/go/feast/typesb\x06proto3') + serialized_pb=_b('\n\x1c\x66\x65\x61st/types/FeatureRow.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/types/Feature.proto\"\x8f\x01\n\nFeatureRow\x12\x11\n\tentityKey\x18\x01 \x01(\t\x12&\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x14.feast.types.Feature\x12\x32\n\x0e\x65ventTimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nentityName\x18\x04 \x01(\tBV\n\x0b\x66\x65\x61st.typesB\x0f\x46\x65\x61tureRowProtoZ6github.com/gojek/feast/protos/generated/go/feast/typesb\x06proto3') , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Feature__pb2.DESCRIPTOR,]) -_FEATUREROWKEY = _descriptor.Descriptor( - name='FeatureRowKey', - full_name='feast.types.FeatureRowKey', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='entityKey', full_name='feast.types.FeatureRowKey.entityKey', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='eventTimestamp', full_name='feast.types.FeatureRowKey.eventTimestamp', index=1, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='entityName', full_name='feast.types.FeatureRowKey.entityName', index=2, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=105, - serialized_end=211, -) - - _FEATUREROW = _descriptor.Descriptor( name='FeatureRow', full_name='feast.types.FeatureRow', @@ -120,24 +76,15 @@ extension_ranges=[], oneofs=[ ], - serialized_start=214, - serialized_end=357, + serialized_start=106, + serialized_end=249, ) -_FEATUREROWKEY.fields_by_name['eventTimestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _FEATUREROW.fields_by_name['features'].message_type = feast_dot_types_dot_Feature__pb2._FEATURE _FEATUREROW.fields_by_name['eventTimestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -DESCRIPTOR.message_types_by_name['FeatureRowKey'] = _FEATUREROWKEY DESCRIPTOR.message_types_by_name['FeatureRow'] = _FEATUREROW _sym_db.RegisterFileDescriptor(DESCRIPTOR) -FeatureRowKey = _reflection.GeneratedProtocolMessageType('FeatureRowKey', (_message.Message,), dict( - DESCRIPTOR = _FEATUREROWKEY, - __module__ = 'feast.types.FeatureRow_pb2' - # @@protoc_insertion_point(class_scope:feast.types.FeatureRowKey) - )) -_sym_db.RegisterMessage(FeatureRowKey) - FeatureRow = _reflection.GeneratedProtocolMessageType('FeatureRow', (_message.Message,), dict( DESCRIPTOR = _FEATUREROW, __module__ = 'feast.types.FeatureRow_pb2' diff --git a/sdk/python/feast/types/FeatureRow_pb2.pyi b/sdk/python/feast/types/FeatureRow_pb2.pyi new file mode 100644 index 00000000000..1b93879d2a1 --- /dev/null +++ b/sdk/python/feast/types/FeatureRow_pb2.pyi @@ -0,0 +1,55 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.types.Feature_pb2 import ( + Feature as feast___types___Feature_pb2___Feature, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class FeatureRow(google___protobuf___message___Message): + entityKey = ... # type: typing___Text + entityName = ... # type: typing___Text + + @property + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[feast___types___Feature_pb2___Feature]: ... + + @property + def eventTimestamp(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + entityKey : typing___Optional[typing___Text] = None, + features : typing___Optional[typing___Iterable[feast___types___Feature_pb2___Feature]] = None, + eventTimestamp : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + entityName : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FeatureRow: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"eventTimestamp"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entityKey",u"entityName",u"eventTimestamp",u"features"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"eventTimestamp",b"eventTimestamp"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"entityKey",b"entityName",b"eventTimestamp",b"features"]) -> None: ... diff --git a/sdk/python/feast/types/Feature_pb2.py b/sdk/python/feast/types/Feature_pb2.py index b880b4747e5..ab4d27415f3 100644 --- a/sdk/python/feast/types/Feature_pb2.py +++ b/sdk/python/feast/types/Feature_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/Feature.proto diff --git a/sdk/python/feast/types/Feature_pb2.pyi b/sdk/python/feast/types/Feature_pb2.pyi new file mode 100644 index 00000000000..5d7fbc46ed5 --- /dev/null +++ b/sdk/python/feast/types/Feature_pb2.pyi @@ -0,0 +1,40 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from feast.types.Value_pb2 import ( + Value as feast___types___Value_pb2___Value, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Feature(google___protobuf___message___Message): + id = ... # type: typing___Text + + @property + def value(self) -> feast___types___Value_pb2___Value: ... + + def __init__(self, + id : typing___Optional[typing___Text] = None, + value : typing___Optional[feast___types___Value_pb2___Value] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Feature: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"id",u"value"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"id",b"value"]) -> None: ... diff --git a/sdk/python/feast/types/Value_pb2.py b/sdk/python/feast/types/Value_pb2.py index 2e9d3177fff..bc0eb519e93 100644 --- a/sdk/python/feast/types/Value_pb2.py +++ b/sdk/python/feast/types/Value_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/Value.proto diff --git a/sdk/python/feast/types/Value_pb2.pyi b/sdk/python/feast/types/Value_pb2.pyi new file mode 100644 index 00000000000..8aa5092fdfd --- /dev/null +++ b/sdk/python/feast/types/Value_pb2.pyi @@ -0,0 +1,267 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.descriptor import ( + EnumDescriptor as google___protobuf___descriptor___EnumDescriptor, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from google.protobuf.timestamp_pb2 import ( + Timestamp as google___protobuf___timestamp_pb2___Timestamp, +) + +from typing import ( + Iterable as typing___Iterable, + List as typing___List, + Optional as typing___Optional, + Text as typing___Text, + Tuple as typing___Tuple, + cast as typing___cast, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class ValueType(google___protobuf___message___Message): + class Enum(int): + DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... + @classmethod + def Name(cls, number: int) -> str: ... + @classmethod + def Value(cls, name: str) -> ValueType.Enum: ... + @classmethod + def keys(cls) -> typing___List[str]: ... + @classmethod + def values(cls) -> typing___List[ValueType.Enum]: ... + @classmethod + def items(cls) -> typing___List[typing___Tuple[str, ValueType.Enum]]: ... + UNKNOWN = typing___cast(Enum, 0) + BYTES = typing___cast(Enum, 1) + STRING = typing___cast(Enum, 2) + INT32 = typing___cast(Enum, 3) + INT64 = typing___cast(Enum, 4) + DOUBLE = typing___cast(Enum, 5) + FLOAT = typing___cast(Enum, 6) + BOOL = typing___cast(Enum, 7) + TIMESTAMP = typing___cast(Enum, 8) + + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ValueType: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class Value(google___protobuf___message___Message): + bytesVal = ... # type: bytes + stringVal = ... # type: typing___Text + int32Val = ... # type: int + int64Val = ... # type: int + doubleVal = ... # type: float + floatVal = ... # type: float + boolVal = ... # type: bool + + @property + def timestampVal(self) -> google___protobuf___timestamp_pb2___Timestamp: ... + + def __init__(self, + bytesVal : typing___Optional[bytes] = None, + stringVal : typing___Optional[typing___Text] = None, + int32Val : typing___Optional[int] = None, + int64Val : typing___Optional[int] = None, + doubleVal : typing___Optional[float] = None, + floatVal : typing___Optional[float] = None, + boolVal : typing___Optional[bool] = None, + timestampVal : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Value: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"boolVal",u"bytesVal",u"doubleVal",u"floatVal",u"int32Val",u"int64Val",u"stringVal",u"timestampVal",u"val"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"boolVal",u"bytesVal",u"doubleVal",u"floatVal",u"int32Val",u"int64Val",u"stringVal",u"timestampVal",u"val"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"boolVal",b"boolVal",u"bytesVal",b"bytesVal",u"doubleVal",b"doubleVal",u"floatVal",b"floatVal",u"int32Val",b"int32Val",u"int64Val",b"int64Val",u"stringVal",b"stringVal",u"timestampVal",b"timestampVal",u"val",b"val"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"boolVal",b"bytesVal",b"doubleVal",b"floatVal",b"int32Val",b"int64Val",b"stringVal",b"timestampVal",b"val"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"val",b"val"]) -> typing_extensions___Literal["bytesVal","stringVal","int32Val","int64Val","doubleVal","floatVal","boolVal","timestampVal"]: ... + +class ValueList(google___protobuf___message___Message): + + @property + def bytesList(self) -> BytesList: ... + + @property + def stringList(self) -> StringList: ... + + @property + def int32List(self) -> Int32List: ... + + @property + def int64List(self) -> Int64List: ... + + @property + def doubleList(self) -> DoubleList: ... + + @property + def floatList(self) -> FloatList: ... + + @property + def boolList(self) -> BoolList: ... + + @property + def timestampList(self) -> TimestampList: ... + + def __init__(self, + bytesList : typing___Optional[BytesList] = None, + stringList : typing___Optional[StringList] = None, + int32List : typing___Optional[Int32List] = None, + int64List : typing___Optional[Int64List] = None, + doubleList : typing___Optional[DoubleList] = None, + floatList : typing___Optional[FloatList] = None, + boolList : typing___Optional[BoolList] = None, + timestampList : typing___Optional[TimestampList] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ValueList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"boolList",u"bytesList",u"doubleList",u"floatList",u"int32List",u"int64List",u"stringList",u"timestampList",u"valueList"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"boolList",u"bytesList",u"doubleList",u"floatList",u"int32List",u"int64List",u"stringList",u"timestampList",u"valueList"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"boolList",b"boolList",u"bytesList",b"bytesList",u"doubleList",b"doubleList",u"floatList",b"floatList",u"int32List",b"int32List",u"int64List",b"int64List",u"stringList",b"stringList",u"timestampList",b"timestampList",u"valueList",b"valueList"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"boolList",b"bytesList",b"doubleList",b"floatList",b"int32List",b"int64List",b"stringList",b"timestampList",b"valueList"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"valueList",b"valueList"]) -> typing_extensions___Literal["bytesList","stringList","int32List","int64List","doubleList","floatList","boolList","timestampList"]: ... + +class BytesList(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[bytes] + + def __init__(self, + val : typing___Optional[typing___Iterable[bytes]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> BytesList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class StringList(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + val : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> StringList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class Int32List(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[int] + + def __init__(self, + val : typing___Optional[typing___Iterable[int]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Int32List: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class Int64List(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[int] + + def __init__(self, + val : typing___Optional[typing___Iterable[int]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Int64List: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class DoubleList(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[float] + + def __init__(self, + val : typing___Optional[typing___Iterable[float]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DoubleList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class FloatList(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[float] + + def __init__(self, + val : typing___Optional[typing___Iterable[float]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> FloatList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class BoolList(google___protobuf___message___Message): + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[bool] + + def __init__(self, + val : typing___Optional[typing___Iterable[bool]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> BoolList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... + +class TimestampList(google___protobuf___message___Message): + + @property + def val(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[google___protobuf___timestamp_pb2___Timestamp]: ... + + def __init__(self, + val : typing___Optional[typing___Iterable[google___protobuf___timestamp_pb2___Timestamp]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> TimestampList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"val"]) -> None: ... diff --git a/sdk/python/setup.py b/sdk/python/setup.py index a8b0c7cdc81..20b4f7ff97d 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -26,7 +26,7 @@ "google-api-core>=1.7.0", "google-auth>=1.6.0", "google-cloud-bigquery>=1.8.0", - "google-cloud-storage>=1.13.0", + "google-cloud-storage>=1.18.0", "googleapis-common-protos>=1.5.5", "grpcio>=1.16.1", "pandas>=0.24.0", diff --git a/sdk/python/tests/sample/valid_feature.yaml b/sdk/python/tests/sample/valid_feature.yaml index a54915641bc..824b1a1ce34 100644 --- a/sdk/python/tests/sample/valid_feature.yaml +++ b/sdk/python/tests/sample/valid_feature.yaml @@ -4,9 +4,4 @@ entity: myentity owner: bob@example.com description: test entity. valueType: BOOL -uri: https://github.com/bob/example -dataStores: - serving: - id: REDIS1 - warehouse: - id: BIGQUERY1 \ No newline at end of file +uri: https://github.com/bob/example \ No newline at end of file diff --git a/sdk/python/tests/sample/valid_feature_group.yaml b/sdk/python/tests/sample/valid_feature_group.yaml index f877d209fec..035010877a1 100644 --- a/sdk/python/tests/sample/valid_feature_group.yaml +++ b/sdk/python/tests/sample/valid_feature_group.yaml @@ -1,7 +1,2 @@ id: my_fg -tags: ["tag1", "tag2"] -dataStores: - serving: - id: "REDIS1" - warehouse: - id: "BIGQUERY1" \ No newline at end of file +tags: ["tag1", "tag2"] \ No newline at end of file diff --git a/sdk/python/tests/sdk/resources/test_feature.py b/sdk/python/tests/sdk/resources/test_feature.py index 7f296db2357..80b57fbc3de 100644 --- a/sdk/python/tests/sdk/resources/test_feature.py +++ b/sdk/python/tests/sdk/resources/test_feature.py @@ -12,22 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from feast.sdk.resources.feature import Feature, Datastore, ValueType +from feast.sdk.resources.feature import Feature, ValueType class TestFeature(object): def dummy_feature(self): - warehouse_data_store = Datastore(id="BIGQUERY1", options={}) - serving_data_store = Datastore(id="REDIS1", options={}) my_feature = Feature( name="my_feature", entity="my_entity", value_type=ValueType.BYTES, owner="feast@web.com", description="test feature", - uri="github.com/feature_repo", - warehouse_store=warehouse_data_store, - serving_store=serving_data_store) + uri="github.com/feature_repo") return my_feature def test_set_name(self): @@ -53,5 +49,3 @@ def test_read_from_yaml(self): assert feature.description == "test entity." assert feature.value_type == ValueType.BOOL assert feature.uri == "https://github.com/bob/example" - assert feature.serving_store.id == "REDIS1" - assert feature.warehouse_store.id == "BIGQUERY1" diff --git a/sdk/python/tests/sdk/resources/test_feature_group.py b/sdk/python/tests/sdk/resources/test_feature_group.py index 244211d1924..c553db6d6d1 100644 --- a/sdk/python/tests/sdk/resources/test_feature_group.py +++ b/sdk/python/tests/sdk/resources/test_feature_group.py @@ -20,6 +20,4 @@ def test_read_from_yaml(self): feature_group = FeatureGroup.from_yaml( "tests/sample/valid_feature_group.yaml") assert feature_group.id == "my_fg" - assert feature_group.serving_store.id == "REDIS1" - assert feature_group.warehouse_store.id == "BIGQUERY1" assert feature_group.tags == ["tag1", "tag2"] diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index 9e76cde301d..cbcc163ec01 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -39,7 +39,7 @@ from feast.sdk.utils.bq_util import TableDownloader from feast.serving.Serving_pb2 import QueryFeaturesRequest, \ QueryFeaturesResponse, FeatureValue -from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore +from feast.specs.FeatureSpec_pb2 import FeatureSpec from feast.specs.ImportSpec_pb2 import ImportSpec from feast.specs.StorageSpec_pb2 import StorageSpec from feast.types.Value_pb2 import Value @@ -92,19 +92,6 @@ def test_apply_single_feature_group(self, client, mocker): name = client.apply(my_feature_group) assert name == "test" - def test_apply_single_storage(self, client, mocker): - my_storage = Storage(id="TEST", type="redis") - grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - - with mocker.patch.object( - grpc_stub, - 'ApplyStorage', - return_value=CoreServiceTypes.ApplyStorageResponse( - storageId="TEST")): - client._core_service_stub = grpc_stub - name = client.apply(my_storage) - assert name == "TEST" - def test_apply_unsupported_object(self, client): with pytest.raises(TypeError) as e_info: client.apply(None) @@ -112,17 +99,11 @@ def test_apply_unsupported_object(self, client): + "following types: [Feature, Entity, FeatureGroup, Storage, Importer]" def test_apply_multiple(self, client, mocker): - my_storage = Storage(id="TEST", type="redis") my_feature_group = FeatureGroup(id="test") my_entity = Entity(name="test") grpc_stub = core.CoreServiceStub(grpc.insecure_channel("")) - mocker.patch.object( - grpc_stub, - 'ApplyStorage', - return_value=CoreServiceTypes.ApplyStorageResponse( - storageId="TEST")) mocker.patch.object( grpc_stub, 'ApplyFeatureGroup', @@ -135,8 +116,8 @@ def test_apply_multiple(self, client, mocker): entityName="test")) client._core_service_stub = grpc_stub - ids = client.apply([my_storage, my_entity, my_feature_group]) - assert ids == ["TEST", "test", "test"] + ids = client.apply([my_entity, my_feature_group]) + assert ids == ["test", "test"] def test_run_job_no_staging(self, client, mocker): grpc_stub = jobs.JobServiceStub(grpc.insecure_channel("")) @@ -493,9 +474,7 @@ def _create_query_features_response(self, entity_name, entities): return response def _create_feature_spec(self, feature_id, wh_id): - wh_store = DataStore(id=wh_id) - datastores = DataStores(warehouse=wh_store) - return FeatureSpec(id=feature_id, dataStores=datastores) + return FeatureSpec(id=feature_id) def _create_bq_spec(self, id, project, dataset): return StorageSpec( diff --git a/sdk/python/tests/sdk/test_importer.py b/sdk/python/tests/sdk/test_importer.py index 6a0c9b07848..aa05bcb82ec 100644 --- a/sdk/python/tests/sdk/test_importer.py +++ b/sdk/python/tests/sdk/test_importer.py @@ -15,8 +15,7 @@ import pandas as pd import pytest import ntpath -from feast.sdk.resources.feature import Feature, ValueType, \ - Datastore +from feast.sdk.resources.feature import Feature, ValueType from feast.sdk.importer import _create_feature, Importer from feast.sdk.utils.gs_utils import is_gs_path @@ -28,9 +27,7 @@ def test_from_csv(self): owner = "owner@feast.com" staging_location = "gs://test-bucket" id_column = "driver_id" - feature_columns = [ - "avg_distance_completed", "avg_customer_distance_completed" - ] + feature_columns = ["avg_distance_completed", "avg_customer_distance_completed"] timestamp_column = "ts" importer = Importer.from_csv( @@ -40,17 +37,25 @@ def test_from_csv(self): staging_location=staging_location, id_column=id_column, feature_columns=feature_columns, - timestamp_column=timestamp_column) + timestamp_column=timestamp_column, + ) - self._validate_csv_importer(importer, csv_path, entity_name, - owner, staging_location, id_column, - feature_columns, timestamp_column) + self._validate_csv_importer( + importer, + csv_path, + entity_name, + owner, + staging_location, + id_column, + feature_columns, + timestamp_column, + ) def test_from_csv_id_column_not_specified(self): - with pytest.raises( - ValueError, match="Column with name driver is not found"): + with pytest.raises(ValueError, match="Column with name driver is not found"): feature_columns = [ - "avg_distance_completed", "avg_customer_distance_completed" + "avg_distance_completed", + "avg_customer_distance_completed", ] csv_path = "tests/data/driver_features.csv" Importer.from_csv( @@ -59,12 +64,14 @@ def test_from_csv_id_column_not_specified(self): owner="owner@feast.com", staging_location="gs://test-bucket", feature_columns=feature_columns, - timestamp_column="ts") + timestamp_column="ts", + ) def test_from_csv_timestamp_column_not_specified(self): feature_columns = [ - "avg_distance_completed", "avg_customer_distance_completed", - "avg_distance_cancelled" + "avg_distance_completed", + "avg_customer_distance_completed", + "avg_distance_cancelled", ] csv_path = "tests/data/driver_features.csv" entity_name = "driver" @@ -77,7 +84,8 @@ def test_from_csv_timestamp_column_not_specified(self): owner=owner, staging_location=staging_location, id_column=id_column, - feature_columns=feature_columns) + feature_columns=feature_columns, + ) self._validate_csv_importer( importer, @@ -86,7 +94,8 @@ def test_from_csv_timestamp_column_not_specified(self): owner, staging_location=staging_location, id_column=id_column, - feature_columns=feature_columns) + feature_columns=feature_columns, + ) def test_from_csv_feature_columns_not_specified(self): csv_path = "tests/data/driver_features.csv" @@ -101,7 +110,8 @@ def test_from_csv_feature_columns_not_specified(self): owner=owner, staging_location=staging_location, id_column=id_column, - timestamp_column=timestamp_column) + timestamp_column=timestamp_column, + ) self._validate_csv_importer( importer, @@ -110,15 +120,17 @@ def test_from_csv_feature_columns_not_specified(self): owner, staging_location=staging_location, id_column=id_column, - timestamp_column=timestamp_column) + timestamp_column=timestamp_column, + ) def test_from_csv_staging_location_not_specified(self): with pytest.raises( - ValueError, - match= - "Specify staging_location for importing local file/dataframe"): + ValueError, + match="Specify staging_location for importing local file/dataframe", + ): feature_columns = [ - "avg_distance_completed", "avg_customer_distance_completed" + "avg_distance_completed", + "avg_customer_distance_completed", ] csv_path = "tests/data/driver_features.csv" Importer.from_csv( @@ -126,12 +138,15 @@ def test_from_csv_staging_location_not_specified(self): entity="driver", owner="owner@feast.com", feature_columns=feature_columns, - timestamp_column="ts") + timestamp_column="ts", + ) with pytest.raises( - ValueError, match="Staging location must be in GCS") as e_info: + ValueError, match="Staging location must be in GCS" + ) as e_info: feature_columns = [ - "avg_distance_completed", "avg_customer_distance_completed" + "avg_distance_completed", + "avg_customer_distance_completed", ] csv_path = "tests/data/driver_features.csv" Importer.from_csv( @@ -140,7 +155,8 @@ def test_from_csv_staging_location_not_specified(self): owner="owner@feast.com", staging_location="/home", feature_columns=feature_columns, - timestamp_column="ts") + timestamp_column="ts", + ) def test_from_df(self): csv_path = "tests/data/driver_features.csv" @@ -154,39 +170,40 @@ def test_from_df(self): owner="owner@feast.com", staging_location=staging_location, id_column="driver_id", - timestamp_column="ts") + timestamp_column="ts", + ) assert importer.require_staging == True - assert ("{}/tmp_{}".format(staging_location, - entity) in importer.remote_path) + assert "{}/tmp_{}".format(staging_location, entity) in importer.remote_path for feature in importer.features.values(): assert feature.name in df.columns assert feature.id == "driver." + feature.name import_spec = importer.spec assert import_spec.type == "file.csv" - assert import_spec.sourceOptions == { - "path": importer.remote_path - } + assert import_spec.sourceOptions == {"path": importer.remote_path} assert import_spec.entities == ["driver"] schema = import_spec.schema assert schema.entityIdColumn == "driver_id" assert schema.timestampValue is not None feature_columns = [ - "completed", "avg_distance_completed", - "avg_customer_distance_completed", "avg_distance_cancelled" + "completed", + "avg_distance_completed", + "avg_customer_distance_completed", + "avg_distance_cancelled", ] for col, field in zip(df.columns.values, schema.fields): assert col == field.name if col in feature_columns: assert field.featureId == "driver." + col - + def test_stage_df_without_timestamp(self, mocker): mocker.patch("feast.sdk.importer.df_to_gcs", return_value=True) feature_columns = [ - "avg_distance_completed", "avg_customer_distance_completed", - "avg_distance_cancelled" + "avg_distance_completed", + "avg_customer_distance_completed", + "avg_distance_cancelled", ] csv_path = "tests/data/driver_features.csv" entity_name = "driver" @@ -199,25 +216,29 @@ def test_stage_df_without_timestamp(self, mocker): owner=owner, staging_location=staging_location, id_column=id_column, - feature_columns=feature_columns) + feature_columns=feature_columns, + ) importer.stage() - def _validate_csv_importer(self, - importer, - csv_path, - entity_name, - owner, - staging_location=None, - id_column=None, - feature_columns=None, - timestamp_column=None, - timestamp_value=None): + def _validate_csv_importer( + self, + importer, + csv_path, + entity_name, + owner, + staging_location=None, + id_column=None, + feature_columns=None, + timestamp_column=None, + timestamp_value=None, + ): df = pd.read_csv(csv_path) assert not importer.require_staging == is_gs_path(csv_path) if importer.require_staging: assert importer.remote_path == "{}/{}".format( - staging_location, ntpath.basename(csv_path)) + staging_location, ntpath.basename(csv_path) + ) # check features created for feature in importer.features.values(): @@ -231,7 +252,9 @@ def _validate_csv_importer(self, assert import_spec.entities == [entity_name] schema = import_spec.schema - assert schema.entityIdColumn == id_column if id_column is not None else entity_name + assert ( + schema.entityIdColumn == id_column if id_column is not None else entity_name + ) if timestamp_column is not None: assert schema.timestampColumn == timestamp_column elif timestamp_value is not None: @@ -246,38 +269,26 @@ def _validate_csv_importer(self, for col, field in zip(df.columns.values, schema.fields): assert col == field.name if col in feature_columns: - assert field.featureId == '{}.{}'.format(entity_name, - col).lower() - + assert field.featureId == "{}.{}".format(entity_name, col).lower() class TestHelpers: def test_create_feature(self): - col = pd.Series([1] * 3, dtype='int32', name="test") + col = pd.Series([1] * 3, dtype="int32", name="test") expected = Feature( - name="test", - entity="test", - owner="person", - value_type=ValueType.INT32) - actual = _create_feature(col, "test", "person", None, None) + name="test", entity="test", owner="person", value_type=ValueType.INT32 + ) + actual = _create_feature(col, "test", "person") assert actual.id == expected.id assert actual.value_type == expected.value_type assert actual.owner == expected.owner def test_create_feature_with_stores(self): - col = pd.Series([1] * 3, dtype='int32', name="test") + col = pd.Series([1] * 3, dtype="int32", name="test") expected = Feature( - name="test", - entity="test", - owner="person", - value_type=ValueType.INT32, - serving_store=Datastore(id="SERVING"), - warehouse_store=Datastore(id="WAREHOUSE")) - actual = _create_feature(col, "test", "person", - Datastore(id="SERVING"), - Datastore(id="WAREHOUSE")) + name="test", entity="test", owner="person", value_type=ValueType.INT32 + ) + actual = _create_feature(col, "test", "person") assert actual.id == expected.id assert actual.value_type == expected.value_type assert actual.owner == expected.owner - assert actual.serving_store == expected.serving_store - assert actual.warehouse_store == expected.warehouse_store diff --git a/serving/pom.xml b/serving/pom.xml index 33e3665f7e1..9e05bae4606 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -169,20 +169,19 @@ io.jaegertracing jaeger-client - 0.31.0 + 0.35.5 io.opentracing opentracing-api - 0.31.0 + 0.33.0 io.opentracing.contrib - opentracing-concurrent - 0.2.0 + opentracing-grpc + 0.1.3 - io.micrometer diff --git a/serving/src/main/java/feast/serving/config/AppConfig.java b/serving/src/main/java/feast/serving/config/AppConfig.java index 77b56116ee9..2796c31ec9b 100644 --- a/serving/src/main/java/feast/serving/config/AppConfig.java +++ b/serving/src/main/java/feast/serving/config/AppConfig.java @@ -29,9 +29,6 @@ public class AppConfig { // number of connection allowed to be idle int redisMaxIdleSize; - // max number of entity per thread. - int maxEntityPerBatch; - // timeout for feature retrieval int timeout; } diff --git a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java index 519f3d30258..6188fc14144 100644 --- a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java +++ b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java @@ -21,6 +21,7 @@ import com.timgroup.statsd.StatsDClient; import io.micrometer.core.instrument.MeterRegistry; import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; import java.net.InetAddress; import java.net.UnknownHostException; import org.springframework.beans.factory.annotation.Value; @@ -42,7 +43,9 @@ public StatsDClient getStatsDClient(@Value("${statsd.host}") String host, public Tracer getTracer() { io.jaegertracing.Configuration tracingConfig = io.jaegertracing.Configuration.fromEnv(APP_NAME); - return tracingConfig.getTracer(); + Tracer tracer = tracingConfig.getTracer(); + GlobalTracer.registerIfAbsent(tracer); + return tracer; } @Bean diff --git a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java index 3b3ae6f35a9..db9883f27ce 100644 --- a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java +++ b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java @@ -18,8 +18,6 @@ package feast.serving.config; import com.google.common.base.Strings; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import feast.serving.service.CachedSpecStorage; @@ -28,7 +26,6 @@ import feast.serving.service.SpecStorage; import feast.specs.StorageSpecProto.StorageSpec; import io.opentracing.Tracer; -import io.opentracing.contrib.concurrent.TracedExecutorService; import java.lang.reflect.Type; import java.util.Collections; import java.util.List; @@ -72,10 +69,8 @@ private static Map convertJsonStringToMap(String jsonString) { public AppConfig getAppConfig( @Value("${feast.redispool.maxsize}") int redisPoolMaxSize, @Value("${feast.redispool.maxidle}") int redisPoolMaxIdle, - @Value("${feast.maxentity}") int maxEntityPerBatch, @Value("${feast.timeout}") int timeout) { return AppConfig.builder() - .maxEntityPerBatch(maxEntityPerBatch) .redisMaxPoolSize(redisPoolMaxSize) .redisMaxIdleSize(redisPoolMaxIdle) .timeout(timeout) @@ -112,10 +107,10 @@ public FeatureStorageRegistry getFeatureStorageRegistry( Map optionsMap = convertJsonStringToMap(storageOptions); StorageSpec storageSpec = StorageSpec.newBuilder() - .setId("SERVING") - .setType(storageType) - .putAllOptions(optionsMap) - .build(); + .setId("SERVING") + .setType(storageType) + .putAllOptions(optionsMap) + .build(); FeatureStorageRegistry registry = new FeatureStorageRegistry(appConfig, tracer); try { @@ -129,11 +124,8 @@ public FeatureStorageRegistry getFeatureStorageRegistry( } @Bean - public ListeningExecutorService getExecutorService( - Tracer tracer, @Value("${feast.threadpool.max}") int maxPoolSize) { - - ExecutorService executor = Executors.newFixedThreadPool(maxPoolSize); - return MoreExecutors.listeningDecorator(new TracedExecutorService(executor, tracer)); + public ExecutorService getExecutorService(@Value("${feast.threadpool.max}") int maxPoolSize) { + return Executors.newFixedThreadPool(maxPoolSize); } @Bean diff --git a/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java b/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java index 51622d8ea65..33bcd8c00c0 100644 --- a/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java +++ b/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java @@ -33,13 +33,15 @@ import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; +import io.opentracing.contrib.grpc.OpenTracingContextKey; +import io.opentracing.contrib.grpc.ServerTracingInterceptor; import lombok.extern.slf4j.Slf4j; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; /** Grpc service implementation for Serving API. */ @Slf4j -@GRpcService +@GRpcService(interceptors = {ServerTracingInterceptor.class}) public class ServingGrpcService extends ServingAPIImplBase { private final FeastServing feast; @@ -57,21 +59,20 @@ public ServingGrpcService(FeastServing feast, Tracer tracer, StatsDClient statsD @Override public void queryFeatures(QueryFeaturesRequest request, StreamObserver responseObserver) { long currentMicro = TimeUtil.microTime(); - Span span = tracer.buildSpan("ServingGrpcService-queryFeatures").start(); + Span span =tracer + .buildSpan("ServingGrpcService.queryFeatures") + .asChildOf(OpenTracingContextKey.activeSpan()) + .start(); String[] tags = makeStatsdTags(request); statsDClient.increment("query_features_count", tags); statsDClient.gauge("query_features_entity_count", request.getEntityIdCount(), tags); statsDClient.gauge("query_features_feature_count", request.getFeatureIdCount(), tags); - try (Scope scope = tracer.scopeManager().activate(span, false)) { - Span innerSpan = scope.span(); + try (Scope scope = tracer.scopeManager().activate(span)) { validateRequest(request); QueryFeaturesResponse response = feast.queryFeatures(request); - innerSpan.log("calling onNext"); responseObserver.onNext(response); - innerSpan.log("calling onCompleted"); responseObserver.onCompleted(); - innerSpan.log("all done"); statsDClient.increment("query_feature_success", tags); } catch (Exception e) { statsDClient.increment("query_feature_failed", tags); @@ -79,6 +80,7 @@ public void queryFeatures(QueryFeaturesRequest request, StreamObserver featureSpecs = getFeatureSpecs(request.getFeatureIdList()); - scope.span().log("start retrieving all feature"); + span.log("start retrieving all feature"); Map result = featureRetrievalDispatcher.dispatchFeatureRetrieval( request.getEntityName(), request.getEntityIdList(), featureSpecs); - scope.span().log("finished retrieving all feature"); + span.log("finished retrieving all feature"); // build response return QueryFeaturesResponse.newBuilder() .setEntityName(request.getEntityName()) .putAllEntities(result) .build(); + } finally { + span.finish(); } } @@ -88,10 +92,15 @@ public QueryFeaturesResponse queryFeatures(QueryFeaturesRequest request) { * @return collection of feature spec */ private Collection getFeatureSpecs(Collection featureIds) { - // dedup feature ID. - Collection featureIdSet = Sets.newHashSet(featureIds); + Span span = tracer.buildSpan("FeastServing.getFeatureSpecs").start(); + try (Scope scope = tracer.scopeManager().activate(span)) { + // dedup feature ID. + Collection featureIdSet = Sets.newHashSet(featureIds); - Map featureSpecMap = specStorage.getFeatureSpecs(featureIdSet); - return featureSpecMap.values(); + Map featureSpecMap = specStorage.getFeatureSpecs(featureIdSet); + return featureSpecMap.values(); + } finally { + span.finish(); + } } } diff --git a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java index e9a9ee67bda..a1175679268 100644 --- a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java +++ b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java @@ -17,12 +17,12 @@ package feast.serving.service; -import com.google.common.collect.Lists; import feast.serving.ServingAPIProto.Entity; import feast.serving.model.FeatureValue; import feast.serving.util.EntityMapBuilder; import feast.specs.FeatureSpecProto.FeatureSpec; import io.opentracing.Scope; +import io.opentracing.Span; import io.opentracing.Tracer; import java.util.Collection; import java.util.List; @@ -52,31 +52,15 @@ public FeatureRetrievalDispatcher( *

If request is small enough (only one request type and one source storage) it will be * executed in the current thread. Otherwise, the execution takes place in separate thread. * - * @param entityName entity name of the feature. - * @param entityIds list of entity ids. + * @param entityName entity name of the feature. + * @param entityIds list of entity ids. * @param featureSpecs list of request. * @return map of entityID and Entity instance. */ public Map dispatchFeatureRetrieval( String entityName, Collection entityIds, Collection featureSpecs) { - - return runInCurrentThread(entityName, entityIds, Lists.newArrayList(featureSpecs)); - } - - /** - * Execute request in current thread. - * - * @param entityName entity name of of the feature. - * @param entityIds list of entity ID of the feature to be retrieved. - * @param featureSpecs list of feature specs - * @return entity map containing the result of feature retrieval. - */ - private Map runInCurrentThread( - String entityName, - Collection entityIds, - List featureSpecs) { - try (Scope scope = - tracer.buildSpan("FeatureRetrievalDispatcher-runInCurrentThread").startActive(true)) { + Span span = tracer.buildSpan("FeatureRetrievalDispatcher.runInCurrentThread").start(); + try (Scope scope = tracer.scopeManager().activate(span)) { String storageId = FeastServing.SERVING_STORAGE_ID; FeatureStorage featureStorage = featureStorageRegistry.get(storageId); @@ -87,6 +71,8 @@ private Map runInCurrentThread( EntityMapBuilder builder = new EntityMapBuilder(); builder.addFeatureValueList(featureValues); return builder.toEntityMap(); + } finally { + span.finish(); } } } diff --git a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java index d481c9554e9..53302056ed5 100644 --- a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java @@ -67,18 +67,21 @@ public RedisFeatureStorage(JedisPool jedisPool, Tracer tracer) { @Override public List getFeature( String entityName, Collection entityIds, Collection featureSpecs) { - try (Scope scope = tracer.buildSpan("Redis-getFeature").startActive(true)) { + Span span = tracer.buildSpan("Redis.getFeature").start(); + try (Scope scope = tracer.scopeManager().activate(span)) { List getRequests = new ArrayList<>(entityIds.size() * featureSpecs.size()); for (FeatureSpec featureSpec : featureSpecs) { String featureId = featureSpec.getId(); String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); for (String entityId : entityIds) { RedisBucketKey key = makeBucketKey(entityId, featureIdSha1Prefix, BUCKET_ID_ZERO); - getRequests.add(new GetRequest(entityId, featureId, key)); + getRequests.add(new GetRequest(entityId, featureId, key)); } } - scope.span().log("completed request creation"); + span.log("completed request creation"); return sendAndProcessMultiGet(getRequests); + } finally { + span.finish(); } } @@ -89,9 +92,8 @@ public List getFeature( * @return list of feature value. */ private List sendAndProcessMultiGet(List getRequests) { - try (Scope scope = tracer.buildSpan("Redis-sendAndProcessMultiGet").startActive(true)) { - Span span = scope.span(); - + Span span = tracer.buildSpan("Redis.sendAndProcessMultiGet").start(); + try (Scope scope = tracer.scopeManager().activate(span)) { if (getRequests.isEmpty()) { return Collections.emptyList(); } @@ -106,13 +108,17 @@ private List sendAndProcessMultiGet(List getRequests) span.log("completed creating mget request"); List binaryValues; - try (Jedis jedis = jedisPool.getResource()) { + Span mgetSpan = tracer.buildSpan("Redis.mget").start(); + try (Jedis jedis = jedisPool.getResource(); + Scope mgetScope = tracer.scopeManager().activate(mgetSpan)) { span.log("sending mget"); binaryValues = jedis.mget(binaryKeys); span.log("completed mget"); } catch (Exception e) { log.error("Exception while retrieving feature from Redis", e); throw new FeatureRetrievalException("Unable to retrieve feature from Redis", e); + } finally { + mgetSpan.finish(); } try { @@ -121,6 +127,8 @@ private List sendAndProcessMultiGet(List getRequests) log.error("Unable to parse protobuf", e); throw new FeatureRetrievalException("Unable to parse protobuf while retrieving feature", e); } + } finally { + span.finish(); } } @@ -134,7 +142,8 @@ private List sendAndProcessMultiGet(List getRequests) */ private List processMGet(List requests, List results) throws InvalidProtocolBufferException { - try (Scope scope = tracer.buildSpan("Redis-processMGet").startActive(true)) { + Span span = tracer.buildSpan("Redis.processMGet").start(); + try (Scope scope = tracer.scopeManager().activate(span)) { int keySize = requests.size(); List featureValues = new ArrayList<>(keySize); @@ -154,6 +163,8 @@ private List processMGet(List requests, List r featureValues.add(featureValue); } return featureValues; + } finally { + span.finish(); } } diff --git a/serving/src/main/resources/application.properties b/serving/src/main/resources/application.properties index f7d00236381..ecf43a5e292 100644 --- a/serving/src/main/resources/application.properties +++ b/serving/src/main/resources/application.properties @@ -23,7 +23,6 @@ feast.store.serving.type = ${STORE_SERVING_TYPE:} feast.store.serving.options = ${STORE_SERVING_OPTIONS:{}} feast.threadpool.max=${FEAST_MAX_NB_THREAD:128} -feast.maxentity=${FEAST_MAX_ENTITY_PER_BATCH:2000} feast.timeout=${FEAST_RETRIEVAL_TIMEOUT:5} feast.redispool.maxsize=${FEAST_REDIS_POOL_MAX_SIZE:128} feast.redispool.maxidle=${FEAST_REDIS_POOL_MAX_IDLE:16}