diff --git a/.prow/config.yaml b/.prow/config.yaml index 944d1baf2b3..927d7f9fee7 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -144,6 +144,26 @@ presubmits: - image: python:3.7 command: ["infra/scripts/test-python-sdk.sh"] + - name: test-telemetry + decorate: true + run_if_changed: "sdk/python/.*" + spec: + containers: + - image: python:3.7 + command: ["infra/scripts/test-telemetry.sh"] + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/gcloud/service-account.json + volumeMounts: + - mountPath: /etc/gcloud/service-account.json + name: service-account + readOnly: true + subPath: service-account.json + volumes: + - name: service-account + secret: + secretName: feast-service-account + - name: test-golang-sdk decorate: true spec: diff --git a/CHANGELOG.md b/CHANGELOG.md index fa6f3018c92..01214478f0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,18 @@ # Changelog +## [v0.9.1](https://github.com/feast-dev/feast/tree/v0.9.1) (2021-01-29) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.9.0...v0.9.1) + +**Implemented enhancements:** + +- Add telemetry to Python SDK [\#1289](https://github.com/feast-dev/feast/pull/1289) ([jklegar](https://github.com/jklegar)) + +**Fixed bugs:** + +- Fix kafka download url [\#1298](https://github.com/feast-dev/feast/pull/1298) ([jklegar](https://github.com/jklegar)) +- disable telemetry in docker-compose test and job\_service [\#1297](https://github.com/feast-dev/feast/pull/1297) ([jklegar](https://github.com/jklegar)) + + ## [v0.9.0](https://github.com/feast-dev/feast/tree/v0.9.0) (2021-01-28) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.8.4...v0.9.0) diff --git a/Makefile b/Makefile index c8dc7f0c3a6..1e3ad22ba1a 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ install-python: compile-protos-python python -m pip install -e sdk/python test-python: - pytest --verbose --color=yes sdk/python/tests + FEAST_TELEMETRY=False pytest --verbose --color=yes sdk/python/tests format-python: # Sort diff --git a/datatypes/java/README.md b/datatypes/java/README.md index c926926f209..54e3c3eb5f2 100644 --- a/datatypes/java/README.md +++ b/datatypes/java/README.md @@ -16,7 +16,7 @@ Dependency Coordinates dev.feast datatypes-java - 0.9.0 + 0.9.1 ``` diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6d76d3f7174..fd8fde1c48b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -43,6 +43,7 @@ * [Security](advanced/security.md) * [Audit Logging](advanced/audit-logging.md) * [Metrics](advanced/metrics.md) +* [Telemetry](advanced/telemetry.md) * [Troubleshooting](advanced/troubleshooting.md) ## Reference diff --git a/docs/advanced/audit-logging.md b/docs/advanced/audit-logging.md index 7f80f105ef9..be7e7955ca2 100644 --- a/docs/advanced/audit-logging.md +++ b/docs/advanced/audit-logging.md @@ -48,7 +48,7 @@ Audit Logs produced by Feast are written to the console similar to normal logs b "service": "CoreService", "component": "feast-core", "id": "45329ea9-0d48-46c5-b659-4604f6193711", - "version": "0.9.0" + "version": "0.9.1" }, "hostname": "feast.core" "timestamp": "2020-10-20T04:45:24Z", diff --git a/docs/advanced/telemetry.md b/docs/advanced/telemetry.md new file mode 100644 index 00000000000..1f5d10f5de0 --- /dev/null +++ b/docs/advanced/telemetry.md @@ -0,0 +1,10 @@ +# Telemetry + +### How telemetry is used + +The Feast maintainers use anonymous usage statistics to help shape the Feast roadmap. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS (using `sys.platform`), and the current time. For more detailed information see [the source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/telemetry.py). + +### How to disable telemetry + +To opt out of telemetry, simply set the environment variable `FEAST_TELEMETRY` to `False` in the environment in which the Feast client is run. + diff --git a/docs/contributing/development-guide.md b/docs/contributing/development-guide.md index b52c66c1e3d..20059aa4494 100644 --- a/docs/contributing/development-guide.md +++ b/docs/contributing/development-guide.md @@ -109,10 +109,10 @@ Feast Serving has a dependency on Feast Core, thus always start Feast Core first ```bash # Start Feast Core locally -java -jar core/target/feast-core-0.9.0-exec.jar +java -jar core/target/feast-core-0.9.1-exec.jar # Start Feast Serving locally -java -jar serving/target/feast-serving-0.9.0-exec.jar +java -jar serving/target/feast-serving-0.9.1-exec.jar ``` Test whether Feast Core, Feast Serving are started and running correctly: @@ -123,8 +123,8 @@ feast version --core-url="localhost:6565" --serving-url="localhost:6566" ```javascript { - 'serving': {'url': 'localhost:6566', 'version': '0.9.0'}, - 'core': {'url': 'localhost:6565', 'version': '0.9.0'} + 'serving': {'url': 'localhost:6566', 'version': '0.9.1'}, + 'core': {'url': 'localhost:6565', 'version': '0.9.1'} } ``` diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 354847f1a81..bc2be109e1e 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feature store for machine learning. name: feast -version: 0.9.0 +version: 0.9.1 diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 2b64b49a97d..f3ddb8a3a62 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -1,7 +1,7 @@ feast ===== -Feature store for machine learning. Current chart version is `0.9.0` +Feature store for machine learning. Current chart version is `0.9.1` ## Installation @@ -11,9 +11,9 @@ https://docs.feast.dev/v/master/getting-started/deploying-feast/kubernetes | Repository | Name | Version | |------------|------|---------| -| | feast-core | 0.9.0 | -| | feast-jupyter | 0.9.0 | -| | feast-serving | 0.9.0 | +| | feast-core | 0.9.1 | +| | feast-jupyter | 0.9.1 | +| | feast-serving | 0.9.1 | | | prometheus-statsd-exporter | 0.1.2 | | https://charts.bitnami.com/bitnami/ | kafka | 11.8.8 | | https://kubernetes-charts.storage.googleapis.com/ | grafana | 5.0.5 | diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml index e8ce2818f97..635e4d134d5 100644 --- a/infra/charts/feast/charts/feast-core/Chart.yaml +++ b/infra/charts/feast/charts/feast-core/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Core registers feature specifications. name: feast-core -version: 0.9.0 +version: 0.9.1 diff --git a/infra/charts/feast/charts/feast-core/README.md b/infra/charts/feast/charts/feast-core/README.md index 01d7e76678a..59b7d5235d3 100644 --- a/infra/charts/feast/charts/feast-core/README.md +++ b/infra/charts/feast/charts/feast-core/README.md @@ -2,7 +2,7 @@ feast-core ========== Feast Core registers feature specifications. -Current chart version is `0.9.0` +Current chart version is `0.9.1` diff --git a/infra/charts/feast/charts/feast-core/values.yaml b/infra/charts/feast/charts/feast-core/values.yaml index 6f5be009d9d..5e056fff854 100644 --- a/infra/charts/feast/charts/feast-core/values.yaml +++ b/infra/charts/feast/charts/feast-core/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image repository repository: feastdev/feast-core # image.tag -- Image tag - tag: 0.9.0 + tag: 0.9.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/feast-jobservice/Chart.yaml b/infra/charts/feast/charts/feast-jobservice/Chart.yaml index 5f8943fbada..8fb7443e8bf 100644 --- a/infra/charts/feast/charts/feast-jobservice/Chart.yaml +++ b/infra/charts/feast/charts/feast-jobservice/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Job Service manage ingestion jobs. name: feast-jobservice -version: 0.9.0 +version: 0.9.1 diff --git a/infra/charts/feast/charts/feast-jobservice/README.md b/infra/charts/feast/charts/feast-jobservice/README.md index 2fe24dc6a70..987f4c13fbf 100644 --- a/infra/charts/feast/charts/feast-jobservice/README.md +++ b/infra/charts/feast/charts/feast-jobservice/README.md @@ -2,7 +2,7 @@ feast-jobservice ================ Feast Job Service manage ingestion jobs. -Current chart version is `0.9.0` +Current chart version is `0.9.1` diff --git a/infra/charts/feast/charts/feast-jobservice/values.yaml b/infra/charts/feast/charts/feast-jobservice/values.yaml index d2ebdc2795d..6c30fa88973 100644 --- a/infra/charts/feast/charts/feast-jobservice/values.yaml +++ b/infra/charts/feast/charts/feast-jobservice/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image repository repository: feastdev/feast-jobservice # image.tag -- Image tag - tag: 0.9.0 + tag: 0.9.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/feast-jupyter/Chart.yaml b/infra/charts/feast/charts/feast-jupyter/Chart.yaml index 3df73756ae8..af18acc6651 100644 --- a/infra/charts/feast/charts/feast-jupyter/Chart.yaml +++ b/infra/charts/feast/charts/feast-jupyter/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Jupyter provides a Jupyter server with pre-installed Feast SDK name: feast-jupyter -version: 0.9.0 +version: 0.9.1 diff --git a/infra/charts/feast/charts/feast-jupyter/README.md b/infra/charts/feast/charts/feast-jupyter/README.md index 548bf17c39f..ed78618f9af 100644 --- a/infra/charts/feast/charts/feast-jupyter/README.md +++ b/infra/charts/feast/charts/feast-jupyter/README.md @@ -2,7 +2,7 @@ feast-jupyter ============= Feast Jupyter provides a Jupyter server with pre-installed Feast SDK -Current chart version is `0.9.0` +Current chart version is `0.9.1` diff --git a/infra/charts/feast/charts/feast-jupyter/values.yaml b/infra/charts/feast/charts/feast-jupyter/values.yaml index b582edacfa9..cccb9ae47d5 100644 --- a/infra/charts/feast/charts/feast-jupyter/values.yaml +++ b/infra/charts/feast/charts/feast-jupyter/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image repository repository: feastdev/feast-jupyter # image.tag -- Image tag - tag: 0.9.0 + tag: 0.9.1 # image.pullPolicy -- Image pull policy pullPolicy: Always diff --git a/infra/charts/feast/charts/feast-serving/Chart.yaml b/infra/charts/feast/charts/feast-serving/Chart.yaml index d753d4b3609..ea2957a7733 100644 --- a/infra/charts/feast/charts/feast-serving/Chart.yaml +++ b/infra/charts/feast/charts/feast-serving/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Serving serves low-latency latest features and historical batch features. name: feast-serving -version: 0.9.0 +version: 0.9.1 diff --git a/infra/charts/feast/charts/feast-serving/README.md b/infra/charts/feast/charts/feast-serving/README.md index 932b778c0ae..5b734509bc2 100644 --- a/infra/charts/feast/charts/feast-serving/README.md +++ b/infra/charts/feast/charts/feast-serving/README.md @@ -2,7 +2,7 @@ feast-serving ============= Feast Serving serves low-latency latest features and historical batch features. -Current chart version is `0.9.0` +Current chart version is `0.9.1` diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml index a0eb8ef2086..a3522b5eb8b 100644 --- a/infra/charts/feast/charts/feast-serving/values.yaml +++ b/infra/charts/feast/charts/feast-serving/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image repository repository: feastdev/feast-serving # image.tag -- Image tag - tag: 0.9.0 + tag: 0.9.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/README.md b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md index 61f8ffe64e3..2b5c27c5c22 100644 --- a/infra/charts/feast/charts/prometheus-statsd-exporter/README.md +++ b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md @@ -38,7 +38,7 @@ $ helm delete my-release |`extraArgs` | key:value list of extra arguments to give the binary | `{}` | |`image.pullPolicy` | Image pull policy | `IfNotPresent` | |`image.repository` | Image repository | `prom/statsd-exporter` | -|`image.tag` | Image tag | `v0.9.0` | +|`image.tag` | Image tag | `v0.9.1` | |`ingress.enabled` | enable ingress | `false` | |`ingress.path` | ingress base path | `/` | |`ingress.host` | Ingress accepted hostnames | `nil` | @@ -54,4 +54,4 @@ $ helm delete my-release ```console $ helm install incubator/prometheus-statsd-exporter --name my-release -f values.yaml ``` -> **Tip**: You can use the default [values.yaml](values.yaml) \ No newline at end of file +> **Tip**: You can use the default [values.yaml](values.yaml) diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock index 8a87c90b5c3..79a9c4be865 100644 --- a/infra/charts/feast/requirements.lock +++ b/infra/charts/feast/requirements.lock @@ -1,13 +1,13 @@ dependencies: - name: feast-core repository: "" - version: 0.9.0 + version: 0.9.1 - name: feast-serving repository: "" - version: 0.9.0 + version: 0.9.1 - name: feast-jupyter repository: "" - version: 0.9.0 + version: 0.9.1 - name: postgresql repository: https://kubernetes-charts.storage.googleapis.com/ version: 8.6.1 diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 9987c968145..2b545df3afa 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,13 +1,13 @@ dependencies: - name: feast-core - version: 0.9.0 + version: 0.9.1 condition: feast-core.enabled - name: feast-serving alias: feast-online-serving - version: 0.9.0 + version: 0.9.1 condition: feast-online-serving.enabled - name: feast-jupyter - version: 0.9.0 + version: 0.9.1 condition: feast-jupyter.enabled - name: postgresql version: 8.6.1 diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index e2454f9fe85..3681255e465 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -70,4 +70,4 @@ docker exec \ -e DISABLE_FEAST_SERVICE_FIXTURES=true \ --user root \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && FEAST_TELEMETRY=False pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' diff --git a/infra/scripts/test-end-to-end-aws.sh b/infra/scripts/test-end-to-end-aws.sh index 807f439f6d6..e77f3b7e460 100755 --- a/infra/scripts/test-end-to-end-aws.sh +++ b/infra/scripts/test-end-to-end-aws.sh @@ -6,6 +6,7 @@ pip install "s3fs" "boto3" "urllib3>=1.25.4" export DISABLE_FEAST_SERVICE_FIXTURES=1 export DISABLE_SERVICE_FIXTURES=1 +export FEAST_TELEMETRY="False" PYTHONPATH=sdk/python pytest tests/e2e/ \ --feast-version develop \ @@ -17,4 +18,4 @@ PYTHONPATH=sdk/python pytest tests/e2e/ \ --redis-url $NODE_IP:32379 \ --emr-region us-west-2 \ --kafka-brokers $NODE_IP:30092 \ - -m "not bq" \ No newline at end of file + -m "not bq" diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh index cc54d9c8f88..d1c51f5815a 100755 --- a/infra/scripts/test-end-to-end-gcp.sh +++ b/infra/scripts/test-end-to-end-gcp.sh @@ -12,6 +12,7 @@ make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt +export FEAST_TELEMETRY="False" su -p postgres -c "PATH=$PATH HOME=/tmp pytest -v tests/e2e/ \ --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e \ diff --git a/infra/scripts/test-end-to-end-sparkop.sh b/infra/scripts/test-end-to-end-sparkop.sh index 035acc39f55..4f12daa33d4 100755 --- a/infra/scripts/test-end-to-end-sparkop.sh +++ b/infra/scripts/test-end-to-end-sparkop.sh @@ -6,6 +6,7 @@ pip install "s3fs" "boto3" "urllib3>=1.25.4" export DISABLE_FEAST_SERVICE_FIXTURES=1 export DISABLE_SERVICE_FIXTURES=1 +export FEAST_TELEMETRY="False" export FEAST_SPARK_K8S_NAMESPACE=sparkop @@ -17,4 +18,4 @@ PYTHONPATH=sdk/python pytest tests/e2e/ \ --staging-path $STAGING_PATH \ --redis-url sparkop-redis-master.sparkop.svc.cluster.local:6379 \ --kafka-brokers sparkop-kafka.sparkop.svc.cluster.local:9092 \ - -m "not bq" \ No newline at end of file + -m "not bq" diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index c4853dcbcf2..8120d83efd4 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -10,5 +10,6 @@ make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt +export FEAST_TELEMETRY="False" -su -p postgres -c "PATH=$PATH HOME=/tmp pytest -v tests/e2e/ --feast-version develop" \ No newline at end of file +su -p postgres -c "PATH=$PATH HOME=/tmp pytest -v tests/e2e/ --feast-version develop" diff --git a/infra/scripts/test-integration.sh b/infra/scripts/test-integration.sh index 2a655cfedde..ad5dd29a4ff 100755 --- a/infra/scripts/test-integration.sh +++ b/infra/scripts/test-integration.sh @@ -4,4 +4,5 @@ python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt -pytest tests/integration --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --dataproc-staging-location gs://feast-templocation-kf-feast \ No newline at end of file +export FEAST_TELEMETRY="False" +pytest tests/integration --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --dataproc-staging-location gs://feast-templocation-kf-feast diff --git a/infra/scripts/test-python-sdk.sh b/infra/scripts/test-python-sdk.sh index 7c264eed642..d80c457c509 100755 --- a/infra/scripts/test-python-sdk.sh +++ b/infra/scripts/test-python-sdk.sh @@ -11,4 +11,6 @@ make lint-python cd sdk/python/ pip install -e . +cd tests/ +export FEAST_TELEMETRY="False" pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/infra/scripts/test-telemetry.sh b/infra/scripts/test-telemetry.sh new file mode 100755 index 00000000000..749b3550d6f --- /dev/null +++ b/infra/scripts/test-telemetry.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +pip install -r sdk/python/requirements-ci.txt +make compile-protos-python +make lint-python + +cd sdk/python/ +pip install -e . +cd telemetry_tests/ +pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/pom.xml b/pom.xml index 49fc65454eb..13ee19c01c5 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ - 0.9.0 + 0.9.1 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 31af5f56f2f..f5e70f59c46 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -19,6 +19,7 @@ import warnings from datetime import datetime from itertools import groupby +from os.path import expanduser, join from typing import Any, Dict, List, Optional, Union import grpc @@ -101,6 +102,7 @@ stage_entities_to_fs, table_reference_from_string, ) +from feast.telemetry import log_usage _logger = logging.getLogger(__name__) @@ -148,6 +150,8 @@ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): if self._config.getboolean(opt.ENABLE_AUTH): self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) + self._configure_telemetry() + @property def _core_service(self): """ @@ -382,6 +386,27 @@ def version(self): return result + def _configure_telemetry(self): + telemetry_filepath = join(expanduser("~"), ".feast", "telemetry") + self._telemetry_enabled = ( + self._config.get(opt.TELEMETRY, "True") == "True" + ) # written this way to turn the env var string into a boolean + if self._telemetry_enabled: + self._telemetry_counter = {"get_online_features": 0} + if os.path.exists(telemetry_filepath): + with open(telemetry_filepath, "r") as f: + self._telemetry_id = f.read() + else: + self._telemetry_id = str(uuid.uuid4()) + print( + "Feast is an open source project that collects anonymized usage statistics. To opt out or learn more see https://docs.feast.dev/v/master/advanced/telemetry" + ) + with open(telemetry_filepath, "w") as f: + f.write(self._telemetry_id) + else: + if os.path.exists(telemetry_filepath): + os.remove(telemetry_filepath) + @property def project(self) -> str: """ @@ -487,6 +512,8 @@ def apply( >>> feast_client.apply(entity) """ + if self._telemetry_enabled: + log_usage("apply", self._telemetry_id, datetime.utcnow(), self.version()) if project is None: project = self.project @@ -594,6 +621,10 @@ def get_entity(self, name: str, project: str = None) -> Entity: none is found """ + if self._telemetry_enabled: + log_usage( + "get_entity", self._telemetry_id, datetime.utcnow(), self.version() + ) if project is None: project = self.project @@ -707,6 +738,13 @@ def get_feature_table(self, name: str, project: str = None) -> FeatureTable: none is found """ + if self._telemetry_enabled: + log_usage( + "get_feature_table", + self._telemetry_id, + datetime.utcnow(), + self.version(), + ) if project is None: project = self.project @@ -835,6 +873,8 @@ def ingest( >>> client.ingest(driver_ft, ft_df) """ + if self._telemetry_enabled: + log_usage("ingest", self._telemetry_id, datetime.utcnow(), self.version()) if project is None: project = self.project if isinstance(feature_table, str): @@ -959,6 +999,15 @@ def get_online_features( {'sales:daily_transactions': [1.1,1.2], 'sales:customer_id': [0,1]} """ + if self._telemetry_enabled: + if self._telemetry_counter["get_online_features"] % 1000 == 0: + log_usage( + "get_online_features", + self._telemetry_id, + datetime.utcnow(), + self.version(), + ) + self._telemetry_counter["get_online_features"] += 1 try: response = self._serving_service.GetOnlineFeaturesV2( GetOnlineFeaturesRequestV2( @@ -1019,6 +1068,13 @@ def get_historical_features( >>> output_file_uri = feature_retrieval_job.get_output_file_uri() "gs://some-bucket/output/ """ + if self._telemetry_enabled: + log_usage( + "get_historical_features", + self._telemetry_id, + datetime.utcnow(), + self.version(), + ) feature_tables = self._get_feature_tables_from_feature_refs( feature_refs, self.project ) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 1510f417c18..39f88e172ff 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -259,6 +259,9 @@ class ConfigOptions(metaclass=ConfigMeta): #: Oauth token request url OAUTH_TOKEN_REQUEST_URL: Optional[str] = None + #: Telemetry enabled + TELEMETRY = "True" + def defaults(self): return { k: getattr(self, k) diff --git a/sdk/python/feast/job_service.py b/sdk/python/feast/job_service.py index bf9f67df584..99a460946c9 100644 --- a/sdk/python/feast/job_service.py +++ b/sdk/python/feast/job_service.py @@ -218,7 +218,7 @@ def start_control_loop() -> None: "which will ensure that stream ingestion jobs are successfully running." ) try: - client = feast.Client() + client = feast.Client(telemetry="False") while True: ensure_stream_ingestion_jobs(client, all_projects=True) time.sleep(1) @@ -248,7 +248,7 @@ def start_job_service() -> None: log_fmt = "%(asctime)s %(levelname)s %(message)s" logging.basicConfig(level=logging.INFO, format=log_fmt) - client = feast.Client() + client = feast.Client(telemetry="False") if client._config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP): # Start the control loop thread only if it's enabled from configs diff --git a/sdk/python/feast/telemetry.py b/sdk/python/feast/telemetry.py new file mode 100644 index 00000000000..485b2a74487 --- /dev/null +++ b/sdk/python/feast/telemetry.py @@ -0,0 +1,45 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from datetime import datetime +from typing import Dict + +import requests + +TELEMETRY_ENDPOINT = ( + "https://us-central1-kf-feast.cloudfunctions.net/bq_telemetry_logger" +) + + +def log_usage( + function_name: str, + telemetry_id: str, + timestamp: datetime, + version: Dict[str, Dict[str, str]], +): + json = { + "function_name": function_name, + "telemetry_id": telemetry_id, + "timestamp": timestamp.isoformat(), + "version": version, + "os": sys.platform, + "is_test": os.getenv("FEAST_IS_TELEMETRY_TEST", "False"), + } + try: + requests.post(TELEMETRY_ENDPOINT, json=json) + except Exception: + pass + return diff --git a/sdk/python/telemetry_tests/test_telemetry.py b/sdk/python/telemetry_tests/test_telemetry.py new file mode 100644 index 00000000000..fe6ef7f2416 --- /dev/null +++ b/sdk/python/telemetry_tests/test_telemetry.py @@ -0,0 +1,70 @@ +# Copyright 2020 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from feast.client import Client +from feast.entity import Entity +from feast.value_type import ValueType +from google.cloud import bigquery +import os +import pytest +from time import sleep + +def test_telemetry_on(): + os.environ["FEAST_IS_TELEMETRY_TEST"] = 'True' + test_client = Client(serving_url=None, core_url=None) + test_client.set_project("project1") + entity = Entity( + name="driver_car_id", + description="Car driver id", + value_type=ValueType.STRING, + labels={"team": "matchmaking"}, + ) + + timestamp = datetime.utcnow() + try: + test_client.apply(entity) + except Exception: + pass + + sleep(30) + bq_client = bigquery.Client() + query = f"select * from `kf-feast.feast_telemetry.cloudfunctions_googleapis_com_cloud_functions` where timestamp >= TIMESTAMP(\"{timestamp.date().isoformat()}\") and JSON_EXTRACT(textPayload, '$.is_test')='\"True\"' and JSON_EXTRACT(textPayload, '$.timestamp')>'\"{timestamp.isoformat()}\"'" + query_job = bq_client.query(query) + rows = query_job.result() + assert(rows.total_rows == 1) + +def test_telemetry_off(): + os.environ["FEAST_IS_TELEMETRY_TEST"] = 'True' + test_client = Client(serving_url=None, core_url=None, telemetry=False) + test_client.set_project("project1") + entity = Entity( + name="driver_car_id", + description="Car driver id", + value_type=ValueType.STRING, + labels={"team": "matchmaking"}, + ) + + timestamp = datetime.utcnow() + try: + test_client.apply(entity) + except Exception: + pass + + sleep(30) + bq_client = bigquery.Client() + query = f"select * from `kf-feast.feast_telemetry.cloudfunctions_googleapis_com_cloud_functions` where timestamp >= TIMESTAMP(\"{timestamp.date().isoformat()}\") and JSON_EXTRACT(textPayload, '$.is_test')='\"True\"' and JSON_EXTRACT(textPayload, '$.timestamp')>'\"{timestamp.isoformat()}\"'" + query_job = bq_client.query(query) + rows = query_job.result() + assert(rows.total_rows == 0) diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala index 290dc07fbfd..cecdf6d2713 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala @@ -31,7 +31,7 @@ object IngestionJob { val parser = new scopt.OptionParser[IngestionJobConfig]("IngestionJob") { // ToDo: read version from Manifest - head("feast.ingestion.IngestionJob", "0.9.0") + head("feast.ingestion.IngestionJob", "0.9.1") opt[Modes]("mode") .action((x, c) => c.copy(mode = x)) diff --git a/tests/e2e/fixtures/services.py b/tests/e2e/fixtures/services.py index aa242288781..a8571289cae 100644 --- a/tests/e2e/fixtures/services.py +++ b/tests/e2e/fixtures/services.py @@ -27,7 +27,7 @@ def download_kafka(version="2.12-2.6.0"): if not os.path.isfile(local_path): r = requests.get( - f"https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz" + f"https://archive.apache.org/dist/kafka/2.6.0/kafka_{version}.tgz" ) with open(local_path, "wb") as f: