diff --git a/.prow.yaml b/.prow.yaml index b973cffbbf2..e614e4a2f96 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -64,13 +64,13 @@ presubmits: branches: - ^v0\.(3|4)-branch$ -- name: test-telemetry +- name: test-usage decorate: true run_if_changed: "sdk/python/.*" spec: containers: - image: python:3.7 - command: ["infra/scripts/test-telemetry.sh"] + command: ["infra/scripts/test-usage.sh"] env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/gcloud/service-account.json diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 30b19cfdcc8..b057db87a7e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -43,7 +43,7 @@ * [.feastignore](reference/feature-repository/feast-ignore.md) * [Feast CLI reference](reference/feast-cli-commands.md) * [Python API reference](http://rtd.feast.dev/) -* [Telemetry](reference/telemetry.md) +* [Usage](reference/usage.md) ## Feast on Kubernetes diff --git a/docs/advanced/telemetry.md b/docs/advanced/telemetry.md deleted file mode 100644 index c817e1ce15a..00000000000 --- a/docs/advanced/telemetry.md +++ /dev/null @@ -1,10 +0,0 @@ -# Telemetry - -## How telemetry is used - -The Feast maintainers use anonymous usage statistics to help shape the Feast roadmap. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. For more detailed information see [the source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/telemetry.py). - -## How to disable telemetry - -To opt out of telemetry, simply set the environment variable `FEAST_TELEMETRY` to `False` in the environment in which the Feast client is run. - diff --git a/docs/reference/usage.md b/docs/reference/usage.md new file mode 100644 index 00000000000..6d37bdfd46a --- /dev/null +++ b/docs/reference/usage.md @@ -0,0 +1,11 @@ +# Usage + +### How Feast SDK usage is measured + +The Feast project logs anonymous usage statistics and errors in order to inform our planning. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. + +The [source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/usage.py) is available here. + +### How to disable usage logging + +Set the environment variable `FEAST_USAGE` to `False`. diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 3681255e465..69cdaac82fa 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -70,4 +70,4 @@ docker exec \ -e DISABLE_FEAST_SERVICE_FIXTURES=true \ --user root \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && FEAST_TELEMETRY=False pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && FEAST_USAGE=False pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 8120d83efd4..1c94f9c02d3 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -10,6 +10,6 @@ make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt -export FEAST_TELEMETRY="False" +export FEAST_USAGE="False" su -p postgres -c "PATH=$PATH HOME=/tmp pytest -v tests/e2e/ --feast-version develop" diff --git a/infra/scripts/test-integration.sh b/infra/scripts/test-integration.sh index ad5dd29a4ff..5e88e0281b2 100755 --- a/infra/scripts/test-integration.sh +++ b/infra/scripts/test-integration.sh @@ -4,5 +4,5 @@ python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt -export FEAST_TELEMETRY="False" +export FEAST_USAGE="False" pytest tests/integration --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --dataproc-staging-location gs://feast-templocation-kf-feast diff --git a/infra/scripts/test-telemetry.sh b/infra/scripts/test-usage.sh similarity index 91% rename from infra/scripts/test-telemetry.sh rename to infra/scripts/test-usage.sh index c17f9b107c2..f70fdd62479 100755 --- a/infra/scripts/test-telemetry.sh +++ b/infra/scripts/test-usage.sh @@ -7,5 +7,5 @@ LOGS_ARTIFACT_PATH=/logs/artifacts cd sdk/python/ pip install -e ".[ci]" -cd telemetry_tests/ +cd usage_tests/ pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 9afc4ba9268..c7cd1259ae2 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -69,7 +69,7 @@ ) from feast.protos.feast.serving.ServingService_pb2_grpc import ServingServiceStub from feast.registry import Registry -from feast.telemetry import Telemetry +from feast.usage import Usage _logger = logging.getLogger(__name__) @@ -117,7 +117,7 @@ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): if self._config.getboolean(opt.ENABLE_AUTH): self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) - self._tele = Telemetry() + self._usage = Usage() @property def config(self) -> Config: @@ -467,7 +467,7 @@ def apply( >>> feast_client.apply(entity) """ - self._tele.log("apply") + self._usage.log("apply") if project is None: project = self.project @@ -581,7 +581,7 @@ def get_entity(self, name: str, project: str = None) -> Entity: none is found """ - self._tele.log("get_entity") + self._usage.log("get_entity") if project is None: project = self.project @@ -706,7 +706,7 @@ def get_feature_table(self, name: str, project: str = None) -> FeatureTable: none is found """ - self._tele.log("get_feature_table") + self._usage.log("get_feature_table") if project is None: project = self.project @@ -847,7 +847,7 @@ def ingest( >>> client.ingest(driver_ft, ft_df) """ - self._tele.log("ingest") + self._usage.log("ingest") if project is None: project = self.project if isinstance(feature_table, str): @@ -972,7 +972,7 @@ def get_online_features( {'sales:daily_transactions': [1.1,1.2], 'sales:customer_id': [0,1]} """ - self._tele.log("get_online_features") + self._usage.log("get_online_features") try: response = self._serving_service.GetOnlineFeaturesV2( GetOnlineFeaturesRequestV2( diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 259920edaee..8bf6ea16ec3 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -260,8 +260,8 @@ class ConfigOptions(metaclass=ConfigMeta): #: Oauth token request url OAUTH_TOKEN_REQUEST_URL: Optional[str] = None - #: Telemetry enabled - TELEMETRY = "True" + #: Usage enabled + USAGE = "True" #: Object store registry REGISTRY_PATH: Optional[str] = None diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 832b9e4db8d..f73e8c70efe 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -23,7 +23,7 @@ from feast.protos.feast.core.Entity_pb2 import Entity as EntityV2Proto from feast.protos.feast.core.Entity_pb2 import EntityMeta as EntityMetaProto from feast.protos.feast.core.Entity_pb2 import EntitySpecV2 as EntitySpecProto -from feast.telemetry import log_exceptions +from feast.usage import log_exceptions from feast.value_type import ValueType diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 83e1cc9a998..e34d314e5fd 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -39,7 +39,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.registry import Registry from feast.repo_config import RepoConfig, load_repo_config -from feast.telemetry import log_exceptions, log_exceptions_and_usage +from feast.usage import log_exceptions, log_exceptions_and_usage from feast.version import get_version diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 114bb37e613..d89a8a7b6be 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -33,7 +33,7 @@ from feast.protos.feast.core.FeatureView_pb2 import ( MaterializationInterval as MaterializationIntervalProto, ) -from feast.telemetry import log_exceptions +from feast.usage import log_exceptions from feast.value_type import ValueType diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 5587fb59053..c680d94d07a 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -7,7 +7,7 @@ from pydantic.typing import Dict, Optional, Union from feast.importer import get_class_from_type -from feast.telemetry import log_exceptions +from feast.usage import log_exceptions # These dict exists so that: # - existing values for the online store type in featurestore.yaml files continue to work in a backwards compatible way diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index f4a44a74559..4f2cb1981d3 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -21,7 +21,7 @@ from feast.names import adjectives, animals from feast.registry import Registry from feast.repo_config import RepoConfig -from feast.telemetry import log_exceptions_and_usage +from feast.usage import log_exceptions_and_usage def py_path_to_module(path: Path, repo_root: Path) -> str: diff --git a/sdk/python/feast/telemetry.py b/sdk/python/feast/usage.py similarity index 67% rename from sdk/python/feast/telemetry.py rename to sdk/python/feast/usage.py index 7c8247f3af0..a089f660db3 100644 --- a/sdk/python/feast/telemetry.py +++ b/sdk/python/feast/usage.py @@ -25,76 +25,72 @@ from feast.version import get_version -TELEMETRY_ENDPOINT = ( - "https://us-central1-kf-feast.cloudfunctions.net/bq_telemetry_logger" -) +USAGE_ENDPOINT = "https://us-central1-kf-feast.cloudfunctions.net/bq_telemetry_logger" _logger = logging.getLogger(__name__) -class Telemetry: +class Usage: def __init__(self): - self._telemetry_enabled: bool = False + self._usage_enabled: bool = False self.check_env_and_configure() def check_env_and_configure(self): - telemetry_enabled = ( - os.getenv("FEAST_TELEMETRY", default="True") == "True" + usage_enabled = ( + os.getenv("FEAST_USAGE", default="True") == "True" ) # written this way to turn the env var string into a boolean # Check if it changed - if telemetry_enabled != self._telemetry_enabled: - self._telemetry_enabled = telemetry_enabled + if usage_enabled != self._usage_enabled: + self._usage_enabled = usage_enabled - if self._telemetry_enabled: + if self._usage_enabled: try: feast_home_dir = join(expanduser("~"), ".feast") Path(feast_home_dir).mkdir(exist_ok=True) - telemetry_filepath = join(feast_home_dir, "telemetry") + usage_filepath = join(feast_home_dir, "usage") - self._is_test = ( - os.getenv("FEAST_IS_TELEMETRY_TEST", "False") == "True" - ) - self._telemetry_counter = {"get_online_features": 0} + self._is_test = os.getenv("FEAST_IS_USAGE_TEST", "False") == "True" + self._usage_counter = {"get_online_features": 0} - if os.path.exists(telemetry_filepath): - with open(telemetry_filepath, "r") as f: - self._telemetry_id = f.read() + if os.path.exists(usage_filepath): + with open(usage_filepath, "r") as f: + self._usage_id = f.read() else: - self._telemetry_id = str(uuid.uuid4()) + self._usage_id = str(uuid.uuid4()) - with open(telemetry_filepath, "w") as f: - f.write(self._telemetry_id) + with open(usage_filepath, "w") as f: + f.write(self._usage_id) print( "Feast is an open source project that collects anonymized error reporting and usage statistics. To opt out or learn" - " more see https://docs.feast.dev/reference/telemetry" + " more see https://docs.feast.dev/reference/usage" ) except Exception as e: - _logger.debug(f"Unable to configure telemetry {e}") + _logger.debug(f"Unable to configure usage {e}") @property - def telemetry_id(self) -> Optional[str]: - if os.getenv("FEAST_FORCE_TELEMETRY_UUID"): - return os.getenv("FEAST_FORCE_TELEMETRY_UUID") - return self._telemetry_id + def usage_id(self) -> Optional[str]: + if os.getenv("FEAST_FORCE_USAGE_UUID"): + return os.getenv("FEAST_FORCE_USAGE_UUID") + return self._usage_id def log(self, function_name: str): self.check_env_and_configure() - if self._telemetry_enabled and self.telemetry_id: + if self._usage_enabled and self.usage_id: if function_name == "get_online_features": - if self._telemetry_counter["get_online_features"] % 10000 != 0: - self._telemetry_counter["get_online_features"] += 1 + if self._usage_counter["get_online_features"] % 10000 != 0: + self._usage_counter["get_online_features"] += 1 return json = { "function_name": function_name, - "telemetry_id": self.telemetry_id, + "telemetry_id": self.usage_id, "timestamp": datetime.utcnow().isoformat(), "version": get_version(), "os": sys.platform, "is_test": self._is_test, } try: - requests.post(TELEMETRY_ENDPOINT, json=json) + requests.post(USAGE_ENDPOINT, json=json) except Exception as e: if self._is_test: raise e @@ -104,17 +100,17 @@ def log(self, function_name: str): def log_exception(self, error_type: str, traceback: List[Tuple[str, int, str]]): self.check_env_and_configure() - if self._telemetry_enabled and self.telemetry_id: + if self._usage_enabled and self.usage_id: json = { "error_type": error_type, "traceback": traceback, - "telemetry_id": self.telemetry_id, + "telemetry_id": self.usage_id, "version": get_version(), "os": sys.platform, "is_test": self._is_test, } try: - requests.post(TELEMETRY_ENDPOINT, json=json) + requests.post(USAGE_ENDPOINT, json=json) except Exception as e: if self._is_test: raise e @@ -141,7 +137,7 @@ def exception_logging_wrapper(*args, **kwargs): ) ) tb = tb.tb_next - tele.log_exception(error_type, trace_to_log) + usage.log_exception(error_type, trace_to_log) raise return result @@ -153,7 +149,7 @@ def log_exceptions_and_usage(func): def exception_logging_wrapper(*args, **kwargs): try: result = func(*args, **kwargs) - tele.log(func.__name__) + usage.log(func.__name__) except Exception as e: error_type = type(e).__name__ trace_to_log = [] @@ -167,7 +163,7 @@ def exception_logging_wrapper(*args, **kwargs): ) ) tb = tb.tb_next - tele.log_exception(error_type, trace_to_log) + usage.log_exception(error_type, trace_to_log) raise return result @@ -178,5 +174,5 @@ def _trim_filename(filename: str) -> str: return filename.split("/")[-1] -# Single global telemetry object -tele = Telemetry() +# Single global usage object +usage = Usage() diff --git a/sdk/python/telemetry_tests/test_telemetry.py b/sdk/python/usage_tests/test_usage.py similarity index 67% rename from sdk/python/telemetry_tests/test_telemetry.py rename to sdk/python/usage_tests/test_usage.py index 9b35bf3c17a..e6b7760fabb 100644 --- a/sdk/python/telemetry_tests/test_telemetry.py +++ b/sdk/python/usage_tests/test_usage.py @@ -25,18 +25,18 @@ from feast import Client, Entity, ValueType, FeatureStore, RepoConfig -TELEMETRY_BIGQUERY_TABLE = ( +USAGE_BIGQUERY_TABLE = ( "kf-feast.feast_telemetry.cloudfunctions_googleapis_com_cloud_functions" ) -def test_telemetry_on_v09(mocker): +def test_usage_on_v09(mocker): # Setup environment old_environ = dict(os.environ) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - test_client = Client(serving_url=None, core_url=None, telemetry=True) + os.environ["FEAST_IS_USAGE_TEST"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + test_client = Client(serving_url=None, core_url=None, usage=True) test_client.set_project("project1") entity = Entity( name="driver_car_id", @@ -54,17 +54,17 @@ def test_telemetry_on_v09(mocker): os.environ.clear() os.environ.update(old_environ) - ensure_bigquery_telemetry_id_with_retry(test_telemetry_id) + ensure_bigquery_usage_id_with_retry(test_usage_id) -def test_telemetry_off_v09(mocker): +def test_usage_off_v09(mocker): old_environ = dict(os.environ) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - os.environ["FEAST_TELEMETRY"] = "False" + os.environ["FEAST_IS_USAGE_TEST"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + os.environ["FEAST_USAGE"] = "False" - test_client = Client(serving_url=None, core_url=None, telemetry=False) + test_client = Client(serving_url=None, core_url=None, usage=False) test_client.set_project("project1") entity = Entity( name="driver_car_id", @@ -82,16 +82,16 @@ def test_telemetry_off_v09(mocker): os.environ.clear() os.environ.update(old_environ) sleep(30) - rows = read_bigquery_telemetry_id(test_telemetry_id) + rows = read_bigquery_usage_id(test_usage_id) assert rows.total_rows == 0 -def test_telemetry_on(): +def test_usage_on(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "True" with tempfile.TemporaryDirectory() as temp_dir: test_feature_store = FeatureStore( @@ -115,15 +115,15 @@ def test_telemetry_on(): os.environ.clear() os.environ.update(old_environ) - ensure_bigquery_telemetry_id_with_retry(test_telemetry_id) + ensure_bigquery_usage_id_with_retry(test_usage_id) -def test_telemetry_off(): +def test_usage_off(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "False" - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "False" + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id with tempfile.TemporaryDirectory() as temp_dir: test_feature_store = FeatureStore( @@ -147,16 +147,16 @@ def test_telemetry_off(): os.environ.clear() os.environ.update(old_environ) sleep(30) - rows = read_bigquery_telemetry_id(test_telemetry_id) + rows = read_bigquery_usage_id(test_usage_id) assert rows.total_rows == 0 -def test_exception_telemetry_on(): +def test_exception_usage_on(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "True" try: test_feature_store = FeatureStore("/tmp/non_existent_directory") @@ -165,15 +165,15 @@ def test_exception_telemetry_on(): os.environ.clear() os.environ.update(old_environ) - ensure_bigquery_telemetry_id_with_retry(test_telemetry_id) + ensure_bigquery_usage_id_with_retry(test_usage_id) -def test_exception_telemetry_off(): +def test_exception_usage_off(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "False" - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "False" + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id try: test_feature_store = FeatureStore("/tmp/non_existent_directory") @@ -183,18 +183,18 @@ def test_exception_telemetry_off(): os.environ.clear() os.environ.update(old_environ) sleep(30) - rows = read_bigquery_telemetry_id(test_telemetry_id) + rows = read_bigquery_usage_id(test_usage_id) assert rows.total_rows == 0 @retry(wait=wait_exponential(multiplier=1, min=1, max=10), stop=stop_after_attempt(5)) -def ensure_bigquery_telemetry_id_with_retry(telemetry_id): - rows = read_bigquery_telemetry_id(telemetry_id) +def ensure_bigquery_usage_id_with_retry(usage_id): + rows = read_bigquery_usage_id(usage_id) if rows.total_rows != 1: - raise Exception(f"Could not find telemetry id: {telemetry_id}") + raise Exception(f"Could not find usage id: {usage_id}") -def read_bigquery_telemetry_id(telemetry_id): +def read_bigquery_usage_id(usage_id): bq_client = bigquery.Client() query = f""" SELECT @@ -203,11 +203,11 @@ def read_bigquery_telemetry_id(telemetry_id): SELECT JSON_EXTRACT(textPayload, '$.telemetry_id') AS telemetry_id FROM - `{TELEMETRY_BIGQUERY_TABLE}` + `{USAGE_BIGQUERY_TABLE}` WHERE timestamp >= TIMESTAMP(\"{datetime.utcnow().date().isoformat()}\")) WHERE - telemetry_id = '\"{telemetry_id}\"' + telemetry_id = '\"{usage_id}\"' """ query_job = bq_client.query(query) return query_job.result()