Skip to content

Commit 2f4a20f

Browse files
Genesis929tswastLinchin
authored
feat: add created/started/ended properties to RowIterator. (#2260)
* feat: add created/started/ended attribute to RowIterator. * fix annotation * links update * mypy fix * Update google/cloud/bigquery/query.py Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> * Update google/cloud/bigquery/table.py Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> * Update google/cloud/bigquery/table.py Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> * Update google/cloud/bigquery/query.py Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> * Update google/cloud/bigquery/query.py Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> * Update google/cloud/bigquery/job/query.py Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> * fix unit test --------- Co-authored-by: Tim Sweña (Swast) <tswast@gmail.com> Co-authored-by: Lingqing Gan <lingqing.gan@gmail.com>
1 parent 8386be6 commit 2f4a20f

File tree

8 files changed

+159
-2
lines changed

8 files changed

+159
-2
lines changed

‎packages/google-cloud-bigquery/google/cloud/bigquery/_job_helpers.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,9 @@ def do_query():
561561
query=query,
562562
total_bytes_processed=query_results.total_bytes_processed,
563563
slot_millis=query_results.slot_millis,
564+
created=query_results.created,
565+
started=query_results.started,
566+
ended=query_results.ended,
564567
)
565568

566569
if job_retry is not None:

‎packages/google-cloud-bigquery/google/cloud/bigquery/client.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4145,6 +4145,9 @@ def _list_rows_from_query_results(
41454145
query: Optional[str] = None,
41464146
total_bytes_processed: Optional[int] = None,
41474147
slot_millis: Optional[int] = None,
4148+
created: Optional[datetime.datetime] = None,
4149+
started: Optional[datetime.datetime] = None,
4150+
ended: Optional[datetime.datetime] = None,
41484151
) -> RowIterator:
41494152
"""List the rows of a completed query.
41504153
See
@@ -4198,6 +4201,12 @@ def _list_rows_from_query_results(
41984201
total bytes processed from job statistics, if present.
41994202
slot_millis (Optional[int]):
42004203
Number of slot ms the user is actually billed for.
4204+
created (Optional[datetime.datetime]):
4205+
Datetime at which the job was created.
4206+
started (Optional[datetime.datetime]):
4207+
Datetime at which the job was started.
4208+
ended (Optional[datetime.datetime]):
4209+
Datetime at which the job finished.
42014210
42024211
Returns:
42034212
google.cloud.bigquery.table.RowIterator:
@@ -4238,6 +4247,9 @@ def _list_rows_from_query_results(
42384247
query=query,
42394248
total_bytes_processed=total_bytes_processed,
42404249
slot_millis=slot_millis,
4250+
created=created,
4251+
started=started,
4252+
ended=ended,
42414253
)
42424254
return row_iterator
42434255

‎packages/google-cloud-bigquery/google/cloud/bigquery/job/query.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1767,6 +1767,9 @@ def is_job_done():
17671767
query=self.query,
17681768
total_bytes_processed=self.total_bytes_processed,
17691769
slot_millis=self.slot_millis,
1770+
created=self.created,
1771+
started=self.started,
1772+
ended=self.ended,
17701773
**list_rows_kwargs,
17711774
)
17721775
rows._preserve_order = _contains_order_by(self.query)

‎packages/google-cloud-bigquery/google/cloud/bigquery/query.py‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1287,7 +1287,7 @@ def slot_millis(self):
12871287
"""Total number of slot ms the user is actually billed for.
12881288
12891289
See:
1290-
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.slot_millis
1290+
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.total_slot_ms
12911291
12921292
Returns:
12931293
Optional[int]: Count generated on the server (None until set by the server).
@@ -1310,6 +1310,56 @@ def num_dml_affected_rows(self):
13101310
if num_dml_affected_rows is not None:
13111311
return int(num_dml_affected_rows)
13121312

1313+
@property
1314+
def created(self):
1315+
"""Creation time of this query.
1316+
1317+
See:
1318+
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.creation_time
1319+
1320+
Returns:
1321+
Optional[datetime.datetime]:
1322+
the creation time (None until set from the server).
1323+
"""
1324+
millis = self._properties.get("creationTime")
1325+
if millis is not None:
1326+
return _helpers._datetime_from_microseconds(int(millis) * 1000.0)
1327+
1328+
@property
1329+
def started(self):
1330+
"""Start time of this query.
1331+
1332+
This field will be present when the query transitions from the
1333+
PENDING state to either RUNNING or DONE.
1334+
1335+
See:
1336+
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.start_time
1337+
1338+
Returns:
1339+
Optional[datetime.datetime]:
1340+
the start time (None until set from the server).
1341+
"""
1342+
millis = self._properties.get("startTime")
1343+
if millis is not None:
1344+
return _helpers._datetime_from_microseconds(int(millis) * 1000.0)
1345+
1346+
@property
1347+
def ended(self):
1348+
"""End time of this query.
1349+
1350+
This field will be present whenever a query is in the DONE state.
1351+
1352+
See:
1353+
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.end_time
1354+
1355+
Returns:
1356+
Optional[datetime.datetime]:
1357+
the end time (None until set from the server).
1358+
"""
1359+
millis = self._properties.get("endTime")
1360+
if millis is not None:
1361+
return _helpers._datetime_from_microseconds(int(millis) * 1000.0)
1362+
13131363
@property
13141364
def rows(self):
13151365
"""Query results.

‎packages/google-cloud-bigquery/google/cloud/bigquery/table.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1788,7 +1788,15 @@ class RowIterator(HTTPIterator):
17881788
query (Optional[str]):
17891789
The query text used.
17901790
total_bytes_processed (Optional[int]):
1791-
total bytes processed from job statistics, if present.
1791+
If representing query results, the total bytes processed by the associated query.
1792+
slot_millis (Optional[int]):
1793+
If representing query results, the number of slot ms billed for the associated query.
1794+
created (Optional[datetime.datetime]):
1795+
If representing query results, the creation time of the associated query.
1796+
started (Optional[datetime.datetime]):
1797+
If representing query results, the start time of the associated query.
1798+
ended (Optional[datetime.datetime]):
1799+
If representing query results, the end time of the associated query.
17921800
"""
17931801

17941802
def __init__(
@@ -1813,6 +1821,9 @@ def __init__(
18131821
query: Optional[str] = None,
18141822
total_bytes_processed: Optional[int] = None,
18151823
slot_millis: Optional[int] = None,
1824+
created: Optional[datetime.datetime] = None,
1825+
started: Optional[datetime.datetime] = None,
1826+
ended: Optional[datetime.datetime] = None,
18161827
):
18171828
super(RowIterator, self).__init__(
18181829
client,
@@ -1843,6 +1854,9 @@ def __init__(
18431854
self._query = query
18441855
self._total_bytes_processed = total_bytes_processed
18451856
self._slot_millis = slot_millis
1857+
self._job_created = created
1858+
self._job_started = started
1859+
self._job_ended = ended
18461860

18471861
@property
18481862
def _billing_project(self) -> Optional[str]:
@@ -1905,6 +1919,21 @@ def slot_millis(self) -> Optional[int]:
19051919
"""Number of slot ms the user is actually billed for."""
19061920
return self._slot_millis
19071921

1922+
@property
1923+
def created(self) -> Optional[datetime.datetime]:
1924+
"""If representing query results, the creation time of the associated query."""
1925+
return self._job_created
1926+
1927+
@property
1928+
def started(self) -> Optional[datetime.datetime]:
1929+
"""If representing query results, the start time of the associated query."""
1930+
return self._job_started
1931+
1932+
@property
1933+
def ended(self) -> Optional[datetime.datetime]:
1934+
"""If representing query results, the end time of the associated query."""
1935+
return self._job_ended
1936+
19081937
def _is_almost_completely_cached(self):
19091938
"""Check if all results are completely cached.
19101939

‎packages/google-cloud-bigquery/tests/unit/job/test_query.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -889,6 +889,9 @@ def test_result_reloads_job_state_until_done(self):
889889
job_resource_done = self._make_resource(started=True, ended=True, location="EU")
890890
job_resource_done["statistics"]["query"]["totalBytesProcessed"] = str(1234)
891891
job_resource_done["statistics"]["query"]["totalSlotMs"] = str(5678)
892+
job_resource_done["statistics"]["creationTime"] = str(11)
893+
job_resource_done["statistics"]["startTime"] = str(22)
894+
job_resource_done["statistics"]["endTime"] = str(33)
892895
job_resource_done["configuration"]["query"]["destinationTable"] = {
893896
"projectId": "dest-project",
894897
"datasetId": "dest_dataset",
@@ -971,6 +974,9 @@ def test_result_reloads_job_state_until_done(self):
971974
self.assertEqual(result.query, job.query)
972975
self.assertEqual(result.total_bytes_processed, 1234)
973976
self.assertEqual(result.slot_millis, 5678)
977+
self.assertEqual(result.created.timestamp() * 1000, 11)
978+
self.assertEqual(result.started.timestamp() * 1000, 22)
979+
self.assertEqual(result.ended.timestamp() * 1000, 33)
974980

975981
query_results_path = f"/projects/{self.PROJECT}/queries/{self.JOB_ID}"
976982
query_results_call = mock.call(

‎packages/google-cloud-bigquery/tests/unit/test_client.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5719,6 +5719,9 @@ def test_query_and_wait_defaults(self):
57195719
"queryId": "job_abcDEF_",
57205720
"totalBytesProcessed": 1234,
57215721
"totalSlotMs": 5678,
5722+
"creationTime": "1437767599006",
5723+
"startTime": "1437767600007",
5724+
"endTime": "1437767601008",
57225725
}
57235726
creds = _make_credentials()
57245727
http = object()
@@ -5737,6 +5740,9 @@ def test_query_and_wait_defaults(self):
57375740
self.assertEqual(rows.query, query)
57385741
self.assertEqual(rows.total_bytes_processed, 1234)
57395742
self.assertEqual(rows.slot_millis, 5678)
5743+
self.assertEqual(rows.created.timestamp() * 1000, 1437767599006)
5744+
self.assertEqual(rows.started.timestamp() * 1000, 1437767600007)
5745+
self.assertEqual(rows.ended.timestamp() * 1000, 1437767601008)
57405746

57415747
# Verify the request we send is to jobs.query.
57425748
conn.api_request.assert_called_once()

‎packages/google-cloud-bigquery/tests/unit/test_query.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2016,6 +2016,54 @@ def test_slot_millis_present_string(self):
20162016
query = self._make_one(resource)
20172017
self.assertEqual(query.slot_millis, 123456)
20182018

2019+
def test_created_missing(self):
2020+
query = self._make_one(self._make_resource())
2021+
self.assertIsNone(query.created)
2022+
2023+
def test_created_present_integer(self):
2024+
resource = self._make_resource()
2025+
resource["creationTime"] = 1437767599006
2026+
query = self._make_one(resource)
2027+
self.assertEqual(query.created.timestamp() * 1000, 1437767599006)
2028+
2029+
def test_created_present_string(self):
2030+
resource = self._make_resource()
2031+
resource["creationTime"] = "1437767599006"
2032+
query = self._make_one(resource)
2033+
self.assertEqual(query.created.timestamp() * 1000, 1437767599006)
2034+
2035+
def test_started_missing(self):
2036+
query = self._make_one(self._make_resource())
2037+
self.assertIsNone(query.started)
2038+
2039+
def test_started_present_integer(self):
2040+
resource = self._make_resource()
2041+
resource["startTime"] = 1437767599006
2042+
query = self._make_one(resource)
2043+
self.assertEqual(query.started.timestamp() * 1000, 1437767599006)
2044+
2045+
def test_started_present_string(self):
2046+
resource = self._make_resource()
2047+
resource["startTime"] = "1437767599006"
2048+
query = self._make_one(resource)
2049+
self.assertEqual(query.started.timestamp() * 1000, 1437767599006)
2050+
2051+
def test_ended_missing(self):
2052+
query = self._make_one(self._make_resource())
2053+
self.assertIsNone(query.ended)
2054+
2055+
def test_ended_present_integer(self):
2056+
resource = self._make_resource()
2057+
resource["endTime"] = 1437767599006
2058+
query = self._make_one(resource)
2059+
self.assertEqual(query.ended.timestamp() * 1000, 1437767599006)
2060+
2061+
def test_ended_present_string(self):
2062+
resource = self._make_resource()
2063+
resource["endTime"] = "1437767599006"
2064+
query = self._make_one(resource)
2065+
self.assertEqual(query.ended.timestamp() * 1000, 1437767599006)
2066+
20192067
def test_num_dml_affected_rows_missing(self):
20202068
query = self._make_one(self._make_resource())
20212069
self.assertIsNone(query.num_dml_affected_rows)

0 commit comments

Comments
 (0)