Skip to content

Commit 38d8f2b

Browse files
tswasttswast
authored
chore(bigquery): run unit tests with Python 3.8 (googleapis#9880)
* chore(bigquery): run unit tests with Python 3.8 Coverage is a bit low because of the missing pyarrow dependency. * doc: add comments about changes for Python 3.8 testing * unit test with fastparquet
1 parent 477ea57 commit 38d8f2b

File tree

5 files changed

+74
-39
lines changed

5 files changed

+74
-39
lines changed

‎bigquery/noxfile.py‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,22 @@ def default(session):
3939
session.install("-e", local_dep)
4040

4141
session.install("-e", os.path.join("..", "test_utils"))
42-
dev_install = ".[all]"
42+
43+
coverage_fail_under = "--cov-fail-under=97"
44+
45+
# fastparquet is not included in .[all] because, in general, it's redundant
46+
# with pyarrow. We still want to run some unit tests with fastparquet
47+
# serialization, though.
48+
dev_install = ".[all,fastparquet]"
49+
50+
# There is no pyarrow or fastparquet wheel for Python 3.8.
51+
if session.python == "3.8":
52+
# Since many tests are skipped due to missing dependencies, test
53+
# coverage is much lower in Python 3.8. Remove once we can test with
54+
# pyarrow.
55+
coverage_fail_under = "--cov-fail-under=92"
56+
dev_install = ".[pandas,tqdm]"
57+
4358
session.install("-e", dev_install)
4459

4560
# IPython does not support Python 2 after version 5.x
@@ -57,19 +72,19 @@ def default(session):
5772
"--cov-append",
5873
"--cov-config=.coveragerc",
5974
"--cov-report=",
60-
"--cov-fail-under=97",
75+
coverage_fail_under,
6176
os.path.join("tests", "unit"),
6277
*session.posargs
6378
)
6479

6580

66-
@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
81+
@nox.session(python=["2.7", "3.5", "3.6", "3.7", "3.8"])
6782
def unit(session):
6883
"""Run the unit test suite."""
6984
default(session)
7085

7186

72-
@nox.session(python=["2.7", "3.6"])
87+
@nox.session(python=["2.7", "3.7"])
7388
def system(session):
7489
"""Run the system test suite."""
7590

@@ -100,7 +115,7 @@ def system(session):
100115
)
101116

102117

103-
@nox.session(python=["2.7", "3.6"])
118+
@nox.session(python=["2.7", "3.7"])
104119
def snippets(session):
105120
"""Run the snippets test suite."""
106121

@@ -121,7 +136,7 @@ def snippets(session):
121136
session.run("py.test", "samples", *session.posargs)
122137

123138

124-
@nox.session(python="3.6")
139+
@nox.session(python="3.7")
125140
def cover(session):
126141
"""Run the final coverage report.
127142
@@ -133,7 +148,7 @@ def cover(session):
133148
session.run("coverage", "erase")
134149

135150

136-
@nox.session(python="3.6")
151+
@nox.session(python="3.7")
137152
def lint(session):
138153
"""Run linters.
139154
@@ -152,15 +167,15 @@ def lint(session):
152167
session.run("black", "--check", *BLACK_PATHS)
153168

154169

155-
@nox.session(python="3.6")
170+
@nox.session(python="3.7")
156171
def lint_setup_py(session):
157172
"""Verify that setup.py is valid (including RST check)."""
158173

159174
session.install("docutils", "Pygments")
160175
session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
161176

162177

163-
@nox.session(python="3.6")
178+
@nox.session(python="3.7")
164179
def blacken(session):
165180
"""Run black.
166181
Format code to uniform standard.
@@ -169,7 +184,7 @@ def blacken(session):
169184
session.run("black", *BLACK_PATHS)
170185

171186

172-
@nox.session(python="3.6")
187+
@nox.session(python="3.7")
173188
def docs(session):
174189
"""Build the docs."""
175190

‎bigquery/tests/unit/test__pandas_helpers.py‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@
3131
import pyarrow
3232
import pyarrow.types
3333
except ImportError: # pragma: NO COVER
34-
pyarrow = None
34+
# Mock out pyarrow when missing, because methods from pyarrow.types are
35+
# used in test parameterization.
36+
pyarrow = mock.Mock()
3537
import pytest
3638
import pytz
3739

@@ -85,7 +87,7 @@ def all_(*functions):
8587
return functools.partial(do_all, functions)
8688

8789

88-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
90+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
8991
def test_is_datetime():
9092
assert is_datetime(pyarrow.timestamp("us", tz=None))
9193
assert not is_datetime(pyarrow.timestamp("ms", tz=None))
@@ -249,15 +251,15 @@ def test_all_():
249251
("UNKNOWN_TYPE", "REPEATED", is_none),
250252
],
251253
)
252-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
254+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
253255
def test_bq_to_arrow_data_type(module_under_test, bq_type, bq_mode, is_correct_type):
254256
field = schema.SchemaField("ignored_name", bq_type, mode=bq_mode)
255257
actual = module_under_test.bq_to_arrow_data_type(field)
256258
assert is_correct_type(actual)
257259

258260

259261
@pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"])
260-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
262+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
261263
def test_bq_to_arrow_data_type_w_struct(module_under_test, bq_type):
262264
fields = (
263265
schema.SchemaField("field01", "STRING"),
@@ -301,7 +303,7 @@ def test_bq_to_arrow_data_type_w_struct(module_under_test, bq_type):
301303

302304

303305
@pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"])
304-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
306+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
305307
def test_bq_to_arrow_data_type_w_array_struct(module_under_test, bq_type):
306308
fields = (
307309
schema.SchemaField("field01", "STRING"),
@@ -345,7 +347,7 @@ def test_bq_to_arrow_data_type_w_array_struct(module_under_test, bq_type):
345347
assert actual.value_type.equals(expected_value_type)
346348

347349

348-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
350+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
349351
def test_bq_to_arrow_data_type_w_struct_unknown_subfield(module_under_test):
350352
fields = (
351353
schema.SchemaField("field1", "STRING"),
@@ -442,7 +444,7 @@ def test_bq_to_arrow_data_type_w_struct_unknown_subfield(module_under_test):
442444
],
443445
)
444446
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
445-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
447+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
446448
def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows):
447449
series = pandas.Series(rows, dtype="object")
448450
bq_field = schema.SchemaField("field_name", bq_type)
@@ -452,7 +454,7 @@ def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows):
452454

453455

454456
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
455-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
457+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
456458
def test_bq_to_arrow_array_w_arrays(module_under_test):
457459
rows = [[1, 2, 3], [], [4, 5, 6]]
458460
series = pandas.Series(rows, dtype="object")
@@ -464,7 +466,7 @@ def test_bq_to_arrow_array_w_arrays(module_under_test):
464466

465467
@pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"])
466468
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
467-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
469+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
468470
def test_bq_to_arrow_array_w_structs(module_under_test, bq_type):
469471
rows = [
470472
{"int_col": 123, "string_col": "abc"},
@@ -486,7 +488,7 @@ def test_bq_to_arrow_array_w_structs(module_under_test, bq_type):
486488

487489

488490
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
489-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
491+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
490492
def test_bq_to_arrow_array_w_special_floats(module_under_test):
491493
bq_field = schema.SchemaField("field_name", "FLOAT64")
492494
rows = [float("-inf"), float("nan"), float("inf"), None]
@@ -503,7 +505,7 @@ def test_bq_to_arrow_array_w_special_floats(module_under_test):
503505
assert roundtrip[3] is None
504506

505507

506-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
508+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
507509
def test_bq_to_arrow_schema_w_unknown_type(module_under_test):
508510
fields = (
509511
schema.SchemaField("field1", "STRING"),
@@ -729,7 +731,7 @@ def test_dataframe_to_bq_schema_dict_sequence(module_under_test):
729731

730732

731733
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
732-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
734+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
733735
def test_dataframe_to_arrow_with_multiindex(module_under_test):
734736
bq_schema = (
735737
schema.SchemaField("str_index", "STRING"),
@@ -796,7 +798,7 @@ def test_dataframe_to_arrow_with_multiindex(module_under_test):
796798

797799

798800
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
799-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
801+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
800802
def test_dataframe_to_arrow_with_required_fields(module_under_test):
801803
bq_schema = (
802804
schema.SchemaField("field01", "STRING", mode="REQUIRED"),
@@ -851,7 +853,7 @@ def test_dataframe_to_arrow_with_required_fields(module_under_test):
851853

852854

853855
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
854-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
856+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
855857
def test_dataframe_to_arrow_with_unknown_type(module_under_test):
856858
bq_schema = (
857859
schema.SchemaField("field00", "UNKNOWN_TYPE"),
@@ -884,7 +886,7 @@ def test_dataframe_to_arrow_with_unknown_type(module_under_test):
884886

885887

886888
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
887-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
889+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
888890
def test_dataframe_to_arrow_dict_sequence_schema(module_under_test):
889891
dict_schema = [
890892
{"name": "field01", "type": "STRING", "mode": "REQUIRED"},
@@ -914,7 +916,7 @@ def test_dataframe_to_parquet_without_pyarrow(module_under_test, monkeypatch):
914916

915917

916918
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
917-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
919+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
918920
def test_dataframe_to_parquet_w_extra_fields(module_under_test, monkeypatch):
919921
with pytest.raises(ValueError) as exc_context:
920922
module_under_test.dataframe_to_parquet(
@@ -926,7 +928,7 @@ def test_dataframe_to_parquet_w_extra_fields(module_under_test, monkeypatch):
926928

927929

928930
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
929-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
931+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
930932
def test_dataframe_to_parquet_w_missing_fields(module_under_test, monkeypatch):
931933
with pytest.raises(ValueError) as exc_context:
932934
module_under_test.dataframe_to_parquet(
@@ -938,7 +940,7 @@ def test_dataframe_to_parquet_w_missing_fields(module_under_test, monkeypatch):
938940

939941

940942
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
941-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
943+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
942944
def test_dataframe_to_parquet_compression_method(module_under_test):
943945
bq_schema = (schema.SchemaField("field00", "STRING"),)
944946
dataframe = pandas.DataFrame({"field00": ["foo", "bar"]})
@@ -985,7 +987,7 @@ def test_dataframe_to_bq_schema_fallback_needed_wo_pyarrow(module_under_test):
985987

986988

987989
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
988-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
990+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
989991
def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test):
990992
dataframe = pandas.DataFrame(
991993
data=[
@@ -1015,7 +1017,7 @@ def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test):
10151017

10161018

10171019
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
1018-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1020+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
10191021
def test_dataframe_to_bq_schema_pyarrow_fallback_fails(module_under_test):
10201022
dataframe = pandas.DataFrame(
10211023
data=[
@@ -1040,7 +1042,7 @@ def test_dataframe_to_bq_schema_pyarrow_fallback_fails(module_under_test):
10401042

10411043

10421044
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
1043-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1045+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
10441046
def test_augment_schema_type_detection_succeeds(module_under_test):
10451047
dataframe = pandas.DataFrame(
10461048
data=[
@@ -1101,7 +1103,7 @@ def test_augment_schema_type_detection_succeeds(module_under_test):
11011103

11021104

11031105
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
1104-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1106+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
11051107
def test_augment_schema_type_detection_fails(module_under_test):
11061108
dataframe = pandas.DataFrame(
11071109
data=[
@@ -1137,7 +1139,7 @@ def test_augment_schema_type_detection_fails(module_under_test):
11371139
assert "struct_field" in warning_msg and "struct_field_2" in warning_msg
11381140

11391141

1140-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1142+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
11411143
def test_dataframe_to_parquet_dict_sequence_schema(module_under_test):
11421144
dict_schema = [
11431145
{"name": "field01", "type": "STRING", "mode": "REQUIRED"},
@@ -1166,7 +1168,7 @@ def test_dataframe_to_parquet_dict_sequence_schema(module_under_test):
11661168
assert schema_arg == expected_schema_arg
11671169

11681170

1169-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1171+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
11701172
def test_download_arrow_tabledata_list_unknown_field_type(module_under_test):
11711173
fake_page = api_core.page_iterator.Page(
11721174
parent=mock.Mock(),
@@ -1202,7 +1204,7 @@ def test_download_arrow_tabledata_list_unknown_field_type(module_under_test):
12021204
assert list(col) == [2.2, 22.22, 222.222]
12031205

12041206

1205-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1207+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
12061208
def test_download_arrow_tabledata_list_known_field_type(module_under_test):
12071209
fake_page = api_core.page_iterator.Page(
12081210
parent=mock.Mock(),
@@ -1237,7 +1239,7 @@ def test_download_arrow_tabledata_list_known_field_type(module_under_test):
12371239
assert list(col) == ["2.2", "22.22", "222.222"]
12381240

12391241

1240-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1242+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
12411243
def test_download_arrow_tabledata_list_dict_sequence_schema(module_under_test):
12421244
fake_page = api_core.page_iterator.Page(
12431245
parent=mock.Mock(),
@@ -1265,7 +1267,7 @@ def test_download_arrow_tabledata_list_dict_sequence_schema(module_under_test):
12651267

12661268

12671269
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
1268-
@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
1270+
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
12691271
def test_download_dataframe_tabledata_list_dict_sequence_schema(module_under_test):
12701272
fake_page = api_core.page_iterator.Page(
12711273
parent=mock.Mock(),

‎bigquery/tests/unit/test_client.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@
3131
import pytest
3232
import pytz
3333

34+
try:
35+
import fastparquet
36+
except (ImportError, AttributeError): # pragma: NO COVER
37+
fastparquet = None
3438
try:
3539
import pandas
3640
except (ImportError, AttributeError): # pragma: NO COVER
@@ -6127,6 +6131,7 @@ def test_load_table_from_dataframe_unknown_table(self):
61276131
)
61286132

61296133
@unittest.skipIf(pandas is None, "Requires `pandas`")
6134+
@unittest.skipIf(fastparquet is None, "Requires `fastparquet`")
61306135
def test_load_table_from_dataframe_no_schema_warning_wo_pyarrow(self):
61316136
client = self._make_client()
61326137

@@ -6317,6 +6322,7 @@ def test_load_table_from_dataframe_w_partial_schema_extra_types(self):
63176322
assert "unknown_col" in message
63186323

63196324
@unittest.skipIf(pandas is None, "Requires `pandas`")
6325+
@unittest.skipIf(fastparquet is None, "Requires `fastparquet`")
63206326
def test_load_table_from_dataframe_w_partial_schema_missing_types(self):
63216327
from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES
63226328
from google.cloud.bigquery import job

0 commit comments

Comments
 (0)