Skip to content

Commit e4e6f50

Browse files
PGijsbersPGijsbers
authored
Add HTTP headers to all requests (#1342)
* Add HTTP headers to all requests This allows us to better understand the traffic we see to our API. It is not identifiable to a person. * Update unit test to pass even with user-agent in header
1 parent 532be7b commit e4e6f50

File tree

6 files changed

+56
-83
lines changed

6 files changed

+56
-83
lines changed

‎doc/progress.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ next
1010
~~~~~~
1111

1212
* MAINT #1340: Add Numpy 2.0 support. Update tests to work with scikit-learn <= 1.5.
13+
* ADD #1342: Add HTTP header to requests to indicate they are from openml-python.
1314

1415
0.14.2
1516
~~~~~~

‎openml/_api_calls.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,16 @@
1919
from urllib3 import ProxyManager
2020

2121
from . import config
22+
from .__version__ import __version__
2223
from .exceptions import (
2324
OpenMLHashException,
2425
OpenMLServerError,
2526
OpenMLServerException,
2627
OpenMLServerNoResult,
2728
)
2829

30+
_HEADERS = {"user-agent": f"openml-python/{__version__}"}
31+
2932
DATA_TYPE = Dict[str, Union[str, int]]
3033
FILE_ELEMENTS_TYPE = Dict[str, Union[str, Tuple[str, str]]]
3134
DATABASE_CONNECTION_ERRCODE = 107
@@ -164,6 +167,7 @@ def _download_minio_file(
164167
bucket_name=bucket,
165168
object_name=object_name,
166169
file_path=str(destination),
170+
request_headers=_HEADERS,
167171
)
168172
if destination.is_file() and destination.suffix == ".zip":
169173
with zipfile.ZipFile(destination, "r") as zip_ref:
@@ -350,11 +354,11 @@ def _send_request( # noqa: C901
350354
for retry_counter in range(1, n_retries + 1):
351355
try:
352356
if request_method == "get":
353-
response = session.get(url, params=data)
357+
response = session.get(url, params=data, headers=_HEADERS)
354358
elif request_method == "delete":
355-
response = session.delete(url, params=data)
359+
response = session.delete(url, params=data, headers=_HEADERS)
356360
elif request_method == "post":
357-
response = session.post(url, data=data, files=files)
361+
response = session.post(url, data=data, files=files, headers=_HEADERS)
358362
else:
359363
raise NotImplementedError()
360364

‎tests/test_datasets/test_dataset_functions.py‎

Lines changed: 12 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1768,11 +1768,9 @@ def test_delete_dataset_not_owned(mock_delete, test_files_directory, test_api_ke
17681768
):
17691769
openml.datasets.delete_dataset(40_000)
17701770

1771-
expected_call_args = [
1772-
("https://test.openml.org/api/v1/xml/data/40000",),
1773-
{"params": {"api_key": test_api_key}},
1774-
]
1775-
assert expected_call_args == list(mock_delete.call_args)
1771+
dataset_url = "https://test.openml.org/api/v1/xml/data/40000"
1772+
assert dataset_url == mock_delete.call_args.args[0]
1773+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
17761774

17771775

17781776
@mock.patch.object(requests.Session, "delete")
@@ -1792,11 +1790,9 @@ def test_delete_dataset_with_run(mock_delete, test_files_directory, test_api_key
17921790
):
17931791
openml.datasets.delete_dataset(40_000)
17941792

1795-
expected_call_args = [
1796-
("https://test.openml.org/api/v1/xml/data/40000",),
1797-
{"params": {"api_key": test_api_key}},
1798-
]
1799-
assert expected_call_args == list(mock_delete.call_args)
1793+
dataset_url = "https://test.openml.org/api/v1/xml/data/40000"
1794+
assert dataset_url == mock_delete.call_args.args[0]
1795+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
18001796

18011797

18021798
@mock.patch.object(requests.Session, "delete")
@@ -1813,11 +1809,9 @@ def test_delete_dataset_success(mock_delete, test_files_directory, test_api_key)
18131809
success = openml.datasets.delete_dataset(40000)
18141810
assert success
18151811

1816-
expected_call_args = [
1817-
("https://test.openml.org/api/v1/xml/data/40000",),
1818-
{"params": {"api_key": test_api_key}},
1819-
]
1820-
assert expected_call_args == list(mock_delete.call_args)
1812+
dataset_url = "https://test.openml.org/api/v1/xml/data/40000"
1813+
assert dataset_url == mock_delete.call_args.args[0]
1814+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
18211815

18221816

18231817
@mock.patch.object(requests.Session, "delete")
@@ -1837,11 +1831,9 @@ def test_delete_unknown_dataset(mock_delete, test_files_directory, test_api_key)
18371831
):
18381832
openml.datasets.delete_dataset(9_999_999)
18391833

1840-
expected_call_args = [
1841-
("https://test.openml.org/api/v1/xml/data/9999999",),
1842-
{"params": {"api_key": test_api_key}},
1843-
]
1844-
assert expected_call_args == list(mock_delete.call_args)
1834+
dataset_url = "https://test.openml.org/api/v1/xml/data/9999999"
1835+
assert dataset_url == mock_delete.call_args.args[0]
1836+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
18451837

18461838

18471839
def _assert_datasets_have_id_and_valid_status(datasets: pd.DataFrame):

‎tests/test_flows/test_flow_functions.py‎

Lines changed: 15 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -460,11 +460,9 @@ def test_delete_flow_not_owned(mock_delete, test_files_directory, test_api_key):
460460
):
461461
openml.flows.delete_flow(40_000)
462462

463-
expected_call_args = [
464-
("https://test.openml.org/api/v1/xml/flow/40000",),
465-
{"params": {"api_key": test_api_key}},
466-
]
467-
assert expected_call_args == list(mock_delete.call_args)
463+
flow_url = "https://test.openml.org/api/v1/xml/flow/40000"
464+
assert flow_url == mock_delete.call_args.args[0]
465+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
468466

469467

470468
@mock.patch.object(requests.Session, "delete")
@@ -482,11 +480,9 @@ def test_delete_flow_with_run(mock_delete, test_files_directory, test_api_key):
482480
):
483481
openml.flows.delete_flow(40_000)
484482

485-
expected_call_args = [
486-
("https://test.openml.org/api/v1/xml/flow/40000",),
487-
{"params": {"api_key": test_api_key}},
488-
]
489-
assert expected_call_args == list(mock_delete.call_args)
483+
flow_url = "https://test.openml.org/api/v1/xml/flow/40000"
484+
assert flow_url == mock_delete.call_args.args[0]
485+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
490486

491487

492488
@mock.patch.object(requests.Session, "delete")
@@ -504,11 +500,9 @@ def test_delete_subflow(mock_delete, test_files_directory, test_api_key):
504500
):
505501
openml.flows.delete_flow(40_000)
506502

507-
expected_call_args = [
508-
("https://test.openml.org/api/v1/xml/flow/40000",),
509-
{"params": {"api_key": test_api_key}},
510-
]
511-
assert expected_call_args == list(mock_delete.call_args)
503+
flow_url = "https://test.openml.org/api/v1/xml/flow/40000"
504+
assert flow_url == mock_delete.call_args.args[0]
505+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
512506

513507

514508
@mock.patch.object(requests.Session, "delete")
@@ -523,11 +517,9 @@ def test_delete_flow_success(mock_delete, test_files_directory, test_api_key):
523517
success = openml.flows.delete_flow(33364)
524518
assert success
525519

526-
expected_call_args = [
527-
("https://test.openml.org/api/v1/xml/flow/33364",),
528-
{"params": {"api_key": test_api_key}},
529-
]
530-
assert expected_call_args == list(mock_delete.call_args)
520+
flow_url = "https://test.openml.org/api/v1/xml/flow/33364"
521+
assert flow_url == mock_delete.call_args.args[0]
522+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
531523

532524

533525
@mock.patch.object(requests.Session, "delete")
@@ -545,8 +537,6 @@ def test_delete_unknown_flow(mock_delete, test_files_directory, test_api_key):
545537
):
546538
openml.flows.delete_flow(9_999_999)
547539

548-
expected_call_args = [
549-
("https://test.openml.org/api/v1/xml/flow/9999999",),
550-
{"params": {"api_key": test_api_key}},
551-
]
552-
assert expected_call_args == list(mock_delete.call_args)
540+
flow_url = "https://test.openml.org/api/v1/xml/flow/9999999"
541+
assert flow_url == mock_delete.call_args.args[0]
542+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")

‎tests/test_runs/test_run_functions.py‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1907,11 +1907,9 @@ def test_delete_run_not_owned(mock_delete, test_files_directory, test_api_key):
19071907
):
19081908
openml.runs.delete_run(40_000)
19091909

1910-
expected_call_args = [
1911-
("https://test.openml.org/api/v1/xml/run/40000",),
1912-
{"params": {"api_key": test_api_key}},
1913-
]
1914-
assert expected_call_args == list(mock_delete.call_args)
1910+
run_url = "https://test.openml.org/api/v1/xml/run/40000"
1911+
assert run_url == mock_delete.call_args.args[0]
1912+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
19151913

19161914

19171915
@mock.patch.object(requests.Session, "delete")
@@ -1926,11 +1924,9 @@ def test_delete_run_success(mock_delete, test_files_directory, test_api_key):
19261924
success = openml.runs.delete_run(10591880)
19271925
assert success
19281926

1929-
expected_call_args = [
1930-
("https://test.openml.org/api/v1/xml/run/10591880",),
1931-
{"params": {"api_key": test_api_key}},
1932-
]
1933-
assert expected_call_args == list(mock_delete.call_args)
1927+
run_url = "https://test.openml.org/api/v1/xml/run/10591880"
1928+
assert run_url == mock_delete.call_args.args[0]
1929+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
19341930

19351931

19361932
@mock.patch.object(requests.Session, "delete")
@@ -1948,8 +1944,6 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key):
19481944
):
19491945
openml.runs.delete_run(9_999_999)
19501946

1951-
expected_call_args = [
1952-
("https://test.openml.org/api/v1/xml/run/9999999",),
1953-
{"params": {"api_key": test_api_key}},
1954-
]
1955-
assert expected_call_args == list(mock_delete.call_args)
1947+
run_url = "https://test.openml.org/api/v1/xml/run/9999999"
1948+
assert run_url == mock_delete.call_args.args[0]
1949+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")

‎tests/test_tasks/test_task_functions.py‎

Lines changed: 12 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -249,11 +249,9 @@ def test_delete_task_not_owned(mock_delete, test_files_directory, test_api_key):
249249
):
250250
openml.tasks.delete_task(1)
251251

252-
expected_call_args = [
253-
("https://test.openml.org/api/v1/xml/task/1",),
254-
{"params": {"api_key": test_api_key}},
255-
]
256-
assert expected_call_args == list(mock_delete.call_args)
252+
task_url = "https://test.openml.org/api/v1/xml/task/1"
253+
assert task_url == mock_delete.call_args.args[0]
254+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
257255

258256

259257
@mock.patch.object(requests.Session, "delete")
@@ -271,11 +269,9 @@ def test_delete_task_with_run(mock_delete, test_files_directory, test_api_key):
271269
):
272270
openml.tasks.delete_task(3496)
273271

274-
expected_call_args = [
275-
("https://test.openml.org/api/v1/xml/task/3496",),
276-
{"params": {"api_key": test_api_key}},
277-
]
278-
assert expected_call_args == list(mock_delete.call_args)
272+
task_url = "https://test.openml.org/api/v1/xml/task/3496"
273+
assert task_url == mock_delete.call_args.args[0]
274+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
279275

280276

281277
@mock.patch.object(requests.Session, "delete")
@@ -290,11 +286,9 @@ def test_delete_success(mock_delete, test_files_directory, test_api_key):
290286
success = openml.tasks.delete_task(361323)
291287
assert success
292288

293-
expected_call_args = [
294-
("https://test.openml.org/api/v1/xml/task/361323",),
295-
{"params": {"api_key": test_api_key}},
296-
]
297-
assert expected_call_args == list(mock_delete.call_args)
289+
task_url = "https://test.openml.org/api/v1/xml/task/361323"
290+
assert task_url == mock_delete.call_args.args[0]
291+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
298292

299293

300294
@mock.patch.object(requests.Session, "delete")
@@ -312,8 +306,6 @@ def test_delete_unknown_task(mock_delete, test_files_directory, test_api_key):
312306
):
313307
openml.tasks.delete_task(9_999_999)
314308

315-
expected_call_args = [
316-
("https://test.openml.org/api/v1/xml/task/9999999",),
317-
{"params": {"api_key": test_api_key}},
318-
]
319-
assert expected_call_args == list(mock_delete.call_args)
309+
task_url = "https://test.openml.org/api/v1/xml/task/9999999"
310+
assert task_url == mock_delete.call_args.args[0]
311+
assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")

0 commit comments

Comments
 (0)