Skip to content

Commit 8cc6429

Browse files
satvshrPGijsbersgeetu040
authored
[MNT] Dockerized tests for CI runs using localhost (openml#1629)
#### Metadata * Reference Issue: fixes openml#1614, stacks on openml#1630 * New Tests Added: No * Documentation Updated: No #### Details * What does this PR implement/fix? Explain your changes. This PR implements the setting up of the v1 and v2 test servers in CI using docker via `localhost`. --------- Co-authored-by: PGijsbers <p.gijsbers@tue.nl> Co-authored-by: Armaghan Shakir <raoarmaghanshakir040@gmail.com>
1 parent e95675a commit 8cc6429

File tree

10 files changed

+271
-19
lines changed

10 files changed

+271
-19
lines changed

‎.github/workflows/test.yml‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -101,22 +101,40 @@ jobs:
101101
echo "BEFORE=$git_status" >> $GITHUB_ENV
102102
echo "Repository status before tests: $git_status"
103103
104+
- name: Clone Services
105+
if: matrix.os == 'ubuntu-latest'
106+
run: |
107+
git clone --depth 1 https://github.com/openml/services.git
108+
109+
- name: Start Docker Services
110+
if: matrix.os == 'ubuntu-latest'
111+
working-directory: ./services
112+
run: |
113+
docker compose --profile rest-api --profile minio up -d
114+
115+
echo "Waiting for PHP API to boot..."
116+
timeout 60s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done'
117+
118+
echo "Final Verification: Gateway Connectivity..."
119+
curl -sSfL http://localhost:8000/api/v1/xml/data/1 | head -n 15
120+
104121
- name: Show installed dependencies
105122
run: python -m pip list
106123

107124
- name: Run tests on Ubuntu Test
108125
if: matrix.os == 'ubuntu-latest'
109126
env:
110127
OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
128+
OPENML_USE_LOCAL_SERVICES: "true"
111129
run: |
112130
if [ "${{ matrix.code-cov }}" = "true" ]; then
113131
codecov="--cov=openml --long --cov-report=xml"
114132
fi
115133
116134
if [ "${{ matrix.sklearn-only }}" = "true" ]; then
117-
marks="sklearn and not production_server and not test_server"
135+
marks="sklearn and not production_server"
118136
else
119-
marks="not production_server and not test_server"
137+
marks="not production_server"
120138
fi
121139
122140
pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -125,15 +143,16 @@ jobs:
125143
if: matrix.os == 'ubuntu-latest'
126144
env:
127145
OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
146+
OPENML_USE_LOCAL_SERVICES: "true"
128147
run: |
129148
if [ "${{ matrix.code-cov }}" = "true" ]; then
130149
codecov="--cov=openml --long --cov-report=xml"
131150
fi
132151
133152
if [ "${{ matrix.sklearn-only }}" = "true" ]; then
134-
marks="sklearn and production_server and not test_server"
153+
marks="sklearn and production_server"
135154
else
136-
marks="production_server and not test_server"
155+
marks="production_server"
137156
fi
138157
139158
pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -145,6 +164,20 @@ jobs:
145164
run: | # we need a separate step because of the bash-specific if-statement in the previous one.
146165
pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"
147166
167+
- name: Upload coverage
168+
if: matrix.code-cov && always()
169+
uses: codecov/codecov-action@v4
170+
with:
171+
files: coverage.xml
172+
token: ${{ secrets.CODECOV_TOKEN }}
173+
fail_ci_if_error: true
174+
verbose: true
175+
176+
- name: Cleanup Docker setup
177+
if: matrix.os == 'ubuntu-latest' && always()
178+
run: |
179+
sudo rm -rf services
180+
148181
- name: Check for files left behind by test
149182
if: matrix.os != 'windows-latest' && always()
150183
run: |
@@ -157,15 +190,6 @@ jobs:
157190
exit 1
158191
fi
159192
160-
- name: Upload coverage
161-
if: matrix.code-cov && always()
162-
uses: codecov/codecov-action@v4
163-
with:
164-
files: coverage.xml
165-
token: ${{ secrets.CODECOV_TOKEN }}
166-
fail_ci_if_error: true
167-
verbose: true
168-
169193
dummy_windows_py_sk024:
170194
name: (windows-latest, Py, sk0.24.*, sk-only:false)
171195
runs-on: ubuntu-latest

‎tests/conftest.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,8 @@ def as_robot() -> Iterator[None]:
272272

273273
@pytest.fixture(autouse=True)
274274
def with_server(request):
275+
if os.getenv("OPENML_USE_LOCAL_SERVICES") == "true":
276+
openml.config.TEST_SERVER_URL = "http://localhost:8000"
275277
if "production_server" in request.keywords:
276278
openml.config.server = "https://www.openml.org/api/v1/xml"
277279
openml.config.apikey = None
@@ -306,4 +308,4 @@ def workdir(tmp_path):
306308
original_cwd = Path.cwd()
307309
os.chdir(tmp_path)
308310
yield tmp_path
309-
os.chdir(original_cwd)
311+
os.chdir(original_cwd)

‎tests/test_datasets/test_dataset_functions.py‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,10 @@ def test_deletion_of_cache_dir_faulty_download(self, patch):
530530
datasets_cache_dir = os.path.join(openml.config.get_cache_directory(), "datasets")
531531
assert len(os.listdir(datasets_cache_dir)) == 0
532532

533+
@pytest.mark.skipif(
534+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
535+
reason="Pending resolution of #1657",
536+
)
533537
@pytest.mark.test_server()
534538
def test_publish_dataset(self):
535539
arff_file_path = self.static_cache_dir / "org" / "openml" / "test" / "datasets" / "2" / "dataset.arff"
@@ -566,6 +570,10 @@ def test__retrieve_class_labels(self):
566570
labels = custom_ds.retrieve_class_labels(target_name=custom_ds.features[31].name)
567571
assert labels == ["COIL", "SHEET"]
568572

573+
@pytest.mark.skipif(
574+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
575+
reason="Pending resolution of #1657",
576+
)
569577
@pytest.mark.test_server()
570578
def test_upload_dataset_with_url(self):
571579
dataset = OpenMLDataset(
@@ -689,6 +697,10 @@ def test_attributes_arff_from_df_unknown_dtype(self):
689697
with pytest.raises(ValueError, match=err_msg):
690698
attributes_arff_from_df(df)
691699

700+
@pytest.mark.skipif(
701+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
702+
reason="Pending resolution of #1657",
703+
)
692704
@pytest.mark.test_server()
693705
def test_create_dataset_numpy(self):
694706
data = np.array([[1, 2, 3], [1.2, 2.5, 3.8], [2, 5, 8], [0, 1, 0]]).T
@@ -723,6 +735,10 @@ def test_create_dataset_numpy(self):
723735
), "Uploaded arff does not match original one"
724736
assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset"
725737

738+
@pytest.mark.skipif(
739+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
740+
reason="Pending resolution of #1657",
741+
)
726742
@pytest.mark.test_server()
727743
def test_create_dataset_list(self):
728744
data = [
@@ -778,6 +794,10 @@ def test_create_dataset_list(self):
778794
), "Uploaded ARFF does not match original one"
779795
assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset"
780796

797+
@pytest.mark.skipif(
798+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
799+
reason="Pending resolution of #1657",
800+
)
781801
@pytest.mark.test_server()
782802
def test_create_dataset_sparse(self):
783803
# test the scipy.sparse.coo_matrix
@@ -926,6 +946,10 @@ def test_get_online_dataset_format(self):
926946
dataset_id
927947
), "The format of the ARFF files is different"
928948

949+
@pytest.mark.skipif(
950+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
951+
reason="Pending resolution of #1657",
952+
)
929953
@pytest.mark.test_server()
930954
def test_create_dataset_pandas(self):
931955
data = [
@@ -1151,6 +1175,10 @@ def test_ignore_attributes_dataset(self):
11511175
paper_url=paper_url,
11521176
)
11531177

1178+
@pytest.mark.skipif(
1179+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
1180+
reason="Pending resolution of #1657",
1181+
)
11541182
@pytest.mark.test_server()
11551183
def test_publish_fetch_ignore_attribute(self):
11561184
"""Test to upload and retrieve dataset and check ignore_attributes"""
@@ -1270,6 +1298,10 @@ def test_create_dataset_row_id_attribute_error(self):
12701298
paper_url=paper_url,
12711299
)
12721300

1301+
@pytest.mark.skipif(
1302+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
1303+
reason="Pending resolution of #1657",
1304+
)
12731305
@pytest.mark.test_server()
12741306
def test_create_dataset_row_id_attribute_inference(self):
12751307
# meta-information
@@ -1438,6 +1470,10 @@ def test_data_edit_non_critical_field(self):
14381470
edited_dataset = openml.datasets.get_dataset(did)
14391471
assert edited_dataset.description == desc
14401472

1473+
@pytest.mark.skipif(
1474+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
1475+
reason="Pending resolution of #1657",
1476+
)
14411477
@pytest.mark.test_server()
14421478
def test_data_edit_critical_field(self):
14431479
# Case 2
@@ -1490,6 +1526,10 @@ def test_data_edit_requires_valid_dataset(self):
14901526
description="xor operation dataset",
14911527
)
14921528

1529+
@pytest.mark.skipif(
1530+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
1531+
reason="Pending resolution of #1657",
1532+
)
14931533
@pytest.mark.test_server()
14941534
def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self):
14951535
# Need to own a dataset to be able to edit meta-data
@@ -2008,4 +2048,4 @@ def test_get_dataset_parquet(requests_mock, test_files_directory):
20082048
assert dataset._parquet_url is not None
20092049
assert dataset.parquet_file is not None
20102050
assert os.path.isfile(dataset.parquet_file)
2011-
assert dataset.data_file is None # is alias for arff path
2051+
assert dataset.data_file is None # is alias for arff path

‎tests/test_flows/test_flow.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import copy
66
import hashlib
77
import re
8+
import os
89
import time
910
from packaging.version import Version
1011
from unittest import mock
@@ -33,7 +34,6 @@
3334
from openml.testing import SimpleImputer, TestBase
3435

3536

36-
3737
class TestFlow(TestBase):
3838
_multiprocess_can_split_ = True
3939

@@ -180,6 +180,10 @@ def test_to_xml_from_xml(self):
180180
openml.flows.functions.assert_flows_equal(new_flow, flow)
181181
assert new_flow is not flow
182182

183+
@pytest.mark.skipif(
184+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
185+
reason="Pending resolution of #1657",
186+
)
183187
@pytest.mark.sklearn()
184188
@pytest.mark.test_server()
185189
def test_publish_flow(self):
@@ -222,6 +226,10 @@ def test_publish_existing_flow(self, flow_exists_mock):
222226
f"collected from {__file__.split('/')[-1]}: {flow.flow_id}",
223227
)
224228

229+
@pytest.mark.skipif(
230+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
231+
reason="Pending resolution of #1657",
232+
)
225233
@pytest.mark.sklearn()
226234
@pytest.mark.test_server()
227235
def test_publish_flow_with_similar_components(self):
@@ -273,6 +281,10 @@ def test_publish_flow_with_similar_components(self):
273281
TestBase._mark_entity_for_removal("flow", flow3.flow_id, flow3.name)
274282
TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}")
275283

284+
@pytest.mark.skipif(
285+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
286+
reason="Pending resolution of #1657",
287+
)
276288
@pytest.mark.sklearn()
277289
@pytest.mark.test_server()
278290
def test_semi_legal_flow(self):
@@ -383,6 +395,10 @@ def get_sentinel():
383395
flow_id = openml.flows.flow_exists(name, version)
384396
assert not flow_id
385397

398+
@pytest.mark.skipif(
399+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
400+
reason="Pending resolution of #1657",
401+
)
386402
@pytest.mark.sklearn()
387403
@pytest.mark.test_server()
388404
def test_existing_flow_exists(self):
@@ -424,6 +440,10 @@ def test_existing_flow_exists(self):
424440
)
425441
assert downloaded_flow_id == flow.flow_id
426442

443+
@pytest.mark.skipif(
444+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
445+
reason="Pending resolution of #1657",
446+
)
427447
@pytest.mark.sklearn()
428448
@pytest.mark.test_server()
429449
def test_sklearn_to_upload_to_flow(self):

‎tests/test_flows/test_flow_functions.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from unittest import mock
1313
from unittest.mock import patch
1414

15+
import os
1516
import pandas as pd
1617
import pytest
1718
import requests
@@ -309,6 +310,10 @@ def test_get_flow1(self):
309310
flow = openml.flows.get_flow(1)
310311
assert flow.external_version is None
311312

313+
@pytest.mark.skipif(
314+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
315+
reason="Pending resolution of #1657",
316+
)
312317
@pytest.mark.sklearn()
313318
@pytest.mark.test_server()
314319
def test_get_flow_reinstantiate_model(self):
@@ -392,6 +397,10 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self):
392397
assert flow.flow_id is None
393398
assert "sklearn==0.19.1" not in flow.dependencies
394399

400+
@pytest.mark.skipif(
401+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
402+
reason="Pending resolution of #1657",
403+
)
395404
@pytest.mark.sklearn()
396405
@pytest.mark.test_server()
397406
def test_get_flow_id(self):

‎tests/test_openml/test_api_calls.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import minio
99
import pytest
10+
import os
1011

1112
import openml
1213
from openml.config import ConfigurationForExamples
@@ -20,6 +21,10 @@ def test_too_long_uri(self):
2021
with pytest.raises(openml.exceptions.OpenMLServerError, match="URI too long!"):
2122
openml.datasets.list_datasets(data_id=list(range(10000)))
2223

24+
@pytest.mark.skipif(
25+
os.getenv("OPENML_USE_LOCAL_SERVICES") == "true",
26+
reason="Pending resolution of #1657",
27+
)
2328
@unittest.mock.patch("time.sleep")
2429
@unittest.mock.patch("requests.Session")
2530
@pytest.mark.test_server()

0 commit comments

Comments
 (0)