diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a54874d4..9a7fdd096 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,15 +45,44 @@ jobs: run: | uv run pre-commit run --all-files --verbose - tests: - name: Python ${{ matrix.python-version}} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} + full-fixture-tests: + name: Preform Full Test Suite (3.14) needs: lint + runs-on: ubuntu-latest + steps: + - name: Checkout Source Files + id: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup Environment + id: setup-environment + uses: ./.github/actions/setup + with: + python-version: 3.14 + uv-install-options: --all-extras + + - name: Run Full PyTest Suite with Code Coverage + id: run-full-pytests-with-code-coverage + shell: bash + run: | + uv run pytest --fixture-set all -n auto --cov kasa --cov-report xml + + - name: Upload Code Coverage to Codecov + id: upload-code-coverage-to-codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + tests: + name: Python ${{ matrix.python-version }} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} + needs: full-fixture-tests runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - python-version: [3.11, 3.12, 3.13, 3.14] os: [ubuntu-latest, macos-latest, windows-latest] extras: [false, true] + python-version: [3.11, 3.12, 3.13, 3.14] exclude: - os: macos-latest extras: true @@ -75,11 +104,7 @@ jobs: id: run-pytests-with-code-coverage shell: bash run: | - if [[ "${{ runner.os }}" == "Windows" ]]; then - uv run pytest -n0 --cov kasa --cov-report xml - else - uv run pytest -n auto --cov kasa --cov-report xml - fi + uv run pytest --fixture-set representative -n auto --cov kasa --cov-report xml - name: Upload Code Coverage to Codecov id: upload-code-coverage-to-codecov diff --git a/pyproject.toml b/pyproject.toml index 2866b9b2e..ae6e65339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,8 @@ testpaths = [ "tests", ] markers = [ + "all_fixtures: always run every parametrized device fixture", + "fixture_representatives(limit): override the representative fixture limit for a test", "requires_dummy: test requires dummy data to pass, skipped on real devices", ] asyncio_mode = "auto" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..36436159b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,60 @@ +# Test fixture tiers + +The test suite has two device-fixture tiers. The tiers keep cross-platform +compatibility coverage broad without multiplying every generic behavior test by +every captured device response. + +## Representative suite + +Run the representative suite with: + +```console +uv run pytest --fixture-set representative +``` + +For each parametrized test function, pytest retains every non-device parameter +combination and selects up to six device fixtures. Selection is deterministic and +favors fixtures that add distinct protocols, models, parent/child roles, +components, component versions, discovery transports, and response shapes. +Small, explicitly parametrized regression fixture sets are retained in full. + +This is the suite used for the Python and operating-system compatibility matrix. + +## Full fixture suite + +Run every collected device-fixture combination with: + +```console +uv run pytest --fixture-set all +``` + +`all` is the local default so an unqualified `uv run pytest` remains exhaustive. +CI runs this tier once on the canonical Ubuntu/Python 3.14 environment with all +optional dependencies installed. + +## Marker overrides + +Use `all_fixtures` when a test is itself an exhaustive fixture contract and must +not be sampled in representative mode: + +```python +@pytest.mark.all_fixtures +async def test_fixture_contract(dev): + ... +``` + +Use `fixture_representatives(limit)` when a broad test needs a different maximum: + +```python +@pytest.mark.fixture_representatives(20) +async def test_behavior_across_more_capabilities(dev): + ... +``` + +Prefer a small explicit `@pytest.mark.parametrize` list for a regression tied to +specific fixture files. Pools at or below the representative limit are never +reduced. + +Do not use representative selection as a substitute for a focused regression +test. When a fixture exists because of a particular bug, pin that fixture in the +regression test so the relationship remains visible in review. diff --git a/tests/conftest.py b/tests/conftest.py index 0d97f16a2..0d109dc3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import os import sys import warnings +from collections import defaultdict from pathlib import Path from unittest.mock import MagicMock, patch @@ -23,6 +24,11 @@ from .device_fixtures import * # noqa: F403 from .discovery_fixtures import * # noqa: F403 +from .fixture_selection import ( + DEFAULT_REPRESENTATIVE_LIMIT, + find_fixture_infos, + select_representative_fixtures, +) from .fixtureinfo import fixture_info # noqa: F401 # Parametrize tests to run with device both on and off @@ -138,9 +144,19 @@ def pytest_addoption(parser): parser.addoption( "--password", action="store", default=None, help="authentication password" ) + parser.addoption( + "--fixture-set", + action="store", + choices=("all", "representative"), + default="all", + help="run all device fixtures or a representative subset per test", + ) def pytest_collection_modifyitems(config, items): + if config.getoption("--fixture-set") == "representative": + _select_representative_fixture_items(config, items) + if not config.getoption("--ip"): print("Testing against fixtures.") # pytest_socket doesn't work properly in windows with asyncio @@ -160,6 +176,71 @@ def pytest_collection_modifyitems(config, items): item.add_marker(pytest.mark.enable_socket) +def _select_representative_fixture_items(config, items): + """Deselect excess fixture variants while preserving parameter coverage.""" + fixture_pools = defaultdict(set) + fixture_items = {} + limits = {} + + for item in items: + if item.get_closest_marker("all_fixtures"): + continue + callspec = getattr(item, "callspec", None) + if callspec is None: + continue + fixture_infos = tuple( + fixture + for value in callspec.params.values() + for fixture in find_fixture_infos(value) + ) + if not fixture_infos: + continue + + test_id = item.nodeid.split("[", maxsplit=1)[0] + fixture_items[item] = (test_id, fixture_infos) + fixture_pools[test_id].update(fixture_infos) + + marker = item.get_closest_marker("fixture_representatives") + limit = ( + marker.args[0] if marker and marker.args else DEFAULT_REPRESENTATIVE_LIMIT + ) + if not isinstance(limit, int) or limit < 1: + raise pytest.UsageError( + "fixture_representatives requires a positive integer limit" + ) + previous_limit = limits.setdefault(test_id, limit) + if previous_limit != limit: + raise pytest.UsageError( + f"conflicting representative fixture limits for {test_id}" + ) + + selected_by_test = { + test_id: set( + select_representative_fixtures( + fixtures, + limit=limits[test_id], + ) + ) + for test_id, fixtures in fixture_pools.items() + } + selected = [] + deselected = [] + for item in items: + fixture_item = fixture_items.get(item) + if fixture_item is None: + selected.append(item) + continue + test_id, fixture_infos = fixture_item + if all(fixture in selected_by_test[test_id] for fixture in fixture_infos): + selected.append(item) + else: + deselected.append(item) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected + + @pytest.fixture(autouse=True, scope="session") def asyncio_sleep_fixture(request): # noqa: PT004 """Patch sleep to prevent tests actually waiting.""" diff --git a/tests/fixture_selection.py b/tests/fixture_selection.py new file mode 100644 index 000000000..891a1ca7d --- /dev/null +++ b/tests/fixture_selection.py @@ -0,0 +1,139 @@ +"""Select deterministic representative device fixtures for broad test matrices.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from .fixtureinfo import FixtureInfo + +DEFAULT_REPRESENTATIVE_LIMIT = 6 + + +def _model_name(fixture: FixtureInfo) -> str: + """Return the model portion of a fixture filename.""" + return fixture.name.split("_", maxsplit=1)[0].split("(", maxsplit=1)[0] + + +def _component_traits(fixture: FixtureInfo) -> set[str]: + """Return component identifiers and versions advertised by a fixture.""" + traits: set[str] = set() + component_nego = fixture.data.get("component_nego", {}) + for component in component_nego.get("component_list", []): + component_id = component.get("id") + version = component.get("ver_code") + if component_id: + traits.add(f"component:{component_id}") + traits.add(f"component-version:{component_id}:{version}") + + app_components = fixture.data.get("getAppComponentList", {}).get( + "app_component", {} + ) + for component in app_components.get("app_component_list", []): + component_id = component.get("name") + version = component.get("version") + if component_id: + traits.add(f"component:{component_id}") + traits.add(f"component-version:{component_id}:{version}") + return traits + + +def _discovery_traits(fixture: FixtureInfo) -> set[str]: + """Return transport and authentication traits from discovery data.""" + traits: set[str] = set() + discovery_result = fixture.data.get("discovery_result", {}).get("result", {}) + encryption = discovery_result.get("mgt_encrypt_schm", {}) + for key in ("encrypt_type", "is_support_https", "lv"): + if key in encryption: + traits.add(f"discovery:{key}:{encryption[key]}") + + for key in ("device_type", "device_model", "result_type"): + if key in discovery_result: + value = discovery_result[key] + if key == "device_model": + value = str(value).split("(", maxsplit=1)[0] + traits.add(f"discovery:{key}:{value}") + return traits + + +def fixture_traits(fixture: FixtureInfo) -> frozenset[str]: + """Return stable behavioral traits used for representative selection.""" + top_level_keys = tuple(sorted(fixture.data)) + traits = { + f"protocol:{fixture.protocol}", + f"model:{_model_name(fixture)}", + f"role:{'child' if '.CHILD' in fixture.protocol else 'parent'}", + f"top-level-shape:{'|'.join(top_level_keys)}", + *(f"top-level-key:{key}" for key in top_level_keys), + } + traits.update(_component_traits(fixture)) + traits.update(_discovery_traits(fixture)) + return frozenset(traits) + + +def select_representative_fixtures( + fixtures: Iterable[FixtureInfo], + *, + limit: int = DEFAULT_REPRESENTATIVE_LIMIT, +) -> tuple[FixtureInfo, ...]: + """Select a deterministic, diverse subset of fixtures. + + Selection happens independently for each parametrized test function. A greedy + set-cover pass favors fixtures that add the most protocol, model, component, + discovery, and response-shape traits. Stable filename/protocol ordering breaks + ties so every xdist worker collects the same tests. + """ + if limit < 1: + raise ValueError("representative fixture limit must be positive") + + fixture_list = sorted(set(fixtures), key=lambda item: (item.name, item.protocol)) + if len(fixture_list) <= limit: + return tuple(fixture_list) + + traits_by_fixture = {fixture: fixture_traits(fixture) for fixture in fixture_list} + uncovered = set().union(*traits_by_fixture.values()) + selected: list[FixtureInfo] = [] + remaining = set(fixture_list) + + # Protocol implementations have different transports, discovery formats, and + # device classes. Reserve one slot for every protocol present in the test's + # fixture pool before the general set-cover pass. + for protocol in sorted({fixture.protocol for fixture in fixture_list}): + if len(selected) >= limit: + break + fixture = min( + (candidate for candidate in remaining if candidate.protocol == protocol), + key=lambda candidate: (-len(traits_by_fixture[candidate]), candidate.name), + ) + selected.append(fixture) + uncovered.difference_update(traits_by_fixture[fixture]) + remaining.remove(fixture) + + while remaining and len(selected) < limit: + fixture = min( + remaining, + key=lambda candidate: ( + -len(traits_by_fixture[candidate] & uncovered), + len(traits_by_fixture[candidate]), + candidate.name, + candidate.protocol, + ), + ) + selected.append(fixture) + uncovered.difference_update(traits_by_fixture[fixture]) + remaining.remove(fixture) + + return tuple(sorted(selected, key=lambda item: (item.name, item.protocol))) + + +def find_fixture_infos(value: Any) -> tuple[FixtureInfo, ...]: + """Recursively find fixture parameters embedded in a pytest callspec value.""" + if isinstance(value, FixtureInfo): + return (value,) + if isinstance(value, dict): + nested = (find_fixture_infos(item) for item in value.values()) + elif isinstance(value, list | tuple | set | frozenset): + nested = (find_fixture_infos(item) for item in value) + else: + return () + return tuple(fixture for group in nested for fixture in group) diff --git a/tests/test_feature.py b/tests/test_feature.py index bb707688e..aea88fdc8 100644 --- a/tests/test_feature.py +++ b/tests/test_feature.py @@ -86,13 +86,13 @@ def test_prop(self): mock_dev_prop.assert_not_called() -def test_feature_value_callable(dev, dummy_feature: Feature): +def test_feature_value_callable(dummy_feature: Feature): """Verify that callables work as *attribute_getter*.""" dummy_feature.attribute_getter = lambda x: "dummy value" assert dummy_feature.value == "dummy value" -async def test_feature_setter(dev, mocker, dummy_feature: Feature): +async def test_feature_setter(mocker, dummy_feature: Feature): """Verify that *set_value* calls the defined method.""" mock_set_dummy = mocker.patch.object( dummy_feature.device, "set_dummy", create=True, new_callable=AsyncMock diff --git a/tests/test_fixture_selection.py b/tests/test_fixture_selection.py new file mode 100644 index 000000000..9173114bb --- /dev/null +++ b/tests/test_fixture_selection.py @@ -0,0 +1,100 @@ +"""Tests for representative device fixture selection.""" + +from __future__ import annotations + +import pytest + +from .fixture_selection import ( + find_fixture_infos, + fixture_traits, + select_representative_fixtures, +) +from .fixtureinfo import FIXTURE_DATA, FixtureInfo + + +def _fixture( + name: str, + protocol: str, + *, + keys: tuple[str, ...] = (), + components: tuple[tuple[str, int], ...] = (), +) -> FixtureInfo: + data: dict[str, object] = {key: {} for key in keys} + if components: + data["component_nego"] = { + "component_list": [ + {"id": component_id, "ver_code": version} + for component_id, version in components + ] + } + return FixtureInfo(name=name, protocol=protocol, data=data) + + +def test_selection_keeps_small_fixture_pools(): + fixtures = ( + _fixture("B.json", "SMART"), + _fixture("A.json", "IOT"), + ) + + assert select_representative_fixtures(fixtures, limit=2) == ( + fixtures[1], + fixtures[0], + ) + + +def test_selection_is_deterministic_and_limited(): + fixtures = tuple( + _fixture( + f"MODEL{index}.json", + "IOT" if index % 2 else "SMART", + keys=(f"method_{index}",), + ) + for index in range(8) + ) + + selected = select_representative_fixtures(fixtures, limit=4) + + assert len(selected) == 4 + assert selected == select_representative_fixtures(reversed(fixtures), limit=4) + assert {fixture.protocol for fixture in selected} == {"IOT", "SMART"} + + +def test_selection_covers_distinct_components(): + fixtures = ( + _fixture("PLUG1.json", "SMART", components=(("device", 1),)), + _fixture("PLUG2.json", "SMART", components=(("device", 2),)), + _fixture("BULB.json", "SMART", components=(("brightness", 1),)), + ) + + selected = select_representative_fixtures(fixtures, limit=2) + selected_traits = set().union(*(fixture_traits(item) for item in selected)) + + assert "component:device" in selected_traits + assert "component:brightness" in selected_traits + + +def test_repository_selection_covers_every_protocol(): + selected = select_representative_fixtures(FIXTURE_DATA) + + assert {fixture.protocol for fixture in selected} == { + fixture.protocol for fixture in FIXTURE_DATA + } + + +def test_selection_rejects_invalid_limit(): + with pytest.raises( + ValueError, + match="representative fixture limit must be positive", + ): + select_representative_fixtures((), limit=0) + + +def test_find_fixture_infos_in_nested_parameters(): + first = _fixture("FIRST.json", "IOT") + second = _fixture("SECOND.json", "SMART") + + assert find_fixture_infos({"first": [first], "nested": ("value", {second})}) == ( + first, + second, + ) + assert find_fixture_infos("not a fixture") == ()