From 7c2c38e295b344700ca1546c8a024d53e847a901 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 15:27:30 -0400 Subject: [PATCH 01/10] ci: shard Windows pytest jobs --- .github/workflows/ci.yml | 115 ++++++++++++++++++++++++++++++++++++--- tests/conftest.py | 37 +++++++++++++ 2 files changed, 144 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a54874d4..13d65d69c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,45 @@ jobs: run: | uv run pre-commit run --all-files --verbose + windows-shards: + name: Calculate Windows Test Shards + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.calculate_windows_shards.outputs.matrix }} + steps: + - name: Checkout Source Files + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup Environment + uses: ./.github/actions/setup + with: + python-version: 3.14 + uv-install-options: --all-extras + + - name: Calculate Windows Test Shards + id: calculate_windows_shards + shell: bash + run: | + shard_size=3800 + collection=$(uv run pytest --collect-only -q) + test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) + if [[ -z "$test_count" ]]; then + echo "Unable to determine the collected test count" >&2 + exit 1 + fi + shard_count=$(( (test_count + shard_size - 1) / shard_size )) + include='[]' + for python_version in 3.11 3.12 3.13 3.14; do + for ((shard_index = 0; shard_index < shard_count; shard_index++)); do + include=$(jq -c \ + --arg python_version "$python_version" \ + --argjson shard_index "$shard_index" \ + '. + [{"python-version": $python_version, "shard_index": $shard_index}]' \ + <<< "$include") + done + done + echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT" + tests: name: Python ${{ matrix.python-version}} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} needs: lint @@ -52,13 +91,11 @@ jobs: strategy: matrix: python-version: [3.11, 3.12, 3.13, 3.14] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] extras: [false, true] exclude: - os: macos-latest extras: true - - os: windows-latest - extras: true steps: - name: Checkout Source Files id: checkout @@ -75,14 +112,76 @@ jobs: id: run-pytests-with-code-coverage shell: bash run: | - if [[ "${{ runner.os }}" == "Windows" ]]; then - uv run pytest -n0 --cov kasa --cov-report xml - else - uv run pytest -n auto --cov kasa --cov-report xml - fi + uv run pytest -n auto --cov kasa --cov-report xml - name: Upload Code Coverage to Codecov id: upload-code-coverage-to-codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} + + windows-tests: + name: Python ${{ matrix.python-version }} on windows-latest (shard ${{ matrix.shard_index }}) + needs: [lint, windows-shards] + runs-on: windows-latest + strategy: + fail-fast: false + max-parallel: 6 + matrix: ${{ fromJSON(needs.windows-shards.outputs.matrix) }} + steps: + - name: Checkout Source Files + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup Environment + uses: ./.github/actions/setup + with: + python-version: ${{ matrix.python-version }} + + - name: Run PyTest Shard with Code Coverage + shell: bash + run: | + uv run pytest \ + -n auto \ + --shard-index ${{ matrix.shard_index }} \ + --shard-size 3800 \ + --cov kasa \ + --cov-report= + mv .coverage ".coverage.windows-py${{ matrix.python-version }}-shard${{ matrix.shard_index }}" + + - name: Upload Code Coverage Data + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: windows-coverage-py${{ matrix.python-version }}-shard${{ matrix.shard_index }} + path: .coverage.windows-py${{ matrix.python-version }}-shard${{ matrix.shard_index }} + if-no-files-found: error + + windows-coverage: + name: Combine Windows Code Coverage + needs: windows-tests + runs-on: windows-latest + steps: + - name: Checkout Source Files + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup Environment + uses: ./.github/actions/setup + with: + python-version: 3.14 + + - name: Download Code Coverage Data + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: windows-coverage-* + path: coverage-data + merge-multiple: true + + - name: Combine Code Coverage + shell: bash + run: | + uv run coverage combine coverage-data + uv run coverage xml + + - name: Upload Code Coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/tests/conftest.py b/tests/conftest.py index 0d97f16a2..d7734b08c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -138,9 +138,46 @@ def pytest_addoption(parser): parser.addoption( "--password", action="store", default=None, help="authentication password" ) + parser.addoption( + "--shard-index", + action="store", + type=int, + default=None, + help="zero-based pytest shard index", + ) + parser.addoption( + "--shard-size", + action="store", + type=int, + default=None, + help="maximum number of tests in a pytest shard", + ) def pytest_collection_modifyitems(config, items): + shard_index = config.getoption("--shard-index") + shard_size = config.getoption("--shard-size") + if (shard_index is None) != (shard_size is None): + raise pytest.UsageError( + "--shard-index and --shard-size must be provided together" + ) + if shard_index is not None and shard_size is not None: + if shard_size < 1 or shard_index < 0: + raise pytest.UsageError( + "--shard-index must be non-negative and --shard-size positive" + ) + + start = shard_index * shard_size + if start >= len(items): + raise pytest.UsageError( + f"shard {shard_index} starts beyond {len(items)} collected tests" + ) + end = min(start + shard_size, len(items)) + deselected = items[:start] + items[end:] + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = items[start:end] + if not config.getoption("--ip"): print("Testing against fixtures.") # pytest_socket doesn't work properly in windows with asyncio From 87bac0e521dccc45295683eda2aae3fb0199fe10 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 15:36:45 -0400 Subject: [PATCH 02/10] ci: run Windows shards without throttling --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13d65d69c..aa1f8b03e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,8 +73,8 @@ jobs: fi shard_count=$(( (test_count + shard_size - 1) / shard_size )) include='[]' - for python_version in 3.11 3.12 3.13 3.14; do - for ((shard_index = 0; shard_index < shard_count; shard_index++)); do + for ((shard_index = 0; shard_index < shard_count; shard_index++)); do + for python_version in 3.11 3.12 3.13 3.14; do include=$(jq -c \ --arg python_version "$python_version" \ --argjson shard_index "$shard_index" \ @@ -126,7 +126,6 @@ jobs: runs-on: windows-latest strategy: fail-fast: false - max-parallel: 6 matrix: ${{ fromJSON(needs.windows-shards.outputs.matrix) }} steps: - name: Checkout Source Files @@ -154,6 +153,7 @@ jobs: name: windows-coverage-py${{ matrix.python-version }}-shard${{ matrix.shard_index }} path: .coverage.windows-py${{ matrix.python-version }}-shard${{ matrix.shard_index }} if-no-files-found: error + include-hidden-files: true windows-coverage: name: Combine Windows Code Coverage From 264f2ddd3b3b2f509cb9a7b66d28e0fd1c9dc5c6 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 15:54:14 -0400 Subject: [PATCH 03/10] ci: run Windows shards within version jobs --- .github/workflows/ci.yml | 109 ++++++++++----------------------------- 1 file changed, 26 insertions(+), 83 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa1f8b03e..f9a520e79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,45 +45,6 @@ jobs: run: | uv run pre-commit run --all-files --verbose - windows-shards: - name: Calculate Windows Test Shards - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.calculate_windows_shards.outputs.matrix }} - steps: - - name: Checkout Source Files - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Setup Environment - uses: ./.github/actions/setup - with: - python-version: 3.14 - uv-install-options: --all-extras - - - name: Calculate Windows Test Shards - id: calculate_windows_shards - shell: bash - run: | - shard_size=3800 - collection=$(uv run pytest --collect-only -q) - test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) - if [[ -z "$test_count" ]]; then - echo "Unable to determine the collected test count" >&2 - exit 1 - fi - shard_count=$(( (test_count + shard_size - 1) / shard_size )) - include='[]' - for ((shard_index = 0; shard_index < shard_count; shard_index++)); do - for python_version in 3.11 3.12 3.13 3.14; do - include=$(jq -c \ - --arg python_version "$python_version" \ - --argjson shard_index "$shard_index" \ - '. + [{"python-version": $python_version, "shard_index": $shard_index}]' \ - <<< "$include") - done - done - echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT" - tests: name: Python ${{ matrix.python-version}} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} needs: lint @@ -121,12 +82,13 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} windows-tests: - name: Python ${{ matrix.python-version }} on windows-latest (shard ${{ matrix.shard_index }}) - needs: [lint, windows-shards] + name: Python ${{ matrix.python-version }} on windows-latest + needs: lint runs-on: windows-latest strategy: fail-fast: false - matrix: ${{ fromJSON(needs.windows-shards.outputs.matrix) }} + matrix: + python-version: [3.11, 3.12, 3.13, 3.14] steps: - name: Checkout Source Files uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -136,52 +98,33 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Run PyTest Shard with Code Coverage - shell: bash - run: | - uv run pytest \ - -n auto \ - --shard-index ${{ matrix.shard_index }} \ - --shard-size 3800 \ - --cov kasa \ - --cov-report= - mv .coverage ".coverage.windows-py${{ matrix.python-version }}-shard${{ matrix.shard_index }}" - - - name: Upload Code Coverage Data - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: windows-coverage-py${{ matrix.python-version }}-shard${{ matrix.shard_index }} - path: .coverage.windows-py${{ matrix.python-version }}-shard${{ matrix.shard_index }} - if-no-files-found: error - include-hidden-files: true - - windows-coverage: - name: Combine Windows Code Coverage - needs: windows-tests - runs-on: windows-latest - steps: - - name: Checkout Source Files - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Setup Environment - uses: ./.github/actions/setup - with: - python-version: 3.14 - - - name: Download Code Coverage Data - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - pattern: windows-coverage-* - path: coverage-data - merge-multiple: true - - - name: Combine Code Coverage + - name: Run PyTests with Code Coverage shell: bash run: | - uv run coverage combine coverage-data + shard_size=3800 + collection=$(uv run pytest --collect-only -q) + test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) + if [[ -z "$test_count" ]]; then + echo "Unable to determine the collected test count" >&2 + exit 1 + fi + shard_count=$(( (test_count + shard_size - 1) / shard_size )) + uv run coverage erase + for ((shard_index = 0; shard_index < shard_count; shard_index++)); do + echo "::group::Shard $((shard_index + 1)) of $shard_count" + uv run pytest \ + -n auto \ + --shard-index "$shard_index" \ + --shard-size "$shard_size" \ + --cov kasa \ + --cov-append \ + --cov-report= + echo "::endgroup::" + done uv run coverage xml - name: Upload Code Coverage to Codecov + id: upload-code-coverage-to-codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} From ff5dd5ef5c91ea16ef93ac5313638417c4df87fe Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 16:04:46 -0400 Subject: [PATCH 04/10] ci: group test matrix jobs by platform --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9a520e79..2fb0f322d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,9 +51,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.11, 3.12, 3.13, 3.14] os: [ubuntu-latest, macos-latest] extras: [false, true] + python-version: [3.11, 3.12, 3.13, 3.14] exclude: - os: macos-latest extras: true From d58c2f5986935f7270698acb225d5ef23cee2543 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 16:17:02 -0400 Subject: [PATCH 05/10] ci: isolate Windows pytest shard state --- .github/workflows/ci.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fb0f322d..cdf9bf19e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: - name: Run PyTests with Code Coverage shell: bash run: | - shard_size=3800 + shard_size=3500 collection=$(uv run pytest --collect-only -q) test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) if [[ -z "$test_count" ]]; then @@ -111,16 +111,26 @@ jobs: shard_count=$(( (test_count + shard_size - 1) / shard_size )) uv run coverage erase for ((shard_index = 0; shard_index < shard_count; shard_index++)); do + shard_temp="$RUNNER_TEMP/pytest-temp-$shard_index" + shard_cache="$RUNNER_TEMP/pytest-cache-$shard_index" echo "::group::Shard $((shard_index + 1)) of $shard_count" - uv run pytest \ + COVERAGE_FILE=".coverage.shard-$shard_index" uv run pytest \ -n auto \ --shard-index "$shard_index" \ --shard-size "$shard_size" \ + --basetemp "$shard_temp" \ + -o cache_dir="$shard_cache" \ --cov kasa \ - --cov-append \ --cov-report= + for cleanup_path in "$shard_temp" "$shard_cache"; do + case "$cleanup_path" in + "$RUNNER_TEMP"/*) rm -rf -- "$cleanup_path" ;; + *) echo "Refusing to clean path outside RUNNER_TEMP: $cleanup_path" >&2; exit 1 ;; + esac + done echo "::endgroup::" done + uv run coverage combine uv run coverage xml - name: Upload Code Coverage to Codecov From 6c2767915bf88c4179214a2020c806e4ac6fe664 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 16:37:02 -0400 Subject: [PATCH 06/10] ci: reset Windows runners between shard batches --- .github/workflows/ci.yml | 65 ++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdf9bf19e..abb0955b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,53 @@ jobs: run: | uv run pre-commit run --all-files --verbose + windows-shards: + name: Calculate Windows Test Shard Batches + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.calculate_windows_shards.outputs.matrix }} + steps: + - name: Checkout Source Files + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup Environment + uses: ./.github/actions/setup + with: + python-version: 3.14 + uv-install-options: --all-extras + + - name: Calculate Windows Test Shard Batches + id: calculate_windows_shards + shell: bash + run: | + shard_size=3500 + shards_per_job=4 + collection=$(uv run pytest --collect-only -q) + test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) + if [[ -z "$test_count" ]]; then + echo "Unable to determine the collected test count" >&2 + exit 1 + fi + shard_count=$(( (test_count + shard_size - 1) / shard_size )) + include='[]' + for ((shard_start = 0; shard_start < shard_count; shard_start += shards_per_job)); do + shard_end=$(( shard_start + shards_per_job )) + if (( shard_end > shard_count )); then + shard_end=$shard_count + fi + shard_group="$((shard_start + 1))-$shard_end" + for python_version in 3.11 3.12 3.13 3.14; do + include=$(jq -c \ + --arg python_version "$python_version" \ + --argjson shard_start "$shard_start" \ + --argjson shard_end "$shard_end" \ + --arg shard_group "$shard_group" \ + '. + [{"python-version": $python_version, "shard_start": $shard_start, "shard_end": $shard_end, "shard_group": $shard_group}]' \ + <<< "$include") + done + done + echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT" + tests: name: Python ${{ matrix.python-version}} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} needs: lint @@ -82,13 +129,12 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} windows-tests: - name: Python ${{ matrix.python-version }} on windows-latest - needs: lint + name: Python ${{ matrix.python-version }} on windows-latest (shards ${{ matrix.shard_group }}) + needs: [lint, windows-shards] runs-on: windows-latest strategy: fail-fast: false - matrix: - python-version: [3.11, 3.12, 3.13, 3.14] + matrix: ${{ fromJSON(needs.windows-shards.outputs.matrix) }} steps: - name: Checkout Source Files uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -102,18 +148,11 @@ jobs: shell: bash run: | shard_size=3500 - collection=$(uv run pytest --collect-only -q) - test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) - if [[ -z "$test_count" ]]; then - echo "Unable to determine the collected test count" >&2 - exit 1 - fi - shard_count=$(( (test_count + shard_size - 1) / shard_size )) uv run coverage erase - for ((shard_index = 0; shard_index < shard_count; shard_index++)); do + for ((shard_index = ${{ matrix.shard_start }}; shard_index < ${{ matrix.shard_end }}; shard_index++)); do shard_temp="$RUNNER_TEMP/pytest-temp-$shard_index" shard_cache="$RUNNER_TEMP/pytest-cache-$shard_index" - echo "::group::Shard $((shard_index + 1)) of $shard_count" + echo "::group::Shard $((shard_index + 1))" COVERAGE_FILE=".coverage.shard-$shard_index" uv run pytest \ -n auto \ --shard-index "$shard_index" \ From b030e0b857a350dd84265889a39f74909cd3cee3 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 17:14:17 -0400 Subject: [PATCH 07/10] ci: group Windows shard batches by Python --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb0955b5..e8b3ecce3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,13 +74,13 @@ jobs: fi shard_count=$(( (test_count + shard_size - 1) / shard_size )) include='[]' - for ((shard_start = 0; shard_start < shard_count; shard_start += shards_per_job)); do - shard_end=$(( shard_start + shards_per_job )) - if (( shard_end > shard_count )); then - shard_end=$shard_count - fi - shard_group="$((shard_start + 1))-$shard_end" - for python_version in 3.11 3.12 3.13 3.14; do + for python_version in 3.11 3.12 3.13 3.14; do + for ((shard_start = 0; shard_start < shard_count; shard_start += shards_per_job)); do + shard_end=$(( shard_start + shards_per_job )) + if (( shard_end > shard_count )); then + shard_end=$shard_count + fi + shard_group=$(( shard_start / shards_per_job + 1 )) include=$(jq -c \ --arg python_version "$python_version" \ --argjson shard_start "$shard_start" \ @@ -129,7 +129,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} windows-tests: - name: Python ${{ matrix.python-version }} on windows-latest (shards ${{ matrix.shard_group }}) + name: Python ${{ matrix.python-version }} on windows-latest (shard group ${{ matrix.shard_group }}) needs: [lint, windows-shards] runs-on: windows-latest strategy: From b1a19069c3bc09611be37b6f9f844e8ec518b9bc Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 18:33:44 -0400 Subject: [PATCH 08/10] tests: add representative fixture suite --- .github/workflows/ci.yml | 113 ++++++++------------------ pyproject.toml | 2 + tests/README.md | 60 ++++++++++++++ tests/conftest.py | 110 +++++++++++++++++-------- tests/fixture_selection.py | 139 ++++++++++++++++++++++++++++++++ tests/test_feature.py | 4 +- tests/test_fixture_selection.py | 100 +++++++++++++++++++++++ 7 files changed, 414 insertions(+), 114 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/fixture_selection.py create mode 100644 tests/test_fixture_selection.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8b3ecce3..533d40664 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,65 +45,41 @@ jobs: run: | uv run pre-commit run --all-files --verbose - windows-shards: - name: Calculate Windows Test Shard Batches - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.calculate_windows_shards.outputs.matrix }} + tests: + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + needs: lint + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: [3.11, 3.12, 3.13, 3.14] steps: - name: Checkout Source Files + id: checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Environment + id: setup-environment uses: ./.github/actions/setup with: - python-version: 3.14 - uv-install-options: --all-extras + python-version: ${{ matrix.python-version }} - - name: Calculate Windows Test Shard Batches - id: calculate_windows_shards + - name: Run PyTests with Code Coverage + id: run-pytests-with-code-coverage shell: bash run: | - shard_size=3500 - shards_per_job=4 - collection=$(uv run pytest --collect-only -q) - test_count=$(grep -oE '[0-9]+ tests? collected' <<< "$collection" | tail -1 | cut -d' ' -f1) - if [[ -z "$test_count" ]]; then - echo "Unable to determine the collected test count" >&2 - exit 1 - fi - shard_count=$(( (test_count + shard_size - 1) / shard_size )) - include='[]' - for python_version in 3.11 3.12 3.13 3.14; do - for ((shard_start = 0; shard_start < shard_count; shard_start += shards_per_job)); do - shard_end=$(( shard_start + shards_per_job )) - if (( shard_end > shard_count )); then - shard_end=$shard_count - fi - shard_group=$(( shard_start / shards_per_job + 1 )) - include=$(jq -c \ - --arg python_version "$python_version" \ - --argjson shard_start "$shard_start" \ - --argjson shard_end "$shard_end" \ - --arg shard_group "$shard_group" \ - '. + [{"python-version": $python_version, "shard_start": $shard_start, "shard_end": $shard_end, "shard_group": $shard_group}]' \ - <<< "$include") - done - done - echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT" + uv run pytest --fixture-set representative -n auto --cov kasa --cov-report xml - tests: - name: Python ${{ matrix.python-version}} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} + - name: Upload Code Coverage to Codecov + id: upload-code-coverage-to-codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + full-fixture-tests: + name: Full Fixture Test Suite on Python 3.14 (all extras) needs: lint - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - extras: [false, true] - python-version: [3.11, 3.12, 3.13, 3.14] - exclude: - - os: macos-latest - extras: true + runs-on: ubuntu-latest steps: - name: Checkout Source Files id: checkout @@ -113,14 +89,14 @@ jobs: id: setup-environment uses: ./.github/actions/setup with: - python-version: ${{ matrix.python-version }} - uv-install-options: ${{ matrix.extras == true && '--all-extras' || '' }} + python-version: 3.14 + uv-install-options: --all-extras - - name: Run PyTests with Code Coverage - id: run-pytests-with-code-coverage + - name: Run Full PyTest Suite with Code Coverage + id: run-full-pytests-with-code-coverage shell: bash run: | - uv run pytest -n auto --cov kasa --cov-report xml + uv run pytest --fixture-set all -n auto --cov kasa --cov-report xml - name: Upload Code Coverage to Codecov id: upload-code-coverage-to-codecov @@ -129,12 +105,13 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} windows-tests: - name: Python ${{ matrix.python-version }} on windows-latest (shard group ${{ matrix.shard_group }}) - needs: [lint, windows-shards] + name: Python ${{ matrix.python-version }} on windows-latest + needs: lint runs-on: windows-latest strategy: fail-fast: false - matrix: ${{ fromJSON(needs.windows-shards.outputs.matrix) }} + matrix: + python-version: [3.11, 3.12, 3.13, 3.14] steps: - name: Checkout Source Files uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -145,32 +122,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run PyTests with Code Coverage + id: run-pytests-with-code-coverage shell: bash run: | - shard_size=3500 - uv run coverage erase - for ((shard_index = ${{ matrix.shard_start }}; shard_index < ${{ matrix.shard_end }}; shard_index++)); do - shard_temp="$RUNNER_TEMP/pytest-temp-$shard_index" - shard_cache="$RUNNER_TEMP/pytest-cache-$shard_index" - echo "::group::Shard $((shard_index + 1))" - COVERAGE_FILE=".coverage.shard-$shard_index" uv run pytest \ - -n auto \ - --shard-index "$shard_index" \ - --shard-size "$shard_size" \ - --basetemp "$shard_temp" \ - -o cache_dir="$shard_cache" \ - --cov kasa \ - --cov-report= - for cleanup_path in "$shard_temp" "$shard_cache"; do - case "$cleanup_path" in - "$RUNNER_TEMP"/*) rm -rf -- "$cleanup_path" ;; - *) echo "Refusing to clean path outside RUNNER_TEMP: $cleanup_path" >&2; exit 1 ;; - esac - done - echo "::endgroup::" - done - uv run coverage combine - uv run coverage xml + uv run pytest --fixture-set representative -n auto --cov kasa --cov-report xml - name: Upload Code Coverage to Codecov id: upload-code-coverage-to-codecov diff --git a/pyproject.toml b/pyproject.toml index 2866b9b2e..ae6e65339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,8 @@ testpaths = [ "tests", ] markers = [ + "all_fixtures: always run every parametrized device fixture", + "fixture_representatives(limit): override the representative fixture limit for a test", "requires_dummy: test requires dummy data to pass, skipped on real devices", ] asyncio_mode = "auto" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..36436159b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,60 @@ +# Test fixture tiers + +The test suite has two device-fixture tiers. The tiers keep cross-platform +compatibility coverage broad without multiplying every generic behavior test by +every captured device response. + +## Representative suite + +Run the representative suite with: + +```console +uv run pytest --fixture-set representative +``` + +For each parametrized test function, pytest retains every non-device parameter +combination and selects up to six device fixtures. Selection is deterministic and +favors fixtures that add distinct protocols, models, parent/child roles, +components, component versions, discovery transports, and response shapes. +Small, explicitly parametrized regression fixture sets are retained in full. + +This is the suite used for the Python and operating-system compatibility matrix. + +## Full fixture suite + +Run every collected device-fixture combination with: + +```console +uv run pytest --fixture-set all +``` + +`all` is the local default so an unqualified `uv run pytest` remains exhaustive. +CI runs this tier once on the canonical Ubuntu/Python 3.14 environment with all +optional dependencies installed. + +## Marker overrides + +Use `all_fixtures` when a test is itself an exhaustive fixture contract and must +not be sampled in representative mode: + +```python +@pytest.mark.all_fixtures +async def test_fixture_contract(dev): + ... +``` + +Use `fixture_representatives(limit)` when a broad test needs a different maximum: + +```python +@pytest.mark.fixture_representatives(20) +async def test_behavior_across_more_capabilities(dev): + ... +``` + +Prefer a small explicit `@pytest.mark.parametrize` list for a regression tied to +specific fixture files. Pools at or below the representative limit are never +reduced. + +Do not use representative selection as a substitute for a focused regression +test. When a fixture exists because of a particular bug, pin that fixture in the +regression test so the relationship remains visible in review. diff --git a/tests/conftest.py b/tests/conftest.py index d7734b08c..0d109dc3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import os import sys import warnings +from collections import defaultdict from pathlib import Path from unittest.mock import MagicMock, patch @@ -23,6 +24,11 @@ from .device_fixtures import * # noqa: F403 from .discovery_fixtures import * # noqa: F403 +from .fixture_selection import ( + DEFAULT_REPRESENTATIVE_LIMIT, + find_fixture_infos, + select_representative_fixtures, +) from .fixtureinfo import fixture_info # noqa: F401 # Parametrize tests to run with device both on and off @@ -139,44 +145,17 @@ def pytest_addoption(parser): "--password", action="store", default=None, help="authentication password" ) parser.addoption( - "--shard-index", - action="store", - type=int, - default=None, - help="zero-based pytest shard index", - ) - parser.addoption( - "--shard-size", + "--fixture-set", action="store", - type=int, - default=None, - help="maximum number of tests in a pytest shard", + choices=("all", "representative"), + default="all", + help="run all device fixtures or a representative subset per test", ) def pytest_collection_modifyitems(config, items): - shard_index = config.getoption("--shard-index") - shard_size = config.getoption("--shard-size") - if (shard_index is None) != (shard_size is None): - raise pytest.UsageError( - "--shard-index and --shard-size must be provided together" - ) - if shard_index is not None and shard_size is not None: - if shard_size < 1 or shard_index < 0: - raise pytest.UsageError( - "--shard-index must be non-negative and --shard-size positive" - ) - - start = shard_index * shard_size - if start >= len(items): - raise pytest.UsageError( - f"shard {shard_index} starts beyond {len(items)} collected tests" - ) - end = min(start + shard_size, len(items)) - deselected = items[:start] + items[end:] - if deselected: - config.hook.pytest_deselected(items=deselected) - items[:] = items[start:end] + if config.getoption("--fixture-set") == "representative": + _select_representative_fixture_items(config, items) if not config.getoption("--ip"): print("Testing against fixtures.") @@ -197,6 +176,71 @@ def pytest_collection_modifyitems(config, items): item.add_marker(pytest.mark.enable_socket) +def _select_representative_fixture_items(config, items): + """Deselect excess fixture variants while preserving parameter coverage.""" + fixture_pools = defaultdict(set) + fixture_items = {} + limits = {} + + for item in items: + if item.get_closest_marker("all_fixtures"): + continue + callspec = getattr(item, "callspec", None) + if callspec is None: + continue + fixture_infos = tuple( + fixture + for value in callspec.params.values() + for fixture in find_fixture_infos(value) + ) + if not fixture_infos: + continue + + test_id = item.nodeid.split("[", maxsplit=1)[0] + fixture_items[item] = (test_id, fixture_infos) + fixture_pools[test_id].update(fixture_infos) + + marker = item.get_closest_marker("fixture_representatives") + limit = ( + marker.args[0] if marker and marker.args else DEFAULT_REPRESENTATIVE_LIMIT + ) + if not isinstance(limit, int) or limit < 1: + raise pytest.UsageError( + "fixture_representatives requires a positive integer limit" + ) + previous_limit = limits.setdefault(test_id, limit) + if previous_limit != limit: + raise pytest.UsageError( + f"conflicting representative fixture limits for {test_id}" + ) + + selected_by_test = { + test_id: set( + select_representative_fixtures( + fixtures, + limit=limits[test_id], + ) + ) + for test_id, fixtures in fixture_pools.items() + } + selected = [] + deselected = [] + for item in items: + fixture_item = fixture_items.get(item) + if fixture_item is None: + selected.append(item) + continue + test_id, fixture_infos = fixture_item + if all(fixture in selected_by_test[test_id] for fixture in fixture_infos): + selected.append(item) + else: + deselected.append(item) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected + + @pytest.fixture(autouse=True, scope="session") def asyncio_sleep_fixture(request): # noqa: PT004 """Patch sleep to prevent tests actually waiting.""" diff --git a/tests/fixture_selection.py b/tests/fixture_selection.py new file mode 100644 index 000000000..891a1ca7d --- /dev/null +++ b/tests/fixture_selection.py @@ -0,0 +1,139 @@ +"""Select deterministic representative device fixtures for broad test matrices.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from .fixtureinfo import FixtureInfo + +DEFAULT_REPRESENTATIVE_LIMIT = 6 + + +def _model_name(fixture: FixtureInfo) -> str: + """Return the model portion of a fixture filename.""" + return fixture.name.split("_", maxsplit=1)[0].split("(", maxsplit=1)[0] + + +def _component_traits(fixture: FixtureInfo) -> set[str]: + """Return component identifiers and versions advertised by a fixture.""" + traits: set[str] = set() + component_nego = fixture.data.get("component_nego", {}) + for component in component_nego.get("component_list", []): + component_id = component.get("id") + version = component.get("ver_code") + if component_id: + traits.add(f"component:{component_id}") + traits.add(f"component-version:{component_id}:{version}") + + app_components = fixture.data.get("getAppComponentList", {}).get( + "app_component", {} + ) + for component in app_components.get("app_component_list", []): + component_id = component.get("name") + version = component.get("version") + if component_id: + traits.add(f"component:{component_id}") + traits.add(f"component-version:{component_id}:{version}") + return traits + + +def _discovery_traits(fixture: FixtureInfo) -> set[str]: + """Return transport and authentication traits from discovery data.""" + traits: set[str] = set() + discovery_result = fixture.data.get("discovery_result", {}).get("result", {}) + encryption = discovery_result.get("mgt_encrypt_schm", {}) + for key in ("encrypt_type", "is_support_https", "lv"): + if key in encryption: + traits.add(f"discovery:{key}:{encryption[key]}") + + for key in ("device_type", "device_model", "result_type"): + if key in discovery_result: + value = discovery_result[key] + if key == "device_model": + value = str(value).split("(", maxsplit=1)[0] + traits.add(f"discovery:{key}:{value}") + return traits + + +def fixture_traits(fixture: FixtureInfo) -> frozenset[str]: + """Return stable behavioral traits used for representative selection.""" + top_level_keys = tuple(sorted(fixture.data)) + traits = { + f"protocol:{fixture.protocol}", + f"model:{_model_name(fixture)}", + f"role:{'child' if '.CHILD' in fixture.protocol else 'parent'}", + f"top-level-shape:{'|'.join(top_level_keys)}", + *(f"top-level-key:{key}" for key in top_level_keys), + } + traits.update(_component_traits(fixture)) + traits.update(_discovery_traits(fixture)) + return frozenset(traits) + + +def select_representative_fixtures( + fixtures: Iterable[FixtureInfo], + *, + limit: int = DEFAULT_REPRESENTATIVE_LIMIT, +) -> tuple[FixtureInfo, ...]: + """Select a deterministic, diverse subset of fixtures. + + Selection happens independently for each parametrized test function. A greedy + set-cover pass favors fixtures that add the most protocol, model, component, + discovery, and response-shape traits. Stable filename/protocol ordering breaks + ties so every xdist worker collects the same tests. + """ + if limit < 1: + raise ValueError("representative fixture limit must be positive") + + fixture_list = sorted(set(fixtures), key=lambda item: (item.name, item.protocol)) + if len(fixture_list) <= limit: + return tuple(fixture_list) + + traits_by_fixture = {fixture: fixture_traits(fixture) for fixture in fixture_list} + uncovered = set().union(*traits_by_fixture.values()) + selected: list[FixtureInfo] = [] + remaining = set(fixture_list) + + # Protocol implementations have different transports, discovery formats, and + # device classes. Reserve one slot for every protocol present in the test's + # fixture pool before the general set-cover pass. + for protocol in sorted({fixture.protocol for fixture in fixture_list}): + if len(selected) >= limit: + break + fixture = min( + (candidate for candidate in remaining if candidate.protocol == protocol), + key=lambda candidate: (-len(traits_by_fixture[candidate]), candidate.name), + ) + selected.append(fixture) + uncovered.difference_update(traits_by_fixture[fixture]) + remaining.remove(fixture) + + while remaining and len(selected) < limit: + fixture = min( + remaining, + key=lambda candidate: ( + -len(traits_by_fixture[candidate] & uncovered), + len(traits_by_fixture[candidate]), + candidate.name, + candidate.protocol, + ), + ) + selected.append(fixture) + uncovered.difference_update(traits_by_fixture[fixture]) + remaining.remove(fixture) + + return tuple(sorted(selected, key=lambda item: (item.name, item.protocol))) + + +def find_fixture_infos(value: Any) -> tuple[FixtureInfo, ...]: + """Recursively find fixture parameters embedded in a pytest callspec value.""" + if isinstance(value, FixtureInfo): + return (value,) + if isinstance(value, dict): + nested = (find_fixture_infos(item) for item in value.values()) + elif isinstance(value, list | tuple | set | frozenset): + nested = (find_fixture_infos(item) for item in value) + else: + return () + return tuple(fixture for group in nested for fixture in group) diff --git a/tests/test_feature.py b/tests/test_feature.py index bb707688e..aea88fdc8 100644 --- a/tests/test_feature.py +++ b/tests/test_feature.py @@ -86,13 +86,13 @@ def test_prop(self): mock_dev_prop.assert_not_called() -def test_feature_value_callable(dev, dummy_feature: Feature): +def test_feature_value_callable(dummy_feature: Feature): """Verify that callables work as *attribute_getter*.""" dummy_feature.attribute_getter = lambda x: "dummy value" assert dummy_feature.value == "dummy value" -async def test_feature_setter(dev, mocker, dummy_feature: Feature): +async def test_feature_setter(mocker, dummy_feature: Feature): """Verify that *set_value* calls the defined method.""" mock_set_dummy = mocker.patch.object( dummy_feature.device, "set_dummy", create=True, new_callable=AsyncMock diff --git a/tests/test_fixture_selection.py b/tests/test_fixture_selection.py new file mode 100644 index 000000000..6ea458b02 --- /dev/null +++ b/tests/test_fixture_selection.py @@ -0,0 +1,100 @@ +"""Tests for representative device fixture selection.""" + +from __future__ import annotations + +import pytest + +from .fixture_selection import ( + find_fixture_infos, + fixture_traits, + select_representative_fixtures, +) +from .fixtureinfo import FIXTURE_DATA, FixtureInfo + + +def _fixture( + name: str, + protocol: str, + *, + keys: tuple[str, ...] = (), + components: tuple[tuple[str, int], ...] = (), +) -> FixtureInfo: + data = {key: {} for key in keys} + if components: + data["component_nego"] = { + "component_list": [ + {"id": component_id, "ver_code": version} + for component_id, version in components + ] + } + return FixtureInfo(name=name, protocol=protocol, data=data) + + +def test_selection_keeps_small_fixture_pools(): + fixtures = ( + _fixture("B.json", "SMART"), + _fixture("A.json", "IOT"), + ) + + assert select_representative_fixtures(fixtures, limit=2) == ( + fixtures[1], + fixtures[0], + ) + + +def test_selection_is_deterministic_and_limited(): + fixtures = tuple( + _fixture( + f"MODEL{index}.json", + "IOT" if index % 2 else "SMART", + keys=(f"method_{index}",), + ) + for index in range(8) + ) + + selected = select_representative_fixtures(fixtures, limit=4) + + assert len(selected) == 4 + assert selected == select_representative_fixtures(reversed(fixtures), limit=4) + assert {fixture.protocol for fixture in selected} == {"IOT", "SMART"} + + +def test_selection_covers_distinct_components(): + fixtures = ( + _fixture("PLUG1.json", "SMART", components=(("device", 1),)), + _fixture("PLUG2.json", "SMART", components=(("device", 2),)), + _fixture("BULB.json", "SMART", components=(("brightness", 1),)), + ) + + selected = select_representative_fixtures(fixtures, limit=2) + selected_traits = set().union(*(fixture_traits(item) for item in selected)) + + assert "component:device" in selected_traits + assert "component:brightness" in selected_traits + + +def test_repository_selection_covers_every_protocol(): + selected = select_representative_fixtures(FIXTURE_DATA) + + assert {fixture.protocol for fixture in selected} == { + fixture.protocol for fixture in FIXTURE_DATA + } + + +def test_selection_rejects_invalid_limit(): + with pytest.raises( + ValueError, + match="representative fixture limit must be positive", + ): + select_representative_fixtures((), limit=0) + + +def test_find_fixture_infos_in_nested_parameters(): + first = _fixture("FIRST.json", "IOT") + second = _fixture("SECOND.json", "SMART") + + assert find_fixture_infos({"first": [first], "nested": ("value", {second})}) == ( + first, + second, + ) + assert find_fixture_infos("not a fixture") == () From 905749595063c609c6f570fe0b1f4e807b3254d6 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 18:35:42 -0400 Subject: [PATCH 09/10] tests: annotate fixture selection test data --- tests/test_fixture_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fixture_selection.py b/tests/test_fixture_selection.py index 6ea458b02..9173114bb 100644 --- a/tests/test_fixture_selection.py +++ b/tests/test_fixture_selection.py @@ -19,7 +19,7 @@ def _fixture( keys: tuple[str, ...] = (), components: tuple[tuple[str, int], ...] = (), ) -> FixtureInfo: - data = {key: {} for key in keys} + data: dict[str, object] = {key: {} for key in keys} if components: data["component_nego"] = { "component_list": [ From 22efa4929e8ec16696fb4aff4c8a188021805770 Mon Sep 17 00:00:00 2001 From: ZeliardM Date: Fri, 10 Jul 2026 19:54:06 -0400 Subject: [PATCH 10/10] ci: gate representative matrix on full suite --- .github/workflows/ci.yml | 51 ++++++++++++---------------------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 533d40664..9a7fdd096 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,39 +45,8 @@ jobs: run: | uv run pre-commit run --all-files --verbose - tests: - name: Python ${{ matrix.python-version }} on ${{ matrix.os }} - needs: lint - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - python-version: [3.11, 3.12, 3.13, 3.14] - steps: - - name: Checkout Source Files - id: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Setup Environment - id: setup-environment - uses: ./.github/actions/setup - with: - python-version: ${{ matrix.python-version }} - - - name: Run PyTests with Code Coverage - id: run-pytests-with-code-coverage - shell: bash - run: | - uv run pytest --fixture-set representative -n auto --cov kasa --cov-report xml - - - name: Upload Code Coverage to Codecov - id: upload-code-coverage-to-codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - full-fixture-tests: - name: Full Fixture Test Suite on Python 3.14 (all extras) + name: Preform Full Test Suite (3.14) needs: lint runs-on: ubuntu-latest steps: @@ -104,22 +73,32 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - windows-tests: - name: Python ${{ matrix.python-version }} on windows-latest - needs: lint - runs-on: windows-latest + tests: + name: Python ${{ matrix.python-version }} on ${{ matrix.os }}${{ fromJSON('[" (extras)", ""]')[matrix.extras == ''] }} + needs: full-fixture-tests + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + extras: [false, true] python-version: [3.11, 3.12, 3.13, 3.14] + exclude: + - os: macos-latest + extras: true + - os: windows-latest + extras: true steps: - name: Checkout Source Files + id: checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Environment + id: setup-environment uses: ./.github/actions/setup with: python-version: ${{ matrix.python-version }} + uv-install-options: ${{ matrix.extras == true && '--all-extras' || '' }} - name: Run PyTests with Code Coverage id: run-pytests-with-code-coverage