Skip to content

Extract the multi-backend test harness from the Vuforia fixtures #3413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adamtheturtle merged 1 commit into from
Aug 11, 2026
Merged

Extract the multi-backend test harness from the Vuforia fixtures #3413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions tests/backend_harness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
"""Run one test suite against several interchangeable backends.

A "backend" is one way of running the system which the tests exercise.
A suite might run against a real remote service, an in-memory fake of
that service, and the same fake behind an HTTP server, and assert the
same things about each. That is how a fake is kept honest.

Nothing in this module knows about Vuforia, and nothing in it may import
from ``mock_vws``. Backends are identified by members of any
:class:`~enum.Enum`: the member name gives the command line option which
deselects it, and the member value gives the ID which ``pytest`` shows
for it.

This module is a candidate for extraction as a ``pytest`` plugin. Keep
it free of anything which is specific to this project so that extracting
it stays a move rather than a rewrite.
"""

import contextlib
from collections.abc import Callable, Generator, Iterable
from enum import Enum

import pytest
from beartype import beartype


@beartype
def _skip_option(*, backend: Enum) -> str:
"""The command line option which deselects a backend.

Args:
backend: The backend to give the option for.

Returns:
The name of the option which deselects the given backend.
"""
return f"--skip-{backend.name.lower()}"


@beartype
def add_skip_options(
*,
parser: pytest.Parser,
backends: Iterable[Enum],
) -> None:
"""Add an option which deselects each backend.

Call this from a ``pytest_addoption`` hook. Tests which use a
deselected backend are skipped rather than deselected, so that a run
which skips a backend still reports the tests which would have used
it.

Args:
parser: The parser to add options to.
backends: The backends to add options for.
"""
for backend in backends:
parser.addoption(
_skip_option(backend=backend),
action="store_true",
default=False,
help=f"Skip tests for {backend.value}",
)


@beartype
def backend_ids(*, backends: Iterable[Enum]) -> list[str]:
"""The IDs which ``pytest`` shows for a set of backends.

Args:
backends: The backends to give IDs for.

Returns:
The ID to show for each given backend.
"""
return [str(object=backend.value) for backend in backends]


@beartype
@contextlib.contextmanager
def running_backend(
*,
backend: Enum,
config: pytest.Config,
setup: Callable[[], Generator[None]],
) -> Generator[None]:
"""Set a backend up for the duration of a test.

Args:
backend: The backend to run the test against.
config: The configuration to look for skip options in.
setup: A generator function which sets the backend up, yields
once while the test runs, and then tears it down. Bind any
arguments it needs with :func:`functools.partial` before
passing it in.

Yields:
``None``, once the backend is set up.
"""
if config.getoption(name=_skip_option(backend=backend)):
pytest.skip()

with contextlib.contextmanager(func=setup)():
yield
151 changes: 91 additions & 60 deletions tests/mock_vws/fixtures/vuforia_backends.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
"""Choose which backends to use for the tests."""

import contextlib
import functools
import logging
from collections.abc import Generator
from collections.abc import Callable, Generator
from enum import Enum

import pytest
Expand All @@ -22,6 +23,11 @@
from mock_vws.database import CloudDatabase, VuMarkDatabase
from mock_vws.states import States
from mock_vws.target import VuMarkTarget
from tests.backend_harness import (
add_skip_options,
backend_ids,
running_backend,
)
from tests.mock_vws.fixtures.credentials import (
InactiveVuMarkCloudDatabase,
VuMarkCloudDatabase,
Expand Down Expand Up @@ -313,20 +319,39 @@ class VuforiaBackend(Enum):
DOCKER_IN_MEMORY = "In Memory version of Docker application"


_ALL_BACKENDS = list(VuforiaBackend)
# The real Vuforia cannot be set up for tests which need to control the
# state of the service.
_MOCK_BACKENDS = [
backend for backend in _ALL_BACKENDS if backend != VuforiaBackend.REAL
]

# These deliberately have no type annotation, so that the keyword
# arguments of the setup functions are still checked where they are
# bound.
_SETUP_FUNCTIONS = {
VuforiaBackend.REAL: _enable_use_real_vuforia,
VuforiaBackend.MOCK: _enable_use_mock_vuforia,
VuforiaBackend.DOCKER_IN_MEMORY: _enable_use_docker_in_memory,
}

_MODEL_TARGET_SETUP_FUNCTIONS = {
VuforiaBackend.REAL: _enable_use_real_model_target_vuforia,
VuforiaBackend.MOCK: _enable_use_mock_model_target_vuforia,
VuforiaBackend.DOCKER_IN_MEMORY: (
_enable_use_docker_in_memory_model_target_vuforia
),
}


@beartype
def pytest_addoption(parser: pytest.Parser) -> None:
"""
Add options to the pytest command line for skipping tests with
particular
backends.
"""
for backend in VuforiaBackend:
parser.addoption(
f"--skip-{backend.name.lower()}",
action="store_true",
default=False,
help=f"Skip tests for {backend.value}",
)
add_skip_options(parser=parser, backends=_ALL_BACKENDS)

parser.addoption(
"--skip-docker_build_tests",
Expand Down Expand Up @@ -355,10 +380,36 @@ def pytest_collection_modifyitems(
item.add_marker(marker=skip_docker_build_tests_marker)


@beartype
def _bind_setup(
*,
backend: VuforiaBackend,
vuforia_database: CloudDatabase,
inactive_cloud_database: CloudDatabase,
vumark_vuforia_database: VuMarkCloudDatabase,
inactive_vumark_database: InactiveVuMarkCloudDatabase,
monkeypatch: pytest.MonkeyPatch,
) -> Callable[[], Generator[None]]:
"""Bind the setup function for a backend to the databases to set
up.

Returns:
A setup function which takes no arguments.
"""
return functools.partial(
_SETUP_FUNCTIONS[backend],
working_database=vuforia_database,
inactive_cloud_database=inactive_cloud_database,
vumark_vuforia_database=vumark_vuforia_database,
inactive_vumark_database=inactive_vumark_database,
monkeypatch=monkeypatch,
)


@pytest.fixture(
name="verify_mock_vuforia",
params=list(VuforiaBackend),
ids=[backend.value for backend in list(VuforiaBackend)],
params=_ALL_BACKENDS,
ids=backend_ids(backends=_ALL_BACKENDS),
)
def fixture_verify_mock_vuforia(
*,
Expand All @@ -379,32 +430,27 @@ def fixture_verify_mock_vuforia(
The backend which the test is running against.
"""
backend: VuforiaBackend = request.param
should_skip = request.config.getoption(
name=f"--skip-{backend.name.lower()}",
)
if should_skip:
pytest.skip()

enable_function = {
VuforiaBackend.REAL: _enable_use_real_vuforia,
VuforiaBackend.MOCK: _enable_use_mock_vuforia,
VuforiaBackend.DOCKER_IN_MEMORY: _enable_use_docker_in_memory,
}[backend]

with contextlib.contextmanager(func=enable_function)(
working_database=vuforia_database,
setup = _bind_setup(
backend=backend,
vuforia_database=vuforia_database,
inactive_cloud_database=inactive_cloud_database,
vumark_vuforia_database=vumark_vuforia_database,
inactive_vumark_database=inactive_vumark_database,
monkeypatch=monkeypatch,
)

with running_backend(
backend=backend,
config=request.config,
setup=setup,
):
yield backend


@pytest.fixture(
name="verify_model_target_mock_vuforia",
params=list(VuforiaBackend),
ids=[backend.value for backend in list(VuforiaBackend)],
params=_ALL_BACKENDS,
ids=backend_ids(backends=_ALL_BACKENDS),
)
def fixture_verify_model_target_mock_vuforia(
*,
Expand All @@ -415,34 +461,22 @@ def fixture_verify_model_target_mock_vuforia(
APIs.
"""
backend: VuforiaBackend = request.param
should_skip = request.config.getoption(
name=f"--skip-{backend.name.lower()}",
setup = functools.partial(
_MODEL_TARGET_SETUP_FUNCTIONS[backend],
monkeypatch=monkeypatch,
)
if should_skip:
pytest.skip()

enable_function = {
VuforiaBackend.REAL: _enable_use_real_model_target_vuforia,
VuforiaBackend.MOCK: _enable_use_mock_model_target_vuforia,
VuforiaBackend.DOCKER_IN_MEMORY: (
_enable_use_docker_in_memory_model_target_vuforia
),
}[backend]

with contextlib.contextmanager(func=enable_function)(
monkeypatch=monkeypatch,
with running_backend(
backend=backend,
config=request.config,
setup=setup,
):
yield backend


@pytest.fixture(
params=[item for item in VuforiaBackend if item != VuforiaBackend.REAL],
ids=[
backend.value
for backend in [
item for item in VuforiaBackend if item != VuforiaBackend.REAL
]
],
params=_MOCK_BACKENDS,
ids=backend_ids(backends=_MOCK_BACKENDS),
)
def mock_only_vuforia(
*,
Expand All @@ -464,21 +498,18 @@ def mock_only_vuforia(
``None``.
"""
backend: VuforiaBackend = request.param
should_skip = request.config.getoption(
name=f"--skip-{backend.name.lower()}",
)
if should_skip:
pytest.skip()

enable_function = {
VuforiaBackend.MOCK: _enable_use_mock_vuforia,
VuforiaBackend.DOCKER_IN_MEMORY: _enable_use_docker_in_memory,
}[backend]

yield from enable_function(
working_database=vuforia_database,
setup = _bind_setup(
backend=backend,
vuforia_database=vuforia_database,
inactive_cloud_database=inactive_cloud_database,
vumark_vuforia_database=vumark_vuforia_database,
inactive_vumark_database=inactive_vumark_database,
monkeypatch=monkeypatch,
)

with running_backend(
backend=backend,
config=request.config,
setup=setup,
):
yield
Loading