diff --git a/deeplabcut/core/config/base_config.py b/deeplabcut/core/config/base_config.py index 13cbd62591..a0af8199ed 100644 --- a/deeplabcut/core/config/base_config.py +++ b/deeplabcut/core/config/base_config.py @@ -98,13 +98,9 @@ def from_any( f"dictionary, string, or Path. Got {type(config)}" ) - # Note @deruyter92 2026-06-15: the ignore_empty option is currently just used to support - # some top-level fields in v0 legacy configs that are often empty. Should be removed in v1. @classmethod - def from_yaml(cls, yaml_path: str | Path, ignore_empty: bool = True) -> Self: + def from_yaml(cls, yaml_path: str | Path) -> Self: yaml_dict = read_config_as_dict(yaml_path) - if ignore_empty: - yaml_dict = {k: v for k, v in yaml_dict.items() if v is not None} cfg = cls.from_dict(yaml_dict) cfg._post_yaml_load_updates(yaml_path=Path(yaml_path)) return cfg diff --git a/deeplabcut/core/config/project_config.py b/deeplabcut/core/config/project_config.py index 40481e0af9..43824ab73b 100644 --- a/deeplabcut/core/config/project_config.py +++ b/deeplabcut/core/config/project_config.py @@ -10,8 +10,9 @@ # """Project configuration classes for DeepLabCut pose estimation models.""" +import logging from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, get_args from pydantic import Field, model_validator from typing_extensions import Self @@ -27,6 +28,8 @@ validate_crop_bounds, ) +logger = logging.getLogger(__name__) + class ProjectConfig(DLCVersionedConfig): """Complete project configuration. @@ -245,6 +248,20 @@ def normalize_legacy_empty_values(cls, data: Any) -> Any: if data.get(fieldname) == "": data.pop(fieldname) + # Support for legacy config.yaml templates that use bare (null) keys as placeholders + # for unset fields. Only fields whose type doesn't already accept None are affected. + for name, value in list(data.items()): + if value is not None or name not in cls.model_fields: + continue + none_is_invalid = type(None) not in get_args(cls.model_fields[name].annotation) + if none_is_invalid: + logger.warning( + f"Found invalid empty/null/None value for `{name}` in the project " + "config. This is only supported for legacy compatibility. " + f"Treating `{name}` as unset and using the field default instead." + ) + data.pop(name) + # NOTE @deruyter92 2026-06-15: This should be removed in v1. if data.get("multianimalproject") and not data.get("bodyparts"): data["bodyparts"] = "MULTI!" diff --git a/deeplabcut/core/config/utils.py b/deeplabcut/core/config/utils.py index cc0005bb76..5c474643c2 100644 --- a/deeplabcut/core/config/utils.py +++ b/deeplabcut/core/config/utils.py @@ -292,7 +292,13 @@ def create_config_template_3d() -> tuple: return cfg_file_3d, ruamelFile_3d -def read_config(configname: str | Path, ignore_empty: bool = True) -> ProjectConfig: +_IGNORE_EMPTY_UNSET = object() + + +def read_config( + configname: str | Path, + ignore_empty: bool | object = _IGNORE_EMPTY_UNSET, +) -> ProjectConfig: """ Reads structured config file defining a project. @@ -301,17 +307,27 @@ def read_config(configname: str | Path, ignore_empty: bool = True) -> ProjectCon Args: configname: Path to the project configuration file (config.yaml). - ignore_empty: If True, empty/None values in the YAML are ignored and - dataclass defaults are used instead. If False, empty values represent None. - Defaults to True. + ignore_empty: Deprecated, has no effect. Empty/None values for + non-optional config.yaml fields are always treated as unset; optional + fields retain their ``None`` values (see `ProjectConfig` validators). Returns: The project configuration as a ProjectConfig instance (supports dict-like access). """ from deeplabcut.core.config.project_config import ProjectConfig + from deeplabcut.core.deprecation import DLCDeprecationWarning + + if ignore_empty is not _IGNORE_EMPTY_UNSET: + warnings.warn( + "read_config(ignore_empty=...) is deprecated and no longer has any effect: " + "empty/None values for non-optional ProjectConfig fields are always treated " + "as unset; optional fields retain their None values.", + DLCDeprecationWarning, + stacklevel=2, + ) path = Path(configname) - project_config = ProjectConfig.from_yaml(path, ignore_empty=ignore_empty) + project_config = ProjectConfig.from_yaml(path) # If necessary, ProjectConfig automatically updates its project path via _post_yaml_load_updates. # if that is the case (marked as dirty), we write the config back to the file. diff --git a/tests/core/config/test_core_config.py b/tests/core/config/test_core_config.py index a5422afa6d..d01c03b5d2 100644 --- a/tests/core/config/test_core_config.py +++ b/tests/core/config/test_core_config.py @@ -11,6 +11,7 @@ """Tests for deeplabcut.core.config.""" import logging +import warnings from collections.abc import Mapping from pathlib import Path @@ -298,6 +299,25 @@ def test_read_config_breaks_for_invalid_fields(tmp_path): read_config(config_path) +def test_read_config_loads_null_non_optional_fields(tmp_path): + """Bare/null keys in config.yaml are treated as unset (legacy template support).""" + config_path = tmp_path / "config.yaml" + config_path.write_text(f"project_path: {tmp_path}\nengine: pytorch\nmultianimalproject:\nidentity:\n") + cfg = read_config(config_path) + assert cfg["multianimalproject"] is False + assert cfg["identity"] is None + + +@pytest.mark.parametrize("ignore_empty", [True, False]) +def test_read_config_ignore_empty_is_deprecated_noop(tmp_path, ignore_empty): + """Any explicit ignore_empty=... warns and has no effect; null keys use defaults.""" + config_path = tmp_path / "config.yaml" + config_path.write_text(f"project_path: {tmp_path}\nengine: pytorch\nmultianimalproject:\n") + with pytest.warns(DLCDeprecationWarning, match="ignore_empty"): + cfg = read_config(config_path, ignore_empty=ignore_empty) + assert cfg["multianimalproject"] is False + + # ----------------------------------------------------------------------------- # write_project_config # ----------------------------------------------------------------------------- @@ -436,7 +456,6 @@ def test_resolve_alias_returns_name_unchanged_for_unknown_key(): def test_resolve_alias_no_warning_when_warn_false(): - import warnings with warnings.catch_warnings(): warnings.simplefilter("error", DLCDeprecationWarning) diff --git a/tests/core/config/test_project_config.py b/tests/core/config/test_project_config.py index 249266d522..2105a6159b 100644 --- a/tests/core/config/test_project_config.py +++ b/tests/core/config/test_project_config.py @@ -165,6 +165,79 @@ def test_project_path_not_updated_when_already_correct(self, tmp_path): assert not cfg.is_dirty +# ----------------------------------------------------------------------------- +# Legacy empty / null values (normalize_legacy_empty_values) +# ----------------------------------------------------------------------------- + + +class TestNormalizeLegacyEmptyValues: + def test_none_for_non_optional_field_uses_default(self): + """Bare/null YAML values become None in dicts; non-optional fields use defaults.""" + cfg = ProjectConfig.from_dict({"multianimalproject": None, "engine": "pytorch"}) + assert cfg.multianimalproject is False + + def test_none_for_optional_field_kept_as_none(self): + """Optional fields (T | None) accept None; it is not rewritten to a non-None default.""" + cfg = ProjectConfig.from_dict({"identity": None, "engine": "pytorch"}) + assert cfg.identity is None + + def test_empty_string_for_legacy_list_fields_uses_default(self): + cfg = ProjectConfig.from_dict( + { + "skeleton": "", + "TrainingFraction": "", + "video_sets": "", + "bodyparts": "", + "engine": "pytorch", + } + ) + defaults = ProjectConfig() + assert cfg.skeleton == defaults.skeleton + assert cfg.TrainingFraction == defaults.TrainingFraction + assert cfg.video_sets == defaults.video_sets + assert cfg.bodyparts == defaults.bodyparts + + def test_from_dict_and_from_yaml_agree_on_null_keys(self, tmp_path): + """Dict path (legacy scripts) must match YAML path for bare/null keys.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "\n".join( + [ + f"project_path: {tmp_path}", + "engine: pytorch", + "multianimalproject:", + "identity:", + "bodyparts:", + " - snout", + "", + ] + ) + ) + from_yaml = ProjectConfig.from_yaml(config_path) + from_dict = ProjectConfig.from_dict(read_config_as_dict(config_path)) + assert from_dict.multianimalproject == from_yaml.multianimalproject is False + assert from_dict.identity == from_yaml.identity is None + assert from_dict.bodyparts == from_yaml.bodyparts == ["snout"] + + def test_openfield_example_loads_via_dict_and_yaml(self): + """Example project configs with empty fields must load via both entrypoints.""" + config_path = Path(__file__).resolve().parents[3] / "examples" / "openfield-Pranav-2018-10-30" / "config.yaml" + assert config_path.is_file() + raw = read_config_as_dict(config_path) + assert raw["multianimalproject"] is None + assert raw["identity"] is None + + from_dict = ProjectConfig.from_dict(raw) + from_yaml = ProjectConfig.from_yaml(config_path) + assert from_dict.multianimalproject == from_yaml.multianimalproject is False + assert from_dict.identity == from_yaml.identity is None + + def test_null_non_optional_emits_warning(self, caplog): + with caplog.at_level(logging.WARNING, logger="deeplabcut.core.config.project_config"): + ProjectConfig.from_dict({"multianimalproject": None, "engine": "pytorch"}) + assert any("multianimalproject" in r.message for r in caplog.records) + + # ----------------------------------------------------------------------------- # YAML round-trip # -----------------------------------------------------------------------------