diff --git a/_toc.yml b/_toc.yml index e24ca7753..6bb51801f 100644 --- a/_toc.yml +++ b/_toc.yml @@ -1,10 +1,12 @@ format: jb-book root: README + parts: - caption: Getting Started chapters: - file: docs/UseOverviewGuide - file: docs/course + - caption: Installation chapters: - file: docs/installation @@ -31,40 +33,41 @@ parts: - file: docs/pytorch/user_guide.md - file: docs/pytorch/pytorch_config.md - file: docs/pytorch/architectures.md + - caption: Quick Start Tutorials chapters: - file: docs/quick-start/single_animal_quick_guide - file: docs/quick-start/tutorial_maDLC -- caption: 🚀 Beginner's Guide to DeepLabCut + +- caption: "🚀 Beginner's Guide to DeepLabCut" chapters: - file: docs/beginner-guides/beginners-guide - file: docs/beginner-guides/manage-project - file: docs/beginner-guides/labeling - file: docs/beginner-guides/Training-Evaluation - file: docs/beginner-guides/video-analysis -- caption: 🚀 Main Demo Notebooks + +- caption: "🚀 Main Demo Notebooks" chapters: - file: examples/COLAB/COLAB_DEMO_SuperAnimal - file: examples/COLAB/COLAB_DEMO_mouse_openfield - file: examples/COLAB/COLAB_3miceDemo - file: examples/COLAB/COLAB_HumanPose_with_RTMPose - - -- caption: 🚀 Notebooks For Your Data +- caption: "🚀 Notebooks For Your Data" chapters: - file: examples/COLAB/COLAB_YOURDATA_SuperAnimal - file: examples/COLAB/COLAB_YOURDATA_TrainNetwork_VideoAnalysis - file: examples/COLAB/COLAB_YOURDATA_maDLC_TrainNetwork_VideoAnalysis -- caption: 🚀 Special Feature Demos +- caption: "🚀 Special Feature Demos" chapters: - file: examples/COLAB/COLAB_transformer_reID - file: examples/COLAB/COLAB_BUCTD_and_CTD_tracking - file: examples/JUPYTER/Demo_3D_DeepLabCut - file: examples/COLAB/COLAB_DLC_ModelZoo -- caption: 🧑‍🍳 Cookbook (detailed helper guides) +- caption: "🧑‍🍳 Cookbook (detailed helper guides)" chapters: - file: docs/convert_maDLC - file: docs/recipes/OtherData @@ -82,6 +85,7 @@ parts: - caption: Hardware Tips chapters: - file: docs/recipes/TechHardware + - caption: DeepLabCut-Live! chapters: - file: docs/dlc-live/deeplabcutlive @@ -110,7 +114,7 @@ parts: - file: docs/benchmark - file: docs/pytorch/Benchmarking_shuffle_guide -- caption: Mission & Contribute +- caption: "Mission & Contribute" chapters: - file: docs/MISSION_AND_VALUES - file: docs/roadmap diff --git a/deeplabcut/core/config/base_config.py b/deeplabcut/core/config/base_config.py index ec89235c3..495511ccc 100644 --- a/deeplabcut/core/config/base_config.py +++ b/deeplabcut/core/config/base_config.py @@ -49,7 +49,7 @@ class DLCBaseConfig(BaseModel): - Field aliases from `json_schema_extra`. """ - model_config = ConfigDict(extra="forbid", validate_assignment=True) + model_config = ConfigDict(extra="forbid", validate_assignment=True, arbitrary_types_allowed=True) # ------------------------------------------------------------------ # Validation (before pydantic field validation) @@ -392,10 +392,16 @@ def record_change_note( include_caller: bool = False, _stack_depth: int = 1, ) -> None: + field_name = self._resolve_alias(field_name) + + if field_name not in type(self).model_fields: + raise KeyError(f"'{type(self).__name__}' has no field '{field_name}'") + if include_caller: frame = sys._getframe(_stack_depth) filename = frame.f_code.co_filename.rsplit("/", 1)[-1] message = f"{message} [{filename}:{frame.f_lineno}]" + self._change_notes[field_name] = message def log_changes(self) -> None: diff --git a/deeplabcut/core/weight_init.py b/deeplabcut/core/weight_init.py index 4f1689b25..19551b9fe 100644 --- a/deeplabcut/core/weight_init.py +++ b/deeplabcut/core/weight_init.py @@ -104,8 +104,8 @@ def from_dict(data: dict) -> WeightInitialization: if "snapshot_path" not in data: return WeightInitialization.from_dict_legacy(data) - snapshot_path = data['snapshot_path'] - if data['snapshot_path'] is not None: + snapshot_path = data["snapshot_path"] + if data["snapshot_path"] is not None: snapshot_path = Path(snapshot_path) detector_snapshot_path = data.get("detector_snapshot_path") diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py index ba4c4ad94..d6ac04d4f 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py @@ -612,7 +612,6 @@ def _generic2coco( os.symlink(src, dest) except Exception as err: print(f"Could not create a symlink from {src} to {dest}: {err}") - pass image["file_name"] = file_name lookuptable[dest] = src diff --git a/deeplabcut/pose_estimation_3d/camera_calibration.py b/deeplabcut/pose_estimation_3d/camera_calibration.py index 75e6752a6..10ee847ce 100644 --- a/deeplabcut/pose_estimation_3d/camera_calibration.py +++ b/deeplabcut/pose_estimation_3d/camera_calibration.py @@ -435,7 +435,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): f2.suptitle("Undistorted corner points on camera-1 and camera-2", fontsize=25) ax1.imshow(cv2.cvtColor(im_remapped1, cv2.COLOR_BGR2RGB)) ax2.imshow(cv2.cvtColor(im_remapped2, cv2.COLOR_BGR2RGB)) - for i in range(0, cam1_undistort.shape[1]): + for i in range(cam1_undistort.shape[1]): ax1.scatter( [cam1_undistort[-1][i, 0, 0]], [cam1_undistort[-1][i, 0, 1]], diff --git a/deeplabcut/pose_estimation_pytorch/config/make_pose_config.py b/deeplabcut/pose_estimation_pytorch/config/make_pose_config.py index 4ac439b25..a60df2d63 100644 --- a/deeplabcut/pose_estimation_pytorch/config/make_pose_config.py +++ b/deeplabcut/pose_estimation_pytorch/config/make_pose_config.py @@ -17,6 +17,7 @@ from deeplabcut.core.config import read_config_as_dict, write_config from deeplabcut.core.weight_init import WeightInitialization +from deeplabcut.pose_estimation_pytorch.config.pose import PoseConfig from deeplabcut.pose_estimation_pytorch.config.utils import ( get_config_folder_path, load_backbones, diff --git a/deeplabcut/pose_estimation_pytorch/data/base.py b/deeplabcut/pose_estimation_pytorch/data/base.py index 51cf65b58..8b6527330 100644 --- a/deeplabcut/pose_estimation_pytorch/data/base.py +++ b/deeplabcut/pose_estimation_pytorch/data/base.py @@ -10,16 +10,16 @@ # from __future__ import annotations +import warnings from abc import ABC, abstractmethod from pathlib import Path -import warnings import albumentations as A import numpy as np -from deeplabcut.pose_estimation_pytorch.config.pose import PoseConfig -import deeplabcut.core.config as config_utils import deeplabcut.pose_estimation_pytorch.config as config +from deeplabcut.core.types import DEPRECATED_ARGUMENT +from deeplabcut.pose_estimation_pytorch.config.pose import PoseConfig from deeplabcut.pose_estimation_pytorch.data.dataset import ( PoseDataset, PoseDatasetParameters, @@ -34,7 +34,6 @@ map_id_to_annotations, ) from deeplabcut.pose_estimation_pytorch.task import Task -from deeplabcut.core.types import DEPRECATED_ARGUMENT class Loader(ABC): @@ -66,18 +65,20 @@ def __init__( Args: project_root: The root directory of the project. image_root: The root directory of the images. - model_config (Path | str | PoseConfig | dict): + model_config (Path | str | PoseConfig | dict): The pose model configuration. Can be a path to a YAML file, a PoseConfig object, or a dictionary. (model_config_path: The path to the pose model configuration. Deprecated, use `model_config` instead.) """ + def _resolve_legacy_args(model_config, model_config_path): """Support for legacy argument `model_config_path`. returns new model_config arg""" if model_config_path: warnings.warn( "argument `model_config_path` in Loader.__init__ is deprecated, use `model_config` instead", DeprecationWarning, + stacklevel=2, ) - if model_config is not None: + if model_config is not None: raise ValueError( "`model_config_path` and `model_config` arguments cannot be provided together! " "Please provide only `model_config`." @@ -86,21 +87,24 @@ def _resolve_legacy_args(model_config, model_config_path): model_config_path = Path(model_config_path) / "pytorch_config.yaml" model_config = model_config_path elif model_config is None: - raise ValueError( - "`model_config` argument must be provided." - ) + raise ValueError("`model_config` argument must be provided.") return model_config - model_config = _resolve_legacy_args(model_config, model_config_path) + + model_config = _resolve_legacy_args(model_config, model_config_path) self.model_cfg: PoseConfig = PoseConfig.from_any(model_config) def _infer_model_config_path(model_config: PoseConfig | dict | Path | str) -> Path: - """Resolve the pose config path. Either the input is a path, or it is specified in `metadata.pose_config_path` field.""" + """ + Resolve the pose config path. + Either the input is a path, or it is specified in `metadata.pose_config_path` field. + """ provided_path = Path(model_config) if isinstance(model_config, (Path, str)) else None specified_path = self.model_cfg.select("metadata.pose_config_path") model_config_path = provided_path or specified_path if model_config_path is None: raise ValueError("`model_config` must contain a `metadata.pose_config_path` field.") return Path(model_config_path) + self.model_config_path = _infer_model_config_path(model_config) self.project_root = Path(project_root) diff --git a/deeplabcut/pose_estimation_pytorch/data/dlcloader.py b/deeplabcut/pose_estimation_pytorch/data/dlcloader.py index 8aec269f1..2391be7b9 100644 --- a/deeplabcut/pose_estimation_pytorch/data/dlcloader.py +++ b/deeplabcut/pose_estimation_pytorch/data/dlcloader.py @@ -20,8 +20,9 @@ import numpy as np import pandas as pd import scipy.io as sio -from deeplabcut.core.config.project_config import ProjectConfig + import deeplabcut.utils.auxiliaryfunctions as af +from deeplabcut.core.config.project_config import ProjectConfig from deeplabcut.core.engine import Engine from deeplabcut.generate_training_dataset.trainingsetmanipulation import drop_likelihood_columns from deeplabcut.pose_estimation_pytorch.data.base import Loader @@ -51,9 +52,7 @@ def __init__( self._project_config: ProjectConfig = ProjectConfig.from_any(config) self._project_root = provided_root_dir or self._project_config.project_path if self._project_root is None: - raise ValueError( - "`config` must contain a `project_path` field." - ) + raise ValueError("`config` must contain a `project_path` field.") self._shuffle = shuffle self._trainset_index = trainset_index diff --git a/deeplabcut/pose_estimation_pytorch/data/postprocessor.py b/deeplabcut/pose_estimation_pytorch/data/postprocessor.py index e408b9b49..211d1c751 100644 --- a/deeplabcut/pose_estimation_pytorch/data/postprocessor.py +++ b/deeplabcut/pose_estimation_pytorch/data/postprocessor.py @@ -39,7 +39,6 @@ def __call__(self, predictions: Any, context: Context) -> Any: Returns: a single post-processed prediction """ - pass def build_bottom_up_postprocessor( diff --git a/deeplabcut/pose_estimation_pytorch/data/preprocessor.py b/deeplabcut/pose_estimation_pytorch/data/preprocessor.py index 762cd632e..024f0260d 100644 --- a/deeplabcut/pose_estimation_pytorch/data/preprocessor.py +++ b/deeplabcut/pose_estimation_pytorch/data/preprocessor.py @@ -51,7 +51,6 @@ def __call__(self, image: Image, context: Context) -> tuple[Image, Context]: Returns: the pre-processed image (or batch of images) and their context """ - pass def build_bottom_up_preprocessor(color_mode: str, transform: A.BaseCompose) -> Preprocessor: diff --git a/deeplabcut/pose_estimation_pytorch/models/backbones/base.py b/deeplabcut/pose_estimation_pytorch/models/backbones/base.py index dedac0f70..9f5a3972a 100644 --- a/deeplabcut/pose_estimation_pytorch/models/backbones/base.py +++ b/deeplabcut/pose_estimation_pytorch/models/backbones/base.py @@ -54,7 +54,6 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: Returns: a feature map for the input, of shape (batch_size, c', h', w') """ - pass def freeze_batch_norm_layers(self) -> None: """Freezes batch norm layers. diff --git a/deeplabcut/pose_estimation_pytorch/models/detectors/base.py b/deeplabcut/pose_estimation_pytorch/models/detectors/base.py index 09bcf4e9f..689702e41 100644 --- a/deeplabcut/pose_estimation_pytorch/models/detectors/base.py +++ b/deeplabcut/pose_estimation_pytorch/models/detectors/base.py @@ -85,7 +85,6 @@ def forward( losses: {'loss_name': loss_value} detections: for each of the b images, {"boxes": bounding_boxes} """ - pass @abstractmethod def get_target(self, labels: dict) -> list[dict]: @@ -97,7 +96,6 @@ def get_target(self, labels: dict) -> list[dict]: Returns: list of dictionaries, each representing target information for a single annotation. """ - pass def freeze_batch_norm_layers(self) -> None: """Freezes batch norm layers. diff --git a/deeplabcut/pose_estimation_pytorch/models/heads/base.py b/deeplabcut/pose_estimation_pytorch/models/heads/base.py index bb459f72d..c4be4f5bc 100644 --- a/deeplabcut/pose_estimation_pytorch/models/heads/base.py +++ b/deeplabcut/pose_estimation_pytorch/models/heads/base.py @@ -97,7 +97,6 @@ def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: Returns: the head outputs (e.g. "heatmap", "locref") """ - pass def get_loss( self, @@ -172,4 +171,3 @@ def convert_weights( ) `` """ - pass diff --git a/deeplabcut/pose_estimation_pytorch/models/modules/conv_block.py b/deeplabcut/pose_estimation_pytorch/models/modules/conv_block.py index 529db6495..8c1e00ee6 100644 --- a/deeplabcut/pose_estimation_pytorch/models/modules/conv_block.py +++ b/deeplabcut/pose_estimation_pytorch/models/modules/conv_block.py @@ -50,7 +50,6 @@ def forward(self, x: torch.Tensor): Returns: Output tensor. """ - pass def _init_weights(self, pretrained: str | None): """Method for initializing block weights from pretrained models. diff --git a/deeplabcut/pose_estimation_pytorch/models/necks/base.py b/deeplabcut/pose_estimation_pytorch/models/necks/base.py index 336b1a9ef..737005e61 100644 --- a/deeplabcut/pose_estimation_pytorch/models/necks/base.py +++ b/deeplabcut/pose_estimation_pytorch/models/necks/base.py @@ -33,7 +33,6 @@ def forward(self, x: torch.Tensor): Returns: Output tensor. """ - pass def _init_weights(self, pretrained: str): """Initialize the Neck with pretrained weights. diff --git a/deeplabcut/pose_estimation_pytorch/models/predictors/base.py b/deeplabcut/pose_estimation_pytorch/models/predictors/base.py index 42c9fb586..c175d7162 100644 --- a/deeplabcut/pose_estimation_pytorch/models/predictors/base.py +++ b/deeplabcut/pose_estimation_pytorch/models/predictors/base.py @@ -61,4 +61,3 @@ def forward(self, stride: float, outputs: dict[str, torch.Tensor]) -> dict[str, Raises: NotImplementedError: This method must be implemented in subclasses. """ - pass diff --git a/deeplabcut/pose_estimation_pytorch/runners/inference.py b/deeplabcut/pose_estimation_pytorch/runners/inference.py index 2a062be15..7f60db8d4 100644 --- a/deeplabcut/pose_estimation_pytorch/runners/inference.py +++ b/deeplabcut/pose_estimation_pytorch/runners/inference.py @@ -256,7 +256,7 @@ def __init__( self.inference_cfg = InferenceConfig.from_dict(inference_cfg) elif inference_cfg is None: self.inference_cfg = InferenceConfig() - else: + else: raise ValueError(f"Invalid inference config: {inference_cfg}") if self.snapshot_path is not None and self.snapshot_path != "": @@ -539,7 +539,11 @@ def _safe_get(self) -> Any: return item except Empty: # check if producer is still running - if self._stop_event.is_set() or self._preprocessing_thread is None or not self._preprocessing_thread.is_alive(): + if ( + self._stop_event.is_set() + or self._preprocessing_thread is None + or not self._preprocessing_thread.is_alive() + ): return None continue diff --git a/deeplabcut/pose_estimation_pytorch/runners/logger.py b/deeplabcut/pose_estimation_pytorch/runners/logger.py index 58250ae4b..215e36762 100644 --- a/deeplabcut/pose_estimation_pytorch/runners/logger.py +++ b/deeplabcut/pose_estimation_pytorch/runners/logger.py @@ -152,7 +152,6 @@ def log_images( targets: the targets for each model head step: the current step """ - pass def select_images_to_log(self, train: DataLoader, valid: DataLoader) -> None: """Selects the train and test images to log. @@ -448,7 +447,6 @@ def log_config(self, config: dict = None) -> None: Args: config: Experiment config file. """ - pass def _load_existing_data(self) -> None: """Loads existing CSV data if the log file exists.""" diff --git a/deeplabcut/pose_estimation_pytorch/runners/schedulers.py b/deeplabcut/pose_estimation_pytorch/runners/schedulers.py index 381a1a939..a677b95c0 100644 --- a/deeplabcut/pose_estimation_pytorch/runners/schedulers.py +++ b/deeplabcut/pose_estimation_pytorch/runners/schedulers.py @@ -13,7 +13,6 @@ from typing import Any import torch -from omegaconf import DictConfig, ListConfig, OmegaConf from torch.optim.lr_scheduler import _LRScheduler @@ -87,9 +86,6 @@ def build_scheduler( # unpicklable omegaconf objects with _parent references. # TODO @deruyter92: decide on typed / plain list / dict in upstream code # Then this check can be removed. - if isinstance(param, (ListConfig, DictConfig)): - param = OmegaConf.to_container(param, resolve=True) - if isinstance(param, list): param = [_parse_scheduler_param(p, optimizer) for p in param] else: diff --git a/deeplabcut/pose_estimation_tensorflow/nnets/utils.py b/deeplabcut/pose_estimation_tensorflow/nnets/utils.py index 1087fef57..5641ffc6c 100644 --- a/deeplabcut/pose_estimation_tensorflow/nnets/utils.py +++ b/deeplabcut/pose_estimation_tensorflow/nnets/utils.py @@ -213,5 +213,3 @@ def drop_connect(inputs, is_training, drop_connect_rate): class DepthwiseConv2D(tf.keras.layers.DepthwiseConv2D, tf.compat.v1.layers.Layer): """Wrap keras DepthwiseConv2D to tf.layers.""" - - pass diff --git a/deeplabcut/refine_training_dataset/outlier_frames.py b/deeplabcut/refine_training_dataset/outlier_frames.py index 7772df983..2a60ed37e 100644 --- a/deeplabcut/refine_training_dataset/outlier_frames.py +++ b/deeplabcut/refine_training_dataset/outlier_frames.py @@ -1122,7 +1122,7 @@ def merge_datasets(config, forceiterate=None): else: print("The following folder was not manually refined,...", folder) flagged = True - pass # this folder does not contain a MachineLabelsRefine file (not updated...) + # this folder does not contain a MachineLabelsRefine file (not updated...) if not flagged: # updates iteration by 1 diff --git a/deeplabcut/refine_training_dataset/tracklets.py b/deeplabcut/refine_training_dataset/tracklets.py index c89e24efb..408a2449b 100644 --- a/deeplabcut/refine_training_dataset/tracklets.py +++ b/deeplabcut/refine_training_dataset/tracklets.py @@ -193,7 +193,7 @@ def get_frame_ind(s): # Map a tracklet # to the animal ID it belongs to or the bodypart # it corresponds to. self.individuals = self.cfg["individuals"] + (["single"] if len(self.cfg["uniquebodyparts"]) else []) - self.tracklet2id = [i for i in range(0, self.nindividuals) for _ in bodyparts_multi] + [ + self.tracklet2id = [i for i in range(self.nindividuals) for _ in bodyparts_multi] + [ self.nindividuals ] * len(bodyparts_single) bps = bodyparts_multi + bodyparts_single diff --git a/deeplabcut/utils/auxiliaryfunctions.py b/deeplabcut/utils/auxiliaryfunctions.py index c230104a2..d9edc8ee3 100644 --- a/deeplabcut/utils/auxiliaryfunctions.py +++ b/deeplabcut/utils/auxiliaryfunctions.py @@ -22,7 +22,6 @@ import os import pickle -import warnings from collections.abc import Sequence from pathlib import Path @@ -48,9 +47,7 @@ def read_plainconfig(configname: str | Path) -> dict: return core_config.read_config_as_dict(config_path=configname) -def write_plainconfig( - configname: str | Path, cfg: dict, overwrite: bool = True -) -> None: +def write_plainconfig(configname: str | Path, cfg: dict, overwrite: bool = True) -> None: """Write a config dict to YAML (alias for write_config). See deeplabcut.core.config.""" core_config.write_config(config_path=configname, config=cfg, overwrite=overwrite) @@ -612,7 +609,7 @@ def find_analyzed_data(folder, videoname: str, scorer: str, filtered=False, trac candidates = [] for file in grab_files_in_folder(folder, "h5"): stem = Path(file).stem.replace("_filtered", "") - starts_by_scorer = file.startswith(videoname + scorer) or file.startswith(videoname + scorer_legacy) + starts_by_scorer = file.startswith((videoname + scorer, videoname + scorer_legacy)) if tracker: matches_tracker = stem.endswith(tracker) else: diff --git a/deeplabcut/utils/auxiliaryfunctions_3d.py b/deeplabcut/utils/auxiliaryfunctions_3d.py index 5d2c22300..58fdd7eee 100644 --- a/deeplabcut/utils/auxiliaryfunctions_3d.py +++ b/deeplabcut/utils/auxiliaryfunctions_3d.py @@ -66,7 +66,7 @@ def create_empty_df(dataframe, scorer, flag): [[scorer], [bodypart], ["x", "y", "z"]], names=["scorer", "bodyparts", "coords"], ) - frame = pd.DataFrame(a, columns=pdindex, index=range(0, df.shape[0])) + frame = pd.DataFrame(a, columns=pdindex, index=range(df.shape[0])) dataFrame = pd.concat([frame, dataFrame], axis=1) return (dataFrame, scorer, bodyparts) diff --git a/deeplabcut/utils/make_labeled_video.py b/deeplabcut/utils/make_labeled_video.py index 959050908..3a786d4e1 100644 --- a/deeplabcut/utils/make_labeled_video.py +++ b/deeplabcut/utils/make_labeled_video.py @@ -1348,12 +1348,10 @@ def create_video_with_all_detections( ) except ValueError as err: # No data stored for that particular frame print(n, f"no data: {err}") - pass try: clip.save_frame(frame) except Exception: print(n, "frame writing error.") - pass clip.close() else: print("Detections already plotted, ", outputname) diff --git a/deeplabcut/utils/skeleton.py b/deeplabcut/utils/skeleton.py index 9b0ccd823..1ad1729af 100644 --- a/deeplabcut/utils/skeleton.py +++ b/deeplabcut/utils/skeleton.py @@ -27,14 +27,12 @@ import pandas as pd from matplotlib.collections import LineCollection from matplotlib.widgets import Button, LassoSelector -from ruamel.yaml import YAML from scipy.spatial import KDTree from skimage import io +from deeplabcut.core.config import read_config, read_config_as_dict, write_config from deeplabcut.generate_training_dataset.trainingsetmanipulation import drop_likelihood_columns -from deeplabcut.core.config import read_config_as_dict, write_config - class SkeletonBuilder: def __init__(self, config_path): diff --git a/deeplabcut/utils/video_processor.py b/deeplabcut/utils/video_processor.py index 5ce7bee90..01c5ff39e 100644 --- a/deeplabcut/utils/video_processor.py +++ b/deeplabcut/utils/video_processor.py @@ -86,27 +86,21 @@ def frame_count(self): def get_video(self): """Implement your own.""" - pass def get_info(self): """Implement your own.""" - pass def create_video(self): """Implement your own.""" - pass def _read_frame(self): """Implement your own.""" - pass def save_frame(self, frame): """Implement your own.""" - pass def close(self): """Implement your own.""" - pass class VideoProcessorCV(VideoProcessor): diff --git a/tests/core/config/test_config_breakage.py b/tests/core/config/test_config_breakage.py new file mode 100644 index 000000000..81cba7204 --- /dev/null +++ b/tests/core/config/test_config_breakage.py @@ -0,0 +1,236 @@ +"""Pathological casess tests for the centralized config model.""" + +from __future__ import annotations + +import enum +import logging +from pathlib import Path + +import pytest +from pydantic import Field, ValidationError + +from deeplabcut.core.config import DLCBaseConfig, DLCVersionedConfig, ProjectConfig +from deeplabcut.utils.deprecation import DLCDeprecationWarning + +# ----------------------------------------------------------------------------- +# In-place nested mutation +# ----------------------------------------------------------------------------- + + +@pytest.mark.xfail( + reason="This may make validation difficult and cause subtle issues " + "if we start changing how functions such as append() operate." + " Avoiding lists in configs could be a better long-term solution." +) +def test_in_place_list_mutation_should_be_validated_like_assignment(): + """Appending to a config list changes config state without calling __setattr__. + + A strict config system should reject invalid values even when users mutate an + existing list instead of assigning a replacement list. + + Note that this pattern would incur validation: + cfg.TrainingFraction = [*cfg.TrainingFraction, 0.8] + cfg.video_sets = {**cfg.video_sets, "video.mp4": {"crop": "0, 100, 0, 100"}} + """ + cfg = ProjectConfig(TrainingFraction=[0.95]) + cfg.mark_clean() + + with pytest.raises(ValidationError): + cfg.TrainingFraction.append(1.5) + + +@pytest.mark.xfail(reason="Not implemented yet") +@pytest.mark.parametrize( + ("field_name", "mutate"), + [ + ("TrainingFraction", lambda cfg: cfg.TrainingFraction.append(0.8)), + ("video_sets", lambda cfg: cfg.video_sets.__setitem__("video.mp4", {"crop": "0, 100, 0, 100"})), + ], +) +def test_in_place_nested_mutation_should_mark_config_dirty(field_name, mutate): + """Mutating nested containers changes the saved config but may bypass dirty tracking. + + Dirty tracking is useful only if semantic config changes are recorded, whether + the user assigns a whole field or mutates a nested list/dict in place. + This ties with the problem that some nested fields are still dict instead of config models, + maybe once we have full config models we can come up with a parent/child system where changes in a child + are detected when querying the parent for its dirty state (or a simpler equivalent) + """ + cfg = ProjectConfig(TrainingFraction=[0.95], video_sets={}) + cfg.mark_clean() + + mutate(cfg) + + assert cfg.is_dirty + assert field_name in cfg.dirty_fields + + +# ----------------------------------------------------------------------------- +# Private state isolation +# ----------------------------------------------------------------------------- + + +class _TrackedForIsolation(DLCVersionedConfig): + name: str = "default" + count: int = 0 + + +def test_dirty_state_is_not_shared_between_instances(): + """Private dirty-field state must be per-instance, not shared via a mutable default.""" + first = _TrackedForIsolation() + second = _TrackedForIsolation() + + first.name = "changed" + + assert "name" in first.dirty_fields + assert not second.is_dirty + assert "name" not in second.dirty_fields + + +def test_change_notes_are_not_shared_between_instances(): + """Private change-note state must be per-instance, not shared via a mutable default.""" + first = _TrackedForIsolation() + second = _TrackedForIsolation() + + first.record_change_note("name", "name changed on first instance") + + assert first.change_notes == ["name changed on first instance"] + assert second.change_notes == [] + + +# ----------------------------------------------------------------------------- +# Changes note consistency +# ----------------------------------------------------------------------------- + + +def test_change_note_should_reject_unknown_field_names(): + """Notes for misspelled fields are silently lost during logging. + + Rejecting unknown field names catches typos at the call site instead of + storing a note that can never match a dirty field. + """ + cfg = ProjectConfig() + + with pytest.raises(KeyError): + cfg.record_change_note("not_a_real_field", "this note should not be accepted") + + +def test_change_note_recorded_with_alias_should_follow_canonical_dirty_field(caplog): + """Aliases should not split notes from the canonical dirty field name. + + If a note is recorded with a deprecated alias, log_changes should still use + that note when the canonical field is modified. + """ + cfg = ProjectConfig() + cfg.mark_clean() + + cfg.record_change_note("with_identity", "identity changed through compatibility alias") + cfg.identity = True + + with caplog.at_level(logging.INFO): + cfg.log_changes() + + assert "identity changed through compatibility alias" in caplog.text + + +# ----------------------------------------------------------------------------- +# Nested YAML comments and nested serialization +# ----------------------------------------------------------------------------- + + +class _CommentedInner(DLCBaseConfig): + threshold: float = Field( + default=0.5, + json_schema_extra={"comment": "Nested threshold comment"}, + ) + + +class _CommentedOuter(DLCBaseConfig): + inner: _CommentedInner = Field( + default_factory=_CommentedInner, + json_schema_extra={"comment": "Inner config section"}, + ) + + +@pytest.mark.xfail(reason="YAML comments are currently applied only to top-level config fields.") +def test_to_yaml_should_emit_comments_for_nested_config_fields(tmp_path): + """Nested config models can have field comments too. + + This protects against only applying YAML comments to the top-level model and + silently dropping useful nested schema documentation. + """ + path = tmp_path / "commented.yaml" + + _CommentedOuter().to_yaml(path) + + text = path.read_text() + assert "Inner config section" in text + assert "Nested threshold comment" in text + + +class _SerializationMode(enum.Enum): + FAST = "fast" + ACCURATE = "accurate" + + +class _SerializableInner(DLCBaseConfig): + output_path: Path = Path("outputs/predictions") + mode: _SerializationMode = _SerializationMode.FAST + + +class _SerializableOuter(DLCBaseConfig): + inner: _SerializableInner = Field(default_factory=_SerializableInner) + + +def test_nested_config_normalization_should_handle_paths_and_enums(): + """Nested model dumps should normalize non-primitive values before YAML output. + + Path and Enum values are common in configs and should become plain YAML-safe + values even when they appear inside nested config models. + """ + cfg = _SerializableOuter() + + assert cfg.to_dict(normalize=True) == { + "inner": { + "output_path": str(Path("outputs/predictions")), + "mode": "fast", + } + } + + +def test_nested_config_to_yaml_should_write_plain_path_and_enum_values(tmp_path): + """YAML output should not rely on Python-specific object representation. + + This catches failures where nested BaseModel values, pathlib paths, or enums + are passed to the YAML dumper without normalization. + """ + path = tmp_path / "serializable.yaml" + + _SerializableOuter().to_yaml(path) + + text = path.read_text() + assert str(Path("outputs/predictions")) in text + assert "fast" in text + + +# ----------------------------------------------------------------------------- +# Alias warnings and canonical writes +# ----------------------------------------------------------------------------- + + +def test_item_assignment_with_alias_should_warn_once_and_track_canonical_field(): + """Dict-style alias assignment should not warn twice or track the alias name. + + __setitem__ resolves aliases before delegating to assignment validation, so + the visible side effects should be one warning and one canonical dirty field. + """ + cfg = ProjectConfig() + cfg.mark_clean() + + with pytest.warns(DLCDeprecationWarning, match="with_identity") as caught: + cfg["with_identity"] = True + + assert len(caught) == 1 + assert cfg.identity is True + assert "identity" in cfg.dirty_fields + assert "with_identity" not in cfg.dirty_fields diff --git a/tests/pose_estimation_pytorch/runners/test_schedulers.py b/tests/pose_estimation_pytorch/runners/test_schedulers.py index 37e98ea6a..af745016a 100644 --- a/tests/pose_estimation_pytorch/runners/test_schedulers.py +++ b/tests/pose_estimation_pytorch/runners/test_schedulers.py @@ -39,7 +39,7 @@ def generate_random_lr_list(num_floats: int): @pytest.mark.parametrize( "milestones, lr_list", - [([10, 430], [[0.05], [0.005]]), (list(sorted(random.sample(range(0, 999), 2))), generate_random_lr_list(2))], + [([10, 430], [[0.05], [0.005]]), (list(sorted(random.sample(range(999), 2))), generate_random_lr_list(2))], ) def test_scheduler(milestones, lr_list): """Testing schedulers.py.