diff --git a/deeplabcut/benchmark/__init__.py b/deeplabcut/benchmark/__init__.py index e663705b8..ab863dec9 100644 --- a/deeplabcut/benchmark/__init__.py +++ b/deeplabcut/benchmark/__init__.py @@ -13,12 +13,13 @@ import json import os from collections.abc import Container +from pathlib import Path from typing import Literal from deeplabcut.benchmark.base import Benchmark, Result, ResultCollection -DATA_ROOT = os.path.join(os.getcwd(), "data") -CACHE = os.path.join(os.getcwd(), ".results") +DATA_ROOT = Path.cwd() / "data" +CACHE = Path.cwd() / ".results" __registry = [] @@ -99,20 +100,20 @@ def evaluate( def get_filepath(basename: str): - return os.path.join(DATA_ROOT, basename) + return DATA_ROOT / basename def savecache(results: ResultCollection): - with open(CACHE, "w") as fh: + with Path(CACHE).open("w") as fh: json.dump(results.todicts(), fh, indent=2) def loadcache(cache=CACHE, on_missing: Literal["raise", "ignore"] = "ignore") -> ResultCollection: - if not os.path.exists(cache): + if not Path(cache).exists(): if on_missing == "raise": raise FileNotFoundError(cache) return ResultCollection() - with open(cache) as fh: + with Path(cache).open() as fh: try: data = json.load(fh) except json.decoder.JSONDecodeError as e: diff --git a/deeplabcut/benchmark/metrics.py b/deeplabcut/benchmark/metrics.py index 91f34a457..39e8133b5 100644 --- a/deeplabcut/benchmark/metrics.py +++ b/deeplabcut/benchmark/metrics.py @@ -11,9 +11,9 @@ """Evaluation metrics for the DeepLabCut benchmark.""" -import os import pickle from collections import defaultdict +from pathlib import Path import numpy as np import pandas as pd @@ -33,7 +33,7 @@ def _format_gt_data(h5file: str, test_indices: list[int] | None = None): except KeyError: n_unique = 0 guarantee_multiindex_rows(df) - file_paths = [os.path.join(*row) for row in df.index.to_list()] + file_paths = [Path(*row) for row in df.index.to_list()] temp = ( df.stack("individuals", dropna=False) .reindex(animals, level="individuals") @@ -248,13 +248,13 @@ def load_test_images(h5file: str, metadata: str) -> list[str]: test_images = [] for img_path in df_test.index: if not isinstance(img_path, str): - img_path = os.path.join(*img_path) + img_path = str(Path(*img_path)) test_images.append(img_path) return test_images def _load_test_indices(shuffle_metadata_path: str) -> list[int]: """Returns the indices of test images in the training dataset dataframe.""" - with open(shuffle_metadata_path, "rb") as f: + with Path(shuffle_metadata_path).open("rb") as f: test_indices = set([int(i) for i in pickle.load(f)[2]]) return list(sorted(test_indices)) diff --git a/deeplabcut/benchmark/utils.py b/deeplabcut/benchmark/utils.py index 9c93fcf57..fc5492ade 100644 --- a/deeplabcut/benchmark/utils.py +++ b/deeplabcut/benchmark/utils.py @@ -18,6 +18,7 @@ import os import pkgutil import sys +from pathlib import Path class RedirectStdStreams: @@ -46,7 +47,7 @@ def __exit__(self, exc_type, exc_value, traceback): class DisableOutput(RedirectStdStreams): def __init__(self): - devnull = open(os.devnull, "w") + devnull = Path(os.devnull).open("w") super().__init__(stdout=devnull, stderr=devnull) diff --git a/deeplabcut/compat.py b/deeplabcut/compat.py index cf511533b..3c11e6a2c 100644 --- a/deeplabcut/compat.py +++ b/deeplabcut/compat.py @@ -26,6 +26,14 @@ DEFAULT_ENGINE = Engine.PYTORCH +def _coerce_video_paths(videos: list[str | Path]) -> list[Path]: + return [Path(v) for v in videos] + + +def _coerce_optional_path(path: str | Path | None) -> Path | None: + return None if path is None else Path(path) + + def get_project_engine(cfg: dict) -> Engine: """ Args: @@ -95,8 +103,8 @@ def train_network( Parameters ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. shuffle: int, optional, default=1 Integer value specifying the shuffle index to select for training. @@ -256,6 +264,7 @@ def train_network( display_iters=50, ) """ + config = Path(config) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -313,7 +322,7 @@ def train_network( def return_train_network_path( - config, + config: str | Path, shuffle: int = 1, trainingsetindex: int = 0, modelprefix: str = "", @@ -324,8 +333,8 @@ def return_train_network_path( Parameters ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. shuffle: int Integer value specifying the shuffle index to select for training. @@ -345,6 +354,7 @@ def return_train_network_path( Returns the triple: trainposeconfigfile, testposeconfigfile, snapshotfolder """ + config = Path(config) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -404,7 +414,7 @@ def evaluate_network( Parameters ---------- - config : string + config : str or Path Full path of the config.yaml file. shuffles: sequence of int, optional, default=[1] @@ -518,6 +528,7 @@ def evaluate_network( Note: This defaults to standard plotting for single-animal projects. """ + config = Path(config) if engine is None: cfg = _load_config(config) engines = set() @@ -576,7 +587,7 @@ def evaluate_network( @renamed_parameter(old="comparisonbodyparts", new="comparison_bodyparts", since="3.0.0") @renamed_parameter(old="Snapindex", new="snapshotindex", since="3.0.0") def return_evaluate_network_data( - config: str, + config: str | Path, shuffle: int = 0, trainingsetindex: int = 0, comparison_bodyparts: str | list[str] = "all", @@ -603,8 +614,8 @@ def return_evaluate_network_data( comparison_bodyparts, cfg, Snapshots[snapshotindex] ) ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. shuffle: integer integers specifying shuffle index of the training dataset. @@ -642,6 +653,7 @@ def return_evaluate_network_data( If you want to plot >>> deeplabcut.evaluate_network('/analysis/project/reaching-task/config.yaml',shuffle=[1],plotting=True) """ + config = Path(config) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -672,15 +684,15 @@ def return_evaluate_network_data( @renamed_parameter(old="batchsize", new="batch_size", since="3.0.0") @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def analyze_videos( - config: str, - videos: list[str], + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, gputouse: str | None = None, save_as_csv: bool = False, in_random_order: bool = True, - destfolder: str | None = None, + destfolder: str | Path | None = None, batch_size: int | None = None, cropping: list[int] | None = None, TFGPUinference: bool = True, @@ -713,12 +725,12 @@ def analyze_videos( Parameters ---------- - config: str + config: str or Path Full path of the config.yaml file. - videos: list[str] - A list of strings containing the full paths to videos for analysis or a path to - the directory, where all the videos with same extension are stored. + videos: list[str] or list[Path] + Full paths to videos for analysis, or a path to the directory where all videos + with the same extension are stored. video_extensions : str | Sequence[str] | None, optional, default=None Controls how ``videos`` are filtered, based on file extension. @@ -751,7 +763,7 @@ def analyze_videos( Whether or not to analyze videos in a random order. This is only relevant when specifying a video directory in `videos`. - destfolder: string or None, optional, default=None + destfolder: str, Path, or None, optional, default=None Specifies the destination folder for analysis data. If ``None``, the path of the video is used. Note that for subsequent analysis this folder also needs to be passed. @@ -915,6 +927,9 @@ def analyze_videos( save_as_csv=True, ) """ + config = Path(config) + videos = _coerce_video_paths(videos) + destfolder = _coerce_optional_path(destfolder) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -999,14 +1014,14 @@ def analyze_videos( @renamed_parameter(old="batchsize", new="batch_size", since="3.0.0") @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def create_tracking_dataset( - config: str, - videos: list[str], + config: str | Path, + videos: list[str | Path], track_method: str, video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, gputouse: int | None = None, - destfolder: str | None = None, + destfolder: str | Path | None = None, batch_size: int | None = None, cropping: list[int] | None = None, TFGPUinference: bool = True, @@ -1019,13 +1034,12 @@ def create_tracking_dataset( Parameters ---------- - config: str + config: str or Path Full path of the config.yaml file. - videos: list[str] - A list of strings containing the full paths to videos from which to create a - tracking dataset, or a path to the directory where all the videos with same - extension are stored. + videos: list[str] or list[Path] + Full paths to videos from which to create a tracking dataset, or a path to the + directory where all videos with the same extension are stored. track_method: str Specifies the tracker used to generate the pose estimation data. Must be either @@ -1088,6 +1102,9 @@ def create_tracking_dataset( DLCScorer: str the scorer used to analyze the videos """ + config = Path(config) + videos = _coerce_video_paths(videos) + destfolder = _coerce_optional_path(destfolder) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -1313,8 +1330,8 @@ def analyze_images( def analyze_time_lapse_frames( - config: str, - directory: str, + config: str | Path, + directory: str | Path, frametype: str = ".png", shuffle: int = 1, trainingsetindex: int = 0, @@ -1340,10 +1357,10 @@ def analyze_time_lapse_frames( Parameters ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. - directory: string + directory: str or Path Full path to directory containing the frames that shall be analyzed frametype: string, optional @@ -1387,6 +1404,8 @@ def analyze_time_lapse_frames( Note: for test purposes one can extract all frames from a video with ffmeg, e.g. >>> ffmpeg -i testvideo.avi "thumb%04d.png" """ + config = Path(config) + directory = Path(directory) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -1427,13 +1446,13 @@ def analyze_time_lapse_frames( @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def convert_detections2tracklets( - config: str, - videos: list[str], + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, overwrite: bool = False, - destfolder: str | None = None, + destfolder: str | Path | None = None, ignore_bodyparts: list[str] | None = None, inferencecfg: dict | None = None, modelprefix: str = "", @@ -1449,12 +1468,12 @@ def convert_detections2tracklets( Parameters ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. - videos : list - A list of strings containing the full paths to videos for analysis or a path to the directory, - where all the videos with same extension are stored. + videos : list[str] or list[Path] + Full paths to videos for analysis, or a path to the directory where all videos + with the same extension are stored. video_extensions : str | Sequence[str] | None, optional, default=None Controls how ``videos`` are filtered, based on file extension. @@ -1476,7 +1495,7 @@ def convert_detections2tracklets( overwrite: bool, optional. Overwrite tracks file i.e. recompute tracks from full detections and overwrite. - destfolder: string, optional + destfolder: str, Path, or None, optional Specifies the destination folder for analysis data (default is the path of the video). Note that for subsequent analysis this folder also needs to be passed. @@ -1536,6 +1555,9 @@ def convert_detections2tracklets( -------- """ + config = Path(config) + videos = _coerce_video_paths(videos) + destfolder = _coerce_optional_path(destfolder) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -1592,7 +1614,7 @@ def convert_detections2tracklets( def extract_maps( - config, + config: str | Path, shuffle: int = 0, trainingsetindex: int = 0, gputouse: int | None = None, @@ -1609,8 +1631,8 @@ def extract_maps( partaffinity_graph, imagename, True/False if this image was in trainingset). ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. shuffle: integer integers specifying shuffle index of the training dataset. The default is 0. @@ -1650,6 +1672,7 @@ def extract_maps( If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0. >>> deeplabcut.extract_maps(configfile,0,Indices=[0,103]) """ + config = Path(config) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -1745,7 +1768,7 @@ def visualize_paf( @renamed_parameter(old="comparisonbodyparts", new="comparison_bodyparts", since="3.0.0") def extract_save_all_maps( - config, + config: str | Path, shuffle: int = 1, trainingsetindex: int = 0, comparison_bodyparts: str | list[str] = "all", @@ -1756,7 +1779,7 @@ def extract_save_all_maps( rescale: bool = False, Indices: list[int] | None = None, modelprefix: str = "", - dest_folder: str = None, + dest_folder: str | Path | None = None, snapshot_index: int | str | None = None, detector_snapshot_index: int | str | None = None, engine: Engine | None = None, @@ -1766,8 +1789,8 @@ def extract_save_all_maps( will be rescaled to the size of the input image and stored in the corresponding model folder in /evaluation-results. ---------- - config : string - Full path of the config.yaml file as a string. + config : str or Path + Full path of the config.yaml file. shuffle: integer integers specifying shuffle index of the training dataset. The default is 1. @@ -1823,6 +1846,8 @@ def extract_save_all_maps( >>> deeplabcut.extract_save_all_maps('/analysis/project/reaching-task/config.yaml', shuffle=1,Indices=[0,1,33]) """ + config = Path(config) + dest_folder = _coerce_optional_path(dest_folder) if engine is None: engine = get_shuffle_engine( _load_config(config), @@ -1870,7 +1895,7 @@ def extract_save_all_maps( def export_model( - cfg_path: str, + cfg_path: str | Path, shuffle: int = 1, trainingsetindex: int = 0, snapshotindex: int | None = None, @@ -1892,7 +1917,7 @@ def export_model( Parameters ----------- - cfg_path : string + cfg_path : str or Path path to the DLC Project config.yaml file shuffle : int, optional @@ -1937,6 +1962,7 @@ def export_model( >>> deeplabcut.export_model('/analysis/project/reaching-task/config.yaml',shuffle=3, snapshotindex=-1) -------- """ + cfg_path = Path(cfg_path) if engine is None: engine = get_shuffle_engine( _load_config(cfg_path), @@ -1995,12 +2021,12 @@ def _gpu_to_use_to_device(gpu_to_use: int | None, device: str | None) -> str | N return device -def _load_config(config: str) -> dict: +def _load_config(config: str | Path) -> dict: config_path = Path(config) if not config_path.exists(): raise FileNotFoundError(f"Config {config} is not found. Please make sure that the file exists.") - with open(config) as f: + with config_path.open() as f: project_config = YAML(typ="safe", pure=True).load(f) return project_config diff --git a/deeplabcut/core/config.py b/deeplabcut/core/config.py index db222def3..722130b53 100644 --- a/deeplabcut/core/config.py +++ b/deeplabcut/core/config.py @@ -26,7 +26,7 @@ def read_config_as_dict(config_path: str | Path) -> dict: Returns: The configuration file with pure Python classes """ - with open(config_path) as f: + with Path(config_path).open() as f: cfg = YAML(typ="safe", pure=True).load(f) return cfg @@ -46,7 +46,7 @@ def write_config(config_path: str | Path, config: dict, overwrite: bool = True) if not overwrite and Path(config_path).exists(): raise FileExistsError(f"Cannot write to {config_path} - set overwrite=True to force") - with open(config_path, "w") as file: + with Path(config_path).open("w") as file: YAML().dump(config, file) diff --git a/deeplabcut/core/crossvalutils.py b/deeplabcut/core/crossvalutils.py index e62f19ddc..ca1447c61 100644 --- a/deeplabcut/core/crossvalutils.py +++ b/deeplabcut/core/crossvalutils.py @@ -10,11 +10,11 @@ # -import os import pickle import shutil from collections import defaultdict from copy import deepcopy +from pathlib import Path import networkx as nx import numpy as np @@ -44,9 +44,10 @@ def _set_up_evaluation(data): def _form_original_path(path): - root, filename = os.path.split(path) - base, ext = os.path.splitext(filename) - return os.path.join(root, filename.split("c")[0] + ext) + p = Path(path) + ext = p.suffix + filename = p.name + return str(p.parent / (filename.split("c")[0] + ext)) def _unsorted_unique(array): @@ -395,9 +396,9 @@ def cross_validate_paf_graphs( inf_cfg_temp = inf_cfg.copy() inf_cfg_temp["pcutoff"] = pcutoff - with open(full_data_file, "rb") as file: + with Path(full_data_file).open("rb") as file: data = pickle.load(file) - with open(metadata_file, "rb") as file: + with Path(metadata_file).open("rb") as file: metadata = pickle.load(file) params = _set_up_evaluation(data) @@ -415,10 +416,8 @@ def cross_validate_paf_graphs( if calibrate: trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) - calibration_file = os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", + calibration_file = str( + Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5") ) else: calibration_file = "" @@ -448,7 +447,7 @@ def cross_validate_paf_graphs( inds = list(paf_inds[size_opt]) auxiliaryfunctions.edit_config(pose_config, {"paf_best": [int(ind) for ind in inds]}) if output_name: - with open(output_name, "wb") as file: + with Path(output_name).open("wb") as file: pickle.dump([results], file) return results[:3], paf_scores, results[3][size_opt] diff --git a/deeplabcut/core/inferenceutils.py b/deeplabcut/core/inferenceutils.py index b7bb82f10..8fb249a94 100644 --- a/deeplabcut/core/inferenceutils.py +++ b/deeplabcut/core/inferenceutils.py @@ -20,6 +20,7 @@ from collections.abc import Iterable from dataclasses import dataclass from math import erf, sqrt +from pathlib import Path from typing import Any import networkx as nx @@ -828,7 +829,7 @@ def wrapped(i): pbar.update() def from_pickle(self, pickle_path): - with open(pickle_path, "rb") as file: + with Path(pickle_path).open("rb") as file: data = pickle.load(file) self.unique = data.pop("single", {}) self.assemblies = data @@ -876,7 +877,7 @@ def to_pickle(self, output_name): data[ind] = [ass.data for ass in assemblies] if self.unique: data["single"] = self.unique - with open(output_name, "wb") as file: + with Path(output_name).open("wb") as file: pickle.dump(data, file, pickle.HIGHEST_PROTOCOL) diff --git a/deeplabcut/create_project/add.py b/deeplabcut/create_project/add.py index 550c5a91e..443831aad 100644 --- a/deeplabcut/create_project/add.py +++ b/deeplabcut/create_project/add.py @@ -9,8 +9,16 @@ # Licensed under GNU Lesser General Public License v3.0 # +from pathlib import Path -def add_new_videos(config, videos, copy_videos=False, coords=None, extract_frames=False): + +def add_new_videos( + config: str | Path, + videos: list[str | Path], + copy_videos=False, + coords=None, + extract_frames=False, +): """Add new videos to the config file at any stage of the project. Parameters @@ -95,7 +103,7 @@ def add_new_videos(config, videos, copy_videos=False, coords=None, extract_frame try: src = str(src) dst = str(dst) - os.symlink(src, dst) + Path(dst).symlink_to(src) print(f"Created the symlink of {src} to {dst}") except OSError: try: @@ -117,7 +125,7 @@ def add_new_videos(config, videos, copy_videos=False, coords=None, extract_frame video_path = str(Path.resolve(Path(video))) # video_path = os.path.realpath(video) except Exception: - video_path = os.readlink(video) + video_path = str(Path(video).readlink()) vid = VideoReader(video_path) if coords is not None: diff --git a/deeplabcut/create_project/demo_data.py b/deeplabcut/create_project/demo_data.py index 30b58390f..a74e9d3f7 100644 --- a/deeplabcut/create_project/demo_data.py +++ b/deeplabcut/create_project/demo_data.py @@ -9,7 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import os from pathlib import Path import deeplabcut @@ -18,7 +17,7 @@ def load_demo_data( - config: str, + config: str | Path, createtrainingset: bool = True, engine: Engine = Engine.PYTORCH, ): @@ -61,9 +60,9 @@ def transform_data(config): cfg["project_path"] = project_path if "Reaching" in project_path: - video_file = os.path.join(project_path, "videos", "reachingvideo1.avi") + video_file = Path(project_path) / "videos" / "reachingvideo1.avi" elif "openfield" in project_path: - video_file = os.path.join(project_path, "videos", "m4s1.mp4") + video_file = Path(project_path) / "videos" / "m4s1.mp4" else: print("This is not an official demo dataset.") diff --git a/deeplabcut/create_project/modelzoo.py b/deeplabcut/create_project/modelzoo.py index b4d615803..85843a438 100644 --- a/deeplabcut/create_project/modelzoo.py +++ b/deeplabcut/create_project/modelzoo.py @@ -46,7 +46,7 @@ def MakeTrain_pose_yaml(itemstochange, saveasconfigfile, defaultconfigfile): - raw = open(defaultconfigfile).read() + raw = Path(defaultconfigfile).open().read() docs = [] for raw_doc in raw.split("\n---"): try: @@ -57,7 +57,7 @@ def MakeTrain_pose_yaml(itemstochange, saveasconfigfile, defaultconfigfile): for key in itemstochange.keys(): docs[0][key] = itemstochange[key] docs[0]["max_input_size"] = 1500 - with open(saveasconfigfile, "w") as f: + with Path(saveasconfigfile).open("w") as f: yaml.dump(docs[0], f) return docs[0] @@ -476,7 +476,7 @@ def _create_inference_config(inference_cfg_path: str | Path, project_cfg: dict): topktoretain=len(project_cfg["individuals"]), withid=project_cfg.get("identity", False), ) - default_inf_path = Path(auxiliaryfunctions.get_deeplabcut_path()) / "inference_cfg.yaml" + default_inf_path = auxiliaryfunctions.get_deeplabcut_path() / "inference_cfg.yaml" MakeInference_yaml(inf_updates, inference_cfg_path, default_inf_path) @@ -561,7 +561,7 @@ def create_pretrained_project_tensorflow( model = "full_human" if model in MODELOPTIONS: - cwd = os.getcwd() + cwd = Path.cwd() cfg = deeplabcut.create_new_project( project, experimenter, videos, working_directory, copy_videos, video_extensions=video_extensions @@ -610,31 +610,27 @@ def create_pretrained_project_tensorflow( auxiliaryfunctions.write_config(cfg, config) config = auxiliaryfunctions.read_config(cfg) - train_dir = Path( - os.path.join( - config["project_path"], - str( - auxiliaryfunctions.get_model_folder( - trainFraction=config["TrainingFraction"][0], - shuffle=1, - cfg=config, - ) - ), - "train", + train_dir = ( + Path(config["project_path"]) + / str( + auxiliaryfunctions.get_model_folder( + trainFraction=config["TrainingFraction"][0], + shuffle=1, + cfg=config, + ) ) + / "train" ) - test_dir = Path( - os.path.join( - config["project_path"], - str( - auxiliaryfunctions.get_model_folder( - trainFraction=config["TrainingFraction"][0], - shuffle=1, - cfg=config, - ) - ), - "test", + test_dir = ( + Path(config["project_path"]) + / str( + auxiliaryfunctions.get_model_folder( + trainFraction=config["TrainingFraction"][0], + shuffle=1, + cfg=config, + ) ) + / "test" ) # Create the model directory @@ -644,8 +640,8 @@ def create_pretrained_project_tensorflow( modelfoldername = auxiliaryfunctions.get_model_folder( trainFraction=config["TrainingFraction"][0], shuffle=1, cfg=config ) - path_train_config = str(os.path.join(config["project_path"], Path(modelfoldername), "train", "pose_cfg.yaml")) - path_test_config = str(os.path.join(config["project_path"], Path(modelfoldername), "test", "pose_cfg.yaml")) + path_train_config = str(Path(config["project_path"]) / Path(modelfoldername) / "train" / "pose_cfg.yaml") + path_test_config = str(Path(config["project_path"]) / Path(modelfoldername) / "test" / "pose_cfg.yaml") # Download the weights and put then in appropriate directory print("Downloading weights...") @@ -668,9 +664,9 @@ def create_pretrained_project_tensorflow( # model_path = auxfun_models.check_for_weights(pose_cfg['net_type'], parent_path) # Updating training and test pose_cfg: - snapshotname = [fn for fn in os.listdir(train_dir) if ".meta" in fn][0].split(".meta")[0] + snapshotname = [p.name for p in Path(train_dir).iterdir() if ".meta" in p.name][0].split(".meta")[0] dict2change = { - "init_weights": str(os.path.join(train_dir, snapshotname)), + "init_weights": str(Path(train_dir) / snapshotname), "project_path": str(config["project_path"]), } diff --git a/deeplabcut/create_project/new.py b/deeplabcut/create_project/new.py index 9fdbe4480..85f7bfb5f 100644 --- a/deeplabcut/create_project/new.py +++ b/deeplabcut/create_project/new.py @@ -26,8 +26,8 @@ def create_new_project( project: str, experimenter: str, - videos: list[str], - working_directory: str | None = None, + videos: list[str | Path], + working_directory: str | Path | None = None, copy_videos: bool = False, video_extensions: str | Sequence[str] | None = None, multianimal: bool = False, @@ -154,7 +154,7 @@ def create_new_project( # Create project and sub-directories if not DEBUG and project_path.exists(): print(f'Project "{project_path}" already exists!') - return os.path.join(str(project_path), "config.yaml") + return str(project_path / "config.yaml") video_path = project_path / "videos" data_path = project_path / "labeled-data" shuffles_path = project_path / "training-datasets" @@ -198,7 +198,7 @@ def create_new_project( try: src = str(src) dst = str(dst) - os.symlink(src, dst) + Path(dst).symlink_to(src) print(f"Created the symlink of {src} to {dst}") except OSError: try: @@ -223,14 +223,14 @@ def create_new_project( # video. [old: rel_video_path = os.path.realpath(video)] rel_video_path = str(Path.resolve(Path(video))) except Exception: - rel_video_path = os.readlink(str(video)) + rel_video_path = str(Path(str(video)).readlink()) try: vid = VideoReader(rel_video_path) video_sets[rel_video_path] = {"crop": ", ".join(map(str, vid.get_bbox()))} except OSError: warnings.warn("Cannot open the video file! Skipping to the next one...", stacklevel=2) - os.remove(video) # Removing the video or link from the project + Path(video).unlink() # Removing the video or link from the project if not len(video_sets): # Silently sweep the files that were already written. @@ -304,7 +304,7 @@ def create_new_project( cfg_file["alphavalue"] = 0.7 # for plots transparency of markers cfg_file["colormap"] = "rainbow" # for plots type of colormap - projconfigfile = os.path.join(str(project_path), "config.yaml") + projconfigfile = project_path / "config.yaml" # Write dictionary to yaml config file auxiliaryfunctions.write_config(projconfigfile, cfg_file) diff --git a/deeplabcut/create_project/new_3d.py b/deeplabcut/create_project/new_3d.py index 780a6c44d..5160ba77c 100644 --- a/deeplabcut/create_project/new_3d.py +++ b/deeplabcut/create_project/new_3d.py @@ -10,7 +10,6 @@ # -import os from pathlib import Path from deeplabcut import DEBUG @@ -122,7 +121,7 @@ def create_new_project_3d(project, experimenter, num_cameras=2, working_director cfg_file_3d.insert(len(cfg_file_3d), str("shuffle_camera-" + str(i + 1)), 1) cfg_file_3d.insert(len(cfg_file_3d), str("trainingsetindex_camera-" + str(i + 1)), 0) - projconfigfile = os.path.join(str(project_path), "config.yaml") + projconfigfile = project_path / "config.yaml" auxiliaryfunctions.write_config_3d(projconfigfile, cfg_file_3d) print('Generated "{}"'.format(project_path / "config.yaml")) diff --git a/deeplabcut/generate_training_dataset/frame_extraction.py b/deeplabcut/generate_training_dataset/frame_extraction.py index 1d4bae857..93422f24f 100755 --- a/deeplabcut/generate_training_dataset/frame_extraction.py +++ b/deeplabcut/generate_training_dataset/frame_extraction.py @@ -9,8 +9,10 @@ # Licensed under GNU Lesser General Public License v3.0 # +from pathlib import Path -def select_cropping_area(config, videos=None): + +def select_cropping_area(config: str | Path, videos=None): """Interactively select the cropping area of all videos in the config. A user interface pops up with a frame to select the cropping parameters. Use the left click to draw a box and hit the button 'set cropping parameters' to store the cropping @@ -62,7 +64,7 @@ def select_cropping_area(config, videos=None): def extract_frames( - config, + config: str | Path, mode="automatic", algo="kmeans", crop=False, @@ -250,8 +252,6 @@ def extract_frames( extracted_cam=0, ) """ - import glob - import os import re import sys from pathlib import Path @@ -331,7 +331,7 @@ def extract_frames( output_path = Path(config).parents[0] / "labeled-data" / fname.stem if output_path.exists(): - if len(os.listdir(output_path)): + if any(output_path.iterdir()): if userfeedback: askuser = input( "The directory already contains some frames. Do you want to add to it?(yes/no): " @@ -463,8 +463,7 @@ def extract_frames( if videos_list is not None: # filter video_list by the ones in the config file videos = [v for v in videos if v in videos_list] project_path = Path(config).parents[0] - labels_path = os.path.join(project_path, "labeled-data/") - os.path.join(project_path, "videos/") + labels_path = project_path / "labeled-data" try: cfg_3d = auxiliaryfunctions.read_config(config3d) except Exception as e: @@ -474,7 +473,7 @@ def extract_frames( cams = cfg_3d["camera_names"] extCam_name = cams[extracted_cam] del cams[extracted_cam] - label_dirs = sorted(glob.glob(os.path.join(labels_path, "*" + extCam_name + "*"))) + label_dirs = sorted(labels_path.glob("*" + extCam_name + "*")) # select crop method crop_list = [] @@ -497,19 +496,19 @@ def extract_frames( crop_list.append(coords) for coords, dirPath in zip(crop_list, label_dirs, strict=False): - extracted_images = glob.glob(os.path.join(dirPath, "*png")) + extracted_images = list(dirPath.glob("*png")) imgPattern = re.compile("[0-9]{1,10}") for cam in cams: - output_path = re.sub(extCam_name, cam, dirPath) + output_path = Path(re.sub(extCam_name, cam, str(dirPath))) - for fname in os.listdir(output_path): - if fname.endswith(".png"): - os.remove(os.path.join(output_path, fname)) + for p in output_path.iterdir(): + if p.name.endswith(".png"): + p.unlink() # Find the matching video from the config `video_sets`, # as it may be stored elsewhere than in the `videos` directory. - video_name = os.path.basename(output_path) + video_name = output_path.name vid = "" for video in cfg["video_sets"]: if video_name in video: @@ -521,12 +520,12 @@ def extract_frames( cap = cv2.VideoCapture(vid) print("\n extracting matched frames from " + video_name) for img in extracted_images: - imgNum = re.findall(imgPattern, os.path.basename(img))[0] + imgNum = re.findall(imgPattern, img.name)[0] cap.set(1, int(imgNum)) ret, frame = cap.read() if ret: image = img_as_ubyte(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - img_name = os.path.join(output_path, "img" + imgNum + ".png") + img_name = str(output_path / ("img" + imgNum + ".png")) if crop: io.imsave( img_name, diff --git a/deeplabcut/generate_training_dataset/metadata.py b/deeplabcut/generate_training_dataset/metadata.py index f854a962a..906a39937 100644 --- a/deeplabcut/generate_training_dataset/metadata.py +++ b/deeplabcut/generate_training_dataset/metadata.py @@ -68,14 +68,14 @@ def load_split(self, cfg: dict, trainset_path: Path) -> ShuffleMetadata: _, doc_path = auxiliaryfunctions.get_data_and_metadata_filenames( trainset_path, self.train_fraction, self.index, cfg ) - if not Path(doc_path).exists(): + if not doc_path.exists(): raise ValueError( f"Could not load the metadata file for {self} as {doc_path} does not " f"exist. If you deleted the shuffle, you also need to delete the " f"shuffle from metadata.yaml or recreate the metadata.yaml file." ) - with open(doc_path, "rb") as f: + with doc_path.open("rb") as f: _, train_idx, test_idx, _ = pickle.load(f) return ShuffleMetadata( name=self.name, @@ -231,7 +231,7 @@ def save(self) -> None: "engine": s.engine.aliases[0], } - with open(self.path(self.project_config), "w") as file: + with self.path(self.project_config).open("w") as file: file.write("\n".join(self.file_header) + "\n") YAML().dump(metadata, file) @@ -254,7 +254,7 @@ def load( metadata_path = TrainingDatasetMetadata.path(cfg) if not metadata_path.exists(): raise FileNotFoundError(f"No metadata.yaml found at {metadata_path}.") - with open(metadata_path) as file: + with metadata_path.open() as file: metadata = YAML(typ="safe", pure=True).load(file) shuffles = [] @@ -307,7 +307,7 @@ def create(config: str | Path | dict) -> TrainingDatasetMetadata: existing_splits: dict[tuple[tuple[int, ...], tuple[int, ...]], int] = {} for doc_path in shuffle_docs: index = int(doc_path.stem.split("shuffle")[-1]) - with open(doc_path, "rb") as f: + with doc_path.open("rb") as f: _, train_idx, test_idx, train_frac = pickle.load(f) engine = Engine.TF diff --git a/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py b/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py index 19b0fedc3..33d3c3518 100755 --- a/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py +++ b/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py @@ -11,7 +11,6 @@ from __future__ import annotations import os -import os.path import re import warnings from itertools import combinations @@ -69,7 +68,7 @@ def format_multianimal_training_data( array = np.round(array, decimals=n_decimals) for i in tqdm(train_inds): filename = filenames[i] - img_shape = read_image_shape_fast(os.path.join(project_path, *filename)) + img_shape = read_image_shape_fast(Path(project_path).joinpath(*filename)) joints = dict() has_data = False for n, xy in enumerate(array[i]): @@ -101,7 +100,7 @@ def format_multianimal_training_data( def create_multianimaltraining_dataset( - config, + config: str | Path, num_shuffles=1, Shuffles=None, windows2linux=False, @@ -367,12 +366,12 @@ def create_multianimaltraining_dataset( # Loading the encoder (if necessary downloading from TF) dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() - defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") + defaultconfigfile = dlcparent_path / "pose_cfg.yaml" if engine == Engine.PYTORCH: model_path = dlcparent_path else: - model_path = auxfun_models.check_for_weights(net_type, Path(dlcparent_path)) + model_path = auxfun_models.check_for_weights(net_type, dlcparent_path) Shuffles = validate_shuffles(cfg, Shuffles, num_shuffles, userfeedback) @@ -431,7 +430,7 @@ def create_multianimaltraining_dataset( # Saving metadata and data file (Pickle file) ################################################################################ auxiliaryfunctions.save_metadata( - os.path.join(project_path, metadatafilename), + Path(project_path) / metadatafilename, data, trainIndices, testIndices, @@ -447,10 +446,10 @@ def create_multianimaltraining_dataset( overwrite=not userfeedback, ) - datafilename = datafilename.split(".mat")[0] + ".pickle" + datafilename = datafilename.with_suffix(".pickle") import pickle - with open(os.path.join(project_path, datafilename), "wb") as f: + with (Path(project_path) / datafilename).open("wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) @@ -469,30 +468,9 @@ def create_multianimaltraining_dataset( auxiliaryfunctions.attempt_to_make_folder(str(Path(config).parents[0] / modelfoldername / "train")) auxiliaryfunctions.attempt_to_make_folder(str(Path(config).parents[0] / modelfoldername / "test")) - path_train_config = str( - os.path.join( - cfg["project_path"], - Path(modelfoldername), - "train", - "pose_cfg.yaml", - ) - ) - path_test_config = str( - os.path.join( - cfg["project_path"], - Path(modelfoldername), - "test", - "pose_cfg.yaml", - ) - ) - path_inference_config = str( - os.path.join( - cfg["project_path"], - Path(modelfoldername), - "test", - "inference_cfg.yaml", - ) - ) + path_train_config = str(Path(cfg["project_path"]) / modelfoldername / "train" / "pose_cfg.yaml") + path_test_config = str(Path(cfg["project_path"]) / modelfoldername / "test" / "pose_cfg.yaml") + path_inference_config = str(Path(cfg["project_path"]) / modelfoldername / "test" / "inference_cfg.yaml") if engine == Engine.TF: jointnames = [str(bpt) for bpt in multianimalbodyparts] @@ -600,7 +578,7 @@ def create_multianimaltraining_dataset( make_pytorch_test_config(pytorch_cfg, path_test_config, save=True) # Setting inference cfg file: - default_inf_path = Path(dlcparent_path) / "inference_cfg.yaml" + default_inf_path = dlcparent_path / "inference_cfg.yaml" inf_updates = dict( minimalnumberofconnections=int(len(cfg["multianimalbodyparts"]) / 2), topktoretain=len(cfg["individuals"]), @@ -646,11 +624,11 @@ def convert_cropped_to_standard_dataset( if delete_crops: print("Deleting crops...") - data_path = os.path.join(project_path, "labeled-data") + data_path = Path(project_path) / "labeled-data" for video in cfg["video_sets"]: _, filename, _ = trainingsetmanipulation._robust_path_split(video) if "_cropped" in video: # One can never be too safe... - shutil.rmtree(os.path.join(data_path, filename), ignore_errors=True) + shutil.rmtree(data_path / filename, ignore_errors=True) cfg["video_sets"] = videos_orig write_config(config_path, cfg) @@ -658,20 +636,18 @@ def convert_cropped_to_standard_dataset( if not recreate_datasets: return - datasets_folder = os.path.join( - project_path, - auxiliaryfunctions.get_training_set_folder(cfg), - ) + datasets_folder = Path(project_path) / auxiliaryfunctions.get_training_set_folder(cfg) df_old = pd.read_hdf( - os.path.join(datasets_folder, "CollectedData_" + cfg["scorer"] + ".h5"), + datasets_folder / ("CollectedData_" + cfg["scorer"] + ".h5"), ) def strip_cropped_image_name(path): - head, filename = os.path.split(path) - head = head.replace("_cropped", "") + path_obj = Path(path) + head = str(path_obj.parent).replace("_cropped", "") + filename = path_obj.name file, ext = filename.split(".") file = file.split("c")[0] - return os.path.join(head, file + "." + ext) + return str(Path(head) / (file + "." + ext)) img_names_old = np.asarray([strip_cropped_image_name(img) for img in df_old.index.to_list()]) df = merge_annotateddatasets(cfg, datasets_folder) @@ -679,12 +655,12 @@ def strip_cropped_image_name(path): train_idx = [] test_idx = [] pickle_files = [] - for filename in os.listdir(datasets_folder): - if filename.endswith("pickle"): - pickle_file = os.path.join(datasets_folder, filename) + for p in datasets_folder.iterdir(): + if p.name.endswith("pickle"): + pickle_file = p pickle_files.append(pickle_file) - if filename.startswith("Docu"): - with open(pickle_file, "rb") as f: + if p.name.startswith("Docu"): + with p.open("rb") as f: _, train_inds, test_inds, train_frac = pickle.load(f) train_inds_temp = np.flatnonzero(np.isin(img_names, img_names_old[train_inds])) test_inds_temp = np.flatnonzero(np.isin(img_names, img_names_old[test_inds])) @@ -694,10 +670,10 @@ def strip_cropped_image_name(path): # Search a pose_config.yaml file to parse missing information pose_config_path = "" - for dirpath, _, filenames in os.walk(os.path.join(project_path, "dlc-models")): + for dirpath, _, filenames in os.walk(Path(project_path) / "dlc-models"): for file in filenames: if file.endswith("pose_cfg.yaml"): - pose_config_path = os.path.join(dirpath, file) + pose_config_path = str(Path(dirpath) / file) break pose_cfg = read_plainconfig(pose_config_path) net_type = pose_cfg["net_type"] @@ -707,8 +683,8 @@ def strip_cropped_image_name(path): # Clean the training-datasets folder prior to recreating the data pickles shuffle_inds = set() for file in pickle_files: - os.remove(file) - shuffle_inds.add(int(re.findall(r"shuffle(\d+)", file)[0])) + file.unlink() + shuffle_inds.add(int(re.findall(r"shuffle(\d+)", str(file))[0])) create_multianimaltraining_dataset( config_path, trainIndices=train_idx, diff --git a/deeplabcut/generate_training_dataset/trainingsetmanipulation.py b/deeplabcut/generate_training_dataset/trainingsetmanipulation.py index e6fa7c913..b23819f36 100755 --- a/deeplabcut/generate_training_dataset/trainingsetmanipulation.py +++ b/deeplabcut/generate_training_dataset/trainingsetmanipulation.py @@ -12,8 +12,6 @@ import logging import math -import os -import os.path import warnings from functools import cache from pathlib import Path @@ -36,7 +34,7 @@ from deeplabcut.utils.auxfun_videos import VideoReader -def comparevideolistsanddatafolders(config): +def comparevideolistsanddatafolders(config: str | Path): """Auxiliary function that compares the folders in labeled-data and the ones listed under video_sets (in the config file). @@ -48,7 +46,7 @@ def comparevideolistsanddatafolders(config): cfg = auxiliaryfunctions.read_config(config) videos = cfg["video_sets"].keys() video_names = [Path(i).stem for i in videos] - alldatafolders = [fn for fn in os.listdir(Path(config).parent / "labeled-data") if "_labeled" not in fn] + alldatafolders = [f.name for f in (Path(config).parent / "labeled-data").iterdir() if "_labeled" not in f.name] print("Config file contains:", len(video_names)) print("Labeled-data contains:", len(alldatafolders)) @@ -62,7 +60,7 @@ def comparevideolistsanddatafolders(config): print(vn, " is missing in config file!") -def adddatasetstovideolistandviceversa(config): +def adddatasetstovideolistandviceversa(config: str | Path): """First run comparevideolistsanddatafolders(config) to compare the folders in labeled-data and the ones listed under video_sets (in the config file). If you detect differences this function can be used to maker sure each folder has a video @@ -86,8 +84,9 @@ def adddatasetstovideolistandviceversa(config): videos = cfg["video_sets"] video_names = [Path(i).stem for i in videos] + labeled_data_dir = Path(config).parent / "labeled-data" alldatafolders = [ - fn for fn in os.listdir(Path(config).parent / "labeled-data") if "_labeled" not in fn and not fn.startswith(".") + f.name for f in labeled_data_dir.iterdir() if "_labeled" not in f.name and not f.name.startswith(".") ] print("Config file contains:", len(video_names)) @@ -111,19 +110,19 @@ def adddatasetstovideolistandviceversa(config): print(vn, " is missing in config file >> adding it!") # Find the corresponding video file found = False - for file in os.listdir(os.path.join(cfg["project_path"], "videos")): - if os.path.splitext(file)[0] == vn: + for file in (Path(cfg["project_path"]) / "videos").iterdir(): + if file.stem == vn: found = True break if found: - video_path = os.path.join(cfg["project_path"], "videos", file) + video_path = str(file) clip = VideoReader(video_path) videos.update({video_path: {"crop": ", ".join(map(str, clip.get_bbox()))}}) auxiliaryfunctions.write_config(config, cfg) -def dropduplicatesinannotatinfiles(config): +def dropduplicatesinannotatinfiles(config: str | Path): """Drop duplicate entries (of images) in annotation files (this should no longer happen, but might be useful). @@ -139,20 +138,20 @@ def dropduplicatesinannotatinfiles(config): for folder in folders: try: - fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5") + fn = folder / ("CollectedData_" + cfg["scorer"] + ".h5") DC = pd.read_hdf(fn) numimages = len(DC.index) DC = DC[~DC.index.duplicated(keep="first")] if len(DC.index) < numimages: print("Dropped", numimages - len(DC.index)) DC.to_hdf(fn, key="df_with_missing", mode="w") - DC.to_csv(os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".csv")) + DC.to_csv(folder / ("CollectedData_" + cfg["scorer"] + ".csv")) except FileNotFoundError: print("Attention:", folder, "does not appear to have labeled data!") -def dropannotationfileentriesduetodeletedimages(config): +def dropannotationfileentriesduetodeletedimages(config: str | Path): """Drop entries for all deleted images in annotation files, i.e. for folders of the type: /labeled-data/*folder*/CollectedData_*scorer*.h5 Will be carried out iteratively for all *folders* in labeled-data. @@ -168,7 +167,7 @@ def dropannotationfileentriesduetodeletedimages(config): folders = [Path(config).parent / "labeled-data" / Path(i) for i in video_names] for folder in folders: - fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5") + fn = folder / ("CollectedData_" + cfg["scorer"] + ".h5") try: DC = pd.read_hdf(fn) except FileNotFoundError: @@ -176,7 +175,7 @@ def dropannotationfileentriesduetodeletedimages(config): continue dropped = False for imagename in DC.index: - if os.path.isfile(os.path.join(cfg["project_path"], *imagename)): + if Path(cfg["project_path"]).joinpath(*imagename).is_file(): pass else: print("Dropping...", imagename) @@ -184,10 +183,10 @@ def dropannotationfileentriesduetodeletedimages(config): dropped = True if dropped: DC.to_hdf(fn, key="df_with_missing", mode="w") - DC.to_csv(os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".csv")) + DC.to_csv(folder / ("CollectedData_" + cfg["scorer"] + ".csv")) -def dropimagesduetolackofannotation(config): +def dropimagesduetolackofannotation(config: str | Path): """ Drop images from corresponding folder for not annotated images: /labeled-data/*folder*/CollectedData_*scorer*.h5 Will be carried out iteratively for all *folders* in labeled-data. @@ -203,7 +202,7 @@ def dropimagesduetolackofannotation(config): folders = [Path(config).parent / "labeled-data" / Path(i) for i in video_names] for folder in folders: - h5file = os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5") + h5file = folder / ("CollectedData_" + cfg["scorer"] + ".h5") try: DC = pd.read_hdf(h5file) except FileNotFoundError: @@ -211,19 +210,19 @@ def dropimagesduetolackofannotation(config): continue conversioncode.guarantee_multiindex_rows(DC) annotatedimages = [fn[-1] for fn in DC.index] - imagelist = [fns for fns in os.listdir(str(folder)) if ".png" in fns] + imagelist = [f.name for f in folder.iterdir() if ".png" in f.name] print("Annotated images: ", len(annotatedimages), " In folder:", len(imagelist)) for imagename in imagelist: if imagename in annotatedimages: pass else: - fullpath = os.path.join(cfg["project_path"], "labeled-data", folder, imagename) - if os.path.isfile(fullpath): + fullpath = folder / imagename + if fullpath.is_file(): print("Deleting", fullpath) - os.remove(fullpath) + fullpath.unlink() annotatedimages = [fn[-1] for fn in DC.index] - imagelist = [fns for fns in os.listdir(str(folder)) if ".png" in fns] + imagelist = [f.name for f in folder.iterdir() if ".png" in f.name] print( "PROCESSED:", folder, @@ -234,7 +233,7 @@ def dropimagesduetolackofannotation(config): ) -def dropunlabeledframes(config): +def dropunlabeledframes(config: str | Path): """Drop entries such that all the bodyparts are not labeled from the annotation files, i.e. h5 and csv files Will be carried out iteratively for all *folders* in labeled-data. @@ -250,7 +249,7 @@ def dropunlabeledframes(config): folders = [Path(config).parent / "labeled-data" / Path(i) for i in video_names] for folder in folders: - h5file = os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5") + h5file = folder / ("CollectedData_" + cfg["scorer"] + ".h5") try: DC = pd.read_hdf(h5file) except FileNotFoundError: @@ -262,7 +261,7 @@ def dropunlabeledframes(config): dropped = before_len - after_len if dropped: DC.to_hdf(h5file, key="df_with_missing", mode="w") - DC.to_csv(os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".csv")) + DC.to_csv(folder / ("CollectedData_" + cfg["scorer"] + ".csv")) print("Dropped ", dropped, "entries in ", folder) @@ -270,7 +269,7 @@ def dropunlabeledframes(config): def check_labels( - config, + config: str | Path, Labels=None, scale=1, dpi=100, @@ -327,11 +326,11 @@ def check_labels( videos = cfg["video_sets"].keys() video_names = [_robust_path_split(video)[1] for video in videos] - folders = [os.path.join(cfg["project_path"], "labeled-data", str(Path(i))) for i in video_names] + folders = [Path(cfg["project_path"]) / "labeled-data" / Path(i) for i in video_names] print("Creating images with labels by {}.".format(cfg["scorer"])) for folder in folders: try: - DataCombined = pd.read_hdf(os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5")) + DataCombined = pd.read_hdf(folder / ("CollectedData_" + cfg["scorer"] + ".h5")) conversioncode.guarantee_multiindex_rows(DataCombined) if cfg.get("multianimalproject", False): color_by = "individual" if visualizeindividuals else "bodypart" @@ -362,7 +361,7 @@ def boxitintoacell(joints): def ParseYaml(configfile): - raw = open(configfile).read() + raw = Path(configfile).open().read() docs = [] for raw_doc in raw.split("\n---"): try: @@ -391,7 +390,7 @@ def MakeTrain_pose_yaml( docs[0][key] = itemstochange[key] if save: - with open(saveasconfigfile, "w") as f: + with Path(saveasconfigfile).open("w") as f: yaml.dump(docs[0], f) return docs[0] @@ -421,7 +420,7 @@ def MakeTest_pose_yaml( dict_test["locref_smooth"] = locref_smooth dict_test["scoremap_dir"] = "test" - with open(saveasfile, "w") as f: + with Path(saveasfile).open("w") as f: yaml.dump(dict_test, f) @@ -430,7 +429,7 @@ def MakeInference_yaml(itemstochange, saveasconfigfile, defaultconfigfile): for key in itemstochange.keys(): docs[0][key] = itemstochange[key] - with open(saveasconfigfile, "w") as f: + with Path(saveasconfigfile).open("w") as f: yaml.dump(docs[0], f) return docs[0] @@ -445,7 +444,7 @@ def _robust_path_split(path): parent, file = splits else: raise (f"Unknown filepath split for path {path}") - filename, ext = os.path.splitext(file) + filename, ext = Path(file).stem, Path(file).suffix return parent, filename, ext @@ -527,11 +526,11 @@ def merge_annotateddatasets(cfg, trainingsetfolder_full): But if someone labels on windows and wants to train on a unix cluster or colab... """ AnnotationData = [] - data_path = Path(os.path.join(cfg["project_path"], "labeled-data")) + data_path = Path(cfg["project_path"]) / "labeled-data" videos = cfg["video_sets"].keys() video_filenames = parse_video_filenames(videos) for filename in video_filenames: - file_path = os.path.join(data_path / filename, f"CollectedData_{cfg['scorer']}.h5") + file_path = data_path / filename / f"CollectedData_{cfg['scorer']}.h5" try: data = pd.read_hdf(file_path) conversioncode.guarantee_multiindex_rows(data) @@ -582,9 +581,9 @@ def merge_annotateddatasets(cfg, trainingsetfolder_full): "Hint: are bodyparts correctly listed in the configuration?" ) - filename = os.path.join(trainingsetfolder_full, f"CollectedData_{cfg['scorer']}") - AnnotationData.to_hdf(filename + ".h5", key="df_with_missing", mode="w") - AnnotationData.to_csv(filename + ".csv") # human readable. + filename = trainingsetfolder_full / f"CollectedData_{cfg['scorer']}" + AnnotationData.to_hdf(str(filename) + ".h5", key="df_with_missing", mode="w") + AnnotationData.to_csv(str(filename) + ".csv") # human readable. return AnnotationData @@ -652,7 +651,7 @@ def pad_train_test_indices(train_inds, test_inds, train_fraction): return train_inds, test_inds -def mergeandsplit(config, trainindex=0, uniform=True): +def mergeandsplit(config: str | Path, trainindex=0, uniform=True): """This function allows additional control over "create_training_dataset". Merge annotated data sets (from different folders) and split data in a specific way, @@ -702,8 +701,8 @@ def mergeandsplit(config, trainindex=0, uniform=True): project_path = cfg["project_path"] # Create path for training sets & store data there trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) # Path concatenation OS platform independent - auxiliaryfunctions.attempt_to_make_folder(Path(os.path.join(project_path, str(trainingsetfolder))), recursive=True) - fn = os.path.join(project_path, trainingsetfolder, "CollectedData_" + cfg["scorer"]) + auxiliaryfunctions.attempt_to_make_folder(Path(project_path) / str(trainingsetfolder), recursive=True) + fn = str(Path(project_path) / trainingsetfolder / ("CollectedData_" + cfg["scorer"])) try: data = pd.read_hdf(fn + ".h5") @@ -711,7 +710,7 @@ def mergeandsplit(config, trainindex=0, uniform=True): except FileNotFoundError: data = merge_annotateddatasets( cfg, - Path(os.path.join(project_path, trainingsetfolder)), + Path(project_path) / trainingsetfolder, ) if data is None: return [], [] @@ -778,7 +777,7 @@ def to_matlab_cell(array): data = dict() filename = df.index[i] data["image"] = filename - img_shape = read_image_shape_fast(os.path.join(project_path, *filename)) + img_shape = read_image_shape_fast(Path(project_path).joinpath(*filename)) data["size"] = img_shape row = df.iloc[i].values @@ -820,7 +819,7 @@ def to_matlab_cell(array): def create_training_dataset( - config, + config: str | Path, num_shuffles=1, Shuffles=None, windows2linux=False, @@ -1070,9 +1069,7 @@ def create_training_dataset( trainingsetfolder = auxiliaryfunctions.get_training_set_folder( cfg ) # Path concatenation OS platform independent - auxiliaryfunctions.attempt_to_make_folder( - Path(os.path.join(project_path, str(trainingsetfolder))), recursive=True - ) + auxiliaryfunctions.attempt_to_make_folder(Path(project_path) / str(trainingsetfolder), recursive=True) # Create the trainset metadata file, if it doesn't yet exist if not metadata.TrainingDatasetMetadata.path(cfg).exists(): @@ -1081,7 +1078,7 @@ def create_training_dataset( Data = merge_annotateddatasets( cfg, - Path(os.path.join(project_path, trainingsetfolder)), + Path(project_path) / trainingsetfolder, ) if Data is None: return @@ -1145,14 +1142,14 @@ def create_training_dataset( # Loading the encoder (if necessary downloading from TF) dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() if not posecfg_template: - defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") + defaultconfigfile = dlcparent_path / "pose_cfg.yaml" elif posecfg_template: defaultconfigfile = posecfg_template if engine == Engine.PYTORCH: model_path = dlcparent_path else: - model_path = auxfun_models.check_for_weights(net_type, Path(dlcparent_path)) + model_path = auxfun_models.check_for_weights(net_type, dlcparent_path) Shuffles = validate_shuffles(cfg, Shuffles, num_shuffles, userfeedback) @@ -1217,13 +1214,13 @@ def create_training_dataset( # Saving data file (convert to training file for deeper cut (*.mat)) ################################################################################ data, MatlabData = format_training_data(Data, trainIndices, nbodyparts, project_path) - sio.savemat(os.path.join(project_path, datafilename), {"dataset": MatlabData}) + sio.savemat(str(Path(project_path) / datafilename), {"dataset": MatlabData}) ################################################################################ # Saving metadata (Pickle file) ################################################################################ auxiliaryfunctions.save_metadata( - os.path.join(project_path, metadatafilename), + Path(project_path) / metadatafilename, data, trainIndices, testIndices, @@ -1253,22 +1250,8 @@ def create_training_dataset( auxiliaryfunctions.attempt_to_make_folder(str(Path(config).parents[0] / modelfoldername) + "/train") auxiliaryfunctions.attempt_to_make_folder(str(Path(config).parents[0] / modelfoldername) + "/test") - path_train_config = str( - os.path.join( - cfg["project_path"], - Path(modelfoldername), - "train", - engine.pose_cfg_name, - ) - ) - path_test_config = str( - os.path.join( - cfg["project_path"], - Path(modelfoldername), - "test", - "pose_cfg.yaml", - ) - ) + path_train_config = str(Path(cfg["project_path"]) / modelfoldername / "train" / engine.pose_cfg_name) + path_test_config = str(Path(cfg["project_path"]) / modelfoldername / "test" / "pose_cfg.yaml") if engine == Engine.TF: if weight_init is not None: raise ValueError( @@ -1361,7 +1344,7 @@ def create_training_dataset( return splits -def get_largestshuffle_index(config): +def get_largestshuffle_index(config: str | Path): """Returns the largest shuffle for all dlc-models in the current iteration.""" shuffle_indices = get_existing_shuffle_indices(config) if len(shuffle_indices) > 0: @@ -1470,7 +1453,7 @@ def validate_shuffles( def create_training_model_comparison( - config, + config: str | Path, trainindex=0, num_shuffles=1, net_types=None, @@ -1578,7 +1561,7 @@ def create_training_model_comparison( ) # create log file - log_file_name = os.path.join(cfg["project_path"], "training_model_comparison.log") + log_file_name = str(Path(cfg["project_path"]) / "training_model_comparison.log") logger = logging.getLogger("training_model_comparison") if not logger.handlers: logger = logging.getLogger("training_model_comparison") @@ -1636,7 +1619,7 @@ def create_training_model_comparison( def create_training_dataset_from_existing_split( - config: str, + config: str | Path, from_shuffle: int, from_trainsetindex: int = 0, num_shuffles: int = 1, diff --git a/deeplabcut/gui/__init__.py b/deeplabcut/gui/__init__.py index 484f13937..e4a425ff5 100644 --- a/deeplabcut/gui/__init__.py +++ b/deeplabcut/gui/__init__.py @@ -10,8 +10,9 @@ # import os +from pathlib import Path os.environ["QT_API"] = "pyside6" import qtpy # Necessary unused import to properly store the env variable -BASE_DIR = os.path.dirname(__file__) +BASE_DIR = Path(__file__).parent diff --git a/deeplabcut/gui/components.py b/deeplabcut/gui/components.py index cb3a74752..cc8fa3d37 100644 --- a/deeplabcut/gui/components.py +++ b/deeplabcut/gui/components.py @@ -10,7 +10,6 @@ # from __future__ import annotations -import os from pathlib import Path from PySide6 import QtWidgets @@ -355,7 +354,7 @@ def update_videos(self): ) if filenames[0]: - abs_files = [os.path.abspath(vid) for vid in filenames[0]] + abs_files = [str(Path(vid).resolve()) for vid in filenames[0]] self.root.add_video_files(abs_files) # Optional safety: sync dropdown to selected file suffix @@ -418,7 +417,7 @@ def _update_selected_snapshot_display(self): self.selected_snapshot_text.setText("") self.clear_snapshot_button.hide() else: - self.selected_snapshot_text.setText(f"{os.path.basename(self.selected_snapshot)}") + self.selected_snapshot_text.setText(f"{Path(self.selected_snapshot).name}") self.clear_snapshot_button.show() def select_snapshot(self): @@ -436,7 +435,7 @@ def select_snapshot(self): ) # When Canceling a file selection, Qt returns an empty string as selected file if selected_snapshot: - self.selected_snapshot = os.path.abspath(selected_snapshot) + self.selected_snapshot = str(Path(selected_snapshot).resolve()) self._update_selected_snapshot_display() @@ -525,7 +524,7 @@ def _is_model_bu(selected_conditions) -> bool: selected_conditions = None # When Canceling a file selection, Qt returns an empty string as selected file - self.selected_conditions = str(os.path.abspath(selected_conditions)) if selected_conditions else None + self.selected_conditions = str(Path(selected_conditions).resolve()) if selected_conditions else None self._update_selected_conditions_display() @@ -672,9 +671,8 @@ def _create_message_box(text, info_text): msg.setWindowTitle("Info") msg.setMinimumWidth(900) - logo_dir = os.path.dirname(os.path.realpath("logo.png")) + os.path.sep - logo = logo_dir + "/assets/logo.png" - msg.setWindowIcon(QIcon(logo)) + logo = Path("logo.png").resolve().parent / "assets" / "logo.png" + msg.setWindowIcon(QIcon(str(logo))) msg.setStandardButtons(QtWidgets.QMessageBox.Ok) return msg @@ -687,9 +685,8 @@ def _create_confirmation_box(title, description): msg.setWindowTitle("Confirmation") msg.setMinimumWidth(900) - logo_dir = os.path.dirname(os.path.realpath("logo.png")) + os.path.sep - logo = logo_dir + "/assets/logo.png" - msg.setWindowIcon(QIcon(logo)) + logo = Path("logo.png").resolve().parent / "assets" / "logo.png" + msg.setWindowIcon(QIcon(str(logo))) msg.setStandardButtons(QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) return msg diff --git a/deeplabcut/gui/displays/selected_shuffle_display.py b/deeplabcut/gui/displays/selected_shuffle_display.py index d1ae3732b..b0d3a2a1a 100644 --- a/deeplabcut/gui/displays/selected_shuffle_display.py +++ b/deeplabcut/gui/displays/selected_shuffle_display.py @@ -117,7 +117,7 @@ def _set_text_error(self, error: str) -> None: self.pose_cfg = None def _read_pose_config(self, pose_cfg_path: Path) -> None: - pose_cfg = auxiliaryfunctions.read_plainconfig(str(pose_cfg_path)) + pose_cfg = auxiliaryfunctions.read_plainconfig(pose_cfg_path) self._engine = Engine.PYTORCH if "pytorch" in pose_cfg_path.stem.lower() else Engine.TF self._net_type = pose_cfg.get("net_type", "UNKNOWN") diff --git a/deeplabcut/gui/displays/shuffle_metadata_viewer.py b/deeplabcut/gui/displays/shuffle_metadata_viewer.py index ec44d5a72..084d91264 100644 --- a/deeplabcut/gui/displays/shuffle_metadata_viewer.py +++ b/deeplabcut/gui/displays/shuffle_metadata_viewer.py @@ -12,6 +12,8 @@ from __future__ import annotations +from pathlib import Path + from PySide6 import QtWidgets from PySide6.QtCore import Qt @@ -57,7 +59,7 @@ def _load_metadata(cfg: dict) -> list[str]: trainset_meta = metadata.TrainingDatasetMetadata.create(cfg) trainset_meta.save() - with open(metadata_path) as file: + with Path(metadata_path).open() as file: raw_metadata = file.read() return raw_metadata.split("\n") diff --git a/deeplabcut/gui/launch_script.py b/deeplabcut/gui/launch_script.py index 698070c0e..d30df067c 100644 --- a/deeplabcut/gui/launch_script.py +++ b/deeplabcut/gui/launch_script.py @@ -19,7 +19,6 @@ """ -import os import sys import PySide6.QtWidgets as QtWidgets @@ -32,16 +31,16 @@ def launch_dlc(): app = QtWidgets.QApplication(sys.argv) - app.setWindowIcon(QIcon(os.path.join(BASE_DIR, "assets", "logo.png"))) + app.setWindowIcon(QIcon(str(BASE_DIR / "assets" / "logo.png"))) screen_size = app.screens()[0].size() - pixmap = QPixmap(os.path.join(BASE_DIR, "assets", "welcome.png")).scaledToWidth( + pixmap = QPixmap(str(BASE_DIR / "assets" / "welcome.png")).scaledToWidth( int(0.7 * screen_size.width()), Qt.SmoothTransformation ) splash = QtWidgets.QSplashScreen(pixmap) splash.show() - stylefile = os.path.join(BASE_DIR, "style.qss") - with open(stylefile) as f: + stylefile = BASE_DIR / "style.qss" + with stylefile.open() as f: app.setStyleSheet(f.read()) dark_stylesheet = qdarkstyle.load_stylesheet_pyside2() diff --git a/deeplabcut/gui/tabs/create_project.py b/deeplabcut/gui/tabs/create_project.py index 615806df2..878888b8b 100644 --- a/deeplabcut/gui/tabs/create_project.py +++ b/deeplabcut/gui/tabs/create_project.py @@ -8,8 +8,8 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os from datetime import datetime +from pathlib import Path from PySide6 import QtCore, QtWidgets from PySide6.QtGui import QBrush, QColor, QDesktopServices, QIcon, QPainter, QPen @@ -235,7 +235,7 @@ def lay_out_user_frame(self): self.loc_line = QtWidgets.QLineEdit(self.loc_default, user_frame) self.loc_line.setReadOnly(True) action = self.loc_line.addAction( - QIcon(os.path.join(BASE_DIR, "assets", "icons", "open2.png")), + QIcon(str(BASE_DIR / "assets" / "icons" / "open2.png")), QtWidgets.QLineEdit.TrailingPosition, ) action.triggered.connect(self.on_click) @@ -437,7 +437,7 @@ def browse_videos(self): folder, relative=False, ): - if os.path.splitext(video)[1][1:].lower() in DLCParams.VIDEOTYPES[1:]: + if Path(video).suffix[1:].lower() in DLCParams.VIDEOTYPES[1:]: self.video_frame.fancy_list.add_item(video) def finalize_project(self): @@ -535,5 +535,5 @@ def update_experimenter_name(self, text): def update_project_location(self): full_name = self.name_default.format(self.proj_default, self.exp_default) - full_path = os.path.join(self.loc_default, full_name) + full_path = str(Path(self.loc_default) / full_name) self.loc_line.setText(full_path) diff --git a/deeplabcut/gui/tabs/create_training_dataset.py b/deeplabcut/gui/tabs/create_training_dataset.py index ff3a35e45..01382e160 100644 --- a/deeplabcut/gui/tabs/create_training_dataset.py +++ b/deeplabcut/gui/tabs/create_training_dataset.py @@ -10,7 +10,6 @@ # from __future__ import annotations -import os import re from importlib import import_module from pathlib import Path @@ -200,7 +199,7 @@ def edit_conversion_table(self): memory_replay_folder = Path(self.root.project_folder) / "memory_replay" conversion_matrix_out_path = str(memory_replay_folder / "confusion_matrix.png") files = [self.root.config] - if os.path.exists(conversion_matrix_out_path): + if Path(conversion_matrix_out_path).exists(): files.append(conversion_matrix_out_path) _ = launch_napari(files) @@ -339,8 +338,8 @@ def create_training_dataset(self): ) ) if self.root.is_multianimal: - filenames[0] = filenames[0].replace("mat", "pickle") - if all(os.path.exists(os.path.join(self.root.project_folder, file)) for file in filenames): + filenames[0] = filenames[0].with_suffix(".pickle") + if all((Path(self.root.project_folder) / file).exists() for file in filenames): self.root.shuffle_created.emit(self.shuffle.value()) msg = _create_message_box( "The training dataset is successfully created.", diff --git a/deeplabcut/gui/tabs/evaluate_network.py b/deeplabcut/gui/tabs/evaluate_network.py index 4eb31b63e..8d3c05534 100644 --- a/deeplabcut/gui/tabs/evaluate_network.py +++ b/deeplabcut/gui/tabs/evaluate_network.py @@ -10,7 +10,6 @@ # from __future__ import annotations -import os from pathlib import Path import matplotlib.image as mpimg @@ -130,12 +129,14 @@ def plot_maps(self): deeplabcut.extract_save_all_maps(config, shuffle=shuffle, Indices=[0, 1, 2]) # Display all images - dest_folder = os.path.join( - self.root.project_folder, - str(auxiliaryfunctions.get_evaluation_folder(self.root.cfg["TrainingFraction"][0], shuffle, self.root.cfg)), - "maps", + dest_folder = ( + Path(self.root.project_folder) + / str( + auxiliaryfunctions.get_evaluation_folder(self.root.cfg["TrainingFraction"][0], shuffle, self.root.cfg) + ) + / "maps" ) - image_paths = [os.path.join(dest_folder, file) for file in os.listdir(dest_folder) if file.endswith(".png")] + image_paths = [str(p) for p in Path(dest_folder).iterdir() if p.name.endswith(".png")] canvas = GridCanvas(image_paths, parent=self) canvas.show() diff --git a/deeplabcut/gui/tabs/label_frames.py b/deeplabcut/gui/tabs/label_frames.py index 509b04fcd..72b105a2a 100644 --- a/deeplabcut/gui/tabs/label_frames.py +++ b/deeplabcut/gui/tabs/label_frames.py @@ -10,7 +10,6 @@ # from __future__ import annotations -import os from pathlib import Path from PySide6 import QtWidgets @@ -22,7 +21,7 @@ from deeplabcut.utils.skeleton import SkeletonBuilder -def label_frames(config_path: str | Path | None = None, image_folder: str | None = None): +def label_frames(config_path: str | Path | None = None, image_folder: str | Path | None = None): """Launches the napari-deeplabcut labelling GUI. For more information on labelling data with napari-deeplabcut, see our docs: @@ -122,12 +121,12 @@ def label_frames(self): dialog = QtWidgets.QFileDialog(self) dialog.setFileMode(QtWidgets.QFileDialog.Directory) dialog.setViewMode(QtWidgets.QFileDialog.Detail) - dialog.setDirectory(os.path.join(os.path.dirname(self.root.config), "labeled-data")) + dialog.setDirectory(str(Path(self.root.config).parent / "labeled-data")) if dialog.exec_(): folder = dialog.selectedFiles()[0] has_h5 = False - for file in os.listdir(folder): - if file.endswith(".h5"): + for file in Path(folder).iterdir(): + if file.name.endswith(".h5"): has_h5 = True break if not has_h5: diff --git a/deeplabcut/gui/tabs/manage_project.py b/deeplabcut/gui/tabs/manage_project.py index b4d05e7d2..5003c2173 100644 --- a/deeplabcut/gui/tabs/manage_project.py +++ b/deeplabcut/gui/tabs/manage_project.py @@ -8,7 +8,7 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os +from pathlib import Path from PySide6.QtCore import Qt from PySide6.QtWidgets import ( @@ -65,7 +65,7 @@ def open_config_editor(self): editor.show() def add_new_videos(self): - cwd = os.getcwd() + cwd = str(Path.cwd()) files = QFileDialog.getOpenFileNames( self, "Select videos to add to the project", diff --git a/deeplabcut/gui/tabs/modelzoo.py b/deeplabcut/gui/tabs/modelzoo.py index b40252354..1ee07772e 100644 --- a/deeplabcut/gui/tabs/modelzoo.py +++ b/deeplabcut/gui/tabs/modelzoo.py @@ -8,7 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os import webbrowser from functools import partial from pathlib import Path @@ -213,7 +212,7 @@ def _add_output_settings_section(self, layout: QtWidgets.QGridLayout): ) self.loc_line.setReadOnly(True) action = self.loc_line.addAction( - QIcon(os.path.join(BASE_DIR, "assets", "icons", "open2.png")), + QIcon(str(BASE_DIR / "assets" / "icons" / "open2.png")), QtWidgets.QLineEdit.TrailingPosition, ) action.triggered.connect(self.select_folder) @@ -251,7 +250,7 @@ def _add_tf_scales_row(self, layout: QtWidgets.QGridLayout): validator.validationChanged.connect(self._handle_validation_change) self.scales_line.setValidator(validator) tooltip_label = QtWidgets.QLabel() - tooltip_label.setPixmap(QPixmap(os.path.join(BASE_DIR, "assets", "icons", "help2.png")).scaledToWidth(30)) + tooltip_label.setPixmap(QPixmap(str(BASE_DIR / "assets" / "icons" / "help2.png")).scaledToWidth(30)) tooltip_label.setToolTip( "Approximate animal sizes in pixels, for spatial pyramid search. If left " "blank, defaults to video height +/- 50 pixels" @@ -269,7 +268,7 @@ def _add_use_adaptation_row(self, layout: QtWidgets.QGridLayout, layout_row: int self.adapt_checkbox.setStyleSheet("font-weight: bold; font-size: 16px; padding: 6px 12px;") # Add help button adapt_help_btn = QtWidgets.QToolButton() - adapt_help_btn.setIcon(QIcon(os.path.join(BASE_DIR, "assets", "icons", "help2.png"))) + adapt_help_btn.setIcon(QIcon(str(BASE_DIR / "assets" / "icons" / "help2.png"))) adapt_help_btn.setIconSize(QSize(24, 24)) adapt_help_btn.setToolTip("What is video adaptation?") diff --git a/deeplabcut/gui/tabs/open_project.py b/deeplabcut/gui/tabs/open_project.py index c21cfc78c..4ff9b2b10 100644 --- a/deeplabcut/gui/tabs/open_project.py +++ b/deeplabcut/gui/tabs/open_project.py @@ -8,7 +8,7 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os +from pathlib import Path from PySide6 import QtCore, QtWidgets from PySide6.QtGui import QIcon @@ -58,7 +58,7 @@ def open_config_name(self): self.open_line.text() def load_config(self): - cwd = os.getcwd() + cwd = str(Path.cwd()) config = QtWidgets.QFileDialog.getOpenFileName( self, "Select a configuration file", cwd, "Config files (*.yaml)" ) @@ -76,7 +76,7 @@ def open_project(self): msg.setWindowTitle("Error") msg.setMinimumWidth(400) - self.logo_dir = os.path.dirname(os.path.realpath("logo.png")) + os.path.sep + self.logo_dir = str(Path("logo.png").resolve().parent) + "/" self.logo = self.logo_dir + "/assets/logo.png" msg.setWindowIcon(QIcon(self.logo)) msg.setStandardButtons(QtWidgets.QMessageBox.Ok) @@ -84,7 +84,7 @@ def open_project(self): self.loaded = False else: - self.logo_dir = os.path.dirname(os.path.realpath("logo.png")) + os.path.sep + self.logo_dir = str(Path("logo.png").resolve().parent) + "/" self.logo = self.logo_dir + "/assets/logo.png" self.loaded = True diff --git a/deeplabcut/gui/tabs/refine_tracklets.py b/deeplabcut/gui/tabs/refine_tracklets.py index 1e841f8b4..cabad6b9b 100644 --- a/deeplabcut/gui/tabs/refine_tracklets.py +++ b/deeplabcut/gui/tabs/refine_tracklets.py @@ -8,7 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os from pathlib import Path from PySide6 import QtWidgets @@ -248,9 +247,9 @@ def refine_tracks(self): video = list(self.files)[0] track_method = cfg.get("default_track_method", "ellipse") method = trackingutils.TRACK_METHODS[track_method] - dest = str(Path(video).parents[0]) + dest = Path(video).parents[0] vname = Path(video).stem - datafile = os.path.join(dest, vname + DLCscorer + f"{method}.h5") + datafile = str(dest / (vname + DLCscorer + f"{method}.h5")) self.manager, self.viz = deeplabcut.refine_tracklets( self.root.config, datafile, diff --git a/deeplabcut/gui/tabs/train_network.py b/deeplabcut/gui/tabs/train_network.py index c7b30595a..7298ab91a 100644 --- a/deeplabcut/gui/tabs/train_network.py +++ b/deeplabcut/gui/tabs/train_network.py @@ -10,8 +10,8 @@ # from __future__ import annotations -import os from dataclasses import dataclass +from pathlib import Path from PySide6 import QtWidgets from PySide6.QtCore import Qt, Slot @@ -244,7 +244,7 @@ def train_network(self): msg.setWindowTitle("Info") msg.setMinimumWidth(900) - self.logo_dir = os.path.dirname(os.path.realpath("logo.png")) + os.path.sep + self.logo_dir = str(Path("logo.png").resolve().parent) + "/" self.logo = self.logo_dir + "/assets/logo.png" msg.setWindowIcon(QIcon(self.logo)) msg.setStandardButtons(QtWidgets.QMessageBox.Ok) diff --git a/deeplabcut/gui/tracklet_toolbox.py b/deeplabcut/gui/tracklet_toolbox.py index 295b853dd..c1e11e164 100644 --- a/deeplabcut/gui/tracklet_toolbox.py +++ b/deeplabcut/gui/tracklet_toolbox.py @@ -8,6 +8,7 @@ # # Licensed under GNU Lesser General Public License v3.0 # +from pathlib import Path as PPath from threading import Event import matplotlib.patches as patches @@ -835,7 +836,6 @@ def save(self, *args): self.manager.save() def export_to_training_data(self, pcutoff=0.1): - import os from skimage import io @@ -846,20 +846,20 @@ def export_to_training_data(self, pcutoff=0.1): # Save additional frames to the labeled-data directory strwidth = int(np.ceil(np.log10(self.nframes))) - tmpfolder = os.path.join(self.manager.cfg["project_path"], "labeled-data", self.video.name) - if os.path.isdir(tmpfolder): + tmpfolder = PPath(self.manager.cfg["project_path"]) / "labeled-data" / self.video.name + if tmpfolder.is_dir(): print( "Frames from video", self.video.name, " already extracted (more will be added)!", ) else: - attempt_to_make_folder(tmpfolder) + attempt_to_make_folder(str(tmpfolder)) index = [] for ind in inds: - imagename = os.path.join(tmpfolder, "img" + str(ind).zfill(strwidth) + ".png") - index.append(tuple((os.path.join(*imagename.rsplit(os.path.sep, 3)[-3:])).split("\\"))) - if not os.path.isfile(imagename): + imagename = tmpfolder / ("img" + str(ind).zfill(strwidth) + ".png") + index.append(tuple(PPath(*PPath(imagename).parts[-3:]).as_posix().split("/"))) + if not imagename.is_file(): self.video.set_to_frame(ind) frame = self.video.read_frame() if frame is None: @@ -869,7 +869,7 @@ def export_to_training_data(self, pcutoff=0.1): if self.manager.cfg["cropping"]: x1, x2, y1, y2 = [int(self.manager.cfg[key]) for key in ("x1", "x2", "y1", "y2")] frame = frame[y1:y2, x1:x2] - io.imsave(imagename, frame) + io.imsave(str(imagename), frame) # Store the newly-refined data data = self.manager.format_data() @@ -884,22 +884,22 @@ def filter_low_prob(cols, prob): df = df.groupby(level="bodyparts", axis=1, group_keys=False).apply(filter_low_prob, prob=pcutoff) df.index = pd.MultiIndex.from_tuples(index) - machinefile = os.path.join(tmpfolder, "machinelabels-iter" + str(self.manager.cfg["iteration"]) + ".h5") - if os.path.isfile(machinefile): + machinefile = tmpfolder / ("machinelabels-iter" + str(self.manager.cfg["iteration"]) + ".h5") + if machinefile.is_file(): df_old = pd.read_hdf(machinefile) df_joint = pd.concat([df_old, df]) df_joint = df_joint[~df_joint.index.duplicated(keep="first")] df_joint.to_hdf(machinefile, key="df_with_missing", mode="w") - df_joint.to_csv(os.path.join(tmpfolder, "machinelabels.csv")) + df_joint.to_csv(tmpfolder / "machinelabels.csv") else: df.to_hdf(machinefile, key="df_with_missing", mode="w") - df.to_csv(os.path.join(tmpfolder, "machinelabels.csv")) + df.to_csv(tmpfolder / "machinelabels.csv") # Merge with the already existing annotated data df.columns = df.columns.set_levels([self.manager.cfg["scorer"]], level="scorer") df.drop("likelihood", level="coords", axis=1, inplace=True) - output_path = os.path.join(tmpfolder, f"CollectedData_{self.manager.cfg['scorer']}.h5") - if os.path.isfile(output_path): + output_path = tmpfolder / f"CollectedData_{self.manager.cfg['scorer']}.h5" + if output_path.is_file(): print( "A training dataset file is already found for this video. The refined machine labels are merged to this" "data!" @@ -911,11 +911,11 @@ def filter_low_prob(cols, prob): df_joint = df_joint[~df_joint.index.duplicated(keep="first")] df_joint.sort_index(inplace=True) df_joint.to_hdf(output_path, key="df_with_missing", mode="w") - df_joint.to_csv(output_path.replace("h5", "csv")) + df_joint.to_csv(output_path.with_suffix(".csv")) else: df.sort_index(inplace=True) df.to_hdf(output_path, key="df_with_missing", mode="w") - df.to_csv(output_path.replace("h5", "csv")) + df.to_csv(output_path.with_suffix(".csv")) def refine_tracklets( diff --git a/deeplabcut/gui/widgets.py b/deeplabcut/gui/widgets.py index f8a1e4c93..a44ca5b53 100644 --- a/deeplabcut/gui/widgets.py +++ b/deeplabcut/gui/widgets.py @@ -10,6 +10,7 @@ # import ast import os +from pathlib import Path from queue import Queue import napari @@ -107,13 +108,13 @@ def dragEnterEvent(self, event): def dropEvent(self, event): for url in event.mimeData().urls(): path = url.toLocalFile() - if os.path.isfile(path): + if Path(path).is_file(): self.add_item(path) - elif os.path.isdir(path): + elif Path(path).is_dir(): for root, _, files in os.walk(path): for file in files: if not file.startswith("."): - self.add_item(os.path.join(root, file)) + self.add_item(str(Path(root) / file)) class ItemSelectionFrame(QtWidgets.QFrame): @@ -275,7 +276,7 @@ def create_item(self): creator.created.connect(self.parent.insert) def fix_path(self): - self.current_item.setText(1, os.path.split(self.parent.filename)[0]) + self.current_item.setText(1, str(Path(self.parent.filename).parent)) class DictViewer(QtWidgets.QWidget): @@ -353,7 +354,7 @@ def cast_to_right_type(val): pass except SyntaxError: # Slashes also raise the error, but no need to print anything since it is then likely to be a path - if os.path.sep not in val: + if "/" not in val and "\\" not in val: print("Consider removing leading zeros or spaces in the string.") return val diff --git a/deeplabcut/gui/window.py b/deeplabcut/gui/window.py index e5b8fb466..2075720dd 100644 --- a/deeplabcut/gui/window.py +++ b/deeplabcut/gui/window.py @@ -9,7 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # import logging -import os import sys import warnings from functools import cached_property @@ -214,7 +213,7 @@ def engine(self, e: Engine) -> None: msg.setWindowTitle("Info") msg.setMinimumWidth(900) - logo_dir = os.path.dirname(os.path.realpath("logo.png")) + os.path.sep + logo_dir = str(Path("logo.png").resolve().parent) + "/" logo = logo_dir + "/assets/logo.png" msg.setWindowIcon(QIcon(logo)) msg.setStandardButtons(QtWidgets.QMessageBox.Ok) @@ -225,7 +224,7 @@ def engine(self, e: Engine) -> None: @property def project_folder(self) -> str: - return self.cfg.get("project_path", os.path.expanduser("~/Desktop")) + return self.cfg.get("project_path", str(Path("~/Desktop").expanduser())) @property def is_multianimal(self) -> bool: @@ -275,15 +274,15 @@ def models_folder(self) -> str: @property def inference_cfg_path(self) -> str: - return os.path.join( - self.cfg["project_path"], - auxiliaryfunctions.get_model_folder( + return str( + Path(self.cfg["project_path"]) + / auxiliaryfunctions.get_model_folder( self.cfg["TrainingFraction"][int(self.trainingset_index)], int(self.shuffle_value), self.cfg, - ), - "test", - "inference_cfg.yaml", + ) + / "test" + / "inference_cfg.yaml" ) def update_cfg(self, text): @@ -512,7 +511,7 @@ def window_set(self): palette.setColor(QtGui.QPalette.Window, QtGui.QColor("#ffffff")) self.setPalette(palette) - icon = os.path.join(BASE_DIR, "assets", "logo.png") + icon = str(BASE_DIR / "assets" / "logo.png") self.setWindowIcon(QIcon(icon)) # Set default window size and allow resizing @@ -546,7 +545,7 @@ def _generate_welcome_page(self): image_widget = QtWidgets.QLabel(self) image_widget.setAlignment(Qt.AlignCenter) image_widget.setContentsMargins(0, 0, 0, 0) - logo = os.path.join(BASE_DIR, "assets", "logo_transparent.png") + logo = str(BASE_DIR / "assets" / "logo_transparent.png") pixmap = QtGui.QPixmap(logo) image_widget.setPixmap(pixmap.scaledToHeight(400, QtCore.Qt.SmoothTransformation)) self.layout.addWidget(image_widget) @@ -603,7 +602,7 @@ def create_actions(self, names): self.newAction = QAction(self) self.newAction.setText("&New Project...") - self.newAction.setIcon(QIcon(os.path.join(BASE_DIR, "assets", "icons", names[0]))) + self.newAction.setIcon(QIcon(str(BASE_DIR / "assets" / "icons" / names[0]))) self.newAction.setShortcut("Ctrl+N") self.newAction.setStatusTip("Create a new project...") @@ -611,7 +610,7 @@ def create_actions(self, names): # Creating actions using the second constructor self.openAction = QAction("&Open...", self) - self.openAction.setIcon(QIcon(os.path.join(BASE_DIR, "assets", "icons", names[1]))) + self.openAction.setIcon(QIcon(str(BASE_DIR / "assets" / "icons" / names[1]))) self.openAction.setShortcut("Ctrl+O") self.openAction.setStatusTip("Open a project...") self.openAction.triggered.connect(self._open_project) @@ -625,7 +624,7 @@ def create_actions(self, names): self.darkmodeAction.triggered.connect(self.darkmode) self.helpAction = QAction("&Help", self) - self.helpAction.setIcon(QIcon(os.path.join(BASE_DIR, "assets", "icons", names[2]))) + self.helpAction.setIcon(QIcon(str(BASE_DIR / "assets" / "icons" / names[2]))) self.helpAction.setStatusTip("Ask for help...") self.helpAction.triggered.connect(self._ask_for_help) diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/base.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/base.py index cde1d0716..815eaacc0 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/base.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/base.py @@ -9,8 +9,8 @@ # Licensed under GNU Lesser General Public License v3.0 # import copy -import os import warnings +from pathlib import Path import numpy as np @@ -30,7 +30,7 @@ def raw_2_imagename_with_id(image): """ file_name = image["file_name"] - image_name = file_name.split(os.sep)[-1] + image_name = Path(file_name).name pre, suffix = image_name.split(".") image_id = image["id"] return f"{pre}_{image_id}.{suffix}" @@ -40,7 +40,7 @@ def raw_2_imagename(image): """Only getting the imagename part from the image object.""" file_name = image["file_name"] - image_name = file_name.split(os.sep)[-1] + image_name = Path(file_name).name return image_name diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/base_dlc.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/base_dlc.py index bdae87671..20ff07c30 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/base_dlc.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/base_dlc.py @@ -8,8 +8,8 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os import pickle +from pathlib import Path import numpy as np import pandas as pd @@ -32,9 +32,9 @@ def __init__(self, proj_root, dataset_name, shuffle=1, modelprefix=""): self.proj_root = proj_root if modelprefix: - config_file = os.path.join(self.proj_root, modelprefix + "_config.yaml") + config_file = Path(self.proj_root) / (modelprefix + "_config.yaml") else: - config_file = os.path.join(self.proj_root, "config.yaml") + config_file = Path(self.proj_root) / "config.yaml" cfg = auxiliaryfunctions.read_config(config_file) @@ -42,27 +42,21 @@ def __init__(self, proj_root, dataset_name, shuffle=1, modelprefix=""): scorer = cfg["scorer"] - datasets_folder = os.path.join( - self.proj_root, - auxiliaryfunctions.GetTrainingSetFolder(cfg), - ) + datasets_folder = Path(self.proj_root) / auxiliaryfunctions.GetTrainingSetFolder(cfg) self.datasets_folder = datasets_folder trainingFraction = int(cfg["TrainingFraction"][0] * 100) - path_dlc_collected = os.path.join(datasets_folder, f"CollectedData_{scorer}.h5") + path_dlc_collected = datasets_folder / f"CollectedData_{scorer}.h5" - path_dlc_document = os.path.join( - datasets_folder, - f"Documentation_data-{task}_{trainingFraction}shuffle{shuffle}.pickle", - ) + path_dlc_document = datasets_folder / f"Documentation_data-{task}_{trainingFraction}shuffle{shuffle}.pickle" df = pd.read_hdf(path_dlc_collected) self.dlc_df = df - with open(path_dlc_document, "rb") as f: + with path_dlc_document.open("rb") as f: document_data = pickle.load(f) train_indices = document_data[1] diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/coco.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/coco.py index 2c6200cc9..4c6fc2917 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/coco.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/coco.py @@ -10,7 +10,7 @@ # import copy import json -import os +from pathlib import Path from deeplabcut.modelzoo.generalized_data_converter.datasets.base import BasePoseDataset @@ -44,8 +44,8 @@ def __init__( self.populate_generic() def _load_json(self, json_fn): - path = os.path.join(self.proj_root, "annotations", json_fn) - with open(path) as f: + path = Path(self.proj_root) / "annotations" / json_fn + with path.open() as f: json_obj = json.load(f) return json_obj @@ -60,7 +60,7 @@ def populate_generic(self): # assuming the file_name is mmpose style, i.e. only the image name is stored # so we need to add back absolute path - image["file_name"] = os.path.join(self.proj_root, "images", image_path) + image["file_name"] = str(Path(self.proj_root) / "images" / image_path) self.generic_train_images = temp_train_images self.generic_test_images = temp_test_images diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc.py index e5f57b0a4..d22b6e05f 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc.py @@ -8,7 +8,7 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os +from pathlib import Path import numpy as np import pandas as pd @@ -124,9 +124,9 @@ def _df2generic(self, df, image_id_offset=0): # I think width and height are important if isinstance(file_name, tuple): - image_path = os.path.join(self.proj_root, *list(file_name)) + image_path = str(Path(self.proj_root).joinpath(*file_name)) else: - image_path = os.path.join(self.proj_root, file_name) + image_path = str(Path(self.proj_root) / file_name) _, height, width = read_image_shape_fast(image_path) diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc_dataframe.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc_dataframe.py index b73036918..e6b5ef7f9 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc_dataframe.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc_dataframe.py @@ -8,7 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os from pathlib import Path import numpy as np @@ -35,11 +34,11 @@ def merge_annotateddatasets(cfg): But if someone labels on windows and wants to train on a unix cluster or colab... """ AnnotationData = [] - data_path = Path(os.path.join(cfg["project_path"], "labeled-data")) + data_path = Path(cfg["project_path"]) / "labeled-data" videos = cfg["video_sets"].keys() video_filenames = parse_video_filenames(videos) for filename in video_filenames: - file_path = os.path.join(data_path / filename, f"CollectedData_{cfg['scorer']}.h5") + file_path = data_path / filename / f"CollectedData_{cfg['scorer']}.h5" try: data = pd.read_hdf(file_path) conversioncode.guarantee_multiindex_rows(data) @@ -238,9 +237,9 @@ def _df2generic(self, df, image_id_offset=0): # I think width and height are important if isinstance(file_name, tuple): - image_path = os.path.join(self.proj_root, *list(file_name)) + image_path = str(Path(self.proj_root).joinpath(*file_name)) else: - image_path = os.path.join(self.proj_root, file_name) + image_path = str(Path(self.proj_root) / file_name) _, height, width = read_image_shape_fast(image_path) diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py index 1098ce8af..0bf2e9b79 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/materialize.py @@ -9,7 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # import json -import os import pickle import shutil from pathlib import Path @@ -35,7 +34,7 @@ def get_filename(filename): if isinstance(filename, tuple): - filename = os.path.join(*filename) + filename = str(Path(*filename)) return filename @@ -81,7 +80,7 @@ def __init__(self): def create_cfg(self, proj_root, kwargs): self.cfg.update(kwargs) - with open(os.path.join(proj_root, "config.yaml"), "w") as f: + with (Path(proj_root) / "config.yaml").open("w") as f: yaml.dump(self.cfg, f) @@ -94,7 +93,7 @@ def __init__(self): def create_cfg(self, proj_root, kwargs): self.cfg.update(kwargs) - with open(os.path.join(proj_root, "config.yaml"), "w") as f: + with (Path(proj_root) / "config.yaml").open("w") as f: yaml.dump(self.cfg, f) @@ -119,7 +118,7 @@ def _generic2madlc( assert full_image_path, "DLC wants full image path" - os.makedirs(os.path.join(proj_root, "labeled-data"), exist_ok=True) + (Path(proj_root) / "labeled-data").mkdir(parents=True, exist_ok=True) cfg_template = MaDLC_config() @@ -157,7 +156,7 @@ def _generic2madlc( imageid2datasetname = meta["imageid2datasetname"] for dataset_name in meta["mat_datasets"]: - os.makedirs(os.path.join(proj_root, "labeled-data", dataset_name), exist_ok=True) + (Path(proj_root) / "labeled-data" / dataset_name).mkdir(parents=True, exist_ok=True) # also, to make sure the split is right, we will have to pass the right indices @@ -175,7 +174,7 @@ def _generic2madlc( for image in total_images: image_id = image["id"] file_name = image["file_name"] - image_name = file_name.split(os.sep)[-1] + image_name = Path(file_name).name pre, suffix = image_name.split(".") if append_image_id: dest_image_name = f"{pre}_{image_id}.{suffix}" @@ -186,16 +185,16 @@ def _generic2madlc( dataset_name = imageid2datasetname[image_id] - dest = os.path.join(proj_root, "labeled-data", dataset_name, dest_image_name) + dest = Path(proj_root) / "labeled-data" / dataset_name / dest_image_name if deepcopy: shutil.copy(file_name, dest) else: try: - os.symlink(file_name, dest) + dest.symlink_to(file_name) except Exception: pass - relative_dest = os.path.join("labeled-data", dataset_name, dest_image_name) + relative_dest = str(Path("labeled-data") / dataset_name / dest_image_name) imageid2relativedest[image_id] = relative_dest @@ -239,29 +238,29 @@ def _generic2madlc( df.loc[file_name][scorer, f"individual{individual_id}", kpt_name, "x"] = -1 df.loc[file_name][scorer, f"individual{individual_id}", kpt_name, "y"] = -1 df.to_hdf( - os.path.join(proj_root, "labeled-data", dataset_name, f"CollectedData_{scorer}.h5"), + str(Path(proj_root) / "labeled-data" / dataset_name / f"CollectedData_{scorer}.h5"), key="df_with_missing", mode="w", ) # paf_graph default as None. But I am not sure how to do better - create_multianimaltraining_dataset(os.path.join(proj_root, "config.yaml"), paf_graph=None) + create_multianimaltraining_dataset(str(Path(proj_root) / "config.yaml"), paf_graph=None) # dlc's merge_annotation messes up my indices, so I will need to overwrite the documentation file # I could have done it in a more elegant way if I could modify part of DLC # source code, but for backward compatibility reasons, overriding # documentation is smarter - config_path = os.path.join(proj_root, "config.yaml") + config_path = str(Path(proj_root) / "config.yaml") cfg = auxiliaryfunctions.read_config(config_path) - train_folder = os.path.join(proj_root, auxiliaryfunctions.GetTrainingSetFolder(cfg)) + train_folder = str(Path(proj_root) / auxiliaryfunctions.GetTrainingSetFolder(cfg)) datafilename, metafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(train_folder, train_fraction, 1, cfg) modify_train_test_cfg(config_path) - dlc_df = pd.read_hdf(os.path.join(train_folder, f"CollectedData_{scorer}.h5")) + dlc_df = pd.read_hdf(str(Path(train_folder) / f"CollectedData_{scorer}.h5")) # I strip off video info from the naming. For horse10, I need to get it back parent_trace = {} @@ -269,8 +268,8 @@ def _generic2madlc( def _filter(image): file_name = image["file_name"] - image_name = file_name.split(os.sep)[-1] - video_folder = file_name.split(os.sep)[-2] + image_name = Path(file_name).name + video_folder = Path(file_name).parts[-2] pre, suffix = image_name.split(".") image_id = image["id"] if append_image_id: @@ -283,23 +282,23 @@ def _filter(image): _filter_train_images = list(map(_filter, train_images)) _filter_test_images = list(map(_filter, test_images)) - with open(os.path.join(train_folder, "parent_trace.pickle"), "wb") as f: + with (Path(train_folder) / "parent_trace.pickle").open("wb") as f: pickle.dump(parent_trace, f) trainIndices = [ - idx for idx, image in enumerate(dlc_df.index) if get_filename(image).split(os.sep)[-1] in _filter_train_images + idx for idx, image in enumerate(dlc_df.index) if Path(get_filename(image)).name in _filter_train_images ] testIndices = [ - idx for idx, image in enumerate(dlc_df.index) if get_filename(image).split(os.sep)[-1] in _filter_test_images + idx for idx, image in enumerate(dlc_df.index) if Path(get_filename(image)).name in _filter_test_images ] - with open(metafilename, "rb") as f: + with Path(metafilename).open("rb") as f: metafile = pickle.load(f) metafile[1] = trainIndices metafile[2] = testIndices - with open(metafilename, "wb") as f: + with Path(metafilename).open("wb") as f: pickle.dump(metafile, f) # need to overwrite the data pickle file too @@ -317,7 +316,7 @@ def _filter(image): print(f"overwriting data file {datafilename}") - with open(os.path.join(proj_root, datafilename), "wb") as f: + with (Path(proj_root) / datafilename).open("wb") as f: pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) @@ -335,7 +334,7 @@ def _generic2sdlc( assert full_image_path, "DLC wants full image path" - os.makedirs(os.path.join(proj_root, "labeled-data"), exist_ok=True) + (Path(proj_root) / "labeled-data").mkdir(parents=True, exist_ok=True) cfg_template = SingleDLC_config() @@ -364,7 +363,7 @@ def _generic2sdlc( imageid2datasetname = meta["imageid2datasetname"] for dataset_name in meta["mat_datasets"]: - os.makedirs(os.path.join(proj_root, "labeled-data", dataset_name), exist_ok=True) + (Path(proj_root) / "labeled-data" / dataset_name).mkdir(parents=True, exist_ok=True) columnindex = pd.MultiIndex.from_product([[scorer], bodyparts, ["x", "y"]], names=["scorer", "bodyparts", "coords"]) @@ -383,7 +382,7 @@ def _generic2sdlc( image_id = image["id"] file_name = image["file_name"] - image_name = file_name.split(os.sep)[-1] + image_name = Path(file_name).name pre, suffix = image_name.split(".") if append_image_id: @@ -395,19 +394,19 @@ def _generic2sdlc( dataset_name = imageid2datasetname[image_id] - dest = os.path.join(proj_root, "labeled-data", dataset_name, dest_image_name) + dest = Path(proj_root) / "labeled-data" / dataset_name / dest_image_name if deepcopy: shutil.copy(file_name, dest) else: try: - os.symlink(file_name, dest) + dest.symlink_to(file_name) except Exception: pass if dataset_name == "AwA-Pose": count += 1 - relative_dest = os.path.join("labeled-data", dataset_name, dest_image_name) + relative_dest = str(Path("labeled-data") / dataset_name / dest_image_name) imageid2relativedest[image_id] = relative_dest # so we know where to put the next annotation if there are multiple individuals in that image @@ -452,36 +451,36 @@ def _generic2sdlc( df = df.dropna(how="all") df.to_hdf( - os.path.join(proj_root, "labeled-data", dataset_name, f"CollectedData_{scorer}.h5"), + str(Path(proj_root) / "labeled-data" / dataset_name / f"CollectedData_{scorer}.h5"), key="df_with_missing", mode="w", ) - create_training_dataset(os.path.join(proj_root, "config.yaml")) + create_training_dataset(str(Path(proj_root) / "config.yaml")) # dlc's merge_annotation messes up my indices, so I will need to overwrite the documentation file # I could have done it in a more elegant way if I could modify part of DLC # source code, but for backward compatibility reasons, overriding # documentation is smarter - config_path = os.path.join(proj_root, "config.yaml") + config_path = str(Path(proj_root) / "config.yaml") cfg = auxiliaryfunctions.read_config(config_path) - train_folder = os.path.join(proj_root, auxiliaryfunctions.GetTrainingSetFolder(cfg)) + train_folder = str(Path(proj_root) / auxiliaryfunctions.GetTrainingSetFolder(cfg)) datafilename, metafilename = auxiliaryfunctions.GetDataandMetaDataFilenames(train_folder, train_fraction, 1, cfg) modify_train_test_cfg(config_path) - dlc_df = pd.read_hdf(os.path.join(train_folder, f"CollectedData_{scorer}.h5")) + dlc_df = pd.read_hdf(str(Path(train_folder) / f"CollectedData_{scorer}.h5")) parent_trace = {} def _filter(image): file_name = image["file_name"] - image_name = file_name.split(os.sep)[-1] - video_folder = file_name.split(os.sep)[-2] + image_name = Path(file_name).name + video_folder = Path(file_name).parts[-2] pre, suffix = image_name.split(".") image_id = image["id"] if append_image_id: @@ -496,23 +495,23 @@ def _filter(image): _filter_train_images = list(map(_filter, train_images)) _filter_test_images = list(map(_filter, test_images)) - with open(os.path.join(train_folder, "parent_trace.pickle"), "wb") as f: + with (Path(train_folder) / "parent_trace.pickle").open("wb") as f: pickle.dump(parent_trace, f) trainIndices = [ - idx for idx, image in enumerate(dlc_df.index) if get_filename(image).split(os.sep)[-1] in _filter_train_images + idx for idx, image in enumerate(dlc_df.index) if Path(get_filename(image)).name in _filter_train_images ] testIndices = [ - idx for idx, image in enumerate(dlc_df.index) if get_filename(image).split(os.sep)[-1] in _filter_test_images + idx for idx, image in enumerate(dlc_df.index) if Path(get_filename(image)).name in _filter_test_images ] - with open(metafilename, "rb") as f: + with Path(metafilename).open("rb") as f: metafile = pickle.load(f) metafile[1] = trainIndices metafile[2] = testIndices - with open(metafilename, "wb") as f: + with Path(metafilename).open("wb") as f: pickle.dump(metafile, f) # need to overwrite the true data file too @@ -522,7 +521,7 @@ def _filter(image): print(f"overwriting data file {datafilename}") - sio.savemat(os.path.join(datafilename), {"dataset": MatlabData}) + sio.savemat(datafilename, {"dataset": MatlabData}) def _generic2coco( @@ -558,8 +557,8 @@ def _generic2coco( the images in the original dataset are used in the annotations. """ - os.makedirs(os.path.join(proj_root, "images"), exist_ok=True) - os.makedirs(os.path.join(proj_root, "annotations"), exist_ok=True) + (Path(proj_root) / "images").mkdir(parents=True, exist_ok=True) + (Path(proj_root) / "annotations").mkdir(parents=True, exist_ok=True) # from new path to old_path lookuptable = {} @@ -611,7 +610,7 @@ def _generic2coco( shutil.copy(src, dest) else: try: - os.symlink(src, dest) + dest.symlink_to(src) except Exception as err: print(f"Could not create a symlink from {src} to {dest}: {err}") pass @@ -622,7 +621,7 @@ def _generic2coco( train_annotations = [train_anno for train_anno in train_annotations if train_anno["image_id"] not in broken_links] test_annotations = [test_anno for test_anno in test_annotations if test_anno["image_id"] not in broken_links] - with open(os.path.join(proj_root, "annotations", "train.json"), "w") as f: + with (Path(proj_root) / "annotations" / "train.json").open("w") as f: train_json_obj = dict( images=train_images, annotations=train_annotations, @@ -631,7 +630,7 @@ def _generic2coco( json.dump(train_json_obj, f, indent=4, cls=NpEncoder) - with open(os.path.join(proj_root, "annotations", "test.json"), "w") as f: + with (Path(proj_root) / "annotations" / "test.json").open("w") as f: test_json_obj = dict( images=test_images, annotations=test_annotations, diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc.py index 405613362..a2de64ba1 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc.py @@ -8,7 +8,7 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os +from pathlib import Path import numpy as np import pandas as pd @@ -108,9 +108,9 @@ def _df2generic(self, df, image_id_offset=0): # I think width and height are important if isinstance(file_name, tuple): - image_path = os.path.join(self.proj_root, *list(file_name)) + image_path = str(Path(self.proj_root).joinpath(*file_name)) else: - image_path = os.path.join(self.proj_root, file_name) + image_path = str(Path(self.proj_root) / file_name) _, height, width = read_image_shape_fast(image_path) diff --git a/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc_dataframe.py b/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc_dataframe.py index 07896698a..69cf94cd5 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc_dataframe.py +++ b/deeplabcut/modelzoo/generalized_data_converter/datasets/single_dlc_dataframe.py @@ -8,7 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os from pathlib import Path import numpy as np @@ -34,11 +33,11 @@ def merge_annotateddatasets(cfg): colab... """ AnnotationData = [] - data_path = Path(os.path.join(cfg["project_path"], "labeled-data")) + data_path = Path(cfg["project_path"]) / "labeled-data" videos = cfg["video_sets"].keys() video_filenames = parse_video_filenames(videos) for filename in video_filenames: - file_path = os.path.join(data_path / filename, f"CollectedData_{cfg['scorer']}.h5") + file_path = data_path / filename / f"CollectedData_{cfg['scorer']}.h5" try: data = pd.read_hdf(file_path) conversioncode.guarantee_multiindex_rows(data) @@ -214,9 +213,9 @@ def _df2generic(self, df, image_id_offset=0): # I think width and height are important if isinstance(file_name, tuple): - image_path = os.path.join(self.proj_root, *list(file_name)) + image_path = str(Path(self.proj_root).joinpath(*file_name)) else: - image_path = os.path.join(self.proj_root, file_name) + image_path = str(Path(self.proj_root) / file_name) _, height, width = read_image_shape_fast(image_path) diff --git a/deeplabcut/modelzoo/generalized_data_converter/utils.py b/deeplabcut/modelzoo/generalized_data_converter/utils.py index fafe97e46..c19610744 100644 --- a/deeplabcut/modelzoo/generalized_data_converter/utils.py +++ b/deeplabcut/modelzoo/generalized_data_converter/utils.py @@ -8,8 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import glob -import os import pickle from pathlib import Path @@ -66,9 +64,9 @@ def create_dummy_config_file_from_h5( print(df) - pattern = glob.glob(os.path.join(proj_root, "labeled-data", "*")) + pattern = list(Path(proj_root).glob("labeled-data/*")) - labeled_folders = [f.split("/")[-1] for f in pattern] + labeled_folders = [f.name for f in pattern] video_sets = {f"{folder}.mp4": {"crop": "0, 400, 0, 400"} for folder in labeled_folders} @@ -102,7 +100,7 @@ def create_dummy_config_file_from_pickle( cfg_template = SingleDLC_config() - with open(reference_pickle, "rb") as f: + with Path(reference_pickle).open("rb") as f: pickle.load(f) # bodyparts = pickle_obj['keypoint_names'] @@ -137,7 +135,7 @@ def create_dummy_config_file_from_pickle( def create_video_h5_from_pickle(proj_root, cfg, reference_pickle, videopath): - with open(reference_pickle, "rb") as f: + with Path(reference_pickle).open("rb") as f: pickle_obj = pickle.load(f) # bodyparts = pickle_obj['keypoint_names'] @@ -179,10 +177,7 @@ def create_video_h5_from_pickle(proj_root, cfg, reference_pickle, videopath): coords = [0, 400, 0, 400] trainFraction = cfg["TrainingFraction"][0] - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, 0, cfg)), - ) + modelfolder = str(Path(cfg["project_path"]) / str(auxiliaryfunctions.get_model_folder(trainFraction, 0, cfg))) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" test_cfg = auxiliaryfunctions.read_plainconfig(path_test_config) @@ -207,11 +202,11 @@ def create_video_h5_from_pickle(proj_root, cfg, reference_pickle, videopath): } metadata = {"data": dictionary} - dataname = os.path.join(proj_root, vname + DLCscorer + ".h5") + dataname = str(Path(proj_root) / (vname + DLCscorer + ".h5")) metadata_path = dataname.split(".h5")[0] + "_meta.pickle" - with open(metadata_path, "wb") as f: + with Path(metadata_path).open("wb") as f: pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) df.to_hdf(dataname, key="df_with_missing", format="table", mode="w") @@ -290,7 +285,7 @@ def create_modelprefix(modelprefix): shutil.copytree( "template-dlc-models", - os.path.join(modelprefix, "dlc-models"), + str(Path(modelprefix) / "dlc-models"), dirs_exist_ok=True, ) diff --git a/deeplabcut/modelzoo/utils.py b/deeplabcut/modelzoo/utils.py index e3bc96c6c..5a285121e 100644 --- a/deeplabcut/modelzoo/utils.py +++ b/deeplabcut/modelzoo/utils.py @@ -10,9 +10,7 @@ # from __future__ import annotations -import os import warnings -from glob import glob from pathlib import Path import numpy as np @@ -31,7 +29,7 @@ def dlc_modelzoo_path() -> Path: """Returns: the path to the `modelzoo` folder in the DeepLabCut installation""" - dlc_root_path = Path(get_deeplabcut_path()) + dlc_root_path = get_deeplabcut_path() return dlc_root_path / "modelzoo" @@ -217,16 +215,16 @@ def parse_project_model_name(superanimal_name: str) -> tuple[str, str]: project_name = superanimal_name.replace(f"_{model_name}", "") dlc_root_path = get_deeplabcut_path() - modelzoo_path = os.path.join(dlc_root_path, "modelzoo") + modelzoo_path = dlc_root_path / "modelzoo" - available_model_configs = glob(os.path.join(modelzoo_path, "model_configs", "*.yaml")) - available_models = [os.path.splitext(os.path.basename(path))[0] for path in available_model_configs] + available_model_configs = list((modelzoo_path / "model_configs").glob("*.yaml")) + available_models = [path.stem for path in available_model_configs] if model_name not in available_models: raise ValueError(f"Model {model_name} not found. Available models are: {available_models}") - available_project_configs = glob(os.path.join(modelzoo_path, "project_configs", "*.yaml")) - [os.path.splitext(os.path.basename(path))[0] for path in available_project_configs] + available_project_configs = list((modelzoo_path / "project_configs").glob("*.yaml")) + [path.stem for path in available_project_configs] return project_name, model_name diff --git a/deeplabcut/modelzoo/video_inference.py b/deeplabcut/modelzoo/video_inference.py index a516cc359..220267360 100644 --- a/deeplabcut/modelzoo/video_inference.py +++ b/deeplabcut/modelzoo/video_inference.py @@ -12,7 +12,6 @@ import json import logging -import os import warnings from collections.abc import Sequence from pathlib import Path @@ -65,7 +64,7 @@ def video_inference_superanimal( detector_name: str | None = None, scale_list: list | None = None, video_extensions: str | Sequence[str] | None = None, - dest_folder: str | None = None, + dest_folder: str | Path | None = None, cropping: list[int] | None = None, video_adapt: bool = False, plot_trajectories: bool = False, @@ -336,11 +335,14 @@ def video_inference_superanimal( """ if scale_list is None: scale_list = [] + if dest_folder is not None: + dest_folder = Path(dest_folder) if not model_name.startswith("fmpose3d"): print(f"Running video inference on {videos} with {superanimal_name}_{model_name}") dlc_root_path = get_deeplabcut_path() - modelzoo_path = os.path.join(dlc_root_path, "modelzoo") - available_architectures = json.load(open(os.path.join(modelzoo_path, "models_to_framework.json"))) + modelzoo_path = dlc_root_path / "modelzoo" + with (modelzoo_path / "models_to_framework.json").open() as _f: + available_architectures = json.load(_f) framework = available_architectures[model_name] print(f"Using {framework} for model {model_name}") if framework == "tensorflow": @@ -504,7 +506,7 @@ def video_inference_superanimal( pseudo_anno_dir = Path(dest_folder) pseudo_anno_name = f"{video_path.stem}_{dlc_scorer}_before_adapt.json" - with open(pseudo_anno_dir / pseudo_anno_name) as f: + with (pseudo_anno_dir / pseudo_anno_name).open() as f: predictions = json.load(f) # make sure we tune parameters inside this function such as pseudo @@ -528,7 +530,7 @@ def video_inference_superanimal( # the model config's parameters need to be updated for adaptation training model_config_path = model_folder / "pytorch_config.yaml" - with open(model_config_path, "w") as f: + with model_config_path.open("w") as f: yaml = YAML() yaml.dump(config, f) @@ -575,7 +577,7 @@ def video_inference_superanimal( print("Running video adaptation with following parameters:\n" + params_msg) train_file = pseudo_dataset_folder / "annotations" / "train.json" - with open(train_file) as f: + with train_file.open() as f: temp_obj = json.load(f) annotations = temp_obj["annotations"] diff --git a/deeplabcut/pose_estimation_3d/camera_calibration.py b/deeplabcut/pose_estimation_3d/camera_calibration.py index 75e6752a6..866321704 100644 --- a/deeplabcut/pose_estimation_3d/camera_calibration.py +++ b/deeplabcut/pose_estimation_3d/camera_calibration.py @@ -9,8 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import glob -import os import pickle from pathlib import Path @@ -25,7 +23,14 @@ matplotlib_axes_logger.setLevel("ERROR") -def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, search_window_size=(11, 11)): +def calibrate_cameras( + config: str | Path, + cbrow=8, + cbcol=6, + calibrate=False, + alpha=0.4, + search_window_size=(11, 11), +): """This function extracts the corners points from the calibration images, calibrates the camera and stores the calibration files in the project folder (defined in the config file). @@ -94,7 +99,7 @@ def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, sear path_removed_images, ) = auxiliaryfunctions_3d.Foldernames3Dproject(cfg_3d) - images = glob.glob(os.path.join(img_path, "*.jpg")) + images = [str(p) for p in Path(img_path).glob("*.jpg")] cam_names = cfg_3d["camera_names"] # update the variable snapshot* in config file according to the name of the cameras @@ -107,7 +112,7 @@ def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, sear pass project_path = cfg_3d["project_path"] - projconfigfile = os.path.join(str(project_path), "config.yaml") + projconfigfile = str(Path(project_path) / "config.yaml") auxiliaryfunctions.write_config_3d(projconfigfile, cfg_3d) # Initialize the dictionary @@ -151,15 +156,12 @@ def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, sear imgpoints[cam].append(corners) # Draw the corners and store the images img = cv2.drawChessboardCorners(img, (cbcol, cbrow), corners, ret) - cv2.imwrite(os.path.join(str(path_corners), filename + "_corner.jpg"), img) + cv2.imwrite(str(Path(path_corners) / (filename + "_corner.jpg")), img) else: print(f"Corners not found for the image {Path(fname).name}") for new_cam in cam_names: remove_fname = Path(fname).name.replace(cam, new_cam) - os.rename( - os.path.join(str(img_path), remove_fname), - os.path.join(str(path_removed_images), remove_fname), - ) + (Path(img_path) / remove_fname).rename(Path(path_removed_images) / remove_fname) if new_cam != cam: skip_images.append(remove_fname) @@ -190,15 +192,9 @@ def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, sear } pickle.dump( dist_pickle, - open( - os.path.join(path_camera_matrix, cam + "_intrinsic_params.pickle"), - "wb", - ), - ) - print( - f"Saving intrinsic camera calibration matrices for {cam}" - f" as a pickle file in {os.path.join(path_camera_matrix)}" + (Path(path_camera_matrix) / (cam + "_intrinsic_params.pickle")).open("wb"), ) + print(f"Saving intrinsic camera calibration matrices for {cam} as a pickle file in {path_camera_matrix}") # Compute mean re-projection errors for individual cameras mean_error = 0 @@ -266,12 +262,9 @@ def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, sear "image_shape": [img_shape[pair[0]], img_shape[pair[1]]], } - print( - "Saving the stereo parameters for every " - f"pair of cameras as a pickle file in {str(os.path.join(path_camera_matrix))}" - ) + print(f"Saving the stereo parameters for every pair of cameras as a pickle file in {path_camera_matrix}") - auxiliaryfunctions.write_pickle(os.path.join(path_camera_matrix, "stereo_params.pickle"), stereo_params) + auxiliaryfunctions.write_pickle(str(Path(path_camera_matrix) / "stereo_params.pickle"), stereo_params) print("Camera calibration done! Use the function ``check_undistortion`` to check the check the calibration") else: print( @@ -282,7 +275,7 @@ def calibrate_cameras(config, cbrow=8, cbcol=6, calibrate=False, alpha=0.4, sear ) -def check_undistortion(config, cbrow=8, cbcol=6, plot=True): +def check_undistortion(config: str | Path, cbrow=8, cbcol=6, plot=True): """This function undistorts the calibration images based on the camera matrices and stores them in the project folder(defined in the config file) to visually check if the camera matrices are correct. @@ -326,7 +319,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): markerColor = cfg_3d["markerColor"] cam_names = cfg_3d["camera_names"] - images = glob.glob(os.path.join(img_path, "*.jpg")) + images = [str(p) for p in Path(img_path).glob("*.jpg")] # Sort the images images.sort(key=lambda f: int("".join(filter(str.isdigit, f)))) @@ -340,7 +333,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) """ camera_pair = [[cam_names[0], cam_names[1]]] - stereo_params = auxiliaryfunctions.read_pickle(os.path.join(path_camera_matrix, "stereo_params.pickle")) + stereo_params = auxiliaryfunctions.read_pickle(str(Path(path_camera_matrix) / "stereo_params.pickle")) for pair in camera_pair: map1_x, map1_y = cv2.initUndistortRectifyMap( @@ -382,7 +375,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): ) cam1_undistort.append(imgpoints_proj_undistort) cv2.imwrite( - os.path.join(str(path_undistort), filename + "_undistort.jpg"), + str(Path(path_undistort) / (filename + "_undistort.jpg")), im_remapped1, ) imgpoints_proj_undistort = [] @@ -406,7 +399,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): ) cam2_undistort.append(imgpoints_proj_undistort2) cv2.imwrite( - os.path.join(str(path_undistort), filename + "_undistort.jpg"), + str(Path(path_undistort) / (filename + "_undistort.jpg")), im_remapped2, ) imgpoints_proj_undistort2 = [] @@ -428,7 +421,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): ax2.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)) mcolors.Normalize(vmin=0.0, vmax=cam1_undistort.shape[1]) - plt.savefig(os.path.join(str(path_undistort), "Original_Image.png")) + plt.savefig(str(Path(path_undistort) / "Original_Image.png")) # Plot the undistorted corner points f2, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10)) @@ -452,7 +445,7 @@ def check_undistortion(config, cbrow=8, cbcol=6, plot=True): color=markerColor, alpha=alphaValue, ) - plt.savefig(os.path.join(str(path_undistort), "undistorted_points.png")) + plt.savefig(str(Path(path_undistort) / "undistorted_points.png")) # Triangulate triangulate = auxiliaryfunctions_3d.compute_triangulation_calibration_images( diff --git a/deeplabcut/pose_estimation_3d/plotting3D.py b/deeplabcut/pose_estimation_3d/plotting3D.py index 5383401a9..985cac41b 100644 --- a/deeplabcut/pose_estimation_3d/plotting3D.py +++ b/deeplabcut/pose_estimation_3d/plotting3D.py @@ -9,8 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import glob -import os from pathlib import Path import matplotlib.pyplot as plt @@ -58,8 +56,8 @@ def set_up_grid(figsize, xlim, ylim, zlim, view): # other API (i.e. videotype: str -> video_extensions: str | Sequence[str] | None) # this requires updating Get_list_of_triangulated_and_videoFiles. def create_labeled_video_3d( - config, - path, + config: str | Path, + path: str | Path, videofolder=None, start=0, end=None, @@ -151,8 +149,6 @@ def create_labeled_video_3d( >>> deeplabcut.create_labeled_video_3d(config,['/data/project1/videos'],start=100, end=500,view=[30,90],xlim=[-12,12],ylim=[15,25],zlim=[20,30]) """ - os.getcwd() - # Read the config file and related variables cfg_3d = auxiliaryfunctions.read_config(config) cam_names = cfg_3d["camera_names"] @@ -184,8 +180,8 @@ def create_labeled_video_3d( # triangulated file is a list which is always sorted as [triangulated.h5,camera-1.videotype,camera-2.videotype] # name for output video file_name = str(Path(triangulate_file).stem) - videooutname = os.path.join(path_h5_file, file_name + ".mp4") - if os.path.isfile(videooutname): + videooutname = path_h5_file / (file_name + ".mp4") + if videooutname.is_file(): print("Video already created...") else: string_to_remove = str(Path(triangulate_file).suffix) @@ -211,41 +207,21 @@ def create_labeled_video_3d( try: print("Looking for filtered predictions...") df_cam1 = pd.read_hdf( - glob.glob( - os.path.join( - path_h5_file, - str("*" + base_filename_cam1 + cam1_scorer + "*filtered.h5"), - ) - )[0] + list(path_h5_file.glob("*" + base_filename_cam1 + cam1_scorer + "*filtered.h5"))[0] ) df_cam2 = pd.read_hdf( - glob.glob( - os.path.join( - path_h5_file, - str("*" + base_filename_cam2 + cam2_scorer + "*filtered.h5"), - ) - )[0] + list(path_h5_file.glob("*" + base_filename_cam2 + cam2_scorer + "*filtered.h5"))[0] ) # print("Found filtered predictions, will be use these for triangulation.") print( "Found the following filtered data: ", - os.path.join( - path_h5_file, - str("*" + base_filename_cam1 + cam1_scorer + "*filtered.h5"), - ), - os.path.join( - path_h5_file, - str("*" + base_filename_cam2 + cam2_scorer + "*filtered.h5"), - ), + path_h5_file / ("*" + base_filename_cam1 + cam1_scorer + "*filtered.h5"), + path_h5_file / ("*" + base_filename_cam2 + cam2_scorer + "*filtered.h5"), ) except IndexError: print("No filtered predictions found, the unfiltered predictions will be used instead.") - df_cam1 = pd.read_hdf( - glob.glob(os.path.join(path_h5_file, str(base_filename_cam1 + cam1_scorer + "*.h5")))[0] - ) - df_cam2 = pd.read_hdf( - glob.glob(os.path.join(path_h5_file, str(base_filename_cam2 + cam2_scorer + "*.h5")))[0] - ) + df_cam1 = pd.read_hdf(list(path_h5_file.glob(base_filename_cam1 + cam1_scorer + "*.h5"))[0]) + df_cam2 = pd.read_hdf(list(path_h5_file.glob(base_filename_cam2 + cam2_scorer + "*.h5"))[0]) df_3d = pd.read_hdf(triangulate_file) try: @@ -258,7 +234,7 @@ def create_labeled_video_3d( end = min(end, min(len(vid_cam1), len(vid_cam2))) frames = list(range(start, end)) - output_folder = Path(os.path.join(path_h5_file, "temp_" + file_name)) + output_folder = path_h5_file / ("temp_" + file_name) output_folder.mkdir(parents=True, exist_ok=True) # Flatten the list of bodyparts to connect @@ -337,7 +313,7 @@ def create_labeled_video_3d( axes3.add_collection(coll_3d) writer = FFMpegWriter(fps=fps) - with writer.saving(fig, videooutname, dpi=dpi): + with writer.saving(fig, str(videooutname), dpi=dpi): for k in tqdm(frames): vid_cam1.set_to_frame(k) vid_cam2.set_to_frame(k) diff --git a/deeplabcut/pose_estimation_3d/triangulation.py b/deeplabcut/pose_estimation_3d/triangulation.py index ca90853ce..f9ef9dabd 100644 --- a/deeplabcut/pose_estimation_3d/triangulation.py +++ b/deeplabcut/pose_estimation_3d/triangulation.py @@ -9,7 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import os import warnings from pathlib import Path @@ -25,8 +24,8 @@ # other API (i.e. videotype: str -> video_extensions: str | Sequence[str] | None) # this requires updating get_camerawise_videos (matching `collect_video_paths`) def triangulate( - config, - video_path, + config: str | Path, + video_path: str | Path | list[str | Path] | list[list[str | Path]], videotype="", filterpredictions=True, filtertype="median", @@ -105,7 +104,7 @@ def triangulate( for cam in cam_names: snapshots[cam] = cfg_3d[str("config_file_" + cam)] # Check if the config file exists - if not os.path.exists(snapshots[cam]): + if not Path(snapshots[cam]).exists(): raise Exception( str("It seems the file specified in the variable config_file_" + str(cam)) + " does not exist. Please edit the config file with correct file path and retry." @@ -157,10 +156,10 @@ def triangulate( trainingsetindex = cfg_3d[str("trainingsetindex_" + cam_names[j])] trainFraction = cfg["TrainingFraction"][trainingsetindex] if flag: - video = os.path.join(video_path, video_list[i][j]) + video = str(Path(video_path) / video_list[i][j]) else: video_path = str(Path(video_list[i][j]).parents[0]) - video = os.path.join(video_path, video_list[i][j]) + video = str(Path(video_path) / video_list[i][j]) if destfolder is None: destfolder = str(Path(video).parents[0]) @@ -179,18 +178,16 @@ def triangulate( suffix = suffix[1:] if prefix == "": - output_file = os.path.join(destfolder, suffix) + output_file = str(Path(destfolder) / suffix) else: if suffix == "": - output_file = os.path.join(destfolder, prefix) + output_file = str(Path(destfolder) / prefix) else: - output_file = os.path.join(destfolder, prefix + "_" + suffix) + output_file = str(Path(destfolder) / (prefix + "_" + suffix)) - output_filename = os.path.join( - output_file + "_" + scorer_3d - ) # Check if the videos are already analyzed for 3d - if os.path.isfile(output_filename + ".h5"): - if save_as_csv is True and not os.path.exists(output_filename + ".csv"): + output_filename = output_file + "_" + scorer_3d # Check if the videos are already analyzed for 3d + if Path(output_filename + ".h5").is_file(): + if save_as_csv is True and not Path(output_filename + ".csv").exists(): # In case user adds save_as_csv is True after triangulating pd.read_hdf(output_filename + ".h5").to_csv(str(output_filename + ".csv")) @@ -208,7 +205,7 @@ def triangulate( path_undistort, _, ) = auxiliaryfunctions_3d.Foldernames3Dproject(cfg_3d) - path_stereo_file = os.path.join(path_camera_matrix, "stereo_params.pickle") + path_stereo_file = str(Path(path_camera_matrix) / "stereo_params.pickle") stereo_file = auxiliaryfunctions.read_pickle(path_stereo_file) cam_pair = str(cam_names[0] + "-" + cam_names[1]) is_video_analyzed = False # variable to keep track if the video was already analyzed @@ -234,7 +231,7 @@ def triangulate( if is_video_analyzed: print("This file is already analyzed!") - dataname.append(os.path.join(destfolder, vname + DLCscorer + tr_method_suffix + ".h5")) + dataname.append(str(Path(destfolder) / (vname + DLCscorer + tr_method_suffix + ".h5"))) scorer_name[cam_names[j]] = DLCscorer else: # Analyze video if score name is different @@ -263,7 +260,7 @@ def triangulate( ) suffix += "_filtered" - dataname.append(os.path.join(destfolder, vname + DLCscorer + suffix + ".h5")) + dataname.append(str(Path(destfolder) / (vname + DLCscorer + suffix + ".h5"))) else: # need to do the whole jam. DLCscorer = analyze_videos( @@ -290,7 +287,7 @@ def triangulate( destfolder=destfolder, ) suffix += "_filtered" - dataname.append(os.path.join(destfolder, vname + DLCscorer + suffix + ".h5")) + dataname.append(str(Path(destfolder) / (vname + DLCscorer + suffix + ".h5"))) if run_triangulate: # if len(dataname)>0: @@ -522,14 +519,14 @@ def undistort_points(config, dataframe, camera_pair): f"needs filenames to two data frames, but got dataframe={dataframe}." ) for filename in dataframe: - if not os.path.exists(filename): + if not Path(filename).exists(): raise FileNotFoundError(f"Dataframe path '{filename}' could not be found in the filesystem.") - if not os.path.exists(path_camera_matrix): + if not Path(path_camera_matrix).exists(): raise FileNotFoundError(f"Camera matrix file '{path_camera_matrix}' could not be found in the filesystem.") # Create an empty dataFrame to store the undistorted 2d coordinates and likelihood dataframe_cam1 = pd.read_hdf(dataframe[0]) dataframe_cam2 = pd.read_hdf(dataframe[1]) - path_stereo_file = os.path.join(path_camera_matrix, "stereo_params.pickle") + path_stereo_file = str(Path(path_camera_matrix) / "stereo_params.pickle") stereo_file = auxiliaryfunctions.read_pickle(path_stereo_file) dataFrame_cam1_undistort, dataFrame_cam2_undistort = _undistort_views( [(dataframe_cam1, dataframe_cam2)], diff --git a/deeplabcut/pose_estimation_pytorch/apis/analyze_images.py b/deeplabcut/pose_estimation_pytorch/apis/analyze_images.py index 25d415848..9e097b448 100644 --- a/deeplabcut/pose_estimation_pytorch/apis/analyze_images.py +++ b/deeplabcut/pose_estimation_pytorch/apis/analyze_images.py @@ -11,10 +11,8 @@ from __future__ import annotations import copy -import glob import json import logging -import os from collections import defaultdict from pathlib import Path @@ -576,7 +574,7 @@ def plot_images_coco( Raises: ValueError: if a top-down model configuration is given but detector_path is None """ - with open(data_json_path) as f: + with Path(data_json_path).open() as f: obj = json.load(f) coco_images = obj["images"] @@ -585,7 +583,7 @@ def plot_images_coco( image_name_to_id = {} for image in coco_images: # only works with relative path as a test image can be in a different folder - image_name = image["file_name"].split(os.sep)[-1] + image_name = Path(image["file_name"]).name image_name_to_id[image_name] = image["id"] image_id_to_annotations = defaultdict(list) @@ -596,11 +594,11 @@ def plot_images_coco( image_id_to_annotations[image_id].append(annotation) # need to support more image types - images_in_folder = glob.glob(str(Path(image_folder) / "*.png")) + images_in_folder = list(Path(image_folder).glob("*.png")) corresponded_images = [] for image in images_in_folder: image_path = image - image_name = image.split(os.sep)[-1] + image_name = Path(image).name if image_name in image_name_to_id: corresponded_images.append(image_path) @@ -617,11 +615,11 @@ def plot_images_coco( cond_provider=cond_provider, ) - os.makedirs(out_path, exist_ok=True) + Path(out_path).mkdir(parents=True, exist_ok=True) coco_format_predictions = [] for image_path, prediction in predictions.items(): - image_name = image_path.split(os.sep)[-1] + image_name = Path(image_path).name coco_prediction = dict( image_id=image_name_to_id[image_name], gt_annotations=image_id_to_annotations[image_name_to_id[image_name]], @@ -656,8 +654,8 @@ def plot_images_coco( rect = plt.Rectangle((xmin, ymin), w, h, fill=False, edgecolor="blue", linewidth=2) ax.add_patch(rect) - image_name = image_path.split("/")[-1] - fig.savefig(os.path.join(out_path, image_name)) + image_name = Path(image_path).name + fig.savefig(Path(out_path) / image_name) return coco_format_predictions diff --git a/deeplabcut/pose_estimation_pytorch/apis/tracking_dataset.py b/deeplabcut/pose_estimation_pytorch/apis/tracking_dataset.py index 97b3c8fad..c1c28d2e9 100644 --- a/deeplabcut/pose_estimation_pytorch/apis/tracking_dataset.py +++ b/deeplabcut/pose_estimation_pytorch/apis/tracking_dataset.py @@ -128,13 +128,13 @@ def extract_features_for_video( @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def create_tracking_dataset( - config: str, + config: str | Path, videos: list[str] | list[Path], track_method: str, video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, - destfolder: str | None = None, + destfolder: str | Path | None = None, batch_size: int | None = None, detector_batch_size: int | None = None, cropping: list[int] | None = None, diff --git a/deeplabcut/pose_estimation_pytorch/apis/tracklets.py b/deeplabcut/pose_estimation_pytorch/apis/tracklets.py index ea7bf881c..7dac3ebae 100644 --- a/deeplabcut/pose_estimation_pytorch/apis/tracklets.py +++ b/deeplabcut/pose_estimation_pytorch/apis/tracklets.py @@ -36,13 +36,13 @@ @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def convert_detections2tracklets( - config: str, + config: str | Path, videos: str | list[str], video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, overwrite: bool = False, - destfolder: str | None = None, + destfolder: str | Path | None = None, ignore_bodyparts: list[str] | None = None, inferencecfg: dict | None = None, modelprefix="", @@ -66,7 +66,7 @@ def convert_detections2tracklets( auxiliaryfunctions.write_config(config, cfg) train_fraction = cfg["TrainingFraction"][trainingsetindex] - start_path = os.getcwd() # record cwd to return to this directory in the end + start_path = Path.cwd() # record cwd to return to this directory in the end # TODO: add cropping as in video analysis! # if cropping is not None: @@ -84,7 +84,7 @@ def convert_detections2tracklets( ) model_dir = Path(cfg["project_path"]) / rel_model_dir path_test_config = model_dir / "test" / "pose_cfg.yaml" - dlc_cfg = auxiliaryfunctions.read_plainconfig(str(path_test_config)) + dlc_cfg = auxiliaryfunctions.read_plainconfig(path_test_config) if "multi-animal" not in dlc_cfg["dataset_type"]: raise ValueError("This function is only required for multianimal projects!") @@ -182,7 +182,7 @@ def convert_detections2tracklets( identity_only=identity_only, ) - with open(track_filename, "wb") as f: + with Path(track_filename).open("wb") as f: pickle.dump(tracklets, f, pickle.HIGHEST_PROTOCOL) os.chdir(str(start_path)) diff --git a/deeplabcut/pose_estimation_pytorch/apis/utils.py b/deeplabcut/pose_estimation_pytorch/apis/utils.py index 879e449e3..4ce23250b 100644 --- a/deeplabcut/pose_estimation_pytorch/apis/utils.py +++ b/deeplabcut/pose_estimation_pytorch/apis/utils.py @@ -126,7 +126,10 @@ def parse_snapshot_index_for_analysis( def return_train_network_path( - config: str, shuffle: int = 1, trainingsetindex: int = 0, modelprefix: str = "" + config: str | Path, + shuffle: int = 1, + trainingsetindex: int = 0, + modelprefix: str = "", ) -> tuple[Path, Path, Path]: """ Args: diff --git a/deeplabcut/pose_estimation_pytorch/apis/videos.py b/deeplabcut/pose_estimation_pytorch/apis/videos.py index 823674be7..89d243a9f 100644 --- a/deeplabcut/pose_estimation_pytorch/apis/videos.py +++ b/deeplabcut/pose_estimation_pytorch/apis/videos.py @@ -244,8 +244,8 @@ def video_inference( @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def analyze_videos( - config: str, - videos: str | list[str], + config: str | Path, + videos: str | Path | list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, @@ -254,7 +254,7 @@ def analyze_videos( snapshot_index: int | str | None = None, detector_snapshot_index: int | str | None = None, device: str | None = None, - destfolder: str | None = None, + destfolder: str | Path | None = None, batch_size: int | None = None, detector_batch_size: int | None = None, dynamic: tuple[bool, float, int] = (False, 0.5, 10), @@ -415,6 +415,8 @@ def analyze_videos( Returns: The scorer used to analyze the videos """ + config = Path(config) + destfolder = Path(destfolder) if destfolder is not None else None # Create the output folder _validate_destfolder(destfolder) @@ -602,7 +604,7 @@ def analyze_videos( robust_nframes=robust_nframes, ) - with open(output_path / f"{output_prefix}_meta.pickle", "wb") as f: + with (output_path / f"{output_prefix}_meta.pickle").open("wb") as f: pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) if use_shelve and save_as_df: @@ -610,7 +612,7 @@ def analyze_videos( if not use_shelve: output_data = _generate_output_data(pose_cfg, predictions) - with open(output_pkl, "wb") as f: + with Path(output_pkl).open("wb") as f: pickle.dump(output_data, f, pickle.HIGHEST_PROTOCOL) if save_as_df: @@ -780,7 +782,7 @@ def _generate_assemblies_file( ) -> None: """Generates the assemblies file from predictions.""" if full_data_path.exists(): - with open(full_data_path, "rb") as f: + with full_data_path.open("rb") as f: data = pickle.load(f) else: @@ -831,7 +833,7 @@ def _generate_assemblies_file( unique_preds = unique_preds.transpose((1, 0, 2)) assemblies["single"][frame_index] = unique_preds[0] # single prediction - with open(output_path, "wb") as file: + with Path(output_path).open("wb") as file: pickle.dump(assemblies, file, pickle.HIGHEST_PROTOCOL) if isinstance(data, shelving.ShelfReader): diff --git a/deeplabcut/pose_estimation_pytorch/config/utils.py b/deeplabcut/pose_estimation_pytorch/config/utils.py index 499420748..0f8dbddcd 100644 --- a/deeplabcut/pose_estimation_pytorch/config/utils.py +++ b/deeplabcut/pose_estimation_pytorch/config/utils.py @@ -189,7 +189,7 @@ def update_config_by_dotpath(config: dict, updates: dict, copy_original: bool = def get_config_folder_path() -> Path: """Returns: the Path to the folder containing the "configs" for DeepLabCut 3.0""" - dlc_parent_path = Path(auxiliaryfunctions.get_deeplabcut_path()) + dlc_parent_path = auxiliaryfunctions.get_deeplabcut_path() return dlc_parent_path / "pose_estimation_pytorch" / "config" diff --git a/deeplabcut/pose_estimation_pytorch/data/cocoloader.py b/deeplabcut/pose_estimation_pytorch/data/cocoloader.py index 7fdea47f4..a613c52ca 100644 --- a/deeplabcut/pose_estimation_pytorch/data/cocoloader.py +++ b/deeplabcut/pose_estimation_pytorch/data/cocoloader.py @@ -11,7 +11,6 @@ from __future__ import annotations import json -import os import warnings from pathlib import Path @@ -107,11 +106,11 @@ def load_json(project_root: str | Path, filename: str) -> dict: Check https://docs.trainingdata.io/v1.0/Export%20Format/COCO/ to see examples of how a json file looks like. """ - json_path = os.path.join(project_root, "annotations", filename) - if not os.path.exists(json_path): + json_path = Path(project_root) / "annotations" / filename + if not json_path.exists(): raise FileNotFoundError(f"File {json_path} does not exist.") - with open(json_path) as f: + with json_path.open() as f: json_obj = json.load(f) if not isinstance(json_obj, dict): diff --git a/deeplabcut/pose_estimation_pytorch/data/ctd.py b/deeplabcut/pose_estimation_pytorch/data/ctd.py index 1b054f9e8..8269b0641 100644 --- a/deeplabcut/pose_estimation_pytorch/data/ctd.py +++ b/deeplabcut/pose_estimation_pytorch/data/ctd.py @@ -359,7 +359,7 @@ def load_conditions_json( A dictionary mapping image paths to condition arrays. Each array has shape (num_conditions, num_bodyparts, 3). """ - with open(filepath) as f: + with Path(filepath).open() as f: conditions = json.load(f) # Parse list and return @@ -421,7 +421,7 @@ def load_conditions_pickle(filepath: str | Path) -> list[np.ndarray]: Args: filepath: Path to the Pickle file containing conditions. """ - with open(filepath, "rb") as f: + with Path(filepath).open("rb") as f: data = pickle.load(f) frames = [f for f in data.keys() if isinstance(f, int)] diff --git a/deeplabcut/pose_estimation_pytorch/data/dlcloader.py b/deeplabcut/pose_estimation_pytorch/data/dlcloader.py index c034f0e74..66988e999 100644 --- a/deeplabcut/pose_estimation_pytorch/data/dlcloader.py +++ b/deeplabcut/pose_estimation_pytorch/data/dlcloader.py @@ -283,7 +283,7 @@ def load_split( shuffle_id = f"{config['Task']}_{train_frac}shuffle{shuffle}.pickle" doc_path = trainset_dir / f"Documentation_data-{shuffle_id}" - with open(doc_path, "rb") as f: + with doc_path.open("rb") as f: meta = pickle.load(f) train_ids = [int(i) for i in meta[1]] @@ -579,7 +579,7 @@ def _load_pickle_dataset( images_sizes: all possible images sizes in the dataset dlc_dataset: the dataset in a DLC-format DataFrame """ - with open(file, "rb") as f: + with Path(file).open("rb") as f: raw_data = pickle.load(f) num_images = len(raw_data) diff --git a/deeplabcut/pose_estimation_pytorch/metrics/scoring.py b/deeplabcut/pose_estimation_pytorch/metrics/scoring.py index c830974ef..cf6667efa 100644 --- a/deeplabcut/pose_estimation_pytorch/metrics/scoring.py +++ b/deeplabcut/pose_estimation_pytorch/metrics/scoring.py @@ -11,6 +11,7 @@ from __future__ import annotations import pickle +from pathlib import Path import numpy as np from sklearn.metrics import accuracy_score @@ -20,7 +21,7 @@ def _match_identity_preds_to_gt(config_path: str, full_pickle_path: str) -> tuple[np.ndarray, list]: - with open(full_pickle_path, "rb") as f: + with Path(full_pickle_path).open("rb") as f: data = pickle.load(f) metadata = data.pop("metadata") cfg = read_config(config_path) diff --git a/deeplabcut/pose_estimation_pytorch/modelzoo/config.py b/deeplabcut/pose_estimation_pytorch/modelzoo/config.py index 697231f6e..48ce570e7 100644 --- a/deeplabcut/pose_estimation_pytorch/modelzoo/config.py +++ b/deeplabcut/pose_estimation_pytorch/modelzoo/config.py @@ -12,7 +12,6 @@ from __future__ import annotations -import os from pathlib import Path from ruamel.yaml import YAML @@ -173,8 +172,8 @@ def write_pytorch_config_for_memory_replay(config_path, shuffle, pytorch_config) model_folder = dlc_proj_root / af.get_model_folder( cfg["TrainingFraction"][trainIndex], shuffle, cfg, engine=Engine.PYTORCH ) - os.makedirs(model_folder / "train", exist_ok=True) + (model_folder / "train").mkdir(parents=True, exist_ok=True) out_path = model_folder / "train" / "pytorch_config.yaml" - with open(str(out_path), "w") as f: + with out_path.open("w") as f: yaml = YAML() yaml.dump(pytorch_config, f) diff --git a/deeplabcut/pose_estimation_pytorch/modelzoo/fmpose_3d/inference.py b/deeplabcut/pose_estimation_pytorch/modelzoo/fmpose_3d/inference.py index df993b332..b6be062a1 100644 --- a/deeplabcut/pose_estimation_pytorch/modelzoo/fmpose_3d/inference.py +++ b/deeplabcut/pose_estimation_pytorch/modelzoo/fmpose_3d/inference.py @@ -219,12 +219,12 @@ def _process_batch( results[video_path] = df output_json = dest_folder / f"{output_prefix}.json" - with open(output_json, "w") as f: + with output_json.open("w") as f: json.dump(predictions_2d, f, cls=NumpyEncoder) poses_3d_serialisable = [pose.tolist() if isinstance(pose, np.ndarray) else pose for pose in all_poses_3d] output_3d_json = dest_folder / f"{output_prefix}_3d.json" - with open(output_3d_json, "w") as f: + with output_3d_json.open("w") as f: json.dump( { "model": model_name, diff --git a/deeplabcut/pose_estimation_pytorch/modelzoo/inference.py b/deeplabcut/pose_estimation_pytorch/modelzoo/inference.py index 1dafa68a0..3256bc6a3 100644 --- a/deeplabcut/pose_estimation_pytorch/modelzoo/inference.py +++ b/deeplabcut/pose_estimation_pytorch/modelzoo/inference.py @@ -193,7 +193,7 @@ def _video_inference_superanimal( ) results[video_path] = df - with open(output_json, "w") as f: + with output_json.open("w") as f: json.dump(predictions, f, cls=NumpyEncoder) if create_labeled_video: diff --git a/deeplabcut/pose_estimation_pytorch/modelzoo/memory_replay.py b/deeplabcut/pose_estimation_pytorch/modelzoo/memory_replay.py index d38dd965d..254132147 100644 --- a/deeplabcut/pose_estimation_pytorch/modelzoo/memory_replay.py +++ b/deeplabcut/pose_estimation_pytorch/modelzoo/memory_replay.py @@ -11,7 +11,6 @@ from __future__ import annotations import json -import os from collections import defaultdict from pathlib import Path @@ -67,7 +66,7 @@ def get_pose_predictions( # COCO-format annotations file containing predictions made by the SuperAnimal model sa_predictions = {} if predictions_file.exists(): - with open(predictions_file) as f: + with predictions_file.open() as f: raw_sa_predictions = json.load(f) # parse predictions to convert lists to numpy arrays @@ -116,7 +115,7 @@ def get_pose_predictions( } for image, predictions in sa_predictions.items() } - with open(predictions_file, "w") as f: + with predictions_file.open("w") as f: json.dump(json_sa_predictions, f, indent=2) return sa_predictions @@ -143,7 +142,7 @@ def prepare_memory_replay_dataset( # Contains the ground truth annotations for the DeepLabCut project # .../dlc-models-pytorch/.../...shuffle0/train/memory_replay/annotations/train.json - with open(source_dataset_folder / "annotations" / train_file) as f: + with (source_dataset_folder / "annotations" / train_file).open() as f: project_gt = json.load(f) # parse the GT so that image paths are in the format (no matter the OS): @@ -240,14 +239,14 @@ def optimal_match(gts_list, preds_list): gts[idx]["keypoints"] = list(matched_gt.flatten()) # memory replay path - memory_replay_train_file_path = os.path.join(source_dataset_folder, "annotations", "memory_replay_train.json") + memory_replay_train_file_path = Path(source_dataset_folder) / "annotations" / "memory_replay_train.json" # parse the GT to put the image paths back into OS-specific format for image in project_gt["images"]: image_rel_path = image["file_name"].split("/") image["file_name"] = str(af.safe_resolve(project_root / Path(*image_rel_path))) - with open(memory_replay_train_file_path, "w") as f: + with memory_replay_train_file_path.open("w") as f: json.dump(project_gt, f, indent=4) diff --git a/deeplabcut/pose_estimation_pytorch/modelzoo/utils.py b/deeplabcut/pose_estimation_pytorch/modelzoo/utils.py index 80288040c..462a51c9d 100644 --- a/deeplabcut/pose_estimation_pytorch/modelzoo/utils.py +++ b/deeplabcut/pose_estimation_pytorch/modelzoo/utils.py @@ -27,17 +27,17 @@ def get_model_configs_folder_path() -> Path: """Returns: the folder containing the SuperAnimal model configuration files""" - return Path(auxiliaryfunctions.get_deeplabcut_path()) / "modelzoo" / "model_configs" + return auxiliaryfunctions.get_deeplabcut_path() / "modelzoo" / "model_configs" def get_project_configs_folder_path() -> Path: """Returns: the folder containing the SuperAnimal project configuration files""" - return Path(auxiliaryfunctions.get_deeplabcut_path()) / "modelzoo" / "project_configs" + return auxiliaryfunctions.get_deeplabcut_path() / "modelzoo" / "project_configs" def get_snapshot_folder_path() -> Path: """Returns: the path to the folder containing the SuperAnimal model snapshots""" - return Path(auxiliaryfunctions.get_deeplabcut_path()) / "modelzoo" / "checkpoints" + return auxiliaryfunctions.get_deeplabcut_path() / "modelzoo" / "checkpoints" def get_super_animal_model_config_path(model_name: str) -> Path: diff --git a/deeplabcut/pose_estimation_pytorch/runners/logger.py b/deeplabcut/pose_estimation_pytorch/runners/logger.py index 53608fcb2..211e9b133 100644 --- a/deeplabcut/pose_estimation_pytorch/runners/logger.py +++ b/deeplabcut/pose_estimation_pytorch/runners/logger.py @@ -310,7 +310,7 @@ def _save_wandb_info(self): } output_path = self.train_folder / "wandb_info.yaml" - with open(output_path, "w") as f: + with output_path.open("w") as f: yaml.safe_dump(wandb_info, f) logging.info(f"WandB run info saved to {output_path}") @@ -435,7 +435,7 @@ def log(self, metrics: dict[str, Any], step: int | None = None) -> None: def save(self): """Saves the metrics to the file system.""" logs = self._prepare_logs() - with open(self.log_file, "w", newline="") as f: + with Path(self.log_file).open("w", newline="") as f: writer = csv.writer(f) writer.writerows(logs) @@ -451,7 +451,7 @@ def _load_existing_data(self) -> None: """Loads existing CSV data if the log file exists.""" logging.info(f"Loading existing CSV data from {self.log_file}") try: - with open(self.log_file, newline="") as f: + with Path(self.log_file).open(newline="") as f: reader = csv.DictReader(f) # Update logged metrics from header diff --git a/deeplabcut/pose_estimation_pytorch/utils.py b/deeplabcut/pose_estimation_pytorch/utils.py index 8b39d8f0f..d96a45301 100644 --- a/deeplabcut/pose_estimation_pytorch/utils.py +++ b/deeplabcut/pose_estimation_pytorch/utils.py @@ -10,8 +10,8 @@ # from __future__ import annotations -import os import random +from pathlib import Path import numpy as np import torch @@ -23,8 +23,8 @@ def create_folder(path_to_folder): Args: path_to_folder: Path to the folder that should be created """ - if not os.path.exists(path_to_folder): - os.makedirs(path_to_folder) + if not Path(path_to_folder).exists(): + Path(path_to_folder).mkdir(parents=True) def fix_seeds(seed: int) -> None: diff --git a/deeplabcut/pose_estimation_tensorflow/backbones/efficientnet_builder.py b/deeplabcut/pose_estimation_tensorflow/backbones/efficientnet_builder.py index d3ba8c89b..19b841df2 100644 --- a/deeplabcut/pose_estimation_tensorflow/backbones/efficientnet_builder.py +++ b/deeplabcut/pose_estimation_tensorflow/backbones/efficientnet_builder.py @@ -14,8 +14,8 @@ # limitations under the License. # import functools -import os import re +from pathlib import Path import tensorflow as tf @@ -238,7 +238,7 @@ def build_model( blocks_args, global_params = get_model_params(model_name, override_params) if model_dir: - param_file = os.path.join(model_dir, "model_params.txt") + param_file = Path(model_dir) / "model_params.txt" if not tf.io.gfile.exists(param_file): if not tf.io.gfile.exists(model_dir): tf.io.gfile.makedirs(model_dir) diff --git a/deeplabcut/pose_estimation_tensorflow/backbones/mobilenet.py b/deeplabcut/pose_estimation_tensorflow/backbones/mobilenet.py index 4dd91a096..be59c2c98 100644 --- a/deeplabcut/pose_estimation_tensorflow/backbones/mobilenet.py +++ b/deeplabcut/pose_estimation_tensorflow/backbones/mobilenet.py @@ -16,7 +16,7 @@ import collections import contextlib import copy -import os +from pathlib import Path import tensorflow as tf import tf_slim as slim @@ -246,7 +246,7 @@ def mobilenet_base( # pylint: disable=invalid-name print(f"Failed to create op {i}: {opdef} params: {params}") raise end_points[end_point] = net - scope = os.path.dirname(net.name) + scope = str(Path(net.name).parent) scopes[scope] = end_point if final_endpoint is not None and end_point == final_endpoint: break @@ -254,8 +254,8 @@ def mobilenet_base( # pylint: disable=invalid-name # Add all tensors that end with 'output' to # endpoints for t in net.graph.get_operations(): - scope = os.path.dirname(t.name) - bn = os.path.basename(t.name) + scope = str(Path(t.name).parent) + bn = Path(t.name).name if scope in scopes and t.name.endswith("output"): end_points[scopes[scope] + "/" + bn] = t.outputs[0] return net, end_points diff --git a/deeplabcut/pose_estimation_tensorflow/config.py b/deeplabcut/pose_estimation_tensorflow/config.py index e44474eab..81196c046 100644 --- a/deeplabcut/pose_estimation_tensorflow/config.py +++ b/deeplabcut/pose_estimation_tensorflow/config.py @@ -14,6 +14,7 @@ import logging import pprint +from pathlib import Path import yaml @@ -42,7 +43,7 @@ def _merge_a_into_b(a, b): def cfg_from_file(filename): """Load a config from file filename and merge it into the default options.""" - with open(filename) as f: + with Path(filename).open() as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) # Update the snapshot path to the corresponding path! diff --git a/deeplabcut/pose_estimation_tensorflow/core/evaluate.py b/deeplabcut/pose_estimation_tensorflow/core/evaluate.py index 74fa483bc..3921084aa 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/evaluate.py +++ b/deeplabcut/pose_estimation_tensorflow/core/evaluate.py @@ -60,7 +60,6 @@ def calculatepafdistancebounds(config, shuffle=0, trainingsetindex=0, modelprefi numdigits: number of digits to round for distances. """ - import os from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.utils import auxfun_multianimal, auxiliaryfunctions @@ -78,18 +77,13 @@ def calculatepafdistancebounds(config, shuffle=0, trainingsetindex=0, modelprefi # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) trainFraction = cfg["TrainingFraction"][trainingsetindex] - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) # Load meta data & annotations Data = pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) + Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5") )[cfg["scorer"]] path_train_config, path_test_config, _ = return_train_network_path( @@ -251,17 +245,15 @@ def return_evaluate_network_data( from deeplabcut.pose_estimation_tensorflow.config import load_config from deeplabcut.utils import auxiliaryfunctions - start_path = os.getcwd() + start_path = Path.cwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) # Data=pd.read_hdf( - # os.path.join( - # cfg["project_path"], - # str(trainingsetfolder - # ),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing' + # Path(cfg["project_path"]) / str(trainingsetfolder) / ('CollectedData_' + cfg["scorer"] + '.h5'), + # 'df_with_missing' # ) # Get list of body parts to evaluate network for @@ -270,9 +262,8 @@ def return_evaluate_network_data( # Load data... ################################################## trainFraction = cfg["TrainingFraction"][trainingsetindex] - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) path_train_config, path_test_config, _ = return_train_network_path( config=config, @@ -299,28 +290,17 @@ def return_evaluate_network_data( scale = test_pose_cfg["global_scale"] print("Rescaling Data to ", scale) Data = ( - pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) - ) + pd.read_hdf(Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5")) * scale ) else: scale = 1 Data = pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) + Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5") ) - evaluationfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + evaluationfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) Snapshots = auxiliaryfunctions.get_snapshots_from_folder( @@ -339,10 +319,10 @@ def return_evaluate_network_data( results = [] resultsfns = [] for snapshot_name in snapshot_names: - test_pose_cfg["init_weights"] = os.path.join( - str(modelfolder), "train", snapshot_name + test_pose_cfg["init_weights"] = str( + Path(modelfolder) / "train" / snapshot_name ) # setting weights to corresponding snapshot. - trainingsiterations = (test_pose_cfg["init_weights"].split(os.sep)[-1]).split("-")[ + trainingsiterations = Path(test_pose_cfg["init_weights"]).name.split("-")[ -1 ] # read how many training siterations that corresponds to. @@ -369,7 +349,7 @@ def return_evaluate_network_data( print(resultsfilename) resultsfns.append(resultsfilename) if not returnjustfns: - if not notanalyzed and os.path.isfile(resultsfilename): # data exists.. + if not notanalyzed and Path(resultsfilename).is_file(): # data exists.. DataMachine = pd.read_hdf(resultsfilename) DataCombined = pd.concat([Data.T, DataMachine.T], axis=0).T RMSE, RMSEpcutoff = pairwisedistances( @@ -607,9 +587,7 @@ def evaluate_network( if plotting not in (True, False, "bodypart", "individual"): raise ValueError(f"Unknown value for `plotting`={plotting}") - import os - - start_path = os.getcwd() + start_path = Path.cwd() from deeplabcut.utils import auxiliaryfunctions cfg = auxiliaryfunctions.read_config(config) @@ -648,7 +626,7 @@ def evaluate_network( os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # # tf.logging.set_verbosity(tf.logging.WARN) - start_path = os.getcwd() + start_path = Path.cwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) if gputouse is not None: # gpu selectinon @@ -670,11 +648,7 @@ def evaluate_network( # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) Data = pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) + Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5") ) # Get list of body parts to evaluate network for @@ -682,7 +656,7 @@ def evaluate_network( cfg, comparisonbodyparts ) # Make folder for evaluation - auxiliaryfunctions.attempt_to_make_folder(str(cfg["project_path"] + "/evaluation-results/")) + auxiliaryfunctions.attempt_to_make_folder(Path(cfg["project_path"]) / "evaluation-results") for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## @@ -726,9 +700,8 @@ def evaluate_network( test_pose_cfg["batch_size"] = 1 # in case this was edited for analysis. # Create folder structure to store results. - evaluationfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + evaluationfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) auxiliaryfunctions.attempt_to_make_folder(evaluationfolder, recursive=True) @@ -754,11 +727,9 @@ def evaluate_network( scale = test_pose_cfg["global_scale"] Data = ( pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) + Path(cfg["project_path"]) + / str(trainingsetfolder) + / ("CollectedData_" + cfg["scorer"] + ".h5") ) * scale ) @@ -770,8 +741,8 @@ def evaluate_network( # Compute predictions over images ################################################## for snapshot_name in snapshot_names: - test_pose_cfg["init_weights"] = os.path.join( - str(modelfolder), "train", snapshot_name + test_pose_cfg["init_weights"] = str( + Path(modelfolder) / "train" / snapshot_name ) # setting weights to corresponding snapshot. training_iterations = int(snapshot_name.split("-")[-1]) @@ -807,7 +778,7 @@ def evaluate_network( print("Running evaluation ...") for imageindex, imagename in tqdm(enumerate(Data.index)): image = imread( - os.path.join(cfg["project_path"], *imagename), + Path(cfg["project_path"]).joinpath(*imagename), mode="skimage", ) if scale != 1: @@ -907,10 +878,7 @@ def evaluate_network( if plotting: print("Plotting...") - foldername = os.path.join( - str(evaluationfolder), - "LabeledImages_" + DLCscorer + "_" + snapshot_name, - ) + foldername = Path(evaluationfolder) / ("LabeledImages_" + DLCscorer + "_" + snapshot_name) auxiliaryfunctions.attempt_to_make_folder(foldername) Plotting( cfg, @@ -928,11 +896,8 @@ def evaluate_network( conversioncode.guarantee_multiindex_rows(DataMachine) if plotting: DataCombined = pd.concat([Data.T, DataMachine.T], axis=0, sort=False).T - foldername = os.path.join( - str(evaluationfolder), - "LabeledImages_" + DLCscorer + "_" + snapshot_name, - ) - if not os.path.exists(foldername): + foldername = Path(evaluationfolder) / ("LabeledImages_" + DLCscorer + "_" + snapshot_name) + if not Path(foldername).exists(): print( "Plotting..." "(warning, scale might be inconsistent in comparison " @@ -986,8 +951,8 @@ def make_results_file(final_result, evaluationfolder, DLCscorer): "Test error with p-cutoff", ] df = pd.DataFrame(final_result, columns=col_names) - output_path = os.path.join(str(evaluationfolder), DLCscorer + "-results.csv") - if os.path.exists(output_path): + output_path = Path(evaluationfolder) / (DLCscorer + "-results.csv") + if Path(output_path).exists(): temp = pd.read_csv(output_path, index_col=0) df = pd.concat((temp, df)).reset_index(drop=True) @@ -996,8 +961,8 @@ def make_results_file(final_result, evaluationfolder, DLCscorer): ## Also storing one "large" table with results: # note: evaluationfolder.parents[0] to get common folder above all shuffle evaluations. df = pd.DataFrame(final_result, columns=col_names) - output_path = os.path.join(str(Path(evaluationfolder).parents[0]), "CombinedEvaluation-results.csv") - if os.path.exists(output_path): + output_path = Path(evaluationfolder).parents[0] / "CombinedEvaluation-results.csv" + if Path(output_path).exists(): temp = pd.read_csv(output_path, index_col=0) df = pd.concat((temp, df)).reset_index(drop=True) diff --git a/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py b/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py index 42b4a7b63..da62373c9 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py +++ b/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py @@ -123,7 +123,7 @@ def evaluate_multianimal_full( if gputouse is not None: # gpu selectinon os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse) - start_path = os.getcwd() + start_path = Path.cwd() if plotting is True: plotting = "bodypart" @@ -139,13 +139,7 @@ def evaluate_multianimal_full( # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) - Data = pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) - ) + Data = pd.read_hdf(Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5")) conversioncode.guarantee_multiindex_rows(Data) # Get list of body parts to evaluate network for @@ -153,7 +147,7 @@ def evaluate_multianimal_full( all_bpts = np.asarray(len(cfg["individuals"]) * cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]) colors = visualization.get_cmap(len(comparisonbodyparts), name=cfg["colormap"]) # Make folder for evaluation - auxiliaryfunctions.attempt_to_make_folder(str(cfg["project_path"] + "/evaluation-results/")) + auxiliaryfunctions.attempt_to_make_folder(Path(cfg["project_path"]) / "evaluation-results") for shuffle in Shuffles: for trainFraction in TrainingFractions: ################################################## @@ -188,7 +182,7 @@ def evaluate_multianimal_full( train_pose_cfg = load_config(str(path_train_config)) # Load meta data _, trainIndices, testIndices, _ = auxiliaryfunctions.load_metadata( - os.path.join(cfg["project_path"], train_pose_cfg["metadataset"]) + Path(cfg["project_path"]) / train_pose_cfg["metadataset"] ) pipeline = iaa.Sequential(random_order=False) @@ -206,9 +200,8 @@ def evaluate_multianimal_full( joints = test_pose_cfg["all_joints_names"] # Create folder structure to store results. - evaluationfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + evaluationfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) auxiliaryfunctions.attempt_to_make_folder(evaluationfolder, recursive=True) @@ -243,8 +236,8 @@ def evaluate_multianimal_full( # Compute predictions over images ################################################## for snapshot_name in snapshot_names: - test_pose_cfg["init_weights"] = os.path.join( - str(modelfolder), "train", snapshot_name + test_pose_cfg["init_weights"] = str( + Path(modelfolder) / "train" / snapshot_name ) # setting weights to corresponding snapshot. training_iterations = int(snapshot_name.split("-")[-1]) @@ -276,15 +269,12 @@ def evaluate_multianimal_full( data_path = resultsfilename.split(".h5")[0] + "_full.pickle" if plotting: - foldername = os.path.join( - str(evaluationfolder), - "LabeledImages_" + DLCscorer + "_" + snapshot_name, - ) + foldername = Path(evaluationfolder) / ("LabeledImages_" + DLCscorer + "_" + snapshot_name) auxiliaryfunctions.attempt_to_make_folder(foldername) if plotting == "bodypart": fig, ax = visualization.create_minimal_figure() - if os.path.isfile(data_path): + if Path(data_path).is_file(): print("Model already evaluated.", resultsfilename) else: ( @@ -299,7 +289,7 @@ def evaluate_multianimal_full( conf = np.full_like(dist, np.nan) print("Network Evaluation underway...") for imageindex, imagename in tqdm(enumerate(Data.index)): - image_path = os.path.join(cfg["project_path"], *imagename) + image_path = Path(cfg["project_path"]).joinpath(*imagename) frame = auxfun_videos.imread(image_path, mode="skimage") GT = Data.iloc[imageindex] @@ -428,7 +418,7 @@ def evaluate_multianimal_full( predicted_poses = np.concatenate((predicted_poses, np.expand_dims(conf, axis=-1)), axis=-1) predicted_poses = predicted_poses.reshape(predicted_poses.shape[0], -1) df_predicted_poses = pd.DataFrame(predicted_poses, columns=poses_multi_index) - write_poses_path = os.path.join(evaluationfolder, f"predicted_poses_{training_iterations}.h5") + write_poses_path = Path(evaluationfolder) / f"predicted_poses_{training_iterations}.h5" df_predicted_poses.to_hdf(write_poses_path, key="df_with_missing") # Compute all distance statistics @@ -447,7 +437,7 @@ def evaluate_multianimal_full( ascending=[True, True], inplace=True, ) - write_path = os.path.join(evaluationfolder, f"dist_{training_iterations}.csv") + write_path = Path(evaluationfolder) / f"dist_{training_iterations}.csv" df_joint.to_csv(write_path) # Calculate overall prediction error @@ -563,7 +553,7 @@ def evaluate_multianimal_full( colors = visualization.get_cmap(n_animals, name=cfg["colormap"]) for k, v in tqdm(assemblies.items()): imname = image_paths[k] - image_path = os.path.join(cfg["project_path"], *imname) + image_path = Path(cfg["project_path"]).joinpath(*imname) frame = auxfun_videos.imread(image_path, mode="skimage") h, w, _ = np.shape(frame) @@ -609,7 +599,7 @@ def evaluate_multianimal_full( df.loc(axis=0)[("mAR_train", "mean")] = [d[0]["mAR"] for d in results[2]] df.loc(axis=0)[("mAP_test", "mean")] = [d[1]["mAP"] for d in results[2]] df.loc(axis=0)[("mAR_test", "mean")] = [d[1]["mAR"] for d in results[2]] - with open(data_path.replace("_full.", "_map."), "wb") as file: + with Path(data_path.replace("_full.", "_map.")).open("wb") as file: pickle.dump((df, paf_scores), file) if len(final_result) > 0: # Only append if results were calculated diff --git a/deeplabcut/pose_estimation_tensorflow/core/openvino/session.py b/deeplabcut/pose_estimation_tensorflow/core/openvino/session.py index c4c32ecc7..ae237cf88 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/openvino/session.py +++ b/deeplabcut/pose_estimation_tensorflow/core/openvino/session.py @@ -8,8 +8,8 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os import subprocess +from pathlib import Path import cv2 import numpy as np @@ -30,18 +30,18 @@ def __init__(self, cfg, device): self.device = device # Convert a frozen graph to OpenVINO IR format - if not os.path.exists(self.xml_path): + if not Path(self.xml_path).exists(): subprocess.run( [ "mo", "--output_dir", - os.path.dirname(cfg["init_weights"]), + str(Path(cfg["init_weights"]).parent), "--input_model", cfg["init_weights"] + ".pb", "--input_shape", "[1, 747, 832, 3]", "--extensions", - os.path.join(os.path.dirname(__file__), "mo_extensions"), + str(Path(__file__).parent / "mo_extensions"), "--data_type", "FP16", ], diff --git a/deeplabcut/pose_estimation_tensorflow/core/test.py b/deeplabcut/pose_estimation_tensorflow/core/test.py index 27bb23a35..1c655f81e 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/test.py +++ b/deeplabcut/pose_estimation_tensorflow/core/test.py @@ -14,7 +14,7 @@ import argparse import logging -import os +from pathlib import Path import numpy as np import scipy.io @@ -44,8 +44,8 @@ def test_net(visualise, cache_scoremaps): if cache_scoremaps: out_dir = cfg["scoremap_dir"] - if not os.path.exists(out_dir): - os.makedirs(out_dir) + if not Path(out_dir).exists(): + Path(out_dir).mkdir(parents=True) num_images = dataset.num_images predictions = np.zeros((num_images,), dtype=np.object) @@ -71,12 +71,11 @@ def test_net(visualise, cache_scoremaps): visualize.waitforbuttonpress() if cache_scoremaps: - base = os.path.basename(batch[Batch.data_item].im_path) - raw_name = os.path.splitext(base)[0] - out_fn = os.path.join(out_dir, raw_name + ".mat") + raw_name = Path(batch[Batch.data_item].im_path).stem + out_fn = Path(out_dir) / (raw_name + ".mat") scipy.io.savemat(out_fn, mdict={"scoremaps": scmap.astype("float32")}) - out_fn = os.path.join(out_dir, raw_name + "_locreg" + ".mat") + out_fn = Path(out_dir) / (raw_name + "_locreg.mat") if cfg["location_refinement"]: scipy.io.savemat(out_fn, mdict={"locreg_pred": locref.astype("float32")}) diff --git a/deeplabcut/pose_estimation_tensorflow/core/train.py b/deeplabcut/pose_estimation_tensorflow/core/train.py index 0f4e94a06..01f9762f8 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/train.py +++ b/deeplabcut/pose_estimation_tensorflow/core/train.py @@ -140,7 +140,7 @@ def train( keepdeconvweights=True, allow_growth=True, ): - start_path = os.getcwd() + start_path = Path.cwd() os.chdir(str(Path(config_yaml).parents[0])) # switch to folder of config_yaml (for logging) setup_logging() @@ -251,7 +251,7 @@ def train( lr_gen = LearningRate(cfg) stats_path = Path(config_yaml).with_name("learning_stats.csv") - lrf = open(str(stats_path), "w") + lrf = stats_path.open("w") print("Training parameter:") print(cfg) diff --git a/deeplabcut/pose_estimation_tensorflow/core/train_multianimal.py b/deeplabcut/pose_estimation_tensorflow/core/train_multianimal.py index 88f7e9bc3..ce94dde40 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/train_multianimal.py +++ b/deeplabcut/pose_estimation_tensorflow/core/train_multianimal.py @@ -51,7 +51,7 @@ def train( # in case there was already a graph tf.compat.v1.reset_default_graph() - start_path = os.getcwd() + start_path = Path.cwd() if modelfolder == "": os.chdir(str(Path(config_yaml).parents[0])) # switch to folder of config_yaml (for logging) else: @@ -78,7 +78,7 @@ def train( cfg["log_dir"] = modelfolder cfg["project_path"] = modelfolder # have to overwrite this - cfg["snapshot_prefix"] = os.path.join("snapshot") + cfg["snapshot_prefix"] = "snapshot" if cfg["optimizer"] != "adam": print( @@ -193,7 +193,7 @@ def train( lrf = None if not isinstance(config_yaml, dict): stats_path = Path(config_yaml).with_name("learning_stats.csv") - lrf = open(str(stats_path), "w") + lrf = stats_path.open("w") print("Training parameters:") print(cfg) diff --git a/deeplabcut/pose_estimation_tensorflow/datasets/pose_deterministic.py b/deeplabcut/pose_estimation_tensorflow/datasets/pose_deterministic.py index 8f67c10cb..1e475376d 100644 --- a/deeplabcut/pose_estimation_tensorflow/datasets/pose_deterministic.py +++ b/deeplabcut/pose_estimation_tensorflow/datasets/pose_deterministic.py @@ -11,7 +11,7 @@ import logging -import os +from pathlib import Path import numpy as np import scipy.io as sio @@ -47,7 +47,7 @@ def __init__(self, cfg): def load_dataset(self): cfg = self.cfg - file_name = os.path.join(self.cfg["project_path"], cfg["dataset"]) + file_name = Path(self.cfg["project_path"]) / cfg["dataset"] mlab = sio.loadmat(file_name) self.raw_data = mlab mlab = mlab["dataset"] @@ -66,7 +66,7 @@ def load_dataset(self): im_path = robust_split_path(im_path) else: im_path = [s.strip() for s in im_path] - item.im_path = os.path.join(*im_path) + item.im_path = str(Path(*im_path)) item.im_size = sample[1][0] if len(sample) >= 3: joints = sample[2][0][0] @@ -168,7 +168,7 @@ def make_batch(self, data_item, scale, mirror): im_file = data_item.im_path logging.debug("image %s", im_file) logging.debug("mirror %r", mirror) - image = imread(os.path.join(self.cfg["project_path"], im_file), mode="skimage") + image = imread(Path(self.cfg["project_path"]) / im_file, mode="skimage") if self.has_gt: joints = np.copy(data_item.joints) diff --git a/deeplabcut/pose_estimation_tensorflow/datasets/pose_imgaug.py b/deeplabcut/pose_estimation_tensorflow/datasets/pose_imgaug.py index 0d05c24af..f5cd3a19a 100644 --- a/deeplabcut/pose_estimation_tensorflow/datasets/pose_imgaug.py +++ b/deeplabcut/pose_estimation_tensorflow/datasets/pose_imgaug.py @@ -15,8 +15,8 @@ """ import logging -import os import pickle +from pathlib import Path import imgaug.augmenters as iaa import numpy as np @@ -78,7 +78,7 @@ def __init__(self, cfg): def load_dataset(self): cfg = self.cfg - file_name = os.path.join(self.cfg["project_path"], cfg["dataset"]) + file_name = str(Path(self.cfg["project_path"]) / cfg["dataset"]) if ".mat" in file_name: # legacy loader mlab = sio.loadmat(file_name) self.raw_data = mlab @@ -98,7 +98,7 @@ def load_dataset(self): im_path = robust_split_path(im_path) else: im_path = [s.strip() for s in im_path] - item.im_path = os.path.join(*im_path) + item.im_path = str(Path(*im_path)) item.im_size = sample[1][0] if len(sample) >= 3: joints = sample[2][0][0] @@ -117,7 +117,7 @@ def load_dataset(self): else: print("Loading pickle data with float coordinates!") file_name = cfg["dataset"].split(".")[0] + ".pickle" - with open(os.path.join(self.cfg["project_path"], file_name), "rb") as f: + with (Path(self.cfg["project_path"]) / file_name).open("rb") as f: pickledata = pickle.load(f) self.raw_data = pickledata @@ -128,7 +128,7 @@ def load_dataset(self): sample = pickledata[i] # mlab[0, i] item = DataItem() item.image_id = i - item.im_path = os.path.join(*sample["image"]) # [0][0] + item.im_path = str(Path(*sample["image"])) # [0][0] item.im_size = sample["size"] # sample[1][0] if len(sample) >= 3: item.num_animals = len(sample["joints"]) @@ -310,7 +310,7 @@ def get_batch(self): im_file = data_item.im_path logging.debug(f"image {im_file}") - image = imread(os.path.join(self.cfg["project_path"], im_file), mode="skimage") + image = imread(Path(self.cfg["project_path"]) / im_file, mode="skimage") if self.has_gt: joints = data_item.joints diff --git a/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py b/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py index 2e0bc0344..ebe5bbfce 100644 --- a/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py +++ b/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py @@ -11,7 +11,6 @@ import logging -import os import pickle from math import sqrt from pathlib import Path @@ -42,7 +41,7 @@ def __init__(self, cfg): self._n_animals = 1 else: - self.main_cfg = auxiliaryfunctions.read_config(os.path.join(self.cfg["project_path"], "config.yaml")) + self.main_cfg = auxiliaryfunctions.read_config(Path(self.cfg["project_path"]) / "config.yaml") animals, unique, multi = auxfun_multianimal.extractindividualsandbodyparts(self.main_cfg) self._n_kpts = len(multi) + len(unique) self._n_animals = len(animals) @@ -88,8 +87,8 @@ def load_dataset(self): mask_kpts_below_thresh=mask_kpts_below_thresh, ) - file_name = os.path.join(self.cfg["project_path"], cfg["dataset"]) - with open(os.path.join(self.cfg["project_path"], file_name), "rb") as f: + file_name = Path(self.cfg["project_path"]) / cfg["dataset"] + with (Path(self.cfg["project_path"]) / file_name).open("rb") as f: # Pickle the 'data' dictionary using the highest protocol available. pickledata = pickle.load(f) @@ -105,7 +104,7 @@ def load_dataset(self): im_path = sample["image"] if isinstance(im_path, str): im_path = robust_split_path(im_path) - item.im_path = os.path.join(*im_path) + item.im_path = str(Path(*im_path)) item.im_size = sample["size"] if "joints" in sample.keys(): Joints = sample["joints"] @@ -123,7 +122,7 @@ def load_dataset(self): def _load_pseudo_data_from_h5(self, cfg, threshold=0.5, mask_kpts_below_thresh=False): gt_file = cfg["pseudo_label"] - assert os.path.exists(gt_file) + assert Path(gt_file).exists() path_ = Path(gt_file) print("Using gt file:", path_.name) len(cfg["all_joints_names"]) @@ -140,12 +139,12 @@ def _load_pseudo_data_from_h5(self, cfg, threshold=0.5, mask_kpts_below_thresh=F item.num_joints = kpts.shape[0] joint_ids = np.arange(item.num_joints)[..., np.newaxis] frame_name = "frame_" + str(int(imagename.split("frame")[1])) + ".png" - item.im_path = os.path.join(video_root, frame_name) + item.im_path = str(Path(video_root) / frame_name) if self.vid: item.im_size = self.video_image_size else: - item.im_size = read_image_shape_fast(os.path.join(video_root, frame_name)) + item.im_size = read_image_shape_fast(Path(video_root) / frame_name) item.joints = {} @@ -366,7 +365,7 @@ def get_batch(self): im_file = data_item.im_path logging.debug("image %s", im_file) - image = imread(os.path.join(self.cfg["project_path"], im_file), mode="skimage") + image = imread(Path(self.cfg["project_path"]) / im_file, mode="skimage") if self.has_gt: joints = data_item.joints kpts = np.full((self._n_kpts * self._n_animals, 2), np.nan) @@ -504,7 +503,7 @@ def next_batch(self, plotting=False): shape=batch_images[i].shape, ) im = kps.draw_on_image(batch_images[i]) - imageio.imwrite(os.path.join(self.cfg["project_path"], str(i) + ".png"), im) + imageio.imwrite(Path(self.cfg["project_path"]) / f"{i}.png", im) batch = {Batch.inputs: batch_images.astype(np.float64)} if self.has_gt: diff --git a/deeplabcut/pose_estimation_tensorflow/datasets/pose_tensorpack.py b/deeplabcut/pose_estimation_tensorflow/datasets/pose_tensorpack.py index 66a279d4e..eb45e19f4 100644 --- a/deeplabcut/pose_estimation_tensorflow/datasets/pose_tensorpack.py +++ b/deeplabcut/pose_estimation_tensorflow/datasets/pose_tensorpack.py @@ -21,6 +21,7 @@ import multiprocessing import os +from pathlib import Path import cv2 import numpy as np @@ -99,7 +100,7 @@ def __init__(self, cfg, shuffle=True, dir=None): def load_dataset(self): cfg = self.cfg - file_name = os.path.join(self.cfg["project_path"], cfg["dataset"]) + file_name = Path(self.cfg["project_path"]) / cfg["dataset"] mlab = sio.loadmat(file_name) self.raw_data = mlab mlab = mlab["dataset"] @@ -119,7 +120,7 @@ def load_dataset(self): im_path = robust_split_path(im_path) else: im_path = [s.strip() for s in im_path] - item.im_path = os.path.join(base, *im_path) + item.im_path = str(Path(base).joinpath(*im_path)) item.im_size = sample[1][0] if len(sample) >= 3: joints = sample[2][0][0] diff --git a/deeplabcut/pose_estimation_tensorflow/export.py b/deeplabcut/pose_estimation_tensorflow/export.py index 7924b9af5..b5e59dac1 100644 --- a/deeplabcut/pose_estimation_tensorflow/export.py +++ b/deeplabcut/pose_estimation_tensorflow/export.py @@ -9,8 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import glob -import os import shutil import tarfile from pathlib import Path @@ -62,7 +60,7 @@ def write_deploy_config(configname, cfg): Write structured config file. """ - with open(configname, "w") as cf: + with Path(configname).open("w") as cf: ruamelFile = ruamel.yaml.YAML() cfg_file, ruamelFile = create_deploy_config_template() for key in cfg.keys(): @@ -107,12 +105,11 @@ def load_model(cfg, shuffle=1, trainingsetindex=0, TFGPUinference=True, modelpre ######################## train_fraction = cfg["TrainingFraction"][trainingsetindex] - model_folder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(train_fraction, shuffle, cfg, modelprefix=modelprefix)), + model_folder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(train_fraction, shuffle, cfg, modelprefix=modelprefix) ) - os.path.normpath(model_folder + "/test/pose_cfg.yaml") - path_train_config = os.path.normpath(model_folder + "/train/pose_cfg.yaml") + Path(model_folder) / "test" / "pose_cfg.yaml" + path_train_config = Path(model_folder) / "train" / "pose_cfg.yaml" try: dlc_cfg = load_config(str(path_train_config)) @@ -137,8 +134,8 @@ def load_model(cfg, shuffle=1, trainingsetindex=0, TFGPUinference=True, modelpre #################################### # Check if data already was generated: - dlc_cfg["init_weights"] = os.path.join(model_folder, "train", Snapshots[snapshotindex]) - (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1] + dlc_cfg["init_weights"] = str(Path(model_folder) / "train" / Snapshots[snapshotindex]) + Path(dlc_cfg["init_weights"]).name.split("-")[-1] dlc_cfg["num_outputs"] = cfg.get("num_outputs", dlc_cfg.get("num_outputs", 1)) dlc_cfg["batch_size"] = None @@ -180,21 +177,21 @@ def tf_to_pb(sess, checkpoint, output, output_dir=None): If None, will export to the directory of the checkpoint file. """ - output_dir = os.path.expanduser(output_dir) if output_dir else os.path.dirname(checkpoint) - ckpt_base = os.path.basename(checkpoint) + output_dir = Path(output_dir).expanduser() if output_dir else Path(checkpoint).parent + ckpt_base = Path(checkpoint).name # save graph to pbtxt file - pbtxt_file = os.path.normpath(output_dir + "/" + ckpt_base + ".pbtxt") + pbtxt_file = str(Path(output_dir) / (ckpt_base + ".pbtxt")) tf.io.write_graph(sess.graph.as_graph_def(), "", pbtxt_file, as_text=True) # create frozen graph from pbtxt file - pb_file = os.path.normpath(output_dir + "/" + ckpt_base + ".pb") + pb_file = Path(output_dir) / (ckpt_base + ".pb") frozen_graph_def = tf.compat.v1.graph_util.convert_variables_to_constants( sess, sess.graph_def, output, ) - with open(pb_file, "wb") as file: + with Path(pb_file).open("wb") as file: file.write(frozen_graph_def.SerializeToString()) @@ -263,7 +260,7 @@ def export_model( except FileNotFoundError: FileNotFoundError(f"The config.yaml file at {cfg_path} does not exist.") - cfg["project_path"] = os.path.dirname(os.path.realpath(cfg_path)) + cfg["project_path"] = str(Path(cfg_path).resolve().parent) cfg["iteration"] = iteration if iteration is not None else cfg["iteration"] cfg["batch_size"] = cfg["batch_size"] if cfg["batch_size"] > 1 else 2 cfg["snapshotindex"] = snapshotindex if snapshotindex is not None else cfg["snapshotindex"] @@ -272,22 +269,21 @@ def export_model( sess, input, output, dlc_cfg = load_model(cfg, shuffle, trainingsetindex, TFGPUinference, modelprefix) ckpt = dlc_cfg["init_weights"] - os.path.dirname(ckpt) ### set up export directory - export_dir = os.path.normpath(cfg["project_path"] + "/" + "exported-models") - if not os.path.isdir(export_dir): - os.mkdir(export_dir) + export_dir = Path(cfg["project_path"]) / "exported-models" + if not export_dir.is_dir(): + export_dir.mkdir() sub_dir_name = f"DLC_{cfg['Task']}_{dlc_cfg['net_type']}_iteration-{cfg['iteration']}_shuffle-{shuffle}" - full_export_dir = os.path.normpath(export_dir + "/" + sub_dir_name) + full_export_dir = export_dir / sub_dir_name - if os.path.isdir(full_export_dir): + if full_export_dir.is_dir(): if not overwrite: raise FileExistsError(f"Export directory {full_export_dir} already exists. Terminating export...") else: - os.mkdir(full_export_dir) + full_export_dir.mkdir() ### write pose config file @@ -303,14 +299,14 @@ def export_model( else: sorted_cfg[key] = value - pose_cfg_file = os.path.normpath(full_export_dir + "/pose_cfg.yaml") + pose_cfg_file = full_export_dir / "pose_cfg.yaml" ruamel_file = ruamel.yaml.YAML() - ruamel_file.dump(sorted_cfg, open(pose_cfg_file, "w")) + ruamel_file.dump(sorted_cfg, pose_cfg_file.open("w")) ### copy checkpoint to export directory - ckpt_files = glob.glob(ckpt + "*") - ckpt_dest = [os.path.normpath(full_export_dir + "/" + os.path.basename(ckf)) for ckf in ckpt_files] + ckpt_files = list(Path(ckpt).parent.glob(Path(ckpt).name + "*")) + ckpt_dest = [full_export_dir / Path(ckf).name for ckf in ckpt_files] for ckf, ckd in zip(ckpt_files, ckpt_dest, strict=False): shutil.copy(ckf, ckd) @@ -321,6 +317,6 @@ def export_model( ### tar export directory if make_tar: - tar_name = os.path.normpath(full_export_dir + ".tar.gz") + tar_name = str(full_export_dir) + ".tar.gz" with tarfile.open(tar_name, "w:gz") as tar: - tar.add(full_export_dir, arcname=os.path.basename(full_export_dir)) + tar.add(full_export_dir, arcname=full_export_dir.name) diff --git a/deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py b/deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py index 8c43d3b5e..91d784272 100644 --- a/deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py +++ b/deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py @@ -8,8 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import glob -import os from collections.abc import Sequence from pathlib import Path @@ -105,11 +103,9 @@ def __init__( if not customized_pose_config: dlc_root_path = get_deeplabcut_path() - project_config = read_config( - os.path.join(dlc_root_path, "modelzoo", "project_configs", f"{project_name}.yaml") - ) + project_config = read_config(dlc_root_path / "modelzoo" / "project_configs" / f"{project_name}.yaml") - model_config = read_config(os.path.join(dlc_root_path, "modelzoo", "model_configs", f"{model_name}.yaml")) + model_config = read_config(dlc_root_path / "modelzoo" / "model_configs" / f"{model_name}.yaml") joints = [i for i in range(len(project_config["bodyparts"]))] num_joints = len(joints) @@ -194,7 +190,7 @@ def adaptation_training(self, displayiters=500, saveiters=1000, **kwargs): _, pseudo_label_path, _, _ = load_analyzed_data(video_root, vname, DLCscorer, False, "") if self.modelfolder != "": - os.makedirs(self.modelfolder, exist_ok=True) + Path(self.modelfolder).mkdir(parents=True, exist_ok=True) self.adapt_iterations = kwargs.get("adapt_iterations", self.adapt_iterations) @@ -206,10 +202,9 @@ def adaptation_training(self, displayiters=500, saveiters=1000, **kwargs): ) def after_adapt_inference(self, create_labeled_video, **kwargs): - pattern = os.path.join(self.modelfolder, f"snapshot-{self.adapt_iterations}.index") ref_proj_config_path = "" - files = glob.glob(pattern) + files = list(Path(self.modelfolder).glob(f"snapshot-{self.adapt_iterations}.index")) if not len(files): raise ValueError("Weights were not found.") diff --git a/deeplabcut/pose_estimation_tensorflow/modelzoo/api/superanimal_inference.py b/deeplabcut/pose_estimation_tensorflow/modelzoo/api/superanimal_inference.py index 8e6444a0a..20bd61812 100644 --- a/deeplabcut/pose_estimation_tensorflow/modelzoo/api/superanimal_inference.py +++ b/deeplabcut/pose_estimation_tensorflow/modelzoo/api/superanimal_inference.py @@ -8,9 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import glob -import os -import os.path import pickle import time import warnings @@ -263,22 +260,8 @@ def video_inference( dlc_root_path = auxiliaryfunctions.get_deeplabcut_path() if customized_test_config == "": - project_cfg = load_config( - os.path.join( - dlc_root_path, - "modelzoo", - "project_configs", - f"{project_name}.yaml", - ) - ) - model_cfg = load_config( - os.path.join( - dlc_root_path, - "modelzoo", - "model_configs", - f"{model_name}.yaml", - ) - ) + project_cfg = load_config(dlc_root_path / "modelzoo" / "project_configs" / f"{project_name}.yaml") + model_cfg = load_config(dlc_root_path / "modelzoo" / "model_configs" / f"{model_name}.yaml") test_cfg = {**project_cfg, **model_cfg} test_cfg["all_joints"] = [i for i in range(len(test_cfg["bobyparts"]))] test_cfg["all_joints_names"] = test_cfg["bobyparts"] @@ -290,8 +273,8 @@ def video_inference( test_cfg = customized_test_config # add a temp folder for checkpoint - weight_folder = str(Path(dlc_root_path) / "modelzoo" / "checkpoints" / f"{project_name}_{model_name}") - snapshots = glob.glob(os.path.join(weight_folder, "snapshot-*.index")) + weight_folder = str(dlc_root_path / "modelzoo" / "checkpoints" / f"{project_name}_{model_name}") + snapshots = list(Path(weight_folder).glob("snapshot-*.index")) test_cfg["partaffinityfield_graph"] = [] test_cfg["partaffinityfield_predict"] = False @@ -301,7 +284,7 @@ def video_inference( if len(snapshots) == 0: raise FileNotFoundError(f"Did not find any super animal snapshots in {weight_folder}") - init_weights = os.path.abspath(snapshots[0]).replace(".index", "") + init_weights = str(Path(snapshots[0]).resolve()).replace(".index", "") test_cfg["init_weights"] = init_weights test_cfg["num_outputs"] = 1 @@ -320,10 +303,10 @@ def video_inference( destfolder = videofolder auxiliaryfunctions.attempt_to_make_folder(destfolder) - dataname = os.path.join(destfolder, vname + DLCscorer + ".h5") + dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5")) datafiles.append(dataname) - if os.path.isfile(dataname): + if Path(dataname).is_file(): print("Video already analyzed!", dataname) else: print("Loading ", video) @@ -395,7 +378,7 @@ def video_inference( metadata_path = dataname.split(".h5")[0] + "_meta.pickle" - with open(metadata_path, "wb") as f: + with Path(metadata_path).open("wb") as f: pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) xyz_labs = ["x", "y", "likelihood"] diff --git a/deeplabcut/pose_estimation_tensorflow/predict_multianimal.py b/deeplabcut/pose_estimation_tensorflow/predict_multianimal.py index 00c23b24d..6dad3fa39 100644 --- a/deeplabcut/pose_estimation_tensorflow/predict_multianimal.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_multianimal.py @@ -48,7 +48,7 @@ def extract_bpt_feature_from_video( protocol=pickle.DEFAULT_PROTOCOL, ) - with open(destfolder / f"{basename}_assemblies.pickle", "rb") as f: + with (destfolder / f"{basename}_assemblies.pickle").open("rb") as f: assemblies = pickle.load(f) print("Loading ", video) @@ -204,7 +204,7 @@ def AnalyzeMultiAnimalVideo( print(f"Video Analyzed. Saving results in {destfolder}") if use_shelve: - with open(destfolder / f"{basename}_meta.pickle", "wb") as f: + with (destfolder / f"{basename}_meta.pickle").open("wb") as f: pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) else: auxfun_multianimal.SaveFullMultiAnimalData(predicted_data, metadata, str(destfolder / f"{basename}.h5")) diff --git a/deeplabcut/pose_estimation_tensorflow/predict_videos.py b/deeplabcut/pose_estimation_tensorflow/predict_videos.py index a3b11440a..93a0171c0 100644 --- a/deeplabcut/pose_estimation_tensorflow/predict_videos.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_videos.py @@ -16,7 +16,6 @@ import argparse import os -import os.path import pickle import re import time @@ -91,7 +90,7 @@ def create_tracking_dataset( auxfun_models.set_visible_devices(gputouse) tf.compat.v1.reset_default_graph() - start_path = os.getcwd() # record cwd to return to this directory in the end + start_path = Path.cwd() # record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg["TrainingFraction"][trainingsetindex] @@ -102,9 +101,8 @@ def create_tracking_dataset( print("Overwriting cropping parameters:", cropping) print("These are used for all videos, but won't be save to the cfg file.") - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" try: @@ -136,8 +134,8 @@ def create_tracking_dataset( ################################################## # Check if data already was generated: - dlc_cfg["init_weights"] = os.path.join(modelfolder, "train", Snapshots[snapshotindex]) - trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1] + dlc_cfg["init_weights"] = str(Path(modelfolder) / "train" / Snapshots[snapshotindex]) + trainingsiterations = Path(dlc_cfg["init_weights"]).name.split("-")[-1] # Update number of output and batchsize dlc_cfg["num_outputs"] = cfg.get("num_outputs", dlc_cfg.get("num_outputs", 1)) @@ -491,7 +489,7 @@ def analyze_videos( auxfun_models.set_visible_devices(gputouse) tf.compat.v1.reset_default_graph() - start_path = os.getcwd() # record cwd to return to this directory in the end + start_path = Path.cwd() # record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg["TrainingFraction"][trainingsetindex] @@ -503,9 +501,8 @@ def analyze_videos( print("Overwriting cropping parameters:", cropping) print("These are used for all videos, but won't be save to the cfg file.") - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" try: @@ -538,8 +535,8 @@ def analyze_videos( ################################################## # Check if data already was generated: - dlc_cfg["init_weights"] = os.path.join(modelfolder, "train", Snapshots[snapshotindex]) - trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1] + dlc_cfg["init_weights"] = str(Path(modelfolder) / "train" / Snapshots[snapshotindex]) + trainingsiterations = Path(dlc_cfg["init_weights"]).name.split("-")[-1] # Update number of output and batchsize dlc_cfg["num_outputs"] = cfg.get("num_outputs", dlc_cfg.get("num_outputs", 1)) @@ -1078,7 +1075,7 @@ def AnalyzeVideo( metadata = {"data": dictionary} print(f"Saving results in {destfolder}...") - dataname = os.path.join(destfolder, vname + DLCscorer + ".h5") + dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5")) auxiliaryfunctions.save_data( PredictedData[:nframes, :], metadata, @@ -1095,7 +1092,7 @@ def GetPosesofFrames(cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, from deeplabcut.utils.auxfun_videos import imread print("Starting to extract posture") - im = imread(os.path.join(directory, framelist[0]), mode="skimage") + im = imread(Path(directory) / framelist[0], mode="skimage") ny, nx, nc = np.shape(im) print( @@ -1132,7 +1129,7 @@ def GetPosesofFrames(cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, if batchsize == 1: for counter, framename in enumerate(framelist): - im = imread(os.path.join(directory, framename), mode="skimage") + im = imread(Path(directory) / framename, mode="skimage") if counter != 0 and counter % step == 0: pbar.update(step) @@ -1147,7 +1144,7 @@ def GetPosesofFrames(cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, else: frames = np.empty((batchsize, ny, nx, 3), dtype="ubyte") # this keeps all the frames of a batch for counter, framename in enumerate(framelist): - im = imread(os.path.join(directory, framename), mode="skimage") + im = imread(Path(directory) / framename, mode="skimage") if counter != 0 and counter % step == 0: pbar.update(step) @@ -1246,13 +1243,12 @@ def analyze_time_lapse_frames( auxfun_models.set_visible_devices(gputouse) tf.compat.v1.reset_default_graph() - start_path = os.getcwd() # record cwd to return to this directory in the end + start_path = Path.cwd() # record cwd to return to this directory in the end cfg = auxiliaryfunctions.read_config(config) trainFraction = cfg["TrainingFraction"][trainingsetindex] - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" try: @@ -1284,8 +1280,8 @@ def analyze_time_lapse_frames( ################################################## # Check if data already was generated: - dlc_cfg["init_weights"] = os.path.join(modelfolder, "train", Snapshots[snapshotindex]) - trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1] + dlc_cfg["init_weights"] = str(Path(modelfolder) / "train" / Snapshots[snapshotindex]) + trainingsiterations = Path(dlc_cfg["init_weights"]).name.split("-")[-1] # update batchsize (based on parameters in config.yaml) dlc_cfg["batch_size"] = cfg["batch_size"] @@ -1320,11 +1316,11 @@ def analyze_time_lapse_frames( # Loading the images ################################################## # checks if input is a directory - if os.path.isdir(directory): + if Path(directory).is_dir(): """Analyzes all the frames in the directory.""" print("Analyzing all frames in the directory: ", directory) os.chdir(directory) - framelist = np.sort([fn for fn in os.listdir(os.curdir) if (frametype in fn)]) + framelist = np.sort([fn.name for fn in Path(directory).iterdir() if (frametype in fn.name)]) vname = Path(directory).stem notanalyzed, dataname, DLCscorer = auxiliaryfunctions.check_if_not_analyzed( directory, vname, DLCscorer, DLCscorerlegacy, flag="framestack" @@ -1441,15 +1437,13 @@ def _convert_detections_to_tracklets( ) if calibrate: trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) - train_data_file = os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", + train_data_file = ( + Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5") ) assembly_builder.calibrate(train_data_file) assembly_builder.assemble() - output_path, _ = os.path.splitext(output_path) + output_path = str(Path(output_path).with_suffix("")) output_path += ".pickle" assembly_builder.to_pickle(output_path.replace(".pickle", "_assemblies.pickle")) @@ -1480,7 +1474,7 @@ def _convert_detections_to_tracklets( names=["scorer", "bodyparts", "coords"], ) tracklets["header"] = pdindex - with open(output_path, "wb") as f: + with Path(output_path).open("wb") as f: pickle.dump(tracklets, f, pickle.HIGHEST_PROTOCOL) @@ -1597,7 +1591,7 @@ def convert_detections2tracklets( auxiliaryfunctions.write_config(config, cfg) trainFraction = cfg["TrainingFraction"][trainingsetindex] - start_path = os.getcwd() # record cwd to return to this directory in the end + start_path = Path.cwd() # record cwd to return to this directory in the end # TODO: add cropping as in video analysis! # if cropping is not None: @@ -1606,9 +1600,8 @@ def convert_detections2tracklets( # print("Overwriting cropping parameters:", cropping) # print("These are used for all videos, but won't be save to the cfg file.") - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" try: @@ -1650,8 +1643,8 @@ def convert_detections2tracklets( snapshotindex = cfg["snapshotindex"] print(f"Using {Snapshots[snapshotindex]}", "for model", modelfolder) - dlc_cfg["init_weights"] = os.path.join(modelfolder, "train", Snapshots[snapshotindex]) - trainingsiterations = (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[-1] + dlc_cfg["init_weights"] = str(Path(modelfolder) / "train" / Snapshots[snapshotindex]) + trainingsiterations = Path(dlc_cfg["init_weights"]).name.split("-")[-1] # Name for scorer: DLCscorer, DLCscorerlegacy = auxiliaryfunctions.get_scorer_name( @@ -1674,7 +1667,7 @@ def convert_detections2tracklets( destfolder = videofolder auxiliaryfunctions.attempt_to_make_folder(destfolder) vname = Path(video).stem - dataname = os.path.join(destfolder, vname + DLCscorer + ".h5") + dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5")) data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(dataname) if track_method == "ellipse": method = "el" @@ -1685,7 +1678,7 @@ def convert_detections2tracklets( trackname = dataname.split(".h5")[0] + f"_{method}.pickle" # NOTE: If dataname line above is changed then line below is obsolete? # trackname = trackname.replace(videofolder, destfolder) - if os.path.isfile(trackname) and not overwrite: # TODO: check if metadata are identical (same parameters!) + if Path(trackname).is_file() and not overwrite: # TODO: check if metadata are identical (same parameters!) print("Tracklets already computed", trackname) print("Set overwrite = True to overwrite.") else: @@ -1741,13 +1734,13 @@ def convert_detections2tracklets( min_n_links=inferencecfg["minimalnumberofconnections"], ) assemblies_filename = dataname.split(".h5")[0] + "_assemblies.pickle" - if not os.path.exists(assemblies_filename) or overwrite: + if not Path(assemblies_filename).exists() or overwrite: if calibrate: trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) - train_data_file = os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", + train_data_file = ( + Path(cfg["project_path"]) + / str(trainingsetfolder) + / ("CollectedData_" + cfg["scorer"] + ".h5") ) assembly_builder.calibrate(train_data_file) assembly_builder.assemble() @@ -1806,7 +1799,7 @@ def convert_detections2tracklets( trackingutils.fill_tracklets(tracklets, trackers, animals, imname) tracklets["header"] = pdindex - with open(trackname, "wb") as f: + with Path(trackname).open("wb") as f: pickle.dump(tracklets, f, pickle.HIGHEST_PROTOCOL) os.chdir(str(start_path)) diff --git a/deeplabcut/pose_estimation_tensorflow/training.py b/deeplabcut/pose_estimation_tensorflow/training.py index 61790e633..5a16d8f35 100644 --- a/deeplabcut/pose_estimation_tensorflow/training.py +++ b/deeplabcut/pose_estimation_tensorflow/training.py @@ -38,9 +38,9 @@ def return_train_network_path(config, shuffle=1, trainingsetindex=0, modelprefix modelfoldername = auxiliaryfunctions.get_model_folder( cfg["TrainingFraction"][trainingsetindex], shuffle, cfg, modelprefix=modelprefix ) - trainposeconfigfile = Path(os.path.join(cfg["project_path"], str(modelfoldername), "train", "pose_cfg.yaml")) - testposeconfigfile = Path(os.path.join(cfg["project_path"], str(modelfoldername), "test", "pose_cfg.yaml")) - snapshotfolder = Path(os.path.join(cfg["project_path"], str(modelfoldername), "train")) + trainposeconfigfile = Path(cfg["project_path"]) / str(modelfoldername) / "train" / "pose_cfg.yaml" + testposeconfigfile = Path(cfg["project_path"]) / str(modelfoldername) / "test" / "pose_cfg.yaml" + snapshotfolder = Path(cfg["project_path"]) / str(modelfoldername) / "train" return trainposeconfigfile, testposeconfigfile, snapshotfolder @@ -164,14 +164,14 @@ def train_network( from deeplabcut.utils import auxiliaryfunctions tf.compat.v1.reset_default_graph() - start_path = os.getcwd() + start_path = Path.cwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) modelfoldername = auxiliaryfunctions.get_model_folder( cfg["TrainingFraction"][trainingsetindex], shuffle, cfg, modelprefix=modelprefix ) - poseconfigfile = Path(os.path.join(cfg["project_path"], str(modelfoldername), "train", "pose_cfg.yaml")) + poseconfigfile = Path(cfg["project_path"]) / str(modelfoldername) / "train" / "pose_cfg.yaml" if not poseconfigfile.is_file(): print("The training datafile ", poseconfigfile, " is not present.") print("Probably, the training dataset for this specific shuffle index was not created.") @@ -189,8 +189,6 @@ def train_network( cfg_dlc = auxiliaryfunctions.read_plainconfig(poseconfigfile) if superanimal_name != "": - import glob - from dlclibrary.dlcmodelzoo.modelzoo_download import ( MODELOPTIONS, download_huggingface_model, @@ -209,7 +207,7 @@ def train_network( ) if superanimal_name in MODELOPTIONS: - if not os.path.exists(weight_folder): + if not Path(weight_folder).exists(): download_huggingface_model(superanimal_name, weight_folder) else: print(f"{weight_folder} exists, using the downloaded weights") @@ -219,8 +217,8 @@ def train_network( MODELOPTIONS, ) - snapshots = glob.glob(os.path.join(weight_folder, "snapshot-*.index")) - init_weights = os.path.abspath(snapshots[0]).replace(".index", "") + snapshots = list(Path(weight_folder).glob("snapshot-*.index")) + init_weights = str(Path(snapshots[0]).resolve()).replace(".index", "") from deeplabcut.pose_estimation_tensorflow.core.train_multianimal import ( train, diff --git a/deeplabcut/pose_estimation_tensorflow/util/logging.py b/deeplabcut/pose_estimation_tensorflow/util/logging.py index 42f0ab7ea..18a18b8c9 100644 --- a/deeplabcut/pose_estimation_tensorflow/util/logging.py +++ b/deeplabcut/pose_estimation_tensorflow/util/logging.py @@ -17,13 +17,12 @@ """ import logging -import os def setup_logging(): FORMAT = "%(asctime)-15s %(message)s" logging.basicConfig( - filename=os.path.join("log.txt"), + filename="log.txt", filemode="a", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO, diff --git a/deeplabcut/pose_estimation_tensorflow/visualizemaps.py b/deeplabcut/pose_estimation_tensorflow/visualizemaps.py index 3cd8b3d6e..dcef085c9 100644 --- a/deeplabcut/pose_estimation_tensorflow/visualizemaps.py +++ b/deeplabcut/pose_estimation_tensorflow/visualizemaps.py @@ -9,6 +9,7 @@ # Licensed under GNU Lesser General Public License v3.0 # import os +from pathlib import Path import matplotlib.pyplot as plt from skimage.transform import resize @@ -61,7 +62,6 @@ def extract_maps( If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0. >>> deeplabcut.extract_maps(configfile,0,Indices=[0,103]) """ - from pathlib import Path import numpy as np import pandas as pd @@ -83,7 +83,7 @@ def extract_maps( os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # # tf.logging.set_verbosity(tf.logging.WARN) - start_path = os.getcwd() + start_path = Path.cwd() # Read file path for pose_config file. >> pass it on cfg = auxiliaryfunctions.read_config(config) @@ -105,16 +105,10 @@ def extract_maps( # Loading human annotatated data trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg) - Data = pd.read_hdf( - os.path.join( - cfg["project_path"], - str(trainingsetfolder), - "CollectedData_" + cfg["scorer"] + ".h5", - ) - ) + Data = pd.read_hdf(Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5")) # Make folder for evaluation - auxiliaryfunctions.attempt_to_make_folder(str(cfg["project_path"] + "/evaluation-results/")) + auxiliaryfunctions.attempt_to_make_folder(Path(cfg["project_path"]) / "evaluation-results") Maps = {} for trainFraction in TrainingFractions: @@ -126,9 +120,8 @@ def extract_maps( trainingsetfolder, trainFraction, shuffle, cfg ) - modelfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + modelfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml" # Load meta data @@ -137,7 +130,7 @@ def extract_maps( trainIndices, testIndices, trainFraction, - ) = auxiliaryfunctions.load_metadata(os.path.join(cfg["project_path"], metadatafn)) + ) = auxiliaryfunctions.load_metadata(Path(cfg["project_path"]) / metadatafn) try: dlc_cfg = load_config(str(path_test_config)) except FileNotFoundError as e: @@ -149,9 +142,8 @@ def extract_maps( dlc_cfg["batch_size"] = 1 # in case this was edited for analysis. # Create folder structure to store results. - evaluationfolder = os.path.join( - cfg["project_path"], - str(auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)), + evaluationfolder = Path(cfg["project_path"]) / str( + auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix) ) auxiliaryfunctions.attempt_to_make_folder(evaluationfolder, recursive=True) @@ -175,12 +167,10 @@ def extract_maps( bptnames = [dlc_cfg["all_joints_names"][i] for i in range(len(dlc_cfg["all_joints"]))] for snapindex in snapindices: - dlc_cfg["init_weights"] = os.path.join( - str(modelfolder), "train", Snapshots[snapindex] + dlc_cfg["init_weights"] = str( + Path(modelfolder) / "train" / Snapshots[snapindex] ) # setting weights to corresponding snapshot. - (dlc_cfg["init_weights"].split(os.sep)[-1]).split("-")[ - -1 - ] # read how many training siterations that corresponds to. + Path(dlc_cfg["init_weights"]).name.split("-")[-1] # read how many training siterations that corresponds to. # Name for deeplabcut net (based on its parameters) # DLCscorer,DLCscorerlegacy = @@ -204,7 +194,7 @@ def extract_maps( DATA = {} for imageindex, imagename in tqdm(Indices): - image = imread(os.path.join(cfg["project_path"], *imagename), mode="skimage") + image = imread(Path(cfg["project_path"]).joinpath(*imagename), mode="skimage") if scale != 1: image = imresize(image, scale) @@ -334,14 +324,14 @@ def extract_save_all_maps( print("Saving plots...") for frac, values in data.items(): if not dest_folder: - dest_folder = os.path.join( - cfg["project_path"], - str(get_evaluation_folder(frac, shuffle, cfg, modelprefix=modelprefix)), - "maps", + dest_folder = ( + Path(cfg["project_path"]) + / str(get_evaluation_folder(frac, shuffle, cfg, modelprefix=modelprefix)) + / "maps" ) attempt_to_make_folder(dest_folder) filepath = "{imname}_{map}_{label}_{shuffle}_{frac}_{snap}.png" - dest_path = os.path.join(dest_folder, filepath) + dest_path = Path(dest_folder) / filepath for snap, maps in values.items(): for imagenr in tqdm(maps): ( diff --git a/deeplabcut/pose_tracking_pytorch/apis.py b/deeplabcut/pose_tracking_pytorch/apis.py index 20a80e708..48ac5b5bf 100644 --- a/deeplabcut/pose_tracking_pytorch/apis.py +++ b/deeplabcut/pose_tracking_pytorch/apis.py @@ -95,7 +95,7 @@ def transformer_reID( >>> ) -------- """ - import os + from pathlib import Path import deeplabcut from deeplabcut.utils import auxiliaryfunctions @@ -146,9 +146,9 @@ def transformer_reID( destfolder=destfolder, ) - transformer_checkpoint = os.path.join(snapshotfolder, f"dlc_transreid_{train_epochs}.pth") + transformer_checkpoint = Path(snapshotfolder) / f"dlc_transreid_{train_epochs}.pth" - if not os.path.exists(transformer_checkpoint): + if not transformer_checkpoint.exists(): raise FileNotFoundError(f"checkpoint {transformer_checkpoint} not found") deeplabcut.stitch_tracklets( diff --git a/deeplabcut/pose_tracking_pytorch/config/__init__.py b/deeplabcut/pose_tracking_pytorch/config/__init__.py index be0434341..933769bfb 100644 --- a/deeplabcut/pose_tracking_pytorch/config/__init__.py +++ b/deeplabcut/pose_tracking_pytorch/config/__init__.py @@ -9,7 +9,7 @@ # Licensed under GNU Lesser General Public License v3.0 # -import os +from pathlib import Path from deeplabcut.utils.auxiliaryfunctions import ( get_deeplabcut_path, @@ -17,5 +17,5 @@ ) dlcparent_path = get_deeplabcut_path() -reid_config = os.path.join(dlcparent_path, "reid_cfg.yaml") +reid_config = dlcparent_path / "reid_cfg.yaml" cfg = read_plainconfig(reid_config) diff --git a/deeplabcut/pose_tracking_pytorch/create_dataset.py b/deeplabcut/pose_tracking_pytorch/create_dataset.py index e7e3e7752..9c9289712 100644 --- a/deeplabcut/pose_tracking_pytorch/create_dataset.py +++ b/deeplabcut/pose_tracking_pytorch/create_dataset.py @@ -9,7 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import os import pickle import shelve from pathlib import Path @@ -61,7 +60,7 @@ def save_train_triplets(feature_fname, triplets, out_name): ret_vecs = np.array(ret_vecs) - with open(out_name, "wb") as f: + with Path(out_name).open("wb") as f: np.save(f, ret_vecs) @@ -79,16 +78,16 @@ def create_triplets_dataset(videos, dlcscorer, track_method, n_triplets=1000, de videofolder = str(Path(video).parents[0]) if destfolder is None: destfolder = videofolder - feature_fname = os.path.join(destfolder, vname + dlcscorer + "_bpt_features.pickle") + feature_fname = str(Path(destfolder) / (vname + dlcscorer + "_bpt_features.pickle")) method = trackingutils.TRACK_METHODS[track_method] - track_file = os.path.join(destfolder, vname + dlcscorer + f"{method}.pickle") - if not Path(track_file).exists(): + track_file = Path(destfolder) / (vname + dlcscorer + f"{method}.pickle") + if not track_file.exists(): raise ValueError( f"Tracklet file {track_file} does not exist. Please run " f"`analyze_videos` with the {method} tracker before using the ReID " "transformer." ) - out_fname = os.path.join(destfolder, vname + dlcscorer + "_triplet_vector.npy") + out_fname = str(Path(destfolder) / (vname + dlcscorer + "_triplet_vector.npy")) create_train_using_pickle(feature_fname, track_file, out_fname, n_triplets=n_triplets) diff --git a/deeplabcut/pose_tracking_pytorch/processor/processor.py b/deeplabcut/pose_tracking_pytorch/processor/processor.py index 05374b54b..5d3302314 100644 --- a/deeplabcut/pose_tracking_pytorch/processor/processor.py +++ b/deeplabcut/pose_tracking_pytorch/processor/processor.py @@ -10,9 +10,9 @@ # import logging -import os import pickle import time +from pathlib import Path import numpy as np import torch @@ -156,7 +156,7 @@ def do_dlc_train( "num_kpts": num_kpts, "feature_dim": feature_dim, }, - os.path.join(ckpt_folder, model_name + f"_{epoch}.pth"), + Path(ckpt_folder) / (model_name + f"_{epoch}.pth"), ) if epoch % eval_period == 0: @@ -193,7 +193,7 @@ def do_dlc_train( plot_dict["test_acc"] = test_acc_list plot_dict["epochs"] = epoch_list - with open(os.path.join(ckpt_folder, "dlc_transreid_results.pickle"), "wb") as handle: + with (Path(ckpt_folder) / "dlc_transreid_results.pickle").open("wb") as handle: pickle.dump(plot_dict, handle, protocol=pickle.HIGHEST_PROTOCOL) @@ -247,9 +247,9 @@ def do_dlc_inference(cfg, model, triplet_loss, val_loader, num_query): val_loss += loss.item() features_list = np.vstack(features_list) - with open("video_trans_features.npy", "wb") as f: + with Path("video_trans_features.npy").open("wb") as f: np.save(f, features_list) - with open("labels.npy", "wb") as f: + with Path("labels.npy").open("wb") as f: np.save(f, labels_list) print(f"validation loss {val_loss / len(val_loader)}") print(f" acc {total_correct / total_n}") diff --git a/deeplabcut/pose_tracking_pytorch/train_dlctransreid.py b/deeplabcut/pose_tracking_pytorch/train_dlctransreid.py index 3ba46b72f..1e987df69 100644 --- a/deeplabcut/pose_tracking_pytorch/train_dlctransreid.py +++ b/deeplabcut/pose_tracking_pytorch/train_dlctransreid.py @@ -15,8 +15,6 @@ import torch except ModuleNotFoundError as e: raise ModuleNotFoundError("Unsupervised identity learning requires PyTorch. Please run `pip install torch`.") from e -import glob -import os from collections.abc import Sequence from pathlib import Path @@ -89,7 +87,7 @@ def train_tracking_transformer( destfolder = videofolder video_name = Path(video).stem # video_name = '.'.join(video.split("/")[-1].split(".")[:-1]) - files = glob.glob(os.path.join(destfolder, video_name + dlcscorer + "*.npy")) + files = [str(p) for p in Path(destfolder).glob(video_name + dlcscorer + "*.npy")] # assuming there is only one match npy_list.append(files[0]) diff --git a/deeplabcut/post_processing/analyze_skeleton.py b/deeplabcut/post_processing/analyze_skeleton.py index 1899bcdb7..0dfa45671 100644 --- a/deeplabcut/post_processing/analyze_skeleton.py +++ b/deeplabcut/post_processing/analyze_skeleton.py @@ -14,7 +14,6 @@ """ import argparse -import os from collections.abc import Sequence from math import atan2, degrees from pathlib import Path @@ -169,8 +168,8 @@ def analyzebone(bp1, bp2): # MAIN FUNC @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def analyzeskeleton( - config, - videos, + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle=1, trainingsetindex=0, @@ -285,7 +284,7 @@ def analyzeskeleton( continue output_name = filepath.replace(".h5", "_skeleton.h5") - if os.path.isfile(output_name): + if Path(output_name).is_file(): print(f"Skeleton in video {vname} already processed. Skipping...") video_to_skeleton_df[video] = pd.read_hdf(output_name, "df_with_missing") continue diff --git a/deeplabcut/post_processing/filtering.py b/deeplabcut/post_processing/filtering.py index cb9c81445..6b8a3a5a6 100644 --- a/deeplabcut/post_processing/filtering.py +++ b/deeplabcut/post_processing/filtering.py @@ -67,8 +67,8 @@ def columnwise_spline_interp(data, max_gap=0): @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def filterpredictions( - config, - video, + config: str | Path, + video: str | Path, video_extensions: str | Sequence[str] | None = None, shuffle=1, trainingsetindex=0, diff --git a/deeplabcut/refine_training_dataset/outlier_frames.py b/deeplabcut/refine_training_dataset/outlier_frames.py index 38fc04084..de1ab8fa0 100644 --- a/deeplabcut/refine_training_dataset/outlier_frames.py +++ b/deeplabcut/refine_training_dataset/outlier_frames.py @@ -11,7 +11,6 @@ import argparse -import os import pickle import re from collections.abc import Sequence @@ -36,9 +35,9 @@ def find_outliers_in_raw_data( - config, - pickle_file, - video_file, + config: str | Path, + pickle_file: str | Path, + video_file: str | Path, pcutoff=0.1, percentiles=(5, 95), with_annotations=True, @@ -86,7 +85,7 @@ def find_outliers_in_raw_data( if not pickle_name.startswith(video_name): raise ValueError("Video and pickle files do not match.") - with open(pickle_file, "rb") as file: + with Path(pickle_file).open("rb") as file: data = pickle.load(file) if pickle_file.endswith("_full.pickle"): inds, data = find_outliers_in_raw_detections(data, threshold=pcutoff) @@ -200,8 +199,8 @@ def _read_video_specific_cropping_margins(config: str | Path | dict, video_path: @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def extract_outlier_frames( - config, - videos, + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle=1, trainingsetindex=0, @@ -417,7 +416,7 @@ def extract_outlier_frames( videofolder = str(Path(video).parents[0]) else: videofolder = destfolder - vname = os.path.splitext(os.path.basename(video))[0] + vname = Path(video).stem try: df, dataname, _, _ = auxiliaryfunctions.load_analyzed_data( @@ -732,11 +731,11 @@ def ExtractFramesbasedonPreselection( str(Path(video).parents[0]) vname = str(Path(video).stem) - tmpfolder = os.path.join(cfg["project_path"], "labeled-data", vname) - if os.path.isdir(tmpfolder): + tmpfolder = Path(cfg["project_path"]) / "labeled-data" / vname + if tmpfolder.is_dir(): print("Frames from video", vname, " already extracted (more will be added)!") else: - auxiliaryfunctions.attempt_to_make_folder(tmpfolder, recursive=True) + auxiliaryfunctions.attempt_to_make_folder(str(tmpfolder), recursive=True) nframes = len(data) print("Loading video...") @@ -866,7 +865,7 @@ def ExtractFramesbasedonPreselection( pass if with_annotations: - machinefile = os.path.join(tmpfolder, "machinelabels-iter" + str(cfg["iteration"]) + ".h5") + machinefile = Path(tmpfolder) / ("machinelabels-iter" + str(cfg["iteration"]) + ".h5") if isinstance(data, pd.DataFrame): df = data.loc[frames2pick] df.index = pd.MultiIndex.from_tuples( @@ -890,7 +889,7 @@ def ExtractFramesbasedonPreselection( for index in frames2pick ] ) - filename = os.path.join(str(tmpfolder), f"CollectedData_{cfg['scorer']}.h5") + filename = str(Path(tmpfolder) / f"CollectedData_{cfg['scorer']}.h5") try: df_temp = pd.read_hdf(filename, "df_with_missing") columns = df_temp.columns @@ -939,11 +938,11 @@ def ExtractFramesbasedonPreselection( DataCombined.to_hdf(machinefile, key="df_with_missing", mode="w") DataCombined.to_csv( - os.path.join(tmpfolder, "machinelabels.csv") + Path(tmpfolder) / "machinelabels.csv" ) # this is always the most current one (as reading is from h5) else: df.to_hdf(machinefile, key="df_with_missing", mode="w") - df.to_csv(os.path.join(tmpfolder, "machinelabels.csv")) + df.to_csv(Path(tmpfolder) / "machinelabels.csv") print(rf"The outlier frames are extracted. They are stored in the subdirectory labeled-data\{vname}.") print("Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels.") @@ -967,13 +966,13 @@ def PlottingSingleFrame( """Label frame and save under imagename / this is already cropped (for clip)""" from skimage import io - imagename1 = os.path.join(tmpfolder, "img" + str(index).zfill(strwidth) + ".png") - imagename2 = os.path.join(tmpfolder, "img" + str(index).zfill(strwidth) + "labeled.png") + imagename1 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + ".png") + imagename2 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + "labeled.png") - if not os.path.isfile(os.path.join(tmpfolder, "img" + str(index).zfill(strwidth) + ".png")): + if not imagename1.is_file(): plt.axis("off") image = img_as_ubyte(clip.get_frame(index * 1.0 / clip.fps)) - io.imsave(imagename1, image) + io.imsave(str(imagename1), image) if savelabeled: if np.ndim(image) > 2: @@ -1028,10 +1027,10 @@ def PlottingSingleFramecv2( """Label frame and save under imagename / cap is not already cropped.""" from skimage import io - imagename1 = os.path.join(tmpfolder, "img" + str(index).zfill(strwidth) + ".png") - imagename2 = os.path.join(tmpfolder, "img" + str(index).zfill(strwidth) + "labeled.png") + imagename1 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + ".png") + imagename2 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + "labeled.png") - if not os.path.isfile(os.path.join(tmpfolder, "img" + str(index).zfill(strwidth) + ".png")): + if not imagename1.is_file(): plt.axis("off") cap.set_to_frame(index) frame = cap.read_frame(crop=True) @@ -1078,7 +1077,7 @@ def PlottingSingleFramecv2( plt.close("all") -def merge_datasets(config, forceiterate=None): +def merge_datasets(config: str | Path, forceiterate=None): """Merge the original training dataset with the newly refined data. Checks if the original training dataset can be merged with the newly refined @@ -1105,17 +1104,15 @@ def merge_datasets(config, forceiterate=None): cfg = auxiliaryfunctions.read_config(config) config_path = Path(config).parents[0] - bf = Path(str(config_path / "labeled-data")) + bf = config_path / "labeled-data" allfolders = [ - os.path.join(bf, fn) for fn in os.listdir(bf) if "_labeled" not in fn and not fn.startswith(".") + p for p in bf.iterdir() if "_labeled" not in p.name and not p.name.startswith(".") ] # exclude labeled data folders and temporary files flagged = False for _findex, folder in enumerate(allfolders): - if os.path.isfile(os.path.join(folder, "MachineLabelsRefine.h5")): # Folder that was manually refine... + if (folder / "MachineLabelsRefine.h5").is_file(): # Folder that was manually refine... pass - elif os.path.isfile( - os.path.join(folder, "CollectedData_" + cfg["scorer"] + ".h5") - ): # Folder that contains human data set... + elif (folder / ("CollectedData_" + cfg["scorer"] + ".h5")).is_file(): # Folder that contains human data set... pass else: print("The following folder was not manually refined,...", folder) diff --git a/deeplabcut/refine_training_dataset/stitch.py b/deeplabcut/refine_training_dataset/stitch.py index 59f792cf3..39b1a4545 100644 --- a/deeplabcut/refine_training_dataset/stitch.py +++ b/deeplabcut/refine_training_dataset/stitch.py @@ -8,7 +8,6 @@ # # Licensed under GNU Lesser General Public License v3.0 # -import os import pickle import re import shelve @@ -494,7 +493,7 @@ def from_pickle( split_tracklets=True, prestitch_residuals=True, ): - with open(pickle_file, "rb") as file: + with Path(pickle_file).open("rb") as file: tracklets = pickle.load(file) class_ = cls.from_dict_of_dict(tracklets, n_tracks, min_length, split_tracklets, prestitch_residuals) class_.filename = pickle_file @@ -961,8 +960,8 @@ def reconstruct_path(self, source): @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def stitch_tracklets( - config_path, - videos, + config_path: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle=1, trainingsetindex=0, @@ -986,11 +985,11 @@ def stitch_tracklets( Parameters ---------- - config_path : str + config_path : str or Path Path to the main project config.yaml file. - videos : list - A list of strings containing the full paths to videos for analysis or a path to the directory, where all the + videos : list[str] or list[Path] + Full paths to videos for analysis, or a path to the directory where all the videos with same extension are stored. video_extensions : str | Sequence[str] | None, optional, default=None @@ -1150,7 +1149,7 @@ def trans_weight_func(tracklet1, tracklet2, nframe, feature_dict): deeplabcut.utils.auxiliaryfunctions.attempt_to_make_folder(dest) vname = Path(video).stem - feature_dict_path = os.path.join(dest, vname + DLCscorer + "_bpt_features.pickle") + feature_dict_path = str(Path(dest) / (vname + DLCscorer + "_bpt_features.pickle")) # should only exist one if transformer_checkpoint: import dbm @@ -1160,7 +1159,7 @@ def trans_weight_func(tracklet1, tracklet2, nframe, feature_dict): except dbm.error as err: raise FileNotFoundError(f"{feature_dict_path} does not exist. Did you run transformer_reID()?") from err - dataname = os.path.join(dest, vname + DLCscorer + ".h5") + dataname = str(Path(dest) / (vname + DLCscorer + ".h5")) method = TRACK_METHODS[track_method] pickle_file = dataname.split(".h5")[0] + f"{method}.pickle" diff --git a/deeplabcut/refine_training_dataset/tracklets.py b/deeplabcut/refine_training_dataset/tracklets.py index c89e24efb..f0d8547bc 100644 --- a/deeplabcut/refine_training_dataset/tracklets.py +++ b/deeplabcut/refine_training_dataset/tracklets.py @@ -10,6 +10,7 @@ # import pickle import re +from pathlib import Path import numpy as np import pandas as pd @@ -217,7 +218,7 @@ def get_frame_ind(s): def load_tracklets_from_pickle(self, filename, auto_fill=True): self.filename = filename - with open(filename, "rb") as file: + with Path(filename).open("rb") as file: tracklets = pickle.load(file) self._load_tracklets(tracklets, auto_fill) self._xy = self.xy.copy() diff --git a/deeplabcut/utils/auxfun_models.py b/deeplabcut/utils/auxfun_models.py index 77cf85217..79f0a8af0 100644 --- a/deeplabcut/utils/auxfun_models.py +++ b/deeplabcut/utils/auxfun_models.py @@ -18,7 +18,6 @@ Licensed under GNU Lesser General Public License v3.0 """ -import os from pathlib import Path from deeplabcut.utils import auxiliaryfunctions @@ -59,8 +58,8 @@ def check_for_weights(modeltype, parent_path): exists = False model_path = parent_path / MODELTYPE_FILEPATH_MAP[modeltype] try: - for file in os.listdir(model_path.parent): - if model_path.name in file: + for file in model_path.parent.iterdir(): + if model_path.name in file.name: exists = True break except FileNotFoundError: @@ -126,13 +125,7 @@ def tarfilenamecutting(tarf): dlc_path = auxiliaryfunctions.get_deeplabcut_path() neturls = auxiliaryfunctions.read_plainconfig( - os.path.join( - dlc_path, - "pose_estimation_tensorflow", - "models", - "pretrained", - "pretrained_model_urls.yaml", - ) + dlc_path / "pose_estimation_tensorflow" / "models" / "pretrained" / "pretrained_model_urls.yaml" ) if modelname in neturls.keys(): url = neturls[modelname] @@ -170,13 +163,10 @@ def smart_restore(restorer, sess, checkpoint_path, net_type): restorer.restore(sess, checkpoint_path) except ValueError as e: # The path may be wrong, or the weights no longer exist dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() - correct_model_path = os.path.join( - dlcparent_path, - MODELTYPE_FILEPATH_MAP[net_type], - ) + correct_model_path = str(dlcparent_path / MODELTYPE_FILEPATH_MAP[net_type]) if checkpoint_path == correct_model_path: # The path is right, hence the weights are missing; we'll download them again. - _ = check_for_weights(net_type, Path(dlcparent_path)) + _ = check_for_weights(net_type, dlcparent_path) restorer.restore(sess, checkpoint_path) else: raise ValueError(e) from e diff --git a/deeplabcut/utils/auxfun_multianimal.py b/deeplabcut/utils/auxfun_multianimal.py index f398a7c59..bc36122bf 100644 --- a/deeplabcut/utils/auxfun_multianimal.py +++ b/deeplabcut/utils/auxfun_multianimal.py @@ -19,7 +19,6 @@ """ import math -import os import pickle import random import shelve @@ -197,9 +196,9 @@ def SaveFullMultiAnimalData(data, metadata, dataname, suffix="_full"): data_path = dataname.split(".h5")[0] + suffix + ".pickle" metadata_path = dataname.split(".h5")[0] + "_meta.pickle" - with open(data_path, "wb") as f: + with Path(data_path).open("wb") as f: pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) - with open(metadata_path, "wb") as f: + with Path(metadata_path).open("wb") as f: pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) return data_path, metadata_path @@ -209,11 +208,11 @@ def LoadFullMultiAnimalData(dataname): predict_videos.py.""" data_file = dataname.split(".h5")[0] + "_full.pickle" try: - with open(data_file, "rb") as handle: + with Path(data_file).open("rb") as handle: data = pickle.load(handle) except (pickle.UnpicklingError, FileNotFoundError): data = shelve.open(data_file, flag="r") - with open(data_file.replace("_full.", "_meta."), "rb") as handle: + with Path(data_file.replace("_full.", "_meta.")).open("rb") as handle: metadata = pickle.load(handle) return data, metadata @@ -228,12 +227,12 @@ def returnlabelingdata(config): print("Do you want to get the data for folder:", folder, "?") askuser = input("yes/no") if askuser == "y" or askuser == "yes" or askuser == "Ja" or askuser == "ha": # multilanguage support :) - fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".h5") + fn = folder / ("CollectedData_" + cfg["scorer"] + ".h5") Data = pd.read_hdf(fn) return Data -def convert2_maDLC(config, userfeedback=True, forceindividual=None): +def convert2_maDLC(config: str | Path, userfeedback=True, forceindividual=None): """ Converts single animal annotation file into a multianimal annotation file, by introducing an individuals column with either the first individual @@ -292,8 +291,8 @@ def convert2_maDLC(config, userfeedback=True, forceindividual=None): askuser = "yes" if askuser == "y" or askuser == "yes" or askuser == "Ja" or askuser == "ha": # multilanguage support :) - fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"]) - Data = pd.read_hdf(fn + ".h5") + fn = folder / ("CollectedData_" + cfg["scorer"]) + Data = pd.read_hdf(str(fn) + ".h5") conversioncode.guarantee_multiindex_rows(Data) imindex = Data.index @@ -380,8 +379,8 @@ def convert_single2multiplelegacyAM(config, userfeedback=True, target=None): askuser = "yes" if askuser == "y" or askuser == "yes" or askuser == "Ja" or askuser == "ha": # multilanguage support :) - fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"]) - Data = pd.read_hdf(fn + ".h5") + fn = folder / ("CollectedData_" + cfg["scorer"]) + Data = pd.read_hdf(str(fn) + ".h5") conversioncode.guarantee_multiindex_rows(Data) imindex = Data.index @@ -525,9 +524,7 @@ def convert_single2multiplelegacyAM(config, userfeedback=True, target=None): def form_default_inferencecfg(cfg): # load defaults - inferencecfg = auxiliaryfunctions.read_plainconfig( - os.path.join(auxiliaryfunctions.get_deeplabcut_path(), "inference_cfg.yaml") - ) + inferencecfg = auxiliaryfunctions.read_plainconfig(auxiliaryfunctions.get_deeplabcut_path() / "inference_cfg.yaml") # set project specific parameters: inferencecfg["minimalnumberofconnections"] = len(cfg["multianimalbodyparts"]) / 2 # reasonable default inferencecfg["topktoretain"] = len(cfg["individuals"]) @@ -544,7 +541,7 @@ def check_inferencecfg_sanity(cfg, inferencecfg): def read_inferencecfg(path_inference_config, cfg): """Load inferencecfg or initialize it.""" try: - inferencecfg = auxiliaryfunctions.read_plainconfig(str(path_inference_config)) + inferencecfg = auxiliaryfunctions.read_plainconfig(path_inference_config) except FileNotFoundError: inferencecfg = form_default_inferencecfg(cfg) auxiliaryfunctions.write_plainconfig(str(path_inference_config), dict(inferencecfg)) diff --git a/deeplabcut/utils/auxfun_videos.py b/deeplabcut/utils/auxfun_videos.py index 5eae5df4e..6806dfdab 100644 --- a/deeplabcut/utils/auxfun_videos.py +++ b/deeplabcut/utils/auxfun_videos.py @@ -20,7 +20,6 @@ """ import datetime -import os import random import subprocess import warnings @@ -42,7 +41,7 @@ class VideoReader: def __init__(self, video_path): - if not os.path.isfile(video_path): + if not Path(video_path).is_file(): raise ValueError(f'Video path "{video_path}" does not point to a file.') self.video_path = video_path self.video = cv2.VideoCapture(video_path) @@ -60,10 +59,10 @@ def __len__(self): return self._n_frames def check_integrity(self): - dest = os.path.join(self.directory, f"{self.name}.log") + dest = str(Path(self.directory) / f"{self.name}.log") command = f'ffmpeg -v error -i "{self.video_path}" -f null - 2>"{dest}"' subprocess.call(command, shell=True) - if os.path.getsize(dest) != 0: + if Path(dest).stat().st_size != 0: warnings.warn(f'Video contains errors. See "{dest}" for a detailed report.', stacklevel=2) def check_integrity_robust(self): @@ -77,15 +76,15 @@ def check_integrity_robust(self): @property def name(self): - return os.path.splitext(os.path.split(self.video_path)[1])[0] + return Path(self.video_path).stem @property def format(self): - return os.path.splitext(self.video_path)[1] + return Path(self.video_path).suffix @property def directory(self): - return os.path.dirname(self.video_path) + return str(Path(self.video_path).parent) @property def metadata(self): @@ -346,7 +345,7 @@ def write_frame(frame, where): def make_output_path(self, suffix, dest_folder): if not dest_folder: dest_folder = self.directory - return os.path.join(dest_folder, f"{self.name}{suffix}{self.format}") + return str(Path(dest_folder) / f"{self.name}{suffix}{self.format}") def check_video_integrity(video_path): diff --git a/deeplabcut/utils/auxiliaryfunctions.py b/deeplabcut/utils/auxiliaryfunctions.py index 416999b58..6c35e9da9 100644 --- a/deeplabcut/utils/auxiliaryfunctions.py +++ b/deeplabcut/utils/auxiliaryfunctions.py @@ -38,6 +38,21 @@ from deeplabcut.utils.deprecation import deprecated +def as_path(path: str | Path) -> Path: + """Coerce a filesystem path argument to :class:`~pathlib.Path`.""" + return Path(path) + + +def as_optional_path(path: str | Path | None) -> Path | None: + """Coerce an optional filesystem path argument to :class:`~pathlib.Path`.""" + return None if path is None else Path(path) + + +def as_path_list(paths: Sequence[str | Path]) -> list[Path]: + """Coerce a sequence of filesystem path arguments to :class:`~pathlib.Path`.""" + return [Path(p) for p in paths] + + def create_config_template(multianimal=False): """Creates a template for config.yaml file. @@ -212,19 +227,19 @@ def safe_resolve(path: Path) -> Path: """ resolved = path.resolve() try: - open(resolved).close() + resolved.open().close() return resolved except OSError: return path.absolute() -def read_config(configname): +def read_config(configname: str | Path): """Reads structured config file defining a project.""" ruamelFile = YAML() path = Path(configname) - if os.path.exists(path): + if path.exists(): try: - with open(path) as f: + with path.open() as f: cfg = ruamelFile.load(f) curr_dir = Path(configname).parent.absolute() @@ -244,7 +259,7 @@ def read_config(configname): except Exception as err: if len(err.args) > 2: if err.args[2] == "could not determine a constructor for the tag '!!python/tuple'": - with open(path) as ymlfile: + with path.open() as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) write_config(configname, cfg) else: @@ -260,7 +275,7 @@ def read_config(configname): def write_config(configname, cfg): """Write structured config file.""" - with open(configname, "w") as cf: + with Path(configname).open("w") as cf: cfg_file, ruamelFile = create_config_template(cfg.get("multianimalproject", False)) for key in cfg.keys(): cfg_file[key] = cfg[key] @@ -353,7 +368,7 @@ def get_unique_bodyparts(cfg: dict) -> list[str]: def write_config_3d(configname, cfg): """Write structured 3D config file.""" - with open(configname, "w") as cf: + with Path(configname).open("w") as cf: cfg_file, ruamelFile = create_config_template_3d() for key in cfg.keys(): cfg_file[key] = cfg[key] @@ -361,19 +376,19 @@ def write_config_3d(configname, cfg): def write_config_3d_template(projconfigfile, cfg_file_3d, ruamelFile_3d): - with open(projconfigfile, "w") as cf: + with Path(projconfigfile).open("w") as cf: ruamelFile_3d.dump(cfg_file_3d, cf) def read_plainconfig(configname): - if not os.path.exists(configname): + if not Path(configname).exists(): raise FileNotFoundError(f"Config {configname} is not found. Please make sure that the file exists.") - with open(configname) as file: + with Path(configname).open() as file: return YAML().load(file) def write_plainconfig(configname, cfg): - with open(configname, "w") as file: + with Path(configname).open("w") as file: YAML().dump(cfg, file) @@ -383,28 +398,28 @@ def attempt_to_make_folder(foldername, recursive=False): Does nothing if it already exists. """ try: - os.path.isdir(foldername) + Path(foldername).is_dir() except TypeError: # https://www.python.org/dev/peps/pep-0519/ foldername = os.fspath(foldername) # https://github.com/DeepLabCut/DeepLabCut/issues/105 (windows) - if os.path.isdir(foldername): + if Path(foldername).is_dir(): pass else: if recursive: - os.makedirs(foldername) + Path(foldername).mkdir(parents=True) else: - os.mkdir(foldername) + Path(foldername).mkdir() def read_pickle(filename): """Read the pickle file.""" - with open(filename, "rb") as handle: + with Path(filename).open("rb") as handle: return pickle.load(handle) def write_pickle(filename, data): """Write the pickle file.""" - with open(filename, "wb") as handle: + with Path(filename).open("wb") as handle: pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL) @@ -430,19 +445,25 @@ def save_data(PredicteData, metadata, dataname, pdindex, imagenames, save_as_csv print("Saving csv poses!") DataMachine.to_csv(dataname.split(".h5")[0] + ".csv") DataMachine.to_hdf(dataname, key="df_with_missing", format="table", mode="w") - with open(dataname.split(".h5")[0] + "_meta.pickle", "wb") as f: + with Path(dataname.split(".h5")[0] + "_meta.pickle").open("wb") as f: # Pickle the 'data' dictionary using the highest protocol available. pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) -def save_metadata(metadatafilename, data, trainIndices, testIndices, trainFraction): - with open(metadatafilename, "wb") as f: +def save_metadata( + metadatafilename: str | Path, + data, + trainIndices, + testIndices, + trainFraction, +): + with Path(metadatafilename).open("wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. pickle.dump([data, trainIndices, testIndices, trainFraction], f, pickle.HIGHEST_PROTOCOL) -def load_metadata(metadatafile): - with open(metadatafile, "rb") as f: +def load_metadata(metadatafile: str | Path): + with Path(metadatafile).open("rb") as f: [ trainingdata_details, trainIndices, @@ -454,16 +475,16 @@ def load_metadata(metadatafile): def get_immediate_subdirectories(a_dir): """Get list of immediate subdirectories.""" - return [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))] + return [p.name for p in Path(a_dir).iterdir() if p.is_dir()] # TODO: @deruyter92 2026-05-20: this function could be updated to match the # signature of collect_video_paths, allowing for multiple extensions. def grab_files_in_folder(folder, ext="", relative=True): """Return the paths of files with extension *ext* present in *folder*.""" - for file in os.listdir(folder): - if file.endswith(ext): - yield file if relative else os.path.join(folder, file) + for file in Path(folder).iterdir(): + if file.name.endswith(ext): + yield file.name if relative else str(file) def filter_files_by_patterns( @@ -528,26 +549,23 @@ def get_training_set_folder(cfg: dict) -> Path: Task = cfg["Task"] date = cfg["date"] iterate = "iteration-" + str(cfg["iteration"]) - return Path(os.path.join("training-datasets", iterate, "UnaugmentedDataSet_" + Task + date)) - - -def get_data_and_metadata_filenames(trainingsetfolder, trainFraction, shuffle, cfg): - # Filename for metadata and data relative to project path for corresponding parameters - metadatafn = os.path.join( - str(trainingsetfolder), - "Documentation_data-" - + cfg["Task"] - + "_" - + str(int(trainFraction * 100)) - + "shuffle" - + str(shuffle) - + ".pickle", + return Path("training-datasets") / iterate / ("UnaugmentedDataSet_" + Task + date) + + +def get_data_and_metadata_filenames( + trainingsetfolder: str | Path, + trainFraction: float, + shuffle: int, + cfg: dict, +) -> tuple[Path, Path]: + """Paths to data and metadata files relative to the project root.""" + base = Path(trainingsetfolder) + datafn = base / ( + cfg["Task"] + "_" + cfg["scorer"] + str(int(100 * trainFraction)) + "shuffle" + str(shuffle) + ".mat" ) - datafn = os.path.join( - str(trainingsetfolder), - cfg["Task"] + "_" + cfg["scorer"] + str(int(100 * trainFraction)) + "shuffle" + str(shuffle) + ".mat", + metadatafn = base / ( + "Documentation_data-" + cfg["Task"] + "_" + str(int(trainFraction * 100)) + "shuffle" + str(shuffle) + ".pickle" ) - return datafn, metadatafn @@ -648,11 +666,11 @@ def get_snapshots_from_folder(train_folder: Path) -> list[str]: return sorted(snapshot_names, key=lambda name: int(name.split("-")[1])) -def get_deeplabcut_path(): +def get_deeplabcut_path() -> Path: """Get path of where deeplabcut is currently running.""" import importlib.util - return os.path.split(importlib.util.find_spec("deeplabcut").origin)[0] + return Path(importlib.util.find_spec("deeplabcut").origin).parent def intersection_of_body_parts_and_ones_given_by_user(cfg, comparisonbodyparts): @@ -672,9 +690,9 @@ def intersection_of_body_parts_and_ones_given_by_user(cfg, comparisonbodyparts): return cpbpts -def get_labeled_data_folder(cfg, video): - videoname = os.path.splitext(os.path.basename(video))[0] - return os.path.join(cfg["project_path"], "labeled-data", videoname) +def get_labeled_data_folder(cfg: dict, video: str | Path) -> Path: + videoname = Path(video).stem + return Path(cfg["project_path"]) / "labeled-data" / videoname def form_data_containers(df, bodyparts): @@ -738,14 +756,14 @@ def get_scorer_name( train_folder = Path(cfg["project_path"]) / model_folder / "train" snapshot_names = get_snapshots_from_folder(train_folder) snapshot_name = snapshot_names[snapshotindex] - trainingsiterations = (snapshot_name.split(os.sep)[-1]).split("-")[-1] + trainingsiterations = Path(snapshot_name).parts[-1].split("-")[-1] dlc_cfg = read_plainconfig( - os.path.join( - cfg["project_path"], - str(get_model_folder(trainFraction, shuffle, cfg, engine=engine, modelprefix=modelprefix)), - "train", - engine.pose_cfg_name, + str( + Path(cfg["project_path"]) + / get_model_folder(trainFraction, shuffle, cfg, engine=engine, modelprefix=modelprefix) + / "train" + / engine.pose_cfg_name ) ) # ABBREVIATE NETWORK NAMES -- esp. for mobilenet! @@ -776,9 +794,10 @@ def check_if_post_processing(folder, vname, DLCscorer, DLCscorerlegacy, suffix=" If not, figures out if data was already analyzed (either with legacy scorer name or new one!) """ - outdataname = os.path.join(folder, vname + DLCscorer + suffix + ".h5") - sourcedataname = os.path.join(folder, vname + DLCscorer + ".h5") - if os.path.isfile(outdataname): # was data already processed? + folder = Path(folder) + outdataname = str(folder / (vname + DLCscorer + suffix + ".h5")) + sourcedataname = str(folder / (vname + DLCscorer + ".h5")) + if Path(outdataname).is_file(): # was data already processed? if suffix == "filtered": print("Video already filtered...", outdataname) elif suffix == "_skeleton": @@ -786,21 +805,21 @@ def check_if_post_processing(folder, vname, DLCscorer, DLCscorerlegacy, suffix=" return False, outdataname, sourcedataname, DLCscorer else: - odn = os.path.join(folder, vname + DLCscorerlegacy + suffix + ".h5") - if os.path.isfile(odn): # was it processed by DLC <2.1 project? + odn = str(folder / (vname + DLCscorerlegacy + suffix + ".h5")) + if Path(odn).is_file(): # was it processed by DLC <2.1 project? if suffix == "filtered": print("Video already filtered...(with DLC<2.1)!", odn) elif suffix == "_skeleton": print("Skeleton in video already processed... (with DLC<2.1)!", odn) return False, odn, odn, DLCscorerlegacy else: - sdn = os.path.join(folder, vname + DLCscorerlegacy + ".h5") + sdn = str(folder / (vname + DLCscorerlegacy + ".h5")) tracks = sourcedataname.replace(".h5", "tracks.h5") - if os.path.isfile(sourcedataname): # Was the video already analyzed? + if Path(sourcedataname).is_file(): # Was the video already analyzed? return True, outdataname, sourcedataname, DLCscorer - elif os.path.isfile(sdn): # was it analyzed with DLC<2.1? + elif Path(sdn).is_file(): # was it analyzed with DLC<2.1? return True, odn, sdn, DLCscorerlegacy - elif os.path.isfile(tracks): # May be a MA project with tracklets + elif Path(tracks).is_file(): # May be a MA project with tracklets return True, tracks.replace(".h5", f"{suffix}.h5"), tracks, DLCscorer else: print("Video not analyzed -- Run analyze_videos first.") @@ -810,7 +829,7 @@ def check_if_post_processing(folder, vname, DLCscorer, DLCscorerlegacy, suffix=" def check_if_not_analyzed(destfolder, vname, DLCscorer, DLCscorerlegacy, flag="video"): h5files = list(grab_files_in_folder(destfolder, "h5", relative=False)) if not len(h5files): - dataname = os.path.join(destfolder, vname + DLCscorer + ".h5") + dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5")) return True, dataname, DLCscorer # Iterate over data files and stop as soon as one matching the scorer is found @@ -829,18 +848,18 @@ def check_if_not_analyzed(destfolder, vname, DLCscorer, DLCscorerlegacy, flag="v return False, h5file, DLCscorerlegacy # If there was no match... - dataname = os.path.join(destfolder, vname + DLCscorer + ".h5") + dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5")) return True, dataname, DLCscorer def check_if_not_evaluated(folder, DLCscorer, DLCscorerlegacy, snapshot): - dataname = os.path.join(folder, DLCscorer + "-" + str(snapshot) + ".h5") - if os.path.isfile(dataname): + dataname = str(Path(folder) / (DLCscorer + "-" + str(snapshot) + ".h5")) + if Path(dataname).is_file(): print("This net has already been evaluated!") return False, dataname, DLCscorer else: - dn = os.path.join(folder, DLCscorerlegacy + "-" + str(snapshot) + ".h5") - if os.path.isfile(dn): + dn = str(Path(folder) / (DLCscorerlegacy + "-" + str(snapshot) + ".h5")) + if Path(dn).is_file(): print("This net has already been evaluated (with DLC<2.1)!") return False, dn, DLCscorerlegacy else: @@ -933,8 +952,9 @@ def load_analyzed_data(folder, videoname, scorer, filtered=False, track_method=" def load_detection_data(video, scorer, track_method): - folder = os.path.dirname(video) - videoname = os.path.splitext(os.path.basename(video))[0] + video = Path(video) + folder = video.parent + videoname = video.stem if track_method == "skeleton": tracker = "sk" elif track_method == "box": @@ -944,8 +964,8 @@ def load_detection_data(video, scorer, track_method): else: raise ValueError(f"Unrecognized track_method={track_method}") - filepath = os.path.splitext(video)[0] + scorer + f"_{tracker}.pickle" - if not os.path.isfile(filepath): + filepath = str(video.with_suffix("")) + scorer + f"_{tracker}.pickle" + if not Path(filepath).is_file(): raise FileNotFoundError( f"No detection data found in {folder} for video {videoname}, scorer {scorer}, and tracker {track_method}" ) @@ -954,7 +974,7 @@ def load_detection_data(video, scorer, track_method): def find_next_unlabeled_folder(config_path, verbose=False): cfg = read_config(config_path) - base_folder = Path(os.path.join(cfg["project_path"], "labeled-data")) + base_folder = Path(cfg["project_path"]) / "labeled-data" h5files = sorted( base_folder.rglob("*.h5"), key=lambda p: p.lstat().st_mtime, diff --git a/deeplabcut/utils/auxiliaryfunctions_3d.py b/deeplabcut/utils/auxiliaryfunctions_3d.py index 5d2c22300..a51dfb9e7 100644 --- a/deeplabcut/utils/auxiliaryfunctions_3d.py +++ b/deeplabcut/utils/auxiliaryfunctions_3d.py @@ -18,7 +18,6 @@ Licensed under GNU Lesser General Public License v3.0 """ -import glob import os import pickle from pathlib import Path @@ -32,11 +31,11 @@ def Foldernames3Dproject(cfg_3d): """Definitions of subfolders in 3D projects.""" - img_path = os.path.join(cfg_3d["project_path"], "calibration_images") - path_corners = os.path.join(cfg_3d["project_path"], "corners") - path_camera_matrix = os.path.join(cfg_3d["project_path"], "camera_matrix") - path_undistort = os.path.join(cfg_3d["project_path"], "undistortion") - path_removed_images = os.path.join(cfg_3d["project_path"], "removed_calibration_images") + img_path = str(Path(cfg_3d["project_path"]) / "calibration_images") + path_corners = str(Path(cfg_3d["project_path"]) / "corners") + path_camera_matrix = str(Path(cfg_3d["project_path"]) / "camera_matrix") + path_undistort = str(Path(cfg_3d["project_path"]) / "undistortion") + path_removed_images = str(Path(cfg_3d["project_path"]) / "removed_calibration_images") return ( img_path, @@ -102,7 +101,7 @@ def compute_triangulation_calibration_images( ax.set_xlabel("X") ax.set_ylabel("Y") ax.set_zlabel("Z") - plt.savefig(os.path.join(str(path_undistort), "checkerboard_3d.png")) + plt.savefig(Path(path_undistort) / "checkerboard_3d.png") return triangulate @@ -121,26 +120,23 @@ def get_camerawise_videos(path, cam_names, videotype): then it will return [['somename-camera-1-othername.avi', 'somename- camera-2-othername.avi']] """ - import glob from pathlib import Path vid = [] # Find videos only specific to the cam names - videos = [glob.glob(os.path.join(path, str("*" + cam_names[i] + "*" + videotype))) for i in range(len(cam_names))] - videos = [y for x in videos for y in x] + videos = [list(Path(path).glob("*" + cam_names[i] + "*" + videotype)) for i in range(len(cam_names))] + videos = [str(y) for x in videos for y in x] # Exclude the labeled video files if "." in videotype: file_to_exclude = str("labeled" + videotype) else: file_to_exclude = str("labeled." + videotype) - videos = [v for v in videos if os.path.isfile(v) and file_to_exclude not in v] + videos = [v for v in videos if Path(v).is_file() and file_to_exclude not in v] video_list = [] cam = cam_names[0] # camera1 - vid.append( - [name for name in glob.glob(os.path.join(path, str("*" + cam + "*" + videotype)))] - ) # all videos with cam + vid.append([str(name) for name in Path(path).glob("*" + cam + "*" + videotype)]) # all videos with cam # print("here is what I found",vid) for k in range(len(vid[0])): if cam in str(Path(vid[0][k]).stem): @@ -151,16 +147,16 @@ def get_camerawise_videos(path, cam_names, videotype): if suf == "": print("Strange naming convention on your part. Respect.") else: - putativecam2name = os.path.join(path, cam_names[1] + suf + ending) + putativecam2name = str(Path(path) / (cam_names[1] + suf + ending)) else: if suf == "": - putativecam2name = os.path.join(path, pref + cam_names[1] + ending) + putativecam2name = str(Path(path) / (pref + cam_names[1] + ending)) else: - putativecam2name = os.path.join(path, pref + cam_names[1] + suf + ending) - # print([os.path.join(path,pref+cam+suf+ending),putativecam2name]) - if os.path.isfile(putativecam2name): + putativecam2name = str(Path(path) / (pref + cam_names[1] + suf + ending)) + # print([str(Path(path) / (pref+cam+suf+ending)), putativecam2name]) + if Path(putativecam2name).is_file(): # found a pair!!! - video_list.append([os.path.join(path, pref + cam + suf + ending), putativecam2name]) + video_list.append([str(Path(path) / (pref + cam + suf + ending)), putativecam2name]) return video_list @@ -173,13 +169,13 @@ def Get_list_of_triangulated_and_videoFiles(filepath, videotype, scorer_3d, cam_ string_to_search = scorer_3d + ".h5" # Checks if filepath is a directory - if [os.path.isdir(i) for i in filepath] == [True]: + if [Path(i).is_dir() for i in filepath] == [True]: """Analyzes all the videos in the directory.""" print("Analyzing all the videos in the directory") videofolder = filepath[0] - cwd = os.getcwd() + cwd = Path.cwd() os.chdir(videofolder) - triangulated_file_list = [fn for fn in os.listdir(os.curdir) if (string_to_search in fn)] + triangulated_file_list = [fn.name for fn in Path(videofolder).iterdir() if (string_to_search in fn.name)] video_list = get_camerawise_videos(videofolder, cam_names, videotype) os.chdir(cwd) triangulated_folder = videofolder @@ -230,12 +226,9 @@ def Get_list_of_triangulated_and_videoFiles(filepath, videotype, scorer_3d, cam_ if (prefix[j][0] in filename[k] and prefix[j][1] in filename[k]) and ( suffix[j][0] in filename[k] and suffix[j][1] in filename[k] ): - triangulated_file = glob.glob( - os.path.join( - triangulated_folder, - str("*" + filename[k] + "*" + string_to_search), - ) - ) + triangulated_file = [ + str(p) for p in Path(triangulated_folder).glob("*" + filename[k] + "*" + string_to_search) + ] vfiles = get_camerawise_videos(videofolder, cam_names, videotype) vfiles = [z for z in vfiles if prefix[j][0] in z[0] and suffix[j][0] in z[1]][0] file_list.append(triangulated_file + vfiles) @@ -244,12 +237,12 @@ def Get_list_of_triangulated_and_videoFiles(filepath, videotype, scorer_3d, cam_ def SaveMetadata3d(metadatafilename, metadata): - with open(metadatafilename, "wb") as f: + with Path(metadatafilename).open("wb") as f: pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL) def LoadMetadata3d(metadatafilename): - with open(metadatafilename, "rb") as f: + with Path(metadatafilename).open("rb") as f: metadata = pickle.load(f) return metadata diff --git a/deeplabcut/utils/conversioncode.py b/deeplabcut/utils/conversioncode.py index 6ff756b24..a12ad49f5 100644 --- a/deeplabcut/utils/conversioncode.py +++ b/deeplabcut/utils/conversioncode.py @@ -9,7 +9,6 @@ # Licensed under GNU Lesser General Public License v3.0 # -import os from itertools import islice from pathlib import Path @@ -23,7 +22,7 @@ SUPPORTED_FILETYPES = "csv", "nwb" -def convertcsv2h5(config, userfeedback=True, scorer=None): +def convertcsv2h5(config: str | Path, userfeedback=True, scorer=None): """ Convert (image) annotation files in folder labeled-data from csv to h5. This function allows the user to manually edit the csv @@ -68,10 +67,10 @@ def convertcsv2h5(config, userfeedback=True, scorer=None): askuser = "yes" if askuser in ("y", "yes", "Ja", "ha", "oui"): # multilanguage support :) - fn = os.path.join(str(folder), "CollectedData_" + cfg["scorer"] + ".csv") + fn = folder / ("CollectedData_" + cfg["scorer"] + ".csv") # Determine whether the data are single- or multi-animal without loading into memory # simply by checking whether 'individuals' is in the second line of the CSV. - with open(fn) as datafile: + with fn.open() as datafile: head = list(islice(datafile, 0, 5)) if "individuals" in head[1]: header = list(range(4)) @@ -90,7 +89,12 @@ def convertcsv2h5(config, userfeedback=True, scorer=None): print("Attention:", folder, "does not appear to have labeled data!") -def adapt_labeled_data_to_new_project(config_path, remove_old_bodyparts=False, other_scorer=False, userfeedback=False): +def adapt_labeled_data_to_new_project( + config_path: str | Path, + remove_old_bodyparts=False, + other_scorer=False, + userfeedback=False, +): """Given the config.yaml file, this function will convert the labels of an ancient project to a new project. For this, the labeled data must be in the project folder, under the labeled-data folder and with the same configuration as all deeplabcut @@ -130,12 +134,12 @@ def adapt_labeled_data_to_new_project(config_path, remove_old_bodyparts=False, o # discard the file extension video_name = video_name.split(".")[0] # Load the csv file - label_path = os.path.join(project_path, "labeled-data", video_name) - csv_files = [file for file in os.listdir(label_path) if file.endswith(".csv")] + label_path = Path(project_path) / "labeled-data" / video_name + csv_files = [f.name for f in label_path.iterdir() if f.name.endswith(".csv")] if not csv_files: print("No csv file in the folder:", label_path) else: - csv_path = os.path.join(label_path, csv_files[0]) + csv_path = str(label_path / csv_files[0]) df = pd.read_csv(csv_path, header=None) # get the scorer @@ -257,8 +261,8 @@ def analyze_videos_converth5_to_csv(video_folder, videotype=".mp4", listofvideos # TODO: @deruyter92 2026-05-20: this function still uses grab_files_in_folder instead # of collect_video_paths and videotype instead of video_extensions. def analyze_videos_converth5_to_nwb( - config, - video_folder, + config: str | Path, + video_folder: str | Path, videotype=".mp4", listofvideos=False, ): @@ -339,11 +343,11 @@ def merge_windowsannotationdataONlinuxsystem(cfg): data_path = Path(cfg["project_path"], "labeled-data") annotationfolders = [] for elem in auxiliaryfunctions.grab_files_in_folder(data_path, relative=False): - if os.path.isdir(elem): + if Path(elem).is_dir(): annotationfolders.append(elem) print("The following folders were found:", annotationfolders) for folder in annotationfolders: - filename = os.path.join(folder, "CollectedData_" + cfg["scorer"] + ".h5") + filename = str(Path(folder) / ("CollectedData_" + cfg["scorer"] + ".h5")) try: data = pd.read_hdf(filename) guarantee_multiindex_rows(data) diff --git a/deeplabcut/utils/make_labeled_video.py b/deeplabcut/utils/make_labeled_video.py index 959050908..43a538506 100644 --- a/deeplabcut/utils/make_labeled_video.py +++ b/deeplabcut/utils/make_labeled_video.py @@ -29,7 +29,6 @@ #################################################### # Dependencies #################################################### -import os.path from collections.abc import Callable, Iterable, Sequence from functools import partial from multiprocessing import Pool, get_start_method @@ -393,8 +392,8 @@ def CreateVideoSlow( @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def create_labeled_video( - config: str, - videos: list[str], + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle: int = 1, trainingsetindex: int = 0, @@ -436,10 +435,10 @@ def create_labeled_video( Parameters ---------- - config : string + config : str or Path Full path of the config.yaml file. - videos : list[str] + videos : list[str] or list[Path] A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored. @@ -638,6 +637,10 @@ def create_labeled_video( video_extensions='mp4', ) """ + if config != "": + config = Path(config) + if destfolder is not None: + destfolder = Path(destfolder) if skeleton is None: skeleton = [] if config == "": @@ -670,7 +673,7 @@ def create_labeled_video( ) model_config_path = Path(config).parent / model_folder / "train" / Engine.PYTORCH.pose_cfg_name if model_config_path.exists(): - model_config = auxiliaryfunctions.read_plainconfig(str(model_config_path)) + model_config = auxiliaryfunctions.read_plainconfig(model_config_path) if model_config["train_settings"].get("weight_init", {}).get("memory_replay", False): superanimal_name = model_config["train_settings"]["weight_init"]["dataset"] if bboxes_pcutoff is None: @@ -702,12 +705,7 @@ def create_labeled_video( if superanimal_name != "": dlc_root_path = auxiliaryfunctions.get_deeplabcut_path() test_cfg = auxiliaryfunctions.read_plainconfig( - os.path.join( - dlc_root_path, - "modelzoo", - "project_configs", - f"{superanimal_name}.yaml", - ) + dlc_root_path / "modelzoo" / "project_configs" / f"{superanimal_name}.yaml" ) bodyparts = test_cfg["bodyparts"] @@ -737,7 +735,7 @@ def create_labeled_video( bodyparts2connect = None skeleton_color = None - start_path = os.getcwd() + start_path = Path.cwd() Videos = collect_video_paths(videos, extensions=video_extensions) if not Videos: @@ -870,7 +868,7 @@ def proc_video( s = "" videooutname = filepath.replace(".h5", f"{s}_p{int(100 * pcutoff)}_labeled.mp4") - if os.path.isfile(videooutname) and not overwrite: + if Path(videooutname).is_file() and not overwrite: print("Labeled video already created. Skipping...") return @@ -923,7 +921,7 @@ def proc_video( ) clip.close() elif not fastmode: - tmpfolder = os.path.join(str(videofolder), "temp-" + vname) + tmpfolder = str(Path(str(videofolder)) / ("temp-" + vname)) if save_frames: auxiliaryfunctions.attempt_to_make_folder(tmpfolder) clip = vp(video) @@ -1151,8 +1149,8 @@ def create_video_with_keypoints_only( @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def create_video_with_all_detections( - config, - videos, + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle=1, trainingsetindex=0, @@ -1243,17 +1241,17 @@ def create_video_with_all_detections( confidence_to_alpha = _get_default_conf_to_alpha(confidence_to_alpha, 0) for video in videos: - videofolder = os.path.splitext(video)[0] + videofolder = str(Path(video).with_suffix("")) if destfolder is None: outputname = f"{videofolder + DLCscorername}_full.mp4" - full_pickle = os.path.join(videofolder + DLCscorername + "_full.pickle") + full_pickle = videofolder + DLCscorername + "_full.pickle" else: auxiliaryfunctions.attempt_to_make_folder(destfolder) - outputname = os.path.join(destfolder, str(Path(video).stem) + DLCscorername + "_full.mp4") - full_pickle = os.path.join(destfolder, str(Path(video).stem) + DLCscorername + "_full.pickle") + outputname = str(Path(destfolder) / (Path(video).stem + DLCscorername + "_full.mp4")) + full_pickle = str(Path(destfolder) / (Path(video).stem + DLCscorername + "_full.pickle")) - if not (os.path.isfile(outputname)): + if not Path(outputname).is_file(): video_name = str(Path(video).stem) print("Creating labeled video for ", video_name) h5file = full_pickle.replace("_full.pickle", ".h5") @@ -1364,8 +1362,8 @@ def _create_video_from_tracks(video, tracks, destfolder, output_name, pcutoff, s from tqdm import tqdm - if not os.path.isdir(destfolder): - os.mkdir(destfolder) + if not Path(destfolder).is_dir(): + Path(destfolder).mkdir() vid = VideoWriter(video) nframes = len(vid) @@ -1384,9 +1382,9 @@ def _create_video_from_tracks(video, tracks, destfolder, output_name, pcutoff, s for index in tqdm(range(nframes)): vid.set_to_frame(index) imname = "frame" + str(index).zfill(strwidth) - image_output = os.path.join(destfolder, imname + ".png") + image_output = str(Path(destfolder) / (imname + ".png")) frame = vid.read_frame() - if frame is not None and not os.path.isfile(image_output): + if frame is not None and not Path(image_output).is_file(): im.set_data(frame[:, X1:X2]) for n, trackid in enumerate(trackids): if imname in tracks[trackid]: @@ -1413,15 +1411,15 @@ def _create_video_from_tracks(video, tracks, destfolder, output_name, pcutoff, s ] ) # remove frames used for video creation - [os.remove(image) for image in os.listdir(destfolder) if "frame" in image] + [p.unlink() for p in Path(destfolder).iterdir() if "frame" in p.name] def create_video_from_pickled_tracks(video, pickle_file, destfolder="", output_name="", pcutoff=0.6): if not destfolder: - destfolder = os.path.splitext(video)[0] + destfolder = str(Path(video).with_suffix("")) if not output_name: - video_name, ext = os.path.splitext(os.path.split(video)[1]) - output_name = video_name + "DLClabeled" + ext + video_path = Path(video) + output_name = video_path.stem + "DLClabeled" + video_path.suffix tracks = auxiliaryfunctions.read_pickle(pickle_file) _create_video_from_tracks(video, tracks, destfolder, output_name, pcutoff) diff --git a/deeplabcut/utils/plotting.py b/deeplabcut/utils/plotting.py index e5745092d..b443661ee 100644 --- a/deeplabcut/utils/plotting.py +++ b/deeplabcut/utils/plotting.py @@ -21,12 +21,10 @@ from __future__ import annotations import argparse -import os #################################################### # Dependencies #################################################### -import os.path import pickle from collections.abc import Sequence from pathlib import Path @@ -149,17 +147,17 @@ def PlottingResults( cbar.set_ticklabels(bodyparts2plot) fig1.savefig( - os.path.join(tmpfolder, "trajectory" + suffix), + Path(tmpfolder) / ("trajectory" + suffix), bbox_inches="tight", dpi=resolution, ) - fig2.savefig(os.path.join(tmpfolder, "plot" + suffix), bbox_inches="tight", dpi=resolution) + fig2.savefig(Path(tmpfolder) / ("plot" + suffix), bbox_inches="tight", dpi=resolution) fig3.savefig( - os.path.join(tmpfolder, "plot-likelihood" + suffix), + Path(tmpfolder) / ("plot-likelihood" + suffix), bbox_inches="tight", dpi=resolution, ) - fig4.savefig(os.path.join(tmpfolder, "hist" + suffix), bbox_inches="tight", dpi=resolution) + fig4.savefig(Path(tmpfolder) / ("hist" + suffix), bbox_inches="tight", dpi=resolution) if showfigures: plt.show() @@ -172,8 +170,8 @@ def PlottingResults( @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0") def plot_trajectories( - config, - videos, + config: str | Path, + videos: list[str | Path], video_extensions: str | Sequence[str] | None = None, shuffle=1, trainingsetindex=0, @@ -194,12 +192,12 @@ def plot_trajectories( Parameters ---------- - config: str + config: str or Path Full path of the config.yaml file. - videos: list[str] - Full paths to videos for analysis or a path to the directory, where all the - videos with same extension are stored. + videos: list[str] or list[Path] + Full paths to videos for analysis, or a path to the directory where all videos + with the same extension are stored. video_extensions : str | Sequence[str] | None, optional, default=None Controls how ``videos`` are filtered, based on file extension. @@ -278,6 +276,9 @@ def plot_trajectories( ['/home/alex/analysis/project/videos/reachingvideo1.avi'], ) """ + config = Path(config) + if destfolder is not None: + destfolder = Path(destfolder) cfg = auxiliaryfunctions.read_config(config) if pcutoff is None: @@ -313,7 +314,7 @@ def plot_trajectories( df, filepath, _, suffix = auxiliaryfunctions.load_analyzed_data( videofolder, vname, DLCscorer, filtered, track_method ) - tmpfolder = os.path.join(videofolder, "plot-poses", vname) + tmpfolder = str(Path(videofolder) / "plot-poses" / vname) _plot_trajectories( filepath, bodyparts, @@ -383,9 +384,9 @@ def _plot_trajectories( except KeyError: individuals = [""] if dest_folder is None: - vname = os.path.basename(h5file).split("DLC")[0] - vid_folder = os.path.dirname(h5file) - dest_folder = os.path.join(vid_folder, "plot-poses", vname) + vname = Path(h5file).name.split("DLC")[0] + vid_folder = Path(h5file).parent + dest_folder = str(vid_folder / "plot-poses" / vname) auxiliaryfunctions.attempt_to_make_folder(dest_folder, recursive=True) # Keep only the individuals and bodyparts that were labeled labeled_bpts = [bp for bp in df.columns.get_level_values("bodyparts").unique() if bp in bodyparts] @@ -459,10 +460,10 @@ def plot_edge_affinity_distributions( Figure size in inches. """ - with open(eval_pickle_file, "rb") as file: + with Path(eval_pickle_file).open("rb") as file: data = pickle.load(file) meta_pickle_file = eval_pickle_file.replace("_full.", "_meta.") - with open(meta_pickle_file, "rb") as file: + with Path(meta_pickle_file).open("rb") as file: metadata = pickle.load(file) (w_train, _), (b_train, _) = crossvalutils._calc_within_between_pafs( data, diff --git a/deeplabcut/utils/pseudo_label.py b/deeplabcut/utils/pseudo_label.py index 77139a172..9843ab582 100644 --- a/deeplabcut/utils/pseudo_label.py +++ b/deeplabcut/utils/pseudo_label.py @@ -10,9 +10,7 @@ # from __future__ import annotations -import glob import json -import os from collections import defaultdict from pathlib import Path @@ -113,7 +111,7 @@ def video_to_frames(input_video, output_folder, cropping: list[int] | None = Non # Save the frame as an image file. frame_str = str(frame_count).zfill(5) - frame_file = os.path.join(output_folder, "images", f"frame_{frame_str}.png") + frame_file = str(Path(output_folder) / "images" / f"frame_{frame_str}.png") cv2.imwrite(frame_file, frame) # Increment the frame counter frame_count += 1 @@ -204,7 +202,7 @@ def keypoint_matching( config = update_config(config, max_individuals, device) individuals = [f"animal{i}" for i in range(max_individuals)] config["metadata"]["individuals"] = individuals - train_file_path = os.path.join(memory_replay_folder, "annotations", train_file) + train_file_path = memory_replay_folder / "annotations" / train_file pose_runner, detector_runner = get_inference_runners( config, @@ -215,7 +213,7 @@ def keypoint_matching( detector_path=detector_path, ) - with open(train_file_path) as f: + with train_file_path.open() as f: train_obj = json.load(f) images = train_obj["images"] @@ -230,7 +228,7 @@ def keypoint_matching( for image in images: # this only works with relative path as the testing image can be at a different folder - name = image["file_name"].split(os.sep)[-1] + name = Path(image["file_name"]).name image_name_to_id[name] = image["id"] image_id_to_name[image["id"]] = name @@ -249,31 +247,31 @@ def keypoint_matching( image_extensions = ["*.png", "*.jpg", "*.jpeg", "*.bmp", "*.gif", "*.tiff"] images_in_folder = [] for ext in image_extensions: - images_in_folder.extend(glob.glob(os.path.join(memory_replay_folder, "images", ext))) + images_in_folder.extend(str(p) for p in (memory_replay_folder / "images").glob(ext)) corresponded_images = [] for image in images_in_folder: image_path = image - name = image.split(os.sep)[-1] + name = Path(image).name if name in image_name_to_id: corresponded_images.append(image_path) images = corresponded_images - bbox_gts = [{"bboxes": np.array(image_name_to_bbox[image.split(os.sep)[-1]])} for image in images] + bbox_gts = [{"bboxes": np.array(image_name_to_bbox[Path(image).name])} for image in images] pose_inputs = list(zip(images, bbox_gts, strict=False)) # pose inference should return meta data for pseudo labeling predictions = pose_runner.inference(pose_inputs) - with open(str(memory_replay_folder / "pseudo_predictions.json"), "w") as f: + with (memory_replay_folder / "pseudo_predictions.json").open("w") as f: json.dump(pose_inputs, f, cls=NumpyEncoder) assert len(images) == len(predictions) image_name_to_pred = {} for image_path, prediction in zip(images, predictions, strict=False): - name = image_path.split(os.sep)[-1] + name = Path(image_path).name image_name_to_pred[name] = prediction pred_keypoint_names = config["metadata"]["bodyparts"] @@ -312,7 +310,7 @@ def keypoint_matching( row_ind, column_ind = linear_sum_assignment(match_matrix * -1) keypoint_mapping_list = [] - conversion_matrix_out_path = os.path.join(memory_replay_folder, "confusion_matrix.png") + conversion_matrix_out_path = memory_replay_folder / "confusion_matrix.png" plot_cost_matrix(match_matrix, gt_keypoint_names, pred_keypoint_names, conversion_matrix_out_path) @@ -329,8 +327,8 @@ def keypoint_matching( for pred, anno in names: conversion_table[pred] = anno - conversion_table_out_path = os.path.join(memory_replay_folder, "conversion_table.csv") - with open(conversion_table_out_path, "w") as f: + conversion_table_out_path = memory_replay_folder / "conversion_table.csv" + with conversion_table_out_path.open("w") as f: out = "gt, MasterName\n" for name in pred_keypoint_names: target = name @@ -376,10 +374,10 @@ def dlc3predictions_2_annotation_from_video( annotations = [] categories = [] annotation_id = 0 - image_folder = os.path.join(dest_proj_folder, "images") + image_folder = Path(dest_proj_folder) / "images" # video_to_frames function by default outputs png or jpg - image_paths = sorted(glob.glob(os.path.join(image_folder, "*.png"))) + image_paths = sorted(str(p) for p in image_folder.glob("*.png")) # Ensure predictions and image_paths have the same length before subsampling if len(predictions) != len(image_paths): @@ -420,7 +418,7 @@ def dlc3predictions_2_annotation_from_video( for image_id, (prediction, image_path) in enumerate(zip(predictions, image_paths, strict=False)): image_obj = cv2.imread(image_path) height, width, channels = image_obj.shape - imagename = image_path.split(os.sep)[-1] + imagename = Path(image_path).name image = { "id": image_id, "file_name": imagename, @@ -481,8 +479,8 @@ def dlc3predictions_2_annotation_from_video( } # there is no 'test' split of video adaptation. This is essentially train.json - with open(os.path.join(dest_proj_folder, "annotations", "test.json"), "w") as f: + with (Path(dest_proj_folder) / "annotations" / "test.json").open("w") as f: json.dump(test_obj, f, indent=4) - with open(os.path.join(dest_proj_folder, "annotations", "train.json"), "w") as f: + with (Path(dest_proj_folder) / "annotations" / "train.json").open("w") as f: json.dump(train_obj, f, indent=4) diff --git a/deeplabcut/utils/skeleton.py b/deeplabcut/utils/skeleton.py index e4e7da8eb..a42af79df 100644 --- a/deeplabcut/utils/skeleton.py +++ b/deeplabcut/utils/skeleton.py @@ -18,8 +18,8 @@ Licensed under GNU Lesser General Public License v3.0 """ -import os import warnings +from pathlib import Path import matplotlib.colors as mcolors import matplotlib.pyplot as plt @@ -37,16 +37,16 @@ # NOTE @C-Achard 2026-03-26 duplicate config read/write functions # should be addressed in config refactor def read_config(configname): - if not os.path.exists(configname): + if not Path(configname).exists(): raise FileNotFoundError(f"Config {configname} is not found. Please make sure that the file exists.") yaml = YAML(typ="rt") - with open(configname, encoding="utf-8") as file: + with Path(configname).open(encoding="utf-8") as file: return yaml.load(file) def write_config(configname, cfg): yaml = YAML(typ="rt") - with open(configname, "w", encoding="utf-8") as file: + with Path(configname).open("w", encoding="utf-8") as file: yaml.dump(cfg, file) @@ -57,11 +57,10 @@ def __init__(self, config_path): # Find uncropped labeled data self.df = None found = False - root = os.path.join(self.cfg["project_path"], "labeled-data") - for dir_ in os.listdir(root): - folder = os.path.join(root, dir_) - if os.path.isdir(folder) and not any(folder.endswith(s) for s in ("cropped", "labeled")): - self.df = pd.read_hdf(os.path.join(folder, f"CollectedData_{self.cfg['scorer']}.h5")) + root = Path(self.cfg["project_path"]) / "labeled-data" + for folder in root.iterdir(): + if folder.is_dir() and not any(folder.name.endswith(s) for s in ("cropped", "labeled")): + self.df = pd.read_hdf(folder / f"CollectedData_{self.cfg['scorer']}.h5") self.df = drop_likelihood_columns(self.df) row, col = self.pick_labeled_frame() if "individuals" in self.df.columns.names: @@ -86,7 +85,7 @@ def __init__(self, config_path): if isinstance(row, str): sep = "/" if "/" in row else "\\" row = row.split(sep) - self.image = io.imread(os.path.join(self.cfg["project_path"], *row)) + self.image = io.imread(Path(self.cfg["project_path"]).joinpath(*row)) self.inds = set() self.segs = set() # Draw the skeleton if already existent diff --git a/deeplabcut/utils/visualization.py b/deeplabcut/utils/visualization.py index e0bbff0b5..9c9956b2d 100644 --- a/deeplabcut/utils/visualization.py +++ b/deeplabcut/utils/visualization.py @@ -227,9 +227,9 @@ def plot_and_save_labeled_frame( scaling=1, ): if isinstance(DataCombined.index[ind], tuple): - image_path = os.path.join(cfg["project_path"], *DataCombined.index[ind]) + image_path = str(Path(cfg["project_path"]).joinpath(*DataCombined.index[ind])) else: - image_path = os.path.join(cfg["project_path"], DataCombined.index[ind]) + image_path = str(Path(cfg["project_path"]) / DataCombined.index[ind]) frame = io.imread(image_path) if np.ndim(frame) > 2: # color image! h, w, numcolors = np.shape(frame) @@ -263,7 +263,7 @@ def save_labeled_frame(fig, image_path, dest_folder, belongs_to_train): dest = "-".join(("Training", imfoldername, imagename)) else: dest = "-".join(("Test", imfoldername, imagename)) - full_path = os.path.join(dest_folder, dest) + full_path = str(Path(dest_folder) / dest) # Windows throws error if file path is > 260 characters, can fix with prefix. # See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file#maximum-path-length-limitation @@ -369,9 +369,9 @@ def make_labeled_images_from_dataframe( bones.extend(zip(match1, match2, strict=False)) ind_bones = tuple(zip(*bones, strict=False)) - images_list = [os.path.join(cfg["project_path"], *tuple_) for tuple_ in df.index.tolist()] + images_list = [str(Path(cfg["project_path"]).joinpath(*tuple_)) for tuple_ in df.index.tolist()] if not destfolder: - destfolder = os.path.dirname(images_list[0]) + destfolder = str(Path(images_list[0]).parent) tmpfolder = destfolder + "_labeled" auxiliaryfunctions.attempt_to_make_folder(tmpfolder) ic = io.imread_collection(images_list) @@ -406,10 +406,10 @@ def make_labeled_images_from_dataframe( pt.set_data(*np.expand_dims(coord, axis=1)) if ind_bones: coll.set_segments(segs[ind]) - imagename = os.path.basename(filename) + imagename = Path(filename).name fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) fig.savefig( - os.path.join(tmpfolder, imagename.replace(".png", f"_{color_by}.png")), + Path(tmpfolder) / imagename.replace(".png", f"_{color_by}.png"), dpi=dpi, ) plt.close(fig) @@ -428,10 +428,10 @@ def make_labeled_images_from_dataframe( if ind_bones: coll = LineCollection(segs[ind], colors=cfg["skeleton_color"], alpha=alpha) ax.add_collection(coll) - imagename = os.path.basename(filename) + imagename = Path(filename).name fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) fig.savefig( - os.path.join(tmpfolder, imagename.replace(".png", f"_{color_by}.png")), + Path(tmpfolder) / imagename.replace(".png", f"_{color_by}.png"), dpi=dpi, ) plt.close(fig) diff --git a/pyproject.toml b/pyproject.toml index a578f17dd..8935b898b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,7 +189,7 @@ target-version = "py310" line-length = 120 fix = true [tool.ruff.lint] -select = [ "E", "F", "B", "I", "UP" ] +select = [ "E", "F", "B", "I", "UP", "PTH" ] ignore = [ "E741", "B007" ] [tool.ruff.lint.per-file-ignores] "__init__.py" = [ "F401", "E402" ]