Skip to content

Commit d0806b7

Browse files
maximpavlivmaximpavliv
authored
Update TensorFlow backend for Python 3.12 compatibility (Keras 2 via tf-keras) (#3087)
* Relax pyside6 * Relax tensorflow non-Windows * Add higher python releases to CI/CD * Relax tensorflow in CI/CD * Relax pytables in CI/CD * Remove pytables from CI/CD installation * Install tf-keras for tensorflow>=2.16 support * Add TF_USE_LEGACY_KERAS env var to CI/CD * TMP Add tables to requirements? * Add _tf_legacy.py * TMP try install specific wrapt version? * Set WRAPT_DISABLE_EXTENSIONS env var * Exclude Windows & python=3.13 from CI/CD * TMP tf macos pin * Remove python 3.12 from cicd * Try pin tensorflow-macos==2.15.0 * TMP try unpin tensorflow-macos and add tf-keras * Replace macos-14 by macos-latest * Set dataloader_workers to 0 * Add tables to setup.py * Relax tensorflow-macos * Update 3mice colab installation * Update modelzoo reid colab installations
1 parent bb9cc68 commit d0806b7

9 files changed

Lines changed: 35 additions & 193 deletions

File tree

‎.github/workflows/python-package.yml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ jobs:
1313
strategy:
1414
fail-fast: false
1515
matrix:
16-
os: [ubuntu-latest, macos-14, windows-latest]
17-
python-version: ["3.10"]
16+
os: [ubuntu-latest, macos-latest, windows-latest]
17+
python-version: ["3.10", "3.11", "3.12"]
1818
include:
1919
- os: ubuntu-latest
2020
path: ~/.cache/pip
21-
- os: macos-14
21+
- os: macos-latest
2222
path: ~/Library/Caches/pip
2323
- os: windows-latest
2424
path: ~\AppData\Local\pip\Cache
@@ -38,18 +38,13 @@ jobs:
3838
shell: bash -el {0} # Important: activates the conda environment
3939
if: runner.os != 'macOS'
4040
run: |
41-
pip install tensorflow==2.12.0 tensorpack==0.11 tf_slim==1.1.0
41+
pip install tensorflow tensorpack==0.11 tf_slim==1.1.0 tf-keras
4242
4343
- name: Install TensorFlow on macOS
4444
shell: bash -el {0} # Important: activates the conda environment
4545
if: runner.os == 'macOS'
4646
run: |
47-
pip install tensorflow-macos==2.12.0 tensorpack==0.11 tf_slim==1.1.0
48-
49-
- name: Install PyTables through Conda
50-
shell: bash -el {0} # Important: activates the conda environment
51-
run: |
52-
conda install pytables==3.8.0 "numpy<2"
47+
pip install tensorflow-macos tensorpack==0.11 tf_slim==1.1.0 tf-keras
5348
5449
- name: Install dependencies
5550
shell: bash -el {0} # Important: activates the conda environment

‎deeplabcut/pose_estimation_tensorflow/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
# Licensed under GNU Lesser General Public License v3.0
1313
#
1414

15+
from . import _tf_legacy
16+
1517
# Suppress tensorflow warning messages
1618
import tensorflow as tf
1719
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

‎deeplabcut/pose_estimation_tensorflow/_tf_legacy.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import os
2+
import sys
3+
4+
os.environ.setdefault("TF_USE_LEGACY_KERAS", "1")
5+
os.environ.setdefault("WRAPT_DISABLE_EXTENSIONS", "1")
6+
7+
try:
8+
import tf_keras.src.legacy_tf_layers as legacy_tf_layers
9+
sys.modules["tf_keras.legacy_tf_layers"] = legacy_tf_layers
10+
except ImportError:
11+
# Older tf-keras didn’t use src/, so nothing to do
12+
pass

‎examples/COLAB/COLAB_3miceDemo.ipynb‎

Lines changed: 2 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -36,32 +36,7 @@
3636
"- a quick guide to maDLC: https://deeplabcut.github.io/DeepLabCut/docs/tutorial.html\n",
3737
"- a demo COLAB for how to use maDLC on your own data: https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_maDLC_TrainNetwork_VideoAnalysis.ipynb\n",
3838
"\n",
39-
"### To get started, please go to \"Runtime\" ->\"change runtime type\"->select \"Python3\", and then select \"GPU\"\n",
40-
"\n",
41-
"As the COLAB environments were updated to CUDA 12.X and Python 3.11, we need to install DeepLabCut and TensorFlow in a distinct way to get TensorFlow to connect to the GPU."
42-
]
43-
},
44-
{
45-
"cell_type": "code",
46-
"execution_count": null,
47-
"metadata": {
48-
"id": "HoNN2_0Z9rr_"
49-
},
50-
"outputs": [],
51-
"source": [
52-
"# Install TensorFlow, tensorpack and tf_slim versions compatible with DeepLabCut\n",
53-
"!pip install \"tensorflow==2.12.1\" \"tensorpack>=0.11\" \"tf_slim>=1.1.0\""
54-
]
55-
},
56-
{
57-
"cell_type": "code",
58-
"execution_count": null,
59-
"metadata": {},
60-
"outputs": [],
61-
"source": [
62-
"# Downgrade PyTorch to a version using CUDA 11.8 and cudnn 8\n",
63-
"# This will also install the required CUDA libraries, for both PyTorch and TensorFlow\n",
64-
"!pip install torch==2.3.1 torchvision --index-url https://download.pytorch.org/whl/cu118"
39+
"### To get started, please go to \"Runtime\" ->\"change runtime type\"->select \"Python3\", and then select \"GPU\""
6540
]
6641
},
6742
{
@@ -71,18 +46,7 @@
7146
"outputs": [],
7247
"source": [
7348
"# Install the correct (older) version of DeepLabCut\n",
74-
"!pip install deeplabcut==2.3.11"
75-
]
76-
},
77-
{
78-
"cell_type": "code",
79-
"execution_count": null,
80-
"metadata": {},
81-
"outputs": [],
82-
"source": [
83-
"# As described in https://www.tensorflow.org/install/pip#step-by-step_instructions, \n",
84-
"# create symbolic links to NVIDIA shared libraries:\n",
85-
"!ln -svf /usr/local/lib/python3.11/dist-packages/nvidia/*/lib/*.so* /usr/local/lib/python3.11/dist-packages/tensorflow"
49+
"!pip install \"deeplabcut[tf]\""
8650
]
8751
},
8852
{

‎examples/COLAB/COLAB_DLC_ModelZoo.ipynb‎

Lines changed: 2 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -45,30 +45,7 @@
4545
"\n",
4646
"## **Let's get going: install DeepLabCut into COLAB:**\n",
4747
"\n",
48-
"*Also, be sure you are connected to a GPU: go to menu, click Runtime > Change Runtime Type > select \"GPU\"*\n",
49-
"\n",
50-
"As the COLAB environments were updated to CUDA 12.X and Python 3.11, we need to install DeepLabCut and TensorFlow in a distinct way to get TensorFlow to connect to the GPU."
51-
]
52-
},
53-
{
54-
"cell_type": "code",
55-
"execution_count": null,
56-
"metadata": {},
57-
"outputs": [],
58-
"source": [
59-
"# Install TensorFlow, tensorpack and tf_slim versions compatible with DeepLabCut\n",
60-
"!pip install \"tensorflow==2.12.1\" \"tensorpack>=0.11\" \"tf_slim>=1.1.0\""
61-
]
62-
},
63-
{
64-
"cell_type": "code",
65-
"execution_count": null,
66-
"metadata": {},
67-
"outputs": [],
68-
"source": [
69-
"# Downgrade PyTorch to a version using CUDA 11.8 and cudnn 8\n",
70-
"# This will also install the required CUDA libraries, for both PyTorch and TensorFlow\n",
71-
"!pip install torch==2.3.1 torchvision --index-url https://download.pytorch.org/whl/cu118"
48+
"*Also, be sure you are connected to a GPU: go to menu, click Runtime > Change Runtime Type > select \"GPU\"*\n"
7249
]
7350
},
7451
{
@@ -78,18 +55,7 @@
7855
"outputs": [],
7956
"source": [
8057
"# Install the latest version of DeepLabCut\n",
81-
"!pip install \"git+https://github.com/DeepLabCut/DeepLabCut.git#egg=deeplabcut[modelzoo]\""
82-
]
83-
},
84-
{
85-
"cell_type": "code",
86-
"execution_count": null,
87-
"metadata": {},
88-
"outputs": [],
89-
"source": [
90-
"# As described in https://www.tensorflow.org/install/pip#step-by-step_instructions, \n",
91-
"# create symbolic links to NVIDIA shared libraries:\n",
92-
"!ln -svf /usr/local/lib/python3.11/dist-packages/nvidia/*/lib/*.so* /usr/local/lib/python3.11/dist-packages/tensorflow"
58+
"!pip install --pre \"deeplabcut[tf,modelzoo]\""
9359
]
9460
},
9561
{

‎examples/COLAB/COLAB_transformer_reID.ipynb‎

Lines changed: 3 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -32,91 +32,18 @@
3232
"- a quick guide to maDLC: https://deeplabcut.github.io/DeepLabCut/docs/tutorial.html\n",
3333
"- a demo COLAB for how to use maDLC on your own data: https://github.com/DeepLabCut/DeepLabCut/blob/main/examples/COLAB/COLAB_maDLC_TrainNetwork_VideoAnalysis.ipynb\n",
3434
"\n",
35-
"### To get started, please go to \"Runtime\" ->\"change runtime type\"->select \"Python3\", and then select \"GPU\"\n",
36-
"\n",
37-
"As the COLAB environments were updated to CUDA 12.X and Python 3.11, we need to install DeepLabCut and TensorFlow in a distinct way to get TensorFlow to connect to the GPU."
35+
"### To get started, please go to \"Runtime\" ->\"change runtime type\"->select \"Python3\", and then select \"GPU\"\n"
3836
]
3937
},
4038
{
4139
"cell_type": "markdown",
4240
"source": [
43-
"‼️ **Attention: this demo is for maDLC, which is version 2.2**\n",
44-
"- the installation is very slow on Colab due to the several steps needed to use older versions of `torch` and `dlc`."
41+
"‼️ **Attention: this demo is for maDLC, which is version 2.2**\n"
4542
],
4643
"metadata": {
4744
"id": "xOe2hvy85EVP"
4845
}
4946
},
50-
{
51-
"cell_type": "code",
52-
"execution_count": null,
53-
"metadata": {
54-
"id": "6_Q4_GKM45Oe"
55-
},
56-
"outputs": [],
57-
"source": [
58-
"# Downgrade PyTorch to a version using CUDA 11.8 and cudnn 8\n",
59-
"# This will also install the required CUDA libraries, for both PyTorch and TensorFlow\n",
60-
"!pip install torch==2.3.1 torchvision --index-url https://download.pytorch.org/whl/cu118"
61-
]
62-
},
63-
{
64-
"cell_type": "code",
65-
"source": [
66-
"import torch\n",
67-
"print(torch.cuda.is_available())\n",
68-
"print(torch.cuda.get_device_name(0))\n",
69-
"!nvcc --version # sometimes not available"
70-
],
71-
"metadata": {
72-
"colab": {
73-
"base_uri": "https://localhost:8080/"
74-
},
75-
"id": "3HLeBRGL9jYJ",
76-
"outputId": "b8e49307-f51c-4f12-8dec-d2c2929928e2"
77-
},
78-
"execution_count": 3,
79-
"outputs": [
80-
{
81-
"output_type": "stream",
82-
"name": "stdout",
83-
"text": [
84-
"True\n",
85-
"NVIDIA A100-SXM4-40GB\n",
86-
"nvcc: NVIDIA (R) Cuda compiler driver\n",
87-
"Copyright (c) 2005-2024 NVIDIA Corporation\n",
88-
"Built on Thu_Jun__6_02:18:23_PDT_2024\n",
89-
"Cuda compilation tools, release 12.5, V12.5.82\n",
90-
"Build cuda_12.5.r12.5/compiler.34385749_0\n"
91-
]
92-
}
93-
]
94-
},
95-
{
96-
"cell_type": "code",
97-
"execution_count": null,
98-
"metadata": {
99-
"id": "Vz3B6Zyf45Od"
100-
},
101-
"outputs": [],
102-
"source": [
103-
"# Install TensorFlow, tensorpack and tf_slim versions compatible with DeepLabCut\n",
104-
"!pip install \"tensorflow==2.12.1\" \"tensorpack>=0.11\" \"tf_slim>=1.1.0\""
105-
]
106-
},
107-
{
108-
"cell_type": "code",
109-
"execution_count": null,
110-
"metadata": {
111-
"id": "UUpac75T45Of"
112-
},
113-
"outputs": [],
114-
"source": [
115-
"# As described in https://www.tensorflow.org/install/pip#step-by-step_instructions,\n",
116-
"# create symbolic links to NVIDIA shared libraries:\n",
117-
"!ln -svf /usr/local/lib/python3.11/dist-packages/nvidia/*/lib/*.so* /usr/local/lib/python3.11/dist-packages/tensorflow"
118-
]
119-
},
12047
{
12148
"cell_type": "code",
12249
"execution_count": null,
@@ -126,15 +53,13 @@
12653
"outputs": [],
12754
"source": [
12855
"# Install DLC version 2.2-2.3 (pre DLC3):\n",
129-
"!pip install deeplabcut==2.3.11"
56+
"!pip install \"deeplabcut[tf]\""
13057
]
13158
},
13259
{
13360
"cell_type": "code",
13461
"source": [
135-
"import tensorflow\n",
13662
"import deeplabcut\n",
137-
"import torch\n",
13863
"import os"
13964
],
14065
"metadata": {
@@ -143,33 +68,6 @@
14368
"execution_count": 3,
14469
"outputs": []
14570
},
146-
{
147-
"cell_type": "code",
148-
"source": [
149-
"print(\"DLC version: \", deeplabcut.__version__)\n",
150-
"print(\"torch version: \",torch.__version__)\n",
151-
"print(\"tensorflow version: \",tensorflow.__version__)\n"
152-
],
153-
"metadata": {
154-
"colab": {
155-
"base_uri": "https://localhost:8080/"
156-
},
157-
"id": "wRojzQO98X2H",
158-
"outputId": "fad889d6-15da-4888-8138-283f1f13cf99"
159-
},
160-
"execution_count": 4,
161-
"outputs": [
162-
{
163-
"output_type": "stream",
164-
"name": "stdout",
165-
"text": [
166-
"DLC version: 2.3.11\n",
167-
"torch version: 2.3.1+cu118\n",
168-
"tensorflow version: 2.12.1\n"
169-
]
170-
}
171-
]
172-
},
17371
{
17472
"cell_type": "markdown",
17573
"metadata": {

‎examples/testscript_pytorch_multi_animal.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,14 @@ def main(
6363
"train_settings.display_iters": 50,
6464
"train_settings.epochs": epochs_,
6565
"train_settings.batch_size": batch_size,
66+
"train_settings.dataloader_workers": 0,
6667
"runner.device": device,
6768
"runner.snapshots.save_epochs": save_epochs,
6869
"runner.snapshots.max_snapshots": max_snapshots_to_keep,
6970
"detector.train_settings.display_iters": 1,
7071
"detector.train_settings.epochs": detector_epochs,
7172
"detector.train_settings.batch_size": detector_batch_size,
73+
"detector.train_settings.dataloader_workers": 0,
7274
"detector.runner.snapshots.save_epochs": save_epochs,
7375
"detector.runner.snapshots.max_snapshots": max_snapshots_to_keep,
7476
"logger": logger,

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ scikit-image>=0.17
2424
scikit-learn>=1.0
2525
scipy>=1.9
2626
statsmodels>=0.11
27-
tables==3.8.0
27+
tables
2828
torch==2.7.1
2929
torchvision
3030
tqdm

‎setup.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,23 +78,26 @@ def pytorch_config_paths() -> list[str]:
7878
"pycocotools",
7979
"pyyaml",
8080
"Pillow>=7.1",
81+
"tables",
8182
],
8283
extras_require={
8384
"gui": [
84-
"pyside6==6.4.2",
85+
"pyside6",
8586
"qdarkstyle==3.1",
8687
"napari-deeplabcut>=0.2.1.6",
8788
],
8889
"openvino": ["openvino-dev==2022.1.0"],
8990
"docs": ["numpydoc"],
9091
"tf": [
9192
"tensorflow>=2.0,<=2.10;platform_system=='Windows'",
92-
"tensorflow>=2.0,<=2.12;platform_system!='Windows'",
93+
"tensorflow>=2.0;platform_system!='Windows'",
94+
"tf-keras;platform_system!='Windows'",
9395
"tensorpack>=0.11",
9496
"tf_slim>=1.1.0",
9597
], # Last supported TF version on Windows Native is 2.10
9698
"apple_mchips": [
97-
"tensorflow-macos<2.13.0",
99+
"tensorflow-macos",
100+
"tf-keras",
98101
"tensorflow-metal",
99102
"tensorpack>=0.11",
100103
"tf_slim>=1.1.0",

0 commit comments

Comments
 (0)