diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 356b979..8d13451 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -4,6 +4,7 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.ref }}
@@ -13,9 +14,9 @@ jobs:
   static-analysis:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python 3.11
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - uses: pre-commit/action@v3.0.0
@@ -32,26 +33,39 @@ jobs:
           - pytest-benchmark
           - valgrind
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
-          - "3.12.0-beta.1"
+          - "3.12"
+        pytest-version:
+          - ">=8.1.1"
+        include:
+          - config: pytest-benchmark
+            python-version: "3.7"
+            pytest-version: "<8.1.1"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: "Set up Python ${{ matrix.python-version }}"
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python-version }}"
       - if: matrix.config == 'valgrind' || matrix.config == 'pytest-benchmark'
         name: Install valgrind
         run: sudo apt-get install valgrind -y
-      - name: Install dependencies
-        run: pip install .[dev,compat]
+      - name: Install dependencies with pytest${{ matrix.pytest-version }}
+        run: pip install .[dev,compat] "pytest${{ matrix.pytest-version }}"
       - if: matrix.config != 'pytest-benchmark'
         name: Uninstall pytest-benchmark
         run: pip uninstall -y pytest-benchmark
       - name: Run tests
         run: pytest -vs
+
+  all-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "All CI checks passed."
+    needs:
+      - static-analysis
+      - tests
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index a633d95..b13938d 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python 3.12
         uses: actions/setup-python@v2
         with:
-          python-version: "3.12.0-beta.3"
+          python-version: "3.12"
       - name: Install local version of pytest-codspeed
         run: pip install .
       - name: Run benchmarks
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0f71438..953997a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -8,15 +8,12 @@ repos:
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-added-large-files
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.3.0
     hooks:
       - id: mypy
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.275
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.3
     hooks:
       - id: ruff
+      - id: ruff-format
diff --git a/README.md b/README.md
index be3d3b3..32ec187 100644
--- a/README.md
+++ b/README.md
@@ -2,11 +2,10 @@
 <h1>pytest-codspeed</h1>
 
 [![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
-<a href="https://pypi.org/project/pytest-codspeed" target="_blank">
-<img src="https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi" alt="Package version">
-</a>
-<img src="https://img.shields.io/badge/python-3.7%20|%203.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12-informational.svg" alt="python-3.7-3.8-3.9-3.10-3.11-3.12">
+[![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
+![Python Version](https://img.shields.io/badge/python-3.7%20|%203.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12-informational.svg)
 [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
+[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
 
 Pytest plugin to create CodSpeed benchmarks
 
@@ -108,7 +107,7 @@ jobs:
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Run benchmarks
-        uses: CodSpeedHQ/action@v1
+        uses: CodSpeedHQ/action@v2
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: pytest tests/ --codspeed
diff --git a/pyproject.toml b/pyproject.toml
index 2411b8b..9032d69 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,14 +35,16 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "cffi ~= 1.15.1",
+    "cffi >= 1.15.1",
+    # cffi doesn't automatically install setuptools with python 3.12+
+    # cf https://github.com/python-cffi/cffi/releases/tag/v1.16.0
+    "setuptools; python_full_version >= '3.12.0'",
     "pytest>=3.8",
-    "filelock ~= 3.12.2",
-    "setuptools ~= 67.8.0; python_full_version >= '3.12.0b1'", # FIXME: remove when cffi supports directly python 3.12
+    "filelock >= 3.12.2",
 ]
 
 [project.optional-dependencies]
-lint = ["black ~= 23.3.0", "isort ~=5.12.0", "mypy ~= 1.3.0", "ruff ~= 0.0.275"]
+lint = ["mypy ~= 1.3.0", "ruff ~= 0.3.3"]
 compat = ["pytest-benchmark ~= 4.0.0", "pytest-xdist ~= 2.0.0"]
 test = ["pytest ~= 7.0", "pytest-cov ~= 4.0.0"]
 
@@ -71,6 +73,7 @@ python_version = "3.12"
 
 [tool.ruff]
 line-length = 88
+[tool.ruff.lint]
 select = ["E", "F", "I", "C"]
 
 [tool.isort]
diff --git a/src/pytest_codspeed/__init__.py b/src/pytest_codspeed/__init__.py
index fda8a71..dcc0ac0 100644
--- a/src/pytest_codspeed/__init__.py
+++ b/src/pytest_codspeed/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.2.0"
+__version__ = "2.2.1"
 
 from .plugin import BenchmarkFixture
 
diff --git a/src/pytest_codspeed/plugin.py b/src/pytest_codspeed/plugin.py
index 13d8e97..24dc656 100644
--- a/src/pytest_codspeed/plugin.py
+++ b/src/pytest_codspeed/plugin.py
@@ -28,6 +28,7 @@
 
 IS_PYTEST_BENCHMARK_INSTALLED = pkgutil.find_loader("pytest_benchmark") is not None
 SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
+BEFORE_PYTEST_8_1_1 = pytest.version_tuple < (8, 1, 1)
 
 
 @pytest.hookimpl(trylast=True)
@@ -95,15 +96,18 @@ def pytest_plugin_registered(plugin, manager: "pytest.PytestPluginManager"):
         codspeed_plugin: CodSpeedPlugin = manager.get_plugin(PLUGIN_NAME)
         if codspeed_plugin.is_codspeed_enabled:
             codspeed_benchmark_fixtures = plugin.getfixturedefs(
-                "codspeed_benchmark", ""
+                "codspeed_benchmark",
+                fixture_manager.session.nodeid
+                if BEFORE_PYTEST_8_1_1
+                else fixture_manager.session,
             )
             assert codspeed_benchmark_fixtures is not None
-            fixture_manager._arg2fixturedefs[
-                "__benchmark"
-            ] = fixture_manager._arg2fixturedefs["benchmark"]
-            fixture_manager._arg2fixturedefs["benchmark"] = list(
-                codspeed_benchmark_fixtures
+            # Archive the pytest-benchmark fixture
+            fixture_manager._arg2fixturedefs["__benchmark"] = (
+                fixture_manager._arg2fixturedefs["benchmark"]
             )
+            # Replace the pytest-benchmark fixture with the codspeed one
+            fixture_manager._arg2fixturedefs["benchmark"] = codspeed_benchmark_fixtures
 
 
 @pytest.hookimpl(trylast=True)
diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py
index ce83032..2997f08 100644
--- a/tests/test_pytest_plugin.py
+++ b/tests/test_pytest_plugin.py
@@ -142,7 +142,7 @@ class TestGroup:
             def setup_method(self):
                 print(); print("Setup called")
 
-            def teardown(self):
+            def teardown_method(self):
                 print(); print("Teardown called")
 
             @pytest.mark.benchmark