Skip to content

Commit 0ce1dd7

Browse files
potiukpotiuk
authored
Run non-provider mypy checks as regular prek static checks instead of separate CI jobs (#64780)
Non-provider mypy checks (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common) now run locally via uv as regular prek hooks in the pre-commit stage, instead of running as separate mypy CI jobs in the CI image checks workflow. This means they run as part of the regular static checks job in CI and automatically on every local commit. The folder-level mypy checks (which check entire directories at once for comprehensive cross-file type checking) replace the previous file-level incremental checks. Provider mypy checks still run via breeze as a dedicated CI job, now embedded directly in the main CI workflow (ci-amd-arm.yml) instead of being dispatched through the ci-image-checks reusable workflow. The selective checks logic skips non-provider mypy hooks when their relevant files haven't changed, unless devel-common/pyproject.toml changes on main (which affects all mypy configurations).
1 parent c3d4905 commit 0ce1dd7

24 files changed

Lines changed: 470 additions & 347 deletions

‎.github/workflows/ci-amd-arm.yml‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@ jobs:
9191
kubernetes-versions-list-as-string: >-
9292
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
9393
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
94-
mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
9594
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
9695
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
9796
platform: ${{ steps.selective-checks.outputs.platform }}
@@ -115,7 +114,7 @@ jobs:
115114
run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }}
116115
run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }}
117116
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
118-
run-mypy: ${{ steps.selective-checks.outputs.run-mypy }}
117+
run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }}
119118
run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
120119
run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
121120
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
@@ -307,8 +306,6 @@ jobs:
307306
with:
308307
runners: ${{ needs.build-info.outputs.runner-type }}
309308
platform: ${{ needs.build-info.outputs.platform }}
310-
run-mypy: ${{ needs.build-info.outputs.run-mypy }}
311-
mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
312309
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
313310
branch: ${{ needs.build-info.outputs.default-branch }}
314311
canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -333,6 +330,51 @@ jobs:
333330
DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
334331
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
335332

333+
mypy-providers:
334+
timeout-minutes: 45
335+
name: "MyPy providers checks"
336+
needs: [build-info, build-ci-images]
337+
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
338+
if: needs.build-info.outputs.run-mypy-providers == 'true'
339+
env:
340+
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
341+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
342+
steps:
343+
- name: "Cleanup repo"
344+
shell: bash
345+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
346+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
347+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
348+
with:
349+
persist-credentials: false
350+
- name: "Free up disk space"
351+
shell: bash
352+
run: ./scripts/tools/free_up_disk_space.sh
353+
- name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}"
354+
uses: ./.github/actions/prepare_breeze_and_image
355+
with:
356+
platform: ${{ needs.build-info.outputs.platform }}
357+
python: "${{ needs.build-info.outputs.default-python-version }}"
358+
use-uv: ${{ needs.build-info.outputs.use-uv }}
359+
make-mnt-writeable-and-cleanup: true
360+
id: breeze
361+
- name: "Install prek"
362+
uses: ./.github/actions/install-prek
363+
id: prek
364+
with:
365+
python-version: ${{steps.breeze.outputs.host-python-version}}
366+
platform: ${{ needs.build-info.outputs.platform }}
367+
save-cache: false
368+
- name: "MyPy checks for providers"
369+
run: prek --color always --verbose --stage manual mypy-providers --all-files
370+
env:
371+
VERBOSE: "false"
372+
COLUMNS: "202"
373+
SKIP_GROUP_OUTPUT: "true"
374+
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
375+
RUFF_FORMAT: "github"
376+
INCLUDE_MYPY_VOLUME: "false"
377+
336378
providers:
337379
name: "provider distributions tests"
338380
uses: ./.github/workflows/test-providers.yml
@@ -895,6 +937,7 @@ jobs:
895937
- build-prod-images
896938
- ci-image-checks
897939
- generate-constraints
940+
- mypy-providers
898941
- providers
899942
- tests-helm
900943
- tests-integration-system

‎.github/workflows/ci-image-checks.yml‎

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,6 @@ on: # yamllint disable-line rule:truthy
2828
description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
2929
required: true
3030
type: string
31-
run-mypy:
32-
description: "Whether to run mypy checks (true/false)"
33-
required: true
34-
type: string
35-
mypy-checks:
36-
description: "List of folders to run mypy checks on"
37-
required: false
38-
type: string
3931
python-versions-list-as-string:
4032
description: "The list of python versions as string separated by spaces"
4133
required: true
@@ -169,55 +161,6 @@ jobs:
169161
run: cat ~/.cache/prek/prek.log || true
170162
if: failure()
171163

172-
mypy:
173-
timeout-minutes: 45
174-
name: "MyPy checks"
175-
runs-on: ${{ fromJSON(inputs.runners) }}
176-
if: inputs.run-mypy == 'true'
177-
strategy:
178-
fail-fast: false
179-
matrix:
180-
mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
181-
env:
182-
PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
183-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
184-
steps:
185-
- name: "Cleanup repo"
186-
shell: bash
187-
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
188-
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
189-
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
190-
with:
191-
persist-credentials: false
192-
- name: "Free up disk space"
193-
shell: bash
194-
run: ./scripts/tools/free_up_disk_space.sh
195-
- name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
196-
uses: ./.github/actions/prepare_breeze_and_image
197-
with:
198-
platform: ${{ inputs.platform }}
199-
python: "${{ inputs.default-python-version }}"
200-
use-uv: ${{ inputs.use-uv }}
201-
make-mnt-writeable-and-cleanup: true
202-
id: breeze
203-
- name: "Install prek"
204-
uses: ./.github/actions/install-prek
205-
id: prek
206-
with:
207-
python-version: ${{steps.breeze.outputs.host-python-version}}
208-
platform: ${{ inputs.platform }}
209-
save-cache: false
210-
- name: "MyPy checks for ${{ matrix.mypy-check }}"
211-
run: prek --color always --verbose --stage manual "$MYPY_CHECK" --all-files
212-
env:
213-
VERBOSE: "false"
214-
COLUMNS: "202"
215-
SKIP_GROUP_OUTPUT: "true"
216-
DEFAULT_BRANCH: ${{ inputs.branch }}
217-
RUFF_FORMAT: "github"
218-
INCLUDE_MYPY_VOLUME: "false"
219-
MYPY_CHECK: ${{ matrix.mypy-check }}
220-
221164
build-docs:
222165
timeout-minutes: 150
223166
name: "Build documentation"

‎.pre-commit-config.yaml‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,39 +1014,23 @@ repos:
10141014
^uv\.lock$
10151015
pass_filenames: false
10161016
require_serial: true
1017-
## ADD MOST PREK HOOK ABOVE THAT LINE
1018-
# The below prek hooks are those requiring CI image to be built
1019-
## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
10201017
- id: mypy-dev
1021-
stages: ['pre-push']
10221018
name: Run mypy for dev
10231019
language: python
1024-
entry: ./scripts/ci/prek/mypy.py
1025-
files: ^dev/.*\.py$|^scripts/.*\.py$
1026-
require_serial: true
1027-
- id: mypy-dev
1028-
stages: ['manual']
1029-
name: Run mypy for dev (manual)
1030-
language: python
1031-
entry: ./scripts/ci/prek/mypy_folder.py dev scripts
1020+
entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts
10321021
pass_filenames: false
10331022
files: ^.*\.py$
10341023
require_serial: true
10351024
- id: mypy-devel-common
1036-
stages: ['pre-push']
10371025
name: Run mypy for devel-common
10381026
language: python
1039-
entry: ./scripts/ci/prek/mypy.py
1040-
files: ^devel-common/.*\.py$
1041-
require_serial: true
1042-
- id: mypy-devel-common
1043-
stages: ['manual']
1044-
name: Run mypy for devel-common (manual)
1045-
language: python
1046-
entry: ./scripts/ci/prek/mypy_folder.py devel-common
1027+
entry: ./scripts/ci/prek/mypy_local_folder.py devel-common
10471028
pass_filenames: false
10481029
files: ^.*\.py$
10491030
require_serial: true
1031+
## ADD MOST PREK HOOK ABOVE THAT LINE
1032+
# The below prek hooks are those requiring CI image to be built
1033+
## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
10501034
- id: check-template-fields-valid
10511035
name: Check templated fields mapped in operators/sensors
10521036
language: python

‎AGENTS.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@
3030
- **Run other suites of tests** `breeze testing <test_group>` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`)
3131
- **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs`
3232
- **Run Airflow CLI:** `breeze run airflow dags list`
33-
- **Type-check:** `breeze run mypy path/to/code`
33+
- **Type-check (non-providers):** `uv run --project <PROJECT> --with "apache-airflow-devel-common[mypy]" mypy path/to/code`
34+
- **Type-check (providers):** `breeze run mypy path/to/code`
3435
- **Lint with ruff only:** `prek run ruff --from-ref <target_branch>`
3536
- **Format with ruff only:** `prek run ruff-format --from-ref <target_branch>`
3637
- **Run regular (fast) static checks:** `prek run --from-ref <target_branch> --stage pre-commit`
@@ -147,7 +148,7 @@ code review checklist in [`.github/instructions/code-review.instructions.md`](.g
147148
3. Confirm the code follows the project's coding standards and architecture boundaries
148149
described in this file.
149150
4. Run regular (fast) static checks (`prek run --from-ref <target_branch> --stage pre-commit`)
150-
and fix any failures.
151+
and fix any failures. This includes mypy checks for non-provider projects (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common).
151152
5. Run manual (slower) checks (`prek run --from-ref <target_branch> --stage manual`) and fix any failures.
152153
6. Run relevant individual tests and confirm they pass.
153154
7. Find which tests to run for the changes with selective-checks and run those tests in parallel to confirm they pass and check for CI-specific issues.

‎airflow-core/.pre-commit-config.yaml‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -221,23 +221,15 @@ repos:
221221
additional_dependencies: ['pnpm@10.25.0']
222222
pass_filenames: true
223223
require_serial: true
224-
## ADD MOST PREK HOOK ABOVE THAT LINE
225-
# The below prek hooks are those requiring CI image to be built
226224
- id: mypy-airflow-core
227-
stages: ['pre-push']
228225
name: Run mypy for airflow-core
229226
language: python
230-
entry: ../scripts/ci/prek/mypy.py
231-
files: ^.*\.py$
232-
require_serial: true
233-
- id: mypy-airflow-core
234-
stages: ['manual']
235-
name: Run mypy for airflow-core (manual)
236-
language: python
237-
entry: ../scripts/ci/prek/mypy_folder.py airflow-core
227+
entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core
238228
pass_filenames: false
239229
files: ^.*\.py$
240230
require_serial: true
231+
## ADD MOST PREK HOOK ABOVE THAT LINE
232+
# The below prek hooks are those requiring CI image to be built
241233
- id: generate-openapi-spec
242234
name: Generate the FastAPI API spec
243235
language: python

‎airflow-core/src/airflow/utils/log/non_caching_file_handler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
def make_file_io_non_caching(io: IO[str]) -> IO[str]:
2626
try:
2727
fd = io.fileno()
28-
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
28+
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) # type: ignore[attr-defined]
2929
except Exception:
3030
# in case either file descriptor cannot be retrieved or fadvise is not available
3131
# we should simply return the wrapper retrieved by FileHandler's open method

‎airflow-ctl/.pre-commit-config.yaml‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,21 +25,9 @@ repos:
2525
- repo: local
2626
hooks:
2727
- id: mypy-airflow-ctl
28-
stages: ['pre-push']
2928
name: Run mypy for airflow-ctl
3029
language: python
31-
entry: ../scripts/ci/prek/mypy.py
32-
files:
33-
(?x)
34-
^src/airflowctl/.*\.py$|
35-
^tests/.*\.py$
36-
exclude: .*generated.py
37-
require_serial: true
38-
- id: mypy-airflow-ctl
39-
stages: ['manual']
40-
name: Run mypy for airflow-ctl (manual)
41-
language: python
42-
entry: ../scripts/ci/prek/mypy_folder.py airflow-ctl
30+
entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl
4331
pass_filenames: false
4432
files: ^.*\.py$
4533
require_serial: true

‎contributing-docs/08_static_code_checks.rst‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -173,17 +173,18 @@ But you can run prek hooks manually as needed.
173173
prek
174174
175175
- Run only mypy check on your staged airflow and dev files by specifying the
176-
``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified):
176+
``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified).
177+
For non-provider projects, mypy runs locally via ``uv`` (no breeze image needed):
177178

178179
.. code-block:: bash
179180
180-
prek mypy-airflow-core mypy-dev --stage pre-push
181+
prek mypy-airflow-core mypy-dev
181182
182183
- Run only mypy airflow checks on all "airflow-core" files by using:
183184

184185
.. code-block:: bash
185186
186-
prek mypy-airflow-core --all-files --stage pre-push
187+
prek mypy-airflow-core --all-files
187188
188189
- Run all pre-commit stage hooks on all files by using:
189190

@@ -279,41 +280,40 @@ them manually by running ``prek --stage manual <hook-id>``.
279280
Mypy checks
280281
-----------
281282

282-
When we run mypy checks locally when pushing a change to PR, the ``mypy-*`` checks is run, ``mypy-airflow``,
283-
``mypy-dev``, ``mypy-providers``, ``mypy-airflow-ctl``, depending on the files you are changing. The mypy checks
284-
are run by passing those changed files to mypy. This is way faster than running checks for all files (even
285-
if mypy cache is used - especially when you change a file in Airflow core that is imported and used by many
286-
files). You also need to have ``breeze ci-image build --python 3.10`` built locally to run the mypy checks.
283+
When we run mypy checks locally, the ``mypy-*`` checks run depending on the files you are changing:
284+
``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``, ``mypy-airflow-ctl``, etc.
287285

288-
However, in some cases, it produces different results than when running checks for the whole set
289-
of files, because ``mypy`` does not even know that some types are defined in other files and it might not
290-
be able to follow imports properly if they are dynamic. Therefore in CI we run ``mypy`` check for whole
291-
directories (``airflow`` - excluding providers, ``providers``, ``dev`` and ``docs``) to make sure
292-
that we catch all ``mypy`` errors - so you can experience different results when running mypy locally and
293-
in CI. If you want to run mypy checks for all files locally, you can do it by running the following
294-
command (example for ``airflow`` files):
286+
For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common), mypy
287+
runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular
288+
prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both
289+
as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs).
290+
You can also run mypy directly. Use ``--frozen`` to avoid updating ``uv.lock``:
295291

296292
.. code-block:: bash
297293
298-
prek --stage manual mypy-<FOLDER> --all-files
294+
uv run --frozen --project <PROJECT> --with "apache-airflow-devel-common[mypy]" mypy path/to/code
299295
300-
For example:
296+
To run the prek hook for a specific project (example for ``airflow-core`` files):
301297

302298
.. code-block:: bash
303299
304-
prek --stage manual mypy-airflow --all-files
300+
prek mypy-airflow-core --all-files
305301
306302
To show unused mypy ignores for any providers/airflow etc, eg: run below command:
307303

308304
.. code-block:: bash
305+
309306
export SHOW_UNUSED_MYPY_WARNINGS=true
310-
prek --stage manual mypy-airflow --all-files
307+
prek mypy-airflow-core --all-files
308+
309+
For non-provider projects, the local mypy cache is stored in ``.mypy_cache`` at the repo root.
310+
311+
For **providers**, mypy still runs via breeze (``breeze run mypy``) as a separate CI job and requires
312+
``breeze ci-image build --python 3.10`` to be built locally. Providers use a separate docker-volume
313+
(called ``mypy-cache-volume``) that keeps the cache of last MyPy execution.
311314

312-
MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy
313-
execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy
314-
will handle refreshing the cache when and if needed, there are some cases when it won't (cache invalidation
315-
is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such
316-
cases you might need to manually remove the cache volume by running ``breeze down --cleanup-mypy-cache``.
315+
To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume), run
316+
``breeze down --cleanup-mypy-cache``.
317317

318318
-----------
319319

‎dev/breeze/src/airflow_breeze/commands/developer_commands.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -932,6 +932,10 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache:
932932
if cleanup_mypy_cache:
933933
command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"]
934934
run_command(command_to_execute)
935+
local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache"
936+
if local_mypy_cache.exists():
937+
console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n")
938+
shutil.rmtree(local_mypy_cache)
935939
if cleanup_build_cache:
936940
command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"]
937941
run_command(command_to_execute)
@@ -1070,6 +1074,10 @@ def doctor(ctx):
10701074
console_print("\n[info]Cleaning mypy cache...\n")
10711075
command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"]
10721076
run_command(command_to_execute)
1077+
local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache"
1078+
if local_mypy_cache.exists():
1079+
console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n")
1080+
shutil.rmtree(local_mypy_cache)
10731081

10741082
console_print("\n[info]Cleaning build cache...\n")
10751083
command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"]

‎dev/breeze/src/airflow_breeze/commands/registry_commands.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def _read_provider_yaml_info(provider_id: str) -> tuple[str, list[str]]:
160160
try:
161161
import tomllib
162162
except ImportError:
163-
import tomli as tomllib
163+
import tomli as tomllib # type: ignore[no-redef]
164164

165165
provider_yaml_path = _find_provider_yaml(provider_id)
166166
with open(provider_yaml_path) as f:

0 commit comments

Comments
 (0)